Source file src/net/http/client.go

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // HTTP client. See RFC 7230 through 7235.
     6  //
     7  // This is the high-level Client interface.
     8  // The low-level implementation is in transport.go.
     9  
    10  package http
    11  
    12  import (
    13  	"context"
    14  	"crypto/tls"
    15  	"encoding/base64"
    16  	"errors"
    17  	"fmt"
    18  	"io"
    19  	"log"
    20  	"net/http/internal/ascii"
    21  	"net/url"
    22  	"reflect"
    23  	"sort"
    24  	"strings"
    25  	"sync"
    26  	"sync/atomic"
    27  	"time"
    28  )
    29  
    30  // A Client is an HTTP client. Its zero value ([DefaultClient]) is a
    31  // usable client that uses [DefaultTransport].
    32  //
    33  // The [Client.Transport] typically has internal state (cached TCP
    34  // connections), so Clients should be reused instead of created as
    35  // needed. Clients are safe for concurrent use by multiple goroutines.
    36  //
    37  // A Client is higher-level than a [RoundTripper] (such as [Transport])
    38  // and additionally handles HTTP details such as cookies and
    39  // redirects.
    40  //
    41  // When following redirects, the Client will forward all headers set on the
    42  // initial [Request] except:
    43  //
    44  //   - when forwarding sensitive headers like "Authorization",
    45  //     "WWW-Authenticate", and "Cookie" to untrusted targets.
    46  //     These headers will be ignored when following a redirect to a domain
    47  //     that is not a subdomain match or exact match of the initial domain.
    48  //     For example, a redirect from "foo.com" to either "foo.com" or "sub.foo.com"
    49  //     will forward the sensitive headers, but a redirect to "bar.com" will not.
    50  //   - when forwarding the "Cookie" header with a non-nil cookie Jar.
    51  //     Since each redirect may mutate the state of the cookie jar,
    52  //     a redirect may possibly alter a cookie set in the initial request.
    53  //     When forwarding the "Cookie" header, any mutated cookies will be omitted,
    54  //     with the expectation that the Jar will insert those mutated cookies
    55  //     with the updated values (assuming the origin matches).
    56  //     If Jar is nil, the initial cookies are forwarded without change.
    57  type Client struct {
    58  	// Transport specifies the mechanism by which individual
    59  	// HTTP requests are made.
    60  	// If nil, DefaultTransport is used.
    61  	Transport RoundTripper
    62  
    63  	// CheckRedirect specifies the policy for handling redirects.
    64  	// If CheckRedirect is not nil, the client calls it before
    65  	// following an HTTP redirect. The arguments req and via are
    66  	// the upcoming request and the requests made already, oldest
    67  	// first. If CheckRedirect returns an error, the Client's Get
    68  	// method returns both the previous Response (with its Body
    69  	// closed) and CheckRedirect's error (wrapped in a url.Error)
    70  	// instead of issuing the Request req.
    71  	// As a special case, if CheckRedirect returns ErrUseLastResponse,
    72  	// then the most recent response is returned with its body
    73  	// unclosed, along with a nil error.
    74  	//
    75  	// If CheckRedirect is nil, the Client uses its default policy,
    76  	// which is to stop after 10 consecutive requests.
    77  	CheckRedirect func(req *Request, via []*Request) error
    78  
    79  	// Jar specifies the cookie jar.
    80  	//
    81  	// The Jar is used to insert relevant cookies into every
    82  	// outbound Request and is updated with the cookie values
    83  	// of every inbound Response. The Jar is consulted for every
    84  	// redirect that the Client follows.
    85  	//
    86  	// If Jar is nil, cookies are only sent if they are explicitly
    87  	// set on the Request.
    88  	Jar CookieJar
    89  
    90  	// Timeout specifies a time limit for requests made by this
    91  	// Client. The timeout includes connection time, any
    92  	// redirects, and reading the response body. The timer remains
    93  	// running after Get, Head, Post, or Do return and will
    94  	// interrupt reading of the Response.Body.
    95  	//
    96  	// A Timeout of zero means no timeout.
    97  	//
    98  	// The Client cancels requests to the underlying Transport
    99  	// as if the Request's Context ended.
   100  	//
   101  	// For compatibility, the Client will also use the deprecated
   102  	// CancelRequest method on Transport if found. New
   103  	// RoundTripper implementations should use the Request's Context
   104  	// for cancellation instead of implementing CancelRequest.
   105  	Timeout time.Duration
   106  }
   107  
   108  // DefaultClient is the default [Client] and is used by [Get], [Head], and [Post].
   109  var DefaultClient = &Client{}
   110  
   111  // RoundTripper is an interface representing the ability to execute a
   112  // single HTTP transaction, obtaining the [Response] for a given [Request].
   113  //
   114  // A RoundTripper must be safe for concurrent use by multiple
   115  // goroutines.
   116  type RoundTripper interface {
   117  	// RoundTrip executes a single HTTP transaction, returning
   118  	// a Response for the provided Request.
   119  	//
   120  	// RoundTrip should not attempt to interpret the response. In
   121  	// particular, RoundTrip must return err == nil if it obtained
   122  	// a response, regardless of the response's HTTP status code.
   123  	// A non-nil err should be reserved for failure to obtain a
   124  	// response. Similarly, RoundTrip should not attempt to
   125  	// handle higher-level protocol details such as redirects,
   126  	// authentication, or cookies.
   127  	//
   128  	// RoundTrip should not modify the request, except for
   129  	// consuming and closing the Request's Body. RoundTrip may
   130  	// read fields of the request in a separate goroutine. Callers
   131  	// should not mutate or reuse the request until the Response's
   132  	// Body has been closed.
   133  	//
   134  	// RoundTrip must always close the body, including on errors,
   135  	// but depending on the implementation may do so in a separate
   136  	// goroutine even after RoundTrip returns. This means that
   137  	// callers wanting to reuse the body for subsequent requests
   138  	// must arrange to wait for the Close call before doing so.
   139  	//
   140  	// The Request's URL and Header fields must be initialized.
   141  	RoundTrip(*Request) (*Response, error)
   142  }
   143  
   144  // refererForURL returns a referer without any authentication info or
   145  // an empty string if lastReq scheme is https and newReq scheme is http.
   146  // If the referer was explicitly set, then it will continue to be used.
   147  func refererForURL(lastReq, newReq *url.URL, explicitRef string) string {
   148  	// https://tools.ietf.org/html/rfc7231#section-5.5.2
   149  	//   "Clients SHOULD NOT include a Referer header field in a
   150  	//    (non-secure) HTTP request if the referring page was
   151  	//    transferred with a secure protocol."
   152  	if lastReq.Scheme == "https" && newReq.Scheme == "http" {
   153  		return ""
   154  	}
   155  	if explicitRef != "" {
   156  		return explicitRef
   157  	}
   158  
   159  	referer := lastReq.String()
   160  	if lastReq.User != nil {
   161  		// This is not very efficient, but is the best we can
   162  		// do without:
   163  		// - introducing a new method on URL
   164  		// - creating a race condition
   165  		// - copying the URL struct manually, which would cause
   166  		//   maintenance problems down the line
   167  		auth := lastReq.User.String() + "@"
   168  		referer = strings.Replace(referer, auth, "", 1)
   169  	}
   170  	return referer
   171  }
   172  
   173  // didTimeout is non-nil only if err != nil.
   174  func (c *Client) send(req *Request, deadline time.Time) (resp *Response, didTimeout func() bool, err error) {
   175  	if c.Jar != nil {
   176  		for _, cookie := range c.Jar.Cookies(req.URL) {
   177  			req.AddCookie(cookie)
   178  		}
   179  	}
   180  	resp, didTimeout, err = send(req, c.transport(), deadline)
   181  	if err != nil {
   182  		return nil, didTimeout, err
   183  	}
   184  	if c.Jar != nil {
   185  		if rc := resp.Cookies(); len(rc) > 0 {
   186  			c.Jar.SetCookies(req.URL, rc)
   187  		}
   188  	}
   189  	return resp, nil, nil
   190  }
   191  
   192  func (c *Client) deadline() time.Time {
   193  	if c.Timeout > 0 {
   194  		return time.Now().Add(c.Timeout)
   195  	}
   196  	return time.Time{}
   197  }
   198  
   199  func (c *Client) transport() RoundTripper {
   200  	if c.Transport != nil {
   201  		return c.Transport
   202  	}
   203  	return DefaultTransport
   204  }
   205  
   206  // ErrSchemeMismatch is returned when a server returns an HTTP response to an HTTPS client.
   207  var ErrSchemeMismatch = errors.New("http: server gave HTTP response to HTTPS client")
   208  
   209  // send issues an HTTP request.
   210  // Caller should close resp.Body when done reading from it.
   211  func send(ireq *Request, rt RoundTripper, deadline time.Time) (resp *Response, didTimeout func() bool, err error) {
   212  	req := ireq // req is either the original request, or a modified fork
   213  
   214  	if rt == nil {
   215  		req.closeBody()
   216  		return nil, alwaysFalse, errors.New("http: no Client.Transport or DefaultTransport")
   217  	}
   218  
   219  	if req.URL == nil {
   220  		req.closeBody()
   221  		return nil, alwaysFalse, errors.New("http: nil Request.URL")
   222  	}
   223  
   224  	if req.RequestURI != "" {
   225  		req.closeBody()
   226  		return nil, alwaysFalse, errors.New("http: Request.RequestURI can't be set in client requests")
   227  	}
   228  
   229  	// forkReq forks req into a shallow clone of ireq the first
   230  	// time it's called.
   231  	forkReq := func() {
   232  		if ireq == req {
   233  			req = new(Request)
   234  			*req = *ireq // shallow clone
   235  		}
   236  	}
   237  
   238  	// Most the callers of send (Get, Post, et al) don't need
   239  	// Headers, leaving it uninitialized. We guarantee to the
   240  	// Transport that this has been initialized, though.
   241  	if req.Header == nil {
   242  		forkReq()
   243  		req.Header = make(Header)
   244  	}
   245  
   246  	if u := req.URL.User; u != nil && req.Header.Get("Authorization") == "" {
   247  		username := u.Username()
   248  		password, _ := u.Password()
   249  		forkReq()
   250  		req.Header = cloneOrMakeHeader(ireq.Header)
   251  		req.Header.Set("Authorization", "Basic "+basicAuth(username, password))
   252  	}
   253  
   254  	if !deadline.IsZero() {
   255  		forkReq()
   256  	}
   257  	stopTimer, didTimeout := setRequestCancel(req, rt, deadline)
   258  
   259  	resp, err = rt.RoundTrip(req)
   260  	if err != nil {
   261  		stopTimer()
   262  		if resp != nil {
   263  			log.Printf("RoundTripper returned a response & error; ignoring response")
   264  		}
   265  		if tlsErr, ok := err.(tls.RecordHeaderError); ok {
   266  			// If we get a bad TLS record header, check to see if the
   267  			// response looks like HTTP and give a more helpful error.
   268  			// See golang.org/issue/11111.
   269  			if string(tlsErr.RecordHeader[:]) == "HTTP/" {
   270  				err = ErrSchemeMismatch
   271  			}
   272  		}
   273  		return nil, didTimeout, err
   274  	}
   275  	if resp == nil {
   276  		return nil, didTimeout, fmt.Errorf("http: RoundTripper implementation (%T) returned a nil *Response with a nil error", rt)
   277  	}
   278  	if resp.Body == nil {
   279  		// The documentation on the Body field says “The http Client and Transport
   280  		// guarantee that Body is always non-nil, even on responses without a body
   281  		// or responses with a zero-length body.” Unfortunately, we didn't document
   282  		// that same constraint for arbitrary RoundTripper implementations, and
   283  		// RoundTripper implementations in the wild (mostly in tests) assume that
   284  		// they can use a nil Body to mean an empty one (similar to Request.Body).
   285  		// (See https://golang.ir/issue/38095.)
   286  		//
   287  		// If the ContentLength allows the Body to be empty, fill in an empty one
   288  		// here to ensure that it is non-nil.
   289  		if resp.ContentLength > 0 && req.Method != "HEAD" {
   290  			return nil, didTimeout, fmt.Errorf("http: RoundTripper implementation (%T) returned a *Response with content length %d but a nil Body", rt, resp.ContentLength)
   291  		}
   292  		resp.Body = io.NopCloser(strings.NewReader(""))
   293  	}
   294  	if !deadline.IsZero() {
   295  		resp.Body = &cancelTimerBody{
   296  			stop:          stopTimer,
   297  			rc:            resp.Body,
   298  			reqDidTimeout: didTimeout,
   299  		}
   300  	}
   301  	return resp, nil, nil
   302  }
   303  
   304  // timeBeforeContextDeadline reports whether the non-zero Time t is
   305  // before ctx's deadline, if any. If ctx does not have a deadline, it
   306  // always reports true (the deadline is considered infinite).
   307  func timeBeforeContextDeadline(t time.Time, ctx context.Context) bool {
   308  	d, ok := ctx.Deadline()
   309  	if !ok {
   310  		return true
   311  	}
   312  	return t.Before(d)
   313  }
   314  
   315  // knownRoundTripperImpl reports whether rt is a RoundTripper that's
   316  // maintained by the Go team and known to implement the latest
   317  // optional semantics (notably contexts). The Request is used
   318  // to check whether this particular request is using an alternate protocol,
   319  // in which case we need to check the RoundTripper for that protocol.
   320  func knownRoundTripperImpl(rt RoundTripper, req *Request) bool {
   321  	switch t := rt.(type) {
   322  	case *Transport:
   323  		if altRT := t.alternateRoundTripper(req); altRT != nil {
   324  			return knownRoundTripperImpl(altRT, req)
   325  		}
   326  		return true
   327  	case *http2Transport, http2noDialH2RoundTripper:
   328  		return true
   329  	}
   330  	// There's a very minor chance of a false positive with this.
   331  	// Instead of detecting our golang.org/x/net/http2.Transport,
   332  	// it might detect a Transport type in a different http2
   333  	// package. But I know of none, and the only problem would be
   334  	// some temporarily leaked goroutines if the transport didn't
   335  	// support contexts. So this is a good enough heuristic:
   336  	if reflect.TypeOf(rt).String() == "*http2.Transport" {
   337  		return true
   338  	}
   339  	return false
   340  }
   341  
   342  // setRequestCancel sets req.Cancel and adds a deadline context to req
   343  // if deadline is non-zero. The RoundTripper's type is used to
   344  // determine whether the legacy CancelRequest behavior should be used.
   345  //
   346  // As background, there are three ways to cancel a request:
   347  // First was Transport.CancelRequest. (deprecated)
   348  // Second was Request.Cancel.
   349  // Third was Request.Context.
   350  // This function populates the second and third, and uses the first if it really needs to.
   351  func setRequestCancel(req *Request, rt RoundTripper, deadline time.Time) (stopTimer func(), didTimeout func() bool) {
   352  	if deadline.IsZero() {
   353  		return nop, alwaysFalse
   354  	}
   355  	knownTransport := knownRoundTripperImpl(rt, req)
   356  	oldCtx := req.Context()
   357  
   358  	if req.Cancel == nil && knownTransport {
   359  		// If they already had a Request.Context that's
   360  		// expiring sooner, do nothing:
   361  		if !timeBeforeContextDeadline(deadline, oldCtx) {
   362  			return nop, alwaysFalse
   363  		}
   364  
   365  		var cancelCtx func()
   366  		req.ctx, cancelCtx = context.WithDeadline(oldCtx, deadline)
   367  		return cancelCtx, func() bool { return time.Now().After(deadline) }
   368  	}
   369  	initialReqCancel := req.Cancel // the user's original Request.Cancel, if any
   370  
   371  	var cancelCtx func()
   372  	if timeBeforeContextDeadline(deadline, oldCtx) {
   373  		req.ctx, cancelCtx = context.WithDeadline(oldCtx, deadline)
   374  	}
   375  
   376  	cancel := make(chan struct{})
   377  	req.Cancel = cancel
   378  
   379  	doCancel := func() {
   380  		// The second way in the func comment above:
   381  		close(cancel)
   382  		// The first way, used only for RoundTripper
   383  		// implementations written before Go 1.5 or Go 1.6.
   384  		type canceler interface{ CancelRequest(*Request) }
   385  		if v, ok := rt.(canceler); ok {
   386  			v.CancelRequest(req)
   387  		}
   388  	}
   389  
   390  	stopTimerCh := make(chan struct{})
   391  	var once sync.Once
   392  	stopTimer = func() {
   393  		once.Do(func() {
   394  			close(stopTimerCh)
   395  			if cancelCtx != nil {
   396  				cancelCtx()
   397  			}
   398  		})
   399  	}
   400  
   401  	timer := time.NewTimer(time.Until(deadline))
   402  	var timedOut atomic.Bool
   403  
   404  	go func() {
   405  		select {
   406  		case <-initialReqCancel:
   407  			doCancel()
   408  			timer.Stop()
   409  		case <-timer.C:
   410  			timedOut.Store(true)
   411  			doCancel()
   412  		case <-stopTimerCh:
   413  			timer.Stop()
   414  		}
   415  	}()
   416  
   417  	return stopTimer, timedOut.Load
   418  }
   419  
   420  // See 2 (end of page 4) https://www.ietf.org/rfc/rfc2617.txt
   421  // "To receive authorization, the client sends the userid and password,
   422  // separated by a single colon (":") character, within a base64
   423  // encoded string in the credentials."
   424  // It is not meant to be urlencoded.
   425  func basicAuth(username, password string) string {
   426  	auth := username + ":" + password
   427  	return base64.StdEncoding.EncodeToString([]byte(auth))
   428  }
   429  
   430  // Get issues a GET to the specified URL. If the response is one of
   431  // the following redirect codes, Get follows the redirect, up to a
   432  // maximum of 10 redirects:
   433  //
   434  //	301 (Moved Permanently)
   435  //	302 (Found)
   436  //	303 (See Other)
   437  //	307 (Temporary Redirect)
   438  //	308 (Permanent Redirect)
   439  //
   440  // An error is returned if there were too many redirects or if there
   441  // was an HTTP protocol error. A non-2xx response doesn't cause an
   442  // error. Any returned error will be of type [*url.Error]. The url.Error
   443  // value's Timeout method will report true if the request timed out.
   444  //
   445  // When err is nil, resp always contains a non-nil resp.Body.
   446  // Caller should close resp.Body when done reading from it.
   447  //
   448  // Get is a wrapper around DefaultClient.Get.
   449  //
   450  // To make a request with custom headers, use [NewRequest] and
   451  // DefaultClient.Do.
   452  //
   453  // To make a request with a specified context.Context, use [NewRequestWithContext]
   454  // and DefaultClient.Do.
   455  func Get(url string) (resp *Response, err error) {
   456  	return DefaultClient.Get(url)
   457  }
   458  
   459  // Get issues a GET to the specified URL. If the response is one of the
   460  // following redirect codes, Get follows the redirect after calling the
   461  // [Client.CheckRedirect] function:
   462  //
   463  //	301 (Moved Permanently)
   464  //	302 (Found)
   465  //	303 (See Other)
   466  //	307 (Temporary Redirect)
   467  //	308 (Permanent Redirect)
   468  //
   469  // An error is returned if the [Client.CheckRedirect] function fails
   470  // or if there was an HTTP protocol error. A non-2xx response doesn't
   471  // cause an error. Any returned error will be of type [*url.Error]. The
   472  // url.Error value's Timeout method will report true if the request
   473  // timed out.
   474  //
   475  // When err is nil, resp always contains a non-nil resp.Body.
   476  // Caller should close resp.Body when done reading from it.
   477  //
   478  // To make a request with custom headers, use [NewRequest] and [Client.Do].
   479  //
   480  // To make a request with a specified context.Context, use [NewRequestWithContext]
   481  // and Client.Do.
   482  func (c *Client) Get(url string) (resp *Response, err error) {
   483  	req, err := NewRequest("GET", url, nil)
   484  	if err != nil {
   485  		return nil, err
   486  	}
   487  	return c.Do(req)
   488  }
   489  
   490  func alwaysFalse() bool { return false }
   491  
   492  // ErrUseLastResponse can be returned by Client.CheckRedirect hooks to
   493  // control how redirects are processed. If returned, the next request
   494  // is not sent and the most recent response is returned with its body
   495  // unclosed.
   496  var ErrUseLastResponse = errors.New("net/http: use last response")
   497  
   498  // checkRedirect calls either the user's configured CheckRedirect
   499  // function, or the default.
   500  func (c *Client) checkRedirect(req *Request, via []*Request) error {
   501  	fn := c.CheckRedirect
   502  	if fn == nil {
   503  		fn = defaultCheckRedirect
   504  	}
   505  	return fn(req, via)
   506  }
   507  
   508  // redirectBehavior describes what should happen when the
   509  // client encounters a 3xx status code from the server.
   510  func redirectBehavior(reqMethod string, resp *Response, ireq *Request) (redirectMethod string, shouldRedirect, includeBody bool) {
   511  	switch resp.StatusCode {
   512  	case 301, 302, 303:
   513  		redirectMethod = reqMethod
   514  		shouldRedirect = true
   515  		includeBody = false
   516  
   517  		// RFC 2616 allowed automatic redirection only with GET and
   518  		// HEAD requests. RFC 7231 lifts this restriction, but we still
   519  		// restrict other methods to GET to maintain compatibility.
   520  		// See Issue 18570.
   521  		if reqMethod != "GET" && reqMethod != "HEAD" {
   522  			redirectMethod = "GET"
   523  		}
   524  	case 307, 308:
   525  		redirectMethod = reqMethod
   526  		shouldRedirect = true
   527  		includeBody = true
   528  
   529  		if ireq.GetBody == nil && ireq.outgoingLength() != 0 {
   530  			// We had a request body, and 307/308 require
   531  			// re-sending it, but GetBody is not defined. So just
   532  			// return this response to the user instead of an
   533  			// error, like we did in Go 1.7 and earlier.
   534  			shouldRedirect = false
   535  		}
   536  	}
   537  	return redirectMethod, shouldRedirect, includeBody
   538  }
   539  
   540  // urlErrorOp returns the (*url.Error).Op value to use for the
   541  // provided (*Request).Method value.
   542  func urlErrorOp(method string) string {
   543  	if method == "" {
   544  		return "Get"
   545  	}
   546  	if lowerMethod, ok := ascii.ToLower(method); ok {
   547  		return method[:1] + lowerMethod[1:]
   548  	}
   549  	return method
   550  }
   551  
   552  // Do sends an HTTP request and returns an HTTP response, following
   553  // policy (such as redirects, cookies, auth) as configured on the
   554  // client.
   555  //
   556  // An error is returned if caused by client policy (such as
   557  // CheckRedirect), or failure to speak HTTP (such as a network
   558  // connectivity problem). A non-2xx status code doesn't cause an
   559  // error.
   560  //
   561  // If the returned error is nil, the [Response] will contain a non-nil
   562  // Body which the user is expected to close. If the Body is not both
   563  // read to EOF and closed, the [Client]'s underlying [RoundTripper]
   564  // (typically [Transport]) may not be able to re-use a persistent TCP
   565  // connection to the server for a subsequent "keep-alive" request.
   566  //
   567  // The request Body, if non-nil, will be closed by the underlying
   568  // Transport, even on errors. The Body may be closed asynchronously after
   569  // Do returns.
   570  //
   571  // On error, any Response can be ignored. A non-nil Response with a
   572  // non-nil error only occurs when CheckRedirect fails, and even then
   573  // the returned [Response.Body] is already closed.
   574  //
   575  // Generally [Get], [Post], or [PostForm] will be used instead of Do.
   576  //
   577  // If the server replies with a redirect, the Client first uses the
   578  // CheckRedirect function to determine whether the redirect should be
   579  // followed. If permitted, a 301, 302, or 303 redirect causes
   580  // subsequent requests to use HTTP method GET
   581  // (or HEAD if the original request was HEAD), with no body.
   582  // A 307 or 308 redirect preserves the original HTTP method and body,
   583  // provided that the [Request.GetBody] function is defined.
   584  // The [NewRequest] function automatically sets GetBody for common
   585  // standard library body types.
   586  //
   587  // Any returned error will be of type [*url.Error]. The url.Error
   588  // value's Timeout method will report true if the request timed out.
   589  func (c *Client) Do(req *Request) (*Response, error) {
   590  	return c.do(req)
   591  }
   592  
   593  var testHookClientDoResult func(retres *Response, reterr error)
   594  
   595  func (c *Client) do(req *Request) (retres *Response, reterr error) {
   596  	if testHookClientDoResult != nil {
   597  		defer func() { testHookClientDoResult(retres, reterr) }()
   598  	}
   599  	if req.URL == nil {
   600  		req.closeBody()
   601  		return nil, &url.Error{
   602  			Op:  urlErrorOp(req.Method),
   603  			Err: errors.New("http: nil Request.URL"),
   604  		}
   605  	}
   606  
   607  	var (
   608  		deadline      = c.deadline()
   609  		reqs          []*Request
   610  		resp          *Response
   611  		copyHeaders   = c.makeHeadersCopier(req)
   612  		reqBodyClosed = false // have we closed the current req.Body?
   613  
   614  		// Redirect behavior:
   615  		redirectMethod string
   616  		includeBody    bool
   617  	)
   618  	uerr := func(err error) error {
   619  		// the body may have been closed already by c.send()
   620  		if !reqBodyClosed {
   621  			req.closeBody()
   622  		}
   623  		var urlStr string
   624  		if resp != nil && resp.Request != nil {
   625  			urlStr = stripPassword(resp.Request.URL)
   626  		} else {
   627  			urlStr = stripPassword(req.URL)
   628  		}
   629  		return &url.Error{
   630  			Op:  urlErrorOp(reqs[0].Method),
   631  			URL: urlStr,
   632  			Err: err,
   633  		}
   634  	}
   635  	for {
   636  		// For all but the first request, create the next
   637  		// request hop and replace req.
   638  		if len(reqs) > 0 {
   639  			loc := resp.Header.Get("Location")
   640  			if loc == "" {
   641  				// While most 3xx responses include a Location, it is not
   642  				// required and 3xx responses without a Location have been
   643  				// observed in the wild. See issues #17773 and #49281.
   644  				return resp, nil
   645  			}
   646  			u, err := req.URL.Parse(loc)
   647  			if err != nil {
   648  				resp.closeBody()
   649  				return nil, uerr(fmt.Errorf("failed to parse Location header %q: %v", loc, err))
   650  			}
   651  			host := ""
   652  			if req.Host != "" && req.Host != req.URL.Host {
   653  				// If the caller specified a custom Host header and the
   654  				// redirect location is relative, preserve the Host header
   655  				// through the redirect. See issue #22233.
   656  				if u, _ := url.Parse(loc); u != nil && !u.IsAbs() {
   657  					host = req.Host
   658  				}
   659  			}
   660  			ireq := reqs[0]
   661  			req = &Request{
   662  				Method:   redirectMethod,
   663  				Response: resp,
   664  				URL:      u,
   665  				Header:   make(Header),
   666  				Host:     host,
   667  				Cancel:   ireq.Cancel,
   668  				ctx:      ireq.ctx,
   669  			}
   670  			if includeBody && ireq.GetBody != nil {
   671  				req.Body, err = ireq.GetBody()
   672  				if err != nil {
   673  					resp.closeBody()
   674  					return nil, uerr(err)
   675  				}
   676  				req.ContentLength = ireq.ContentLength
   677  			}
   678  
   679  			// Copy original headers before setting the Referer,
   680  			// in case the user set Referer on their first request.
   681  			// If they really want to override, they can do it in
   682  			// their CheckRedirect func.
   683  			copyHeaders(req)
   684  
   685  			// Add the Referer header from the most recent
   686  			// request URL to the new one, if it's not https->http:
   687  			if ref := refererForURL(reqs[len(reqs)-1].URL, req.URL, req.Header.Get("Referer")); ref != "" {
   688  				req.Header.Set("Referer", ref)
   689  			}
   690  			err = c.checkRedirect(req, reqs)
   691  
   692  			// Sentinel error to let users select the
   693  			// previous response, without closing its
   694  			// body. See Issue 10069.
   695  			if err == ErrUseLastResponse {
   696  				return resp, nil
   697  			}
   698  
   699  			// Close the previous response's body. But
   700  			// read at least some of the body so if it's
   701  			// small the underlying TCP connection will be
   702  			// re-used. No need to check for errors: if it
   703  			// fails, the Transport won't reuse it anyway.
   704  			const maxBodySlurpSize = 2 << 10
   705  			if resp.ContentLength == -1 || resp.ContentLength <= maxBodySlurpSize {
   706  				io.CopyN(io.Discard, resp.Body, maxBodySlurpSize)
   707  			}
   708  			resp.Body.Close()
   709  
   710  			if err != nil {
   711  				// Special case for Go 1 compatibility: return both the response
   712  				// and an error if the CheckRedirect function failed.
   713  				// See https://golang.ir/issue/3795
   714  				// The resp.Body has already been closed.
   715  				ue := uerr(err)
   716  				ue.(*url.Error).URL = loc
   717  				return resp, ue
   718  			}
   719  		}
   720  
   721  		reqs = append(reqs, req)
   722  		var err error
   723  		var didTimeout func() bool
   724  		if resp, didTimeout, err = c.send(req, deadline); err != nil {
   725  			// c.send() always closes req.Body
   726  			reqBodyClosed = true
   727  			if !deadline.IsZero() && didTimeout() {
   728  				err = &httpError{
   729  					err:     err.Error() + " (Client.Timeout exceeded while awaiting headers)",
   730  					timeout: true,
   731  				}
   732  			}
   733  			return nil, uerr(err)
   734  		}
   735  
   736  		var shouldRedirect bool
   737  		redirectMethod, shouldRedirect, includeBody = redirectBehavior(req.Method, resp, reqs[0])
   738  		if !shouldRedirect {
   739  			return resp, nil
   740  		}
   741  
   742  		req.closeBody()
   743  	}
   744  }
   745  
   746  // makeHeadersCopier makes a function that copies headers from the
   747  // initial Request, ireq. For every redirect, this function must be called
   748  // so that it can copy headers into the upcoming Request.
   749  func (c *Client) makeHeadersCopier(ireq *Request) func(*Request) {
   750  	// The headers to copy are from the very initial request.
   751  	// We use a closured callback to keep a reference to these original headers.
   752  	var (
   753  		ireqhdr  = cloneOrMakeHeader(ireq.Header)
   754  		icookies map[string][]*Cookie
   755  	)
   756  	if c.Jar != nil && ireq.Header.Get("Cookie") != "" {
   757  		icookies = make(map[string][]*Cookie)
   758  		for _, c := range ireq.Cookies() {
   759  			icookies[c.Name] = append(icookies[c.Name], c)
   760  		}
   761  	}
   762  
   763  	preq := ireq // The previous request
   764  	return func(req *Request) {
   765  		// If Jar is present and there was some initial cookies provided
   766  		// via the request header, then we may need to alter the initial
   767  		// cookies as we follow redirects since each redirect may end up
   768  		// modifying a pre-existing cookie.
   769  		//
   770  		// Since cookies already set in the request header do not contain
   771  		// information about the original domain and path, the logic below
   772  		// assumes any new set cookies override the original cookie
   773  		// regardless of domain or path.
   774  		//
   775  		// See https://golang.ir/issue/17494
   776  		if c.Jar != nil && icookies != nil {
   777  			var changed bool
   778  			resp := req.Response // The response that caused the upcoming redirect
   779  			for _, c := range resp.Cookies() {
   780  				if _, ok := icookies[c.Name]; ok {
   781  					delete(icookies, c.Name)
   782  					changed = true
   783  				}
   784  			}
   785  			if changed {
   786  				ireqhdr.Del("Cookie")
   787  				var ss []string
   788  				for _, cs := range icookies {
   789  					for _, c := range cs {
   790  						ss = append(ss, c.Name+"="+c.Value)
   791  					}
   792  				}
   793  				sort.Strings(ss) // Ensure deterministic headers
   794  				ireqhdr.Set("Cookie", strings.Join(ss, "; "))
   795  			}
   796  		}
   797  
   798  		// Copy the initial request's Header values
   799  		// (at least the safe ones).
   800  		for k, vv := range ireqhdr {
   801  			if shouldCopyHeaderOnRedirect(k, preq.URL, req.URL) {
   802  				req.Header[k] = vv
   803  			}
   804  		}
   805  
   806  		preq = req // Update previous Request with the current request
   807  	}
   808  }
   809  
   810  func defaultCheckRedirect(req *Request, via []*Request) error {
   811  	if len(via) >= 10 {
   812  		return errors.New("stopped after 10 redirects")
   813  	}
   814  	return nil
   815  }
   816  
   817  // Post issues a POST to the specified URL.
   818  //
   819  // Caller should close resp.Body when done reading from it.
   820  //
   821  // If the provided body is an [io.Closer], it is closed after the
   822  // request.
   823  //
   824  // Post is a wrapper around DefaultClient.Post.
   825  //
   826  // To set custom headers, use [NewRequest] and DefaultClient.Do.
   827  //
   828  // See the [Client.Do] method documentation for details on how redirects
   829  // are handled.
   830  //
   831  // To make a request with a specified context.Context, use [NewRequestWithContext]
   832  // and DefaultClient.Do.
   833  func Post(url, contentType string, body io.Reader) (resp *Response, err error) {
   834  	return DefaultClient.Post(url, contentType, body)
   835  }
   836  
   837  // Post issues a POST to the specified URL.
   838  //
   839  // Caller should close resp.Body when done reading from it.
   840  //
   841  // If the provided body is an [io.Closer], it is closed after the
   842  // request.
   843  //
   844  // To set custom headers, use [NewRequest] and [Client.Do].
   845  //
   846  // To make a request with a specified context.Context, use [NewRequestWithContext]
   847  // and [Client.Do].
   848  //
   849  // See the Client.Do method documentation for details on how redirects
   850  // are handled.
   851  func (c *Client) Post(url, contentType string, body io.Reader) (resp *Response, err error) {
   852  	req, err := NewRequest("POST", url, body)
   853  	if err != nil {
   854  		return nil, err
   855  	}
   856  	req.Header.Set("Content-Type", contentType)
   857  	return c.Do(req)
   858  }
   859  
   860  // PostForm issues a POST to the specified URL, with data's keys and
   861  // values URL-encoded as the request body.
   862  //
   863  // The Content-Type header is set to application/x-www-form-urlencoded.
   864  // To set other headers, use [NewRequest] and DefaultClient.Do.
   865  //
   866  // When err is nil, resp always contains a non-nil resp.Body.
   867  // Caller should close resp.Body when done reading from it.
   868  //
   869  // PostForm is a wrapper around DefaultClient.PostForm.
   870  //
   871  // See the [Client.Do] method documentation for details on how redirects
   872  // are handled.
   873  //
   874  // To make a request with a specified [context.Context], use [NewRequestWithContext]
   875  // and DefaultClient.Do.
   876  func PostForm(url string, data url.Values) (resp *Response, err error) {
   877  	return DefaultClient.PostForm(url, data)
   878  }
   879  
   880  // PostForm issues a POST to the specified URL,
   881  // with data's keys and values URL-encoded as the request body.
   882  //
   883  // The Content-Type header is set to application/x-www-form-urlencoded.
   884  // To set other headers, use [NewRequest] and [Client.Do].
   885  //
   886  // When err is nil, resp always contains a non-nil resp.Body.
   887  // Caller should close resp.Body when done reading from it.
   888  //
   889  // See the Client.Do method documentation for details on how redirects
   890  // are handled.
   891  //
   892  // To make a request with a specified context.Context, use [NewRequestWithContext]
   893  // and Client.Do.
   894  func (c *Client) PostForm(url string, data url.Values) (resp *Response, err error) {
   895  	return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
   896  }
   897  
   898  // Head issues a HEAD to the specified URL. If the response is one of
   899  // the following redirect codes, Head follows the redirect, up to a
   900  // maximum of 10 redirects:
   901  //
   902  //	301 (Moved Permanently)
   903  //	302 (Found)
   904  //	303 (See Other)
   905  //	307 (Temporary Redirect)
   906  //	308 (Permanent Redirect)
   907  //
   908  // Head is a wrapper around DefaultClient.Head.
   909  //
   910  // To make a request with a specified [context.Context], use [NewRequestWithContext]
   911  // and DefaultClient.Do.
   912  func Head(url string) (resp *Response, err error) {
   913  	return DefaultClient.Head(url)
   914  }
   915  
   916  // Head issues a HEAD to the specified URL. If the response is one of the
   917  // following redirect codes, Head follows the redirect after calling the
   918  // [Client.CheckRedirect] function:
   919  //
   920  //	301 (Moved Permanently)
   921  //	302 (Found)
   922  //	303 (See Other)
   923  //	307 (Temporary Redirect)
   924  //	308 (Permanent Redirect)
   925  //
   926  // To make a request with a specified [context.Context], use [NewRequestWithContext]
   927  // and [Client.Do].
   928  func (c *Client) Head(url string) (resp *Response, err error) {
   929  	req, err := NewRequest("HEAD", url, nil)
   930  	if err != nil {
   931  		return nil, err
   932  	}
   933  	return c.Do(req)
   934  }
   935  
   936  // CloseIdleConnections closes any connections on its [Transport] which
   937  // were previously connected from previous requests but are now
   938  // sitting idle in a "keep-alive" state. It does not interrupt any
   939  // connections currently in use.
   940  //
   941  // If [Client.Transport] does not have a [Client.CloseIdleConnections] method
   942  // then this method does nothing.
   943  func (c *Client) CloseIdleConnections() {
   944  	type closeIdler interface {
   945  		CloseIdleConnections()
   946  	}
   947  	if tr, ok := c.transport().(closeIdler); ok {
   948  		tr.CloseIdleConnections()
   949  	}
   950  }
   951  
   952  // cancelTimerBody is an io.ReadCloser that wraps rc with two features:
   953  //  1. On Read error or close, the stop func is called.
   954  //  2. On Read failure, if reqDidTimeout is true, the error is wrapped and
   955  //     marked as net.Error that hit its timeout.
   956  type cancelTimerBody struct {
   957  	stop          func() // stops the time.Timer waiting to cancel the request
   958  	rc            io.ReadCloser
   959  	reqDidTimeout func() bool
   960  }
   961  
   962  func (b *cancelTimerBody) Read(p []byte) (n int, err error) {
   963  	n, err = b.rc.Read(p)
   964  	if err == nil {
   965  		return n, nil
   966  	}
   967  	if err == io.EOF {
   968  		return n, err
   969  	}
   970  	if b.reqDidTimeout() {
   971  		err = &httpError{
   972  			err:     err.Error() + " (Client.Timeout or context cancellation while reading body)",
   973  			timeout: true,
   974  		}
   975  	}
   976  	return n, err
   977  }
   978  
   979  func (b *cancelTimerBody) Close() error {
   980  	err := b.rc.Close()
   981  	b.stop()
   982  	return err
   983  }
   984  
   985  func shouldCopyHeaderOnRedirect(headerKey string, initial, dest *url.URL) bool {
   986  	switch CanonicalHeaderKey(headerKey) {
   987  	case "Authorization", "Www-Authenticate", "Cookie", "Cookie2":
   988  		// Permit sending auth/cookie headers from "foo.com"
   989  		// to "sub.foo.com".
   990  
   991  		// Note that we don't send all cookies to subdomains
   992  		// automatically. This function is only used for
   993  		// Cookies set explicitly on the initial outgoing
   994  		// client request. Cookies automatically added via the
   995  		// CookieJar mechanism continue to follow each
   996  		// cookie's scope as set by Set-Cookie. But for
   997  		// outgoing requests with the Cookie header set
   998  		// directly, we don't know their scope, so we assume
   999  		// it's for *.domain.com.
  1000  
  1001  		ihost := idnaASCIIFromURL(initial)
  1002  		dhost := idnaASCIIFromURL(dest)
  1003  		return isDomainOrSubdomain(dhost, ihost)
  1004  	}
  1005  	// All other headers are copied:
  1006  	return true
  1007  }
  1008  
  1009  // isDomainOrSubdomain reports whether sub is a subdomain (or exact
  1010  // match) of the parent domain.
  1011  //
  1012  // Both domains must already be in canonical form.
  1013  func isDomainOrSubdomain(sub, parent string) bool {
  1014  	if sub == parent {
  1015  		return true
  1016  	}
  1017  	// If sub contains a :, it's probably an IPv6 address (and is definitely not a hostname).
  1018  	// Don't check the suffix in this case, to avoid matching the contents of a IPv6 zone.
  1019  	// For example, "::1%.www.example.com" is not a subdomain of "www.example.com".
  1020  	if strings.ContainsAny(sub, ":%") {
  1021  		return false
  1022  	}
  1023  	// If sub is "foo.example.com" and parent is "example.com",
  1024  	// that means sub must end in "."+parent.
  1025  	// Do it without allocating.
  1026  	if !strings.HasSuffix(sub, parent) {
  1027  		return false
  1028  	}
  1029  	return sub[len(sub)-len(parent)-1] == '.'
  1030  }
  1031  
  1032  func stripPassword(u *url.URL) string {
  1033  	_, passSet := u.User.Password()
  1034  	if passSet {
  1035  		return strings.Replace(u.String(), u.User.String()+"@", u.User.Username()+":***@", 1)
  1036  	}
  1037  	return u.String()
  1038  }
  1039  

View as plain text