Golang Response.Body方法代码示例

func (c *Client) logResponse(resp *http.Response) error {
	if c.logHTTP {
		var err error
		save := resp.Body
		savecl := resp.ContentLength
		body := true
		if !body {
			resp.Body = failureToReadBody{}
		} else if resp.Body == nil {
			resp.Body = emptyBody
		} else {
			save, resp.Body, err = drainBody(resp.Body)
			if err != nil {
				return err
			}
		}
		fmt.Println("----------- response start -----------")
		err = resp.Write(os.Stderr)
		if err == errNoBody {
			err = nil
		}
		resp.Body = save
		resp.ContentLength = savecl
		if err != nil {
			return err
		}
		fmt.Println("----------- response end -----------")
	}
	return nil
}

// DumpResponse is like DumpRequest but dumps a response.
func DumpResponse(resp *http.Response, body bool) (dump []byte, err error) {
	var b bytes.Buffer
	save := resp.Body
	savecl := resp.ContentLength

	if !body {
		// For content length of zero. Make sure the body is an empty
		// reader, instead of returning error through failureToReadBody{}.
		if resp.ContentLength == 0 {
			resp.Body = emptyBody
		} else {
			resp.Body = failureToReadBody{}
		}
	} else if resp.Body == nil {
		resp.Body = emptyBody
	} else {
		save, resp.Body, err = drainBody(resp.Body)
		if err != nil {
			return nil, err
		}
	}
	err = resp.Write(&b)
	if err == errNoBody {
		err = nil
	}
	resp.Body = save
	resp.ContentLength = savecl
	if err != nil {
		return nil, err
	}
	return b.Bytes(), nil
}

// DumpResponse is like DumpRequest but dumps a response.
func DumpResponse(resp *http.Response, body bool) (dump []byte, err error) { // dump出响应内容
	var b bytes.Buffer
	save := resp.Body
	savecl := resp.ContentLength

	if !body {
		resp.Body = failureToReadBody{}
	} else if resp.Body == nil {
		resp.Body = emptyBody
	} else {
		save, resp.Body, err = drainBody(resp.Body)
		if err != nil {
			return
		}
	}
	err = resp.Write(&b)
	if err == errNoBody {
		err = nil
	}
	resp.Body = save
	resp.ContentLength = savecl
	if err != nil {
		return nil, err
	}
	return b.Bytes(), nil
}

func traceHttpResponse(c *Configuration, res *http.Response) {
	if res == nil {
		return
	}

	tracerx.Printf("HTTP: %d", res.StatusCode)

	if c.isTracingHttp == false {
		return
	}

	fmt.Fprintf(os.Stderr, "\n")
	fmt.Fprintf(os.Stderr, "< %s %s\n", res.Proto, res.Status)
	for key, _ := range res.Header {
		fmt.Fprintf(os.Stderr, "< %s: %s\n", key, res.Header.Get(key))
	}

	traceBody := false
	ctype := strings.ToLower(strings.SplitN(res.Header.Get("Content-Type"), ";", 2)[0])
	for _, tracedType := range tracedTypes {
		if strings.Contains(ctype, tracedType) {
			traceBody = true
		}
	}

	res.Body = newCountedResponse(res)
	if traceBody {
		res.Body = newTracedBody(res.Body)
	}

	fmt.Fprintf(os.Stderr, "\n")
}

func CopyHttpResponse(r *http.Response) *http.Response {

	resCopy := new(http.Response)
	if r == nil {
		return resCopy
	}

	*resCopy = *r

	if r.Body != nil {
		defer r.Body.Close()

		// Buffer body data
		var bodyBuffer bytes.Buffer
		bodyBuffer2 := new(bytes.Buffer)

		io.Copy(&bodyBuffer, r.Body)
		*bodyBuffer2 = bodyBuffer

		// Create new ReadClosers so we can split output
		r.Body = ioutil.NopCloser(&bodyBuffer)
		resCopy.Body = ioutil.NopCloser(bodyBuffer2)
	}

	return resCopy
}

func (srv *Server) handlerExecutePipeline(request *Request, keepAlive bool) *http.Response {

	var res *http.Response
	// execute the pipeline
	if res = srv.Pipeline.execute(request); res == nil {
		res = StringResponse(request.HttpRequest, 404, nil, "Not Found")
	}

	// The res.Write omits Content-length on 0 length bodies, and by spec,
	// it SHOULD. While this is not MUST, it's kinda broken.  See sec 4.4
	// of rfc2616 and a 200 with a zero length does not satisfy any of the
	// 5 conditions if Connection: keep-alive is set :(
	// I'm forcing chunked which seems to work because I couldn't get the
	// content length to write if it was 0.
	// Specifically, the android http client waits forever if there's no
	// content-length instead of assuming zero at the end of headers. der.
	if res.Body == nil {
		if request.HttpRequest.Method != "HEAD" {
			res.ContentLength = 0
		}
		res.TransferEncoding = []string{"identity"}
		res.Body = ioutil.NopCloser(bytes.NewBuffer([]byte{}))
	} else if res.ContentLength == 0 && len(res.TransferEncoding) == 0 && !((res.StatusCode-100 < 100) || res.StatusCode == 204 || res.StatusCode == 304) {
		// the following is copied from net/http/transfer.go
		// in the std lib, this is only applied to a request.  we need it on a response

		// Test to see if it's actually zero or just unset.
		var buf [1]byte
		n, _ := io.ReadFull(res.Body, buf[:])
		if n == 1 {
			// Oh, guess there is data in this Body Reader after all.
			// The ContentLength field just wasn't set.
			// Stich the Body back together again, re-attaching our
			// consumed byte.
			res.ContentLength = -1
			res.Body = &lengthFixReadCloser{io.MultiReader(bytes.NewBuffer(buf[:]), res.Body), res.Body}
		} else {
			res.TransferEncoding = []string{"identity"}
		}
	}
	if res.ContentLength < 0 && request.HttpRequest.Method != "HEAD" {
		res.TransferEncoding = []string{"chunked"}
	}

	// For HTTP/1.0 and Keep-Alive, sending the Connection: Keep-Alive response header is required
	// because close is default (opposite of 1.1)
	if keepAlive && !request.HttpRequest.ProtoAtLeast(1, 1) {
		res.Header.Set("Connection", "Keep-Alive")
	}

	// cleanup
	request.HttpRequest.Body.Close()
	return res
}

func CopyResponse(dest *http.Response, src *http.Response) {

	*dest = *src
	var bodyBytes []byte

	if src.Body != nil {
		bodyBytes, _ = ioutil.ReadAll(src.Body)
	}

	// Restore the io.ReadCloser to its original state
	src.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))
	dest.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))
}

// readResponse is like Read, but for HTTP response. Upon return, the body is
// replaced wit a buffer and can be read again.
func (srv *Server) readResponse(resp *http.Response) ([]byte, error) {
	p, err := srv.read(resp.Body, resp.ContentLength)
	resp.Body.Close()
	switch e := srv.uninstrument(err); {
	case e == io.EOF || e == io.ErrUnexpectedEOF || len(p) == 0:
		resp.Body = ioutil.NopCloser(eofReader{})
		return nil, io.EOF
	case err != nil:
		return nil, err
	case len(p) != 0:
		resp.Body = ioutil.NopCloser(bytes.NewReader(p))
	}
	return p, nil
}

func TestGetUser(t *testing.T) {

	g := New("clientID", "secret", "http://myapp.com/")
	creds := &common.Credentials{Map: objx.MSI()}

	testTripperFactory := new(test.TestTripperFactory)
	testTripper := new(test.TestTripper)
	testTripperFactory.On("NewTripper", mock.Anything, g).Return(testTripper, nil)
	testResponse := new(http.Response)
	testResponse.Header = make(http.Header)
	testResponse.Header.Set("Content-Type", "application/json")
	testResponse.StatusCode = 200
	testResponse.Body = ioutil.NopCloser(strings.NewReader(`{"full_name":"their-name","id":"uniqueid","username":"loginname","avatar_url":"http://myface.com/","online":true}`))
	testTripper.On("RoundTrip", mock.Anything).Return(testResponse, nil)

	g.tripperFactory = testTripperFactory

	user, err := g.GetUser(creds)

	if assert.NoError(t, err) && assert.NotNil(t, user) {
		assert.Equal(t, user.Name(), "their-name")
		assert.Equal(t, user.AuthCode(), "")
		assert.Equal(t, user.Nickname(), "loginname")
		assert.Equal(t, user.Email(), "") // doesn't come from soundcloud
		assert.Equal(t, user.AvatarURL(), "http://myface.com/")
		assert.Equal(t, user.Data()["online"], true)
	}

}

// Do sends an HTTP request with the provided http.Client and returns an HTTP response.
// If the client is nil, http.DefaultClient is used.
// If the context is canceled or times out, ctx.Err() will be returned.
func Do(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
	if client == nil {
		client = http.DefaultClient
	}

	// Request cancelation changed in Go 1.5, see cancelreq.go and cancelreq_go14.go.
	cancel := canceler(client, req)

	type responseAndError struct {
		resp *http.Response
		err  error
	}
	result := make(chan responseAndError, 1)

	// Make local copies of test hooks closed over by goroutines below.
	// Prevents data races in tests.
	testHookDoReturned := testHookDoReturned
	testHookDidBodyClose := testHookDidBodyClose

	go func() {
		resp, err := client.Do(req)
		testHookDoReturned()
		result <- responseAndError{resp, err}
	}()

	var resp *http.Response

	select {
	case <-ctx.Done():
		testHookContextDoneBeforeHeaders()
		cancel()
		// Clean up after the goroutine calling client.Do:
		go func() {
			if r := <-result; r.resp != nil {
				testHookDidBodyClose()
				r.resp.Body.Close()
			}
		}()
		return nil, ctx.Err()
	case r := <-result:
		var err error
		resp, err = r.resp, r.err
		if err != nil {
			return resp, err
		}
	}

	c := make(chan struct{})
	go func() {
		select {
		case <-ctx.Done():
			cancel()
		case <-c:
			// The response's Body is closed.
		}
	}()
	resp.Body = &notifyingReader{resp.Body, c}

	return resp, nil
}

func (self *Surfer) Download(sp *spider.Spider, cReq *request.Request) *spider.Context {
	ctx := spider.GetContext(sp, cReq)

	var resp *http.Response
	var err error

	switch cReq.GetDownloaderID() {
	case SURF_ID:
		resp, err = self.surf.Download(cReq)

	case PHANTOM_ID:
		resp, err = self.phantom.Download(cReq)
	}

	if resp.StatusCode >= 400 {
		err = errors.New("响应状态 " + resp.Status)
	}

	if resp.Header.Get("Content-Encoding") == "gzip" {
		var gzipReader *gzip.Reader
		gzipReader, err = gzip.NewReader(resp.Body)
		resp.Body.Close()
		if err == nil {
			resp.Body = ioutil.NopCloser(gzipReader)
		}
	}

	ctx.SetResponse(resp).SetError(err)

	return ctx
}

func fockHTTPServer(req *http.Request, support_version bool) (error, *http.Response) {
	if support_version {
		contentType := req.Header.Get("Content-Type:")
		if contentType != "application/octet-stream" {
			return errors.New("Content-Type: unmatched"), nil
		}
		if strings.EqualFold(req.Method, "POST") {
			return errors.New("method unmatched"), nil
		}
		protocol := req.Header.Get("application/X-DNSoverHTTP")
		if strings.EqualFold(protocol, "UDP") || strings.EqualFold(protocol, "TCP") {
			return errors.New("protocol isn't UDP or TCP"), nil
		}
		return nil, new(http.Response)
	} else {
		if strings.EqualFold(req.Method, "POST") {
			return errors.New("method unmatched"), nil
		}
		contentType := req.Header.Get("Content-Type:")
		if contentType != "application/X-DNSoverHTTP" {
			return errors.New("Content-Type: unmatched"), nil
		}
		protocol := req.Header.Get("X-Proxy-DNS-Transport")
		if strings.EqualFold(protocol, "UDP") || strings.EqualFold(protocol, "TCP") {
			return errors.New("protocol isn't UDP or TCP"), nil
		}
		res := new(http.Response)
		res.Body = req.Body
		return nil, res
	}
}

func buildTestHTTPResponse(body string) *http.Response {
	rsp := new(http.Response)

	rsp.Body = noopCloser{bytes.NewBufferString(body)}

	return rsp
}

// Get the robots.txt group for this crawler.
func (this *worker) getRobotsTxtGroup(b []byte, res *http.Response) (g *robotstxt.Group) {
	var data *robotstxt.RobotsData
	var e error

	if res != nil {
		// Get the bytes from the response body
		b, e = ioutil.ReadAll(res.Body)
		// Rewind the res.Body (by re-creating it from the bytes)
		res.Body = ioutil.NopCloser(bytes.NewBuffer(b))
		// Error or not, the robots.txt has been fetched, so notify
		this.extender.FetchedRobots(res)
	}

	if e == nil {
		data, e = robotstxt.FromBytes(b)
	}

	// If robots data cannot be parsed, will return nil, which will allow access by default.
	// Reasonable, since by default no robots.txt means full access, so invalid
	// robots.txt is similar behavior.
	if e != nil {
		this.extender.Error(newCrawlError(e, CekParseRobots, nil))
		this.logFunc(LogError, "ERROR parsing robots.txt for host %s: %s", this.host, e.Error())
	} else {
		g = data.FindGroup(this.robotUserAgent)
	}
	return
}

// save saves the body of a response corresponding to a request.
func (c *CachedRoundTrip) save(req *http.Request, resp *http.Response) error {
	if resp.StatusCode == http.StatusMovedPermanently || resp.StatusCode == http.StatusTemporaryRedirect {
		u, err := resp.Location()
		if err != nil {
			return err
		}

		err = c.Cache.Put(req.URL, c.newEntry([]byte("REDIRECT:"+u.String()), resp))
		if err != nil {
			return err
		}
		return nil
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	resp.Body.Close()
	err = c.Cache.Put(req.URL, c.newEntry(body, resp))
	if err != nil {
		return err
	}
	resp.Body = ioutil.NopCloser(bytes.NewReader(body))
	return nil
}