Python httplib.HTTPResponse类代码示例

 def parseNewResponse(self, packetString, src, dst, tcp):
     responseCode = packetString[9:12]
     if responseCode == '200': # just okay responses for now        
         if '\r\n\r\n' in packetString: # only proceed if the response has a body
             bodyIndex = packetString.index('\r\n\r\n') + 4
             body = packetString[bodyIndex:]
             socket = FakeSocket(packetString)
             response = HTTPResponse(socket)
             response.begin()
             headerArray = response.getheaders()
             for item in headerArray:
                 flowKey = (src, dst)
                 if item[0] == 'content-type' and 'text/html' in item[1]: # accept any kind of text content
                     print headerArray
                     for item in headerArray:
                         if item[0] == 'content-length':
                             print 'found fixed length'
                             length = int(item[1])
                             if length is not 0:
                                 self.parseFixedLengthResponse(flowKey, body, length, src, dst, tcp, responseCode)
                             else:
                                 print "warning, content-length is zero!"
                         elif item[0] == 'transfer-encoding' and item[1] == 'chunked':
                             print 'found chunked'
                             self.parseChunkedResponse(flowKey, body, src, dst, tcp, responseCode)
         else:
             print "body not found"

    def send_request(self, request):
        self.request_num += 1

        # Sanity check: if we're sending a request with a content-length but
        # we don't have that many bytes to send, we'll just get a 504.  Don't
        # send it and instead report a client error.

        parts = request.split('\r\n\r\n', 1)

        if len(parts) > 1:
            req, body = parts

            match = content_length_re.search(req)
            if match:
                if len(body) < int(match.group(1)):
                    self.error("request body of incorrect size")

                    return True

        if not self.connection:
            self._connect()

        if self.connection:
            # tally request method
            #self.tally(request.split(" ", 1)[0])

            try:
                self.connection.sendall(request)

                response = HTTPResponse(self.connection)
                response.begin()

                self.tally(response.status)

                while response.read():
                    pass

                if response.will_close:
                    # We hope our Connection: keep-alive won't be ignored, but
                    # deal with it if it does.
                    self._disconnect()

                if self.options.speedup < 0.8:
                    # if we're slowing down by too much, keep-alive will just
                    # result in the server getting bored between requests and
                    # dropping the connection, so disable it.
                    self._disconnect()

                return True
            except IncompleteRead:
                self.error("error while reading response: IncompleteRead (terminating job)")
                self._disconnect()

            except Exception, e:  # TODO: more restrictive error catching?
                self.error("error while sending request and reading response: %s %s" % (type(e), e))
                self._disconnect()

                if self.connection:
                    self.connection.close()
                    self.connection = None

 def do_proxy(self):
     try:
         if self.s == 0:
             self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
             self.s.connect((proxy_host, proxy_port))
         self.s.send(self.requestline.encode('ascii') + b'\r\n')
         # Add Sogou Verification Tags
         self.headers['X-Sogou-Auth'] = x_sogou_auth
         t = hex(int(time.time()))[2:].rstrip('L').zfill(8)
         self.headers['X-Sogou-Tag'] = calc_sogou_hash(t, self.headers['Host'])
         self.headers['X-Sogou-Timestamp'] = t
         self.s.send(str(self.headers).encode('ascii') + b'\r\n')
         # Send Post data
         if(self.command=='POST'):
             self.s.send(self.rfile.read(int(self.headers['Content-Length'])))
         response = HTTPResponse(self.s, method=self.command, buffering=True)
         response.begin()
         # Reply to the browser
         status = 'HTTP/1.1 ' + str(response.status) + ' ' + response.reason
         self.wfile.write(status.encode('ascii') + b'\r\n')
         h = ''
         for hh, vv in response.getheaders():
             if hh.upper()!='TRANSFER-ENCODING':
                 h += hh + ': ' + vv + '\r\n'
         self.wfile.write(h.encode('ascii') + b'\r\n')
         while True:
             response_data = response.read(8192)
             if(len(response_data) == 0):
                 break
             self.wfile.write(response_data)
     except socket.error:
         print('socket error for ' + self.requestline)

    def sogouProxy(self):
        if self.headers["Host"].startswith('chrome_dcp_proxy_pac.cnbeining'):  #Give a PAC file
            self.wfile.write("HTTP/1.1 200 OK".encode('ascii') + b'\r\n')
            hstr = '''Host: 127.0.0.1

function FindProxyForURL(url, host) {
  if (url.substring(0,5) == 'http:' && 
      !isPlainHostName(host) && 
      !shExpMatch(host, '*.local') && 
      !isInNet(dnsResolve(host), '10.0.0.0', '255.0.0.0') && 
      !isInNet(dnsResolve(host), '172.16.0.0',  '255.240.0.0') && 
      !isInNet(dnsResolve(host), '192.168.0.0',  '255.255.0.0') && 
      !isInNet(dnsResolve(host), '127.0.0.0', '255.255.255.0') ) 
    return 'PROXY ''' + server_ip + ':' + str(server_port) + '''; DIRECT';
  return 'DIRECT';
}'''
            self.wfile.write(hstr + b'\r\n')
            return
            
        if self.remote is None or self.lastHost != self.headers["Host"]:
            if PROXY_MODE == 'HTTPS':
                context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
                context.verify_mode = ssl.CERT_REQUIRED
                context.check_hostname = True
                context.load_default_certs()
                s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                s.settimeout(RemoteTimeout)
                self.remote = context.wrap_socket(s, server_hostname='proxy.googlezip.net')
                self.remote.connect(('proxy.googlezip.net', 443))
            else:  #HTTP
                self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                self.remote.settimeout(RemoteTimeout)
                self.remote.connect(("compress.googlezip.net", 80))
        self.remote.sendall(self.requestline.encode('ascii') + b"\r\n")
        # Add Verification Tags
        self.headers["Chrome-Proxy"] = get_google_header()
        headerstr = str(self.headers).replace("\r\n", "\n").replace("\n", "\r\n")
        self.remote.sendall(headerstr.encode('ascii') + b"\r\n")
        # Send Post data
        if self.command == 'POST':
            self.remote.sendall(self.rfile.read(int(self.headers['Content-Length'])))
        response = HTTPResponse(self.remote, method=self.command)
        response.begin()

        # Reply to the browser
        status = "HTTP/1.1 " + str(response.status) + " " + response.reason
        self.wfile.write(status.encode('ascii') + b'\r\n')
        hlist = []
        for line in response.msg.headers: # Fixed multiple values of a same name
            if 'TRANSFER-ENCODING' not in line.upper():
                hlist.append(line)
        self.wfile.write("".join(hlist) + b'\r\n')

        if self.command == "CONNECT":  # NO HTTPS, as Chrome DCP does not allow HTTPS traffic
            return
        else:
            while True:
                response_data = response.read(BufferSize)
                if not response_data: break
                self.wfile.write(response_data)

 def proxy(self):
     if self.remote is None or self.lastHost != self.headers['Host']:
         self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         self.remote.connect((proxy_host, proxy_port))
     self.remote.sendall(self.requestline.encode('ascii') + b'\r\n')
     # Add auth inf.
     self.headers['Proxy-Authorization'] = 'Basic %s' % base64.b64encode(auth)
     #self.remote.sendall('Proxy-Authorization: Basic %s\r\n' % base64.b64encode(auth))
     headerstr = str(self.headers).replace('\r\n', '\n').replace('\n', '\r\n')
     self.remote.sendall(headerstr.encode('ascii') + b"\r\n")
     # Send Post data
     if self.command == 'POST':
         self.remote.sendall(self.rfile.read(int(self.headers['Content-Length'])))
     response = HTTPResponse(self.remote, method=self.command)
     response.begin()
     
     # Reply to the browser
     status = 'HTTP/1.1 ' + str(response.status) + ' ' + response.reason
     self.wfile.write(status.encode('ascii') + b'\r\n')
     hlist = []
     for line in response.msg.headers: # Fixed multiple values of a same name
         if 'TRANSFER-ENCODING' not in line.upper():
             hlist.append(line)
     self.wfile.write(''.join(hlist) + b'\r\n')
     
     if self.command == 'CONNECT' and response.status == 200:
         return self.transfer(self.remote, self.connection)
     else:
         while True:
             response_data = response.read(BufferSize)
             if not response_data: break
             self.wfile.write(response_data)

 def sogouProxy(self):
     if self.remote is None or self.lastHost != self.headers["Host"]:
         self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         self.remote.settimeout(RemoteTimeout)
         self.remote.connect((proxy_host, proxy_port))
     self.remote.sendall(self.requestline.encode('ascii').replace('baike.baidu.com', '220.181.111.247') + b"\r\n")
     # Add Sogou Verification Tags
     self.headers["X-Sogou-Auth"] = x_sogou_auth
     t = hex(int(time.time()))[2:].rstrip('L').zfill(8)
     self.headers["X-Sogou-Tag"] = calc_sogou_hash(t, self.headers['Host'])
     self.headers["X-Sogou-Timestamp"] = t
     headerstr = str(self.headers).replace("\r\n", "\n").replace("\n", "\r\n")
     self.remote.sendall(headerstr.encode('ascii') + b"\r\n")
     # Send Post data
     if self.command == 'POST':
         self.remote.sendall(self.rfile.read(int(self.headers['Content-Length'])))
     response = HTTPResponse(self.remote, method=self.command)
     response.begin()
     
     # Reply to the browser
     status = "HTTP/1.1 " + str(response.status) + " " + response.reason
     self.wfile.write(status.encode('ascii') + b'\r\n')
     hlist = []
     for line in response.msg.headers: # Fixed multiple values of a same name
         if 'TRANSFER-ENCODING' not in line.upper():
             hlist.append(line)
     self.wfile.write("".join(hlist) + b'\r\n')
     
     if self.command == "CONNECT" and response.status == 200:
         return self.transfer(self.remote, self.connection)
     else:
         while True:
             response_data = response.read(BufferSize)
             if not response_data: break
             self.wfile.write(response_data)

def issue_row(raw_row):
    issue_row = {}
    for column in COLUMN_HEADERS:
        column_data_raw = raw_row.findtext(column)
        if column_data_raw:
            if column in ['issueDetail', 'issueBackground', 'remediationBackground']:
                issue_row[column] = htmltext(column_data_raw)
            else:
                issue_row[column] = column_data_raw

            if len(issue_row[column]) > 32000:
                issue_row[column] = "".join(issue_row[column][:32000], " [Text Cut Due To Length]")

    request = raw_row.findtext('./requestresponse/request')
    if request:
        parsed_request = HTTPRequest(binascii.a2b_base64(request))
        formatted_request_a = "command : {}\nuri : {}\nrequest_version : {}".format(parsed_request.command, parsed_request.path, parsed_request.request_version)
        formatted_request_b = "\n".join("{}: {}".format(header, parsed_request.headers[header]) for header in parsed_request.headers.keys())
        issue_row['requestHeaders'] = "{}\n{}".format(formatted_request_a, formatted_request_b)

    response = raw_row.findtext('./requestresponse/response')
    if response:
        parsed_response = HTTPResponse(FakeSocket(binascii.a2b_base64(response)))
        parsed_response.begin()
        formatted_response = "\n".join(["{} : {}".format(header_item[0], header_item[1]) for header_item in parsed_response.getheaders()])
        issue_row['responseHeaders'] = formatted_response

    return issue_row

def httpparse(fp):
    try:
        contents = fp.read()
    except AttributeError:
        contents = fp
    socket = FakeSocket(contents)
    response = HTTPResponse(socket)
    response.begin()
    return response

 def _parse_http(self, text):
     # if the response text starts with a 302, skip to the next non-302 header
     if re.match(r'^HTTP/.*?\s302 Found', text):
         m = re.search(r'(HTTP/\d+\.\d+\s(?!302 Found).*$)', text, re.S)
         if not m:
             raise Exception("Unrecognized response: %s" % text)
         else:
             text = m.group(1)
     socket = self.FakeSocket(text)
     response = HTTPResponse(socket)
     response.begin()
     return response

 def proccessHttpResponse(self, burpHttpReqResp):
     """ Processes only the HTTP repsonses with a CSP header """
     byteResponse = burpHttpReqResp.getResponse()
     httpSocket = HttpDummySocket(bytearray(byteResponse))
     response = HTTPResponse(httpSocket)
     response.begin()
     issues = []
     for header in response.getheaders():
         if header[0].lower() in ContentSecurityPolicy.HEADERS:
             findings = self.parseContentSecurityPolicy(header, burpHttpReqResp)
             issues.extend(findings)
     return issues

    def begin(self):
        HTTPResponse.begin(self)
        # To void the broken logic at the end of begin, because
        # but the connection may stay explicitly open with a connect
        # of for some other reason
        self.will_close = self._check_close()

        # it is sensible to assume that after a connect if
        # 200 is returned, the connection will not close
        # even if issued as 1.0 and no specific connection
        # header came back - a binary connection is now open
        if self._method == 'CONNECT' and self.status == 200:
            self.will_close = 0

    def setUp(self):
        from httplib import HTTPResponse

        with Stub() as HTTPResponse:
            HTTPResponse.status >> 200
            HTTPResponse.read() >> '{"id": "THE-PRECIOUS-GUID", "title": "example title", "subtitle": "example title - subtitle", "description": "a description", "user": "lfalvarez", "tags": [], "created_at": 1329767353.0, "source": "http://www.example.com/example.csv", "link": "http://www.junar.com/datastreams/some-url"}'

        with Stub() as conn:
            from httplib import HTTPConnection

            conn = HTTPConnection("api.junar.com")
            conn.request("POST", "/datastreams/publish", self.params, self.headers)
            conn.getresponse() >> HTTPResponse

    def get(self, request):
 
        # Format and pass along the initial request
        request = self.format_request(request)
        self.outsocket.sendall(request)
 
        # Wait for request
        while(True):
            r,w,x = select.select([self.insocket, self.outsocket],[],[],0)
            if self.insocket in r:
                request = self.insocket.recv(buff_size)
                if request == "":
                    print "Got end message from browser"
                    self.kill()
                else:
                    try:
                        request = self.format_request(request)
                    except MismatchedHost:
                        print "Host changed. Getting new end socket"
                        self.insocket.sendall("")
                        self.kill()                
                self.outsocket.sendall(request)
            if self.outsocket in r: 
                httpRes = HTTPResponse(self.outsocket)
                response = ""
                try:
                    httpRes.begin()
                    
                    headers = str(httpRes.msg)
                    content = httpRes.read()
                    
                    if headers.find("Transfer-Encoding: chunked") != -1:
                        headers = headers.replace("Transfer-Encoding: chunked\r\n", "")
                        headers += "Content Length: " + str(len(content)) + "\r\n"
                    
                    if httpRes.version == 10:
                        response += "HTTP/1.0 "
                    elif httpRes.version == 11:
                        response += "HTTP/1.1 "
                    response += str(httpRes.status) + " " + str(httpRes.reason) + nl
                    response += headers + nl
                    response += content
                    #print response
                except BadStatusLine:
                    self.kill()
                self.insocket.sendall(response)
                self.kill()
                #'''
            if self.killed:
                self.end()
                return           

    def testKeepaliveHttp10(self):
        # Handling of Keep-Alive within HTTP 1.0
        data = "Default: Don't keep me alive"
        s = ("GET / HTTP/1.0\n"
             "Content-Length: %d\n"
             "\n"
             "%s") % (len(data), data)
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect((LOCALHOST, self.port))
        sock.send(s)
        response = ClientHTTPResponse(sock)
        response.begin()
        self.failUnlessEqual(int(response.status), 200)
        connection = response.getheader('Connection', '')
        # We sent no Connection: Keep-Alive header
        # Connection: close (or no header) is default.
        self.failUnless(connection != 'Keep-Alive')

        # If header Connection: Keep-Alive is explicitly sent,
        # we want to keept the connection open, we also need to return
        # the corresponding header
        data = "Keep me alive"
        s = ("GET / HTTP/1.0\n"
             "Connection: Keep-Alive\n"
             "Content-Length: %d\n"
             "\n"
             "%s") % (len(data), data)
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect((LOCALHOST, self.port))
        sock.send(s)
        response = ClientHTTPResponse(sock)
        response.begin()
        self.failUnlessEqual(int(response.status), 200)
        connection = response.getheader('Connection', '')
        self.failUnlessEqual(connection, 'Keep-Alive')

    def _do_handshake(self, url, version=6, origin=None, cookie=None):
        """http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-06"""
        urlParts = urlparse.urlparse(url)
        key = b64encode(urandom(16))
        request  = "GET %s HTTP/1.1\r\n" \
                   "Host: %s\r\n" \
                   "Upgrade: WebSocket\r\n" \
                   "Connection: Upgrade\r\n" \
                   "Sec-WebSocket-Key: %s\r\n" \
                   "Sec-WebSocket-Version: %d\r\n" \
                   % (urlParts.path, self.host, key, version)

        if origin is not None:
            request += "Sec-WebSocket-Origin: %s\r\n" % (origin,)

        if cookie is not None:
            request += cookie.output(header="Cookie:") + "\r\n"

        # Authenticate if required
        if self.username is not None:
            request += "Authorization: Basic %s\r\n" \
                    % b64encode('%s:%s' % (self.username, self.password or ''))

        # Finish request
        request += "\r\n"
        if _debug:
            print >> sys.stderr, '\x1B[D\x1B[31m%s\x1B[m' % (request,),
        self.socket.sendall(request)

        response = HTTPResponse(self.socket)
        response.begin()

        if _debug:
            print >> sys.stderr, '\x1B[D\x1B[34m%s' % ({9: 'HTTP/0.9', 10: 'HTTP/1.0', 11: 'HTTP/1.1'}[response.version],), response.status, response.reason
            print >> sys.stderr, '%s\x1B[m' % (response.msg,)

        if response.status != 101:
            self.socket.close()
            raise RuntimeError("WebSocket upgrade failed: %d %s" % (response.status, response.reason))

        expected = sha1(key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").digest()
        assert len(expected) == 20
        expected = b64encode(expected)

        if 'Sec-WebSocket-Accept' not in response.msg:
            raise RuntimeError, "Expected WebSocket header not present: Sec-WebSocket-Accept"

        if response.msg['Sec-WebSocket-Accept'].strip() != expected:
            raise RuntimeError, "Invalid WebSocket accept returned: %s %s %s" % (key, expected, response.msg['Sec-WebSocket-Accept'])