Python HTTPResponse.begin方法代码示例

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import begin [as 别名]
 def parseNewResponse(self, packetString, src, dst, tcp):
     responseCode = packetString[9:12]
     if responseCode == '200': # just okay responses for now        
         if '\r\n\r\n' in packetString: # only proceed if the response has a body
             bodyIndex = packetString.index('\r\n\r\n') + 4
             body = packetString[bodyIndex:]
             socket = FakeSocket(packetString)
             response = HTTPResponse(socket)
             response.begin()
             headerArray = response.getheaders()
             for item in headerArray:
                 flowKey = (src, dst)
                 if item[0] == 'content-type' and 'text/html' in item[1]: # accept any kind of text content
                     print headerArray
                     for item in headerArray:
                         if item[0] == 'content-length':
                             print 'found fixed length'
                             length = int(item[1])
                             if length is not 0:
                                 self.parseFixedLengthResponse(flowKey, body, length, src, dst, tcp, responseCode)
                             else:
                                 print "warning, content-length is zero!"
                         elif item[0] == 'transfer-encoding' and item[1] == 'chunked':
                             print 'found chunked'
                             self.parseChunkedResponse(flowKey, body, src, dst, tcp, responseCode)
         else:
             print "body not found"

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import begin [as 别名]
    def send_request(self, request):
        self.request_num += 1

        # Sanity check: if we're sending a request with a content-length but
        # we don't have that many bytes to send, we'll just get a 504.  Don't
        # send it and instead report a client error.

        parts = request.split('\r\n\r\n', 1)

        if len(parts) > 1:
            req, body = parts

            match = content_length_re.search(req)
            if match:
                if len(body) < int(match.group(1)):
                    self.error("request body of incorrect size")

                    return True

        if not self.connection:
            self._connect()

        if self.connection:
            # tally request method
            #self.tally(request.split(" ", 1)[0])

            try:
                self.connection.sendall(request)

                response = HTTPResponse(self.connection)
                response.begin()

                self.tally(response.status)

                while response.read():
                    pass

                if response.will_close:
                    # We hope our Connection: keep-alive won't be ignored, but
                    # deal with it if it does.
                    self._disconnect()

                if self.options.speedup < 0.8:
                    # if we're slowing down by too much, keep-alive will just
                    # result in the server getting bored between requests and
                    # dropping the connection, so disable it.
                    self._disconnect()

                return True
            except IncompleteRead:
                self.error("error while reading response: IncompleteRead (terminating job)")
                self._disconnect()

            except Exception, e:  # TODO: more restrictive error catching?
                self.error("error while sending request and reading response: %s %s" % (type(e), e))
                self._disconnect()

                if self.connection:
                    self.connection.close()
                    self.connection = None

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import begin [as 别名]
 def sogouProxy(self):
     if self.remote is None or self.lastHost != self.headers["Host"]:
         self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         self.remote.settimeout(RemoteTimeout)
         self.remote.connect((proxy_host, proxy_port))
     self.remote.sendall(self.requestline.encode('ascii').replace('baike.baidu.com', '220.181.111.247') + b"\r\n")
     # Add Sogou Verification Tags
     self.headers["X-Sogou-Auth"] = x_sogou_auth
     t = hex(int(time.time()))[2:].rstrip('L').zfill(8)
     self.headers["X-Sogou-Tag"] = calc_sogou_hash(t, self.headers['Host'])
     self.headers["X-Sogou-Timestamp"] = t
     headerstr = str(self.headers).replace("\r\n", "\n").replace("\n", "\r\n")
     self.remote.sendall(headerstr.encode('ascii') + b"\r\n")
     # Send Post data
     if self.command == 'POST':
         self.remote.sendall(self.rfile.read(int(self.headers['Content-Length'])))
     response = HTTPResponse(self.remote, method=self.command)
     response.begin()
     
     # Reply to the browser
     status = "HTTP/1.1 " + str(response.status) + " " + response.reason
     self.wfile.write(status.encode('ascii') + b'\r\n')
     hlist = []
     for line in response.msg.headers: # Fixed multiple values of a same name
         if 'TRANSFER-ENCODING' not in line.upper():
             hlist.append(line)
     self.wfile.write("".join(hlist) + b'\r\n')
     
     if self.command == "CONNECT" and response.status == 200:
         return self.transfer(self.remote, self.connection)
     else:
         while True:
             response_data = response.read(BufferSize)
             if not response_data: break
             self.wfile.write(response_data)

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import begin [as 别名]
    def send_and_recv(self):
        try:
            # because www.dream-pro.info is tlanslated to 127.0.0.1 using hosts' entry,
            # send message to www.dream-pro.info with socket.socket to make
            # http connection
            sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
            sock.connect((WWW_DREAM_PRO_INFO,80))
            sock.sendall(str(self))
        except:
            print 'SocketError'
            return

        res = HTTPResponse(sock)
        res.begin()
        res_body = res.read()
        res.close()

        if 'transfer-encoding' in res.msg:
            # httplib.HTTPResponse automatically concatenate chunked response
            # but do not delete 'transfer-encoding' header
            # so the header must be deleted
            res.msg.__delitem__('transfer-encoding')
        compmeth = res.msg.getheader('content-encoding','').lower()
        if compmeth and compmeth.find('identity') != 0 :
            # response body is compressed with some method
            offset = 0
            if compmeth.find('gzip') != -1:
                # if body is gziped, header offset value is 47
                # if not, offset value is 0
                # this server does not support sdch...
                offset += 47
            res_body = decompress(res_body,offset)
            res.msg['content-encoding'] = 'identity'

        return res, res_body

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import begin [as 别名]
    def testKeepaliveHttp10(self):
        # Handling of Keep-Alive within HTTP 1.0
        data = "Default: Don't keep me alive"
        s = ("GET / HTTP/1.0\n"
             "Content-Length: %d\n"
             "\n"
             "%s") % (len(data), data)
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect((LOCALHOST, self.port))
        sock.send(s)
        response = ClientHTTPResponse(sock)
        response.begin()
        self.failUnlessEqual(int(response.status), 200)
        connection = response.getheader('Connection', '')
        # We sent no Connection: Keep-Alive header
        # Connection: close (or no header) is default.
        self.failUnless(connection != 'Keep-Alive')

        # If header Connection: Keep-Alive is explicitly sent,
        # we want to keept the connection open, we also need to return
        # the corresponding header
        data = "Keep me alive"
        s = ("GET / HTTP/1.0\n"
             "Connection: Keep-Alive\n"
             "Content-Length: %d\n"
             "\n"
             "%s") % (len(data), data)
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect((LOCALHOST, self.port))
        sock.send(s)
        response = ClientHTTPResponse(sock)
        response.begin()
        self.failUnlessEqual(int(response.status), 200)
        connection = response.getheader('Connection', '')
        self.failUnlessEqual(connection, 'Keep-Alive')

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import begin [as 别名]
 def __init__(self, payload):
     r = HTTPResponse(self._fakeSocket(payload))
     r.begin()
     self.st = r.getheader("st") or None
     self.usn = r.getheader("usn") or None
     self.server = r.getheader("server") or None
     self.location = r.getheader("location") or None

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import begin [as 别名]
    def process_record(self, record):
        content = None
        try:
            payload = record.payload.read()
            s = FakeSocket(payload)
            response = HTTPResponse(s)
            response.begin()
            status_code = response.status
            if status_code != 200:
                return
            content_type = response.getheader('Content-Type', '')
            if 'text/html' not in content_type:
                return

            headers = response.getheaders()
            content = response.read(len(payload))
        except Exception:
            self.increment_counter('errors', 'process_record', 1)
            logging.error('Error processing record: {}', traceback.format_exc())
            return

        if content is not None:
            content = content.strip()

        if not content:
            return

        for item in self.process_content(record.url, headers, content):
            yield item

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import begin [as 别名]
    def sogouProxy(self):
        if self.headers["Host"].startswith('chrome_dcp_proxy_pac.cnbeining'):  #Give a PAC file
            self.wfile.write("HTTP/1.1 200 OK".encode('ascii') + b'\r\n')
            hstr = '''Host: 127.0.0.1

function FindProxyForURL(url, host) {
  if (url.substring(0,5) == 'http:' && 
      !isPlainHostName(host) && 
      !shExpMatch(host, '*.local') && 
      !isInNet(dnsResolve(host), '10.0.0.0', '255.0.0.0') && 
      !isInNet(dnsResolve(host), '172.16.0.0',  '255.240.0.0') && 
      !isInNet(dnsResolve(host), '192.168.0.0',  '255.255.0.0') && 
      !isInNet(dnsResolve(host), '127.0.0.0', '255.255.255.0') ) 
    return 'PROXY ''' + server_ip + ':' + str(server_port) + '''; DIRECT';
  return 'DIRECT';
}'''
            self.wfile.write(hstr + b'\r\n')
            return
            
        if self.remote is None or self.lastHost != self.headers["Host"]:
            if PROXY_MODE == 'HTTPS':
                context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
                context.verify_mode = ssl.CERT_REQUIRED
                context.check_hostname = True
                context.load_default_certs()
                s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                s.settimeout(RemoteTimeout)
                self.remote = context.wrap_socket(s, server_hostname='proxy.googlezip.net')
                self.remote.connect(('proxy.googlezip.net', 443))
            else:  #HTTP
                self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                self.remote.settimeout(RemoteTimeout)
                self.remote.connect(("compress.googlezip.net", 80))
        self.remote.sendall(self.requestline.encode('ascii') + b"\r\n")
        # Add Verification Tags
        self.headers["Chrome-Proxy"] = get_google_header()
        headerstr = str(self.headers).replace("\r\n", "\n").replace("\n", "\r\n")
        self.remote.sendall(headerstr.encode('ascii') + b"\r\n")
        # Send Post data
        if self.command == 'POST':
            self.remote.sendall(self.rfile.read(int(self.headers['Content-Length'])))
        response = HTTPResponse(self.remote, method=self.command)
        response.begin()

        # Reply to the browser
        status = "HTTP/1.1 " + str(response.status) + " " + response.reason
        self.wfile.write(status.encode('ascii') + b'\r\n')
        hlist = []
        for line in response.msg.headers: # Fixed multiple values of a same name
            if 'TRANSFER-ENCODING' not in line.upper():
                hlist.append(line)
        self.wfile.write("".join(hlist) + b'\r\n')

        if self.command == "CONNECT":  # NO HTTPS, as Chrome DCP does not allow HTTPS traffic
            return
        else:
            while True:
                response_data = response.read(BufferSize)
                if not response_data: break
                self.wfile.write(response_data)

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import begin [as 别名]
def process_http(raw):
    headers, body = raw.split(BODY_SEP, 1)
    if headers.startswith('CONNECT'):
        return None
    
    if not len(body):
         return {
            'headers': headers,
            'body':  None
        }
    
    source = FakeSocket(raw)
    response = HTTPResponse(source)
    response.begin()
    body = None
    if response.getheader('Content-Encoding') == 'gzip':
        buf = StringIO(response.read(len(raw)))
        f = gzip.GzipFile(fileobj=buf)
        body = f.read()
    else:
        body = response.read(len(raw))
    
    return {
        'headers': headers,
        'body': body if len(body) else None
    }

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import begin [as 别名]
def issue_row(raw_row):
    issue_row = {}
    for column in COLUMN_HEADERS:
        column_data_raw = raw_row.findtext(column)
        if column_data_raw:
            if column in ['issueDetail', 'issueBackground', 'remediationBackground']:
                issue_row[column] = htmltext(column_data_raw)
            else:
                issue_row[column] = column_data_raw

            if len(issue_row[column]) > 32000:
                issue_row[column] = "".join(issue_row[column][:32000], " [Text Cut Due To Length]")

    request = raw_row.findtext('./requestresponse/request')
    if request:
        parsed_request = HTTPRequest(binascii.a2b_base64(request))
        formatted_request_a = "command : {}\nuri : {}\nrequest_version : {}".format(parsed_request.command, parsed_request.path, parsed_request.request_version)
        formatted_request_b = "\n".join("{}: {}".format(header, parsed_request.headers[header]) for header in parsed_request.headers.keys())
        issue_row['requestHeaders'] = "{}\n{}".format(formatted_request_a, formatted_request_b)

    response = raw_row.findtext('./requestresponse/response')
    if response:
        parsed_response = HTTPResponse(FakeSocket(binascii.a2b_base64(response)))
        parsed_response.begin()
        formatted_response = "\n".join(["{} : {}".format(header_item[0], header_item[1]) for header_item in parsed_response.getheaders()])
        issue_row['responseHeaders'] = formatted_response

    return issue_row

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import begin [as 别名]
 def do_proxy(self):
     try:
         if self.s == 0:
             self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
             self.s.connect((proxy_host, proxy_port))
         self.s.send(self.requestline.encode('ascii') + b'\r\n')
         # Add Sogou Verification Tags
         self.headers['X-Sogou-Auth'] = x_sogou_auth
         t = hex(int(time.time()))[2:].rstrip('L').zfill(8)
         self.headers['X-Sogou-Tag'] = calc_sogou_hash(t, self.headers['Host'])
         self.headers['X-Sogou-Timestamp'] = t
         self.s.send(str(self.headers).encode('ascii') + b'\r\n')
         # Send Post data
         if(self.command=='POST'):
             self.s.send(self.rfile.read(int(self.headers['Content-Length'])))
         response = HTTPResponse(self.s, method=self.command, buffering=True)
         response.begin()
         # Reply to the browser
         status = 'HTTP/1.1 ' + str(response.status) + ' ' + response.reason
         self.wfile.write(status.encode('ascii') + b'\r\n')
         h = ''
         for hh, vv in response.getheaders():
             if hh.upper()!='TRANSFER-ENCODING':
                 h += hh + ': ' + vv + '\r\n'
         self.wfile.write(h.encode('ascii') + b'\r\n')
         while True:
             response_data = response.read(8192)
             if(len(response_data) == 0):
                 break
             self.wfile.write(response_data)
     except socket.error:
         print('socket error for ' + self.requestline)

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import begin [as 别名]
 def proxy(self):
     if self.remote is None or self.lastHost != self.headers['Host']:
         self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         self.remote.connect((proxy_host, proxy_port))
     self.remote.sendall(self.requestline.encode('ascii') + b'\r\n')
     # Add auth inf.
     self.headers['Proxy-Authorization'] = 'Basic %s' % base64.b64encode(auth)
     #self.remote.sendall('Proxy-Authorization: Basic %s\r\n' % base64.b64encode(auth))
     headerstr = str(self.headers).replace('\r\n', '\n').replace('\n', '\r\n')
     self.remote.sendall(headerstr.encode('ascii') + b"\r\n")
     # Send Post data
     if self.command == 'POST':
         self.remote.sendall(self.rfile.read(int(self.headers['Content-Length'])))
     response = HTTPResponse(self.remote, method=self.command)
     response.begin()
     
     # Reply to the browser
     status = 'HTTP/1.1 ' + str(response.status) + ' ' + response.reason
     self.wfile.write(status.encode('ascii') + b'\r\n')
     hlist = []
     for line in response.msg.headers: # Fixed multiple values of a same name
         if 'TRANSFER-ENCODING' not in line.upper():
             hlist.append(line)
     self.wfile.write(''.join(hlist) + b'\r\n')
     
     if self.command == 'CONNECT' and response.status == 200:
         return self.transfer(self.remote, self.connection)
     else:
         while True:
             response_data = response.read(BufferSize)
             if not response_data: break
             self.wfile.write(response_data)

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import begin [as 别名]
def httpparse(fp):
    try:
        contents = fp.read()
    except AttributeError:
        contents = fp
    socket = FakeSocket(contents)
    response = HTTPResponse(socket)
    response.begin()
    return response

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import begin [as 别名]
    def __init__(self, response_text):
        self.fp = FakeSocket(response_text)
        res = HTTPR(self.fp)
        res.begin()

        headers = res.getheaders()
        for header in headers:
            self.headers[header[0]] = header[1]

        self.length = res.getheader('Content-Length')
        self.chunked = res.getheader('Transfer-Encoding')

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import begin [as 别名]
 def _parse_http(self, text):
     # if the response text starts with a 302, skip to the next non-302 header
     if re.match(r'^HTTP/.*?\s302 Found', text):
         m = re.search(r'(HTTP/\d+\.\d+\s(?!302 Found).*$)', text, re.S)
         if not m:
             raise Exception("Unrecognized response: %s" % text)
         else:
             text = m.group(1)
     socket = self.FakeSocket(text)
     response = HTTPResponse(socket)
     response.begin()
     return response