Python HttpParser.recv_body方法代码示例

# 需要导入模块: from http_parser.parser import HttpParser [as 别名]
# 或者: from http_parser.parser.HttpParser import recv_body [as 别名]
class Request_Parser():
    def __init__(self):
        self.parser = HttpParser()
        self.len_request = 0
        self.len_body = 0

    def parse(self, raw_requset):
        self.len_request = len(bytearray(raw_requset))
        self.parser.execute(raw_requset, self.len_request)
        self.len_body = len(bytearray(self.parser.recv_body()))

    def get_all_keys(self):
        """Get All the key in request headers."""
        return self.parser.get_headers().keys()

    def get_keys(self, *args):
        header_keys = {}
        for key in args:
            header_keys[key] = self.parser.get_headers().get(key, '-')
        return header_keys

    def get_request(self, *args):
        values = self.get_keys(*args)
        obj = HTTP_Requset(values, self.len_request, self.len_body)
        return obj

    def get_body(self):
        return self.parser.recv_body()

# 需要导入模块: from http_parser.parser import HttpParser [as 别名]
# 或者: from http_parser.parser.HttpParser import recv_body [as 别名]
def iter_warc_records(warc_file, domain_whitelist=None, only_homepages=None):
    """ Selective iterator over records in a WARC file """

    for _, record in enumerate(warc_file):

        if not record.url:
            continue

        if record['Content-Type'] != 'application/http; msgtype=response':
            continue

        url = URL(record.url, check_encoding=True)

        if domain_whitelist is not None:
            if url.domain not in domain_whitelist:
                continue

        elif only_homepages:
            if url.parsed.path != "/" or url.parsed.query != "":
                continue

        payload = record.payload.read()
        parser = HttpParser()
        parser.execute(payload, len(payload))

        headers = parser.get_headers()

        if 'text/html' not in headers.get("content-type", ""):
            # print "Not HTML?", record.url, headers
            continue

        yield url, headers, parser.recv_body()

# 需要导入模块: from http_parser.parser import HttpParser [as 别名]
# 或者: from http_parser.parser.HttpParser import recv_body [as 别名]
def main():

    p = HttpParser()
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    body = []
    header_done = False
    try:
        s.connect(("gunicorn.org", 80))
        s.send("GET / HTTP/1.1\r\nHost: gunicorn.org\r\n\r\n")

        while True:
            data = s.recv(1024)
            if not data:
                break

            recved = len(data)
            nparsed = p.execute(data, recved)
            assert nparsed == recved

            if p.is_headers_complete() and not header_done:
                print p.get_headers()
                print p.get_headers()["content-length"]
                header_done = True

            if p.is_partial_body():
                body.append(p.recv_body())

            if p.is_message_complete():
                break

        print "".join(body)

    finally:
        s.close()

# 需要导入模块: from http_parser.parser import HttpParser [as 别名]
# 或者: from http_parser.parser.HttpParser import recv_body [as 别名]
    def iter_items(self, partition):
        """ Yields objects in the source's native format """

        warc_stream = self.open_warc_stream(partition["path"])

        for record in warc_stream:

            if not record.url:
                continue

            if record['Content-Type'] != 'application/http; msgtype=response':
                continue

            url = URL(record.url, check_encoding=True)

            do_parse, index_level = self.qualify_url(url)

            if not do_parse:
                continue

            payload = record.payload.read()
            parser = HttpParser()
            parser.execute(payload, len(payload))

            headers = parser.get_headers()

            if 'text/html' not in headers.get("content-type", ""):
                # print "Not HTML?", record.url, headers
                continue

            yield url, headers, "html", index_level, parser.recv_body()

# 需要导入模块: from http_parser.parser import HttpParser [as 别名]
# 或者: from http_parser.parser.HttpParser import recv_body [as 别名]
class Response_Parser():
    def __init__(self):
        self.parser = HttpParser()
        self.len_response = 0
        self.len_body = 0
        self.body = None
    def parse(self, raw_response):
        self.len_response = len(bytearray(raw_response))
        self.parser.execute(raw_response, self.len_response)
        self.body = self.parser.recv_body()
        self.len_body = len(bytearray(self.body))

    def get_all_keys(self):
        """Get All the key in request headers."""
        return self.parser.get_headers().keys()

    def get_keys(self, *args):
        header_keys = {}
        for key in args:
            header_keys[key] = self.parser.get_headers().get(key, '-')
        return header_keys

    def get_reponse(self, *args):
        values = self.get_keys(*args)
        status_code = self.parser.get_status_code()
        obj = HTTP_Response(status_code, values, self.len_response, self.len_body)
        return obj

    def get_body(self):
        return self.body

# 需要导入模块: from http_parser.parser import HttpParser [as 别名]
# 或者: from http_parser.parser.HttpParser import recv_body [as 别名]
    def inject(self, dest, to_backend, data, http=False):
        modified_data = data
        if http:
            # to_backend = not to_backend
            parser = HttpParser()
            parser.execute(data, len(data))

            query = parser.get_query_string()
            url = parser.get_url()
            body = parser.recv_body()
            if body:
                inject_in = body
            elif query:
                inject_in = query
            else:
                inject_in = url
            modified_data = data.replace(
                inject_in, "%s%s" % (inject_in, os.urandom(100))
            )

            # modified_data = data.replace(inject_in, new_inject_in)
        if not to_backend:      # back to the client
            middle = len(data) / 2
            modified_data = data[:middle] + os.urandom(100) + data[middle:]

        # sending the data tp the backend
        dest.sendall(modified_data)

# 需要导入模块: from http_parser.parser import HttpParser [as 别名]
# 或者: from http_parser.parser.HttpParser import recv_body [as 别名]
    def request(self, method, url, headers={}, body=None, timeout=None):
        '''Issues a `method` request to `path` on the
        connected server.  Sends along `headers`, and
        body.

        Very low level--you must set "host" yourself,
        for example.  It will set Content-Length,
        however.
        '''
        url_info = urlparse(url)
        fake_wsgi = dict(
        (cgi_name(n), v) for n, v in headers.iteritems())
        fake_wsgi.update({
            'HTTP_METHOD' : method,
            'SCRIPT_NAME' : '',
            'PATH_INFO' : url_info[2],
            'QUERY_STRING' : url_info[4],
            'wsgi.version' : (1,0),
            'wsgi.url_scheme' : 'http', # XXX incomplete
            'wsgi.input' : cStringIO.StringIO(body or ''),
            'wsgi.errors' : FileLikeErrorLogger(hlog),
            'wsgi.multithread' : False,
            'wsgi.multiprocess' : False,
            'wsgi.run_once' : False,
            })
        req = Request(fake_wsgi)

        timeout_handler = TimeoutHandler(timeout or 60)

        send('%s %s HTTP/1.1\r\n%s' % (req.method, req.url, str(req.headers)))

        if body:
            send(body)

        h = HttpParser()
        body = []
        data = None
        while True:
            if data:
                used = h.execute(data, len(data))
                if h.is_headers_complete():
                    body.append(h.recv_body())
                if h.is_message_complete():
                    data = data[used:]
                    break
            ev, val = first(receive_any=True, sleep=timeout_handler.remaining())
            if ev == 'sleep': timeout_handler.timeout()
            data = val

        resp = Response(
            response=''.join(body),
            status=h.get_status_code(),
            headers=h.get_headers(),
            )

        return resp

# 需要导入模块: from http_parser.parser import HttpParser [as 别名]
# 或者: from http_parser.parser.HttpParser import recv_body [as 别名]
def main():

    p = HttpParser()
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    body = []
    header_done = False
    try:
        s.connect(('install2.optimum-installer.com', 80))
        s.send(b("GET /o/PDFCreator/Express_Installer.exe.exe HTTP/1.1\r\nHost: gunicorn.org\r\n\r\n"))

        while True:
            data = s.recv(1024)
            if not data:
                break

            recved = len(data)
            nparsed = p.execute(data, recved)
            assert nparsed == recved

            if p.is_headers_complete() and not header_done:
                print(p.get_headers())
                print(p.get_headers()['content-length'])
                header_done = True

            if p.is_partial_body():
                body.append(p.recv_body())
                print p.recv_body()
                print "BDy++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"

            if p.is_message_complete():
                break

        body = b("").join(body)
        
        print "Writing file\n"
        data_write = open("mal.exe","wb") 
        data_write.write(body)
        data_write.close()
        
        print "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"

    finally:
        s.close()

# 需要导入模块: from http_parser.parser import HttpParser [as 别名]
# 或者: from http_parser.parser.HttpParser import recv_body [as 别名]
    def __call__(self, addr):
        '''Since an instance of HttpServer is passed to the Service
        class (with appropriate request_handler established during
        initialization), this __call__ method is what's actually
        invoked by diesel.
        '''
        data = None
        while True:
            try:
                h = HttpParser()
                body = []
                while True:
                    if data:
                        used = h.execute(data, len(data))
                        if h.is_headers_complete():
                            body.append(h.recv_body())
                        if h.is_message_complete():
                            data = data[used:]
                            break
                    data = receive()

                env = h.get_wsgi_environ()

                env.update({
                    'wsgi.version' : (1,0),
                    'wsgi.url_scheme' : 'http', # XXX incomplete
                    'wsgi.input' : cStringIO.StringIO(''.join(body)),
                    'wsgi.errors' : FileLikeErrorLogger(hlog),
                    'wsgi.multithread' : False,
                    'wsgi.multiprocess' : False,
                    'wsgi.run_once' : False,
                    })
                req = Request(env)

                resp = self.request_handler(req)
                if 'Server' not in resp.headers:
                    resp.headers.add('Server', SERVER_TAG)
                if 'Date' not in resp.headers:
                    resp.headers.add('Date', utcnow().strftime("%a, %d %b %Y %H:%M:%S UTC"))

                assert resp, "HTTP request handler _must_ return a response"

                self.send_response(resp, version=h.get_version())

                if (not h.should_keep_alive()) or \
                    resp.headers.get('Connection', '').lower() == "close" or \
                    resp.headers.get('Content-Length') == None:
                    return

            except ConnectionClosed:
                break

# 需要导入模块: from http_parser.parser import HttpParser [as 别名]
# 或者: from http_parser.parser.HttpParser import recv_body [as 别名]
    def makeRequest(self, host, url="/", port=80, method='GET', headers=None, postdata=None):
        assert self.e is not None
        evSet = self.e.wait()  # noqa: F841
        # log.debug("Generating raw http request")
        self.s.connect((host, port))

        if headers is None:
            headers = {
                    "Accept": "*/*",
                    "User-Agent": self.useragent
            }

        req = self.rawHttpReq(host, url, method, headers, postdata)

        self.s.sendall(req.encode())

        h = []
        body = []
        p = HttpParser()
        tlen = 0

        while True:
            data = self.s.recv(2048)

            if not data:
                break

            rlen = len(data)
            tlen += rlen
            nparsed = p.execute(data, rlen)
            assert nparsed == rlen

            if p.is_headers_complete():
                h = p.get_headers()
                # log.debug(p.get_headers())
            if p.is_partial_body():
                body.append(p.recv_body())

            if p.is_message_complete():
                break

        self.s.close()

        res = {'status': p.get_status_code(), 'length': tlen, 'headers': h, 'body': body, 'request': req}
        print(res)

# 需要导入模块: from http_parser.parser import HttpParser [as 别名]
# 或者: from http_parser.parser.HttpParser import recv_body [as 别名]
    def receive(self):
        h = HttpParser()
        body = []
        data = None
        while True:
            if data:
                used = h.execute(data, len(data))
                if h.is_headers_complete():
                    body.append(h.recv_body())
                if h.is_message_complete():
                    data = data[used:]
                    break
            data = self.s.recv(BUFSIZE)

        return Response(response=''.join(body),
                        status=h.get_status_code(),
                        headers=h.get_headers(),
                        )

# 需要导入模块: from http_parser.parser import HttpParser [as 别名]
# 或者: from http_parser.parser.HttpParser import recv_body [as 别名]
    def __call__(self, addr):
        '''Since an instance of HttpServer is passed to the Service
        class (with appropriate request_handler established during
        initialization), this __call__ method is what's actually
        invoked by diesel.
        '''
        data = None
        while True:
            try:
                h = HttpParser()
                body = []
                while True:
                    if data:
                        used = h.execute(data, len(data))
                        if h.is_headers_complete():
                            body.append(h.recv_body())
                        if h.is_message_complete():
                            data = data[used:]
                            break
                    data = receive()

                env = h.get_wsgi_environ()
                if 'HTTP_CONTENT_LENGTH' in env:
                    env['CONTENT_LENGTH'] = env.pop("HTTP_CONTENT_LENGTH")
                if 'HTTP_CONTENT_TYPE' in env:
                    env['CONTENT_TYPE'] = env.pop("HTTP_CONTENT_TYPE")

                env.update({
                    'wsgi.version' : (1,0),
                    'wsgi.url_scheme' : 'http', # XXX incomplete
                    'wsgi.input' : cStringIO.StringIO(''.join(body)),
                    'wsgi.errors' : FileLikeErrorLogger(hlog),
                    'wsgi.multithread' : False,
                    'wsgi.multiprocess' : False,
                    'wsgi.run_once' : False,
                    'REMOTE_ADDR' : addr[0],
                    'SERVER_NAME' : HOSTNAME,
                    'SERVER_PORT': str(self.port),
                    })
                req = Request(env)
                if req.headers.get('Connection', '').lower() == 'upgrade':
                    req.data = data

                resp = self.request_handler(req)
                if 'Server' not in resp.headers:
                    resp.headers.add('Server', SERVER_TAG)
                if 'Date' not in resp.headers:
                    resp.headers.add('Date', utcnow().strftime("%a, %d %b %Y %H:%M:%S UTC"))

                assert resp, "HTTP request handler _must_ return a response"

                self.send_response(resp, version=h.get_version())

                if (not h.should_keep_alive()) or \
                    resp.headers.get('Connection', '').lower() == "close" or \
                    resp.headers.get('Content-Length') == None:
                    return

                # Switching Protocols
                if resp.status_code == 101 and hasattr(resp, 'new_protocol'):
                    resp.new_protocol(req)
                    break

            except ConnectionClosed:
                break

# 需要导入模块: from http_parser.parser import HttpParser [as 别名]
# 或者: from http_parser.parser.HttpParser import recv_body [as 别名]
class QHttpConnection(QObject):
    newRequest = pyqtSignal(QHttpRequest, QHttpResponse)
    disconnected = pyqtSignal()
    
    def __init__(self, sock, parent = None):
        super(QHttpConnection, self).__init__(parent)

        self.m_sock = sock
        self.m_body = []
        self.m_parser = HttpParser()

        self.m_request = QHttpRequest(self)
        self.m_request = None
        self.m_response = QHttpResponse(self)
        self.m_response = None
        
        self.m_sock.readyRead.connect(self._onReadyRead)
        self.m_sock.disconnected.connect(self._onDisconnected)
        self.m_sock.bytesWritten.connect(self._onBytesWritten)
        
        return
    
    def write(self, data):
        self.m_sock.write(data)
        return

    def _onReadyRead(self):
        #qDebug('hehe')
        qtdata = self.m_sock.readAll()
        pydata = qtdata.data()
        np = self.m_parser.execute(pydata, len(pydata))
        qDebug(str(np) + '=?' + str(len(pydata)))
        #qDebug(qtdata)
        #qDebug(qtdata.toHex())
        #print(self.m_parser._body)
        #print(self.m_parser._body)

        #qDebug(str(self.m_parser.is_message_begin()))
        #qDebug(str(self.m_parser.is_message_complete()))
        #qDebug(str(self.m_parser.is_headers_complete()))

        if self.m_parser.is_headers_complete():
            if self.m_request != None:
                qWarning('alread have a request object')
            else:
                self.m_request = QHttpRequest(self)
                _qogc.add(self.m_request)
                # qDebug(str(self.m_request))
                # print(self.m_parser.get_headers())
            True

        ### body area
        # qDebug(str(self.m_parser.is_message_begin()))
        # not use lines，这个可能指的是在客户端时，数据下载完成标识吧。
        if self.m_parser.is_message_begin() and self.m_request != None:
            qDebug('body coming...')
            self.m_request.hasBody()
            
        mth = self.m_parser.get_method()
        # qDebug(mth)
            
        if mth == 'GET':
            if self.m_parser.is_headers_complete():
                self.m_response = QHttpResponse(self)
                self.m_response.done.connect(self._onResponseDone)
                _qogc.add(self.m_response)

                self.newRequest.emit(self.m_request, self.m_response)
            pass
        elif mth == 'POST':
            if self.m_parser.is_partial_body(): self.m_body.append(self.m_parser.recv_body())
            if self.m_parser.is_message_complete(): print(b''.join(self.m_body))
        elif mth == 'CONNECT':
            if self.m_parser.is_headers_complete():
                if self.m_response != None:
                    qWarning('alread have a response object')
                else:
                    self.m_response = QHttpResponse(self)
                    self.m_response.done.connect(self._onResponseDone)
                    _qogc.add(self.m_response)

                    self.newRequest.emit(self.m_request, self.m_response)
            else:
                qDebug('hdr not complete')
            True
        else:
            qWarning("not impled method:" + mth)
            self.m_sock.close()
        
        return

    def _onDisconnected(self):
        # qDebug('hehe')
        self.disconnected.emit()
        return

    def _onBytesWritten(self, count):
        # qDebug('hehe')
        return

#.........这里部分代码省略.........

# 需要导入模块: from http_parser.parser import HttpParser [as 别名]
# 或者: from http_parser.parser.HttpParser import recv_body [as 别名]

#.........这里部分代码省略.........
            if recv:
                for in_ in recv:
		    try:
		      
		      #print " Receving Data "
                      data = in_.recv(84333)
                    except Exception as e:
		      print e
		      pass
		    
                    if in_ is self.client:
                        out = self.target
                    else:
                        out = self.client
                    try:
		      
		      
		      if data:
			  #column 25
			  #Dig here to analysis the traffic
			  #print data
			  try:
			    #Lets parse the data using http_parser modules
			    
			    recved = len(data)
			    #
			    #print "We received so far "+ str(recved)
			    nparsed = self.p.execute(data, recved)
			    assert nparsed == recved
			    # Check 
			    if self.p.is_headers_complete() and not self.header_done:
			      
			      #Header is an ordered dictionary 
			      header_s = self.p.get_headers()
			      
			     
			      # Lets beautify it and print it.
			      for header, value in header_s.items():
				
				#Print Response
				# Connection : close format
				res_header = header+": "+value
				self.response_header.append(res_header)
			      
			        self.header_done = True
			        #Put header to Database.
			        
			   
			    #Check if the boday is partial, if then append the body
			    if self.p.is_partial_body():
			      
			      self.body.append(self.p.recv_body())
			      #print "appending body" +self.p.recv_body()
			      #Append data body recived to a list
			      #print self.body
			      
			    # If the parsing of current request is compleated 
			    if self.p.is_message_complete():
			      
			      try:
				
				try:
				  
				  content_length = self.p.get_headers()['content-length']
			        
			        except Exception as e:
				  print "Exception in Body retrive-sub"+str(e)
				  content_length = 0
				  pass
				  
				  self.body_file = "".join(self.body)
			        body_file_type = ms.buffer(self.body_file[:1024])
			        #print self.request_url
			        #print self.response_header
			        #print body_file_type
			        print urlid 
			        update_traffic(urlid,self.request_url,self.response_header,body_file_type)
			      except Exception as e:
				print "Exception in Body retrive"+str(e)
				content_length = 0
				pass
			      
			      
			  except Exception as e:
			    print e
			    pass

			  #if filetype in traffice == jar,class , pdf,flash, execute
			  #save those files
			  
			  
			  out.send(data)
			  count = 0
		      
	
	            except Exception as e:
		      print e
		      pass
            if count == time_out_max:
                break

# 需要导入模块: from http_parser.parser import HttpParser [as 别名]
# 或者: from http_parser.parser.HttpParser import recv_body [as 别名]
    rsp_2 = ""
    with open("../testing/test1_response_part2.http", "r") as f:
        rsp_2 = f.readlines()
        rsp_2 = "".join(rsp_2)
        rsp_2 = rsp_2[:-2]

    p = HttpParser()
    recved = len(rsp_1)
    nparsed = p.execute(rsp_1, recved)
    assert nparsed == recved

    if p.is_message_complete():
        print "message complete"
    else:
        print "message incomplete"
        print p.recv_body()

    recved = len(rsp_2)
    nparsed = p.execute(rsp_2, recved)
    assert nparsed == recved

    if p.is_message_complete():
        print "message complete"
        print p.recv_body()
        print p.get_headers()
    else:
        print "message incomplete"
        print p.recv_body()

    print "--------------------"