Python HttpParser.get_url方法代码示例

# 需要导入模块: from http_parser.parser import HttpParser [as 别名]
# 或者: from http_parser.parser.HttpParser import get_url [as 别名]
def proxy(data):
    recved = len(data)

    idx = data.find("\r\n")
    if idx <= 0:
        return

    line, rest = data[:idx], data[idx:]
    if line.startswith("CONNECT"):
        parts = line.split(None)
        netloc = parts[1]
        remote = parse_address(netloc, 80)

        reply_msg = "%s 200 OK\r\n\r\n" % parts[2]
        return {"remote": remote, 
                "reply": reply_msg,
                "data": ""}


    parser = HttpParser()
    parsed = parser.execute(data, recved)
    if parsed != recved:
        return  { 'close':'HTTP/1.0 502 Gateway Error\r\n\r\nError parsing request'}

    if not parser.get_url():
        return

    parsed_url = urlparse.urlparse(parser.get_url())

    is_ssl = parsed_url.scheme == "https"
    remote = parse_address(parsed_url.netloc, 80)

    return {"remote": remote, 
            "ssl": is_ssl}

# 需要导入模块: from http_parser.parser import HttpParser [as 别名]
# 或者: from http_parser.parser.HttpParser import get_url [as 别名]
def proxy(data):
    """
    the function called by tproxy to determine where to send traffic

    tproxy will call this function repeatedly for the same connection, as we
    receive more incoming data, until we return something other than None.

    typically our response tells tproxy where to proxy the connection to, but
    may also tell it to hang up, or respond with some error message.
    """

    log = logging.getLogger("proxy")

    bytes_received = len(data)

    parser =  HttpParser()
    bytes_parsed = parser.execute(data, bytes_received)

    if bytes_parsed != bytes_received:
        return { 'close': 
            'HTTP/1.0 400 Bad Request\r\n\r\nParse error' }

    if not parser.is_headers_complete(): 
        if bytes_received > MAX_HEADER_LENGTH:
            return { 'close': 
                'HTTP/1.0 400 Bad Request\r\n'
                '\r\nHeaders are too large' }
        return None

    headers = parser.get_headers()

    # the hostname may be in the form of hostname:port, in which case we want
    # to discard the port, and route just on hostname
    route_host = headers.get('HOST', None)
    if route_host:
        match = _HOST_PORT_REGEXP.match(route_host)
        if match:
            route_host = match.group(1)

    try:
        log.debug("Routing %r" % ( parser.get_url(), ))
        return _ROUTER.route(
            route_host,
            parser.get_method(),
            parser.get_path(),
            parser.get_query_string())
    except Exception, err:
        log.error("error routing %r, %s" % (
            parser.get_url(), traceback.format_exc(), ))
        gevent.sleep(ERROR_DELAY)
        return { 'close': 
            'HTTP/1.0 502 Gateway Error\r\n'
            '\r\nError routing request' }

# 需要导入模块: from http_parser.parser import HttpParser [as 别名]
# 或者: from http_parser.parser.HttpParser import get_url [as 别名]
    def inject(self, dest, to_backend, data, http=False):
        modified_data = data
        if http:
            # to_backend = not to_backend
            parser = HttpParser()
            parser.execute(data, len(data))

            query = parser.get_query_string()
            url = parser.get_url()
            body = parser.recv_body()
            if body:
                inject_in = body
            elif query:
                inject_in = query
            else:
                inject_in = url
            modified_data = data.replace(
                inject_in, "%s%s" % (inject_in, os.urandom(100))
            )

            # modified_data = data.replace(inject_in, new_inject_in)
        if not to_backend:      # back to the client
            middle = len(data) / 2
            modified_data = data[:middle] + os.urandom(100) + data[middle:]

        # sending the data tp the backend
        dest.sendall(modified_data)

# 需要导入模块: from http_parser.parser import HttpParser [as 别名]
# 或者: from http_parser.parser.HttpParser import get_url [as 别名]
 def __init__(self, raw):
     self.raw = raw
     req = HttpParser()
     req.execute(raw.request, len(raw.request))
     self.headers = req.get_headers()
     self.body = b"".join(req._body)
     self.url = req.get_url()
     self.path = req.get_path()
     self.method = req.get_method()
     self.arguments = req.get_query_string()
     self.slug = [a for a in self.path.split('/') if a != '']

# 需要导入模块: from http_parser.parser import HttpParser [as 别名]
# 或者: from http_parser.parser.HttpParser import get_url [as 别名]
class HttpStream(object):
    """ An HTTP parser providing higher-level access to a readable,
    sequential io.RawIOBase object. You can use implementions of
    http_parser.reader (IterReader, StringReader, SocketReader) or
    create your own.
    """

    def __init__(self, stream, kind=HTTP_BOTH, decompress=False):
        """ constructor of HttpStream.

        :attr stream: an io.RawIOBase object
        :attr kind: Int,  could be 0 to parseonly requests,
        1 to parse only responses or 2 if we want to let
        the parser detect the type.
        """
        self.parser = HttpParser(kind=kind, decompress=decompress)
        self.stream = stream

    def _check_headers_complete(self):
        if self.parser.is_headers_complete():
            return

        while True:
            try:
                next(self)
            except StopIteration:
                if self.parser.is_headers_complete():
                    return
                raise NoMoreData("Can't parse headers")

            if self.parser.is_headers_complete():
                return


    def _wait_status_line(self, cond):
        if self.parser.is_headers_complete():
            return True

        data = []
        if not cond():
            while True:
                try:
                    d = next(self)
                    data.append(d)
                except StopIteration:
                    if self.parser.is_headers_complete():
                        return True
                    raise BadStatusLine(b"".join(data))
                if cond():
                    return True
        return True

    def _wait_on_url(self):
        return self._wait_status_line(self.parser.get_url)

    def _wait_on_status(self):
        return self._wait_status_line(self.parser.get_status_code)

    def url(self):
        """ get full url of the request """
        self._wait_on_url()
        return self.parser.get_url()

    def path(self):
        """ get path of the request (url without query string and
        fragment """
        self._wait_on_url()
        return self.parser.get_path()

    def query_string(self):
        """ get query string of the url """
        self._wait_on_url()
        return self.parser.get_query_string()

    def fragment(self):
        """ get fragment of the url """
        self._wait_on_url()
        return self.parser.get_fragment()

    def version(self):
        self._wait_on_status()
        return self.parser.get_version()

    def status_code(self):
        """ get status code of a response as integer """
        self._wait_on_status()
        return self.parser.get_status_code()

    def status(self):
        """ return complete status with reason """
        status_code = self.status_code()
        reason = status_reasons.get(int(status_code), 'unknown')
        return "%s %s" % (status_code, reason)


    def method(self):
        """ get HTTP method as string"""
        self._wait_on_status()
        return self.parser.get_method()

#.........这里部分代码省略.........

# 需要导入模块: from http_parser.parser import HttpParser [as 别名]
# 或者: from http_parser.parser.HttpParser import get_url [as 别名]
class HttpProxyProtocol(asyncio.Protocol):
    ''' Implement HTTP(S) proxy behavior. '''

    def __init__(self, loop, config, token_store):
        ''' Constructor. '''

        self._parser = HttpParser()
        self._body = b''
        self._config = config
        self._loop = loop
        self._mitm = None
        self._mitm_host = None
        self._token_store = token_store

        self._instagram = InstagramApi(
            client_id=config['Instagram']['ClientID'],
            client_secret=config['Instagram']['ClientSecret'],
        )

        self._twitter = TwitterApi(
            consumer_key=config['Twitter']['ConsumerKey'],
            consumer_secret=config['Twitter']['ConsumerSecret'],
            app_token=config['Twitter']['AppToken'],
            app_token_secret=config['Twitter']['AppTokenSecret'],
        )

    def connection_made(self, transport):
        ''' Save a reference to the transport so that we can send a reply. '''
        log.debug('Connection opened.')
        self._transport = transport

    def connection_lost(self, exc):
        log.debug('Connection closed.')

    def data_received(self, data):
        ''' Parse incoming HTTP request. '''

        log.debug('Data received: {}'.format(data))
        self._parser.execute(data, len(data))

        if self._parser.is_partial_body():
            self._body += self._parser.recv_body()

        if self._parser.is_message_complete():
            method = self._parser.get_method()
            uri = self._parser.get_url()
            version = self._parser.get_version()
            headers = self._parser.get_headers()
            content_type = headers.get('Content-type', '')
            charset = _get_charset(content_type)
            body = self._body.decode(charset)

            log.debug('Client charset: {}'.format(charset))
            log.debug('Client status: method={} uri={} version={}' \
                      .format(method, uri, version))
            log.debug('Client headers: {}'.format(headers))
            log.debug('Client body: {}...'.format(body[:1000]))

            if method == 'CONNECT':
                asyncio.async(self._start_mitm(uri, version))
                self._parser = HttpParser()
            else:
                asyncio.async(
                    self._request_upstream(
                        method,
                        uri,
                        version,
                        headers,
                        body
                    )
                )


    def start_tls(self, version):
        '''
        Initiate TLS session with the client.

        This part is completely hacky! We mess around with the
        transport's internals in order to wrap the current transport in TLS.
        Python doesn't have an official way to do this, although it *might*
        get fixed in 3.6: http://bugs.python.org/issue23749
        '''

        log.debug('The proxy is starting TLS with its client.')

        status_line = 'HTTP/{}.{} {} {}\r\n\r\n' \
                      .format(version[0], version[1], 200, 'OK')
        self._transport.write(status_line.encode('ascii'))

        ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
        ssl_context.set_ciphers('HIGH:!aNull:!eNull')
        ssl_context.load_cert_chain('ssl/server.crt', 'ssl/server.key')

        original_socket = self._transport._sock
        self._transport = self._loop._make_ssl_transport(
            original_socket,
            self,
            ssl_context,
            server_side=True
        )
#.........这里部分代码省略.........