Python HTTPResponse.getheader方法代码示例

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import getheader [as 别名]
    def testKeepaliveHttp10(self):
        # Handling of Keep-Alive within HTTP 1.0
        data = "Default: Don't keep me alive"
        s = ("GET / HTTP/1.0\n"
             "Content-Length: %d\n"
             "\n"
             "%s") % (len(data), data)
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect((LOCALHOST, self.port))
        sock.send(s)
        response = ClientHTTPResponse(sock)
        response.begin()
        self.failUnlessEqual(int(response.status), 200)
        connection = response.getheader('Connection', '')
        # We sent no Connection: Keep-Alive header
        # Connection: close (or no header) is default.
        self.failUnless(connection != 'Keep-Alive')

        # If header Connection: Keep-Alive is explicitly sent,
        # we want to keept the connection open, we also need to return
        # the corresponding header
        data = "Keep me alive"
        s = ("GET / HTTP/1.0\n"
             "Connection: Keep-Alive\n"
             "Content-Length: %d\n"
             "\n"
             "%s") % (len(data), data)
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect((LOCALHOST, self.port))
        sock.send(s)
        response = ClientHTTPResponse(sock)
        response.begin()
        self.failUnlessEqual(int(response.status), 200)
        connection = response.getheader('Connection', '')
        self.failUnlessEqual(connection, 'Keep-Alive')

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import getheader [as 别名]
 def __init__(self, payload):
     r = HTTPResponse(self._fakeSocket(payload))
     r.begin()
     self.st = r.getheader("st") or None
     self.usn = r.getheader("usn") or None
     self.server = r.getheader("server") or None
     self.location = r.getheader("location") or None

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import getheader [as 别名]
    def __init__(self, response_text):
        self.fp = FakeSocket(response_text)
        res = HTTPR(self.fp)
        res.begin()

        headers = res.getheaders()
        for header in headers:
            self.headers[header[0]] = header[1]

        self.length = res.getheader('Content-Length')
        self.chunked = res.getheader('Transfer-Encoding')

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import getheader [as 别名]
    def testKeepaliveHttp11(self):
        # Handling of Keep-Alive within HTTP 1.1

        # All connections are kept alive, unless stated otherwise
        data = "Default: Keep me alive"
        s = ("GET / HTTP/1.1\n"
             "Content-Length: %d\n"
             "\n"
             "%s") % (len(data), data)
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect((LOCALHOST, self.port))
        sock.send(s)
        response = ClientHTTPResponse(sock)
        response.begin()
        self.failUnlessEqual(int(response.status), 200)
        self.failUnless(response.getheader('connection') != 'close')

        # Explicitly set keep-alive
        data = "Default: Keep me alive"
        s = ("GET / HTTP/1.1\n"
             "Connection: keep-alive\n"
             "Content-Length: %d\n"
             "\n"
             "%s") % (len(data), data)
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect((LOCALHOST, self.port))
        sock.send(s)
        response = ClientHTTPResponse(sock)
        response.begin()
        self.failUnlessEqual(int(response.status), 200)
        self.failUnless(response.getheader('connection') != 'close')

        # no idea why the test publisher handles this request incorrectly
        # it would be less typing in the test :)
        # h = HTTPConnection(LOCALHOST, self.port)
        # h.request("GET", "/")
        # response = h.getresponse()
        # self.failUnlessEqual(int(response.status), 200)
        # self.failUnless(response.getheader('connection') != 'close')

        # specifying Connection: close explicitly
        data = "Don't keep me alive"
        s = ("GET / HTTP/1.1\n"
             "Connection: close\n"
             "Content-Length: %d\n"
             "\n"
             "%s") % (len(data), data)
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect((LOCALHOST, self.port))
        sock.send(s)
        response = ClientHTTPResponse(sock)
        response.begin()
        self.failUnlessEqual(int(response.status), 200)
        self.failUnlessEqual(response.getheader('connection'), 'close')

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import getheader [as 别名]
    def process_record(self, record):
        content = None
        try:
            payload = record.payload.read()
            s = FakeSocket(payload)
            response = HTTPResponse(s)
            response.begin()
            status_code = response.status
            if status_code != 200:
                return
            content_type = response.getheader('Content-Type', '')
            if 'text/html' not in content_type:
                return

            headers = response.getheaders()
            content = response.read(len(payload))
        except Exception:
            self.increment_counter('errors', 'process_record', 1)
            logging.error('Error processing record: {}', traceback.format_exc())
            return

        if content is not None:
            content = content.strip()

        if not content:
            return

        for item in self.process_content(record.url, headers, content):
            yield item

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import getheader [as 别名]
def process_http(raw):
    headers, body = raw.split(BODY_SEP, 1)
    if headers.startswith('CONNECT'):
        return None
    
    if not len(body):
         return {
            'headers': headers,
            'body':  None
        }
    
    source = FakeSocket(raw)
    response = HTTPResponse(source)
    response.begin()
    body = None
    if response.getheader('Content-Encoding') == 'gzip':
        buf = StringIO(response.read(len(raw)))
        f = gzip.GzipFile(fileobj=buf)
        body = f.read()
    else:
        body = response.read(len(raw))
    
    return {
        'headers': headers,
        'body': body if len(body) else None
    }

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import getheader [as 别名]
    def _command(self, uri, method='GET', body='', **kwargs):
        """Makes an HTTP request through to an AirPlay server

        Args:
            uri(string):    The URI to request
            method(string): The HTTP verb to use when requesting `uri`, defaults to GET
            body(string):   If provided, will be sent witout alteration as the request body.
                            Content-Length header will be set to len(`body`)
            **kwargs:       If provided, Will be converted to a query string and appended to `uri`

        Returns:
            True: Request returned 200 OK, with no response body
            False: Request returned something other than 200 OK, with no response body

            Mixed: The body of the HTTP response
        """

        # generate the request
        if len(kwargs):
            uri = uri + '?' + urlencode(kwargs)

        request = method + " " + uri + " HTTP/1.1\r\nContent-Length: " + str(len(body)) + "\r\n\r\n" + body

        try:
            request = bytes(request, 'UTF-8')
        except TypeError:
            pass

        # send it
        self.control_socket.send(request)

        # parse our response
        result = self.control_socket.recv(self.RECV_SIZE)
        resp = HTTPResponse(FakeSocket(result))
        resp.begin()

        # if our content length is zero, then return bool based on result code
        if int(resp.getheader('content-length', 0)) == 0:
            if resp.status == 200:
                return True
            else:
                return False

        # else, parse based on provided content-type
        # and return the response body
        content_type = resp.getheader('content-type')

        if content_type is None:
            raise RuntimeError('Response returned without a content type!')

        if content_type == 'text/parameters':
            body = resp.read()
            try:
                body = str(body, 'UTF-8')
            except TypeError:
                pass

            return email.message_from_string(body)

        if content_type == 'text/x-apple-plist+xml':
            return plist_loads(resp.read())

        raise RuntimeError('Response received with unknown content-type: {0}'.format(content_type))

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import getheader [as 别名]
    def _command(self, uri, method='GET', body='', **kwargs):
        """Makes an HTTP request through to an AirPlay server

        Args:
            uri(string):    The URI to request
            method(string): The HTTP verb to use when requesting `uri`, defaults to GET
            body(string):   If provided, will be sent witout alteration as the request body.
                            Content-Length header will be set to len(`body`)
            **kwargs:       If provided, Will be converted to a query string and appended to `uri`

        Returns:
            True: Request returned 200 OK, with no response body
            False: Request returned something other than 200 OK, with no response body

            Mixed: The body of the HTTP response
        """

        # generate the request
        if len(kwargs):
            uri = uri + '?' + urlencode(kwargs)

        request = method + " " + uri + " HTTP/1.1\r\nContent-Length: " + str(len(body)) + "\r\n"
        request+="Host: %s:%s\r\n" % (self.host,self.port)
        request+="User-Agent: MediaControl/1.0\r\n"
        request+="X-Apple-Session-ID: c6c0033e-96f9-11e6-b0a4-a45e60c9debb\r\n"
        request+="Connection: close\r\n"
        request+="\r\n"
        request+=body


        try:
            if self.airplaySocket is None:
                self.airplaySocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                self.airplaySocket.settimeout(self.timeout)
                self.airplaySocket.connect((self.host, self.port))
        except socket.error as exc:
            self.airplaySocket=None
            raise ValueError("Unable to connect to {0}:{1}: {2}".format(self.host, self.port, exc))
        # send it
        rs=self.airplaySocket.sendall(request)

        # parse our response
        result = self.airplaySocket.recv(self.RECV_SIZE)
        resp = HTTPResponse(FakeSocket(result))
        resp.begin()

        # if our content length is zero, then return bool based on result code
        if int(resp.getheader('content-length', 0)) == 0:
            if resp.status == 200:
                return True
            else:
                return False

        # else, parse based on provided content-type
        # and return the response body
        content_type = resp.getheader('content-type')

        if content_type is None:
            raise RuntimeError('Response returned without a content type!')

        return resp.read()

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import getheader [as 别名]

#.........这里部分代码省略.........

	filepath = BASE_DIR + filename

	# if site was already cached
	if (filename in cachedSites or os.path.exists(filepath)):
		if ((calendar.timegm(time.gmtime()) - cachedSites[filename].lru) < cachedSites[filename].ttl) :
		# if (1 > 0):
			#if ttl hasn't expired
			#increment the frequency
			cachedSites[filename].frequency += 1
			print "The ttl hasn't expired"
			print "Reading content from file"
			file = open(filepath, 'r')
			content = file.read()
			print "Finished reading content from file"
			# then send the content to the client
			conn.send(content)
			return
	
	try:
		# create a socket to connect to the web server
		s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)  
		s.connect((webserver, port))
		s.send(request)         # send request to webserver
		
		while 1:
			# make sure cache has enough space 
			if (get_size(BASE_DIR) >= MAX_CACHE_SIZE):
				free_cache_memory()
				return
				
			# receive data from web server
			data = s.recv(MAX_DATA_RECV)			
			
			if (len(data) > 0):
				# create a temp source socket to parse the response
				source = FakeSocket(data)
				response = HTTPResponse(source)
				response.begin()


				cacheHeader = response.getheader('Cache-Control')
				maxAge = 10

				if (cacheHeader != 'no-cache' and (cacheHeader != None and 'must-revalidate' not in cacheHeader)):

					if 'max-age' in cacheHeader:
						maxAge = int(cacheHeader.split('=')[1])

					# check if the file exists in the dictionary
					if filename in cachedSites:

						if ((calendar.timegm(time.gmtime()) - cachedSites[filename].lru) >= cachedSites[filename].ttl) :
						# if (1 < 0):
							#if ttl has expired
							print "The ttl has expired"
							freq = cachedSites[filename].frequency + 1
							site = make_site(maxAge, freq, calendar.timegm(time.gmtime()))
							cachedSites[filename] = site							
							file = open(filepath, 'a')
							file.write(data)

						elif beganWritingToFile == True: 
								#if file is being read for the first time and we're still in the process of caching it
								print "Still in the process of caching the file"
								file = open(filepath, 'a')
								file.write(data)
								# then send the data to the client
								conn.send(data)

					else:
						print "Adding the site to the cache"
						print filename
						site = make_site(maxAge, 1, calendar.timegm(time.gmtime()))
						cachedSites[filename] = site
						beganWritingToFile = True
						file = open(filepath, 'a')
						file.write(data)
						# then send the data to the client
						conn.send(data)
				else:
					# just send the data to the client
					print "No cache, therefore, just sending the data directly to client"
					conn.send(data)

			else:
				beganWritingToFile = False
				break
				sys.exit(1)

		s.close()
		conn.close()

	except socket.error, (value, message):
		if s:
				s.close()
		if conn:
			conn.close()
		printout("Peer Reset",first_line,client_addr)
		sys.exit(1)

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import getheader [as 别名]
 lengths = []
 num_words = []
 distinct_words = []
 for record in f:
   if record['Content-Type'] != 'application/http; msgtype=response':
       print "Skipped", record['Content-Type']
       continue
   if 'WARC-Target-URI' in record:
     lengths.append(len(record['WARC-Target-URI']))
     content = record.payload.read()
     
     src = FakeSocket(content)
     response = HTTPResponse(src)
     response.begin()
     
     content_type = response.getheader('Content-Type')
     if content_type is not None and "text/html" not in content_type:
         print "Skipped content", response.getheader('Content-Type')
         continue
     
     #content_lengths.append(len(content))
     try:
         content = response.read()
         content_lengths.append(len(content))
         h = BeautifulSoup(content, "lxml")
     except IncompleteRead:
         continue
     for title in h.findAll('title'):
         print "TITLE", title.text.encode('utf-8')
           
     for body in h.findAll("body"):

# 需要导入模块: from httplib import HTTPResponse [as 别名]
# 或者: from httplib.HTTPResponse import getheader [as 别名]
  all_words = set([])

  for fileno, f in enumerate(files):
    f = warc.open(f)
    record_count = 0
    for record in f:
        if record['Content-Type'] != 'application/http; msgtype=response':
            continue
        if 'WARC-Target-URI' in record:
            content = record.payload.read()

            src = FakeSocket(content)
            response = HTTPResponse(src)
            response.begin()

            content_type = response.getheader('Content-Type')
            if content_type is not None and "text/html" not in content_type and "xml" not in content_type and "text" not in content_type:
                if "image" not in content_type:
                  print content_type
                  continue

            try:
                content = response.read()
                h = BeautifulSoup(content, "lxml")
            except IncompleteRead:
                continue

            for body in h.findAll("body"):
                #text = [i for i in body.recursiveChildGenerator() if type(i) == NavigableString]

                #text = [t.split() for t in text if len(t) > 0]