Python MozillaCookieJar.add_cookie_header方法代码示例

# 需要导入模块: from cookielib import MozillaCookieJar [as 别名]
# 或者: from cookielib.MozillaCookieJar import add_cookie_header [as 别名]
class SimpleCrawler:

  USER_AGENT = 'SimpleCrawler/0.1'
  HEADERS = {
    'User-Agent': USER_AGENT,
    'Accept-Encoding': 'gzip',
    'Connection': 'keep-alive'
    }
  CONTENT_TYPE_PAT = re.compile(r'([^\s;]+)(.*charset=([^\s;]+))?', re.I)
  
  def __init__(self, starturl, index_html='', maxlevel=1,
               cookie_file=None, acldb=None, urldb=None, default_charset=None,
               delay=0, timeout=300, debug=0):
    (proto, self.hostport, _x, _y, _z) = urlsplit(starturl)
    assert proto == 'http'
    #Thread.__init__(self)
    self.debug = debug
    self.index_html = index_html
    if cookie_file:
      self.cookiejar = MozillaCookieJar(cookie_file)
      self.cookiejar.load()
    else:
      self.cookiejar = None
    self.robotstxt = RobotFileParser()
    self.robotstxt.set_url(urljoin(starturl, '/robots.txt'))
    self.robotstxt.read()
    self.conn = None
    self.urldb = urldb
    self.acldb = acldb
    self.curlevel = 0
    self.delay = delay
    self.timeout = timeout
    self.default_charset = default_charset
    if starturl.endswith('/'):
      starturl += self.index_html
    self.urls = [(starturl, maxlevel)]
    self.crawled = {}                   # 1:injected, 2:crawled
    return

  def accept_url(self, url):
    if url.endswith('/'):
      url += self.index_html
    if self.acldb and not self.acldb.allowed(url):
      return None
    return url
  
  def inject_url(self, url):
    if (not self.curlevel) or (not url) or (url in self.crawled): return False
    if not self.robotstxt.can_fetch(self.USER_AGENT, url):
      if self.debug:
        print >>stderr, 'DISALLOW: %r' % url
      return None
    if self.debug:
      print >>stderr, 'INJECT: %r' % url
    self.crawled[url] = 1
    self.urls.append((url, self.curlevel-1))
    return True

  def get1(self, url, maxretry=3, maxredirect=3):
    if self.debug:
      print >>stderr, 'GET: %r' % url
    # loop
    for rtry in range(maxredirect):
      # forge urllib2.Request object.
      req = Request(url)
      # add cookie headers if necessary.
      if self.cookiejar:
        self.cookiejar.add_cookie_header(req)
        headers = req.unredirected_hdrs
        headers.update(self.HEADERS)
      else:
        headers = self.HEADERS
      # get response.
      for ctry in range(maxretry):
        try:
          if not self.conn:
            print >>stderr, 'Making connection: %r...' % (self.hostport,)
            self.conn = HTTPConnection(self.hostport)
          self.conn.request('GET', req.get_selector().replace(' ',''), '', headers)
	  self.conn.sock.settimeout(self.timeout)
          resp = self.conn.getresponse()
          break
        except BadStatusLine, x:
          # connection closed unexpectedly
          print >>stderr, 'Connection closed unexpectedly.'
          # it restarts the connection...
          self.conn.close()
          self.conn = None
        except socket.error, x:
          # connection closed unexpectedly
          print >>stderr, 'Socket error:', x
          self.conn.close()
          self.conn = None
      else:

# 需要导入模块: from cookielib import MozillaCookieJar [as 别名]
# 或者: from cookielib.MozillaCookieJar import add_cookie_header [as 别名]

#.........这里部分代码省略.........
                    ("workout", ""),
                    ("workout", ""),
                    ("title", route),
                    ("show_on_menu", 0),
                    ("date", date_string),
                    ("warmup_mins", ""),
                    ("warmup_secs", ""),
                    ("warmup_distance", ""),
                    ("warmup_dist_unit", "miles"),
                    ("warmup_shoe_id", ""),
                    ("add_log", ""),
                    ("interval[0][reps]", ""),
                    ("interval[0][distance]", ""),
                    ("interval[0][dist_unit]", "miles"),
                    ("interval[0][rest]", ""),
                    ("interval[0][rest_unit]", "secs"),
                    ("interval[0][calculate_pace]", 1),
                    ("interval[0][shoe_id]", ""),
                    ("cooldown_mins", ""),
                    ("cooldown_secs", ""),
                    ("cooldown_distance", ""),
                    ("cooldown_dist_unit", "miles"),
                    ("cooldown_shoe_id", ""),
                    ("mins", int(minutes_component)),
                    ("secs", int(seconds_component)),
                    ("distance", distance_miles),
                    ("dist_unit", "miles"),
                    ("calculate_pace", 0),
                    ("calculate_pace", 1),
                    ("feel", ""),
                    ("field1", ""),
                    ("field2", ""),
                    ("shoe_id", ""),
                    ("notes", notes),
                    ("add_log", ""),]
        for i in range(len(run_info)):
            key = run_info[i][0]
            value = run_info[i][1]
            if key != "add_log" and "interval[0]" not in key:
                key = "RunningLogResource[%s]" % key
                run_info[i] = (key, value)
        return url_encode(run_info)

    def get_running_log_url(self, date):
        date_string = format(date, "%Y/%m/%d")
        return self.running_log_url % date_string

    def login(self, username, password):
        connection = HttpConnection(self.flotrack_domain)
        connection.set_debuglevel(self.debug_level)
        connection.connect()
        # Configure login parameters (this is the content of the HTTP request)
        params = { "LoginForm[username]": username,
                   "LoginForm[password]": password,
                   "LoginForm[rememberMe]": 1, }
        encoded_params = url_encode(params)
        # Get the HTTP headers to use
        headers = self.get_default_headers(encoded_params)
        del headers["Cookie"]
        # Configure the cookie jar to store the login information
        cookie_request = HttpRequestAdapter(self.flotrack_domain, self.login_url,
                                            headers)
        self.cookie_jar.add_cookie_header(cookie_request)
        # Issue the HTTP request
        request = connection.request("POST", self.login_url, encoded_params, headers)
        response = connection.getresponse()
        if response.status == OK:
            return False
        if response.status == FOUND:
            response.read()
            response.info = lambda: response.msg
            # Extract the login cookies
            self.cookie_jar.extract_cookies(response, cookie_request)
            self.connection = connection
            return True
        raise Exception("Flotrack connection failed during login.")

    def logout(self):
        request = self.connection.request("GET", self.logout_url)
        response = self.connection.getresponse()
        if response.status == OK:
            return False
        if response.status == FOUND:
            return True
        raise Exception("Flotrack connection failed during logout.")

    def record_run(self, date, route, distance_miles, time_minutes, notes=""):
        # Create parameters to pass to the log
        encoded_params = self.get_running_log_params(date, route, distance_miles, time_minutes, notes)
        # Post the data to the server
        headers = self.get_default_headers(encoded_params)
        running_log_url = self.get_running_log_url(date)
        request = self.connection.request("POST", running_log_url, encoded_params, headers)
        response = self.connection.getresponse()
        if response.status == OK:
            return False
        if response.status == FOUND:
            response.read()
            return True
        raise Exception("Flotrack connection failed while recording run.")