Python pycurl.URL属性代码示例

# 需要导入模块: import pycurl [as 别名]
# 或者: from pycurl import URL [as 别名]
def urlEncodeParams(data):
    """
    Return data URL-encoded.

    This function specifically handles None and boolean values
    to convert them to JSON-friendly strings (e.g. None -> 'null').
    """
    copy = dict()
    for key, value in six.iteritems(data):
        if value is None:
            copy[key] = 'null'
        elif isinstance(value, bool):
            copy[key] = json.dumps(value)
        else:
            copy[key] = value
    return urllib.urlencode(copy, doseq=True) 

# 需要导入模块: import pycurl [as 别名]
# 或者: from pycurl import URL [as 别名]
def _create_curl_conn(self, url):
        """
        Create a cURL connection object with useful default settings.
        """
        headers = []
        if self.user is not None and self.secret is not None:
            b64cred = base64.b64encode("{}:{}".format(self.user, self.secret))
            headers.append("Authorization: Basic {}".format(b64cred))

        conn = pycurl.Curl()

        if len(headers) > 0:
            conn.setopt(pycurl.HTTPHEADER, headers)

        conn.setopt(pycurl.URL, url)

        # github often redirects
        conn.setopt(pycurl.FOLLOWLOCATION, 1)

        return conn 

# 需要导入模块: import pycurl [as 别名]
# 或者: from pycurl import URL [as 别名]
def query(url):
  """
  Uses pycurl to fetch a site using the proxy on the SOCKS_PORT.
  """

  output = StringIO.StringIO()

  query = pycurl.Curl()
  query.setopt(pycurl.URL, url)
  query.setopt(pycurl.PROXY, 'localhost')
  query.setopt(pycurl.PROXYPORT, SOCKS_PORT)
  query.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5_HOSTNAME)
  query.setopt(pycurl.CONNECTTIMEOUT, CONNECTION_TIMEOUT)
  query.setopt(pycurl.WRITEFUNCTION, output.write)

  try:
    query.perform()
    return output.getvalue()
  except pycurl.error as exc:
    raise ValueError("Unable to reach %s (%s)" % (url, exc)) 

# 需要导入模块: import pycurl [as 别名]
# 或者: from pycurl import URL [as 别名]
def query(url):
  """
  Uses pycurl to fetch a site using the proxy on the SOCKS_PORT.
  """

  output = io.BytesIO()

  query = pycurl.Curl()
  query.setopt(pycurl.URL, url)
  query.setopt(pycurl.PROXY, 'localhost')
  query.setopt(pycurl.PROXYPORT, SOCKS_PORT)
  query.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5_HOSTNAME)
  query.setopt(pycurl.WRITEFUNCTION, output.write)

  try:
    query.perform()
    return output.getvalue()
  except pycurl.error as exc:
    return "Unable to reach %s (%s)" % (url, exc)


# Start an instance of Tor configured to only exit through Russia. This prints
# Tor's bootstrap information as it starts. Note that this likely will not
# work if you have another Tor instance running. 

# 需要导入模块: import pycurl [as 别名]
# 或者: from pycurl import URL [as 别名]
def _handle_request_error(self, e, url):
        if isinstance(e, urlfetch.InvalidURLError):
            msg = ("The Stripe library attempted to fetch an "
                   "invalid URL (%r). This is likely due to a bug "
                   "in the Stripe Python bindings. Please let us know "
                   "at support@stripe.com." % (url,))
        elif isinstance(e, urlfetch.DownloadError):
            msg = "There was a problem retrieving data from Stripe."
        elif isinstance(e, urlfetch.ResponseTooLargeError):
            msg = ("There was a problem receiving all of your data from "
                   "Stripe.  This is likely due to a bug in Stripe. "
                   "Please let us know at support@stripe.com.")
        else:
            msg = ("Unexpected error communicating with Stripe. If this "
                   "problem persists, let us know at support@stripe.com.")

        msg = textwrap.fill(msg) + "\n\n(Network error: " + str(e) + ")"
        raise error.APIConnectionError(msg) 

# 需要导入模块: import pycurl [as 别名]
# 或者: from pycurl import URL [as 别名]
def _check_status(self):
        """
        For internal use only. Only members of :class:`Tunnel`
        """
        try:
            output = io.BytesIO()

            query = pycurl.Curl()
            query.setopt(pycurl.URL, "%s:%s" % (self.addr, self.status_server.port))
            query.setopt(pycurl.PROXY, "127.0.0.1")
            query.setopt(pycurl.PROXYPORT, config.tor_proxy_port)
            query.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5_HOSTNAME)
            query.setopt(pycurl.WRITEFUNCTION, output.write)
            query.perform()

            return "ONIONGROK_CLIENT_STATUS_SUCCESS" in output.getvalue().decode("utf8")
        except Exception as e:
            return False 

# 需要导入模块: import pycurl [as 别名]
# 或者: from pycurl import URL [as 别名]
def __call__(self, json, throw):
        """Fetch the URL"""
        try:
            self.curl.perform()
            status = self.curl.getinfo(pycurl.HTTP_CODE)
            text = self.buf.getvalue()
        except pycurl.error as ex:
            (code, message) = ex
            status = 400
            text = message
        finally:
            self.curl.close()
            self.buf.close()
            
        #If status is outside the HTTP 2XX success  range
        if status < 200 or status > 299:
            if throw:
                raise URLException(text.strip())
            else:
                return (status, text)

        if json:
            return (status, json_load(text))
        else:
            return (status, text) 

# 需要导入模块: import pycurl [as 别名]
# 或者: from pycurl import URL [as 别名]
def url_get( url,          # GET URL
             params=None,  # GET parameters
             bind=None,    # Bind request to specified address
             json=True,    # Interpret result as JSON
             throw=True,   # Throw if status isn't 200
             timeout=None, # Seconds before giving up
             allow_redirects=True, # Allows URL to be redirected
             headers=None, # Hash of HTTP headers
             verify_keys=verify_keys_default  # Verify SSL keys
             ):
    """
    Fetch a URL using GET with parameters, returning whatever came back.
    """

    curl = PycURLRunner(url, params, bind, timeout, allow_redirects, headers, verify_keys)
    return curl(json, throw) 

# 需要导入模块: import pycurl [as 别名]
# 或者: from pycurl import URL [as 别名]
def url_post( url,          # GET URL
              params={},    # GET parameters
              data=None,    # Data to post
              content_type=None,  # Content type
              bind=None,    # Bind request to specified address
              json=True,    # Interpret result as JSON
              throw=True,   # Throw if status isn't 200
              timeout=None, # Seconds before giving up
              allow_redirects=True, #Allows URL to be redirected
              headers={},   # Hash of HTTP headers
              verify_keys=verify_keys_default  # Verify SSL keys
              ):
    """
    Post to a URL, returning whatever came back.
    """

    content_type, data = __content_type_data(content_type, headers, data)
    headers["Content-Type"] = content_type

    curl = PycURLRunner(url, params, bind, timeout, allow_redirects, headers, verify_keys)

    curl.curl.setopt(pycurl.POSTFIELDS, data)

    return curl(json, throw) 

# 需要导入模块: import pycurl [as 别名]
# 或者: from pycurl import URL [as 别名]
def url_put( url,          # GET URL
             params={},    # GET parameters
             data=None,    # Data for body
             content_type=None,  # Content type
             bind=None,    # Bind request to specified address
             json=True,    # Interpret result as JSON
             throw=True,   # Throw if status isn't 200
             timeout=None, # Seconds before giving up
             allow_redirects=True, #Allows URL to be redirected
             headers={},   # Hash of HTTP headers
             verify_keys=verify_keys_default  # Verify SSL keys
             ):
    """
    PUT to a URL, returning whatever came back.
    """

    content_type, data = __content_type_data(content_type, headers, data)
    headers["Content-Type"] = content_type

    curl = PycURLRunner(url, params, bind, timeout, allow_redirects, headers, verify_keys)

    curl.curl.setopt(pycurl.CUSTOMREQUEST, "PUT")
    curl.curl.setopt(pycurl.POSTFIELDS, data)

    return curl(json, throw) 

# 需要导入模块: import pycurl [as 别名]
# 或者: from pycurl import URL [as 别名]
def url_delete_list(
        urls,
        params=None,
        bind=None,
        timeout=None, # Seconds before giving up
        allow_redirects=True, #Allows URL to be redirected
        headers=None, # Hash of HTTP headers
        verify_keys=verify_keys_default  # Verify SSL keys
        ):
    """
    Delete a list of URLs and return tuples of the status and error for
    each.  Note that the timeout is per delete, not for the aggregated
    operation.
    """
    return [ url_delete(url, throw=False, timeout=timeout, params=params,
                        bind=bind, headers=headers, verify_keys=verify_keys, 
                        allow_redirects=allow_redirects)
             for url in urls ] 

# 需要导入模块: import pycurl [as 别名]
# 或者: from pycurl import URL [as 别名]
def perform_fp(fp, method, url, header='', body=''):
    #logger.debug('perform: %s' % url)
    fp.setopt(pycurl.URL, url)

    if method == 'GET':
      fp.setopt(pycurl.HTTPGET, 1)

    elif method == 'POST':
      fp.setopt(pycurl.POST, 1)
      fp.setopt(pycurl.POSTFIELDS, body)

    elif method == 'HEAD':
      fp.setopt(pycurl.NOBODY, 1)

    else:
      fp.setopt(pycurl.CUSTOMREQUEST, method)

    headers = [h.strip('\r') for h in header.split('\n') if h]
    fp.setopt(pycurl.HTTPHEADER, headers)

    fp.perform() 

# 需要导入模块: import pycurl [as 别名]
# 或者: from pycurl import URL [as 别名]
def put(url, data, encoding, headers=None):
    """Make a PUT request to the url, using data in the message body,
    with the additional headers, if any"""

    if headers is None:
        headers = {}
    reply = -1  # default, non-http response

    curl = pycurl.Curl()
    curl.setopt(pycurl.URL, url)
    if len(headers) > 0:
        curl.setopt(pycurl.HTTPHEADER, [k + ': ' + v for k, v in list(headers.items())])
    curl.setopt(pycurl.PUT, 1)
    curl.setopt(pycurl.INFILESIZE, len(data))
    databuffer = BytesIO(data.encode(encoding))
    curl.setopt(pycurl.READDATA, databuffer)
    try:
        curl.perform()
        reply = curl.getinfo(pycurl.HTTP_CODE)
    except Exception:
        pass
    curl.close()

    return reply 

# 需要导入模块: import pycurl [as 别名]
# 或者: from pycurl import URL [as 别名]
def fetch_many_async(urls, callback=None, errback=None, **kwargs):
    """
    Retrieve a list of URLs asynchronously.

    @param callback: Optionally, a function that will be fired one time for
        each successful URL, and will be passed its content and the URL itself.
    @param errback: Optionally, a function that will be fired one time for each
        failing URL, and will be passed the failure and the URL itself.
    @return: A C{DeferredList} whose callback chain will be fired as soon as
        all downloads have terminated. If an error occurs, the errback chain
        of the C{DeferredList} will be fired immediatly.
    """
    results = []
    for url in urls:
        result = fetch_async(url, **kwargs)
        if callback:
            result.addCallback(callback, url)
        if errback:
            result.addErrback(errback, url)
        results.append(result)
    return DeferredList(results, fireOnOneErrback=True, consumeErrors=True) 

# 需要导入模块: import pycurl [as 别名]
# 或者: from pycurl import URL [as 别名]
def fetch_to_files(urls, directory, logger=None, **kwargs):
    """
    Retrieve a list of URLs and save their content as files in a directory.

    @param urls: The list URLs to fetch.
    @param directory: The directory to save the files to, the name of the file
        will equal the last fragment of the URL.
    @param logger: Optional function to be used to log errors for failed URLs.
    """

    def write(data, url):
        filename = url_to_filename(url, directory=directory)
        fd = open(filename, "wb")
        fd.write(data)
        fd.close()

    def log_error(failure, url):
        if logger:
            logger("Couldn't fetch file from %s (%s)" % (
                url, str(failure.value)))
        return failure

    return fetch_many_async(urls, callback=write, errback=log_error, **kwargs)