Python request.urlopen方法代码示例

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlopen [as 别名]
def raw_request(self, base_url, method_name, params, response_format):
        """
        Make a request to the Mixpanel API and return a raw urllib2/url.request file-like
        response object.
        """
        params['format'] = response_format
        # Getting rid of the None params
        params = self.check_params(params)
        url_without_params = '{base_url}/{version}/{method_name}/'.format(
            base_url=base_url,
            version=self.VERSION,
            method_name=method_name,
        )
        request_obj = self.client.auth.authenticate(url_without_params, params)
        effective_timeout = self.DEFAULT_TIMEOUT if self.client.timeout is None else self.client.timeout
        return url_request.urlopen(request_obj, timeout=effective_timeout) 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlopen [as 别名]
def _download_all_metadata(url):
    """
    Downloads the json file that contains all of the metadata for a specific file type (read:
    audio files, benchmark files, or trained models) that is on the EFZ server. This is retrieved
    from one of following three URLs (which are stored in nussl.constants):
    NUSSL_EFZ_AUDIO_METADATA_URL, NUSSL_EFZ_BENCHMARK_METADATA_URL, or NUSSL_EFZ_MODEL_METADATA_URL.

    Args:
        url (str):  URL for the EFZ server that has metadata. One of these three:
            NUSSL_EFZ_AUDIO_METADATA_URL, NUSSL_EFZ_BENCHMARK_METADATA_URL, or
            NUSSL_EFZ_MODEL_METADATA_URL.

    Returns:
        (list): List of dicts with metadata for the desired file type.

    """
    request = Request(url)

    # Make sure to get the newest data
    request.add_header('Pragma', 'no-cache')
    request.add_header('Cache-Control', 'max-age=0')
    try:
        return json.loads(urlopen(request).read())
    except:
        raise NoConnectivityError("Can't connect to internet") 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlopen [as 别名]
def download_sifts_xml(pdb_id, outdir='', force_rerun=False):
    """Download the SIFTS file for a PDB ID.

    Args:
        pdb_id (str): PDB ID
        outdir (str): Output directory, current working directory if not specified.
        force_rerun (bool): If the file should be downloaded again even if it exists

    Returns:
        str: Path to downloaded file

    """
    baseURL = 'ftp://ftp.ebi.ac.uk/pub/databases/msd/sifts/xml/'
    filename = '{}.xml.gz'.format(pdb_id.lower())

    outfile = op.join(outdir, filename.split('.')[0] + '.sifts.xml')

    if ssbio.utils.force_rerun(flag=force_rerun, outfile=outfile):
        response = urlopen(baseURL + filename)
        with open(outfile, 'wb') as f:
            f.write(gzip.decompress(response.read()))

    return outfile 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlopen [as 别名]
def available_examples():
    """
    Return a list of available example data files from the nbodykit
    data repository on NERSC.

    Returns
    -------
    examples : list
        list of the available file names for download
    """
    # read the contents of the main data URL
    response = urlopen(data_url)
    contents = response.read().decode()

    # parse the available files
    parser = ListingParser(data_url)
    parser.feed(contents)

    # get relative paths and remove bad links
    available = [os.path.relpath(link, data_url) for link in parser.links]
    available = [link for link in available if not any(link.startswith(bad) for bad in ['.', '?'])]
    return sorted(available) 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlopen [as 别名]
def check_url(docker_ip, public_port):
    """Check if a service is reachable.

    Makes a simple GET request to '/' of the HTTP endpoint. Service is
    available if returned status code is < 500.
    """
    url = 'http://{}:{}'.format(docker_ip, public_port)
    try:
        r = urlopen(url)
        return r.code < 500
    except HTTPError as e:
        # If service returns e.g. a 404 it's ok
        return e.code < 500
    except Exception:
        # Possible service not yet started
        return False 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlopen [as 别名]
def download_itis_database():
    print("Downloading ITIS data from: " + ITIS_URL)
    try:
        url = request.urlopen(ITIS_URL)
    except URLError:
        print("If you are operating behind a firewall, try setting the HTTP_PROXY/HTTPS_PROXY environment variables.")
        raise
    zipfile = ZipFile(BytesIO(url.read(int(url.headers['content-length']))))
    print("Download complete")
    named_temp_file = NamedTemporaryFile()
    itis_version = zipfile.filelist[0].filename.split('/')[0]
    db_file = None
    for f in zipfile.filelist:
        if f.filename.endswith('.sqlite'):
            db_file = f
            break
    with zipfile.open(db_file) as open_db_file:
        named_temp_file.write(open_db_file.read())
        named_temp_file.flush()
    return named_temp_file, itis_version 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlopen [as 别名]
def _getversions(self):
        """Fetch 'versions.csv'

        from versions_url config option and return the contained data as
        a dict with package name as a key.
        """
        versions_url = self.config_options.versions_url
        if not versions_url:
            fail_req_config_missing('versions_url')

        # return versions.csv as a dict with package name as a key
        vers = {}
        r = urlopen(versions_url)
        content = [x.decode('utf-8') for x in r.readlines()]
        # first line is headers
        for row in csv.reader(content[1:]):
            vers[row[0]] = row[1:]
        return vers 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlopen [as 别名]
def _readFromUrl(cls, url, writable):
        for attempt in retry_http():
            # We can only retry on errors that happen as responses to the request.
            # If we start getting file data, and the connection drops, we fail.
            # So we don't have to worry about writing the start of the file twice.
            with attempt:
                with closing(urlopen(url.geturl())) as readable:
                    # Make something to count the bytes we get
                    # We need to put the actual count in a container so our
                    # nested function can modify it without creating its own
                    # local with the same name.
                    size = [0]
                    def count(l):
                        size[0] += l
                    counter = WriteWatchingStream(writable)
                    counter.onWrite(count)
                    
                    # Do the download
                    shutil.copyfileobj(readable, counter)
                    return size[0] 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlopen [as 别名]
def _startMesos(self, numCores=None):
        if numCores is None:
            numCores = cpu_count()
        shutil.rmtree('/tmp/mesos', ignore_errors=True)
        self.master = self.MesosMasterThread(numCores)
        self.master.start()
        self.agent = self.MesosAgentThread(numCores)
        self.agent.start()
        
        # Wait for the master to come up.
        # Bad Things will happen if the master is not yet ready when Toil tries to use it.
        for attempt in retry(predicate=lambda e: True):
            with attempt:
                log.info('Checking if Mesos is ready...')
                with closing(urlopen('http://127.0.0.1:5050/version')) as content:
                    content.read()
        
        log.info('Mesos is ready! Running test.') 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlopen [as 别名]
def take_action(self, parsed_args):
        self.log.debug("take_action(%s)" % parsed_args)
        plan = parsed_args.plan
        outfile = parsed_args.output_file or '%s.tar.gz' % plan

        if os.path.exists(outfile) and not parsed_args.force_overwrite:
            raise exceptions.PlanExportError(
                "File '%s' already exists, not exporting." % outfile)

        print("Exporting plan %s..." % plan)

        tempurl = plan_management.export_deployment_plan(
            self.app.client_manager, plan
        )
        f = request.urlopen(tempurl)
        tarball_contents = f.read()
        f.close()

        with open(outfile, 'wb') as f:
            f.write(tarball_contents) 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlopen [as 别名]
def _is_pingable(self, mgmt_ip='', retry=5, timeout=5, port=80, **kwargs):
        """Checks whether the server is reachable by using urllib.

        Waits for connectivity for `timeout` seconds,
        and if connection refused, it will retry `retry`
        times.
        :param mgmt_ip: IP to check
        :param retry: times to reconnect if connection refused
        :param timeout: seconds to wait for connection
        :param port: port number to check connectivity
        :return: bool - True or False depending on pingability.
        """
        url = 'http://' + mgmt_ip + ':' + str(port)
        if netaddr.valid_ipv6(mgmt_ip):
            url = 'http://[' + mgmt_ip + ']:' + str(port)

        for retry_index in range(int(retry)):
            try:
                urlreq.urlopen(url, timeout=timeout)
                return True
            except urlerr.URLError:
                LOG.warning('Unable to reach to the url %s', url)
        return 'failure' 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlopen [as 别名]
def test_app(self):
        self.skipTest("Not ready yet")
        greetings = 'Hello, World!!!'

        def hello_world(env, start_response):
            if env['PATH_INFO'] != '/':
                start_response('404 Not Found',
                               [('Content-Type', 'text/plain')])
                return ['Not Found\r\n']
            start_response('200 OK', [('Content-Type', 'text/plain')])
            return [greetings]

        server = wsgi.Server("test_app")
        server.start(hello_world, 0, host="127.0.0.1")

        response = urllibrequest.urlopen('http://127.0.0.1:%d/' % server.port)
        self.assertEqual(greetings, response.read())

        server.stop() 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlopen [as 别名]
def test_app_using_ssl(self):
        CONF.set_default('use_ssl', True)
        CONF.set_default("ssl_cert_file",
                         os.path.join(TEST_VAR_DIR, 'certificate.crt'))
        CONF.set_default("ssl_key_file",
                         os.path.join(TEST_VAR_DIR, 'privatekey.key'))

        greetings = 'Hello, World!!!'

        @webob.dec.wsgify
        def hello_world(req):
            return greetings

        server = wsgi.Server("test_app")
        server.start(hello_world, 0, host="127.0.0.1")

        response = urllibrequest.urlopen('https://127.0.0.1:%d/' % server.port)
        self.assertEqual(greetings, response.read())

        server.stop() 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlopen [as 别名]
def data_is_valid(post_data, postback_server=POSTBACK_SERVER):
    """
    Validates data via the postback. Returns True if data is valid,
    False if data is invalid and None if the request failed.
    """
    post_str = urlencode(_values_to_encode(post_data))  # type: str
    # FIXME: No Content-Type header.
    post_bytes = post_str.encode(settings.DEFAULT_CHARSET)  # type: bytes

    postback_url = postback_server.rstrip('/') + POSTBACK_URL
    try:
        response = urlopen(postback_url, data=post_bytes)
        result = response.read().decode('utf-8')  # XXX: Assumed encoding
    except HTTPError:
        # XXX: Just re-raise for now.
        raise

    if result == 'VALID':
        return True
    elif result == 'INVALID':
        return False
    else:
        raise NotImplementedError('Unexpected result from PayFast validation: {!r}'.format(result)) 

# 需要导入模块: from six.moves.urllib import request [as 别名]
# 或者: from six.moves.urllib.request import urlopen [as 别名]
def _get_content(url):
  """Opens the url and loads the response into json."""
  logging.info('opening url %s', url)
  req = request.Request(url)
  resp = request.urlopen(req)
  resp_text = _as_text(resp.read())
  logging.info('response text = %s', resp_text)
  return json.loads(resp_text)