Python urlparse.urljoin方法代码示例

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urljoin [as 别名]
def maybe_download_mnist_file(file_name, datadir=None, force=False):
    try:
        from urllib.parse import urljoin
        from urllib.request import urlretrieve
    except ImportError:
        from urlparse import urljoin
        from urllib import urlretrieve

    if not datadir:
        datadir = tempfile.gettempdir()
    dest_file = os.path.join(datadir, file_name)

    if force or not os.path.isfile(file_name):
        url = urljoin('http://yann.lecun.com/exdb/mnist/', file_name)
        urlretrieve(url, dest_file)
    return dest_file 

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urljoin [as 别名]
def instance_endpoint_by_id(self, id=None):
        """
        Returns the endpoint designating some instantiated API resource of the
        same kind. If no `id` is provided, will use the `id` of the currently
        instantiated resource. If this is called from a static object, then
        returns `None`.
        """
        _id = self._get_id(id=id)

        if _id:
            # CASE 1: The class end point might have a formatting parameter
            # NOTE: This is for the weird case of submissions of an assignment
            try:
                tmp = self.class_endpoint.format(_id)
                if tmp != self.class_endpoint:
                    return tmp
            except IndexError: # means formatting didn't work
                pass

            # CASE 2: The class end point has not formatting parameter
            # NOTE: Trailing slash important (API bug)
            return urljoin(self.class_endpoint, "{}/".format(_id)) 

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urljoin [as 别名]
def loginredirect():
    """Initialize OAuth login."""
    app.session_interface.abandon_session(app, session)

    redirecturl, request_token = handshaker.initiate()
    session['request_token_key'], session['request_token_secret'] = \
        request_token.key, request_token.secret
    session['return_to_url'] = url_for('main')

    returnto = request.args.get('returnto')
    if returnto:
        ref_url = urlparse(request.url_root)
        test_url = urlparse(urljoin(request.host_url, returnto))
        if (
            test_url.scheme == ref_url.scheme and
            test_url.netloc == ref_url.netloc and
            test_url.path.startswith(ref_url.path)
        ):
            session['return_to_url'] = returnto

    return redirect(redirecturl) 

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urljoin [as 别名]
def send_data_to_if(chunk_metric_data):
    send_data_time = time.time()

    # prepare data for metric streaming agent
    data_to_post = initialize_api_post_data()
    if 'DEPLOYMENT' in if_config_vars['project_type'] or 'INCIDENT' in if_config_vars['project_type']:
        for chunk in chunk_metric_data:
            chunk['data'] = json.dumps(chunk['data'])
    data_to_post[get_data_field_from_project_type()] = json.dumps(chunk_metric_data)

    logger.debug('First:\n' + str(chunk_metric_data[0]))
    logger.debug('Last:\n' + str(chunk_metric_data[-1]))
    logger.debug('Total Data (bytes): ' + str(get_json_size_bytes(data_to_post)))
    logger.debug('Total Lines: ' + str(track['line_count']))

    # do not send if only testing
    if cli_config_vars['testing']:
        return

    # send the data
    post_url = urlparse.urljoin(if_config_vars['if_url'], get_api_from_project_type())
    send_request(post_url, 'POST', 'Could not send request to IF',
                 str(get_json_size_bytes(data_to_post)) + ' bytes of data are reported.',
                 data=data_to_post, proxies=if_config_vars['if_proxies'])
    logger.debug('--- Send data time: %s seconds ---' % round(time.time() - send_data_time, 2)) 

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urljoin [as 别名]
def check_project(project_name):
    if 'token' in if_config_vars and len(if_config_vars['token']) != 0:
        logger.debug(project_name)
        try:
            # check for existing project
            check_url = urlparse.urljoin(if_config_vars['if_url'], '/api/v1/getprojectstatus')
            output_check_project = subprocess.check_output('curl "' + check_url + '?userName=' + if_config_vars['user_name'] + '&token=' + if_config_vars['token'] + '&projectList=%5B%7B%22projectName%22%3A%22' + project_name + '%22%2C%22customerName%22%3A%22' + if_config_vars['user_name'] + '%22%2C%22projectType%22%3A%22CUSTOM%22%7D%5D&tzOffset=-14400000"', shell=True)
            # create project if no existing project
            if project_name not in output_check_project:
                logger.debug('creating project')
                create_url = urlparse.urljoin(if_config_vars['if_url'], '/api/v1/add-custom-project')
                output_create_project = subprocess.check_output('no_proxy= curl -d "userName=' + if_config_vars['user_name'] + '&token=' + if_config_vars['token'] + '&projectName=' + project_name + '&instanceType=PrivateCloud&projectCloudType=PrivateCloud&dataType=' + get_data_type_from_project_type() + '&samplingInterval=' + str(if_config_vars['sampling_interval'] / 60) +  '&samplingIntervalInSeconds=' + str(if_config_vars['sampling_interval']) + '&zone=&email=&access-key=&secrete-key=&insightAgentType=' + get_insight_agent_type_from_project_type() + '" -H "Content-Type: application/x-www-form-urlencoded" -X POST ' + create_url + '?tzOffset=-18000000', shell=True)
            # set project name to proposed name
            if_config_vars['project_name'] = project_name
            # try to add new project to system
            if 'system_name' in if_config_vars and len(if_config_vars['system_name']) != 0:
                system_url = urlparse.urljoin(if_config_vars['if_url'], '/api/v1/projects/update')
                output_update_project = subprocess.check_output('no_proxy= curl -d "userName=' + if_config_vars['user_name'] + '&token=' + if_config_vars['token'] + '&operation=updateprojsettings&projectName=' + project_name + '&systemName=' + if_config_vars['system_name'] + '" -H "Content-Type: application/x-www-form-urlencoded" -X POST ' + system_url + '?tzOffset=-18000000', shell=True)
        except subprocess.CalledProcessError as e:
            logger.error('Unable to create project for ' + project_name + '. Data will be sent to ' + if_config_vars['project_name']) 

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urljoin [as 别名]
def get_hosts_for_app(base_url, headers, data, metrics_list, app_id):
    api = '/v2/applications/' + app_id + '/hosts.json'
    url = urlparse.urljoin(base_url, api)
    response = send_request(url, headers=headers, proxies=agent_config_vars['proxies'])
    try:
        response_json = json.loads(response.text)
        filter_hosts(base_url, headers, data, metrics_list, app_id, response_json['application_hosts'])
    # response = -1
    except TypeError as te:
        logger.warn('Failure when contacting NewRelic API when fetching hosts for app ' + app_id + '].')
        logger.warn(str(te))
        logger.warn(response.text)
    # malformed response_json
    # handles errors from filter_hosts
    except KeyError as ke:
        logger.warn('NewRelic API returned malformed data when fetching hosts for app ' + app_id + '].'
                    'Please contact support if this problem persists.')
        logger.warn(str(ke))
        logger.warn(response.text) 

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urljoin [as 别名]
def get_metrics_for_app_host(base_url, headers, data, metrics_list, app_id, instance='', host_id='', hostname=''):
    api = get_metrics_api(app_id, host_id)
    url = urlparse.urljoin(base_url, api)
    for metric in metrics_list:
        data_copy = data
        data_copy['names[]'] = metric
        data_copy['values[]'] = metrics_list[metric]
        response = send_request(url, headers=headers, proxies=agent_config_vars['proxies'], data=data_copy)
        try:
            metric_data = json.loads(response.text)
            parse_metric_data(metric_data['metric_data']['metrics'], instance, hostname)
        # response = -1
        except TypeError as te:
            logger.warn('Failure when contacting NewRelic API while fetching metrics ' +
                        'for app ' + app_id + ' & host ' + host_id + '.')
            logger.warn(str(te))
            logger.warn(response.text)
        # malformed response_json
        # handles errors from parse_metric_data as well
        except KeyError as ke:
            logger.warn('NewRelic API returned malformed data when fetching metrics ' +
                        'for app ' + app_id + ' & host ' + host_id + '.' +
                        'Please contact support if this problem persists.')
            logger.warn(str(ke))
            logger.warn(response.text) 

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urljoin [as 别名]
def send_data_to_if(chunk_metric_data):
    send_data_time = time.time()

    # prepare data for metric streaming agent
    data_to_post = initialize_api_post_data()
    if 'DEPLOYMENT' in if_config_vars['project_type'] or 'INCIDENT' in if_config_vars['project_type']:
        for chunk in chunk_metric_data:
            chunk['data'] = json.dumps(chunk['data'])
    data_to_post[get_data_field_from_project_type()] = json.dumps(chunk_metric_data)

    logger.debug('First:\n' + str(chunk_metric_data[0]))
    logger.debug('Last:\n' + str(chunk_metric_data[-1]))
    logger.info('Total Data (bytes): ' + str(get_json_size_bytes(data_to_post)))
    logger.info('Total Lines: ' + str(track['line_count']))

    # do not send if only testing
    if cli_config_vars['testing']:
        return

    # send the data
    post_url = urlparse.urljoin(if_config_vars['if_url'], get_api_from_project_type())
    send_request(post_url, 'POST', 'Could not send request to IF',
                 str(get_json_size_bytes(data_to_post)) + ' bytes of data are reported.',
                 data=data_to_post, verify=False, proxies=if_config_vars['if_proxies'])
    logger.info('--- Send data time: %s seconds ---' % round(time.time() - send_data_time, 2)) 

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urljoin [as 别名]
def zipkin_get_traces():
    """ get traces from zipkin for the last reporting period """
    data = {
        'lookback': int(if_config_vars['samplingInterval']) * 60 * 1000,
        'limit': int(if_config_vars['chunkLines'])
    }
    url = urlparse.urljoin(agent_config_vars['url'], '/api/v2/traces')
    resp = send_request(url, "GET", data, agent_config_vars['proxies'],
                        'Could not reach out to Zipkin APIs',
                        'Fetched traces from Zipkin')
    return json.loads(resp.text)


#################
# Configuration #
################# 

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urljoin [as 别名]
def send_data_to_if(chunk_metric_data, mode):
    """ sends data to IF. valid modes are METRIC, METRICREPLAY, LOG, LOGREPLAY, INCIDENT, INCIDENTREPLAY """
    send_data_time = time.time()
    # prepare data for metric streaming agent
    to_send_data_dict = initialize_api_post_data()
    to_send_data_dict["metricData"] = json.dumps(chunk_metric_data)
    to_send_data_dict["agentType"] = get_agent_type_from_mode(mode)

    to_send_data_json = json.dumps(to_send_data_dict)
    logger.debug("TotalData: " + str(len(bytearray(to_send_data_json))))
    logger.debug(to_send_data_json)

    if cli_config_vars['testing']:
        return

    # send the data
    post_url = urlparse.urljoin(if_config_vars['ifURL'], get_api_from_mode(mode))
    send_request(post_url, "POST", json.loads(to_send_data_json), if_config_vars['ifProxies'],
                 'Could not send request to IF',
                 str(len(bytearray(to_send_data_json))) + " bytes of data are reported.")
    logger.debug("--- Send data time: %s seconds ---" % (time.time() - send_data_time)) 

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urljoin [as 别名]
def get_links_from_url(url):
    """Download the page at `url` and parse it for links.

    Returned links have had the fragment after `#` removed, and have been made
    absolute so, e.g. the URL 'gen.html#tornado.gen.coroutine' becomes
    'http://www.tornadoweb.org/en/stable/gen.html'.
    """
    try:
        response = yield httpclient.AsyncHTTPClient().fetch(url)
        print('fetched %s' % url)

        html = response.body if isinstance(response.body, str) \
            else response.body.decode()
        urls = [urljoin(url, remove_fragment(new_url))
                for new_url in get_links(html)]
    except Exception as e:
        print('Exception: %s %s' % (e, url))
        raise gen.Return([])

    raise gen.Return(urls) 

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urljoin [as 别名]
def kat_page(root, page, sort_field, sort_order):
    from urlparse import urljoin
    from kmediatorrent.scrapers import rss
    from kmediatorrent.utils import url_get

    content_type = plugin.request.args_dict.get("content_type")
    if content_type:
        plugin.set_content(content_type)

    page = int(page)
    page_data = url_get(urljoin(BASE_URL, "%s/%d" % (root, page)), headers=HEADERS, params={
        "rss": "1",
        "field": sort_field,
        "sorder": sort_order
    })
    for item in rss.parse(page_data, content_type):
        yield item
    yield {
        "label": ">> Next page",
        "path": plugin.url_for("kat_page", root=root, page=page + 1, sort_field=sort_field, sort_order=sort_order, **plugin.request.args_dict),
    }