Python parse.unquote_plus方法代码示例

# 需要导入模块: from six.moves.urllib import parse [as 别名]
# 或者: from six.moves.urllib.parse import unquote_plus [as 别名]
def deurlquote(d, plus=False):
    """
    Decode a percent encoded string.

    Args:
        d(str): The percent encoded value to decode.
        plus(bool): Parse a plus symbol as a space.

    Returns:
        str: The decoded version of the percent encoded of ``d``.

    Example:
        >>> from pwny import *
        >>> deurlquote('Foo+Bar/Baz')
        'Foo Bar/Baz'
    """
    return unquote_plus(d) if plus else unquote(d) 

# 需要导入模块: from six.moves.urllib import parse [as 别名]
# 或者: from six.moves.urllib.parse import unquote_plus [as 别名]
def _parse_query(self):
        # type: () -> URI
        query = self.query if self.query is not None else ""
        query_dict = omdict()
        queries = query.split("&")
        query_items = []
        subdirectory = self.subdirectory if self.subdirectory else None
        for q in queries:
            key, _, val = q.partition("=")
            val = unquote_plus(val)
            if key == "subdirectory" and not subdirectory:
                subdirectory = val
            else:
                query_items.append((key, val))
        query_dict.load(query_items)
        return attr.evolve(
            self, query_dict=query_dict, subdirectory=subdirectory, query=query
        ) 

# 需要导入模块: from six.moves.urllib import parse [as 别名]
# 或者: from six.moves.urllib.parse import unquote_plus [as 别名]
def get_all(self, external_project_id, **kw):
        LOG.debug('Start certificate_authorities on_get')

        plugin_name = kw.get('plugin_name')
        if plugin_name is not None:
            plugin_name = parse.unquote_plus(plugin_name)

        plugin_ca_id = kw.get('plugin_ca_id', None)
        if plugin_ca_id is not None:
            plugin_ca_id = parse.unquote_plus(plugin_ca_id)

        # refresh CA table, in case plugin entries have expired
        cert_resources.refresh_certificate_resources()

        project_model = res.get_or_create_project(external_project_id)

        cas, offset, limit, total = self._get_subcas_and_root_cas(
            offset=kw.get('offset', 0),
            limit=kw.get('limit', None),
            plugin_name=plugin_name,
            plugin_ca_id=plugin_ca_id,
            project_id=project_model.id)

        return self._display_cas(cas, offset, limit, total) 

# 需要导入模块: from six.moves.urllib import parse [as 别名]
# 或者: from six.moves.urllib.parse import unquote_plus [as 别名]
def test_task_xhr_delete_a_task_with_job(app, client):
    task_id = metadata['task_id']
    # 'schedule.check' in test_check_with_task()
    text, __ = req_single_scrapyd(app, client, view='schedule.run', kws=dict(node=NODE),
                                  data=dict(filename=FILENAME_DV),
                                  location=metadata['location'])
    new_task_id = int(re.search(cst.TASK_NEXT_RUN_TIME_PATTERN, unquote_plus(text)).group(1))
    assert new_task_id - task_id == 1

    sleep()  # Wait until the first execution finish
    __, js = req_single_scrapyd(app, client, view='tasks.xhr', kws=dict(node=NODE, action='dump', task_id=new_task_id))
    check_dumped_task_data(js, version=cst.DEFAULT_LATEST_VERSION)

    __, js = req_single_scrapyd(app, client, view='tasks.xhr', kws=dict(node=NODE, action='list'))
    assert new_task_id in js['ids']
    tip = "apscheduler_job #{id} removed. Task #{id} deleted".format(id=new_task_id)
    req_single_scrapyd(app, client, view='tasks.xhr', kws=dict(node=NODE, action='delete', task_id=new_task_id),
                       jskws=dict(status=cst.OK, tip=tip))
    __, js = req_single_scrapyd(app, client, view='tasks.xhr', kws=dict(node=NODE, action='list'))
    assert new_task_id not in js['ids']

    message = "apscheduler_job #{id} not found. Task #{id} not found. ".format(id=new_task_id)
    __, js = req_single_scrapyd(app, client, view='tasks.xhr', kws=dict(node=NODE, action='dump', task_id=new_task_id),
                                jskws=dict(status=cst.ERROR, message=message))
    assert js['data'] is None 

# 需要导入模块: from six.moves.urllib import parse [as 别名]
# 或者: from six.moves.urllib.parse import unquote_plus [as 别名]
def test_run_with_task(app, client):
    # ScrapydWeb_demo.egg: custom_settings = {}, also output specific settings & arguments in the log
    upload_file_deploy(app, client, filename='ScrapydWeb_demo_no_request.egg', project=cst.PROJECT,
                       redirect_project=cst.PROJECT)

    req(app, client, view='tasks.xhr', kws=dict(node=NODE, action='enable'), ins='STATE_RUNNING', nos='STATE_PAUSED')

    with app.test_request_context():
        text, __ = req(app, client, view='schedule.run', kws=dict(node=NODE), data=run_data,
                       location=url_for('tasks', node=NODE))
    m = re.search(cst.TASK_NEXT_RUN_TIME_PATTERN, unquote_plus(text))
    task_id = int(m.group(1))
    print("task_id: %s" % task_id)
    metadata['task_id'] = task_id

    __, js = req(app, client, view='tasks.xhr', kws=dict(node=NODE, action='dump', task_id=task_id))
    assert js['data']['selected_nodes'] == [1, 2] 

# 需要导入模块: from six.moves.urllib import parse [as 别名]
# 或者: from six.moves.urllib.parse import unquote_plus [as 别名]
def _parse_uri(cls, server, sanitize_username=False):
        """
        Parses a MongoDB-formatted URI (e.g. mongodb://user:pass@server/db) and returns parsed elements
        and a sanitized URI.
        """
        parsed = pymongo.uri_parser.parse_uri(server)

        username = parsed.get('username')
        password = parsed.get('password')
        db_name = parsed.get('database')
        nodelist = parsed.get('nodelist')
        auth_source = parsed.get('options', {}).get('authsource')

        # Remove password (and optionally username) from sanitized server URI.
        # To ensure that the `replace` works well, we first need to url-decode the raw server string
        # since the password parsed by pymongo is url-decoded
        decoded_server = unquote_plus(server)
        clean_server_name = decoded_server.replace(password, "*" * 5) if password else decoded_server

        if sanitize_username and username:
            username_pattern = u"{}[@:]".format(re.escape(username))
            clean_server_name = re.sub(username_pattern, "", clean_server_name)

        return username, password, db_name, nodelist, clean_server_name, auth_source 

# 需要导入模块: from six.moves.urllib import parse [as 别名]
# 或者: from six.moves.urllib.parse import unquote_plus [as 别名]
def _parse_fragment(self):
        # type: () -> URI
        subdirectory = self.subdirectory if self.subdirectory else ""
        fragment = self.fragment if self.fragment else ""
        if self.fragment is None:
            return self
        fragments = self.fragment.split("&")
        fragment_items = {}
        name = self.name if self.name else ""
        extras = self.extras
        for q in fragments:
            key, _, val = q.partition("=")
            val = unquote_plus(val)
            fragment_items[key] = val
            if key == "egg":
                from .utils import parse_extras

                name, stripped_extras = pip_shims.shims._strip_extras(val)
                if stripped_extras:
                    extras = tuple(parse_extras(stripped_extras))
            elif key == "subdirectory":
                subdirectory = val
        return attr.evolve(
            self,
            fragment_dict=fragment_items,
            subdirectory=subdirectory,
            fragment=fragment,
            extras=extras,
            name=name,
        ) 

# 需要导入模块: from six.moves.urllib import parse [as 别名]
# 或者: from six.moves.urllib.parse import unquote_plus [as 别名]
def on_get(self, external_project_id, **kw):
        LOG.debug('Start certificate_authorities on_get (limited)')

        plugin_name = kw.get('plugin_name')
        if plugin_name is not None:
            plugin_name = parse.unquote_plus(plugin_name)

        plugin_ca_id = kw.get('plugin_ca_id', None)
        if plugin_ca_id is not None:
            plugin_ca_id = parse.unquote_plus(plugin_ca_id)

        # refresh CA table, in case plugin entries have expired
        cert_resources.refresh_certificate_resources()

        project_model = res.get_or_create_project(external_project_id)

        if self._project_cas_defined(project_model.id):
            cas, offset, limit, total = self._get_subcas_and_project_cas(
                offset=kw.get('offset', 0),
                limit=kw.get('limit', None),
                plugin_name=plugin_name,
                plugin_ca_id=plugin_ca_id,
                project_id=project_model.id)
        else:
            cas, offset, limit, total = self._get_subcas_and_root_cas(
                offset=kw.get('offset', 0),
                limit=kw.get('limit', None),
                plugin_name=plugin_name,
                plugin_ca_id=plugin_ca_id,
                project_id=project_model.id)

        return self._display_cas(cas, offset, limit, total) 

# 需要导入模块: from six.moves.urllib import parse [as 别名]
# 或者: from six.moves.urllib.parse import unquote_plus [as 别名]
def on_get(self, external_project_id, **kw):
        LOG.debug('Start transport_keys on_get')

        plugin_name = kw.get('plugin_name', None)
        if plugin_name is not None:
            plugin_name = parse.unquote_plus(plugin_name)

        result = self.repo.get_by_create_date(
            plugin_name=plugin_name,
            offset_arg=kw.get('offset', 0),
            limit_arg=kw.get('limit', None),
            suppress_exception=True
        )

        transport_keys, offset, limit, total = result

        if not transport_keys:
            transport_keys_resp_overall = {'transport_keys': [],
                                           'total': total}
        else:
            transport_keys_resp = [
                hrefs.convert_transport_key_to_href(s.id)
                for s in transport_keys
            ]
            transport_keys_resp_overall = hrefs.add_nav_hrefs(
                'transport_keys',
                offset,
                limit,
                total,
                {'transport_keys': transport_keys_resp}
            )
            transport_keys_resp_overall.update({'total': total})

        return transport_keys_resp_overall 

# 需要导入模块: from six.moves.urllib import parse [as 别名]
# 或者: from six.moves.urllib.parse import unquote_plus [as 别名]
def receive(method, query_string, body):
    # Determine the protocol used and pare the appropriate data.
    method = method.upper()
    if method == 'GET':
        data = parse_qs(_text(query_string))
        binding = 'artifact' if 'SAMLArtifact' in data else 'redirect'

    elif method == 'POST':
        data = parse_qs(_text(body))
        binding = 'post'

    else:
        # Unknown method used.
        return None

    if binding in ('redirect', 'post'):
        # Pull the text out of the query.
        encoded = data.get('SAMLResponse', data.get('SAMLRequest'))
        if not encoded:
            # No SAML message found.
            return None

        # Decode the text.
        text = base64.b64decode(encoded[0])
        if binding == "redirect":
            text = zlib.decompress(text, -15)

        # Parse the text into xml.
        message = etree.XML(text)

        # Get the relay state if present.
        relay_state = data.get('RelayState')
        if relay_state:
            relay_state = unquote_plus(relay_state[0])

        # Return the message and the relay state.
        return message, relay_state 

# 需要导入模块: from six.moves.urllib import parse [as 别名]
# 或者: from six.moves.urllib.parse import unquote_plus [as 别名]
def test_edit_to_update_a_task(app, client):
    day_of_week = 'mon-fri'
    task_id = metadata['task_id']
    __, js = req_single_scrapyd(app, client, view='tasks.xhr', kws=dict(node=NODE, action='dump', task_id=task_id))
    create_time = js['data']['create_time']
    update_time = js['data']['update_time']

    data = dict(DATA)
    data.update(task_id=task_id, day_of_week=day_of_week)  # modify day_of_week only
    req_single_scrapyd(app, client, view='schedule.check', kws=dict(node=NODE), data=data,
                       jskws=dict(cmd="-d _version=%s" % cst.VERSION, filename=FILENAME))
    text, __ = req_single_scrapyd(app, client, view='schedule.run', kws=dict(node=NODE),
                                  data=dict(filename=FILENAME),
                                  location=metadata['location'])
    assert int(re.search(cst.TASK_NEXT_RUN_TIME_PATTERN, unquote_plus(text)).group(1)) == task_id
    __, js = req_single_scrapyd(app, client, view='tasks.xhr', kws=dict(node=NODE, action='list'))
    assert task_id in js['ids'] and (task_id + 1) not in js['ids']

    sleep()
    __, js = req_single_scrapyd(app, client, view='tasks.xhr', kws=dict(node=NODE, action='dump', task_id=task_id))
    assert js['data']['create_time'] == create_time
    assert js['data']['update_time'] > update_time
    check_dumped_task_data(js, day_of_week=day_of_week)

    req_single_scrapyd(app, client, view='tasks.xhr', kws=dict(node=NODE, action='dump', task_id=task_id + 1),
                       jskws=dict(data=None, status=cst.ERROR, message="Task #%s not found" % (task_id + 1)))
    req_single_scrapyd(app, client, view='tasks', kws=dict(node=NODE),
                       ins=["id: %s," % task_id, "day_of_week: '%s'," % day_of_week], nos="id: %s," % (task_id + 1)) 

# 需要导入模块: from six.moves.urllib import parse [as 别名]
# 或者: from six.moves.urllib.parse import unquote_plus [as 别名]
def test_edit_to_new_a_task(app, client):
    day_of_week = '*'
    task_id = metadata['task_id']
    __, js = req_single_scrapyd(app, client, view='tasks.xhr', kws=dict(node=NODE, action='dump', task_id=task_id))
    create_time = js['data']['create_time']
    update_time = js['data']['update_time']

    data = dict(DATA)
    data.update(task_id=task_id, replace_existing='False', day_of_week=day_of_week)
    req_single_scrapyd(app, client, view='schedule.check', kws=dict(node=NODE), data=data,
                       jskws=dict(cmd="-d _version=%s" % cst.VERSION, filename=FILENAME))
    text, __ = req_single_scrapyd(app, client, view='schedule.run', kws=dict(node=NODE),
                                  data=dict(filename=FILENAME),
                                  location=metadata['location'])
    new_task_id = int(re.search(cst.TASK_NEXT_RUN_TIME_PATTERN, unquote_plus(text)).group(1))
    print("new_task_id: %s" % new_task_id)
    # assert new_task_id == task_id + 1
    # For compatibility with postgresql, though test_task_xhr_delete_a_task_with_job is executed before
    # https://stackoverflow.com/questions/9984196/postgresql-gapless-sequences
    assert new_task_id > task_id
    __, js = req_single_scrapyd(app, client, view='tasks.xhr', kws=dict(node=NODE, action='list'))
    assert task_id in js['ids'] and new_task_id in js['ids']

    __, js = req_single_scrapyd(app, client, view='tasks.xhr', kws=dict(node=NODE, action='dump', task_id=task_id))
    assert js['data']['create_time'] == create_time
    assert js['data']['update_time'] == update_time
    check_dumped_task_data(js, day_of_week='mon-fri')

    sleep()
    __, js = req_single_scrapyd(app, client, view='tasks.xhr', kws=dict(node=NODE, action='dump', task_id=new_task_id))
    check_dumped_task_data(js, day_of_week=day_of_week)

    req_single_scrapyd(app, client, view='tasks', kws=dict(node=NODE),
                       ins=["id: %s," % task_id, "day_of_week: 'mon-fri',",
                            "id: %s," % new_task_id, "day_of_week: '%s'," % day_of_week])
    req_single_scrapyd(app, client, view='tasks.xhr', kws=dict(node=NODE, action='delete', task_id=new_task_id),
                       jskws=dict(status=cst.OK, tip="Task #%s deleted" % new_task_id))


# POST data contains "task_id": "1", "replace_existing": "False", 

# 需要导入模块: from six.moves.urllib import parse [as 别名]
# 或者: from six.moves.urllib.parse import unquote_plus [as 别名]
def test_add_task_action_add(app, client):
    data_dv = dict(DATA_DV)
    data_dv.update(action='add', hour='*', minute='*', second='*')
    req_single_scrapyd(app, client, view='schedule.check', kws=dict(node=NODE), data=data_dv,
                       jskws=dict(filename=FILENAME_DV))
    text, __ = req_single_scrapyd(app, client, view='schedule.run', kws=dict(node=NODE),
                                  data=dict(filename=FILENAME_DV),
                                  location=metadata['location'])

    m = re.search(cst.TASK_NEXT_RUN_TIME_PATTERN, unquote_plus(text))
    task_id = int(m.group(1))
    print("task_id: %s" % task_id)
    next_run_time = m.group(2)
    print("next_run_time: %s" % next_run_time)
    assert next_run_time == "2036-12-31 00:00:00+08:00"

    __, js = req_single_scrapyd(app, client, view='tasks.xhr', kws=dict(node=NODE, action='list'))
    assert task_id in js['ids']
    __, js = req_single_scrapyd(app, client, view='tasks.xhr', kws=dict(node=NODE, action='dump', task_id=task_id))
    assert js['data']['apscheduler_job']['next_run_time'] == next_run_time

    with app.test_request_context():
        url_pause = url_for('tasks.xhr', node=NODE, action='pause', task_id=task_id)
        url_resume = url_for('tasks.xhr', node=NODE, action='resume', task_id=task_id)

    req_single_scrapyd(app, client, view='tasks', kws=dict(node=NODE),
                       ins=[url_pause, "prev_run_result: 'N/A',", "next_run_time: '%s'," % next_run_time,
                            "fail_times: 0,", "run_times: 0,"],
                       nos=url_resume)
    req_single_scrapyd(app, client, view='tasks', kws=dict(node=NODE, task_id=task_id),
                       ins=[TITLE_DV, 'label="Pass count"', ":total='0'"])
    req_single_scrapyd(app, client, view='tasks', kws=dict(node=NODE, task_id=task_id, task_result_id=cst.BIGINT),
                       ins=[TITLE_DV, 'label="Server"', ":total='0'"])
    req_single_scrapyd(app, client, view='tasks.xhr', kws=dict(node=NODE, action='delete', task_id=task_id))


# check POST  "action": "add_pause" 

# 需要导入模块: from six.moves.urllib import parse [as 别名]
# 或者: from six.moves.urllib.parse import unquote_plus [as 别名]
def test_add_task_with_default_values(app, client):
    data = dict(DATA)
    for k in data.keys():
        if k in ['jitter', 'misfire_grace_time', 'max_instances']:
            data[k] = 'invalid int'
        elif k not in ['project', '_version', 'spider', 'trigger']:  # if not request.form.get('trigger'): return
            data[k] = ''
    req_single_scrapyd(app, client, view='schedule.check', kws=dict(node=NODE), data=data,
                       jskws=dict(cmd="-d _version=%s" % cst.VERSION, filename=FILENAME))
    text, __ = req_single_scrapyd(app, client, view='schedule.run', kws=dict(node=NODE),
                                  data=dict(filename=FILENAME),
                                  location=metadata['location'])
    sleep()
    task_id = int(re.search(cst.TASK_NEXT_RUN_TIME_PATTERN, unquote_plus(text)).group(1))
    print("task_id: %s" % task_id)
    __, js = req_single_scrapyd(app, client, view='tasks.xhr', kws=dict(node=NODE, action='dump', task_id=task_id))
    assert js['data']['settings_arguments'] == {'setting': []}
    assert js['data']['selected_nodes'] == [1]
    assert js['data']['timezone'] is None
    assert js['data']['apscheduler_job']['misfire_grace_time'] == 600
    assert js['data']['apscheduler_job']['coalesce'] is True
    assert js['data']['apscheduler_job']['max_instances'] == 1
    assert js['data']['apscheduler_job']['name'] == 'task_%s' % task_id
    assert ':00:00' in js['data']['apscheduler_job']['next_run_time']
    for k, v in js['data']['apscheduler_job']['trigger'].items():
        if k in ['start_date', 'end_date']:
            assert v is None
        elif k in ['minute', 'second']:
            assert v == '0'
        elif k == 'jitter':
            assert v == 0
        elif k == 'timezone':
            assert v == str(get_localzone())
        else:
            assert v == '*'
    req_single_scrapyd(app, client, view='tasks.xhr', kws=dict(node=NODE, action='delete', task_id=task_id)) 

# 需要导入模块: from six.moves.urllib import parse [as 别名]
# 或者: from six.moves.urllib.parse import unquote_plus [as 别名]
def test_execute_task_fail(app, client):
    data = dict(DATA)
    # set_to_second
    req_single_scrapyd(app, client, view='schedule.check', kws=dict(node=NODE), data=data, set_to_second=True,
                       jskws=dict(cmd="-d _version=%s" % cst.VERSION, filename=FILENAME))
    text, __ = req_single_scrapyd(app, client, view='schedule.run', kws=dict(node=NODE),
                                  data=dict(filename=FILENAME),
                                  location=metadata['location'])
    task_id = int(re.search(cst.TASK_NEXT_RUN_TIME_PATTERN, unquote_plus(text)).group(1))
    print("task_id: %s" % task_id)
    # For compatibility with postgresql
    metadata['task_id'] = task_id
    sleep(2)
    # The first execution has not finished yet
    req_single_scrapyd(app, client, view='tasks', kws=dict(node=NODE),
                       ins=["id: %s," % task_id, "prev_run_result: 'FAIL 0, PASS 0',"])
    text, __ = req_single_scrapyd(app, client, view='tasks', kws=dict(node=NODE, task_id=task_id),
                                  ins=["fail_count: 0,", "pass_count: 0,", ":total='1'"])
    # in the task results page: url_action: '/1/tasks/xhr/delete/5/10/',
    with app.test_request_context():
        url_delete = url_for('tasks.xhr', node=NODE, action='delete', task_id=task_id)
    task_result_id = int(re.search(r'%s(\d+)/' % url_delete, text).group(1))
    print("task_result_id: %s" % task_result_id)
    sleep(28)
    req_single_scrapyd(app, client, view='tasks', kws=dict(node=NODE),
                       ins=["id: %s," % task_id, "prev_run_result: 'FAIL 1, PASS 0',"])
    req_single_scrapyd(app, client, view='tasks', kws=dict(node=NODE, task_id=task_id),
                       ins=["status_code: -1,", "status: 'error',", "Max retries exceeded", ":total='1'"],
                       nos="node: %s," % NODE)
    req_single_scrapyd(app, client, view='tasks', kws=dict(node=NODE, task_id=task_id, task_result_id=task_result_id),
                       ins=["node: %s," % NODE, "status_code: -1,", "status: 'error',",
                            "Max retries exceeded", ":total='1'"])
    req_single_scrapyd(app, client, view='tasks.xhr', kws=dict(node=NODE, action='delete', task_id=task_id))