Python FuturesSession.mount方法代码示例

# 需要导入模块: from requests_futures.sessions import FuturesSession [as 别名]
# 或者: from requests_futures.sessions.FuturesSession import mount [as 别名]
class APNsClient(object):
    def __init__(self, cert_file, use_sandbox=False, use_alternative_port=False):
        server = 'api.development.push.apple.com' if use_sandbox else 'api.push.apple.com'
        port = 2197 if use_alternative_port else 443
        self.cert = cert_file
        self.server = "https://{}:{}".format(server, port)
        self.__connection = FuturesSession()
        self.__connection.mount('https://', HTTP20Adapter())

    def send_notification(self, tokens, notification, priority=NotificationPriority.Immediate, topic=None):
        # print(notification.dict())
        json_payload = json.dumps(notification.dict(), ensure_ascii=False, separators=(',', ':')).encode('utf-8')

        headers = {
            'apns-priority': priority.value
        }
        if topic:
            headers['apns-topic'] = topic

        if not isinstance(tokens, list):
            tokens = [tokens]

        for token in tokens:
            url = '{}/3/device/{}'.format(self.server, token)
            self.__connection.post(url, json_payload, headers=headers, cert=self.cert, background_callback=req_callback)

# 需要导入模块: from requests_futures.sessions import FuturesSession [as 别名]
# 或者: from requests_futures.sessions.FuturesSession import mount [as 别名]
    def _get_raw_data(self):
        docktor_config = providers_config.providers['docktor']
        apps = []
        session = FuturesSession(max_workers=CONCURRENT_JOBS_LIMIT)
        session.mount('https://', self.__requests_http_adapter)
        session.mount('http://', self.__requests_http_adapter)
        for stage in docktor_config:
            for zone in docktor_config[stage]:
                apps_uri = '{uri}/apps/'.format(uri=docktor_config[stage][zone]['uri'])
                try:
                    r = session.get(apps_uri, timeout=REQUEST_TIMEOUT).result()
                    r.raise_for_status()
                    apps_list = r.json()
                except ValueError as e:
                    logger.error("Non json response {} from {}-{} docktor".format(r.content, stage, zone))
                    raise e
                except Exception as e:
                    logger.error("Exception raised on {}-{} docktor".format(stage, zone))
                    raise e

                future_apps_details = [session.get('{apps_uri}{app}'.format(apps_uri=apps_uri, app=app), timeout=REQUEST_TIMEOUT) for app in apps_list]

                try:
                    apps_details = [a.result() for a in future_apps_details]
                except Exception as e:
                    logger.error("Exception raised on {}-{} docktor".format(stage, zone))
                    raise e

                partial_get_app_info = partial(self.get_app_info, stage, zone)

                apps.extend(map(lambda a: partial_get_app_info(a), apps_details))
        return apps

# 需要导入模块: from requests_futures.sessions import FuturesSession [as 别名]
# 或者: from requests_futures.sessions.FuturesSession import mount [as 别名]
class Scraper(object):
    def __init__(self, url):
        self.url = url
        self.session = FuturesSession(max_workers=100)
        adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100)
        self.session.mount('http://', adapter)
        self.session.mount('https://', adapter)

    def _extract_image_urls(self, soup):
        for img in soup.findAll("img", src=True):
            yield urljoin(self.url, img["src"])

    def _find_thumbnail_image(self):
        content_type, content = _fetch_url(self.url, session=self.session)
        soup = BeautifulSoup(content)
        image_urls = self._extract_image_urls(soup)
        image_urls = [u for u in image_urls] # turn to list
        image_urls = list(set(image_urls)) # lose duplicates
        image_sizes = _parallel_get_sizes(image_urls, self.session)
        logging.debug('got sizes for {} images'.format(len(image_sizes)))
        # find biggest
        max_area = 0
        max_url = None
        for image_url in image_urls:
            size = image_sizes[image_url]
            if not size:
                continue

            # ignore little images
            area = size[0] * size[1]
            if area < 5000:
                logging.debug('ignore little {}'.format(image_url))
                continue

            # ignore excessively long/wide images
            if max(size) / min(size) > 1.5:
                logging.debug('ignore dimensions {}'.format(image_url))
                continue

            # penalize images with "sprite" in their name
            if 'sprite' in image_url.lower():
                logging.debug('penalizing sprite {}'.format(image_url))
                area /= 10

            if area > max_area:
                max_area = area
                max_url = image_url
        return max_url


    def scrape(self):
        thumbnail_url = self._find_thumbnail_image()
        #thumbnail = _make_thumbnail_from_url(thumbnail_url, referer=self.url)
        return thumbnail_url

# 需要导入模块: from requests_futures.sessions import FuturesSession [as 别名]
# 或者: from requests_futures.sessions.FuturesSession import mount [as 别名]
class CrashInfo(object):

    # TODO: count is probably erroneous since there is a range by default in supersearch...
    CRASH_STATS_URL = 'https://crash-stats.mozilla.com'
    SUPERSEARCH_URL = CRASH_STATS_URL + '/api/SuperSearch'
    TIMEOUT = 5
    MAX_RETRIES = 5
    MAX_WORKERS = multiprocessing.cpu_count()

    def __init__(self, paths, credentials = None):
        self.results = [ ]
        self.credentials = credentials
        self.info = { }
        self.paths = [paths] if type(paths) == str else paths 
        for path in self.paths:
            self.info[path] = { 'crashes': -1 }
                                
        self.session = FuturesSession(max_workers = self.MAX_WORKERS)
        self.session.mount(self.CRASH_STATS_URL, HTTPAdapter(max_retries = self.MAX_RETRIES))
        self.__get_info()

    def get(self):
        for r in self.results:
            r.result()
        return self.info

    def __get_apikey(self):
        if self.credentials:
            return self.credentials['tokens'][self.CRASH_STATS_URL]
        else:
            return ''
    
    def __info_cb(self, path):
        def cb(sess, res):
            self.info[path]['crashes'] = res.json()['total']

        return cb

    def __get_info(self):
        header = { 'Auth-Token': self.__get_apikey() }
        for path in self.paths:
            filename = os.path.basename(path)
            self.results.append(self.session.get(self.SUPERSEARCH_URL,
                                                 params = { 'product': 'Firefox',
                                                            'topmost_filenames': filename, 
                                                            '_results_number': 0,
                                                            '_facets': 'product',
                                                            '_facets_size': 1 },
                                                 headers = header,
                                                 timeout = self.TIMEOUT,
                                                 background_callback = self.__info_cb(path)))

# 需要导入模块: from requests_futures.sessions import FuturesSession [as 别名]
# 或者: from requests_futures.sessions.FuturesSession import mount [as 别名]
class FXRevision(object):

    ARCHIVES_URL = 'http://archive.mozilla.org'
    NIGHTLY_URL = ARCHIVES_URL + '/pub/firefox/nightly/'
    TIMEOUT = 5
    MAX_RETRIES = 5

    def __init__(self, versions, fx_version, os):
        self.results = [ ]
        self.dates = { }
        self.fx_version = fx_version
        self.os = os
        self.info = { }
        pattern = re.compile('([0-9]{4})([0-9]{2})([0-9]{2})([0-9]{2})([0-9]{2})([0-9]{2})')
        for version in versions:
            m = pattern.search(version)
            self.dates[version] = [m.group(i) for i in range(1, 7)]

        self.session = FuturesSession()
        self.session.mount(self.ARCHIVES_URL, HTTPAdapter(max_retries = self.MAX_RETRIES))
        self.__get_info()

    def get(self):
        for r in self.results:
            r.result()
        return self.info
        
    def __make_url(self, l):
        return self.NIGHTLY_URL + l[0] + '/' + l[1] + '/' + '-'.join(l) + '-mozilla-central/firefox-' + self.fx_version + '.en-US.' + self.os + '.json'

    def __info_cb(self, sess, res):
        json = res.json()
        self.info[json['buildid']] = json['moz_source_stamp']
    
    def __get_info(self):
        for date in self.dates.itervalues():
            self.results.append(self.session.get(self.__make_url(date),
                                                 timeout = self.TIMEOUT,
                                                 background_callback = self.__info_cb))

# 需要导入模块: from requests_futures.sessions import FuturesSession [as 别名]
# 或者: from requests_futures.sessions.FuturesSession import mount [as 别名]
class HTTPDriver(BaseDriver):
  """HTTPDriver

  The :class:`HTTPDriver` class reads SBP messages from an HTTP
  service for a device and writes out to a stream. This driver is like
  a file-handle with read and writes over two separately HTTP
  connections, but can also be enabled and disabled by its consumer.

  Parameters
  ----------
  device_uid : uid
    Device unique id
  url : str
    HTTP endpoint
  retries : tuple
    Configure connect and read retry count. Defaults to
    (MAX_CONNECT_RETRIES, MAX_READ_RETRIES).
  timeout : tuple
    Configure connect and read timeouts. Defaults to
    (DEFAULT_CONNECT_TIMEOUT, DEFAULT_READ_TIMEOUT).

  """

  def __init__(self,
               device_uid=None,
               url="https://broker.staging.skylark.swiftnav.com",
               retries=DEFAULT_RETRIES,
               timeout=DEFAULT_TIMEOUT,):
    self._retry = Retry(connect=DEFAULT_RETRIES[0],
                        read=DEFAULT_RETRIES[1],
                        redirect=MAX_REDIRECTS,
                        status_forcelist=[500],
                        backoff_factor=DEFAULT_BACKOFF_FACTOR)
    self.url = url
    self.read_session = requests.Session()
    self.read_session.mount("http://",
                            HTTPAdapter(pool_connections=DEFAULT_POOLSIZE,
                                        pool_maxsize=DEFAULT_POOLSIZE,
                                        pool_block=DEFAULT_POOLBLOCK,
                                        max_retries=self._retry))
    self.read_session.mount("https://",
                            HTTPAdapter(pool_connections=DEFAULT_POOLSIZE,
                                        pool_maxsize=DEFAULT_POOLSIZE,
                                        pool_block=DEFAULT_POOLBLOCK,
                                        max_retries=self._retry))
    self.write_session = None
    self.device_uid = device_uid
    self.timeout = timeout
    self.read_response = None
    self.write_response = None
    self.source = None

  def flush(self):
    """File-flush wrapper (noop).

    """
    pass

  def close(self):
    """File-handle close wrapper (noop).

    """
    try:
      self.read_close()
      self.write_close()
    except:
      pass

  @property
  def write_ok(self):
    """
    Are we connected for writes?
    """
    # Note that self.write_response is either None or a Response
    # object, which cast to False for 4xx and 5xx HTTP codes.
    return bool(self.write_response)

  def connect_write(self, source, whitelist, device_uid=None, pragma=None):
    """Initialize a streaming write HTTP response. Manually connects the
    underlying file-handle. In the event of a network disconnection,
    use to manually reinitiate an HTTP session.

    Parameters
    ----------
    source : sbp.client.handler.Handler
      Iterable source of SBP messages.
    whitelist : [int]
      Whitelist of messages to write

    """
    header_device_uid = device_uid or self.device_uid
    headers = {'Device-Uid': header_device_uid, 'Content-Type': BROKER_SBP_TYPE, 'Pragma': pragma}
    if not pragma:
      del headers['Pragma']
    try:
      self.executor = ThreadPoolExecutor(max_workers=DEFAULT_POOLSIZE)
      self.write_session = FuturesSession(executor=self.executor)
      self.write_session.mount("http://",
                               HTTPAdapter(pool_connections=DEFAULT_POOLSIZE,
                                           pool_maxsize=DEFAULT_POOLSIZE,
#.........这里部分代码省略.........

# 需要导入模块: from requests_futures.sessions import FuturesSession [as 别名]
# 或者: from requests_futures.sessions.FuturesSession import mount [as 别名]
    it is simply not checked
    
    Generously provided by Juan Luis Boya
    """
    def cert_verify(self, conn, *args, **kwargs):
        """
        Avoids the verification of the SSL Hostname field

        :param Connection conn: The connection object
        """
        super(NotCheckingHostnameHTTPAdapter, self).cert_verify(conn, *args, **kwargs)
        conn.assert_hostname = False

# By changing the adapter no hostname is checked
futures_session = FuturesSession()
futures_session.mount('https://', NotCheckingHostnameHTTPAdapter())

#Creation of the temporal directory if it does not exists
if not os.path.exists(conf.TMPDIR):
    os.makedirs(conf.TMPDIR)

__UPLOADS__ = conf.TMPDIR # temporal directory were files will be stored

open_ws = set() #Set of the current alive websockets

class BaseHandler(RequestHandler):
    """
    The base class which the rest of HTTP handlers extends.
    Provides secure cookie decryption and error handling
    """
    def get_current_user(self):

# 需要导入模块: from requests_futures.sessions import FuturesSession [as 别名]
# 或者: from requests_futures.sessions.FuturesSession import mount [as 别名]
)
# Enable For Debugging:
logging.disable(logging.INFO)

DELAY = .05  # Second delay between calculating trades.
RGAP = .005 # Max gap before cancelling a robux split trade
TGAP = .0025 # Max gap before cancelling a tix split trade
TRADE_LAG_TIME = 1.25 # Estimate of how long it takes for Roblox to process our requests
RESET_TIME = 240 # Number of seconds the bot goes without trading before resetting last rates to be able to trade again (might result in loss)
DEQUE_SIZE = 15 # Max number of past trade rates to keep track of to money prevent loss
NUM_TRADES = 19 # Number of trades that display on the trade currency page
# Initializing requests.Session for frozen application
os.environ["REQUESTS_CA_BUNDLE"] = find_data_file('cacert.pem')
session = FuturesSession(max_workers=15)
adapter = requests.adapters.HTTPAdapter(max_retries=Retry(total=20,connect=10,read=10,backoff_factor=.5))
session.mount("http://", adapter)
session.mount("https://", adapter)
# Storing variables since they can't be stored in QObject
rates = DottedDict(
    dict(
        last_tix_rate = 0,
        last_robux_rate = 0,
        current_tix_rate = 0,
        current_robux_rate = 0,
        past_tix_rates = deque(maxlen=DEQUE_SIZE),
        past_robux_rates = deque(maxlen=DEQUE_SIZE),
    )
)

class Trader(QtCore.QObject):

# 需要导入模块: from requests_futures.sessions import FuturesSession [as 别名]
# 或者: from requests_futures.sessions.FuturesSession import mount [as 别名]
class BZInfo(object):

    BZ_URL = 'https://bugzilla.mozilla.org'
    API_URL = BZ_URL + '/rest/bug'
    TIMEOUT = 60
    MAX_RETRIES = 5
    MAX_WORKERS = multiprocessing.cpu_count()
    CHUNK_SIZE = 8
    
    def __init__(self, bugids, credentials = None):
        self.results = [ ]
        self.credentials = credentials
        self.bugids = map(str, bugids)
        self.info = { }
        for bugid in self.bugids:
            self.info[bugid] = { 'ownership': [],
                                 'reviewers': set(),
                                 'commenters': { },
                                 'authorized': False }
        self.session = FuturesSession(max_workers = self.MAX_WORKERS)
        self.session.mount(self.BZ_URL, HTTPAdapter(max_retries = self.MAX_RETRIES))
        self.reply_pattern = re.compile('^\(In reply to .* comment #([0-9]+)\)')
        self.dupbug_pattern = re.compile('\*\*\* Bug [0-9]+ has been marked as a duplicate of this bug. \*\*\*')
        self.review_pattern= re.compile('review\?\(([^\)]+)\)')
        self.needinfo_pattern= re.compile('needinfo\?\(([^\)]+)\)')
        self.feedback_pattern= re.compile('feedback\?\(([^\)]+)\)')
        self.__get_info()
        self.__analyze_history()
        self.__analyze_comment()

    def get(self):
        for r in self.results:
            r.result()
        return self.info

    def get_best_collaborator(self):
        # a collaboration between A & B is when A reviews a patch of B (or reciprocally)
        # in term of graph:
        #   - each node represents a reviewer or a writter (owner)
        #   - each edge represents a collaboration
        # here we count the degree of each node and find out who's the best collaborator
        # TODO: use this graph to get other metrics (??)

        # it could be interesting to weight each contribution according to its date
        # someone who made 20 contribs recently is probably better than someone 50 contribs
        # two years ago...
        # TODO: We could weight a contrib with a gaussian which depends to the time
        
        collaborations = { }
        for info in self.get().itervalues():
            if info['authorized']:
                owner = info['owner']
                if owner not in collaborations:
                    collaborations[owner] = 0
                reviewers = info['reviewers']
                feedbacks = info['feedbacks']
                collabs = set()
                if reviewers and owner in reviewers:
                    collabs |= reviewers[owner]
                if feedbacks and owner in feedbacks:
                    collabs |= feedbacks[owner]
                if collabs:
                    collaborations[owner] += len(collabs)
                    for person in collabs:
                        collaborations[person] = collaborations[person] + 1 if person in collaborations else 1
 
        # maybe we should compute the percentage of collaborations just to give an idea
 
        return utils.get_best(collaborations)

    def get_best_component_product(self):
        # Just get stats on components and products
        comps_prods = { }
        for info in self.get().itervalues():
            if info['authorized']:
                comp_prod = (info['component'], info['product'])
                comps_prods[comp_prod] = comps_prods[comp_prod] + 1 if comp_prod in comps_prods else 1

        if comps_prods:
            return utils.get_best(comps_prods)
        else:        
            return None

    def __get_apikey(self):
        if self.credentials:
            return self.credentials['tokens'][self.BZ_URL]
        else:
            return ''

    def __info_cb(self, sess, res):
        bugs = res.json()['bugs']
        for bug in bugs:
            self.info[str(bug['id'])].update({ 'authorized': True,
                                               'severity': bug['severity'],
                                               'votes': bug['votes'],
                                               'component': bug['component'],
                                               'product': bug['product'],
                                               'nbcc': len(bug['cc']),
                                               'reporter': bug['creator'],
                                               'owner': bug['assigned_to_detail']['email']})
#.........这里部分代码省略.........

# 需要导入模块: from requests_futures.sessions import FuturesSession [as 别名]
# 或者: from requests_futures.sessions.FuturesSession import mount [as 别名]
class Track(object):

    CRASH_STATS_URL = 'https://crash-stats.mozilla.com'
    SUPERSEARCH_URL = CRASH_STATS_URL + '/api/SuperSearch'
    TIMEOUT = 5
    MAX_RETRIES = 5
    MAX_WORKERS = multiprocessing.cpu_count()
    HG_PATTERN = re.compile('hg:hg.mozilla.org/mozilla-central:([^:]*):([a-z0-9]+)')
    
    def __init__(self, signature, day, day_delta = 1, credentials = None):
        self.results = [ ]
        self.credentials = credentials
        self.has_results = False
        self.day_delta = day_delta
        self.signature = signature
        self.info = { }
        self.date = utils.get_date_ymd(day)
        self.session = FuturesSession(max_workers = self.MAX_WORKERS)
        self.session.mount(self.CRASH_STATS_URL, HTTPAdapter(max_retries = self.MAX_RETRIES))
        self.__get_info()

    def get(self):
        if not self.has_results:
            for r in self.results:
                r.result()
            self.has_results = True
        return self.info

    def has_addons(self):
        return len(self.get()['addons']) != 0

    def __get_apikey(self):
        if self.credentials:
            return self.credentials['tokens'][self.CRASH_STATS_URL]
        else:
            return ''

    @staticmethod
    def __get_stats(info, field):
        l = info[field]
        total = float(info['total'])
        stats = { }
        for e in l:
            stats[e['term']] = utils.percent(float(e['count']) / total)
        return stats
    
    @staticmethod
    def __get_system_memory_use_mean(info):
        l = info['system_memory_use_percentage']
        total = float(info['total'])
        l = [(float(e['count']) / total, float(e['term'])) for e in l]  
        m = 0.
        for e in l:
            m += e[0] * e[1]

        v = 0.
        for e in l:
            v += e[0] * (m - e[1]) ** 2

        return {'mean': utils.simple_percent(round(m, 0)), 'stddev': utils.simple_percent(round(math.sqrt(v), 0))}

    @staticmethod
    def __is_weird_address(addr, cpu_name):
        if addr == '0x0':
            return True
        if utils.is64(cpu_name):
            if len(addr) <= 10:
                val = long(addr, 16)
                if val <= 1L << 16: # val <= 0xffff (ie: first 64k)
                    return True
            elif addr.startswith('0xffffffff'):
                addr = addr[10:] # 10 == len('0xffffffff')
                val = long(addr, 16)
                if val >= ((1L << 32) - (1L << 16)): # val >= 0xfffffffffff0000 (ie: last 64k)
                    return True

# 需要导入模块: from requests_futures.sessions import FuturesSession [as 别名]
# 或者: from requests_futures.sessions.FuturesSession import mount [as 别名]
class Backtrace(object):

    CRASH_STATS_URL = 'https://crash-stats.mozilla.com'
    PROCESSED_URL = CRASH_STATS_URL + '/api/ProcessedCrash/'
    TIMEOUT = 5
    MAX_RETRIES = 5
    MAX_WORKERS = multiprocessing.cpu_count()
    
    def __init__(self, uuids, fraction = 0.2, just_hg = False, credentials = None):
        self.just_hg = just_hg
        self.results = [ ]
        self.credentials = credentials
        self.uuids = uuids
        self.fraction = max(0., min(fraction, 1.))
        self.info = { }
        self.session = FuturesSession(max_workers = self.MAX_WORKERS)
        self.session.mount(self.CRASH_STATS_URL, HTTPAdapter(max_retries = self.MAX_RETRIES))
        self.__get_info()

    def get(self):
        for r in self.results:
            r.result()
        return self.info

    def __get_apikey(self):
        if self.credentials:
            return self.credentials['tokens'][self.CRASH_STATS_URL]
        else:
            return ''
    
    @staticmethod
    def __cycles_detection(funs):
        # TODO: improve this algorithm (not sure that's a good one)
        positions = { }
        # we get the function positions in the trace
        for i in range(len(funs)):
            fun = funs[i]
            if fun in positions:
                positions[fun].append(i)
            else:
                positions[fun] = [ i ]

        lengths = { }
        for k, v in positions.iteritems():
            if len(v) >= 2:
                l = v[1] - v[0]
                good = True
                for i in range(2, len(v)):
                    if v[i] - v[i - 1] != l:
                        good = False
                        break
                if good:
                    if l in lengths:
                        lengths[l].append((k, v))
                    else:
                        lengths[l] = [ (k, v) ]

        cycles = [ ]
        for k, v in lengths.iteritems():
            l = sorted(v, cmp = lambda x, y: cmp(x[1][0], y[1][0]))
            pat = [ ]
            container = [ l[0][0] ]
            pos = l[0][1][0]
            for i in range(1, len(l)):
                _pos = l[i][1][0]
                if _pos == pos + 1:
                    container.append(l[i][0])
                    pos = _pos
                else:
                    pat.append(tuple(container))
                    container = [ l[i][0] ]
                    pos = _pos

            pat.append(tuple(container))
            cycles += pat

        cycles = tuple(cycles)
        
        return cycles
    
    def __info_cb(self, sess, res):
        json = res.json()
        if 'json_dump' in json:
            uuid = json['uuid']
            jd = json['json_dump']
            if 'crashedThread' in json and 'threads' in jd:
                ct = json['crashedThread']
                ct = jd['threads'][ct]
                self.info[uuid] = { 'cycles': [ ],
                                        'address': '',
                                        'cpu_name': json['cpu_name'],
                                        'cpu_info': json['cpu_info'],
                                        'reason': json['reason'],
                                        'os': json['os_pretty_version'] }
                if 'frames' in ct:
                    frames = ct['frames']
                    functions = [ ]
                    # we get the functions in the backtrace (to check if there is a recursion)
                    for frame in frames:
                        if 'function' in frame:
#.........这里部分代码省略.........

# 需要导入模块: from requests_futures.sessions import FuturesSession [as 别名]
# 或者: from requests_futures.sessions.FuturesSession import mount [as 别名]
    def retrieve(self, catalog, *, dry_run=False, media_type=''):
        if not dry_run:
            distributions = Distribution.objects.filter(
                division_id=catalog.division_id, http_status_code__isnull=True)

            if media_type:
                distributions = distributions.filter(mediaType=media_type)

            if not distributions.exists():
                return

            # Collect the distribution-response pairs.
            def callback(distribution, response):
                results.append([distribution, response])

            # Create a closure.
            def factory(distribution):
                return lambda session, response: callback(distribution, response)

            # @see http://docs.python-requests.org/en/latest/api/#requests.adapters.HTTPAdapter
            # @see https://github.com/ross/requests-futures/blob/master/requests_futures/sessions.py
            session = FuturesSession()
            # Avoids "Connection pool is full, discarding connection" warnings.
            adapter_kwargs = {'pool_maxsize': 10}
            session.mount('https://',
                          requests.adapters.HTTPAdapter(**adapter_kwargs))
            session.mount('http://',
                          requests.adapters.HTTPAdapter(**adapter_kwargs))

            # @see https://djangosnippets.org/snippets/1949/
            pk = 0
            last_pk = distributions.order_by('-pk')[0].pk
            distributions = distributions.order_by('pk')
            while pk < last_pk:
                # @see https://github.com/ross/requests-futures/issues/18
                # @see https://github.com/ross/requests-futures/issues/5
                futures = []
                results = []

                # If an exception occurs, we lose progress on at most 100 requests.
                for distribution in distributions.filter(pk__gt=pk)[:100]:
                    pk = distribution.pk

                    # @see http://docs.python-requests.org/en/latest/user/advanced/#body-content-workflow
                    # @see http://stackoverflow.com/a/845595/244258
                    futures.append(
                        session.get(
                            quote(
                                distribution.accessURL,
                                safe="%/:=&?~#+!$,;'@()*[]"),
                            stream=True,
                            verify=False,
                            background_callback=factory(distribution)))

                for future in futures:
                    try:
                        future.result()
                    except (requests.exceptions.ConnectionError,
                            requests.exceptions.InvalidSchema,
                            requests.exceptions.InvalidURL,
                            requests.exceptions.MissingSchema,
                            requests.exceptions.ReadTimeout,
                            requests.exceptions.SSLError,
                            requests.exceptions.TooManyRedirects,
                            requests.packages.urllib3.exceptions.ProtocolError
                            ):
                        self.exception('')

                for distribution, response in results:
                    status_code = response.status_code
                    charset = ''

                    content_length = response.headers.get('content-length')
                    if content_length:
                        content_length = int(content_length)

                    # @see https://github.com/kennethreitz/requests/blob/b137472936cbe6a6acabab538c1d05ed4c7da638/requests/utils.py#L308
                    content_type = response.headers.get('content-type', '')
                    if content_type:
                        content_type, params = cgi.parse_header(content_type)
                        if 'charset' in params:
                            charset = params['charset'].strip("'\"")

                    distribution.http_headers = dict(response.headers)
                    distribution.http_status_code = status_code
                    distribution.http_content_length = content_length
                    distribution.http_content_type = content_type
                    distribution.http_charset = charset
                    distribution.save()

                    self.debug('{} {} {}'.format(
                        status_code, number_to_human_size(content_length),
                        content_type))

                    response.close()
        self.info('{} done'.format(catalog))

# 需要导入模块: from requests_futures.sessions import FuturesSession [as 别名]
# 或者: from requests_futures.sessions.FuturesSession import mount [as 别名]
class ResourceSyncPuSH(object):
    """
    The base class for the publisher, hub and resource. Contains
    methods for reading config files, making http requests, error handling,
    etc.
    """

    def __init__(self):
        """
        Inititalizes the Futures-Requests session with the
        max number of workers and retires.
        """

        # max workers and retries should be configurable?
        self.session = FuturesSession(max_workers=10)
        adapter = HTTPAdapter(max_retries=3)
        self.session.mount("http://", adapter)
        self._start_response = None

        # config parameters
        self.config = {}
        self.config['log_mode'] = ""
        self.config['mimetypes'] = []
        self.config['trusted_publishers'] = []
        self.config['trusted_topics'] = []
        self.config['my_url'] = ""
        self.config['hub_url'] = ""
        self.config['topic_url'] = ""
        self.config['subscribers_file'] = ""
        self.config['server_path'] = ""

        # logging messages
        self.log_msg = {}
        self.log_msg['payload'] = ""
        self.log_msg['msg'] = []
        self.log_msg['link_header'] = ""
        self.log_msg['module'] = ""

    def get_config(self, classname=None):
        """
        Finds and reads the config file. Reads the appropriate config values
        for the classname provided. For eg: if the classname is hub, it will
        read from the [hub] section in the config file.
        """

        if not classname:
            classname = self.__class__.__name__.lower()

        self.log_msg['module'] = classname

        # NOTE: more paths can be added to look for the config files.
        # order of files matter, the config in the first file
        # will be overwritten by the values in the next file.
        cnf_file = []
        cnf_file.extend([
            os.path.join(os.path.dirname(__file__),
                         "../conf/resourcesync_push.ini"),
            "/etc/resourcesync_push.ini",
            "/etc/resourcesync_push/resourcesync_push.ini",
        ])

        # loading values from configuration file
        conf = ConfigParser.ConfigParser()
        conf.read(cnf_file)
        if not conf:
            raise IOError("Unable to read config file")

        if classname == "hub":
            self.get_hub_config(conf)
        elif classname == "publisher":
            self.get_publisher_config(conf)
        elif classname == "subscriber":
            try:
                self.config['my_url'] = conf.get("subscriber", "url")
            except (NoSectionError, NoOptionError):
                print("The url value for subscriber is required \
                      in the config file.")
                raise

        self.get_demo_config(conf)

    def get_demo_config(self, conf):
        """
        Reads the [demo_hub] section from the config file if the
        log mode is set to 'demo'.
        """
        try:
            self.config['log_mode'] = conf.get("general", "log_mode")
        except (NoSectionError, NoOptionError):
            pass

        if not self.config['log_mode'] == "demo":
            return

        try:
            self.config['demo_hub_url'] = conf.get("demo_mode", "hub_url")
        except (NoSectionError, NoOptionError):
            print("Demo log mode requires a hub_url in the \
                  [demo_mode] section")
            raise
#.........这里部分代码省略.........

# 需要导入模块: from requests_futures.sessions import FuturesSession [as 别名]
# 或者: from requests_futures.sessions.FuturesSession import mount [as 别名]
class Connection(object):
    """Represents a connection to a server
    """

    TIMEOUT = 30
    MAX_RETRIES = 256
    MAX_WORKERS = multiprocessing.cpu_count()
    CHUNK_SIZE = 32
    TOKEN = ''

    # Error 429 is for 'Too many requests' => we retry
    STATUS_FORCELIST = [429]

    def __init__(self, base_url, queries=None, **kwargs):
        """Constructor

        Args:
            base_url (str): the server's url
            queries (Optional[Query]): the queries
        """

        self.session = FuturesSession(max_workers=self.MAX_WORKERS)
        retries = Retry(total=Connection.MAX_RETRIES, backoff_factor=1, status_forcelist=Connection.STATUS_FORCELIST)
        self.session.mount(base_url, HTTPAdapter(max_retries=retries))
        self.results = []
        self.queries = queries

        if kwargs:
            if 'timeout' in kwargs:
                self.TIMEOUT = kwargs['timeout']
            if 'max_retries' in kwargs:
                self.MAX_RETRIES = kwargs['max_retries']
            if 'max_workers' in kwargs:
                self.MAX_WORKERS = kwargs['max_workers']

        self.exec_queries()

    def __get_cb(self, query):
        """Get the callback to use when data have been retrieved

        Args:
            query (Query): the query

        Returns:
            function: the callback for the query
        """
        def cb(sess, res):
            if res.status_code == 200:
                try:
                    response = res.json()
                except:
                    response = res.text

                if query.handlerdata is not None:
                    query.handler(response, query.handlerdata)
                else:
                    query.handler(response)
            else:
                print('Connection error:')
                print('   url: ', res.url)
                print('   text: ', res.text)

        return cb

    def wait(self):
        """Just wait that all the queries have been treated
        """
        for r in self.results:
            r.result()

    def get_apikey(self):
        """Get the api key

        Returns:
            str: the api key
        """
        return self.TOKEN

    def get_header(self):
        """Get the header to use each query

        Returns:
            dict: the header
        """
        return {'User-Agent': 'clouseau', 'Connection': 'close'}

    def get_auth(self):
        """Get the auth to use each query

        Returns:
            dict: the auth
        """
        return None

    def exec_queries(self, queries=None):
        """Set and exec some queries

        Args:
            queries (Optional[Query]): the queries to exec
        """
#.........这里部分代码省略.........