Python Library.id方法代码示例

# 需要导入模块: from datamodel import Library [as 别名]
# 或者: from datamodel.Library import id [as 别名]
  def update_metadata(self):
    headers = {'Accept': 'application/vnd.github.drax-preview+json'}
    response = util.github_get('repos', self.owner, self.repo, etag=self.library.metadata_etag, headers=headers)
    if response.status_code == 200:
      try:
        metadata = json.loads(response.content)
      except ValueError:
        return self.error("could not parse metadata")

      repo = metadata.get('name', '').lower()
      owner = metadata.get('owner', {}).get('login', '').lower()
      if repo != '' and owner != '' and (repo != self.repo or owner != self.owner):
        logging.info('deleting renamed repo %s', Library.id(self.owner, self.repo))
        delete_library(self.library.key)
        task_url = util.ensure_library_task(owner, repo)
        util.new_task(task_url, target='manage')
        raise RequestAborted('repo has been renamed to %s', Library.id(owner, repo))

      self.library.metadata = response.content
      self.library.metadata_etag = response.headers.get('ETag', None)
      self.library.metadata_updated = datetime.datetime.now()
      self.library_dirty = True
    elif response.status_code == 404:
      logging.info('deleting non-existing repo %s', Library.id(self.owner, self.repo))
      delete_library(self.library.key)
      raise RequestAborted('repo no longer exists')
    elif response.status_code != 304:
      return self.retry('could not update repo metadata (%d)' % response.status_code)

    response = util.github_get('repos', self.owner, self.repo, 'contributors', etag=self.library.contributors_etag)
    if response.status_code == 200:
      try:
        json.loads(response.content)
      except ValueError:
        return self.error("could not parse contributors")
      self.library.contributors = response.content
      self.library.contributors_etag = response.headers.get('ETag', None)
      self.library.contributors_updated = datetime.datetime.now()
      self.library_dirty = True
    elif response.status_code != 304:
      return self.retry('could not update contributors (%d)' % response.status_code)

    response = util.github_get('repos', self.owner, self.repo, 'stats/participation ', etag=self.library.participation_etag)
    if response.status_code == 200:
      try:
        json.loads(response.content)
      except ValueError:
        return self.error("could not parse stats/participation")
      self.library.participation = response.content
      self.library.participation_etag = response.headers.get('ETag', None)
      self.library.participation_updated = datetime.datetime.now()
      self.library_dirty = True
    elif response.status_code == 202:
      # GitHub is "computing" the data. We'll try again next update cycle.
      # TODO: Alternatively we could retry this task
      pass
    elif response.status_code != 304:
      return self.retry('could not update stats/participation (%d)' % response.status_code)

# 需要导入模块: from datamodel import Library [as 别名]
# 或者: from datamodel.Library import id [as 别名]
 def init_library(self, owner, repo, create=True):
   self.owner = owner.lower()
   self.repo = repo.lower()
   if create:
     self.library = Library.get_or_insert(Library.id(owner, repo))
     self.is_new = self.library.metadata is None and self.library.error is None
   else:
     self.library = Library.get_by_id(Library.id(owner, repo))
   if self.library.status == Status.suppressed:
     raise RequestAborted('library is suppressed')

# 需要导入模块: from datamodel import Library [as 别名]
# 或者: from datamodel.Library import id [as 别名]
  def init_library(self, scope, package, create=True):
    self.scope = scope.lower()
    self.package = package.lower()

    if create:
      self.library = Library.get_or_insert(Library.id(self.scope, self.package))
      self.is_new = self.library.metadata is None and self.library.error is None
    else:
      self.library = Library.get_by_id(Library.id(self.scope, self.package))
    if self.library.status == Status.suppressed:
      raise RequestAborted('library is suppressed')

# 需要导入模块: from datamodel import Library [as 别名]
# 或者: from datamodel.Library import id [as 别名]
 def handle_get(self, owner, repo, version):
   # FIXME: Make deletion transactional with check on library that tag is excluded.
   version_key = ndb.Key(Library, Library.id(owner, repo), Version, version)
   ndb.delete_multi(ndb.Query(ancestor=version_key).iter(keys_only=True))
   if VersionCache.update(version_key.parent()):
     task_url = util.update_indexes_task(owner, repo)
     util.new_task(task_url, target='manage')

# 需要导入模块: from datamodel import Library [as 别名]
# 或者: from datamodel.Library import id [as 别名]
  def update_readme(self, is_npm_package):
    if is_npm_package:
      # Load registry metadata to fetch readme path.
      library = Library.get_by_id(Library.id(self.owner, self.repo))
      registry_metadata = json.loads(library.registry_metadata) if library.registry_metadata else None
      readme_path = registry_metadata.get('readmeFilename', 'README.md')
      response = util.unpkg_get(self.owner, self.repo, self.version, readme_path)
      readme = response.content
    else:
      # Load readme from GitHub endpoint.
      response = util.github_get('repos', self.owner, self.repo, 'readme', params={"ref": self.sha})

      if response.status_code == 200:
        readme = base64.b64decode(json.loads(response.content)['content'])
      elif response.status_code == 404:
        readme = None
      else:
        return self.retry('error fetching readme (%d)' % response.status_code)

    if readme is not None:
      # Store the raw readme markdown.
      try:
        Content(parent=self.version_key, id='readme', content=readme,
                status=Status.ready, etag=response.headers.get('ETag', None)).put()
      except db.BadValueError:
        return self.error("Could not store README.md as a utf-8 string", ErrorCodes.Version_utf)

      # Convert markdown to HTML and store the result.
      response = util.github_markdown(readme)
      if response.status_code == 200:
        Content(parent=self.version_key, id='readme.html', content=response.content,
                status=Status.ready, etag=response.headers.get('ETag', None)).put()
      else:
        return self.retry('error converting readme to markdown (%d)' % response.status_code)

# 需要导入模块: from datamodel import Library [as 别名]
# 或者: from datamodel.Library import id [as 别名]
  def handle_get(self, owner, repo, scope, package):
    library = Library.get_by_id(Library.id(owner, repo))

    if library is None:
      return

    library.npm_package = scope + '/' + package
    library.put()

    # Remove from search indexes.
    index = search.Index('repo')
    index.delete(Library.id(owner, repo))

    npm_library = Library.get_by_id(Library.id(scope, package))
    if npm_library is not None:
      npm_library.migrated_from_bower = True
      npm_library.put()

# 需要导入模块: from datamodel import Library [as 别名]
# 或者: from datamodel.Library import id [as 别名]
  def test_ensure_when_present(self):
    Library(id=Library.id('owner', 'repo')).put()
    response = self.app.get(util.ensure_library_task('owner', 'repo'), headers={'X-AppEngine-QueueName': 'default'})

    self.assertEqual(response.status_int, 200)

    tasks = self.tasks.get_filtered_tasks()
    self.assertEqual([], [task.url for task in tasks])

# 需要导入模块: from datamodel import Library [as 别名]
# 或者: from datamodel.Library import id [as 别名]
  def update_collection_dependencies(self, collection_version_key, bower):
    dependencies = bower.get('dependencies', {})
    for name in dependencies.keys():
      dep = Dependency.from_string(dependencies[name])
      if dep is None:
        continue
      library_key = ndb.Key(Library, Library.id(dep.owner, dep.repo))
      CollectionReference.ensure(library_key, collection_version_key, semver=dep.version)

      task_url = util.ensure_library_task(dep.owner.lower(), dep.repo.lower())
      util.new_task(task_url, target='manage')

# 需要导入模块: from datamodel import Library [as 别名]
# 或者: from datamodel.Library import id [as 别名]
  def test_update_all(self):
    library_key = Library(id='owner/repo').put()
    author_key = Author(id='owner').put()

    response = self.app.get('/manage/update-all', headers={'X-AppEngine-QueueName': 'default'})
    self.assertEqual(response.status_int, 200)

    tasks = self.tasks.get_filtered_tasks()
    self.assertEqual([
        util.update_library_task(library_key.id()),
        util.update_author_task(author_key.id()),
    ], [task.url for task in tasks])

# 需要导入模块: from datamodel import Library [as 别名]
# 或者: from datamodel.Library import id [as 别名]
  def handle_post(self):
    message_json = json.loads(urllib.unquote(self.request.body).rstrip('='))
    message = message_json['message']
    data = base64.b64decode(str(message['data']))
    attributes = message['attributes']
    owner = attributes['owner']
    repo = attributes['repo']
    version = attributes['version']
    error = attributes.get('error', None)

    version_key = ndb.Key(Library, Library.id(owner, repo), Version, version)

    content = Content.get_by_id('analysis', parent=version_key)
    if content is None:
      return
    if data == '':
      content.content = None
    elif len(data) > 500000:
      # Max entity size is only 1MB.
      logging.error('content was too large: %d %s %s', len(data), Library.id(owner, repo), version)
      error = 'content was too large: %d' % len(data)
    else:
      content.content = data

    if error is None:
      content.status = Status.ready
      content.error = None
    else:
      content.status = Status.error
      content.error = error

    content.put()

    if version_key.id() == Library.default_version_for_key_async(version_key.parent()).get_result():
      task_url = util.update_indexes_task(owner, repo)
      util.new_task(task_url, target='manage')

# 需要导入模块: from datamodel import Library [as 别名]
# 或者: from datamodel.Library import id [as 别名]
  def update_search_index(self, owner, repo, version_key, library, bower):
    metadata = json.loads(library.metadata)
    fields = [
        search.AtomField(name='owner', value=owner),
        search.TextField(name='repo', value=repo),
        search.AtomField(name='kind', value=library.kind),
        search.AtomField(name='version', value=version_key.id()),
        search.TextField(name='github_description', value=metadata.get('description', '')),
        search.TextField(name='bower_description', value=bower.get('description', '')),
        search.TextField(name='bower_keywords', value=' '.join(bower.get('keywords', []))),
        search.TextField(name='prefix_matches', value=' '.join(util.generate_prefixes_from_list(
            util.safe_split_strip(metadata.get('description')) + util.safe_split_strip(bower.get('description')) +
            util.safe_split_strip(repo)))),
    ]

    # Generate weighting field
    weights = [(repo, 10)]

    analysis = Content.get_by_id('analysis', parent=version_key)
    if analysis is not None and analysis.status == Status.ready:
      analysis = json.loads(analysis.content)
      elements = analysis.get('elementsByTagName', {}).keys()
      if elements != []:
        fields.append(search.TextField(name='element', value=' '.join(elements)))
        weights.append((' '.join(elements), 5))
      behaviors = analysis.get('behaviorsByName', {}).keys()
      if behaviors != []:
        fields.append(search.TextField(name='behavior', value=' '.join(behaviors)))
        weights.append((' '.join(behaviors), 5))

    weighted = []
    for value, weight in weights:
      for _ in range(0, weight):
        weighted.append(value)
    fields.append(search.TextField(name='weighted_fields', value=' '.join(weighted)))

    rank = int((library.updated - datetime.datetime(2016, 1, 1)).total_seconds())
    document = search.Document(doc_id=Library.id(owner, repo), fields=fields, rank=rank)
    index = search.Index('repo')
    index.put(document)

# 需要导入模块: from datamodel import Library [as 别名]
# 或者: from datamodel.Library import id [as 别名]
  def handle_post(self):
    # Ignore payloads larger than 5 MB.
    if len(self.request.body) > 1048487 * 5:
      return
    message_json = json.loads(urllib.unquote(self.request.body).rstrip('='))
    message = message_json['message']
    data = base64.b64decode(str(message['data']))
    attributes = message['attributes']
    if len(attributes) == 0:
      logging.error(message)
      return
    owner = attributes['owner']
    repo = attributes['repo']
    version = attributes['version']
    error = attributes.get('error', None)

    version_key = ndb.Key(Library, Library.id(owner, repo), Version, version)

    content = Content.get_by_id('analysis', parent=version_key)
    if content is None:
      return
    if data == '':
      content.set_json(None)
    else:
      content.set_json(json.loads(data))

    if error is None:
      content.status = Status.ready
      content.error = None
    else:
      content.status = Status.error
      content.error = error

    content.put()

    if version_key.id() == Library.default_version_for_key_async(version_key.parent()).get_result():
      task_url = util.update_indexes_task(owner, repo)
      util.new_task(task_url, target='manage')

# 需要导入模块: from datamodel import Library [as 别名]
# 或者: from datamodel.Library import id [as 别名]
  def update_metadata(self):
    # Query NPM registry API for packages
    is_npm_package = self.scope.startswith('@')
    if is_npm_package:
      self.update_registry_info()
    else:
      self.owner = self.scope
      self.repo = self.package

    # Fetch GitHub metadata
    headers = {'Accept': 'application/vnd.github.drax-preview+json'}
    response = util.github_get('repos', self.owner, self.repo, etag=self.library.metadata_etag, headers=headers)
    if response.status_code == 200:
      try:
        metadata = json.loads(response.content)
      except ValueError:
        return self.error("could not parse metadata", ErrorCodes.Library_parse_metadata)

      self.owner = metadata.get('owner', {}).get('login', '').lower()
      self.repo = metadata.get('name', '').lower()

      # Deleting is only necessary if Library entity is a GitHub repo
      if (not is_npm_package) and self.repo != '' and self.owner != '' and (self.repo != self.package or self.owner != self.scope):
        logging.info('deleting renamed repo %s', Library.id(self.owner, self.repo))
        delete_library(self.library.key)
        task_url = util.ensure_library_task(self.owner, self.repo)
        util.new_task(task_url, target='manage')
        raise RequestAborted('repo has been renamed to %s', Library.id(self.owner, self.repo))

      # If adding a NPM package that a Bower repo already points to, remove the bower one.
      bower_library_id = Library.id(self.owner, self.repo)
      if is_npm_package and bower_library_id is not None:
        task_url = util.migrate_library_task(self.owner, self.repo, self.scope, self.package)
        util.new_task(task_url, target='manage')

      self.library.github_owner = self.owner
      self.library.github_repo = self.repo

      self.library.metadata = response.content
      self.library.metadata_etag = response.headers.get('ETag', None)
      self.library.metadata_updated = datetime.datetime.now()
      self.library_dirty = True
    elif response.status_code == 404:
      logging.info('deleting non-existing repo %s', Library.id(self.owner, self.repo))
      delete_library(self.library.key)
      raise RequestAborted('repo no longer exists')
    elif response.status_code != 304:
      return self.retry('could not update repo metadata (%d)' % response.status_code)

    response = util.github_get('repos', self.owner, self.repo, 'contributors', etag=self.library.contributors_etag)
    if response.status_code == 200:
      try:
        json.loads(response.content)
      except ValueError:
        return self.error("could not parse contributors", ErrorCodes.Library_parse_contributors)
      self.library.contributors = response.content
      self.library.contributors_etag = response.headers.get('ETag', None)
      self.library.contributors_updated = datetime.datetime.now()
      self.library_dirty = True
    elif response.status_code != 304:
      return self.retry('could not update contributors (%d)' % response.status_code)

    response = util.github_get('repos', self.owner, self.repo, 'stats/participation ', etag=self.library.participation_etag)
    if response.status_code == 200:
      try:
        json.loads(response.content)
      except ValueError:
        return self.error("could not parse stats/participation", ErrorCodes.Library_parse_stats)
      self.library.participation = response.content
      self.library.participation_etag = response.headers.get('ETag', None)
      self.library.participation_updated = datetime.datetime.now()
      self.library_dirty = True
    elif response.status_code == 202:
      # GitHub is "computing" the data. We'll try again next update cycle.
      # TODO: Alternatively we could retry this task
      pass
    elif response.status_code != 304:
      return self.retry('could not update stats/participation (%d)' % response.status_code)

# 需要导入模块: from datamodel import Library [as 别名]
# 或者: from datamodel.Library import id [as 别名]
  def update_search_index(self, owner, repo, version_key, library, bower):
    metadata = json.loads(library.metadata)
    registry_metadata = json.loads(library.registry_metadata) if library.registry_metadata else None
    npm_description = registry_metadata.get('description', '') if registry_metadata else ''
    npm_keywords = registry_metadata.get('keywords', []) if registry_metadata else []
    fields = [
        search.AtomField(name='owner', value=owner),
        search.AtomField(name='github_owner', value=library.github_owner),
        search.TextField(name='repo', value=repo),
        search.AtomField(name='kind', value=library.kind),
        search.AtomField(name='version', value=version_key.id()),
        search.TextField(name='github_description', value=metadata.get('description', '')),
        search.TextField(name='bower_description', value=bower.get('description', '')),
        search.TextField(name='npm_description', value=npm_description),
        search.TextField(name='bower_keywords', value=' '.join(bower.get('keywords', []))),
        search.TextField(name='npm_keywords', value=' '.join(npm_keywords)),
        search.TextField(name='prefix_matches', value=' '.join(util.generate_prefixes_from_list(
            util.safe_split_strip(metadata.get('description')) + util.safe_split_strip(bower.get('description')) +
            util.safe_split_strip(repo)))),
    ]

    # Generate weighting field
    weights = [(repo, 10)]

    analysis = Content.get_by_id('analysis', parent=version_key)
    if analysis is not None and analysis.status == Status.ready:
      data = analysis.get_json()
      if data.get('analyzerData', None) is not None:
        # Use analyzer data for search index
        element_objects = data.get('analyzerData', {}).get('elements', [])
        elements = [element.get('tagname', '') or element.get('classname', '') for element in element_objects]
        if elements != []:
          fields.append(search.TextField(name='element', value=' '.join(elements)))
          weights.append((' '.join(elements), 5))

        behavior_objects = data.get('analyzerData', {}).get('metadata', {}).get('polymer', {}).get('behaviors', [])
        behaviors = [behavior.get('name', '') for behavior in behavior_objects]
        if behaviors != []:
          fields.append(search.TextField(name='behavior', value=' '.join(behaviors)))
          weights.append((' '.join(behaviors), 5))
      else:
        # Use hydrolysis data for search index
        elements = data.get('elementsByTagName', {}).keys()
        if elements != []:
          fields.append(search.TextField(name='element', value=' '.join(elements)))
          weights.append((' '.join(elements), 5))
        behaviors = data.get('behaviorsByName', {}).keys()
        if behaviors != []:
          fields.append(search.TextField(name='behavior', value=' '.join(behaviors)))
          weights.append((' '.join(behaviors), 5))

    weighted = []
    for value, weight in weights:
      for _ in range(0, weight):
        weighted.append(value)
    fields.append(search.TextField(name='weighted_fields', value=' '.join(weighted)))

    rank = int((library.updated - datetime.datetime(2016, 1, 1)).total_seconds())
    document = search.Document(doc_id=Library.id(owner, repo), fields=fields, rank=rank)
    index = search.Index('repo')
    index.put(document)