Python datamodel.Library类代码示例

  def test_normal(self):
    headers = {'X-Github-Event': 'pull_request'}
    payload = {
        'action': 'opened',
        'repository': {
            'owner': {'login': 'owner'},
            'name': 'repo',
            'full_name': 'owner/repo'
        },
        'pull_request': {
            'head': {
                'sha': 'sha',
                'repo': {
                    'owner': {'login': 'pull_owner'},
                    'name': 'pull_repo',
                    'full_name': 'pull_owner/pull_repo'
                }
            },
            'url': 'github_pr_url'
        }
    }
    library = Library(id='owner/repo')
    library.put()

    self.respond_to('https://api.github.com/repos/owner/repo/statuses', {'status': 201})
    self.app.post('/api/preview-event', params=json.dumps(payload), headers=headers, status=200)
    tasks = self.tasks.get_filtered_tasks()
    self.assertEqual(len(tasks), 1)

  def test_update_collection(self):
    library_key = Library(id='org/repo', tags=['v0.0.1'], collection_sequence_number=1, kind='collection', spdx_identifier='MIT').put()
    Version(id='v0.0.1', parent=library_key, sha="old", status=Status.ready).put()

    self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304})
    self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304})
    self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
    self.respond_to_github('https://api.github.com/repos/org/repo/git/refs/heads/master', """{
      "ref": "refs/heads/master",
      "object": {"sha": "new-master-sha"}
    }""")

    response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'})
    self.assertEqual(response.status_int, 200)
    library = library_key.get()
    self.assertEqual(library.error, None)
    self.assertEqual(library.status, Status.ready)

    tasks = self.tasks.get_filtered_tasks()
    self.assertEqual([
        util.ingest_analysis_task('org', 'repo', 'v0.0.2', 'new-master-sha'),
        util.ingest_version_task('org', 'repo', 'v0.0.2'),
    ], [task.url for task in tasks])

    version = Version.get_by_id('v0.0.2', parent=library_key)
    self.assertEqual(version.sha, 'new-master-sha')
    self.assertEqual(version.status, Status.pending)

  def handle_get(self):
    keys = (Library.query()
            .filter(Library.kind == 'element')
            # pylint: disable=singleton-comparison
            .filter(Library.shallow_ingestion == False)
            .filter(Library.status == Status.ready)
            .fetch(keys_only=True, read_policy=ndb.EVENTUAL_CONSISTENCY))
    elements = Sitemap(id='elements')
    elements.pages = [key.id() for key in keys]
    elements.put()
    logging.info('%d elements', len(elements.pages))

    keys = (Library.query()
            .filter(Library.kind == 'collection')
            # pylint: disable=singleton-comparison
            .filter(Library.shallow_ingestion == False)
            .filter(Library.status == Status.ready)
            .fetch(keys_only=True, read_policy=ndb.EVENTUAL_CONSISTENCY))
    collections = Sitemap(id='collections')
    collections.pages = [key.id() for key in keys]
    collections.put()
    logging.info('%d collections', len(collections.pages))

    keys = Author.query().fetch(keys_only=True, read_policy=ndb.EVENTUAL_CONSISTENCY)
    authors = Sitemap(id='authors')
    authors.pages = [key.id() for key in keys]
    authors.put()
    logging.info('%d authors', len(authors.pages))

  def update_readme(self, is_npm_package):
    if is_npm_package:
      # Load registry metadata to fetch readme path.
      library = Library.get_by_id(Library.id(self.owner, self.repo))
      registry_metadata = json.loads(library.registry_metadata) if library.registry_metadata else None
      readme_path = registry_metadata.get('readmeFilename', 'README.md')
      response = util.unpkg_get(self.owner, self.repo, self.version, readme_path)
      readme = response.content
    else:
      # Load readme from GitHub endpoint.
      response = util.github_get('repos', self.owner, self.repo, 'readme', params={"ref": self.sha})

      if response.status_code == 200:
        readme = base64.b64decode(json.loads(response.content)['content'])
      elif response.status_code == 404:
        readme = None
      else:
        return self.retry('error fetching readme (%d)' % response.status_code)

    if readme is not None:
      # Store the raw readme markdown.
      try:
        Content(parent=self.version_key, id='readme', content=readme,
                status=Status.ready, etag=response.headers.get('ETag', None)).put()
      except db.BadValueError:
        return self.error("Could not store README.md as a utf-8 string", ErrorCodes.Version_utf)

      # Convert markdown to HTML and store the result.
      response = util.github_markdown(readme)
      if response.status_code == 200:
        Content(parent=self.version_key, id='readme.html', content=response.content,
                status=Status.ready, etag=response.headers.get('ETag', None)).put()
      else:
        return self.retry('error converting readme to markdown (%d)' % response.status_code)

 def init_library(self, owner, repo, kind=None, create=True):
   self.owner = owner.lower()
   self.repo = repo.lower()
   if create:
     assert kind is not None
     self.library = Library.maybe_create_with_kind(self.owner, self.repo, kind)
   else:
     self.library = Library.get_by_id('%s/%s' % (owner, repo))

  def update_metadata(self):
    headers = {'Accept': 'application/vnd.github.drax-preview+json'}
    response = util.github_get('repos', self.owner, self.repo, etag=self.library.metadata_etag, headers=headers)
    if response.status_code == 200:
      try:
        metadata = json.loads(response.content)
      except ValueError:
        return self.error("could not parse metadata")

      repo = metadata.get('name', '').lower()
      owner = metadata.get('owner', {}).get('login', '').lower()
      if repo != '' and owner != '' and (repo != self.repo or owner != self.owner):
        logging.info('deleting renamed repo %s', Library.id(self.owner, self.repo))
        delete_library(self.library.key)
        task_url = util.ensure_library_task(owner, repo)
        util.new_task(task_url, target='manage')
        raise RequestAborted('repo has been renamed to %s', Library.id(owner, repo))

      self.library.metadata = response.content
      self.library.metadata_etag = response.headers.get('ETag', None)
      self.library.metadata_updated = datetime.datetime.now()
      self.library_dirty = True
    elif response.status_code == 404:
      logging.info('deleting non-existing repo %s', Library.id(self.owner, self.repo))
      delete_library(self.library.key)
      raise RequestAborted('repo no longer exists')
    elif response.status_code != 304:
      return self.retry('could not update repo metadata (%d)' % response.status_code)

    response = util.github_get('repos', self.owner, self.repo, 'contributors', etag=self.library.contributors_etag)
    if response.status_code == 200:
      try:
        json.loads(response.content)
      except ValueError:
        return self.error("could not parse contributors")
      self.library.contributors = response.content
      self.library.contributors_etag = response.headers.get('ETag', None)
      self.library.contributors_updated = datetime.datetime.now()
      self.library_dirty = True
    elif response.status_code != 304:
      return self.retry('could not update contributors (%d)' % response.status_code)

    response = util.github_get('repos', self.owner, self.repo, 'stats/participation ', etag=self.library.participation_etag)
    if response.status_code == 200:
      try:
        json.loads(response.content)
      except ValueError:
        return self.error("could not parse stats/participation")
      self.library.participation = response.content
      self.library.participation_etag = response.headers.get('ETag', None)
      self.library.participation_updated = datetime.datetime.now()
      self.library_dirty = True
    elif response.status_code == 202:
      # GitHub is "computing" the data. We'll try again next update cycle.
      # TODO: Alternatively we could retry this task
      pass
    elif response.status_code != 304:
      return self.retry('could not update stats/participation (%d)' % response.status_code)

 def init_library(self, owner, repo, create=True):
   self.owner = owner.lower()
   self.repo = repo.lower()
   if create:
     self.library = Library.get_or_insert(Library.id(owner, repo))
     self.is_new = self.library.metadata is None and self.library.error is None
   else:
     self.library = Library.get_by_id(Library.id(owner, repo))
   if self.library.status == Status.suppressed:
     raise RequestAborted('library is suppressed')

  def test_update_suppressed_is_noop(self):
    library = Library(id='org/repo', status=Status.suppressed, spdx_identifier='MIT')
    library.put()
    response = self.app.get('/task/update/org/repo', headers={'X-AppEngine-QueueName': 'default'})
    self.assertEqual(response.status_int, 200)
    tasks = self.tasks.get_filtered_tasks()
    self.assertEqual(len(tasks), 0)

    library = library.key.get()
    self.assertEqual(library.status, Status.suppressed)

  def init_library(self, scope, package, create=True):
    self.scope = scope.lower()
    self.package = package.lower()

    if create:
      self.library = Library.get_or_insert(Library.id(self.scope, self.package))
      self.is_new = self.library.metadata is None and self.library.error is None
    else:
      self.library = Library.get_by_id(Library.id(self.scope, self.package))
    if self.library.status == Status.suppressed:
      raise RequestAborted('library is suppressed')

  def test_update_respects_304(self):
    library = Library(id='org/repo', metadata_etag='a', contributors_etag='b', tags_etag='c', tag_map='{}', spdx_identifier='MIT')
    library.put()
    self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304})
    self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304})
    self.respond_to_github('https://api.github.com/repos/org/repo/tags', {'status': 304})
    self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')

    response = self.app.get('/task/update/org/repo', headers={'X-AppEngine-QueueName': 'default'})
    self.assertEqual(response.status_int, 200)
    tasks = self.tasks.get_filtered_tasks()
    self.assertEqual(len(tasks), 0)

  def test_update_all(self):
    library_key = Library(id='owner/repo').put()
    author_key = Author(id='owner').put()

    response = self.app.get('/manage/update-all', headers={'X-AppEngine-QueueName': 'default'})
    self.assertEqual(response.status_int, 200)

    tasks = self.tasks.get_filtered_tasks()
    self.assertEqual([
        util.update_library_task(library_key.id()),
        util.update_author_task(author_key.id()),
    ], [task.url for task in tasks])

  def test_update_deletes_missing_repo(self):
    library = Library(id='org/repo', metadata_etag='a', contributors_etag='b', tags_etag='c', spdx_identifier='MIT')
    library.put()
    version = Version(parent=library.key, id='v1.0.0', sha='lol')
    version.put()

    self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 404})
    response = self.app.get('/task/update/org/repo', headers={'X-AppEngine-QueueName': 'default'})
    self.assertEqual(response.status_int, 200)

    version = version.key.get()
    library = library.key.get()

    self.assertIsNone(library)
    self.assertIsNone(version)

  def test_ingest_collection(self):
    self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{"keywords": ["element-collection"], "license": "MIT"}')
    self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"org"},"name":"repo"}')
    self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]')
    self.respond_to_github('https://api.github.com/repos/org/repo/git/refs/heads/master', '{"ref": "refs/heads/master", "object": {"sha": "master-sha"}}')
    self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
    response = self.app.get(util.ingest_library_task('org', 'repo'), headers={'X-AppEngine-QueueName': 'default'})

    self.assertEqual(response.status_int, 200)
    library = Library.get_by_id('org/repo')
    self.assertIsNotNone(library)
    self.assertIsNone(library.error)
    self.assertEqual(library.metadata, '{"owner":{"login":"org"},"name":"repo"}')
    self.assertEqual(library.contributors, '["a"]')
    self.assertEqual(library.tags, ['v0.0.1'])

    version = ndb.Key(Library, 'org/repo', Version, 'v0.0.1').get()
    self.assertIsNone(version.error)
    self.assertEqual(version.status, Status.pending)
    self.assertEqual(version.sha, 'master-sha')

    tasks = self.tasks.get_filtered_tasks()
    self.assertEqual([
        util.ingest_analysis_task('org', 'repo', 'v0.0.1', 'master-sha'),
        util.ensure_author_task('org'),
        util.ingest_version_task('org', 'repo', 'v0.0.1'),
    ], [task.url for task in tasks])

  def test_add_element(self):
    response = self.app.get('/manage/add/element/org/repo')
    self.assertEqual(response.status_int, 200)
    self.assertEqual(response.normal_body, 'OK')

    tasks = self.tasks.get_filtered_tasks()
    self.assertEqual(len(tasks), 1)
    self.assertEqual(tasks[0].url, util.ingest_library_task('org', 'repo', 'element'))

    self.respond_to_github('https://api.github.com/repos/org/repo', 'metadata bits')
    self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]')
    self.respond_to_github('https://api.github.com/repos/org/repo/git/refs/tags', '[{"ref": "refs/tags/v1.0.0", "object": {"sha": "lol"}}]')
    response = self.app.get(util.ingest_library_task('org', 'repo', 'element'))
    self.assertEqual(response.status_int, 200)
    library = Library.get_by_id('org/repo')
    self.assertIsNotNone(library)
    self.assertIsNone(library.error)
    self.assertEqual(library.kind, 'element')
    self.assertEqual(library.metadata, 'metadata bits')
    self.assertEqual(library.contributors, '["a"]')
    self.assertEqual(library.contributor_count, 1)

    version = ndb.Key(Library, 'org/repo', Version, 'v1.0.0').get()
    self.assertIsNone(version.error)
    self.assertEqual(version.sha, 'lol')

    tasks = self.tasks.get_filtered_tasks()
    self.assertEqual(len(tasks), 2)
    self.assertEqual(tasks[1].url, util.ingest_version_task('org', 'repo', 'v1.0.0') + '?latestVersion=True')

  def test_ingest_version(self):
    library_key = Library(id='org/repo', metadata='{"full_name": "NSS Bob", "stargazers_count": 420, "subscribers_count": 419, "forks": 418, "updated_at": "2011-8-10T13:47:12Z"}').put()
    Version(id='v1.0.0', parent=library_key, sha='sha').put()

    self.respond_to_github(r'https://api.github.com/repos/org/repo/readme\?ref=sha', '{"content":"%s"}' % b64encode('README'))
    self.respond_to('https://raw.githubusercontent.com/org/repo/sha/bower.json', '{}')
    self.respond_to_github('https://api.github.com/markdown', '<html>README</html>')

    response = self.app.get(util.ingest_version_task('org', 'repo', 'v1.0.0'), headers={'X-AppEngine-QueueName': 'default'})
    self.assertEqual(response.status_int, 200)

    version = Version.get_by_id('v1.0.0', parent=library_key)
    self.assertIsNone(version.error)
    self.assertEqual(version.status, Status.ready)
    self.assertFalse(version.preview)

    versions = Library.versions_for_key_async(library_key).get_result()
    self.assertEqual(['v1.0.0'], versions)

    readme = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'readme').get()
    self.assertEqual(readme.content, 'README')
    readme_html = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'readme.html').get()
    self.assertEqual(readme_html.content, '<html>README</html>')
    bower = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'bower').get()
    self.assertEqual(bower.content, '{}')