Python model.Archival类代码示例

def save_archival(resource, status_id, reason, url_redirected_to,
                  download_result, archive_result, log):
    '''Writes to the archival table the result of an attempt to download
    the resource.

    May propagate a CkanError.
    '''
    now = datetime.datetime.now()

    from ckanext.archiver.model import Archival, Status
    from ckan import model

    archival = Archival.get_for_resource(resource['id'])
    first_archival = not archival
    previous_archival_was_broken = None
    if not archival:
        archival = Archival.create(resource['id'])
        model.Session.add(archival)
    else:
        log.info('Archival from before: %r', archival)
        previous_archival_was_broken = archival.is_broken

    revision = model.Session.query(model.Revision).get(resource['revision_id'])
    archival.resource_timestamp = revision.timestamp

    # Details of the latest archival attempt
    archival.status_id = status_id
    archival.is_broken = Status.is_status_broken(status_id)
    archival.reason = reason
    archival.url_redirected_to = url_redirected_to

    # Details of successful archival
    if archival.is_broken is False:
        archival.cache_filepath = archive_result['cache_filepath']
        archival.cache_url = archive_result['cache_url']
        archival.size = download_result['size']
        archival.mimetype = download_result['mimetype']
        archival.hash = download_result['hash']
        archival.etag = download_result['headers'].get('etag')
        archival.last_modified = download_result['headers'].get('last-modified')

    # History
    if archival.is_broken is False:
        archival.last_success = now
        archival.first_failure = None
        archival.failure_count = 0
    else:
        log.info('First_archival=%r Previous_broken=%r Failure_count=%r' %
                 (first_archival, previous_archival_was_broken,
                  archival.failure_count))
        if first_archival or previous_archival_was_broken is False:
            # i.e. this is the first failure (or the first archival)
            archival.first_failure = now
            archival.failure_count = 1
        else:
            archival.failure_count += 1

    archival.updated = now
    log.info('Archival saved: %r', archival)
    model.repo.commit_and_remove()

def qa_resource_show(context, data_dict):
    '''
    Returns the QA and Archival information for a package or resource.
    '''
    model = context['model']
    session = context['session']
    # user = context.get('user')
    # p.toolkit.check_access('qa_resource_show', context, data_dict)

    res_id = p.toolkit.get_or_bust(data_dict, 'id')
    res = session.query(model.Resource).get(res_id)
    if not res:
        raise p.toolkit.ObjectNotFound

    archival = Archival.get_for_resource(res_id)
    qa = QA.get_for_resource(res_id)
    pkg = res.resource_group.package
    return_dict = {
        'name': pkg.name,
        'title': pkg.title,
        'id': res.id
        }
    return_dict['archival'] = archival.as_dict()
    return_dict.update(qa.as_dict())
    return return_dict

 def test_not_available_any_more(self):
     # A cache of the data still exists from the previous run, but this
     # time, the archiver found the file gave a 404.
     # The record of the previous (successful) run of QA.
     res = self._test_resource(license_id=None, format=None)
     qa = qa_model.QA.create(res.id)
     qa.format = 'CSV'
     model.Session.add(qa)
     model.Session.commit()
     # cache still exists from the previous run, but this time, the archiver
     # found the file gave a 404.
     archival = Archival.get_for_resource(res.id)
     archival.cache_filepath = __file__
     archival.status_id = Status.by_text('Download error')
     archival.reason = 'Server returned 404 error'
     archival.last_success = datetime.datetime(year=2008, month=10, day=1)
     archival.first_failure = datetime.datetime(year=2008, month=10, day=2)
     archival.failure_count = 1
     archival.is_broken = True
     result = resource_score(res)
     assert result['openness_score'] == 0, result
     assert_equal(result['format'], 'CSV')
     # in preference it should report that it is not available
     assert_equal(result['openness_score_reason'], 'File could not be downloaded. '
                                                   'Reason: Download error. Error details: Server returned 404 error.'
                                                   ' Attempted on 10/10/2008. This URL last worked on: 01/10/2008.')

def qa_resource_show(context, data_dict):
    '''
    Returns the QA and Archival information for a package or resource.
    '''
    model = context['model']
    session = context['session']
    #user = context.get('user')
    #p.toolkit.check_access('qa_resource_show', context, data_dict)

    res_id = p.toolkit.get_or_bust(data_dict, 'id')
    res = session.query(model.Resource).get(res_id)
    if not res:
        raise p.toolkit.ObjectNotFound

    archival = Archival.get_for_resource(res_id)
    qa = QA.get_for_resource(res_id)
    pkg = res.resource_group.package
    return {'name': pkg.name,
            'title': pkg.title,
            'id': res.id,
            'archival_updated': archival.updated.isoformat() if archival and archival.updated else None,
            'archival_is_broken': archival.is_broken if archival else None,
            'archival_reason': archival.reason if archival else None,
            'archival_url_redirected_to': archival.url_redirected_to if archival else None,
            'openness_score': qa.openness_score if qa else None,
            'openness_score_reason': qa.openness_score_reason if qa else None,
            'updated': qa.updated.isoformat() if qa and qa.updated else None,
            'format': qa.format if qa else None,
            }

def resource_score(resource):
    """
    Score resource on Sir Tim Berners-Lee\'s five stars of openness.

    Returns a dict with keys:

        'openness_score': score (int)
        'openness_score_reason': the reason for the score (string)
        'format': format of the data (string)
        'archival_timestamp': time of the archival that this result is based on (iso string)

    Raises QAError for reasonable errors
    """
    score = 0
    score_reason = ''
    format_ = None

    register_translator()

    try:
        score_reasons = []  # a list of strings detailing how we scored it
        archival = Archival.get_for_resource(resource_id=resource.id)
        if not resource:
            raise QAError('Could not find resource "%s"' % resource.id)

        score, format_ = score_if_link_broken(archival, resource, score_reasons)
        if score is None:
            # we don't want to take the publisher's word for it, in case the link
            # is only to a landing page, so highest priority is the sniffed type
            score, format_ = score_by_sniffing_data(archival, resource,
                                                    score_reasons)
            if score is None:
                # Fall-backs are user-given data
                score, format_ = score_by_url_extension(resource, score_reasons)
                if score is None:
                    score, format_ = score_by_format_field(resource, score_reasons)
                    if score is None:
                        log.warning('Could not score resource: "%s" with url: "%s"',
                                    resource.id, resource.url)
                        score_reasons.append(_('Could not understand the file format, therefore score is 1.'))
                        score = 1
                        if format_ is None:
                            # use any previously stored format value for this resource
                            format_ = get_qa_format(resource.id)
        score_reason = ' '.join(score_reasons)
        format_ = format_ or None
    except Exception, e:
        log.error('Unexpected error while calculating openness score %s: %s\nException: %s',
                  e.__class__.__name__,  unicode(e), traceback.format_exc())
        score_reason = _("Unknown error: %s") % str(e)
        raise

 def _test_resource(self, url='anything', format='TXT', archived=True, cached=True, license_id='uk-ogl'):
     context = {'model': model, 'ignore_auth': True, 'session': model.Session, 'user': 'test'}
     pkg = {'name': 'testpkg', 'license_id': license_id, 'resources': [
         {'url': url, 'format': format, 'description': 'Test'}
         ]}
     pkg = get_action('package_create')(context, pkg)
     res_id = pkg['resources'][0]['id']
     if archived:
         archival = Archival.create(res_id)
         archival.cache_filepath = __file__ if cached else None  # just needs to exist
         archival.updated = TODAY
         model.Session.add(archival)
         model.Session.commit()
     return model.Resource.get(res_id)

 def _test_resource(self, url='anything', format='TXT', archived=True, cached=True, license_id='uk-ogl'):
     pkg = {'license_id': license_id,
            'resources': [
                {'url': url, 'format': format, 'description': 'Test'}
            ]}
     pkg = ckan_factories.Dataset(**pkg)
     res_id = pkg['resources'][0]['id']
     if archived:
         archival = Archival.create(res_id)
         archival.cache_filepath = __file__ if cached else None  # just needs to exist
         archival.updated = TODAY
         model.Session.add(archival)
         model.Session.commit()
     return model.Resource.get(res_id)

def archiver_resource_show(context, data_dict=None):
    '''Return a details of the archival of a resource

    :param id: the id of the resource
    :type id: string

    :rtype: dictionary
    '''
    id_ = _get_or_bust(data_dict, 'id')
    archival = Archival.get_for_resource(id_)
    if archival is None:
        raise ObjectNotFound
    archival_dict = archival.as_dict()
    p.toolkit.check_access('archiver_resource_show', context, data_dict)
    return archival_dict

 def test_not_available_and_not_open(self):
     res = self._test_resource(license_id=None, format=None, cached=False)
     archival = Archival.get_for_resource(res.id)
     archival.status_id = Status.by_text('Download error')
     archival.reason = 'Server returned 500 error'
     archival.last_success = None
     archival.first_failure = datetime.datetime(year=2008, month=10, day=1, hour=6, minute=30)
     archival.failure_count = 16
     archival.is_broken = True
     model.Session.commit()
     result = resource_score(res, log)
     assert result['openness_score'] == 0, result
     assert_equal(result['format'], None)
     # in preference it should report that it is not available
     assert_equal(result['openness_score_reason'], 'File could not be downloaded. Reason: Download error. Error details: Server returned 500 error. Attempted on 10/10/2008. Tried 16 times since 01/10/2008. This URL has not worked in the history of this tool.')

def archiver_dataset_show(context, data_dict=None):
    '''Return a details of the archival of a dataset, aggregated across its
    resources.

    :param id: the name or id of the dataset
    :type id: string

    :rtype: dictionary
    '''
    id_ = _get_or_bust(data_dict, 'id')
    dataset = model.Package.get(id_)
    if not dataset:
        raise ObjectNotFound
    archivals = Archival.get_for_package(dataset.id)
    archival_dict = aggregate_archivals_for_a_dataset(archivals)
    p.toolkit.check_access('archiver_dataset_show', context, data_dict)
    return archival_dict

def qa_package_broken_show(context, data_dict):
    '''
    Returns the Archival is_broken information for a package, aggregating
    across its resources.

    is_broken - True (all), 'some', False or None (not sure)
    '''
    model = context['model']
    session = context['session']
    #user = context.get('user')
    #p.toolkit.check_access('qa_package_broken_show', context, data_dict)

    pkg_id = p.toolkit.get_or_bust(data_dict, 'id')
    pkg = session.query(model.Package).get(pkg_id)
    if not pkg:
        raise p.toolkit.ObjectNotFound

    if pkg.resources:
        # Are any broken?
        any_resources_broken = False
        any_resources_ok = False
        for archival in Archival.get_for_package(pkg_id):
            if archival.is_broken is True:
                any_resources_broken = True
            elif archival.is_broken is False:
                any_resources_ok = True
        if any_resources_broken and any_resources_ok:
            is_broken = 'some'  # i.e. some broken
        elif any_resources_broken:
            is_broken = True  # all broken
        elif any_resources_ok:
            is_broken = False  # all ok
        else:
            is_broken = None  # not sure / not recorded
    else:
        is_broken = False
    return {'name': pkg.name,
            'title': pkg.title,
            'id': pkg.id,
            'archival_is_broken': is_broken,
            }

    def test_trigger_on_archival(cls):
        # create package
        context = {'model': model, 'ignore_auth': True, 'session': model.Session, 'user': 'test'}
        pkg = {'name': 'testpkg', 'license_id': 'uk-ogl', 'resources': [
            {'url': 'http://test.com/', 'format': 'CSV', 'description': 'Test'}
            ]}
        pkg = get_action('package_create')(context, pkg)
        resource_dict = pkg['resources'][0]
        res_id = resource_dict['id']
        # create record of archival
        archival = Archival.create(res_id)
        cache_filepath = __file__  # just needs to exist
        archival.cache_filepath = cache_filepath
        archival.updated = TODAY
        model.Session.add(archival)
        model.Session.commit()
        # TODO show that QA hasn't run yet

        # create a send_data from ckanext-archiver, that gets picked up by
        # ckanext-qa to put a task on the queue
        ckanext.archiver.tasks.notify_package(pkg, 'priority', cache_filepath)

 def after_show(self, context, pkg_dict):
     # Insert the archival info into the package_dict so that it is
     # available on the API.
     # When you edit the dataset, these values will not show in the form,
     # it they will be saved in the resources (not the dataset). I can't see
     # and easy way to stop this, but I think it is harmless. It will get
     # overwritten here when output again.
     archivals = Archival.get_for_package(pkg_dict['id'])
     if not archivals:
         return
     # dataset
     dataset_archival = aggregate_archivals_for_a_dataset(archivals)
     pkg_dict['archiver'] = dataset_archival
     # resources
     archivals_by_res_id = dict((a.resource_id, a) for a in archivals)
     for res in pkg_dict['resources']:
         archival = archivals_by_res_id.get(res['id'])
         if archival:
             archival_dict = archival.as_dict()
             del archival_dict['id']
             del archival_dict['package_id']
             del archival_dict['resource_id']
             res['archiver'] = archival_dict

 def assert_archival_error(self, error_message_fragment, resource_id):
     archival = Archival.get_for_resource(resource_id)
     if error_message_fragment not in archival.reason:
         print 'ERROR: %s (%s)' % (archival.reason, archival.status)
         raise AssertionError(archival.reason)

 def is_res_broken(resource):
     archival = Archival.get_for_resource(resource.id)
     if not archival:
         return None
     return archival.is_broken