Python RQueues.enqueue方法代码示例

# 需要导入模块: from opac_proc.datastore.redis_queues import RQueues [as 别名]
# 或者: from opac_proc.datastore.redis_queues.RQueues import enqueue [as 别名]
def task_article_update(ids=None):
    get_db_connection()
    stage = 'transform'
    model = 'article'
    r_queues = RQueues()
    r_queues.create_queues_for_stage(stage)

    collection = models.TransformCollection.objects.all().first()

    if ids is None:  # update all collections
        models.TransformArticle.objects.all().update(must_reprocess=True)
        for article in models.TransformArticle.objects.all():
            r_queues.enqueue(
                stage, model,
                task_transform_article, collection.acronym, article.pid)
    else:
        for oid in ids:
            try:
                obj = models.TransformArticle.objects.get(pk=oid)
                obj.update(must_reprocess=True)
                obj.reload()
                r_queues.enqueue(
                    stage, model,
                    task_transform_article, collection.acronym, obj.pid)
            except Exception as e:
                logger.error('models.TransformArticle %s. pk: %s', str(e), oid)

# 需要导入模块: from opac_proc.datastore.redis_queues import RQueues [as 别名]
# 或者: from opac_proc.datastore.redis_queues.RQueues import enqueue [as 别名]
def task_consume_diff_update(stage, model_name):
    """
    Task que consume os registros UPDATE dos diff filtrando pelos parametros:
    - @param stage: fase do ETL
    - @param model_name: nome do modelo ETL
    """

    if stage not in ETL_STAGE_LIST:
        raise ValueError(u'param stage: %s é inválido' % stage)

    if model_name not in ETL_MODEL_NAME_LIST:
        raise ValueError(u'param model_name: %s é inválido' % model_name)

    action = 'update'
    SLICE_SIZE = 1000
    r_queues = RQueues()
    get_db_connection()
    diff_class = ETL_DIFFERS_BY_MODEL[model_name]
    diff_class_instance = diff_class()
    full_uuids_to_process = diff_class_instance.get_uuids_unapplied(stage, action)
    list_of_list_of_uuids = list(chunks(full_uuids_to_process, SLICE_SIZE))

    for list_of_uuids in list_of_list_of_uuids:
        list_of_uuids_flat = [str(uuid) for uuid in list_of_uuids]
        logger.info(u'enfilerando: consumo de UUUIDs selecionados (stage:%s, model: %s, action: %s)' % (stage, model_name, action))
        r_queues.enqueue(
            'sync_ids', model_name,
            task_differ_apply_for_selected_uuids,
            stage, model_name, action, list_of_uuids_flat)

# 需要导入模块: from opac_proc.datastore.redis_queues import RQueues [as 别名]
# 或者: from opac_proc.datastore.redis_queues.RQueues import enqueue [as 别名]
def task_clean_diff_models(stage='all', model_name='all', action='all'):
    """
    task que enfilera funções para remover modelos Diffs.
    Param:
    - `stage`: filtro do campo stage: "extract", "transform" ou "load"
    - `model_name`: modelo a ser removido: "collection", "journal", etc.
    - `action` filtro do campo: "add" | "update" | "delete"
    """
    r_queues = RQueues()

    if model_name == 'all':
        model_name_list = DIFF_MODEL_CLASS.keys()
    else:
        model_name_list = [model_name]

    if stage == 'all':
        stages_list = STAGE_LIST
    else:
        stages_list = [stage, ]

    if action == 'all':
        actions_list = ACTION_LIST
    else:
        actions_list = [action, ]

    for model_ in model_name_list:
        for stage_ in stages_list:
            for action_ in actions_list:
                msg = u'Enfilerando task para remover o diff model, modelo: %s, stage: %s, action: %s' % (model_, stage_, action_)
                logger.info(msg)
                create_sync_event_record('sync_ids', model_, 'delete_diff_models', msg)
                r_queues.enqueue('sync_ids', model_, delete_diff_models, stage_, model_, action_)
                msg = u'Fim: Enfilerando task para remover o diff model, modelo: %s, stage: %s, action: %s' % (model_, stage_, action_)
                logger.info(msg)
                create_sync_event_record('sync_ids', model_, 'delete_diff_models', msg)

# 需要导入模块: from opac_proc.datastore.redis_queues import RQueues [as 别名]
# 或者: from opac_proc.datastore.redis_queues.RQueues import enqueue [as 别名]
def task_delete_selected_collections(selected_uuids):
    """
        Task para apagar Coleções Carregadas.
        @param:
        - selected_uuids: lista de UUIDs dos documentos a serem removidos

        Se a lista `selected_uuids` for maior a SLICE_SIZE
            A lista será fatiada em listas de tamanho: SLICE_SIZE
        Se a lista `selected_uuids` for < a SLICE_SIZE
            Será feito uma delete direto no queryset
    """

    stage = 'load'
    model = 'collection'
    model_class = LoadCollection
    get_db_connection()
    r_queues = RQueues()
    SLICE_SIZE = 1000

    if len(selected_uuids) > SLICE_SIZE:
        list_of_list_of_uuids = list(chunks(selected_uuids, SLICE_SIZE))
        for list_of_uuids in list_of_list_of_uuids:
            uuid_as_string_list = [str(uuid) for uuid in list_of_uuids]
            r_queues.enqueue(stage, model, task_delete_selected_collections, uuid_as_string_list)
    else:
        # removemos o conjunto de documentos do LoadCollection indicados pelos uuids
        documents_to_delete = model_class.objects.filter(uuid__in=selected_uuids)
        documents_to_delete.delete()

        # convertemos os uuid para _id e filtramos esses documentos no OPAC
        register_connections()
        opac_pks = [str(uuid).replace('-', '') for uuid in selected_uuids]
        with switch_db(opac_models.Collection, OPAC_WEBAPP_DB_NAME) as opac_model:
            selected_opac_records = opac_model.objects.filter(pk__in=opac_pks)
            selected_opac_records.delete()

# 需要导入模块: from opac_proc.datastore.redis_queues import RQueues [as 别名]
# 或者: from opac_proc.datastore.redis_queues.RQueues import enqueue [as 别名]
def process_issue_article(collection, issns, stage, task_issue, task_article):
    r_queues = RQueues()
    r_queues.create_queues_for_stage(stage)

    search_issn = [child['issn'] for child in collection['children_ids']]

    print u"Lista de ISSNs: %s" % search_issn

    children_found = [child for child in collection['children_ids'] if child['issn'] in issns]

    print u"ISSNs encontrados: %s" % [child['issn'] for child in children_found]

    for child in children_found:

        # children_ids, pegamos os articles_ids e issues_ids
        articles_ids = child['articles_ids']
        issues_ids = child['issues_ids']
        print u"Processando %s pids de issues do periódico %s" % (len(issues_ids), child['issn'])
        print u"Processando %s pids de artigos do periódico %s" % (len(articles_ids), child['issn'])

        # para cada pid de issues, enfileramos na queue certa:
        for issue_pid in issues_ids:
            r_queues.enqueue(stage, 'issue', task_issue, collection.acronym, issue_pid)
        print u"Enfilerados %s PIDs de issues do periódico %s" % (len(r_queues.queues[stage]['issue']), child['issn'])

        # para cada pid de artigos, enfileramos na queue certa:
        for article_pid in articles_ids:
            r_queues.enqueue(stage, 'article', task_article, collection.acronym, article_pid)
        print u"Enfilerados %s PIDs de artigos do periódico %s" % (len(r_queues.queues[stage]['article']), child['issn'])

# 需要导入模块: from opac_proc.datastore.redis_queues import RQueues [as 别名]
# 或者: from opac_proc.datastore.redis_queues.RQueues import enqueue [as 别名]
def task_delete_selected_diff_etl_model(stage, model_name, action, selected_uuids):
    if stage not in ETL_STAGE_LIST:
        raise ValueError(u'param stage: %s é inválido' % stage)
    if model_name not in ETL_MODEL_NAME_LIST:
        raise ValueError(u'param model_name: %s é inválido' % model_name)
    if action not in ACTION_LIST:
        raise ValueError(u'param action: %s é inválido' % action)

    get_db_connection()
    model_class = DIFF_MODEL_CLASS_BY_NAME[model_name]
    r_queues = RQueues()
    SLICE_SIZE = 1000

    if len(selected_uuids) > SLICE_SIZE:
        list_of_list_of_uuids = list(chunks(selected_uuids, SLICE_SIZE))
        for list_of_uuids in list_of_list_of_uuids:
            uuid_as_string_list = [str(uuid) for uuid in list_of_uuids]
            r_queues.enqueue(
                'sync_ids',
                model_name,
                task_delete_selected_diff_etl_model,
                stage, model_name, action, uuid_as_string_list)  # args da task
    else:
        documents_to_delete = model_class.objects.filter(uuid__in=selected_uuids)
        documents_to_delete.delete()

# 需要导入模块: from opac_proc.datastore.redis_queues import RQueues [as 别名]
# 或者: from opac_proc.datastore.redis_queues.RQueues import enqueue [as 别名]
def task_load_selected_news(selected_uuids):
    """
        Task para processar Carga de um LISTA de UUIDs do modelo: News
    """
    r_queues = RQueues()
    for uuid in selected_uuids:
        r_queues.enqueue('load', 'news', task_load_one_news, uuid)

# 需要导入模块: from opac_proc.datastore.redis_queues import RQueues [as 别名]
# 或者: from opac_proc.datastore.redis_queues.RQueues import enqueue [as 别名]
def task_load_selected_articles(selected_uuids):
    """
        Task para processar Carga de um LISTA de UUIDs do modelo: Article
    """
    r_queues = RQueues()
    for uuid in selected_uuids:
        r_queues.enqueue('load', 'article', task_load_one_article, uuid)

# 需要导入模块: from opac_proc.datastore.redis_queues import RQueues [as 别名]
# 或者: from opac_proc.datastore.redis_queues.RQueues import enqueue [as 别名]
def task_load_selected_press_releases(selected_uuids):
    """
        Task para processar Carga de um LISTA de UUIDs do modelo: Press Release
    """
    r_queues = RQueues()
    for uuid in selected_uuids:
        r_queues.enqueue('load', 'press_release', task_load_one_press_release, uuid)

# 需要导入模块: from opac_proc.datastore.redis_queues import RQueues [as 别名]
# 或者: from opac_proc.datastore.redis_queues.RQueues import enqueue [as 别名]
def task_clean_etl_models_by_stage(stage='all', model_name='all'):
    """
    task que enfilera funções para remover modelos da fase (stage) indicada.
    Param:
    - `model_name`: modelo a ser removido: "collection", "journal", etc.
    """
    r_queues = RQueues()
    if stage == 'all':
        stages_list = STAGE_LIST
    else:
        stages_list = [stage, ]

    if model_name == 'all':
        model_name_list = ID_MODEL_CLASS.keys()
    else:
        model_name_list = [model_name]

    for model_name_ in model_name_list:
        for stage_ in stages_list:
            msg = u'Enfilerando task para remover o modelo: %s da stage: %s' % (model_name_, stage_)
            logger.info(msg)
            create_sync_event_record('sync_ids', model_name_, 'delete_identifiers', msg)

            r_queues.enqueue('sync_ids', model_name_, delete_etl_models, stage_, model_name_)

            msg = u'Fim: Enfilerando task para remover o modelo: %s da stage: %s' % (model_name_, stage_)
            logger.info(msg)
            create_sync_event_record('sync_ids', model_name_, 'delete_identifiers', msg)

# 需要导入模块: from opac_proc.datastore.redis_queues import RQueues [as 别名]
# 或者: from opac_proc.datastore.redis_queues.RQueues import enqueue [as 别名]
def task_load_selected_journals(selected_uuids):
    """
        Task para processar Carga de um LISTA de UUIDs do modelo: Journal
    """
    r_queues = RQueues()
    for uuid in selected_uuids:
        r_queues.enqueue('load', 'journal', task_load_one_journal, uuid)

# 需要导入模块: from opac_proc.datastore.redis_queues import RQueues [as 别名]
# 或者: from opac_proc.datastore.redis_queues.RQueues import enqueue [as 别名]
def task_delete_selected_collections_identifiers(selected_ids):
    """
        Task para apagar identificadores de Coleção.
        @param:
        - selected_ids: lista de pk dos documentos a serem removidos

        Se a lista `selected_ids` for maior a SLICE_SIZE
            A lista será fatiada em listas de tamanho: SLICE_SIZE
        Se a lista `selected_ids` for < a SLICE_SIZE
            Será feito uma delete direto no queryset
    """

    model_class = CollectionIdModel
    get_db_connection()
    r_queues = RQueues()
    SLICE_SIZE = 1000

    if len(selected_ids) > SLICE_SIZE:
        list_of_list_of_uuids = list(chunks(selected_ids, SLICE_SIZE))
        for list_of_uuids in list_of_list_of_uuids:
            uuid_as_string_list = [str(uuid) for uuid in list_of_uuids]
            r_queues.enqueue(
                'sync_ids', 'collection',
                task_delete_selected_collections_identifiers, uuid_as_string_list)
    else:
        documents_to_delete = model_class.objects.filter(pk__in=selected_ids)
        documents_to_delete.delete()

# 需要导入模块: from opac_proc.datastore.redis_queues import RQueues [as 别名]
# 或者: from opac_proc.datastore.redis_queues.RQueues import enqueue [as 别名]
def task_journal_update(ids=None):
    get_db_connection()
    stage = 'transform'
    model = 'journal'
    r_queues = RQueues()
    r_queues.create_queues_for_stage(stage)
    collection = models.TransformCollection.objects.all().first()
    if ids is None:  # update all collections
        models.TransformJournal.objects.all().update(must_reprocess=True)
        for journal in models.TransformJournal.objects.all():
            issn = journal.get('scielo_issn', False) or \
                   journal.get('print_issn', False) or \
                   journal.get('eletronic_issn', False)
            if not issn:
                raise ValueError(u'Journal sem issn')
            r_queues.enqueue(
                stage, model,
                task_transform_journal, collection.acronym, issn)
    else:
        for oid in ids:
            try:
                obj = models.TransformJournal.objects.get(pk=oid)
                obj.update(must_reprocess=True)
                obj.reload()
                issn = obj.get('scielo_issn', False) or \
                    obj.get('print_issn', False) or \
                    obj.get('eletronic_issn', False)
                if not issn:
                    raise ValueError(u'Journal sem issn')
                r_queues.enqueue(
                    stage, model,
                    task_transform_journal, collection.acronym, issn)
            except Exception as e:
                logger.error('models.TransformJournal %s. pk: %s', str(e), oid)

# 需要导入模块: from opac_proc.datastore.redis_queues import RQueues [as 别名]
# 或者: from opac_proc.datastore.redis_queues.RQueues import enqueue [as 别名]
def task_delete_selected_collections(selected_uuids):
    """
        Task para apagar Coleções Transformadas.
        @param:
        - selected_uuids: lista de UUIDs dos documentos a serem removidos

        Se a lista `selected_uuids` for maior a SLICE_SIZE
            A lista será fatiada em listas de tamanho: SLICE_SIZE
        Se a lista `selected_uuids` for < a SLICE_SIZE
            Será feito uma delete direto no queryset
    """

    stage = 'transform'
    model = 'collection'
    model_class = TransformCollection
    get_db_connection()
    r_queues = RQueues()
    SLICE_SIZE = 1000

    if len(selected_uuids) > SLICE_SIZE:
        list_of_list_of_uuids = list(chunks(selected_uuids, SLICE_SIZE))
        for list_of_uuids in list_of_list_of_uuids:
            uuid_as_string_list = [str(uuid) for uuid in list_of_uuids]
            r_queues.enqueue(stage, model, task_delete_selected_collections, uuid_as_string_list)
    else:
        documents_to_delete = model_class.objects.filter(uuid__in=selected_uuids)
        documents_to_delete.delete()

# 需要导入模块: from opac_proc.datastore.redis_queues import RQueues [as 别名]
# 或者: from opac_proc.datastore.redis_queues.RQueues import enqueue [as 别名]
def enqueue_full_populate_task_by_model(model_name='all'):
    logger.info("Inicinado: enqueue_full_populate_task_by_model para modelo: %s", model_name)
    create_sync_event_record(
        'sync_ids', model_name, 'enqueue_full_populate_task_by_model',
        u'Inciando enfileramento para preencher datas dos registros diff model: %s' % model_name)

    # setup
    get_db_connection()
    stage = 'sync_ids'
    r_queues = RQueues()
    model_class = None
    task_fn = None

    options = {
        'collection': {
            'model_class': models.ExtractCollection,
            'task_fn': task_populate_collections
        },
        'journal': {
            'model_class': models.ExtractJournal,
            'task_fn': task_populate_journals
        },
        'issue': {
            'model_class': models.ExtractIssue,
            'task_fn': task_populate_issues
        },
        'article': {
            'model_class': models.ExtractArticle,
            'task_fn': task_populate_articles
        },
        'news': {
            'model_class': models.ExtractNews,
            'task_fn': task_populate_news
        },
        'press_release': {
            'model_class': models.ExtractPressRelease,
            'task_fn': task_populate_press_release
        }
    }

    if model_name == 'all':
        for k, v in options.items():
            model_class = v['model_class']
            task_fn = v['task_fn']
            logger.info("enfilerando stage: %s model_name: %s model_class: %s" % (stage, k, model_class))
            r_queues.enqueue(stage, k, task_fn)
    elif model_name not in options.keys():
        raise ValueError('Param: model_name: %s inesperado' % model_name)
    else:
        model_class = options[model_name]['model_class']
        task_fn = options[model_name]['task_fn']
        logger.info("enfilerando stage: %s model_name: %s model_class: %s" % (stage, model_name, model_class))
        r_queues.enqueue(stage, model_name, task_fn)

    create_sync_event_record(
        'sync_ids', model_name, 'enqueue_full_populate_task_by_model',
        u'Fim do enfileramento para preencher datas dos registros diff model: %s' % model_name)

    logger.info("Fim: enqueue_full_populate_task_by_model para modelo: %s", model_name)