Python redis_queues.RQueues类代码示例

def task_delete_selected_collections(selected_uuids):
    """
        Task para apagar Coleções Carregadas.
        @param:
        - selected_uuids: lista de UUIDs dos documentos a serem removidos

        Se a lista `selected_uuids` for maior a SLICE_SIZE
            A lista será fatiada em listas de tamanho: SLICE_SIZE
        Se a lista `selected_uuids` for < a SLICE_SIZE
            Será feito uma delete direto no queryset
    """

    stage = 'load'
    model = 'collection'
    model_class = LoadCollection
    get_db_connection()
    r_queues = RQueues()
    SLICE_SIZE = 1000

    if len(selected_uuids) > SLICE_SIZE:
        list_of_list_of_uuids = list(chunks(selected_uuids, SLICE_SIZE))
        for list_of_uuids in list_of_list_of_uuids:
            uuid_as_string_list = [str(uuid) for uuid in list_of_uuids]
            r_queues.enqueue(stage, model, task_delete_selected_collections, uuid_as_string_list)
    else:
        # removemos o conjunto de documentos do LoadCollection indicados pelos uuids
        documents_to_delete = model_class.objects.filter(uuid__in=selected_uuids)
        documents_to_delete.delete()

        # convertemos os uuid para _id e filtramos esses documentos no OPAC
        register_connections()
        opac_pks = [str(uuid).replace('-', '') for uuid in selected_uuids]
        with switch_db(opac_models.Collection, OPAC_WEBAPP_DB_NAME) as opac_model:
            selected_opac_records = opac_model.objects.filter(pk__in=opac_pks)
            selected_opac_records.delete()

def task_consume_diff_update(stage, model_name):
    """
    Task que consume os registros UPDATE dos diff filtrando pelos parametros:
    - @param stage: fase do ETL
    - @param model_name: nome do modelo ETL
    """

    if stage not in ETL_STAGE_LIST:
        raise ValueError(u'param stage: %s é inválido' % stage)

    if model_name not in ETL_MODEL_NAME_LIST:
        raise ValueError(u'param model_name: %s é inválido' % model_name)

    action = 'update'
    SLICE_SIZE = 1000
    r_queues = RQueues()
    get_db_connection()
    diff_class = ETL_DIFFERS_BY_MODEL[model_name]
    diff_class_instance = diff_class()
    full_uuids_to_process = diff_class_instance.get_uuids_unapplied(stage, action)
    list_of_list_of_uuids = list(chunks(full_uuids_to_process, SLICE_SIZE))

    for list_of_uuids in list_of_list_of_uuids:
        list_of_uuids_flat = [str(uuid) for uuid in list_of_uuids]
        logger.info(u'enfilerando: consumo de UUUIDs selecionados (stage:%s, model: %s, action: %s)' % (stage, model_name, action))
        r_queues.enqueue(
            'sync_ids', model_name,
            task_differ_apply_for_selected_uuids,
            stage, model_name, action, list_of_uuids_flat)

def task_load_selected_articles(selected_uuids):
    """
        Task para processar Carga de um LISTA de UUIDs do modelo: Article
    """
    r_queues = RQueues()
    for uuid in selected_uuids:
        r_queues.enqueue('load', 'article', task_load_one_article, uuid)

def task_load_selected_press_releases(selected_uuids):
    """
        Task para processar Carga de um LISTA de UUIDs do modelo: Press Release
    """
    r_queues = RQueues()
    for uuid in selected_uuids:
        r_queues.enqueue('load', 'press_release', task_load_one_press_release, uuid)

def task_clean_etl_models_by_stage(stage='all', model_name='all'):
    """
    task que enfilera funções para remover modelos da fase (stage) indicada.
    Param:
    - `model_name`: modelo a ser removido: "collection", "journal", etc.
    """
    r_queues = RQueues()
    if stage == 'all':
        stages_list = STAGE_LIST
    else:
        stages_list = [stage, ]

    if model_name == 'all':
        model_name_list = ID_MODEL_CLASS.keys()
    else:
        model_name_list = [model_name]

    for model_name_ in model_name_list:
        for stage_ in stages_list:
            msg = u'Enfilerando task para remover o modelo: %s da stage: %s' % (model_name_, stage_)
            logger.info(msg)
            create_sync_event_record('sync_ids', model_name_, 'delete_identifiers', msg)

            r_queues.enqueue('sync_ids', model_name_, delete_etl_models, stage_, model_name_)

            msg = u'Fim: Enfilerando task para remover o modelo: %s da stage: %s' % (model_name_, stage_)
            logger.info(msg)
            create_sync_event_record('sync_ids', model_name_, 'delete_identifiers', msg)

def task_load_selected_journals(selected_uuids):
    """
        Task para processar Carga de um LISTA de UUIDs do modelo: Journal
    """
    r_queues = RQueues()
    for uuid in selected_uuids:
        r_queues.enqueue('load', 'journal', task_load_one_journal, uuid)

def task_delete_selected_collections(selected_uuids):
    """
        Task para apagar Coleções Transformadas.
        @param:
        - selected_uuids: lista de UUIDs dos documentos a serem removidos

        Se a lista `selected_uuids` for maior a SLICE_SIZE
            A lista será fatiada em listas de tamanho: SLICE_SIZE
        Se a lista `selected_uuids` for < a SLICE_SIZE
            Será feito uma delete direto no queryset
    """

    stage = 'transform'
    model = 'collection'
    model_class = TransformCollection
    get_db_connection()
    r_queues = RQueues()
    SLICE_SIZE = 1000

    if len(selected_uuids) > SLICE_SIZE:
        list_of_list_of_uuids = list(chunks(selected_uuids, SLICE_SIZE))
        for list_of_uuids in list_of_list_of_uuids:
            uuid_as_string_list = [str(uuid) for uuid in list_of_uuids]
            r_queues.enqueue(stage, model, task_delete_selected_collections, uuid_as_string_list)
    else:
        documents_to_delete = model_class.objects.filter(uuid__in=selected_uuids)
        documents_to_delete.delete()

def task_delete_selected_collections_identifiers(selected_ids):
    """
        Task para apagar identificadores de Coleção.
        @param:
        - selected_ids: lista de pk dos documentos a serem removidos

        Se a lista `selected_ids` for maior a SLICE_SIZE
            A lista será fatiada em listas de tamanho: SLICE_SIZE
        Se a lista `selected_ids` for < a SLICE_SIZE
            Será feito uma delete direto no queryset
    """

    model_class = CollectionIdModel
    get_db_connection()
    r_queues = RQueues()
    SLICE_SIZE = 1000

    if len(selected_ids) > SLICE_SIZE:
        list_of_list_of_uuids = list(chunks(selected_ids, SLICE_SIZE))
        for list_of_uuids in list_of_list_of_uuids:
            uuid_as_string_list = [str(uuid) for uuid in list_of_uuids]
            r_queues.enqueue(
                'sync_ids', 'collection',
                task_delete_selected_collections_identifiers, uuid_as_string_list)
    else:
        documents_to_delete = model_class.objects.filter(pk__in=selected_ids)
        documents_to_delete.delete()

def task_delete_selected_diff_etl_model(stage, model_name, action, selected_uuids):
    if stage not in ETL_STAGE_LIST:
        raise ValueError(u'param stage: %s é inválido' % stage)
    if model_name not in ETL_MODEL_NAME_LIST:
        raise ValueError(u'param model_name: %s é inválido' % model_name)
    if action not in ACTION_LIST:
        raise ValueError(u'param action: %s é inválido' % action)

    get_db_connection()
    model_class = DIFF_MODEL_CLASS_BY_NAME[model_name]
    r_queues = RQueues()
    SLICE_SIZE = 1000

    if len(selected_uuids) > SLICE_SIZE:
        list_of_list_of_uuids = list(chunks(selected_uuids, SLICE_SIZE))
        for list_of_uuids in list_of_list_of_uuids:
            uuid_as_string_list = [str(uuid) for uuid in list_of_uuids]
            r_queues.enqueue(
                'sync_ids',
                model_name,
                task_delete_selected_diff_etl_model,
                stage, model_name, action, uuid_as_string_list)  # args da task
    else:
        documents_to_delete = model_class.objects.filter(uuid__in=selected_uuids)
        documents_to_delete.delete()

def task_load_selected_news(selected_uuids):
    """
        Task para processar Carga de um LISTA de UUIDs do modelo: News
    """
    r_queues = RQueues()
    for uuid in selected_uuids:
        r_queues.enqueue('load', 'news', task_load_one_news, uuid)

def task_clean_diff_models(stage='all', model_name='all', action='all'):
    """
    task que enfilera funções para remover modelos Diffs.
    Param:
    - `stage`: filtro do campo stage: "extract", "transform" ou "load"
    - `model_name`: modelo a ser removido: "collection", "journal", etc.
    - `action` filtro do campo: "add" | "update" | "delete"
    """
    r_queues = RQueues()

    if model_name == 'all':
        model_name_list = DIFF_MODEL_CLASS.keys()
    else:
        model_name_list = [model_name]

    if stage == 'all':
        stages_list = STAGE_LIST
    else:
        stages_list = [stage, ]

    if action == 'all':
        actions_list = ACTION_LIST
    else:
        actions_list = [action, ]

    for model_ in model_name_list:
        for stage_ in stages_list:
            for action_ in actions_list:
                msg = u'Enfilerando task para remover o diff model, modelo: %s, stage: %s, action: %s' % (model_, stage_, action_)
                logger.info(msg)
                create_sync_event_record('sync_ids', model_, 'delete_diff_models', msg)
                r_queues.enqueue('sync_ids', model_, delete_diff_models, stage_, model_, action_)
                msg = u'Fim: Enfilerando task para remover o diff model, modelo: %s, stage: %s, action: %s' % (model_, stage_, action_)
                logger.info(msg)
                create_sync_event_record('sync_ids', model_, 'delete_diff_models', msg)

def enqueue_full_populate_task_by_model(model_name='all'):
    logger.info("Inicinado: enqueue_full_populate_task_by_model para modelo: %s", model_name)
    create_sync_event_record(
        'sync_ids', model_name, 'enqueue_full_populate_task_by_model',
        u'Inciando enfileramento para preencher datas dos registros diff model: %s' % model_name)

    # setup
    get_db_connection()
    stage = 'sync_ids'
    r_queues = RQueues()
    model_class = None
    task_fn = None

    options = {
        'collection': {
            'model_class': models.ExtractCollection,
            'task_fn': task_populate_collections
        },
        'journal': {
            'model_class': models.ExtractJournal,
            'task_fn': task_populate_journals
        },
        'issue': {
            'model_class': models.ExtractIssue,
            'task_fn': task_populate_issues
        },
        'article': {
            'model_class': models.ExtractArticle,
            'task_fn': task_populate_articles
        },
        'news': {
            'model_class': models.ExtractNews,
            'task_fn': task_populate_news
        },
        'press_release': {
            'model_class': models.ExtractPressRelease,
            'task_fn': task_populate_press_release
        }
    }

    if model_name == 'all':
        for k, v in options.items():
            model_class = v['model_class']
            task_fn = v['task_fn']
            logger.info("enfilerando stage: %s model_name: %s model_class: %s" % (stage, k, model_class))
            r_queues.enqueue(stage, k, task_fn)
    elif model_name not in options.keys():
        raise ValueError('Param: model_name: %s inesperado' % model_name)
    else:
        model_class = options[model_name]['model_class']
        task_fn = options[model_name]['task_fn']
        logger.info("enfilerando stage: %s model_name: %s model_class: %s" % (stage, model_name, model_class))
        r_queues.enqueue(stage, model_name, task_fn)

    create_sync_event_record(
        'sync_ids', model_name, 'enqueue_full_populate_task_by_model',
        u'Fim do enfileramento para preencher datas dos registros diff model: %s' % model_name)

    logger.info("Fim: enqueue_full_populate_task_by_model para modelo: %s", model_name)

def task_load_selected_collections(selected_uuids):
    """
        Task para processar Carga de um LISTA de UUIDs do modelo: Collection
    """

    r_queues = RQueues()
    for uuid in selected_uuids:
        r_queues.enqueue('load', 'collection', task_load_one_collection, uuid)

def task_collection_create():
    get_db_connection()
    stage = 'load'
    model = 'collection'
    r_queues = RQueues()

    for collection in models.TransformCollection.objects.all():
        r_queues.enqueue(stage, model, task_load_collection, collection.uuid)

def task_journal_create():
    get_db_connection()
    stage = 'load'
    model = 'journal'
    r_queues = RQueues()

    for journal in models.TransformJournal.objects.all():
        r_queues.enqueue(
            stage, model,
            task_load_journal, uuid=journal.uuid)