Python redis_helper.ExpKeys类代码示例

    def remove_block(self, block):
        r = get_redis_instance()
        block_key = ExpKeys.get_block_key(block.uuid)
        pipe = r.pipeline(transaction=True)
        if block.create_new_scope:
            for sub_block_uuid in block.children_blocks:
                sub_block = self.get_block(sub_block_uuid)
                self.remove_block(sub_block)

            pipe.hdel(ExpKeys.get_scope_creating_block_uuid_keys(self.pk), block.sub_scope_name)

        block.on_remove(exp=self)

        # find all bound variables, that provided by this block
        for other_uuid, other_block in self.get_blocks(self.get_all_block_uuids()):
            if other_uuid == block.uuid:
                continue

            for f_name, bound_var in other_block.bound_inputs.items():
                if bound_var.block_uuid == block.uuid:
                    other_block.bound_inputs.pop(f_name)

            self.store_block(other_block)

        # Remove information related to block from redis
        pipe.lrem(ExpKeys.get_exp_blocks_list_key(self.pk), 0, block.uuid)
        pipe.srem(ExpKeys.get_all_exp_keys_key(self.pk), block_key)
        pipe.hdel(ExpKeys.get_blocks_uuid_by_alias(self.pk), block.base_name)

        scope = Scope(self, block.scope_name)
        scope.remove_vars_from_block(block)

        pipe.execute()

def delete_exp(exp):
    """
        We need to clean 3 areas:
            - keys in redis storage
            - uploaded and created files
            - delete exp object through ORM

        @param exp: Instance of Experiment  to be deleted
        @return: None

    """
    # redis
    r = get_redis_instance()
    all_exp_keys = ExpKeys.get_all_exp_keys_key(exp.pk)
    keys_to_delete = r.smembers(all_exp_keys)
    keys_to_delete.update(all_exp_keys)
    r.delete(keys_to_delete)

    # uploaded data
    data_files = UploadedData.objects.filter(exp=exp)
    for f in data_files:
        try:
            os.remove(f.data.path)
        except:
            pass
        f.delete()
    try:
        shutil.rmtree(exp.get_data_folder())
    except:
        pass

    # deleting an experiment
    exp.delete()

    def get_all_scopes_with_block_uuids(self, redis_instance=None):
        if redis_instance is None:
            r = get_redis_instance()
        else:
            r = redis_instance

        return r.hgetall(ExpKeys.get_scope_creating_block_uuid_keys(self.pk))

    def store(self, redis_instance=None):
        if redis_instance is None:
            r = get_redis_instance()
        else:
            r = redis_instance

        key = ExpKeys.get_scope_key(self.exp.pk, scope_name=self.name)
        r.set(key, pickle.dumps(self.scope_vars))
        log.debug("Scope `%s` was saved to storage", self.name)

    def change_block_alias(self, block, new_base_name):
        r = get_redis_instance()

        key = ExpKeys.get_blocks_uuid_by_alias(self.pk)
        pipe = r.pipeline()
        pipe.hdel(key, block.base_name)
        pipe.hset(key, new_base_name, block.uuid)
        pipe.execute()
        block.base_name = new_base_name
        self.store_block(block, redis_instance=r)

def auto_exec_task(exp, scope_name, is_init=False):
    r = get_redis_instance()

    lock_key = ExpKeys.get_auto_exec_task_lock_key(exp.pk, scope_name)
    with redis_lock.Lock(r, lock_key):
        try:
            sr = ScopeRunner(exp, scope_name)
            sr.execute(is_init)
        except Exception, e:
            log.exception(e)

    def get_meta_block_by_sub_scope(self, scope_name, redis_instance=None):
        if redis_instance is None:
            r = get_redis_instance()
        else:
            r = redis_instance

        block_uuid = r.hget(ExpKeys.get_scope_creating_block_uuid_keys(self.pk), scope_name)
        if not block_uuid:
            raise KeyError("Doesn't have a scope with name %s" % scope_name)
        else:
            return self.get_block(block_uuid, r)

def auto_exec_task(exp, scope_name, is_init=False):
    r = get_redis_instance()
    lock_key = ExpKeys.get_auto_exec_task_lock_key(exp.pk, scope_name)
    with redis_lock.Lock(r, lock_key):
        try:
            sr = ScopeRunner(exp, scope_name)
            sr.execute(is_init)
            exp.log("root", "Scope %s finished" % scope_name, "INFO")
        except Exception, e:
            exp.error()
            exp.log("root", e.message, "CRITICAL")
            log.exception(e)

    def get_all_block_uuids(self, redis_instance=None):
        """
        @param redis_instance: Redis client

        @return: list of block uuids
        """
        if redis_instance is None:
            r = get_redis_instance()
        else:
            r = redis_instance

        return r.lrange(ExpKeys.get_exp_blocks_list_key(self.pk), 0, -1) or []

    def save_file_input(self, exp, field_name, file_obj, multiple=False, upload_meta=None):
        if upload_meta is None:
            upload_meta = {}

        if not hasattr(self, field_name):
            raise Exception("Block doesn't have field: %s" % field_name)

        orig_name = file_obj.name
        local_filename = "%s_%s_%s" % (self.uuid[:8], field_name, file_obj.name)

        if not multiple:
            exp.log(self.uuid, "Storing single upload to field: %s" % field_name)
            log.debug("Storing single upload to field: %s", field_name)
            ud, is_created = UploadedData.objects.get_or_create(
                exp=exp, block_uuid=self.uuid, var_name=field_name)

            file_obj.name = local_filename
            ud.data = file_obj
            ud.save()

            ufw = UploadedFileWrapper(ud.pk)
            ufw.orig_name = orig_name
            setattr(self, field_name, ufw)
            exp.store_block(self)
        else:
            exp.log(self.uuid, "Adding upload to field: %s" % field_name)
            log.debug("Adding upload to field: %s", field_name)

            ud, is_created = UploadedData.objects.get_or_create(
                exp=exp, block_uuid=self.uuid, var_name=field_name, filename=orig_name)

            file_obj.name = local_filename
            ud.data = file_obj
            ud.filename = orig_name
            ud.save()

            ufw = UploadedFileWrapper(ud.pk)
            ufw.orig_name = orig_name

            r = get_redis_instance()
            with redis_lock.Lock(r, ExpKeys.get_block_global_lock_key(self.exp_id, self.uuid)):
                exp.log(self.uuid, "Enter lock, file: %s" % orig_name)
                log.debug("Enter lock, file: %s", orig_name)
                block = exp.get_block(self.uuid)
                attr = getattr(block, field_name)

                attr[orig_name] = ufw
                exp.log(self.uuid, "Added upload `%s` to collection: %s" % (orig_name, attr.keys()))
                log.debug("Added upload `%s` to collection: %s", orig_name, attr.keys())
                exp.store_block(block)
                exp.log(self.uuid, "Exit lock, file: %s" % orig_name)
                log.debug("Exit lock, file: %s", orig_name)

    def load(self, redis_instance=None):
        if redis_instance is None:
            r = get_redis_instance()
        else:
            r = redis_instance

        key = ExpKeys.get_scope_key(self.exp.pk, scope_name=self.name)
        raw = r.get(key)
        if raw is not None:
            self.scope_vars = pickle.loads(raw)
            # TODO: set scope name during scope_var creation
            for scope_var in self.scope_vars:
                scope_var.scope_name = self.name

    def get_ctx(self, redis_instance=None):
        if redis_instance is None:
            r = get_redis_instance()
        else:
            r = redis_instance

        key_context = ExpKeys.get_context_store_key(self.pk)
        pickled_ctx = r.get(key_context)
        if pickled_ctx is not None:
            ctx = pickle.loads(pickled_ctx)
        else:
            raise KeyError("Context wasn't found for exp_id: %s" % self.pk)
        return ctx

    def get_all_block_uuids(self, redis_instance=None):
        """
        @type included_inner_blocks: list of str
        @param included_inner_blocks: uuids of inner blocks to be included

        @param redis_instance: Redis client

        @return: list of block uuids
        """
        if redis_instance is None:
            r = get_redis_instance()
        else:
            r = redis_instance

        return r.lrange(ExpKeys.get_exp_blocks_list_key(self.pk), 0, -1) or []

    def get_block_aliases_map(self, redis_instance=None):
        """
        @param redis_instance: Redis

        @return: Map { uuid -> alias }
        @rtype: dict
        """
        if redis_instance is None:
            r = get_redis_instance()
        else:
            r = redis_instance

        orig_map = r.hgetall(ExpKeys.get_blocks_uuid_by_alias(self.pk))
        return dict([
            (uuid, alias)
            for alias, uuid in orig_map.iteritems()
        ])

    def get_block(block_uuid, redis_instance=None):
        """
            @type  block_uuid: str
            @param block_uuid: Block instance identifier

            @type  redis_instance: Redis
            @param redis_instance: Instance of redis client

            @rtype: GenericBlock
            @return: Block instance
        """
        if redis_instance is None:
            r = get_redis_instance()
        else:
            r = redis_instance

        return pickle.loads(r.get(ExpKeys.get_block_key(block_uuid)))