Python filecacher.FileCacher类代码示例

def clean_files(session, dry_run):
    filecacher = FileCacher()
    files = set(file[0] for file in filecacher.list())
    logger.info("A total number of %d files are present in the file store",
                len(files))
    for cls in [Attachment, Executable, File, Manager, PrintJob,
                Statement, Testcase, UserTest, UserTestExecutable,
                UserTestFile, UserTestManager, UserTestResult]:
        for col in ["input", "output", "digest"]:
            if hasattr(cls, col):
                found_digests = set()
                digests = session.query(cls).all()
                digests = [getattr(obj, col) for obj in digests]
                found_digests |= set(digests)
                found_digests.discard(FileCacher.TOMBSTONE_DIGEST)
                logger.info("Found %d digests while scanning %s.%s",
                            len(found_digests), cls.__name__, col)
                files -= found_digests
    logger.info("%d digests are orphan.", len(files))
    total_size = 0
    for orphan in files:
        total_size += filecacher.get_size(orphan)
    logger.info("Orphan files take %s bytes of disk space",
                "{:,}".format(total_size))
    if not dry_run:
        for count, orphan in enumerate(files):
            filecacher.delete(orphan)
            if count % 100 == 0:
                logger.info("%d files deleted from the file store", count)
        logger.info("All orphan files have been deleted")

    def __init__(self, shard):
        logger.initialize(ServiceCoord("Worker", shard))
        Service.__init__(self, shard, custom_logger=logger)
        self.file_cacher = FileCacher(self)

        self.work_lock = gevent.coros.RLock()
        self._ignore_job = False

    def __init__(self, contest_ids, export_target,
                 dump_files, dump_model, skip_generated,
                 skip_submissions, skip_user_tests):
        if contest_ids is None:
            with SessionGen() as session:
                contests = session.query(Contest).all()
                self.contests_ids = [contest.id for contest in contests]
                users = session.query(User).all()
                self.users_ids = [user.id for user in users]
                tasks = session.query(Task)\
                    .filter(Task.contest_id.is_(None)).all()
                self.tasks_ids = [task.id for task in tasks]
        else:
            # FIXME: this is ATM broken, because if you export a contest, you
            # then export the users who participated in it and then all of the
            # contests those users participated in.
            self.contests_ids = contest_ids
            self.users_ids = []
            self.tasks_ids = []
        self.dump_files = dump_files
        self.dump_model = dump_model
        self.skip_generated = skip_generated
        self.skip_submissions = skip_submissions
        self.skip_user_tests = skip_user_tests
        self.export_target = export_target

        # If target is not provided, we use the contest's name.
        if export_target == "":
            self.export_target = "dump_%s.tar.gz" % date.today().isoformat()
            logger.warning("export_target not given, using \"%s\"",
                           self.export_target)

        self.file_cacher = FileCacher()

    def __init__(self, contest_id, spool_dir):
        self.contest_id = contest_id
        self.spool_dir = spool_dir
        self.upload_dir = os.path.join(self.spool_dir, "upload")
        self.contest = None
        self.submissions = None

        self.file_cacher = FileCacher()

def extract_complexity(task_id, file_lengther=None):
    """Extract the complexity of all submissions of the task. The
    results are stored in a file task_<id>.info

    task_id (int): the id of the task we are interested in.
    file_lengther (type): a File-like object that tell the dimension
        of the input (see example above for how to write one).

    return (int): 0 if operation was successful.

    """
    with SessionGen() as session:
        task = Task.get_from_id(task_id, session)
        if task is None:
            return -1

        # Extracting the length of the testcase.
        file_cacher = FileCacher()
        testcases_lengths = [file_length(testcase.input,
                                         file_cacher, file_lengther)
                             for testcase in task.testcases]
        file_cacher.purge_cache()

        # Compute the complexity of the solutions.
        with open("task_%s.info" % task_id, "wt") as info:
            for submission in task.contest.get_submissions():
                if submission.task_id == task_id and \
                       submission.evaluated():
                    print submission.user.username
                    result = extract_complexity_submission(testcases_lengths,
                                                           submission)
                    if result[1] is None:
                        continue
                    info.write("Submission: %s" % submission.id)
                    info.write(" - user: %15s" % submission.user.username)
                    info.write(" - task: %s" % task.name)
                    if result[0] is not None:
                        info.write(" - score: %6.2lf" % result[0])
                    info.write(" - complexity: %20s" %
                               complexity_to_string(result[1]))
                    if result[2] is not None:
                        info.write(" - confidence %5.1lf" % result[2])
                    info.write("\n")

    return 0

def add_statement(task_name, language_code, statement_file, overwrite):
    logger.info("Adding the statement(language: %s) of task %s "
                "in the database.", language_code, task_name)

    if not os.path.exists(statement_file):
        logger.error("Statement file (path: %s) does not exist.",
                     statement_file)
        return False
    if not statement_file.endswith(".pdf"):
        logger.error("Statement file should be a pdf file.")
        return False

    with SessionGen() as session:
        task = session.query(Task)\
            .filter(Task.name == task_name).first()
        if not task:
            logger.error("No task named %s", task_name)
            return False
        try:
            file_cacher = FileCacher()
            digest = file_cacher.put_file_from_path(
                statement_file,
                "Statement for task %s (lang: %s)" %
                (task_name, language_code))
        except Exception:
            logger.error("Task statement storage failed.", exc_info=True)
        arr = session.query(Statement)\
            .filter(Statement.language == language_code)\
            .filter(Statement.task == task)\
            .all()
        if arr:  # Statement already exists
            if overwrite:
                logger.info("Overwriting already existing statement.")
                session.delete(arr[0])
                session.commit()
            else:
                logger.error("A statement with given language already exists. "
                             "Not overwriting.")
                return False
        statement = Statement(language_code, digest, task=task)
        session.add(statement)
        session.commit()

    logger.info("Statement added.")
    return True

 def setUp(self):
     self.file_cacher = FileCacher()
     #self.file_cacher = FileCacher(self, path="fs-storage")
     self.cache_base_path = self.file_cacher.file_dir
     self.cache_path = None
     self.content = None
     self.fake_content = None
     self.digest = None
     self.file_obj = None

    def __init__(self, shard):
        Service.__init__(self, shard)
        self.file_cacher = FileCacher(self)

        self.work_lock = gevent.coros.RLock()
        self._last_end_time = None
        self._total_free_time = 0
        self._total_busy_time = 0
        self._number_execution = 0

def file_length(digest, file_cacher=None, file_lengther=None):
    """Compute the length of the file identified by digest.

    digest (string): the digest of the file.
    file_cacher (FileCacher): the cacher to use, or None.
    file_lengther (type): a File-like object that tell the dimension
        of the input (see example above for how to write one).

    return (int): the length of the tile.

    """
    if file_cacher is None:
        file_cacher = FileCacher()
    if file_lengther is None:
        file_lengther = FileLengther
    lengther = file_lengther()
    file_cacher.get_file_to_fobj(digest, lengther)
    return lengther.tell()

    def make(self):
        # Unset stack size limit
        resource.setrlimit(resource.RLIMIT_STACK, (resource.RLIM_INFINITY,
                                                   resource.RLIM_INFINITY))

        if not os.path.exists(os.path.join(self.odir, "contest-config.py")):
            raise Exception("Directory doesn't contain contest-config.py")
        self.wdir = os.path.join(self.odir, "build")
        if self.clean:
            shutil.rmtree(self.wdir)
        if not os.path.exists(self.wdir):
            os.mkdir(self.wdir)
        # We have to avoid copying the folder contest/build
        # or contest/task/build into contest/build.
        # For this reason, we ignore all files and directories named "build"
        # when copying recursively.
        copyrecursivelyifnecessary(self.odir, self.wdir, set(["build"]))
        self.wdir = os.path.abspath(self.wdir)
        file_cacher = FileCacher(path=os.path.join(self.wdir, ".cache"))
        try:
            with chdir(self.wdir):
                contestconfig = ContestConfig(
                    os.path.join(self.wdir, ".rules"),
                    os.path.basename(self.odir),
                    ignore_latex=self.no_latex,
                    onlytask=self.task)
                contestconfig._readconfig("contest-config.py")
                if self.task is not None and len(contestconfig.tasks) == 0:
                    raise Exception("Task {} not found".format(self.task))
                cdb = contestconfig._makecontest()
                test_udb = contestconfig._makeuser(
                    contestconfig._mytestuser.username)
                test_gdb = contestconfig._makegroup(
                    contestconfig._mytestuser.group.name, cdb)
                # We're not putting the test user on any team for testing
                # (shouldn't be needed).
                test_pdb = contestconfig._makeparticipation(
                    contestconfig._mytestuser.username, cdb,
                    test_udb, test_gdb, None)
                for t in contestconfig.tasks.values():
                    tdb = t._makedbobject(cdb, file_cacher)
                    t._make_test_submissions(test_pdb, tdb, self.local_test)
        finally:
            file_cacher.destroy_cache()

    def build(self):
        file_cacher = FileCacher(path=os.path.join(self.wdir, ".cache"))

        try:
            with chdir(self.wdir):
                contestconfig = \
                    ContestConfig(os.path.join(self.wdir, ".rules"),
                                  "hidden contest", minimal=self.minimal)
                copyifnecessary(os.path.join(contestconfig._get_ready_dir(),
                                             "contest-template.py"),
                                os.path.join(self.wdir, "c.py"))
                contestconfig._readconfig("c.py")
                contestconfig._task(
                    self.task, contestconfig.full_feedback, self.minimal)

                if not self.minimal:
                    cdb = contestconfig._makecontest()
                    test_udb = contestconfig._makeuser(
                        contestconfig._mytestuser.username)
                    test_gdb = contestconfig._makegroup(
                        contestconfig._mytestuser.group.name, cdb)
                    # We're not putting the test user on any team for testing
                    # (shouldn't be needed).
                    test_pdb = contestconfig._makeparticipation(
                        contestconfig._mytestuser.username, cdb,
                        test_udb, test_gdb, None)
                    for t in contestconfig.tasks.values():
                        tdb = t._makedbobject(cdb, file_cacher)
                        t._make_test_submissions(
                            test_pdb, tdb, self.local_test)

        finally:
            file_cacher.destroy_cache()

        primary_statements = [s for s in list(list(contestconfig.tasks.values())[
            0]._statements.values()) if s.primary]
        if len(primary_statements) == 0:
            return None
        elif len(primary_statements) == 1:
            return os.path.abspath(primary_statements[0].file_)
        else:
            raise Exception("More than one primary statement")

def clean_files(session, dry_run):
    filecacher = FileCacher()
    files = set(file[0] for file in filecacher.list())
    logger.info("A total number of %d files are present in the file store",
                len(files))
    found_digests = enumerate_files(session)
    logger.info("Found %d digests while scanning", len(found_digests))
    files -= found_digests
    logger.info("%d digests are orphan.", len(files))
    total_size = 0
    for orphan in files:
        total_size += filecacher.get_size(orphan)
    logger.info("Orphan files take %s bytes of disk space",
                "{:,}".format(total_size))
    if not dry_run:
        for count, orphan in enumerate(files):
            filecacher.delete(orphan)
            if count % 100 == 0:
                logger.info("%d files deleted from the file store", count)
        logger.info("All orphan files have been deleted")

    def __init__(self, shard):
        logger.initialize(ServiceCoord("TestFileCacher", shard))
        TestService.__init__(self, shard, custom_logger=logger)

        # Assume we store the cache in "./cache/fs-cache-TestFileCacher-0/"
        self.cache_base_path = os.path.join(config.cache_dir, "fs-cache-TestFileCacher-0")
        self.cache_path = None
        self.content = None
        self.fake_content = None
        self.digest = None
        self.file_obj = None
        self.file_cacher = FileCacher(self)

    def __init__(self, drop, import_source, load_files, load_model, skip_generated, skip_submissions, skip_user_tests):
        self.drop = drop
        self.load_files = load_files
        self.load_model = load_model
        self.skip_generated = skip_generated
        self.skip_submissions = skip_submissions
        self.skip_user_tests = skip_user_tests

        self.import_source = import_source
        self.import_dir = import_source

        self.file_cacher = FileCacher()

    def __init__(self, contest_id, export_target,
                 dump_files, dump_model, light,
                 skip_submissions, skip_user_tests):
        self.contest_id = contest_id
        self.dump_files = dump_files
        self.dump_model = dump_model
        self.light = light
        self.skip_submissions = skip_submissions
        self.skip_user_tests = skip_user_tests

        # If target is not provided, we use the contest's name.
        if export_target == "":
            with SessionGen(commit=False) as session:
                contest = Contest.get_from_id(self.contest_id, session)
                self.export_target = "dump_%s.tar.gz" % contest.name
                logger.warning("export_target not given, using \"%s\""
                               % self.export_target)
        else:
            self.export_target = export_target

        self.file_cacher = FileCacher()