Python os.walk方法代码示例

# 需要导入模块: import os [as 别名]
# 或者: from os import walk [as 别名]
def validate_data_source(cls,pipeline_type):
        dirs = os.walk("{0}/pipelines/".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))).next()[1]
        is_type_ok = True if pipeline_type in dirs else False
        return is_type_ok


#class NewFileEvent(FileSystemEventHandler):
#
#    pipeline_instance = None
#    def __init__(self,pipeline_instance):
#        self.pipeline_instance = pipeline_instance
#
#    def on_moved(self,event):
#        if not event.is_directory:
#            self.pipeline_instance.new_file_detected(event.dest_path)
#
#    def on_created(self,event):
#        if not event.is_directory:
#            self.pipeline_instance.new_file_detected(event.src_path) 

# 需要导入模块: import os [as 别名]
# 或者: from os import walk [as 别名]
def validate_parameters_values(args,logger):

    logger.info("Validating input parameter values")

    #date.
    is_date_ok = True if len(args.date) == 8 else False

    # type
    dirs = os.walk(script_path).next()[1]
    is_type_ok = True if args.type in dirs else False

    #limit
    try:
        int(args.limit)
        is_limit_ok = True
    except ValueError:
        is_limit_ok = False

    if not is_date_ok: logger.error("date parameter is not correct, please validate it")
    if not is_type_ok: logger.error("type parameter is not supported, please select a valid type")
    if not is_limit_ok: logger.error("limit parameter is not correct, please select a valid limit")
    if not is_date_ok or not is_type_ok or not is_limit_ok: sys.exit(1) 

# 需要导入模块: import os [as 别名]
# 或者: from os import walk [as 别名]
def test_pep8(self):
        """Test method to check PEP8 compliance over the entire project."""
        self.file_structure = dirname(dirname(abspath(__file__)))
        print("Testing for PEP8 compliance of python files in {}".format(
            self.file_structure))
        style = pep8.StyleGuide()
        style.options.max_line_length = 100  # Set this to desired maximum line length
        filenames = []
        # Set this to desired folder location
        for root, _, files in os.walk(self.file_structure):
            python_files = [f for f in files if f.endswith(
                '.py') and "examples" not in root]
            for file in python_files:
                if len(root.split('samples')) != 2:     # Ignore samples directory
                    filename = '{0}/{1}'.format(root, file)
                    filenames.append(filename)
        check = style.check_files(filenames)
        self.assertEqual(check.total_errors, 0, 'PEP8 style errors: %d' %
                         check.total_errors) 

# 需要导入模块: import os [as 别名]
# 或者: from os import walk [as 别名]
def zip_stream(src_dir, output_stream):
        """

        @param src_dir:
        @type src_dir: str
        @param output_stream:
        @type output_stream: zipfile.ZipFile
        @return:
        """
        root_path = os.path.dirname(src_dir)
        assert os.path.isdir(src_dir), "Invalid, not a directory: '{}'".format(src_dir)
        for root, directories, files in os.walk(src_dir):
            for file_name in files:
                file_path = os.path.join(root, file_name)
                relative_path = os.path.relpath(file_path, root_path)
                output_stream.write(file_path, arcname=relative_path) 

# 需要导入模块: import os [as 别名]
# 或者: from os import walk [as 别名]
def normal_run():
    LOGGER.force_log("info", "normal selection %i started... DON'T INTERRUPT"%numberOfSteps, stdout=True, file=False)
    # load fresh engine
    engine = create_engine()
    # delete existing log files
    normalLogs = [fn for fn in next(os.walk("."))[2] if ".log" in fn and normalSelLog in fn]
    [os.remove(l) for l in normalLogs]
    # set log file name
    LOGGER.set_log_file_basename(normalSelLog)
    # set random group selector
    engine.set_group_selector(RandomSelector(engine))
    # run engine
    engine.run(numberOfSteps=numberOfSteps, saveFrequency=2*numberOfSteps, restartPdb=None)
    LOGGER.force_log("info", "normal selection finished", stdout=True, file=False)

############### run machine learning selector ############### 

# 需要导入模块: import os [as 别名]
# 或者: from os import walk [as 别名]
def ML_run():
    LOGGER.force_log("info", "machine learning selection %i started... DON'T INTERRUPT"%numberOfSteps, stdout=True, file=False)
    # load fresh engine
    engine = create_engine()
    # delete existing log files
    MLLogs = [fn for fn in next(os.walk("."))[2] if ".log" in fn and MLSelLog in fn]
    [os.remove(l) for l in MLLogs]
    # set log file name
    LOGGER.set_log_file_basename(MLSelLog)
    # set smart group selector
    engine.set_group_selector(SmartRandomSelector(engine))
    # run engine
    engine.run(numberOfSteps=numberOfSteps, saveFrequency=2*numberOfSteps, restartPdb=None)
    LOGGER.force_log("info", "machine learning selection finished", stdout=True, file=False)

##########################################################################################
#####################################  RUN SIMULATION  ################################### 

# 需要导入模块: import os [as 别名]
# 或者: from os import walk [as 别名]
def make_dataset(dir, class_to_idx):
    images = []
    dir = os.path.expanduser(dir)
    for target in sorted(os.listdir(dir)):
        d = os.path.join(dir, target)
        if not os.path.isdir(d):
            continue

        for root, _, fnames in sorted(os.walk(d)):
            for fname in sorted(fnames):
                if is_image_file(fname):
                    path = os.path.join(root, fname)
                    item = (path, class_to_idx[target])
                    images.append(item)

    return images 

# 需要导入模块: import os [as 别名]
# 或者: from os import walk [as 别名]
def find_test_path(test_file):
    """Searches for the test file and returns the path if found
    As a default, the currend working directory is the top of the search.
    If a directory was provided as part of the argument, the directory will be
    joined with cwd unless it was an absolute path, in which case, the
    absolute path will be used instead. 
    """
    test_file += ".py"
    test_path = os.path.split(test_file)
    top = os.path.join(os.getcwd(), test_path[0])

    for (path, dirs, files) in os.walk(top):
        if test_path[1] in files:
            return  os.path.join(path, test_path[1])
    raise FileNotFoundError("Could not find " + test_path[1] + 
                            "in directory: " + top) 

# 需要导入模块: import os [as 别名]
# 或者: from os import walk [as 别名]
def get_unhidden_ungenerated_python_files(directory: str) -> Iterable[str]:
    """Iterates through relevant python files within the given directory.

    Args:
        directory: The top-level directory to explore.

    Yields:
        File paths.
    """
    for dirpath, dirnames, filenames in os.walk(directory, topdown=True):
        if os.path.split(dirpath)[-1].startswith('.'):
            dirnames.clear()
            continue

        for filename in filenames:
            if filename.endswith('.py') and not filename.endswith('_pb2.py'):
                yield os.path.join(dirpath, filename) 

# 需要导入模块: import os [as 别名]
# 或者: from os import walk [as 别名]
def get_file_list_for_directory(root_path='/', file_name_regex=''):
    if root_path is None or len(root_path) == 0:
        return []

    if root_path[-1] != '/':
        if os.path.exists(root_path) and (not file_name_regex or (file_name_regex and re.match(file_name_regex, root_path))):
            return [root_path]
        else:
            return []

    file_list = []
    for path, subdirs, files in os.walk(root_path):
        for name in files:
            if not file_name_regex or (file_name_regex and re.match(file_name_regex, name)):
                file_list.append(os.path.join(path, name))
    return file_list 

# 需要导入模块: import os [as 别名]
# 或者: from os import walk [as 别名]
def remove_old_files_and_update_filesnames(filenames):
    all_directories = os.listdir(csvpath)
    # aggregate cou for collectd version < 5.5
    aggregate_cpu = False
    # remove old csv files in datadir
    remove_old_files(os.path.join(home_path, data_dir), 'csv')

    for each_dir in all_directories:
        # remove old collectd log files
        remove_old_files(os.path.join(csvpath, each_dir), None)

        if "disk" in each_dir:
            filenames[each_dir + "/disk_octets-"] = [each_dir +
                                                     '_DiskWrite', each_dir + '_DiskRead']
        if "interface" in each_dir:
            filenames[each_dir + "/if_octets-"] = [each_dir +
                                                   '_NetworkIn', each_dir + '_NetworkOut']

    for fEntry in os.walk(os.path.join(csvpath)):
        if "cpu-" in fEntry[0]:
            aggregate_cpu = True
            filenames['aggregation-cpu-average/cpu-system-'] = ['CPU']

    return aggregate_cpu 

# 需要导入模块: import os [as 别名]
# 或者: from os import walk [as 别名]
def find_image_filepaths(dataset_dir):
    """Load image filepaths from the 10k cats dataset."""
    result = []
    for root, dirs, files in os.walk(dataset_dir):
        if "/CAT_" in root:
            for name in files:
                fp = os.path.join(root, name)
                if name.endswith(".jpg") and os.path.isfile("%s.cat" % (fp,)):
                    result.append(fp)
    return result 

# 需要导入模块: import os [as 别名]
# 或者: from os import walk [as 别名]
def get_bootstrap_files(rootfs_skel_dirs, dest="cloudinit"):
    manifest = OrderedDict()
    aegea_conf = os.getenv("AEGEA_CONFIG_FILE")
    targz = io.BytesIO()
    tar = tarfile.open(mode="w:gz", fileobj=targz) if dest == "tarfile" else None

    for rootfs_skel_dir in rootfs_skel_dirs:
        if rootfs_skel_dir == "auto":
            fn = os.path.join(os.path.dirname(__file__), "..", "rootfs.skel")
        elif aegea_conf:
            # FIXME: not compatible with colon-separated AEGEA_CONFIG_FILE
            fn = os.path.join(os.path.dirname(aegea_conf), rootfs_skel_dir)
        elif os.path.exists(rootfs_skel_dir):
            fn = os.path.abspath(os.path.normpath(rootfs_skel_dir))
        else:
            raise Exception("rootfs_skel directory {} not found".format(fn))
        logger.debug("Trying rootfs.skel: %s" % fn)
        if not os.path.exists(fn):
            raise Exception("rootfs_skel directory {} not found".format(fn))
        for root, dirs, files in os.walk(fn):
            for file_ in files:
                path = os.path.join("/", os.path.relpath(root, fn), file_)
                if dest == "cloudinit":
                    add_file_to_cloudinit_manifest(os.path.join(root, file_), path, manifest)
                elif dest == "tarfile":
                    tar.add(os.path.join(root, file_), path)

    if dest == "cloudinit":
        return list(manifest.values())
    elif dest == "tarfile":
        tar.close()
        return targz.getvalue() 

# 需要导入模块: import os [as 别名]
# 或者: from os import walk [as 别名]
def rm_full_dir(path, ignore_errors=False):
    """
    This function is used to remove a directory and all files and
    directories within it (like `rm -rf`).
    """
    if os.path.isdir(path):
        try:
            os.chmod(path, os.stat(path).st_mode | stat.S_IRWXU
                                                 & ~stat.S_ISVTX)
        except OSError:
            pass
        f_last = 0
        while True:
            f_count = 0
            for root, d_names, f_names in os.walk(path):
                try:
                    os.chmod(root, os.stat(root).st_mode | stat.S_IRWXU
                                                         & ~stat.S_ISVTX)
                except OSError:
                    pass
                for fs_name in f_names + d_names:
                    target = os.path.join(root, fs_name)
                    try:
                        os.chmod(target, os.stat(target).st_mode
                                              | stat.S_IRWXU
                                              & ~stat.S_ISVTX)
                    except OSError:
                        pass
                    f_count += 1
                f_count += 1
            # do this until we get the same count twice, ie. all files we can
            # chmod our way into have been found
            if f_last == f_count:
                break
            f_last = f_count
        shutil.rmtree(path, ignore_errors) 

# 需要导入模块: import os [as 别名]
# 或者: from os import walk [as 别名]
def get_names(file_dir, files):
    """
    Get the annotator name list based on a list of files
    Args:
    file_dir: AMR file folder
    files: a list of AMR names, e.g. nw_wsj_0001_1

    Returns:
   a list of user names who annotate all the files
    """
    # for each user, check if they have files available
    # return user name list
    total_list = []
    name_list = []
    get_sub = False
    for path, subdir, dir_files in os.walk(file_dir):
        if not get_sub:
            total_list = subdir[:]
            get_sub = True
        else:
            break
    for user in total_list:
        has_file = True
        for f in files:
            file_path = file_dir + user + "/" + f + ".txt"
            if not os.path.exists(file_path):
                has_file = False
                break
        if has_file:
            name_list.append(user)
    if len(name_list) == 0:
        print("********Error: Cannot find any user who completes the files*************", file=ERROR_LOG)
    return name_list