Python filesystem.create_directory函数代码示例

 def store_state(self, result_dir, index=None): 
     """ Stores this node in the given directory *result_dir* """
     if self.store and self.kernel_type == 'LINEAR':
         node_dir = os.path.join(result_dir, self.__class__.__name__)
         from pySPACE.tools.filesystem import create_directory
         create_directory(node_dir)
         try:
             self.features
         except:
             if type(self.w) == FeatureVector:
                 self.features = self.w
             elif not self.w is None:
                 self.features = FeatureVector(self.w.T, self.feature_names)
             else:
                 self.features=None
         if not self.features is None:
             # This node stores the learned features
             name = "%s_sp%s.pickle" % ("features", self.current_split)
             result_file = open(os.path.join(node_dir, name), "wb")
             result_file.write(cPickle.dumps(self.features, protocol=2))
             result_file.close()
             name = "%s_sp%s.yaml" % ("features", self.current_split)
             result_file = open(os.path.join(node_dir, name), "wb")
             result_file.write(str(self.features))
             result_file.close()
             del self.features

    def __init__(
        self,
        node_chain_spec,
        parameter_setting,
        rel_dataset_dir,
        run,
        split,
        storage_format,
        result_dataset_directory,
        store_node_chain=False,
        hide_parameters=[],
    ):

        super(NodeChainProcess, self).__init__()

        self.node_chain_spec = node_chain_spec
        self.parameter_setting = parameter_setting
        self.rel_dataset_dir = rel_dataset_dir
        self.storage = pySPACE.configuration.storage
        self.run = run
        self.storage_format = storage_format
        self.result_dataset_directory = result_dataset_directory
        self.persistency_dir = os.sep.join([result_dataset_directory, "persistency_run%s" % run])
        create_directory(self.persistency_dir)
        self.store_node_chain = store_node_chain
        self.hide_parameters = hide_parameters

        # reduce_log_level for process creation
        try:
            console_log_level = (
                eval(pySPACE.configuration.console_log_level)
                if hasattr(pySPACE.configuration, "console_log_level")
                else logging.WARNING
            )
        except (AttributeError, NameError):
            console_log_level = logging.WARNING
        try:
            file_log_level = (
                eval(pySPACE.configuration.file_log_level)
                if hasattr(pySPACE.configuration, "file_log_level")
                else logging.INFO
            )
        except (AttributeError, NameError):
            file_log_level = logging.INFO

        self.min_log_level = min(console_log_level, file_log_level)
        pySPACE.configuration.min_log_level = self.min_log_level
        # Replace parameters in spec file
        #        self.node_chain_spec = replace_parameters_and_convert(
        #            self.node_chain_spec, self.parameter_setting)
        self.node_chain_spec = replace_parameters2(self.node_chain_spec, self.parameter_setting)
        # Create node chain
        self.node_chain = NodeChainFactory.flow_from_yaml(Flow_Class=BenchmarkNodeChain, flow_spec=self.node_chain_spec)

        for node in self.node_chain:
            node.current_split = split
        # Remove pseudo parameter "__PREPARE_OPERATION__"
        if "__PREPARE_OPERATION__" in self.parameter_setting:
            self.parameter_setting = copy.deepcopy(self.parameter_setting)
            self.parameter_setting.pop("__PREPARE_OPERATION__")

    def __init__(self, dataset_dir, command_template, parametrization,
                 run_number, split_number, operation_result_dir,
                 hide_parameters = []):
        
        super(WEKAFilterProcess, self).__init__()
        
        # Determine the directory in which the of the process' results
        # are stored
        result_collection_name = dataset_dir.split(os.sep)[-2]
        for parameter_name, parameter_value in parametrization.iteritems():
            # If this is a parameter that should not be hidden, then we have to
            # encode it in the result collection name 
            if not parameter_name in hide_parameters:
                result_collection_name += "{__%s__:%s}" % (parameter_name.upper(),
                                                           parameter_value)
                                                                     
        self.result_directory = os.path.join(operation_result_dir,
                                             result_collection_name)
        
        # Create directory for intermediate results if it does not exist yet
        create_directory(self.result_directory 
                              + os.sep + "data_run%s" % run_number)
                
        # Create collection
        collection = BaseDataset.load(dataset_dir)
        
        # The parametrization that is independent of the collection type 
        # and the specific weka command template that is executed
        self.params = {"dataset_name": dataset_dir.replace('/','_'),
                       "dataset_dir": dataset_dir,
                       "run_number": run_number,
                       "split_number": split_number,
                       "weka_class_path": pySPACE.configuration.weka_class_path,
                       "temp_results": self.result_directory}

        # Load the abbreviations
        abbreviations_file = open(os.path.join(pySPACE.configuration.spec_dir,
                                               'operations/weka_templates',
                                               'abbreviations.yaml'), 'r')
        self.abbreviations = yaml.load(abbreviations_file)
        # Add custom parameters for the weka command template
        for parameter_name, parameter_value in parametrization.iteritems():
            # Auto-expand abbreviations
            if parameter_value in self.abbreviations:
                parameter_value = self.abbreviations[parameter_value]
            self.params[parameter_name] = parameter_value
            
        # Build the WEKA command by repeatedly replacing all placeholders in 
        # the template 
        while True:
            instantiated_template = command_template % self.params
            if instantiated_template == command_template:
                # All placeholders replace 
                self.weka_command = instantiated_template
                break
            else:
                # We have to continue since we are not converged
                command_template = instantiated_template
        
        self.handler_class = None

 def _createProcesses(cls, processes, result_dir, data_dict, parameters,
                      metrics, top_level):
     """ Recursive function that is used to create the analysis processes
     
     Each process creates one plot for each numeric parameter, each pair of
     numeric parameters, and each nominal parameter based on the data 
     contained in the *data_dict*. The results are stored in *result_dir*.
     The method calls itself recursively for each value of each parameter.
     """
     # Create the analysis process for the given parameters and the
     # given data
     process = AnalysisProcess(result_dir, data_dict, parameters, metrics)
     processes.put(process)
             
     # If we have less than two parameters it does not make sense to
     # split further
     if len(parameters) < 2:
         if top_level == True:
             # If we have only one parameter to visualize,
             # we don't need to create any further processes,
             # and we have to finish the creating process.
             processes.put(False)
         return 
     
     # For each parameter
     for proj_parameter in parameters:
         # We split the data based on the values of this parameter
         remaining_parameters = [parameter for parameter in parameters 
                                     if parameter != proj_parameter]
         # For each value the respective projection parameter can take on
         for value in set(data_dict[proj_parameter]):
             # Project the result dict onto the rows where the respective 
             # parameter takes on the given value
             projected_dict = defaultdict(list)
             entries_added = False
             for i in range(len(data_dict[parameter])):
                 if data_dict[proj_parameter][i] == value:
                     entries_added = True
                     for column_key in data_dict.keys():
                         if column_key == proj_parameter: continue
                         projected_dict[column_key].append(data_dict[column_key][i])
             # If the projected_dict is empty we continue
             if not entries_added:
                 continue
             
             # Create result_dir and do the recursive call for the 
             # projected data 
             # Parameter is seperated via #
             proj_result_dir = result_dir + os.sep + "%s#%s" % (proj_parameter,
                                                                value)
             create_directory(proj_result_dir)
             cls._createProcesses(processes, proj_result_dir, projected_dict,
                                  remaining_parameters, metrics, False)
     if top_level == True:
         # print "last process created"
         # give executing process the sign that creation is now finished
         processes.put(False)

 def store_state(self, result_dir, index=None): 
     """ Stores the projection in the given directory *result_dir* """
     if self.store:
         node_dir = os.path.join(result_dir, self.__class__.__name__)
         create_directory(node_dir)
         name = "%s_sp%s.pickle" % ("projection", self.current_split)
         result_file = open(os.path.join(node_dir, name), "wb")
         result_file.write(cPickle.dumps(self.projection, protocol=2))
         result_file.close()

    def create(cls, operation_spec, base_result_dir=None):
        """
        A factory method that calls the responsible method
        for creating an operation of the type specified in
        the operation specification dictionary (*operation_spec*).
        """
        # Determine result directory
        result_directory = cls.get_unique_result_dir(base_result_dir)
        print("--> Results will be stored at: \n\t\t %s"%str(result_directory))
        # Check if the required directories exist 
        # and create them if necessary
        create_directory(result_directory)

        # Determine all input datasets (note: they can be specified by
        # extended syntax for the glob package)
        storage = pySPACE.configuration.storage
        if not operation_spec.has_key("input_path"):
            warnings.warn("No input path found in operation specification.")
        input_path_pattern = os.sep.join([storage,
                                          operation_spec.get("input_path", ""),
                                          "*", ""])
        input_paths = glob.glob(input_path_pattern)
        obsolete_paths=[]
        for path in input_paths:
            file_path = os.sep.join([path,"metadata.yaml"])
            if os.path.isfile(os.sep.join([path,"metadata.yaml"])):
                continue
            elif os.path.isfile(os.sep.join([path,"collection.yaml"])):
                continue # warning comes, when data is loaded
            else:
                obsolete_paths.append(path)
                warnings.warn('Folder' + str(path) + ' seems not to be a pySPACE'+
                              ' dataset (no "metadata.yaml" found)! '+
                              'Skipping this folder in operation...')
        for path in obsolete_paths:
            input_paths.remove(path)

        op_type = operation_spec["type"]
        if op_type.endswith("_operation"):
            l=len("_operation")*-1
            op_type=op_type[:l]
            operation_spec["type"] = op_type
            warnings.warn("'%s_operation' has the wrong ending. Using '%s' instead."%(op_type,op_type),DeprecationWarning)
        op_class_name = ''.join([x.title() for x in op_type.split('_')])
        op_class_name += "Operation"
        # dynamic class import: from data_mod_name import col_class_name
        try:
            op_module = __import__('pySPACE.missions.operations.%s' % op_type,
                                        fromlist=[op_class_name])
        except:
            msg = "Operation module %s is unknown. Trying to use node_chain." % (op_type)
            from pySPACE.missions.operations.node_chain import NodeChainOperation
            op_class = NodeChainOperation
        else:
            op_class = getattr(op_module,op_class_name)
        return op_class.create(operation_spec, result_directory,
                               input_paths=input_paths)

    def _get_result_dataset_dir(base_dir, input_dataset_dir,
                                   parameter_setting, hide_parameters):
        """ Determines the name of the result directory

        Determines the name of the result directory based on the
        input_dataset_dir, the node_chain_name and the parameter setting.
        """
        input_name = input_dataset_dir.strip(os.sep).split(os.sep)[-1]
        input_name = input_name.strip("{}")
        # If the input is already the result of an operation
        if input_name.count("}{") > 0:
            input_name_parts = input_name.split("}{")
            input_name = input_name_parts[0]

        # Load the input meta data
        dataset_dir = os.sep.join([pySPACE.configuration.storage,
                                                input_dataset_dir])
        dataset_md = BaseDataset.load_meta_data(dataset_dir)

        # We are going to change the parameter_setting and don't want to
        # interfere with later runs so we work on a copy
        parameter_setting = copy.deepcopy(parameter_setting)

        # Ignore pseudo parameter "__PREPARE_OPERATION__"
        if "__PREPARE_OPERATION__" in parameter_setting:
            parameter_setting.pop("__PREPARE_OPERATION__")

        # Add the input parameters meta data to the given parameter setting
        if "parameter_setting" in dataset_md:
            parameter_setting.update(dataset_md["parameter_setting"])

        # We have to remove ' characters from the parameter value since
        # Weka does ignore them
        for key, value in parameter_setting.iteritems():
            if isinstance(value, basestring) and value.count("'") > 1:
                parameter_setting[key] = eval(value)

        # Determine the result_directory name
        # String between Key and value changed from ":" to "#",
        # because ot problems in windows and with windows file servers
        parameter_str = "}{".join(("%s#%s" % (key, value))
                                        for key, value in parameter_setting.iteritems()
                                            if key not in hide_parameters)

        result_name =  "{%s}" % input_name

        if parameter_str != "":
            result_name += "{%s}" % (parameter_str)

        # Determine the path where this result will be stored
        # and create the directory if necessary
        result_dir = base_dir
        result_dir += os.sep + result_name
        create_directory(result_dir)

        return result_dir

    def store_state(self, result_dir, index=None):
        """ Stores this node in the given directory *result_dir* """
        from pySPACE.tools.filesystem import  create_directory
        node_dir = os.path.join(result_dir, self.__class__.__name__)
        create_directory(node_dir)

        result_file = open(os.path.join(node_dir, "window_definitions.txt"), "w")
        for window_def in self.window_definition:
            result_file.write(str(window_def))
        result_file.close()

 def store_state(self, result_dir, index=None):
     """ Stores this node in the given directory *result_dir*. """
     if self.store:
         node_dir = os.path.join(result_dir, self.__class__.__name__)
         create_directory(node_dir)
         # This node only stores the learned eigenvector and eigenvalues
         name = "%s_sp%s.pickle" % ("eigenmatrix", self.current_split)
         result_file = open(os.path.join(node_dir, name), "wb")
         result_file.write(cPickle.dumps((self.avg, self.v), protocol=2))
         result_file.close()

 def store_state(self, result_dir, index=None):
     """ Stores *scikit_alg* """
     if self.store:
         node_dir = os.path.join(result_dir, self.__class__.__name__)
         create_directory(node_dir)
         name = "%s_sp%s.pickle" % ("Model", self.current_split)
         result_file = open(os.path.join(node_dir, name), "wb")
         result_file.write(cPickle.dumps(self.scikit_alg, protocol=2))
         result_file.close()
     super(ScikitPredictor,self).store_state(result_dir, index)

    def store_state(self, result_dir, index=None): 
        """ Stores this node in the given directory *result_dir* """
        if self.store:
            node_dir = os.path.join(result_dir, self.__class__.__name__)
            create_directory(node_dir)

            # This node only stores which electrodes have been selected
            name = "%s_sp%s.txt" % ("electrode_selection", self.current_split)
            result_file = open(os.path.join(node_dir, name), "wi")

            result_file.write(str(self.selected_channels))
            result_file.close()

    def __init__(self, processes, operation_spec, result_directory):
        self.processes = processes
        self.operation_spec = operation_spec
        self.result_directory = result_directory

        # Check if the required directories exist
        # and create them if necessary
        create_directory(self.result_directory)

        # Store the specification of this operation in the directory
        source_operation_file = open(os.sep.join([self.result_directory, "source_operation.yaml"]), "w")
        yaml.dump(self.operation_spec, source_operation_file)
        source_operation_file.close()

 def store_state(self, result_dir, index=None): 
     """ Stores transformation and feature names in the given directory *result_dir* """
     if self.store:
         node_dir = os.path.join(result_dir, self.__class__.__name__)
         # self.__class__.__name__)
         create_directory(node_dir)
         name = "%s_sp%s.pickle" % ("FN", self.current_split)
         result_file = open(os.path.join(node_dir, name), "wb")
         result_file.write(cPickle.dumps((self.translation, 
                                          self.mult, 
                                          self.feature_names), protocol=2))
         result_file.close()
     super(FeatureNormalizationNode,self).store_state(result_dir)

 def store_state(self, result_dir, index=None):
     """ Store this node in the given directory *result_dir* """
     # ..todo ::  mapping of flow_id and parameterization?!
     if self.store:
         for node in self.flow:
             node.store_state(result_dir, index)
         class_dir = os.path.join(result_dir, self.__class__.__name__)
         create_directory(class_dir)
         # Store the search history
         name = "search_history_sp%d.pickle" % self.current_split
         result_file = open(os.path.join(class_dir, name), "wb")
         result_file.write(cPickle.dumps(self.search_history, 
                                         protocol=cPickle.HIGHEST_PROTOCOL))
         result_file.close()

 def store_state(self, result_dir, index=None): 
     """ Stores this node in the given directory *result_dir* """
     if self.store or self.visualize_pattern:
         node_dir = os.path.join(result_dir, self.__class__.__name__)
         create_directory(node_dir)
     if self.store:
         # This node only stores the learned CSP patterns
         name = "%s_sp%s.pickle" % ("patterns", self.current_split)
         result_file = open(os.path.join(node_dir, name), "wb")
         result_file.write(cPickle.dumps(self.filters, 
                                         protocol=cPickle.HIGHEST_PROTOCOL))
         result_file.close()
         # Store spatial filter plots if desired
     if self.visualize_pattern:
         CSPNode._store_spatial_filter_plots(self.filters,
                                             self.channel_names,
                                             node_dir)