Python nemoa.log函数代码示例

    def csvGetData(self, name):
        conf    = self.cfg['table'][name]['source']
        file    = conf['file']
        delim   = conf['delimiter'] if 'delimiter' in conf \
            else nemoa.common.csvGetDelimiter(file)
        cols    = conf['usecols']
        names   = tuple(self.getColLabels())
        formats = tuple(['<f8' for x in names])
        if not 'rows' in conf or conf['rows']:
            cols = (0,) + cols
            names = ('label',) + names
            formats = ('<U12',) + formats
        dtype = {'names': names, 'formats': formats}

        nemoa.log("import data from csv file: " + file)

        try:
            #data = numpy.genfromtxt(file, skiprows = 1, delimiter = delim,
                #usecols = cols, dtype = dtype)
            data = numpy.loadtxt(file, skiprows = 1, delimiter = delim,
                usecols = cols, dtype = dtype)
        except:
            nemoa.log('error', 'could not import data from file!')
            return None

        return data

    def loadProject(self, project):
        """Import configuration files from user project."""

        nemoa.log('import private resources')
        nemoa.setLog(indent = '+1')

        # check if project exists
        if not project in self.__listUserWorkspaces():
            nemoa.log('warning', """
                could not open project '%s':
                project folder could not be found in '%s'!
                """ % (project, self.__basepath['user']))
            return False
        
        # set project
        self.__project = project

        # update paths
        self.__updatePaths(base = 'user')

        # update path of cache
        self.__updateCachePaths()

        # update logger to current logfile
        nemoa.initLogger(logfile = self.__path['logfile'])

        # import object configurations for current project
        self.__scanForConfigFiles()
        self.__scanForScripts()
        self.__scanForNetworks()

        nemoa.setLog(indent = '-1')
        return True

    def __importModelFromFile(self, file):
        """Return new model instance and set configuration and parameters from file."""

        nemoa.log('import model from file')
        nemoa.setLog(indent = '+1')

        # check file
        if not os.path.exists(file):
            if os.path.exists(
                nemoa.workspace.path('models') + file + '.mp'):
                file = nemoa.workspace.path('models') + file + '.mp'
            else: return nemoa.log('error', """
                could not load model '%s':
                file does not exist.""" % file)

        # load model parameters and configuration from file
        nemoa.log('load model: \'%s\'' % file)
        modelDict = nemoa.common.dictFromFile(file)

        model = self.__getModelInstance(
            name    = modelDict['config']['name'],
            config  = modelDict['config'],
            dataset = modelDict['dataset']['cfg'],
            network = modelDict['network']['cfg'],
            system  = modelDict['system']['config'])

        if nemoa.type.isModel(model): model._set(modelDict)
        else: return None 

        nemoa.setLog(indent = '-1')
        return model

    def _isNetworkMLPCompatible(self, network=None):
        """Check if a network is compatible to multilayer perceptrons.

        Keyword Arguments:
            network -- nemoa network instance

        Description:
            Return True if the following conditions are satisfied:
            (1) The network contains at least three layers
            (2) All layers of the network are not empty
            (3) The first and last layer of the network are visible,
                all middle layers of the network are hidden"""
        if not nemoa.type.isNetwork(network):
            return nemoa.log("error", "could not test network: invalid network instance given!")
        if len(network.layers()) < 3:
            return nemoa.log("error", "Multilayer networks need at least three layers!")
        for layer in network.layers():
            if not len(network.layer(layer)["nodes"]) > 0:
                return nemoa.log("error", "Feedforward networks do not allow empty layers!")
            if not network.layer(layer)["visible"] == (layer in [network.layers()[0], network.layers()[-1]]):
                return nemoa.log(
                    "error",
                    """The first and the last layer
                    of a multilayer feedforward network have to be visible,
                    middle layers have to be hidden!""",
                )
        return True

    def _isNetworkDBNCompatible(self, network=None):
        """Check if a network is compatible to deep beliefe networks.

        Keyword Arguments:
            network -- nemoa network instance

        Description:
            Return True if the following conditions are satisfied:
            (1) The network is MPL compatible
            (2) The network contains an odd number of layers
            (3) The hidden layers are symmetric to the central layer
                related to their number of nodes"""
        if not nemoa.type.isNetwork(network):
            return nemoa.log("error", "could not test network: invalid network instance given!")
        if not self._isNetworkMLPCompatible(network):
            return False
        if not len(network.layers()) % 2 == 1:
            return nemoa.log("error", "DBN / Autoencoder networks expect an odd number of layers!")

        layers = network.layers()
        size = len(layers)
        for id in range(1, (size - 1) / 2):
            symmetric = len(network.layer(layers[id])["nodes"]) == len(network.layer(layers[-id - 1])["nodes"])
            if not symmetric:
                return nemoa.log(
                    "error",
                    """DBN / Autoencoder networks expect a symmetric
                number of hidden nodes, related to their central layer!""",
                )
        return True

    def stratifyData(self, algorithm = 'auto'):
        """Stratify data.

        Keyword arguments:
            algorithm -- name of algorithm used for stratification
                'none':
                    probabilities of sources are
                    number of all samples / number of samples in source
                'auto':
                    probabilities of sources are hierarchical distributed
                    as defined in the configuration
                'equal':
                    probabilities of sources are
                    1 / number of sources"""
        nemoa.log('stratify data using \'%s\'' % (algorithm))

        if algorithm.lower() in ['none']:
            allSize = 0
            for src in self.data: allSize += self.data[src]['array'].shape[0]
            for src in self.data: self.data[src]['fraction'] = \
                float(allSize) / float(self.data[src]['array'].shape[0])
            return True
        if algorithm.lower() in ['auto']: return True
        if algorithm.lower() in ['equal']:
            frac = 1.0 / float(len(self.data))
            for src in self.data: self.data[src]['fraction'] = frac
            return True
        return False

    def normalizeData(self, algorithm = 'gauss'):
        """Normalize stratified data

        Keyword arguments:
            algorithm -- name of algorithm used for data normalization
                'gauss', 'z-trans':
                    Gaussian normalization (aka z-transformation)"""

        nemoa.log('normalize data using \'%s\'' % (algorithm))

        if algorithm.lower() in ['gauss', 'z-trans']:

            # get data for calculation of mean and variance
            # for single source datasets take all data
            # for multi source datasets take a big bunch of stratified data
            if len(self.data.keys()) > 1: data = \
                self.getData(size = 1000000, output = 'recarray')
            else: data = self.getSingleSourceData(source = self.data.keys()[0])

            # iterative update sources
            # get mean and standard deviation per column (recarray)
            # and update the values
            for src in self.data:
                if self.data[src]['array'] == None: continue
                for col in self.data[src]['array'].dtype.names[1:]:
                    self.data[src]['array'][col] = \
                        (self.data[src]['array'][col] - data[col].mean()) \
                        / data[col].std()
            return True

        return False

    def _set(self, dict):
        """Set configuration, parameters and data of model from given dictionary
        return true if no error occured"""

        # get config from dict
        config = self.importConfigFromDict(dict)

        # check self
        if not nemoa.type.isDataset(self.dataset): return nemoa.log('error',
            'could not configure dataset: model does not contain dataset instance!')
        if not nemoa.type.isNetwork(self.network): return nemoa.log('error',
            'could not configure network: model does not contain network instance!')
        if not nemoa.type.isSystem(self.system): return nemoa.log('error',
            'could not configure system: model does not contain system instance!')

        self.__config = config['config'].copy()
        self.network._set(**config['network'])
        self.dataset._set(**config['dataset'])

        ## prepare
        if not 'update' in config['system']['config']:
            config['system']['config']['update'] = {'A': False}

        ## 2do system._set(...) shall create system
        ## and do something like self.configure ...

        # create system
        self.system = nemoa.system.new(
            config  = config['system']['config'].copy(),
            network = self.network,
            dataset = self.dataset )
        self.system._set(**config['system'])

        return self

    def __importConfigFile(self, file):
        """Import configuration (.ini) file."""

        # search definition file
        if os.path.isfile(file):
            configFile = file
        elif os.path.isfile(self.__basepath['workspace'] + file):
            configFile = self.__basepath['workspace'] + file
        else:
            nemoa.log('warning', "configuration file '%s' does not exist!" % (file))
            return False

        # logger info
        nemoa.log("parsing configuration file: '" + configFile + "'")
        nemoa.setLog(indent = '+1')

        # import and register objects without testing
        importer = configFileImporter(self)
        objConfList = importer.load(configFile)

        for objConf in objConfList:
            self.__addObjToStore(objConf)

        self.__check(objConfList)

        nemoa.setLog(indent = '-1')
        return True

    def getFormatedData(self, data, cols = '*', output = 'array'):
        """Return data in given format.

        Keyword Arguments:
            cols -- name of column group
                default: value '*' does not filter columns
            format"""
        
        # check columns
        if cols == '*': cols = self.getColLabels()
        elif not len(cols) == len(set(cols)): return nemoa.log('error',
            'could not retrieve data: columns are not unique!')
        elif [c for c in cols if c not in self.getColLabels()]: \
            return nemoa.log('error',
            'could not retrieve data: unknown columns!')
    
        # check format
        if isinstance(output, str): fmtTuple = (output, )
        elif isinstance(output, tuple): fmtTuple = output
        else: return nemoa.log('error',
            "could not retrieve data: invalid 'format' argument!")

        # format data
        retTuple = ()
        for fmtStr in fmtTuple:
            if fmtStr == 'array': retTuple += (
                data[cols].view('<f8').reshape(data.size, len(cols)), )
            elif fmtStr == 'recarray': retTuple += (
                data[['label'] + cols], )
            elif fmtStr == 'cols': retTuple += (
                [col.split(':')[1] for col in cols], )
            elif fmtStr in ['rows', 'list']: retTuple += (
                data['label'].tolist(), )
        if isinstance(output, str): return retTuple[0]
        return retTuple

    def __confDataset(self, dataset = None, network = None, **kwargs):
        """Configure model.dataset to given dataset and network.

        Keyword Arguments:
            dataset -- dataset instance
            network -- network instance
        """
        dataset = self.dataset
        network = self.network

        # link dataset instance
        if nemoa.type.isDataset(dataset):
            self.dataset = dataset

        # check if dataset instance is valid
        if not nemoa.type.isDataset(self.dataset):
            nemoa.log('error',
            'could not configure dataset: no dataset instance available!')
            return False

        # check if dataset is empty
        if self.dataset.isEmpty():
            return True

        # prepare params
        if not network and not self.network:
            nemoa.log('error',
            'could not configure dataset: no network instance available!')
            return False

        return self.dataset.configure(network = network \
            if not network == None else self.network)

    def __addObjToStore(self, objConf):
        """link object configuration to object dictionary."""
        if not isinstance(objConf, dict):
            return False
        
        type = objConf['class']
        name = objConf['name']
        config = objConf['config']

        nemoa.log('adding %s: %s' % (type, name))

        key = None
        objID = 0

        if not type in self.__store.keys():
            nemoa.log('error', 'could not register object \'%s\': not supported object type \'%s\'!' % (name, type))
            return False

        key = self.__getNewKey(self.__store[type], name)
        objID = self.__getObjIDByName(type, key)

        # add configuration to object tree
        self.__store[type][key] = config
        self.__store[type][key]['id'] = objID

        # add entry to index
        self.__index[objID] = {'type': type, 'name': key, 'project': objConf['project']}

        return objID

def convert(list, input, output = None, filter = False, quiet = True):

    genericClasses = ['number', 'string', 'float']

    if isinstance(list, (numpy.ndarray)):
        list = list.tolist()
        inputDtype = 'nparray'
    else:
        inputDtype = 'list'

    # 'input'
    if input in genericClasses:
        inputClass  = 'generic'
        inputFormat = input
    elif ':' in input:
        inputClass  = input.lower().split(':')[0].strip()
        inputFormat = input.lower().split(':')[1].strip()
    else:
        nemoa.log('warning', 'could not convert list: unknown input format "' + input + '"!')
        return None

    # 'output'
    if output in genericClasses:
        outputClass  = 'generic'
        outputFormat = output
    elif output == None:
        outputClass  = inputClass
        outputFormat = None
    elif ':' in input: 
        outputClass  = output.lower().split(':')[0].strip()
        outputFormat = output.lower().split(':')[1].strip()
    else:
        mp_warnung('could not convert list: unknown output format "' + output + '"!')
        return None

    # 'input' vs 'output'
    if inputClass != outputClass:
        nemoa.log('warning', '"' + inputClass + '" can not be converted to "' + outputClass + '"')
        return None

    # trivial cases
    if inputClass == 'generic' or inputFormat == outputFormat:
        if inputDtype == 'nparray':
            return numpy.asarray(list), numpy.asarray([])
        else:
            return list, []

    # import annotation module
    modName = inputClass.lower()
    import importlib
    module = importlib.import_module('nemoa.annotation.' + modName)
    converter = getattr(module, modName)()
    
    outList, outLost = converter.convert_list(list, inputFormat, outputFormat, filter, quiet = quiet)

    if inputDtype == 'nparray':
        return numpy.asarray(outList), numpy.asarray(outLost)

    return outList, outLost

def interactive():
    try:
        call(["ipython", "nemoa.interactive.py", "-i"])
    except:
        nemoa.log(
            "error",
            """
            could not start interactive nemoa shell:
            you have to install ipython!""",
        )

    def __init__(self):

        stdout = sys.stdout
        try:
            sys.stdout = NullDevice()
            import rpy2.robjects
            self.robjects = rpy2.robjects
            sys.stdout = stdout
        except:
            sys.stdout = stdout
            nemoa.log('error', "could not import python package rpy2!")