Python PathDefaults.PathDefaults类代码示例

    def __init__(self, maxIter=None, iterStartTimeStamp=None): 
        outputDir = PathDefaults.getOutputDir() + "recommend/erasm/"

        if not os.path.exists(outputDir): 
            os.mkdir(outputDir)
            
        #iterStartDate is the starting date of the iterator 
        if iterStartTimeStamp != None: 
            self.iterStartTimeStamp = iterStartTimeStamp
        else: 
            self.iterStartTimeStamp = 1286229600
            
        self.timeStep = timedelta(30).total_seconds()             
                
        self.ratingFileName = outputDir + "data.npz"          
        self.userDictFileName = outputDir + "userIdDict.pkl"   
        self.groupDictFileName = outputDir + "groupIdDict.pkl" 
        self.isTrainRatingsFileName = outputDir + "is_train.npz"
    
        self.dataDir = PathDefaults.getDataDir() + "erasm/"
        self.dataFileName = self.dataDir + "groupMembers-29-11-12" 
        
        self.maxIter = maxIter 
        self.trainSplit = 4.0/5 
        
        self.processRatings()
        self.splitDataset()        
        self.loadProcessedData()

    def __init__(self, field):
        numpy.random.seed(21)        
        
        dataDir = PathDefaults.getDataDir() + "dblp/"
        self.xmlFileName = dataDir + "dblp.xml"
        self.xmlCleanFilename = dataDir + "dblpClean.xml"        

        resultsDir = PathDefaults.getDataDir() + "reputation/" + field + "/"
        self.expertsFileName = resultsDir + "experts.txt"
        self.expertMatchesFilename = resultsDir + "experts_matches.csv"
        self.trainExpertMatchesFilename = resultsDir + "experts_train_matches.csv"
        self.testExpertMatchesFilename = resultsDir + "experts_test_matches.csv"
        self.coauthorsFilename = resultsDir + "coauthors.csv"
        self.publicationsFilename = resultsDir + "publications.csv"
        
        self.stepSize = 100000
        self.numLines = 33532888
        self.publicationTypes = set(["article" , "inproceedings", "proceedings", "book", "incollection", "phdthesis", "mastersthesis", "www"])
        self.p = 0.5     
        self.matchCutoff = 0.95
        
        
        self.cleanXML()
        self.matchExperts()
        logging.warning("Now you must disambiguate the matched experts if not ready done")        

def processSimpleDataset(name, numRealisations, split, ext=".csv", delimiter=",", usecols=None, skiprows=1, converters=None):
    numpy.random.seed(21)
    dataDir = PathDefaults.getDataDir() + "modelPenalisation/regression/"
    fileName = dataDir + name + ext
    
    print("Loading data from file " + fileName)
    outputDir = PathDefaults.getDataDir() + "modelPenalisation/regression/" + name + "/"

    XY = numpy.loadtxt(fileName, delimiter=delimiter, skiprows=skiprows, usecols=usecols, converters=converters)
    X = XY[:, :-1]
    y = XY[:, -1]
    idx = Sampling.shuffleSplit(numRealisations, X.shape[0], split)
    preprocessSave(X, y, outputDir, idx)

    def testGenerateRandomGraph(self):
        egoFileName = PathDefaults.getDataDir() + "infoDiffusion/EgoData.csv"
        alterFileName = PathDefaults.getDataDir()  + "infoDiffusion/AlterData.csv"
        numVertices = 1000
        infoProb = 0.1

        
        p = 0.1
        neighbours = 10
        generator = SmallWorldGenerator(p, neighbours)
        graph = SparseGraph(VertexList(numVertices, 0))
        graph = generator.generate(graph)

        self.svmEgoSimulator.generateRandomGraph(egoFileName, alterFileName, infoProb, graph)

def saveRatingMatrix(): 
    """
    Take the coauthor graph above and make vertices indexed from 0 then save 
    as matrix market format. 
    """    
    edgeFileName = PathDefaults.getOutputDir() + "erasm/edges2.txt"
    
    logging.debug("Reading edge list")
    edges = numpy.loadtxt(edgeFileName, delimiter=",", dtype=numpy.int)
    logging.debug("Total number of edges: " + str(edges.shape[0]))
    
    vertexIdDict = {} 
    vertexIdSet = set([])
    
    i = 0 
        
    for edge in edges:
        if edge[0] not in vertexIdSet: 
            vertexIdDict[edge[0]] = i
            vertexIdSet.add(edge[0])
            i += 1 
         
        if edge[1] not in vertexIdSet: 
            vertexIdDict[edge[1]] = i 
            vertexIdSet.add(edge[1])
            i += 1 

    n = len(vertexIdDict)    
    R = scipy.sparse.lil_matrix((n, n))
    logging.debug("Creating sparse matrix")
    
    for edge in edges:
        R[vertexIdDict[edge[0]], vertexIdDict[edge[1]]] += 1 
        R[vertexIdDict[edge[1]], vertexIdDict[edge[0]]] += 1 
        
    logging.debug("Created matrix " + str(R.shape) + " with " + str(R.getnnz()) + " non zeros")    

    R = R.tocsr()    
    
    minCoauthors = 20
    logging.debug("Removing vertices with <" + str(minCoauthors) + " coauthors")
    nonzeros = R.nonzero()    
    inds = numpy.arange(nonzeros[0].shape[0])[numpy.bincount(nonzeros[0]) >= minCoauthors]
    R = R[inds, :][:, inds]
    logging.debug("Matrix has shape " + str(R.shape) + " with " + str(R.getnnz()) + " non zeros")    
        
    matrixFileName = PathDefaults.getOutputDir() + "erasm/R"
    scipy.io.mmwrite(matrixFileName, R)
    logging.debug("Wrote matrix to file " + matrixFileName)

    def __init__(self, trainXIteratorFunc, testXIteratorFunc, cmdLine=None, defaultAlgoArgs = None, dirName=""):
        """ priority for default args
         - best priority: command-line value
         - middle priority: set-by-function value
         - lower priority: class value
        """
        # Parameters to choose which methods to run
        # Obtained merging default parameters from the class with those from the user
        self.algoArgs = RecommendExpHelper.newAlgoParams(defaultAlgoArgs)
        
        #Function to return iterators to the training and test matrices  
        self.trainXIteratorFunc = trainXIteratorFunc
        self.testXIteratorFunc = testXIteratorFunc
        
        #How often to print output 
        self.logStep = 10
        
        #The max number of observations to use for model selection
        self.sampleSize = 5*10**6

        # basic resultsDir
        self.resultsDir = PathDefaults.getOutputDir() + "recommend/" + dirName + "/"

        # update algoParams from command line
        self.readAlgoParams(cmdLine)

    def testComputeIdealPenalty(self):
        dataDir = PathDefaults.getDataDir() + "modelPenalisation/toy/"
        data = numpy.load(dataDir + "toyData.npz")
        gridPoints, X, y, pdfX, pdfY1X, pdfYminus1X = data["arr_0"], data["arr_1"], data["arr_2"], data["arr_3"], data["arr_4"], data["arr_5"]

        sampleSize = 100
        trainX, trainY = X[0:sampleSize, :], y[0:sampleSize]
        testX, testY = X[sampleSize:, :], y[sampleSize:]

        #We form a test set from the grid points
        fullX = numpy.zeros((gridPoints.shape[0]**2, 2))
        for m in range(gridPoints.shape[0]):
            fullX[m*gridPoints.shape[0]:(m+1)*gridPoints.shape[0], 0] = gridPoints
            fullX[m*gridPoints.shape[0]:(m+1)*gridPoints.shape[0], 1] = gridPoints[m]

        C = 1.0
        gamma = 1.0
        args = (trainX, trainY, fullX, C, gamma, gridPoints, pdfX, pdfY1X, pdfYminus1X)
        penalty = computeIdealPenalty(args)


        #Now compute penalty using data
        args = (trainX, trainY, testX, testY, C, gamma)
        penalty2 = computeIdealPenalty2(args)

        self.assertAlmostEquals(penalty2, penalty, 2)

    def testReadFromMatFile(self):
        numExamples = 10
        dir = PathDefaults.getTempDir()
        fileName = dir + "examplesList1"
        X = rand(numExamples, 10)
        
        ml = ExamplesList(numExamples)
        ml.addDataField("X", X)
        ml.writeToMatFile(fileName)
        
        ml2 = ExamplesList.readFromMatFile(fileName)
        self.assertTrue(ml == ml2)

        Y = rand(numExamples, 20)

        ml.addDataField("Y", Y)
        ml.writeToMatFile(fileName)
        
        ml2 = ExamplesList.readFromMatFile(fileName)
        self.assertTrue(ml == ml2)
        
        Z = rand(numExamples, 50)

        ml.addDataField("Z", Z)
        ml.writeToMatFile(fileName)
        
        ml2 = ExamplesList.readFromMatFile(fileName)
        self.assertTrue(ml == ml2)

    def __init__(self, maxIter=None, iterStartTimeStamp=None):
        """
        Return a training and test set for movielens based on the time each 
        rating was made. 
        """
        self.timeStep = timedelta(30).total_seconds()

        # iterStartDate is the starting date of the iterator
        if iterStartTimeStamp != None:
            self.iterStartTimeStamp = iterStartTimeStamp
        else:
            self.iterStartTimeStamp = 789652009

        outputDir = PathDefaults.getOutputDir() + "recommend/erasm/"

        self.numRatings = 402872
        self.minContacts = 10

        if not os.path.exists(outputDir):
            os.mkdir(outputDir)

        self.ratingFileName = outputDir + "data.npz"
        self.userDictFileName = outputDir + "userIdDict.pkl"
        self.isTrainRatingsFileName = outputDir + "is_train.npz"

        self.maxIter = maxIter
        self.trainSplit = 4.0 / 5

        self.processRatings()
        self.splitDataset()
        self.loadProcessedData()

        if self.maxIter != None:
            logging.debug("Maximum number of iterations: " + str(self.maxIter))

 def testGraphFromMatFile(self):
     matFileName = PathDefaults.getDataDir() +  "infoDiffusion/EgoAlterTransmissions1000.mat"
     sGraph = EgoUtils.graphFromMatFile(matFileName)
     
     examplesList = ExamplesList.readFromMatFile(matFileName)
     numFeatures = examplesList.getDataFieldSize("X", 1)
     
     self.assertEquals(examplesList.getNumExamples(), sGraph.getNumEdges())
     self.assertEquals(examplesList.getNumExamples()*2, sGraph.getNumVertices())
     self.assertEquals(numFeatures/2+1, sGraph.getVertexList().getNumFeatures())
     
     #Every even vertex has information, odd does not 
     for i in range(0, sGraph.getNumVertices()): 
         vertex = sGraph.getVertex(i)
         
         if i%2 == 0: 
             self.assertEquals(vertex[sGraph.getVertexList().getNumFeatures()-1], 1)
         else: 
             self.assertEquals(vertex[sGraph.getVertexList().getNumFeatures()-1], 0)
             
     #Test the first few vertices are the same 
     for i in range(0, 10): 
         vertex1 = sGraph.getVertex(i*2)[0:numFeatures/2]
         vertex2 = sGraph.getVertex(i*2+1)[0:numFeatures/2]
         vertexEx1 = examplesList.getSubDataField("X", numpy.array([i])).ravel()[0:numFeatures/2]
         vertexEx2 = examplesList.getSubDataField("X", numpy.array([i])).ravel()[numFeatures/2:numFeatures]
         
         self.assertTrue((vertex1 == vertexEx1).all())
         self.assertTrue((vertex2 == vertexEx2).all())

    def testEdgeFile(self):
        """
        Figure out the problem with the edge file 
        """
        dataDir = PathDefaults.getDataDir() + "cluster/"
        edgesFilename = dataDir + "Cit-HepTh.txt"

        edges = {}
        file = open(edgesFilename, 'r')
        file.readline()
        file.readline()
        file.readline()
        file.readline()

        vertices = {}

        for line in file:
            (vertex1, sep, vertex2) = line.partition("\t")
            vertex1 = vertex1.strip()
            vertex2 = vertex2.strip()
            edges[(vertex1, vertex2)] = 0
            vertices[vertex1] = 0
            vertices[vertex2] = 0

        #It says there are 352807 edges in paper and 27770 vertices
        self.assertEquals(len(edges), 352807)
        self.assertEquals(len(vertices), 27770)

    def __init__(self, YList, X, featuresName, ages, args):
        super(MetabolomicsExpRunner, self).__init__(args=args)
        self.X = X
        self.YList = YList #The list of concentrations 
        self.featuresName = featuresName
        self.args = args
        self.ages = ages 

        self.maxDepth = 10
        self.numTrees = 10
        self.sampleSize = 1.0
        self.sampleReplace = True
        self.folds = 5
        self.resultsDir = PathDefaults.getOutputDir() + "metabolomics/"

        self.leafRankGenerators = []
        self.leafRankGenerators.append((LinearSvmGS.generate(), "SVM"))
        self.leafRankGenerators.append((SvcGS.generate(), "RBF-SVM"))
        self.leafRankGenerators.append((DecisionTree.generate(), "CART"))

        self.pcaLeafRankGenerators = [(LinearSvmPca.generate(), "LinearSVM-PCA")]

        self.funcLeafRankGenerators = []
        self.funcLeafRankGenerators.append((LinearSvmFGs.generate, "SVMF"))
        self.funcLeafRankGenerators.append((SvcFGs.generate, "RBF-SVMF"))
        self.funcLeafRankGenerators.append((DecisionTreeF.generate, "CARTF"))

        #Store all the label vectors and their missing values
        YIgf1Inds, YICortisolInds, YTestoInds = MetabolomicsUtils.createIndicatorLabels(YList)
        self.hormoneInds = [YIgf1Inds, YICortisolInds, YTestoInds]
        self.hormoneNames = MetabolomicsUtils.getLabelNames()

 def getLsos(self):
     """
     Return a function to display R memory usage
     """
     fileName = PathDefaults.getSourceDir() + "/apgl/metabolomics/R/Util.R"
     robjects.r["source"](fileName)
     return robjects.r['lsos']

 def main(argv=None):
     if argv is None:
         argv = sys.argv
     try:
         # read options
         try:
             opts, args = getopt.getopt(argv[1:], "hd:n:D", ["help", "dir=", "nb_user=", "debug"])
         except getopt.error as msg:
              raise RGUsage(msg)
         # apply options
         dir = PathDefaults.getDataDir() + "cluster/"
         nb_user = None
         log_level = logging.INFO
         for o, a in opts:
             if o in ("-h", "--help"):
                 print(__doc__)
                 return 0
             elif o in ("-d", "--dir"):
                 dir = a
             elif o in ("-n", "--nb_user"):
                 nb_user = int(a)
             elif o in ("-D", "--debug"):
                 log_level = logging.DEBUG
         logging.basicConfig(stream=sys.stdout, level=log_level, format='%(levelname)s (%(asctime)s):%(message)s')
         # process: generate data files
         BemolData.generate_data_file(dir, nb_user)
     except RGUsage as err:
         logging.error(err.msg)
         logging.error("for help use --help")
         return 2

    def testLoadParams(self):
        try:
            lmbda = 0.01

            alterRegressor = PrimalRidgeRegression(lmbda)
            egoRegressor = PrimalRidgeRegression(lmbda)
            predictor = EgoEdgeLabelPredictor(alterRegressor, egoRegressor)

            params = [0.1, 0.2]
            paramFuncs = [egoRegressor.setLambda, alterRegressor.setLambda]
            fileName = PathDefaults.getTempDir() + "tempParams.pkl"

            predictor.saveParams(params, paramFuncs, fileName)

            params2 = predictor.loadParams(fileName)

            self.assertTrue(params2[0][0] == "apgl.predictors.PrimalRidgeRegression")
            self.assertTrue(params2[0][1] == "setLambda")
            self.assertTrue(params2[0][2] == 0.1)

            self.assertTrue(params2[1][0] == "apgl.predictors.PrimalRidgeRegression")
            self.assertTrue(params2[1][1] == "setLambda")
            self.assertTrue(params2[1][2] == 0.2)
        except IOError as e:
            logging.warn(e)