Python common.FascinatorHome类代码示例

    def __createFromSelected(self):
        self.vc("log").debug("Creating package from selected...")
        packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile()
        #self.vc("log").debug("packageType = '{}'", packageType)
        #self.vc("log").debug("jsonConfigFile = '{}'", jsonConfigFile)

        # if modifying existing manifest, we already have an identifier,
        # otherwise create a new one
        manifestId = self.__getActiveManifestId()
        if manifestId is None:
            manifestHash = "%s.tfpackage" % uuid.uuid4()
        else:
            manifestHash = self.__getActiveManifestPid()

        # store the manifest file for harvesting
        packageDir = FascinatorHome.getPathFile("packages")
        packageDir.mkdirs()
        manifestFile = File(packageDir, manifestHash)
        outStream = FileOutputStream(manifestFile)
        outWriter = OutputStreamWriter(outStream, "UTF-8")
        manifest = self.__getActiveManifest()
        oldType = manifest.getType()
        if oldType is None:
            manifest.setType(packageType)
        else:
            manifest.setType(oldType)

        #self.vc("log").debug("Manifest: {}", manifest)
        outWriter.write(manifest.toString(True))
        outWriter.close()

        try:
            if manifestId is None:
                # harvest the package as an object
                username = self.vc("sessionState").get("username")
                if username is None:
                    username = "guest" # necessary?
                harvester = None
                # set up config files, and make sure they are both deployed
                workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
                configFile = self.__getFile(workflowsDir, jsonConfigFile)
                rulesFile = self.__getFile(workflowsDir, "packaging-rules.py")
                # run the harvest client with our packaging workflow config
                harvester = HarvestClient(configFile, manifestFile, username)
                harvester.start()
                manifestId = harvester.getUploadOid()
                harvester.shutdown()
            else:
                # update existing object
                object = StorageUtils.getDigitalObject(Services.getStorage(), manifestId)
                manifestStream = FileUtils.openInputStream(manifestFile)
                StorageUtils.createOrUpdatePayload(object, manifestHash, manifestStream)
                manifestStream.close()
                object.close()
        except Exception, ex:
            error = "Packager workflow failed: %s" % str(ex)
            self.vc("log").error(error, ex)
            if harvester is not None:
                harvester.shutdown()
            return '{ "status": "failed" }'

    def __createNew(self):
        self.vc("log").debug("Creating a new package...")
        packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile()
        self.vc("log").debug("packageType = '{}'", packageType)
        self.vc("log").debug("jsonConfigFile = '{}'", jsonConfigFile)

        manifestHash = "%s.tfpackage" % uuid.uuid4()
        # store the manifest file for harvesting
        packageDir = FascinatorHome.getPathFile("packages")
        packageDir.mkdirs()
        manifestFile = File(packageDir, manifestHash)
        outStream = FileOutputStream(manifestFile)
        outWriter = OutputStreamWriter(outStream, "UTF-8")

        self.vc("sessionState").set("package/active", None)
        manifest = self.__getActiveManifest()
        manifest.setType(packageType)
        metaList = list(self.vc("formData").getValues("metaList"))
        jsonObj = manifest.getJsonObject()
        for metaName in metaList:
            value = self.vc("formData").get(metaName)
            jsonObj.put(metaName, value)
        if self.vc("formData").getValues("sequencesMetaList") != None:    
            sequenceService = ApplicationContextProvider.getApplicationContext().getBean("sequenceService")
            sequencesMetaList = list(self.vc("formData").getValues("sequencesMetaList"))
            for sequenceInfo in sequencesMetaList:   
                sequenceInfoJson = JsonSimple(sequenceInfo)
                sequenceIndex = sequenceService.getSequence(sequenceInfoJson.getString(None,"sequenceName"))
                jsonObj.put(sequenceInfoJson.getString(None,"metadataName"), String.format(sequenceInfoJson.getString(None,"stringFormat"),sequenceIndex))
        self.vc("log").debug("json object created is: %r" % jsonObj)
        outWriter.write(manifest.toString(True))
        outWriter.close()
        # adding ability to set access plugin
        accessPlugin = self.vc("formData").get("access_plugin", "derby")
        if accessPlugin is not None:
            self.vc("page").authentication.set_access_plugin(accessPlugin)
        try:
            # harvest the package as an object
            username = self.vc("sessionState").get("username")
            if username is None:
                username = "guest" # necessary?
            harvester = None
            # set up config files, creating if necessary
            workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
            configFile = self.__getFile(workflowsDir, jsonConfigFile)
            self.__getFile(workflowsDir, "packaging-rules.py")
            # run the harvest client with our packaging workflow config
            harvester = HarvestClient(configFile, manifestFile, username)
            harvester.start()
            manifestId = harvester.getUploadOid()
            harvester.shutdown()
        except Exception, ex:
            error = "Packager workflow failed: %s" % str(ex)
            self.vc("log").error(error, ex)
            if harvester is not None:
                harvester.shutdown()
            return '{ "status": "failed" }'

    def __ingestJson(self, fileName, jsonObject, move):
        if self.configFile is None:
            self.configFile = FascinatorHome.getPathFile("harvest/workflows/dataset.json")

        harvester = None
        try:
            ## Cache the file out to disk... although requires it
            ## .tfpackage extension due to jsonVelocity transformer
            jsonPath = self.pTemp(fileName)
            jsonFile = open(jsonPath, "wb")
            jsonFile.write(jsonObject.toString(True))
            jsonFile.close()

            ## Now instantiate a HarvestClient just for this File.
            harvester = HarvestClient(self.configFile, File(jsonPath), "guest")
            harvester.start()

            ## And cleanup afterwards
            oid = harvester.getUploadOid()
            self.log.info("Harvested alert '{}' to '{}'", fileName, oid)
            if move:
                shutil.move(self.pBase(fileName), self.pDone(fileName))
            return True

        except Exception, e:
            ## TODO: This block looks to just be a copy of the
            ##  top-level one, yet it runs per ROW, not for the
            ##  whole File. Just the JSON data should be stored

            ## Move the CSV to the 'failed' directory
            shutil.move(self.pBase(fileName), self.pFail(fileName))
            ## And write our error data to disk beside it
            self.writeError(fileName, e)
            return False

 def storeToken(self, tokenObject):
     self.resetErrors()
     index = "resumptionTokens-STORE"
     table = "resumptionTokens"
     fields = {
         "token": tokenObject.getToken(),
         "metadataPrefix": tokenObject.getMetadataPrefix(),
         "expiry": Timestamp(tokenObject.getExpiry()),
         "nextToken": tokenObject.getNextToken(),
         "resultJson": ""
     }
     FileUtils.writeStringToFile(File(FascinatorHome.getPath("oaipmh-results")+ "/"+tokenObject.getToken()),tokenObject.getResultJson())
     #self.log.debug("=== storeToken()")
     #self.log.debug("=== TOKEN: '{}'", tokenObject.getToken())
     #self.log.debug("=== METADATAPREFIX: '{}'", tokenObject.getMetadataPrefix())
     #self.log.debug("=== EXPIRY: '{}'", tokenObject.getExpiry())
     #self.log.debug("=== TOTALFOUND: '{}'", tokenObject.getTotalFound())
     #self.log.debug("=== START: '{}'", tokenObject.getStart())
     try:
         self.db.insert(self.dbName, index, table, fields)
         
     except Exception, e:
         msg = self.parseError(e)
         if msg == "Duplicate record!":
             # Expected failure
             self.log.error("Duplicate record already exists in table!")
         else:
             # Something is wrong
             self.log.error("ERROR: ", e)
         self.error = True
         self.errorMsg = msg

    def __activate__(self, context):
        self.None = context["log"]
        self.systemConfig = context["systemConfig"]
        self.sessionState = context["sessionState"]
        self.response = context["response"]
        self.request = context["request"]
        self.services = context["Services"]

        self.sessionState.set("username", "admin")
        self.writer = self.response.getPrintWriter("text/plain; charset=UTF-8")

        curationJobDao = ApplicationContextProvider.getApplicationContext().getBean("curationJobDao")
        publicationHandler = ApplicationContextProvider.getApplicationContext().getBean("publicationHandler")
        jobs = JsonSimple(File(FascinatorHome.getPath() + "/curation-status-responses/inProgressJobs.json")).getArray("inProgressJobs")
        self.writer.println(jobs.size())


        for curationJob in jobs:
            curationJob
            jobStatus = self.queryJobStatus(curationJob)
            self.writer.println(jobStatus.toString())
            status = jobStatus.getString("failed", "status")
            self.writeResponseToStatusResponseCache(jobStatus.getInteger(None, "job_id"), jobStatus)
            self.writer.println(status)
            if "complete" == status:
                publicationHandler.publishRecords(jobStatus.getArray("job_items"))
                self.updateLocalRecordRelations(jobStatus.getArray("job_items"))
           
            self.writer.close()
            self.sessionState.remove("username")

 def updateObjectMetadata(self, objectMetaData, toWorkflowId):
     packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile(toWorkflowId)
     
     workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
     configFile = File(workflowsDir, jsonConfigFile)
     configObject = StorageUtils.checkHarvestFile(self.storage, configFile);
     if configObject is None:
         oid = StorageUtils.generateOid(configFile);
         configObject = StorageUtils.getDigitalObject(self.storage, oid);
         
     objectMetaData.setProperty("jsonConfigPid", jsonConfigFile)
     objectMetaData.setProperty("jsonConfigOid", configObject.getId())
     
     configJson = JsonSimple(configFile)
     rulesFileName = configJson.getString(None, "indexer","script","rules")
     rulesFile = File(workflowsDir,rulesFileName)
     rulesObject = StorageUtils.checkHarvestFile(self.storage, rulesFile);
     if rulesObject is None:
         oid = StorageUtils.generateOid(rulesFile);
         rulesObject = StorageUtils.getDigitalObject(self.storage, oid);
     
     objectMetaData.setProperty("rulesPid", rulesFileName)
     objectMetaData.setProperty("rulesOid", rulesObject.getId())
     objectMetaData.setProperty("workflowTransitioned", "true")
     objectMetaData.setProperty("date_transitioned", time.strftime("%Y-%m-%dT%H:%M:%SZ", time.localtime()))

    def __activate__(self, context):
        self.log = context["log"]
        config = context["systemConfig"]
        response = context["response"]

        ## Variable prep
        self.redboxVersion = self.config.getString("", "redbox.version.string")
        self.defaultPath = FascinatorHome.getPath("alerts")
            
        ## This is the older-style config that allowed 1 folder for alert
        alertsPath = self.config.getString(None, ["alerts", "path"])
        if alertsPath is None:
            ## The newer config allows for alerts to come from several folders
            self.alertSet = self.config.getJsonSimpleList(defaultPath, ["alerts", "alert-set"])
            for alertItem in self.alertSet:
                self.log.info("Processing alert: {}. Log file: {}", alertItem["name"], )
                try:
                    alert = Alert(redboxVersion, alertItem)
                    alert.processAlert()
                except Exception as e:
                    #The Alert class will log this for us so continue to the next alert
                    #Some exceptions stop an alert from running at all so log them just in case
                    self.log.error("Alert [{}] encountered problems - please review the log files in the associated .processed directory. Exception was: {}", alertItem["name"], e.message)
        else:
            try:
                alertItem = self.__prepareAlertFromOldConfig(alertsPath)
                alert = Alert(redboxVersion, alertItem)
                alert.processAlert()
            except Exception as e:
                    #The Alert class will log this for us
                    pass
        return

    def processdocx(self, inputfilepath):
        """
        Convert a docx to html format, and calling
        """

        zipFile = ZipFile(inputfilepath)
        entry = zipFile.getEntry("word/document.xml")
        stream = zipFile.getInputStream(entry)
        text = StreamSource(stream)

        factory = TransformerFactoryImpl()
        xslt = StreamSource(File(join(FascinatorHome.getPath(), "lib", "xslt", "docx2html.xsl")))
        transformer = factory.newTransformer(xslt)
        tf = "/tmp/%s.html" % uuid.uuid4()
        transformer.transform(text, StreamResult(File(tf)))
        parser = DocxHtmlParser()
        parser.init()
        f = open(tf, "r")
        parser.feed(unescape(f.read()))
        f.close()
        try:
            remove(tf)
        except Exception, e:
            self.log.error("Failed to remove uploaded word file: %s." % tf)
            self.log.error(str(e))

    def __activate__(self, context):
        self.log = context["log"]
        self.config = context["systemConfig"]
        response = context["response"]

        try:
            ## Variable prep
            defaultPath = FascinatorHome.getPath("alerts")
            self.alertsPath = self.config.getString(defaultPath, ["alerts", "path"])
            self.configFile = None # We'll allocate this later... if needed
            self.redboxVersion = self.config.getString("", "redbox.version.string")
            self.csvDialect = csv.excel
            self.csvDialect.skipinitialspace = True

            ## XML Parsing
            docFactory = DocumentFactory()
            ##docFactory.setXPathNamespaceURIs(namespaces)
            self.saxReader = SAXReader(docFactory)

            ## Do our job
            (success, failed) = self.__processDir()

            ## Send response to the client (if debugging in browser)
            writer = response.getPrintWriter("text/plain; charset=UTF-8")
            writer.println("%s successful, %s failed" % (success, failed))
            writer.close()

        except Exception,e:
            response.setStatus(500)
            writer = response.getPrintWriter("text/plain; charset=UTF-8")
            writer.println("Unexpected error during script execution:\n%s" % str(e))
            writer.close()

 def __activate__(self, context):
     self.log = context["log"]
     self.request = context["request"]
     self.sessionState = context["sessionState"]
     self.sessionState.set("username","admin")
     processingSet = self.request.getParameter("processingSet")
     self.procMsg = None
     # read configuration and trigger processing stream sets
     # storing the return object on the map
     configFilePath = FascinatorHome.getPath("process")+"/processConfig.json"
     procConfigFile = File(configFilePath)
     if procConfigFile.exists() == True:
         self.dataMap = HashMap()
         self.dataMap.put("indexer", context['Services'].getIndexer())
         self.procConfigJson = JsonSimple(procConfigFile)
         for configObj in self.procConfigJson.getJsonArray():
             configJson = JsonSimple(configObj)
             procId = configJson.getString("", "id")
             if processingSet is not None: 
                 if procId == processingSet:
                     self.execProcSet(procId, configJson)
             else:
                 self.execProcSet(procId, configJson)
         if self.procMsg is None:
             self.procMsg = "Processing complete!"
     else:
         self.procMsg = "Configuration file does not exist: " + configFilePath

    def __createNew(self):
        self.vc("log").debug("Creating a new package...")
        packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile()
        #self.vc("log").debug("packageType = '{}'", packageType)
        #self.vc("log").debug("jsonConfigFile = '{}'", jsonConfigFile)

        manifestHash = "%s.tfpackage" % uuid.uuid4()
        # store the manifest file for harvesting
        packageDir = FascinatorHome.getPathFile("packages")
        packageDir.mkdirs()
        manifestFile = File(packageDir, manifestHash)
        outStream = FileOutputStream(manifestFile)
        outWriter = OutputStreamWriter(outStream, "UTF-8")

        self.vc("sessionState").set("package/active", None)
        manifest = self.__getActiveManifest()
        manifest.setType(packageType)
        metaList = list(self.vc("formData").getValues("metaList"))
        jsonObj = manifest.getJsonObject()
        for metaName in metaList:
            value = self.vc("formData").get(metaName)
            jsonObj.put(metaName, value)

        outWriter.write(manifest.toString(True))
        outWriter.close()

        try:
            # harvest the package as an object
            username = self.vc("sessionState").get("username")
            if username is None:
                username = "guest" # necessary?
            harvester = None
            # set up config files, creating if necessary
            workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
            configFile = self.__getFile(workflowsDir, jsonConfigFile)
            self.__getFile(workflowsDir, "packaging-rules.py")
            # run the harvest client with our packaging workflow config
            harvester = HarvestClient(configFile, manifestFile, username)
            harvester.start()
            manifestId = harvester.getUploadOid()
            harvester.shutdown()
        except Exception, ex:
            error = "Packager workflow failed: %s" % str(ex)
            log.error(error, ex)
            if harvester is not None:
                harvester.shutdown()
            return '{ "status": "failed" }'

    def __activate__(self, context):
        self.velocityContext = context
        self.vc("sessionState").remove("fq")

        self.__myPlans = None
        self.__sharedPlans = None
        self.__myDrafts = None
        self.__myDatasets = None
        self.__stages = JsonSimple(FascinatorHome.getPathFile("harvest/workflows/dataset.json")).getArray("stages")
     
        self.__search()

 def __activate__(self, context):
     self.velocityContext = context
     self.request = context["request"]
     self.response = context["response"]
     self.sessionState = context["sessionState"]
     self.errorMsg = ""
     
     action = self.request.getParameter("action")
     homeConfigFilePath = FascinatorHome.getPath("process")+"/notification/homeInstitutions.json"
     self.homeConfigFile = File(homeConfigFilePath)
     
     if self.homeConfigFile.exists() == False:
         self.errorMsg = "Configuration path does not exist: %s" % homeConfigFilePath
         return
     
     if action is None or action == "list":
         self.listHomes()
     else:
         self.errorMsg = "Invalid action."

    def getToken(self, tokenId):
        self.resetErrors()
        index = "resumptionTokens-GET"
        sql = """
SELECT *
FROM   resumptionTokens
WHERE  token = ?
"""
        fields = [tokenId]
        try:
            result = self.db.select(self.dbName, index, sql, fields)
            # Make sure we got a response
            if result is None or result.isEmpty():
                return None
            # Create the new token to return
            metadataPrefix = result.get(0).get("METADATAPREFIX")
            expiryStr = result.get(0).get("EXPIRY")
            # Jython does not support %f microseconds in time parsing, makes
            # this more awkward then it should be in 2.6+ Python
            # 1: split into basic time + micro seconds
            (basicTime, mSecs) = expiryStr.strip().split(".")
            # 2: Parse the basic time
            expiryDt = datetime.strptime(basicTime, "%Y-%m-%d %H:%M:%S")
            # 3: Convert into a 'epoch' long and then to a string (has an extra ".0" on the end)
            epoch = "%s" % time.mktime(expiryDt.timetuple())
            # 4: Remove the extraneous trailing zero and re-attach microseconds
            expiry = "%s%s" % (epoch.replace(".0", ""), mSecs)

            nextToken = result.get(0).get("NEXTTOKEN")
            file = File(FascinatorHome.getPath("oaipmh-results")+ "/"+tokenId)
            resultJson = FileUtils.readFileToString(file)
            FileUtils.deleteQuietly(file)
            token = ResumptionToken(tokenId, metadataPrefix,nextToken,resultJson)
            token.setExpiry(expiry)
            
            return token
        except Exception, e:
            # Something is wrong
            self.log.error("ERROR: ", e)
            self.error = True
            self.errorMsg = self.parseError(e)
            return None

 def removeToken(self, tokenObject):
     self.resetErrors()
     index = "resumptionTokens-DELETE"
     table = "resumptionTokens"
     fields = {
         "token": tokenObject.getToken()
     }
     try:
         self.db.delete(self.dbName, index, table, fields)
         file = File(FascinatorHome.getPath("oaipmh-results")+ "/"+tokenObject.getToken())
         FileUtils.deleteQuietly(file)
         self.log.info("Delete successful! TOKEN='{}'", tokenObject.getToken())
         return True
     except Exception, e:
         # Something is wrong
         self.log.error("Delete failed! TOKEN='{}'", tokenObject.getToken())
         self.log.error("ERROR: ", e)
         self.error = True
         self.errorMsg = self.parseError(e)
         return False