Python transform.transformDirToInternal函数代码示例

 def processItem(self, job, item):
     monitoringFullPath = transform.transformDirToInternal(item["monitoringPath"])
     fullPath = transform.transformDirToInternal(item["fullPath"])
     # Check if item exists in local file sytem
     if not os.path.exists(fullPath):
         job.delete()
         return
     if not self.itemToProcess.has_key(monitoringFullPath):
         self.itemToProcess[monitoringFullPath] = {}
     #############################################
     # Start processing
     #############################################
     # If the full path already in tube, check if the timestamp is updated
     if self.itemToProcess[monitoringFullPath].has_key(fullPath):
         savedItem = self.itemToProcess[monitoringFullPath][fullPath]
         if savedItem["timestamp"] == item["timestamp"]:
             # Item not updated for time out time, add it to output queue
             self.outputBeanstalk.put(job.body)
             print "output item:", item
             job.delete()
         elif savedItem["timestamp"] < item["timestamp"]:
             # Received a new notification for an path, update saved info
             self.itemToProcess[monitoringFullPath][fullPath] = item
             job.release(priority=beanstalkc.DEFAULT_PRIORITY, delay=gItemDelayTime)
             print "item updated"
         else:
             job.delete()
     else:
         # New notification, add it
         self.itemToProcess[monitoringFullPath][fullPath] = item
         # print item, job, gItemDelayTime
         # priority is necessary to avoid error for requesting priority to be an int in beanstalkc
         job.release(priority=beanstalkc.DEFAULT_PRIORITY, delay=gItemDelayTime)
         print "new item added"

 def processItem(self, job, item):
     #fullPath = transform.transformDirToInternal(item["fullPath"])
     #monitoringFullPath = transform.transformDirToInternal(item["monitoringPath"])
     
     #source_dir = item["SourceDir"]
     #misc.ensureDir(source_dir)
     
     tag = item["tag"]
     
     working_dir = item["WorkingDir"]
     misc.ensureDir(transform.transformDirToInternal(working_dir))
     
     target_dir = item["TargetDir"]
     misc.ensureDir(transform.transformDirToInternal(target_dir))
     
     import wwjufsdatabase.libs.services.servicesV2 as service
     req = service.req()
     t = tagSystem.getTagSysObj(req.getDbSys())
     e = t.getObjs(unicode(tag))
     for i in e:
         print i
         source_dir = transform.transformDirToInternal(i)
         
         AutoArchiveThumb(source_dir, target_dir, working_dir)
     job.delete()
     return False
     #Return true only when the item should be kept in the tube
     return True

 def encInfoZip(self, pendingCollection):
     ############################
     # Save info for zipped files
     ############################
     logFilePath = transform.transformDirToInternal(
         fileTools.getTimestampWithFreeName(self.workingDir, '.log'))
     s = json.dumps(self.zippedFileInfo, sort_keys=True, indent=4)
     f = open(logFilePath,'w')
     f.write(s)
     f.close()
     logZipPath = logFilePath.replace(u'.log',u'.log.zip')
     logZip = zipClass.ZFile(logZipPath, 'w')
     logZip.addfile(unicode(logFilePath), os.path.basename(logFilePath))
     logZip.close()
     
     gTimeV = time.gmtime()
     yearStr = time.strftime("%Y", gTimeV)
     monthStr = time.strftime("%m", gTimeV)
     dayStr = time.strftime("%d", gTimeV)
     dateTimeDir = yearStr+"/"+monthStr+"/"+dayStr
     newEncDir = unicode(os.path.join(self.zipStorageDir, dateTimeDir))
     misc.ensureDir(newEncDir)
     targetPath = transform.transformDirToInternal(
             fileTools.getTimestampWithFreeName(newEncDir, '.enc'))
     self.encCopier.copy(logZipPath, targetPath.replace('.enc', '.encziplog'))
     
     
     ############################
     # Update state in storage state
     ############################
     self.updateZipLog(self.zippedFileInfo, pendingCollection)
     #Clean the current zipped file info
     self.zippedFileInfo = {}

def AutoArchiveThumb(source_folder = gAutoArchiveFullPath, target_dir = g_default_target_dir,
                     workingDir = gWorkingDir,taskUuid = str(uuid.uuid4())):
    inputTubeName = "collectionListTube"+taskUuid
    delayedCollectionListTubeName = "delayedCollectionListTubeName"+taskUuid
    
    #s1 = FolderScanner.FolderScanner()
    #s1.addItem({"command": "folderScanner", "fullPath":source_folder,
    #           "targetTubeName": inputTubeName,"blackList":g_ignore_file_type_list})
    target_dir = transform.transformDirToInternal(target_dir)
    source_folder = transform.transformDirToInternal(source_folder)
    
    #storage_state_collection_name = "storage_state://"+source_folder+":"+target_dir
    s1 = FolderEnumeratingService.FolderEnumeratingService()
    s1.addItem({"full_path": source_folder, "black_list":[], 
                "target_tube_name": inputTubeName})

    s2 = monitorService.monitorService()
    s2.addItem({"command": "monitorService", "fullPath":source_folder,
               "targetTubeName": inputTubeName,"blackList":g_ignore_file_type_list})
    
    s3 = tubeDelayService()
    s3.addItem({"inputTubeName":inputTubeName,
               "outputTubeName": delayedCollectionListTubeName,"blackList":g_ignore_file_type_list})
    
    s4 = FolderInfoArchiveService()
    s4.addItem({"InputTubeName":delayedCollectionListTubeName, "WorkingDir":workingDir, "TargetDir": target_dir})

 def __init__(self, rootPath, fullPath, itemInfo, zipFileObj, pathInZipFile):
     folderStorage.folderStorageItem.__init__(self, rootPath, fullPath)
     self.itemInfo = itemInfo
     self.rootPath = transform.transformDirToInternal(rootPath)
     self.fullPath = transform.transformDirToInternal(fullPath)
     self.zipFileObj = zipFileObj
     self.pathInZipFile = pathInZipFile

 def __init__(self, rootDir, backupDir, syncFolderCollectionId, dbInst):
     '''
     syncFolderCollectionId is a virtual collection which contains all items with synced info
     '''
     collectionDatabase.collectionOnMongoDbBase.__init__(self, syncFolderCollectionId, dbInst.getCollectionDb())
     self.rootDir = transform.transformDirToInternal(rootDir)
     self.backupDir = transform.transformDirToInternal(backupDir)
     self.objDb = dbInst
     self.folderCollection = folderRecursiveEnumCollection.folderRecursiveEnumCollection(self.rootDir, dbInst)

    def processItem(self, job, item):
        #fullPath = transform.transformDirToInternal(item["fullPath"])
        #monitoringFullPath = transform.transformDirToInternal(item["monitoringPath"])
        
        #source_dir = item["SourceDir"]
        #misc.ensureDir(source_dir)
        
        tag = item["tag"]
        
        task_item = item
        
        if item.has_key("output_tube_name"):
            #################################
            # Adding tag processing task
            #################################
            task_item = item
            self.processing_tag_dict[tag] = item
            import wwjufsdatabase.libs.services.servicesV2 as service
            req = service.req()
            t = tagSystem.getTagSysObj(req.getDbSys())
            tagged_item_list = t.getObjs(unicode(tag))
        else:
            #################################
            # A new tag added for existing tag processing task
            #################################
            if self.processing_tag_dict.has_key(tag):
                #Task exist, add the new tagged elment for processing
                task_item = self.processing_tag_dict[tag]
                tagged_item_list = [transform.transformDirToInternal(item["url"])]
            else:
                #Not a valid item, return
                print "not a valid item or tag not have processor yet"
                job.delete()
                return False
            
            
        output_tube_name = task_item["output_tube_name"]
        
        working_dir = task_item["working_dir"]
        misc.ensureDir(transform.transformDirToInternal(working_dir))
        
        target_dir = task_item["target_dir"]
        misc.ensureDir(transform.transformDirToInternal(target_dir))
        
        
        b = beanstalkServiceBase(output_tube_name)

        for i in tagged_item_list:
            info(i)
            source_dir = transform.transformDirToInternal(i)
            
            b.addItem({"source_dir":source_dir, "working_dir": working_dir, "target_dir":target_dir})

        job.delete()
        return False

 def callback(self, pathToWatch, relativePath, changeType):
     fullPath = transform.transformDirToInternal(os.path.join(pathToWatch, relativePath))
     itemDict = {"monitoringPath": transform.transformDirToInternal(pathToWatch),
                     "fullPath": fullPath, "changeType":changeType,
                     "timestamp": time.time()}
     s = json.dumps(itemDict, sort_keys=True, indent=4)
     beanstalk = beanstalkc.Connection(host=gBeanstalkdServerHost, port=gBeanstalkdServerPort)
     beanstalk.use(self.targetTube)
     #print beanstalk.using()
     s = json.dumps(itemDict, sort_keys=True, indent=4)
     job = beanstalk.put(s)

 def __init__(self, srcRoot, storageRoot, stateStoragePath = 'd:/state.txt'):
     print "src root is:", srcRoot
     self.srcRoot = transform.transformDirToInternal(srcRoot)
     self.storageRoot = transform.transformDirToInternal(storageRoot)
     self.stateStoragePath = stateStoragePath
     try:
         f = open(self.stateStoragePath,'r')
         self.config = json.load(f)
         f.close()
     except IOError:
         self.config = {}

 def store(self, element):
     #print 'storing....'
     fullPath = transform.transformDirToInternal(element.getAbsPath())
     relPath = fullPath.replace(self.srcRoot, '')
     if (self.curArchive is None) or (self.curArchivedSize > MAX_SINGLE_ARCHIVE_SIZE):
         self.curArchiveName = transform.transformDirToInternal(
             fileTools.getTimestampWithFreeName(self.storageRoot, '.zip'))
         self.curArchive = zipClass.ZFile(self.curArchiveName, 'w')
         self.curArchivedSize = 0
     #print 'copying "%s" to "%s"'%(fullPath, relPath)
     self.curArchive.addfile(unicode(fullPath).encode('gbk'), unicode(relPath).encode('gbk'))
     self.curArchivedSize += os.stat(fullPath).st_size

 def processItem(self, job, item):
     monitoringFullPath = transform.transformDirToInternal(item['monitoringPath'])
     archiveId = gZipFolderCollectionPrefix + monitoringFullPath
     if not self.collectionInDbForMonitoringPath.has_key(monitoringFullPath):
         self.collectionInDbForMonitoringPath[monitoringFullPath] = collectionDatabase.collectionOnMongoDbBase(archiveId, self.dbInst.getCollectionDb())
         objUuid = self.dbInst.addVirtualObj({"monitoringPath": monitoringFullPath, "zippedInfoCollectionId": archiveId});
         idInCol = objUuid
         self.zippedInfoCollectionList.addObj(idInCol, objUuid)
     #Save the item in the archive collection: zippedInfoColllection://D:/tmp/
     fullPath = transform.transformDirToInternal(item["fullPath"])
     relativePath = transform.getRelativePathFromFull(fullPath, monitoringFullPath)
     if not os.path.exists(fullPath):
         job.delete()
         return False#No job release, job was deleted.
     #################################################################
     # Start process the 
     #################################################################
     if not self.collectionInDbForMonitoringPath[monitoringFullPath].exists(relativePath):
         #This item is not in the collection, so we need to extract info from this item
         newObj = self.dbInst.getFsObjFromFullPath(fullPath)
         self.collectionInDbForMonitoringPath[monitoringFullPath].addObj(relativePath, newObj["uuid"])
         zipFilePath = transform.transformDirToInternal(
             fileTools.getTimestampWithFreeName(self.workingDir, gInfoFileDecryptedExt, gInfoFilePrefix))
         self.decCopier.copy(fullPath, zipFilePath)
         for i in zippedInfo(self.workingDir).enumItems(zipFilePath):
             print '--------------------------------------------------'
             print i
             fp = open(i, 'r')
             loadedFileInfo = json.load(fp)
             print loadedFileInfo
         for i in zippedInfo(self.workingDir).enumZippedFiles(zipFilePath):
             fp = open(i, 'r')
             print 'data file extracted:', i
     '''
     else:
         #This item is not in the collection, so we need to extract info from this item
         newObj = self.dbInst.getFsObjFromFullPath(fullPath)
         self.collectionInDbForMonitoringPath[monitoringFullPath].addObj(relativePath, newObj["uuid"])
         zipFilePath = transform.transformDirToInternal(
             fileTools.getTimestampWithFreeName(self.workingDir, gInfoFileDecryptedExt, gInfoFilePrefix))
         self.decCopier.copy(fullPath, zipFilePath)
         for i in zippedInfo(self.workingDir).enumItems(zipFilePath):
             print '--------------------------------------------------'
             print i
             fp = open(i, 'r')
             loadedFileInfo = json.load(fp)
             print loadedFileInfo
         for i in zippedInfo(self.workingDir).enumZippedFiles(zipFilePath):
             fp = open(i, 'r')
             print 'data file extracted:', i
     '''
     return True#Release job

def checkDirChanges(path_to_watch, busname = BUS_NAME_NAME, interfacename = INTERFACE_NAME, objname = OBJ_NAME):
    path_to_watch = transform.transformDirToInternal(os.path.abspath (path_to_watch))
    need_to_quit = False
    print "Watching %s at %s" % (path_to_watch, time.asctime ())
    hDir = win32file.CreateFile(
        path_to_watch,
        win32con.GENERIC_READ,
        win32con.FILE_SHARE_READ|win32con.FILE_SHARE_WRITE,
        None,
        win32con.OPEN_EXISTING,
        win32con.FILE_FLAG_BACKUP_SEMANTICS,
        None
    )
    cnt = 0
    bus = dbus.SessionBus()
    proxy = bus.get_object(busname, objname)
    ###############################################
    #Scan for existing files
    ###############################################
    for i in os.walk(path_to_watch):
        print i
        for j in i[2]:
            fullPath = transform.transformDirToInternal(os.path.join(i[0], j))
            #print fullPath
            proxy.notify(path_to_watch, fullPath, "Existing", False, dbus_interface = interfacename)

    while not need_to_quit:
#            print "new watch\n"
        results = win32file.ReadDirectoryChangesW(
            hDir,
            1024*256,
            True,
            win32con.FILE_NOTIFY_CHANGE_FILE_NAME
            | win32con.FILE_NOTIFY_CHANGE_DIR_NAME
            | win32con.FILE_NOTIFY_CHANGE_ATTRIBUTES
            | win32con.FILE_NOTIFY_CHANGE_SIZE
            | win32con.FILE_NOTIFY_CHANGE_LAST_WRITE
            | win32con.FILE_NOTIFY_CHANGE_SECURITY,
            None,
            None
        )
        if not need_to_quit:
            for action, file in results:
                #full_filename = os.path.join (self.path_to_watch, file)
                #print full_filename, ACTIONS.get (action, "Unknown")
                #callback(self.path_to_watch, file, ACTIONS.get (action, "Unknown"))
                print 'filechanged called:', path_to_watch, file, ACTIONS.get (action, "Unknown")
                proxy.notify(path_to_watch, file, ACTIONS.get (action, "Unknown"), True, dbus_interface = interfacename)

def genPicThumb(local_path, dest_dir, mime_type = None):
    #If no thumbnail exist, create one
    #print '-----------------------localpath:',local_path
    basename = os.path.basename(local_path)
    #print "basename:" + basename
    
    ext = basename.split(".")[-1]
    #print ext
    #if picFormatSupported(ext):
    if picFormatSupportedV2(local_path, mime_type = None):
        #It is a jpeg file, currently no other type supported
        import Image #Using PIL lib 
        im = Image.open(local_path)
        # convert to thumbnail image
        im.thumbnail((g_default_thumb_size, g_default_thumb_size), Image.ANTIALIAS)
        # don't save if thumbnail already exists
        #Use _T as the thumb file end to indicate the end of the original firl
        thumb_path_without_ext = os.path.join(dest_dir, basename.split(".")[0]+"_T")
        import random
        while os.path.exists(thumb_path_without_ext+".jpg"):
            thumb_path_without_ext += str(random.randint(0,10))
        thumb_path = thumb_path_without_ext+'.jpg'
        #print thumb_path.encode("utf8","replace")
        if im.mode != "RGB":
            im = im.convert("RGB")
        im.save(thumb_path,  "JPEG")
        return transform.transformDirToInternal(thumb_path)
    else:
        print 'non jpeg file not supported'
        raise pictureFormatNotSupported

    def store(self, processingObj, pendingCollection):
        '''
        processingObj = {"fullPath": "D:/tmp/good.txt", "size":100}
        '''
        ncl(processingObj)
        #relaPath = transform.formatRelativePath(item.getRelaPath())
        relaPath = processingObj.getIdInCol()
        ncl('Got relaPath')
        if (pendingCollection.has_key(relaPath)) and (pendingCollection[relaPath] != processingObj["uuid"]):
            #Item exists in pending but uuid is not the same, update the uuid for the pending item
            pendingCollection[relaPath] = processingObj["uuid"]
            cl('Added to pending')
        fullPath = transform.transformDirToInternal(processingObj["fullPath"])

        #Add the file to zip
        try:
            #If there is already an item with the same name, ignore the current?
            existingElem = self.zippedFileInfo[relaPath]
            return
        except:
            pass


        if (self.fileCnt > MAX_FILE_CNT_IN_INFO_FILE):
            self.encInfoZip(pendingCollection)

        processingObj["parentEncZip"] = self.targetPath.replace(".zip", ".enc")
        self.zippedFileInfo[relaPath] = processingObj.getItemInfo()
        cl('return from store')

    def run(self):
        print 'Start scanning'
        if not os.path.isdir(self.rootFolder):
            print "not a folder"
            if filter(self.rootFolder, self.blackList):
                return
            paramDict = {"fullPath": self.rootFolder, "timestamp": os.stat(self.rootFolder)[ST_MTIME],
                             "monitoringPath": self.rootFolder}
            self.addItem(paramDict)
        else:
            for root, dirs, files in os.walk(self.rootFolder):
                #Break if quit called
                if self.quit_flag:
                    break
                #cl("remaining:", dirs)
                #Process files
                for j in dirs:
                    info(j)
                    if filter(j, self.blackList):
                        info("ignoring: ", j, "\n")
                        continue
                    
                    fullPath = transform.transformDirToInternal(os.path.join(root, j))

                    paramDict = {"fullPath": fullPath, "timestamp": os.stat(fullPath)[ST_MTIME],
                                 "monitoringPath": self.rootFolder}
                    self.addItem(paramDict)
        print "process complete, quitting thread"