Python filetool.walk函数代码示例

    def scanRepository(self):
        console.info("Scanning %s" % self.dir)
        console.indent()
        demoDir = "%sdemo%s" % (os.sep, os.sep)
        manifestPaths = []

        for root, dirs, files in filetool.walk(self.dir, topdown=True):
            for dir in dirs:

                if dir == "qooxdoo" or dir == "demo":
                    dirs.remove(dir)
                    continue

                path = os.path.join(root, dir)
                manifestPath = os.path.join(path, "Manifest.json")

                if demoDir in manifestPath:
                    dirs.remove(dir)
                    continue

                if os.path.isfile(manifestPath):
                    console.debug("Found manifest: " + repr(manifestPath))
                    manifestPaths.append(manifestPath)

        console.info("Found %s manifests" % len(manifestPaths))
        console.outdent()
        return manifestPaths

 def testTwoLinksSameTarget(self):
     """
     dir1
         link1 -> ../dir2
         link2 -> ../dir2
     dir2
         foo.txt
     Should follow the symlinks and find foo.txt twice
     """
     
     self.tempDirs.append(tempfile.mkdtemp())
     symTargetFile = file(os.path.join(self.tempDirs[1], "foo.txt"), "w")
     symTarget1 = "link1"
     symTarget1Path = os.path.join(self.tempDirs[0], symTarget1)
     os.symlink(self.tempDirs[1], symTarget1Path)
     symTarget2 = "link2"
     symTarget2Path = os.path.join(self.tempDirs[0], symTarget2)
     os.symlink(self.tempDirs[1], symTarget2Path)
     
     expectedFiles = [ os.path.join(self.tempDirs[0], symTarget2, "foo.txt"),
                       os.path.join(self.tempDirs[0], symTarget1, "foo.txt") ]
     
     foundFiles = []
     for root, dirs, files in filetool.walk(self.tempDirs[0]):
         for filename in files:
             foundFiles.append(os.path.join(root, filename))
     
     self.failUnlessEqual(foundFiles, expectedFiles)

 def testSymlinkCyclic(self):
     """
     dir
         link -> ../dir
         foo.txt
     
     Should find the directories link and link/link, but not link/link/link
     Should find the files foo.txt and link/foo.txt, but not link/link/foo.txt
     """
     symTarget = "link"
     fileInSource = file(os.path.join(self.tempDirs[0], "foo.txt"), "w")
     fileInSource.close()
     symTargetPath = os.path.join(self.tempDirs[0], symTarget)
     os.symlink(self.tempDirs[0], symTargetPath)
     
     expectedDirs = [os.path.join(symTargetPath, symTarget), symTargetPath]
     expectedFiles = [ os.path.join(self.tempDirs[0], symTarget,"foo.txt"),
                       os.path.join(self.tempDirs[0], "foo.txt") ]
     
     foundDirs = []
     foundFiles = []
     for root, dirs, files in filetool.walk(self.tempDirs[0]):
         for dirname in dirs:
             foundDirs.append(os.path.join(root, dirname))
         for filename in files:
             foundFiles.append(os.path.join(root, filename))
     
     self.failUnlessEqual(foundDirs, expectedDirs)
     self.failUnlessEqual(foundFiles, expectedFiles)

    def _scanTranslationPath(self):
        translations = {}  # reset
        if self.assets['translations']['path'] is None or not os.path.isdir(
                os.path.join(self.path,self.assets['translations']['path'])):
            self._console.info("Lib<%s>: Skipping non-existing translation path" % self.namespace)
            return translations

        path = os.path.join(self.path,self.assets['translations']['path'])
        self._console.debug("Scanning translation folder...")

        # Iterate...
        for root, dirs, files in filetool.walk(path):
            # Filter ignored directories
            for ignoredDir in dirs:
                if self._ignoredDirEntries.match(ignoredDir):
                    dirs.remove(ignoredDir)

            # Searching for files
            for fileName in files:
                # Ignore non-po and dot files
                if os.path.splitext(fileName)[-1] != ".po" or fileName.startswith("."):
                    continue

                filePath = os.path.join(root, fileName)
                fileLocale = os.path.splitext(fileName)[0]

                translations[fileLocale] = self.translationEntry(fileLocale, filePath, self.namespace)

        self._console.indent()
        self._console.debug("Found %s translations" % len(translations))
        self._console.outdent()

        return translations

    def _scanResourcePath(self, path):
        if not os.path.exists(path):
            raise ValueError("The given resource path does not exist: %s" % path)

        self._console.debug("Scanning resource folder...")

        path = os.path.abspath(path)
        lib_prefix_len = len(path)
        if not path.endswith(os.sep):
            lib_prefix_len += 1

        for root, dirs, files in filetool.walk(path):
            # filter ignored directories
            for dir in dirs:
                if self._ignoredDirectories.match(dir):
                    dirs.remove(dir)

            for file in files:
                fpath = os.path.join(root, file)
                fpath = os.path.normpath(fpath)
                if Image.isImage(fpath):
                    if CombinedImage.isCombinedImage(fpath):
                        res = CombinedImage(fpath)
                    else:
                        res = Image(fpath)
                    res.analyzeImage()
                else:
                    res = Resource(fpath)
                
                res.id = Path.posifyPath(fpath[lib_prefix_len:])
                res.library= self

                self.resources.add(res)

        return

 def testSymlinkMulti(self):
     self.tempDirs.append(tempfile.mkdtemp())
     symTarget = os.path.basename(self.tempDirs[1])
     symTargetFile = file(os.path.join(self.tempDirs[1], "foo.txt"), "w") 
     symTargetPath = os.path.join(self.tempDirs[0], symTarget)
     os.symlink(self.tempDirs[1], symTargetPath)
     
     found1 = []
     for root, dirs, files in filetool.walk(self.tempDirs[0]):
         for filename in files:
             found1.append(filename)
     
     found2 = []
     for root, dirs, files in filetool.walk(self.tempDirs[0]):
         for filename in files:
             found2.append(filename)
     
     self.failUnlessEqual(found1, found2)

 def testWalkFiles(self):
     fooPath = os.path.join(self.tempDirs[0], "foo.txt")
     foo = file(fooPath, "w")
     foo.close()
     barPath = os.path.join(self.tempDirs[0], "bar.txt")
     bar = file(barPath, "w")
     bar.close()
     
     found = []
     for root, dirs, files in filetool.walk(self.tempDirs[0]):
         for filename in files:
             found.append(filename)
     
     self.failUnlessEqual(found, ["foo.txt", "bar.txt"], "filetool.walk returned unexpected result %s" %repr(found))    

 def _getDemos(self):
   if not self.hasDemoDir:
     return False
   
   demos = {}
   demoPath = os.path.join(self.path, "demo")
   for root, dirs, files in filetool.walk(demoPath, topdown=True):
     for name in dirs:        
       if root == demoPath and name[0] != "." and name != "source" and name != "build":
         foundPath = os.path.join(demoPath, name)
         foundDemo = Demo(name, foundPath, self)
         demos[name] = foundDemo
   
   return demos

    def _getDemos(self):
        if not self.hasDemoDir:
            return False

        demos = {}
        demoPath = os.path.join(self.path, "demo")
        for root, dirs, files in filetool.walk(demoPath, topdown=True):
            for name in dirs:
                path = os.path.join(root, name)
                manifestPath = os.path.join(path, "Manifest.json")
                if os.path.isfile(manifestPath):
                    foundDemo = Demo(name, path, self)
                    demos[name] = foundDemo

        return demos

    def _scanResourcePath(self):
        resources = set()
        if self.resourcePath is None:
            return resources
        if not os.path.isdir(os.path.join(self.path,self.resourcePath)):
            self._console.info("Lib<%s>: Skipping non-existing resource path" % self.namespace)
            return resources

        path = os.path.join(self.path,self.resourcePath)
        self._console.debug("Scanning resource folder...")

        path = os.path.abspath(path)
        lib_prefix_len = len(path)
        if not path.endswith(os.sep):
            lib_prefix_len += 1

        for root, dirs, files in filetool.walk(path):
            # filter ignored directories
            for dir in dirs:
                if self._ignoredDirEntries.match(dir):
                    dirs.remove(dir)

            for file in files:
                if self._ignoredDirEntries.match(file):
                    continue
                fpath = os.path.join(root, file)
                fpath = os.path.normpath(fpath)
                if Image.isImage(fpath):
                    if CombinedImage.isCombinedImage(fpath):
                        res = CombinedImage(fpath)
                    else:
                        res = Image(fpath)
                    res.analyzeImage()
                elif FontMap.isFontMap(fpath):
                    res = FontMap(fpath)
                else:
                    res = Resource(fpath)

                res.set_id(Path.posifyPath(fpath[lib_prefix_len:]))
                res.library= self

                resources.add(res)

        self._console.indent()
        self._console.debug("Found %s resources" % len(resources))
        self._console.outdent()
        return resources

    def classPathIterator(self):
        if self.classPath is None:
            return
        classRoot = os.path.join(self.path, self.classPath)
        for root, dirs, files in filetool.walk(classRoot):
            # Filter ignored directories
            for ignoredDir in dirs:
                if self._ignoredDirEntries.match(ignoredDir):
                    dirs.remove(ignoredDir)

            # Searching for files
            for fileName in files:
                # ignore dot files
                if fileName.startswith(".") or self._ignoredDirEntries.match(fileName):
                    continue
                filePath = os.path.join(root, fileName)
                filePathId = filePath.replace(classRoot + os.sep, '')
                yield (filePathId, filePath)

    def _scanResourcePath(self, path):
        if not os.path.exists(path):
            raise ValueError("The given resource path does not exist: %s" % path)

        self._console.debug("Scanning resource folder...")

        for root, dirs, files in filetool.walk(path):
            # filter ignored directories
            for dir in dirs:
                if self._ignoredDirectories.match(dir):
                    dirs.remove(dir)

            for file in files:
                fpath = os.path.join(root, file)
                #self._resources.append(fpath)  # avoiding this currently, as it is not used
                if CombinedImage.isCombinedImage(fpath):
                    self.resources.combImages.add(os.path.normpath(fpath))

        return

 def testSymlink(self):
     """
     dir1
         dir2 -> ../dir2
     dir2
         foo.txt
     Should follow the symlink and find foo.txt
     """
     
     self.tempDirs.append(tempfile.mkdtemp())
     symTarget = os.path.basename(self.tempDirs[1])
     symTargetFile = file(os.path.join(self.tempDirs[1], "foo.txt"), "w") 
     symTargetPath = os.path.join(self.tempDirs[0], symTarget)
     os.symlink(self.tempDirs[1], symTargetPath)
     
     
     found = []
     for root, dirs, files in filetool.walk(self.tempDirs[0]):
         for filename in files:
             found.append(filename)
     
     self.failUnlessEqual(found, ["foo.txt"])

    def _scanClassPath(self, path, uri, encoding):
        if not os.path.exists(path):
            raise ValueError("The given class path does not exist: %s" % path)

        self._console.debug("Scanning class folder...")

        # Iterate...
        for root, dirs, files in filetool.walk(path):
            # Filter ignored directories
            for ignoredDir in dirs:
                if self._ignoredDirectories.match(ignoredDir):
                    dirs.remove(ignoredDir)

            # Searching for files
            for fileName in files:
                # Ignore dot files
                if fileName.startswith("."):
                    continue

                # Process path data
                filePath = os.path.join(root, fileName)
                fileRel  = filePath.replace(path + os.sep, "")
                fileExt  = os.path.splitext(fileName)[-1]
                fileSize = os.stat(filePath).st_size

                # Compute full URI from relative path
                fileUri = uri + "/" + fileRel.replace(os.sep, "/")

                # Compute identifier from relative path
                filePathId = fileRel.replace(fileExt, "").replace(os.sep, ".")

                # Extract package ID
                filePackage = filePathId[:filePathId.rfind(".")]

                # Handle doc files
                if fileName == self._docFilename:
                    fileFor = filePathId[:filePathId.rfind(".")]
                    self._docs[filePackage] = {
                        "relpath" : fileRel,
                        "path" : filePath,
                        "encoding" : encoding,
                        "namespace" : self._namespace,
                        "id" : filePathId,
                        "package" : filePackage,
                        "size" : fileSize
                    }

                    # Stop further processing
                    continue

                # Ignore non-script
                if os.path.splitext(fileName)[-1] != ".js":
                    continue

                # Read content
                fileContent = filetool.read(filePath, encoding)

                # Extract code ID (e.g. class name, mixin name, ...)
                try:
                    fileCodeId = self._getCodeId(fileContent)
                except ValueError, e:
                    raise ValueError, e.message + u' (%s)' % fileName

                # Ignore all data files (e.g. translation, doc files, ...)
                if fileCodeId == None:
                    continue

                # Compare path and content
                if fileCodeId != filePathId:
                    self._console.error("Detected conflict between filename and classname!")
                    self._console.indent()
                    self._console.error("Classname: %s" % fileCodeId)
                    self._console.error("Path: %s" % fileRel)
                    self._console.outdent()
                    raise RuntimeError()

                # Store file data
                self._console.debug("Adding class %s" % filePathId)
                self._classes[filePathId] = {
                    "relpath" : fileRel,
                    "path" : filePath,
                    "encoding" : encoding,
                    "namespace" : self._namespace,
                    "id" : filePathId,
                    "package" : filePackage,
                    "size" : fileSize
                }

    def _scanClassPath(self, timeOfLastScan=0):

        ##
        # provide a default context dict
        def get_contextdict():
            contextdict = {}
            contextdict["console"] = context.console
            contextdict["cache"] = context.cache
            contextdict["jobconf"] = context.jobconf
            contextdict["envchecksmap"] = {}
            return contextdict

        ##
        # check class path is on file system
        def check_class_path(classRoot):
            if not os.path.isdir(classRoot):
                raise ConfigurationError("Class path from Manifest doesn't exist: %s" % self.classPath)

        ##
        # check single subdirectory from class path
        def check_multiple_namespaces(classRoot):
            if not len([d for d in os.listdir(classRoot) if not d.startswith(".")]) == 1:
                self._console.warn("The class path should contain exactly one namespace: '%s'" % (classRoot,))

        ##
        # check manifest namespace is on file system
        def check_manifest_namespace(classRoot):
            nsPrefix = self.namespace.replace(".", os.sep)
            classNSRoot = os.path.join(classRoot, nsPrefix)
            if not os.path.isdir(classNSRoot):
                raise ValueError("Manifest namespace does not exist on file system:  '%s'" % (classNSRoot))

        # ----------------------------------------------------------------------

        codeIdFromTree = True  # switch between regex- and tree-based codeId search
        classList = []
        existClassIds = dict([(x.id, x) for x in self._classes])  # if we scanned before
        docs = {}
        contextdict = get_contextdict()  # TODO: Clazz() still relies on a context dict!
        classRoot = os.path.join(self.path, self.classPath)

        check_class_path(classRoot)
        check_multiple_namespaces(classRoot)
        check_manifest_namespace(classRoot)

        self._console.debug("Scanning class folder...")

        # Iterate...
        for root, dirs, files in filetool.walk(classRoot):
            # Filter ignored directories
            for ignoredDir in dirs:
                if self._ignoredDirEntries.match(ignoredDir):
                    dirs.remove(ignoredDir)

            # Searching for files
            for fileName in files:
                # ignore dot files
                if fileName.startswith(".") or self._ignoredDirEntries.match(fileName):
                    continue
                self._console.dot()

                # basic attributes
                filePath = os.path.join(root, fileName)  # /foo/bar/baz/source/class/my/space/AppClass.js
                fileRel = filePath.replace(classRoot + os.sep, "")  # my/space/AppClass.js
                fileExt = os.path.splitext(fileName)[-1]  # "js"
                filePathId = fileRel.replace(fileExt, "").replace(os.sep, ".")  # my.space.AppClass
                filePathId = unidata.normalize("NFC", filePathId)  # o" -> ö
                filePackage = filePathId[: filePathId.rfind(".")] if "." in filePathId else ""  # my.space
                fileStat = os.stat(filePath)
                fileSize = fileStat.st_size
                fileMTime = fileStat.st_mtime

                # ignore non-script
                if fileExt != ".js":
                    continue

                # check if known and fresh
                if filePathId in existClassIds and fileMTime < timeOfLastScan:
                    classList.append(existClassIds[filePathId])
                    continue  # re-use known class

                # handle doc files
                if fileName == self._docFilename:
                    docs[filePackage] = {
                        "relpath": fileRel,
                        "path": filePath,
                        "encoding": self.encoding,
                        "namespace": self.namespace,
                        "id": filePathId,
                        "package": filePackage,
                        "size": fileSize,
                    }
                    # stop further processing
                    continue

                if filePathId == "qx.core.Environment":
                    clazz = qcEnvClass(filePathId, filePath, self, contextdict)
                else:
                    clazz = Class(filePathId, filePath, self, contextdict)

#.........这里部分代码省略.........