Python ImagesLoader.saveFromStack方法代码示例

# 需要导入模块: from thunder.rdds.fileio.imagesloader import ImagesLoader [as 别名]
# 或者: from thunder.rdds.fileio.imagesloader.ImagesLoader import saveFromStack [as 别名]

#.........这里部分代码省略.........
            where the first dimension changes most rapidly. For 'png' or 'tif' data dimensions
            will be read from the image file headers.

        inputFormat: str, optional, default = 'stack'
            Expected format of the input data: 'stack', 'png', or 'tif'. 'stack' indicates flat binary stacks.
            'png' or 'tif' indicate image formats. Page of a multipage tif file will be extend along
            the third dimension. Separate files interpreted as distinct records, with ordering
            given by lexicographic sorting of file names.

        ext: string, optional, default = None
            File extension, default will be "bin" if inputFormat=="stack", "tif" for inputFormat=='tif',
            and 'png' for inputFormat=="png".

        dtype: string or numpy dtype. optional, default 'int16'
            Data type of the image files to be loaded, specified as a numpy "dtype" string.
            Ignored for 'tif' or 'png' (data will be inferred from image formats).

        blockSize: string or positive int, optional, default "150M"
            Requested size of blocks (e.g "64M", "512k", "2G"). If shuffle=True, can also be a
            tuple of int specifying the number of pixels or splits per dimension. Indirectly
            controls the number of Spark partitions, with one partition per block.

        blockSizeUnits: string, either "pixels" or "splits", default "pixels"
            Units for interpreting a tuple passed as blockSize when shuffle=True.

        startIdx: nonnegative int, optional, default = None
            Convenience parameters to read only a subset of input files. Uses python slice conventions
            (zero-based indexing with exclusive final position). These parameters give the starting
            and final index after lexicographic sorting.

        stopIdx: nonnegative int, optional, default = None
            See startIdx.

        shuffle: boolean, optional, default = True
            Controls whether the conversion from Images to Series formats will use of a Spark shuffle-based method.

        overwrite: boolean, optional, default False
            If true, the directory specified by outputDirPath will be deleted (recursively) if it
            already exists. (Use with caution.)

        recursive: boolean, optional, default = False
            If true, will recursively descend directories rooted at dataPath, loading all files
            in the tree with an appropriate extension.

        nplanes: positive integer, optional, default = None
            Subdivide individual image files. Every `nplanes` from each file will be considered a new record.
            With nplanes=None (the default), a single file will be considered as representing a single record.
            If the number of records per file is not the same across all files, then `renumber` should be set
            to True to ensure consistent keys.

        npartitions: positive int, optional, default = None
            Specify number of partitions for the RDD, if unspecified will use 1 partition per image.

        renumber: boolean, optional, default = False
            Recalculate keys for records after images are loading. Only necessary if different files contain
            different number of records (e.g. due to specifying nplanes). See Images.renumber().

        confFilename : string, optional, default = 'conf.json'
            Name of conf file if using to specify parameters for binary stack data

        """
        checkParams(inputFormat, ['stack', 'tif', 'tif-stack'])

        if not overwrite:
            raiseErrorIfPathExists(outputDirPath, awsCredentialsOverride=self._credentials)
            overwrite = True  # prevent additional downstream checks for this path

        if not ext:
            ext = DEFAULT_EXTENSIONS.get(inputFormat.lower(), None)

        if shuffle:
            from thunder.rdds.fileio.imagesloader import ImagesLoader
            loader = ImagesLoader(self._sc)
            if inputFormat.lower() == 'stack':
                images = loader.fromStack(dataPath, dims, ext=ext, dtype=dtype, startIdx=startIdx, stopIdx=stopIdx,
                                          recursive=recursive, nplanes=nplanes, npartitions=npartitions,
                                          confFilename=confFilename)
            else:
                # 'tif' or 'tif-stack'
                images = loader.fromTif(dataPath, ext=ext, startIdx=startIdx, stopIdx=stopIdx,
                                        recursive=recursive, nplanes=nplanes, npartitions=npartitions)
            if renumber:
                images = images.renumber()
            images.toBlocks(blockSize, units=blockSizeUnits).saveAsBinarySeries(outputDirPath, overwrite=overwrite)
        else:
            from thunder.rdds.fileio.seriesloader import SeriesLoader
            if nplanes is not None:
                raise NotImplementedError("nplanes is not supported with shuffle=False")
            if npartitions is not None:
                raise NotImplementedError("npartitions is not supported with shuffle=False")
            loader = SeriesLoader(self._sc)
            if inputFormat.lower() == 'stack':
                loader.saveFromStack(dataPath, outputDirPath, dims, ext=ext, dtype=dtype,
                                     blockSize=blockSize, overwrite=overwrite, startIdx=startIdx,
                                     stopIdx=stopIdx, recursive=recursive)
            else:
                # 'tif' or 'tif-stack'
                loader.saveFromTif(dataPath, outputDirPath, ext=ext, blockSize=blockSize,
                                   startIdx=startIdx, stopIdx=stopIdx, overwrite=overwrite,
                                   recursive=recursive)

# 需要导入模块: from thunder.rdds.fileio.imagesloader import ImagesLoader [as 别名]
# 或者: from thunder.rdds.fileio.imagesloader.ImagesLoader import saveFromStack [as 别名]
    def convertImagesToSeries(self, datapath, outputdirpath, dims=None, inputformat='stack',
                              dtype='int16', blocksize="150M", startidx=None, stopidx=None,
                              shuffle=False, overwrite=False):
        """
        Write out Images data as Series data, saved in a flat binary format.

        The resulting Series data files may subsequently be read in using the loadSeries() method. The Series data
        object that results will be equivalent to that which would be generated by loadImagesAsSeries(). It is expected
        that loading Series data directly from the series flat binary format, using loadSeries(), will be faster than
        converting image data to a Series object through loadImagesAsSeries().

        Parameters
        ----------
        datapath: string
            Path to data files or directory, specified as either a local filesystem path or in a URI-like format,
            including scheme. A datapath argument may include a single '*' wildcard character in the filename. Examples
            of valid datapaths include 'a/local/relative/directory/*.stack", "s3n:///my-s3-bucket/data/mydatafile.tif",
            "/mnt/my/absolute/data/directory/", or "file:///mnt/another/data/directory/".

        outputdirpath: string
            Path to a directory into which to write Series file output. An outputdir argument may be either a path
            on the local file system or a URI-like format, as in datapath. Examples of valid outputdirpaths include
            "a/relative/directory/", "s3n:///my-s3-bucket/data/myoutput/", or "file:///mnt/a/new/directory/". If the
            directory specified by outputdirpath already exists and the 'overwrite' parameter is False, this method
            will throw a ValueError. If the directory exists and 'overwrite' is True, the existing directory and all
            its contents will be deleted and overwritten.

        dims: tuple of positive int, optional (but required if inputformat is 'stack')
            Dimensions of input image data, for instance (1024, 1024, 48). Binary stack data will be interpreted as
            coming from a multidimensional array of the specified dimensions.

            The first dimension of the passed dims tuple should be the one that is changing most rapidly
            on disk. So for instance given dims of (x, y, z), the coordinates of the data in a binary stack file
            should be ordered as [(x0, y0, z0), (x1, y0, z0), ..., (xN, y0, z0), (x0, y1, z0), (x1, y1, z0), ...,
            (xN, yM, z0), (x0, y0, z1), ..., (xN, yM, zP)]. This is the opposite convention from that used by numpy,
            which by default has the fastest-changing dimension listed last (column-major convention). Thus, if loading
            a numpy array `ary`, where `ary.shape == (z, y, x)`, written to disk by `ary.tofile("myarray.stack")`, the
            corresponding dims parameter should be (x, y, z).
            If inputformat is 'tif-stack', the dims parameter (if any) will be ignored; data dimensions will instead
            be read out from the tif file headers.

        inputformat: {'stack', 'tif-stack'}. optional, default 'stack'
            Expected format of the input data. 'stack' indicates flat files of raw binary data, while 'tif-stack'
            indicates a sequence of multipage tif files, with each page of the tif corresponding to a separate z-plane.
            For both stacks and tif stacks, separate files are interpreted as distinct time points, with ordering
            given by lexicographic sorting of file names.
            This method assumes that stack data consists of signed 16-bit integers in native byte order. The lower-level
            API method SeriesLoader.saveFromStack() allows alternative data types to be read in.

        dtype: string or numpy dtype. optional, default 'int16'
            Data type of the image files to be loaded, specified as a numpy "dtype" string. If inputformat is
            'tif-stack', the dtype parameter (if any) will be ignored; data type will instead be read out from the
            tif headers.

        blocksize: string formatted as e.g. "64M", "512k", "2G", or positive int. optional, default "150M"
            Requested size of individual output files in bytes (or kilobytes, megabytes, gigabytes). This parameter
            also indirectly controls the number of Spark partitions to be used, with one partition used per block
            created.

        startidx: nonnegative int, optional
            startidx and stopidx are convenience parameters to allow only a subset of input files to be read in. These
            parameters give the starting index (inclusive) and final index (exclusive) of the data files to be used
            after lexicographically sorting all input data files matching the datapath argument. For example,
            startidx=None (the default) and stopidx=10 will cause only the first 10 data files in datapath to be read
            in; startidx=2 and stopidx=3 will cause only the third file (zero-based index of 2) to be read in. startidx
            and stopidx use the python slice indexing convention (zero-based indexing with an exclusive final position).

        stopidx: nonnegative int, optional
            See startidx.

        shuffle: boolean, optional, default False
            Controls whether the conversion from Images to Series formats will make use of a Spark shuffle-based method.
            The default at present is not to use a shuffle. The shuffle-based method may lead to higher performance in
            some cases, but the default method appears to be more stable with larger data set sizes. This default may
            change in future releases.

        overwrite: boolean, optional, default False
            If true, the directory specified by outputdirpath will first be deleted, along with all its contents, if it
            already exists. (Use with caution.) If false, a ValueError will be thrown if outputdirpath is found to
            already exist.
        """
        checkparams(inputformat, ['stack', 'tif-stack'])

        if inputformat.lower() == 'stack' and not dims:
            raise ValueError("Dimensions ('dims' parameter) must be specified if loading from binary image stack" +
                             " ('stack' value for 'inputformat' parameter)")

        if shuffle:
            from thunder.rdds.fileio.imagesloader import ImagesLoader
            loader = ImagesLoader(self._sc)
            if inputformat.lower() == 'stack':
                loader.fromStack(datapath, dims, dtype=dtype, startidx=startidx, stopidx=stopidx)\
                    .saveAsBinarySeries(outputdirpath, blockSize=blocksize, overwrite=overwrite)
            else:
                loader.fromMultipageTif(datapath, startidx=startidx, stopidx=stopidx)\
                    .saveAsBinarySeries(outputdirpath, blockSize=blocksize, overwrite=overwrite)
        else:
            from thunder.rdds.fileio.seriesloader import SeriesLoader
            loader = SeriesLoader(self._sc)
            if inputformat.lower() == 'stack':
#.........这里部分代码省略.........

# 需要导入模块: from thunder.rdds.fileio.imagesloader import ImagesLoader [as 别名]
# 或者: from thunder.rdds.fileio.imagesloader.ImagesLoader import saveFromStack [as 别名]

#.........这里部分代码省略.........
        blockSize: string formatted as e.g. "64M", "512k", "2G", or positive int, tuple of positive int, or instance of
                   BlockingStrategy. optional, default "150M"
            Requested size of individual output files in bytes (or kilobytes, megabytes, gigabytes). blockSize can also
            be an instance of blockingStrategy, or a tuple of int specifying either the number of pixels or of splits
            per dimension to apply to the loaded images. Whether a tuple of int is interpreted as pixels or as splits
            depends on the value of the blockSizeUnits parameter.  This parameter also indirectly controls the number
            of Spark partitions to be used, with one partition used per block created.

        blockSizeUnits: string, either "pixels" or "splits" (or unique prefix of each, such as "s"), default "pixels"
            Specifies units to be used in interpreting a tuple passed as blockSizeSpec when shuffle=True. If a string
            or a BlockingStrategy instance is passed as blockSizeSpec, or if shuffle=False, this parameter has no
            effect.

        startIdx: nonnegative int, optional
            startIdx and stopIdx are convenience parameters to allow only a subset of input files to be read in. These
            parameters give the starting index (inclusive) and final index (exclusive) of the data files to be used
            after lexicographically sorting all input data files matching the dataPath argument. For example,
            startIdx=None (the default) and stopIdx=10 will cause only the first 10 data files in dataPath to be read
            in; startIdx=2 and stopIdx=3 will cause only the third file (zero-based index of 2) to be read in. startIdx
            and stopIdx use the python slice indexing convention (zero-based indexing with an exclusive final position).

        stopIdx: nonnegative int, optional
            See startIdx.

        shuffle: boolean, optional, default True
            Controls whether the conversion from Images to Series formats will make use of a Spark shuffle-based method.

        overwrite: boolean, optional, default False
            If true, the directory specified by outputDirPath will first be deleted, along with all its contents, if it
            already exists. (Use with caution.) If false, a ValueError will be thrown if outputDirPath is found to
            already exist.

        recursive: boolean, default False
            If true, will recursively descend directories rooted at dataPath, loading all files in the tree that
            have an appropriate extension. Recursive loading is currently only implemented for local filesystems
            (not s3), and only with shuffle=True.

        nplanes: positive integer, default None
            If passed, will cause a single image file to be subdivided into multiple records. Every
            `nplanes` z-planes (or multipage tif pages) in the file will be taken as a new record, with the
            first nplane planes of the first file being record 0, the second nplane planes being record 1, etc,
            until the first file is exhausted and record ordering continues with the first nplane planes of the
            second file, and so on. With nplanes=None (the default), a single file will be considered as
            representing a single record. Keys are calculated assuming that all input files contain the same
            number of records; if the number of records per file is not the same across all files,
            then `renumber` should be set to True to ensure consistent keys. nplanes is only supported for
            shuffle=True (the default).

        npartitions: positive int, optional
            If specified, request a certain number of partitions for the underlying Spark RDD. Default is 1
            partition per image file. Only applies when shuffle=True.

        renumber: boolean, optional, default False
            If renumber evaluates to True, then the keys for each record will be explicitly recalculated after
            all images are loaded. This should only be necessary at load time when different files contain
            different number of records. renumber is only supported for shuffle=True (the default). See
            Images.renumber().
        """
        checkParams(inputFormat, ['stack', 'tif', 'tif-stack'])

        if inputFormat.lower() == 'stack' and not dims:
            raise ValueError("Dimensions ('dims' parameter) must be specified if loading from binary image stack" +
                             " ('stack' value for 'inputFormat' parameter)")

        if not overwrite:
            raiseErrorIfPathExists(outputDirPath, awsCredentialsOverride=self.awsCredentials)
            overwrite = True  # prevent additional downstream checks for this path

        if not ext:
            ext = DEFAULT_EXTENSIONS.get(inputFormat.lower(), None)

        if shuffle:
            from thunder.rdds.fileio.imagesloader import ImagesLoader
            loader = ImagesLoader(self._sc)
            if inputFormat.lower() == 'stack':
                images = loader.fromStack(dataPath, dims, ext=ext, dtype=dtype, startIdx=startIdx, stopIdx=stopIdx,
                                          recursive=recursive, nplanes=nplanes, npartitions=npartitions)
            else:
                # 'tif' or 'tif-stack'
                images = loader.fromTif(dataPath, ext=ext, startIdx=startIdx, stopIdx=stopIdx,
                                        recursive=recursive, nplanes=nplanes, npartitions=npartitions)
            if renumber:
                images = images.renumber()
            images.toBlocks(blockSize, units=blockSizeUnits).saveAsBinarySeries(outputDirPath, overwrite=overwrite)
        else:
            from thunder.rdds.fileio.seriesloader import SeriesLoader
            if nplanes is not None:
                raise NotImplementedError("nplanes is not supported with shuffle=False")
            if npartitions is not None:
                raise NotImplementedError("npartitions is not supported with shuffle=False")
            loader = SeriesLoader(self._sc)
            if inputFormat.lower() == 'stack':
                loader.saveFromStack(dataPath, outputDirPath, dims, ext=ext, dtype=dtype,
                                     blockSize=blockSize, overwrite=overwrite, startIdx=startIdx,
                                     stopIdx=stopIdx, recursive=recursive)
            else:
                # 'tif' or 'tif-stack'
                loader.saveFromTif(dataPath, outputDirPath, ext=ext, blockSize=blockSize,
                                   startIdx=startIdx, stopIdx=stopIdx, overwrite=overwrite,
                                   recursive=recursive)