Python FilesHelper.copy_file方法代码示例

# 需要导入模块: from tvb.core.entities.file.files_helper import FilesHelper [as 别名]
# 或者: from tvb.core.entities.file.files_helper.FilesHelper import copy_file [as 别名]
class ImportService():
    """
    Service for importing TVB entities into system.
    It supports TVB exported H5 files as input, but it should also handle H5 files 
    generated outside of TVB, as long as they respect the same structure.
    """


    def __init__(self):
        self.logger = get_logger(__name__)
        self.user_id = None
        self.files_helper = FilesHelper()
        self.created_projects = []


    def _download_and_unpack_project_zip(self, uploaded, uq_file_name, temp_folder):

        if isinstance(uploaded, FieldStorage) or isinstance(uploaded, Part):
            if not uploaded.file:
                raise ProjectImportException("Please select the archive which contains the project structure.")
            with open(uq_file_name, 'wb') as file_obj:
                self.files_helper.copy_file(uploaded.file, file_obj)
        else:
            shutil.copy2(uploaded, uq_file_name)

        try:
            self.files_helper.unpack_zip(uq_file_name, temp_folder)
        except FileStructureException, excep:
            self.logger.exception(excep)
            raise ProjectImportException("Bad ZIP archive provided. A TVB exported project is expected!")

# 需要导入模块: from tvb.core.entities.file.files_helper import FilesHelper [as 别名]
# 或者: from tvb.core.entities.file.files_helper.FilesHelper import copy_file [as 别名]
def search_and_export_ts(project_id, export_folder=os.path.join("~", "TVB")):

    #### This is the simplest filter you could write: filter and entity by Subject
    filter_connectivity = FilterChain(fields=[FilterChain.datatype + '.subject'],
                                      operations=["=="],
                                      values=[DataTypeMetaData.DEFAULT_SUBJECT])

    connectivities = _retrieve_entities_by_filters(Connectivity, project_id, filter_connectivity)


    #### A more complex filter: by linked entity (connectivity), BOLD monitor, sampling, operation param:
    filter_timeseries = FilterChain(fields=[FilterChain.datatype + '._connectivity',
                                            FilterChain.datatype + '._title',
                                            FilterChain.datatype + '._sample_period',
                                            FilterChain.datatype + '._sample_rate',
                                            FilterChain.operation + '.parameters'
                                            ],
                                    operations=["==", "like", ">=", "<=", "like"],
                                    values=[connectivities[0].gid,
                                            "Bold",
                                            "500", "0.002",
                                            '"conduction_speed": "3.0"'
                                            ]
                                    )

    #### If you want to filter another type of TS, change the kind class bellow,
    #### instead of TimeSeriesRegion use TimeSeriesEEG, or TimeSeriesSurface, etc.
    timeseries = _retrieve_entities_by_filters(TimeSeriesRegion, project_id, filter_timeseries)

    for ts in timeseries:
        print("=============================")
        print(ts.summary_info)
        print(" Original file: " + str(ts.get_storage_file_path()))
        destination_file = os.path.expanduser(os.path.join(export_folder, ts.get_storage_file_name()))
        FilesHelper.copy_file(ts.get_storage_file_path(), destination_file)
        if os.path.exists(destination_file):
            print(" TS file copied at: " + destination_file)
        else:
            print(" Some error happened when trying to copy at destination folder!!")

# 需要导入模块: from tvb.core.entities.file.files_helper import FilesHelper [as 别名]
# 或者: from tvb.core.entities.file.files_helper.FilesHelper import copy_file [as 别名]
    def _process_input_zip(self, zip_arch, result_folder, remote_prefix, 
                           file_name_base, expected_pairs, fix_number=True):
        """
        Read entries in uploaded ZIP.
        Raise Exception in case pairs HDR/IMG are not matched or number "expected_pairs" is not met.
        :returns: string with HDR list (to be passed to DTI pipeline).
        """
        
        hdr_files = []
        for file_name in zip_arch.namelist():
            if not file_name.startswith(file_name_base) or file_name.endswith("/"):
                continue
            if file_name.endswith(".hdr"):
                pair_img = file_name.replace(".hdr", ".img")
                if pair_img not in zip_arch.namelist():
                    raise ConnectException("Could not find pair for HDR file :" + str(file_name))
                
                new_file_name = os.path.join(result_folder, file_name_base + str(len(hdr_files)) + ".hdr")
                src = zip_arch.open(file_name, 'rU')
                FilesHelper.copy_file(src, new_file_name)
                hdr_files.append(os.path.join(remote_prefix, os.path.split(new_file_name)[1]))
                new_file_name = new_file_name.replace(".hdr", ".img")
                src = zip_arch.open(pair_img, 'rU')
                FilesHelper.copy_file(src, new_file_name)
                
            elif not file_name.endswith(".img"):
                self.logger.warning("Ignored file :" + str(file_name))
            
        if len(hdr_files) < expected_pairs or (fix_number and len(hdr_files) > expected_pairs):
            raise ConnectException("Invalid number of files:" + str(len(hdr_files)) +
                                   " expected:" + str(expected_pairs))

        result = ""
        for hdr_name in hdr_files:
            result = result + hdr_name + " "
        return result

# 需要导入模块: from tvb.core.entities.file.files_helper import FilesHelper [as 别名]
# 或者: from tvb.core.entities.file.files_helper.FilesHelper import copy_file [as 别名]
class CSVConnectivityImporterTest(TransactionalTestCase):
    """
    Unit-tests for csv connectivity importer.
    """

    def setUp(self):
        self.test_user = TestFactory.create_user()
        self.test_project = TestFactory.create_project(self.test_user)
        self.helper = FilesHelper()


    def tearDown(self):
        """
        Clean-up tests data
        """
        FilesHelper().remove_project_structure(self.test_project.name)


    def _import_csv_test_connectivity(self, reference_connectivity_gid, subject):

        ### First prepare input data:
        data_dir = path.abspath(path.dirname(tvb_data.__file__))

        torronto_dir = path.join(data_dir, 'dti_pipeline', 'Output_Toronto')
        weights = path.join(torronto_dir, 'output_ConnectionCapacityMatrix.csv')
        tracts = path.join(torronto_dir, 'output_ConnectionDistanceMatrix.csv')
        weights_tmp = weights + '.tmp'
        tracts_tmp = tracts + '.tmp'
        self.helper.copy_file(weights, weights_tmp)
        self.helper.copy_file(tracts, tracts_tmp)

        ### Find importer and Launch Operation
        importer = TestFactory.create_adapter('tvb.adapters.uploaders.csv_connectivity_importer',
                                              'CSVConnectivityImporter')
        FlowService().fire_operation(importer, self.test_user, self.test_project.id,
                                     weights=weights_tmp, tracts=tracts_tmp, Data_Subject=subject,
                                     input_data=reference_connectivity_gid)


    def test_happy_flow_import(self):
        """
        Test that importing a CFF generates at least one DataType in DB.
        """
        ConnectivityZipTest.import_test_connectivity96(self.test_user,
                                                       self.test_project,
                                                       subject=TEST_SUBJECT_A)

        field = FilterChain.datatype + '.subject'
        filters = FilterChain('', [field], [TEST_SUBJECT_A], ['=='])
        reference_connectivity = TestFactory.get_entity(self.test_project, Connectivity(), filters)

        dt_count_before = TestFactory.get_entity_count(self.test_project, Connectivity())

        self._import_csv_test_connectivity(reference_connectivity.gid, TEST_SUBJECT_B)

        dt_count_after = TestFactory.get_entity_count(self.test_project, Connectivity())
        self.assertEqual(dt_count_before + 1, dt_count_after)

        filters = FilterChain('', [field], [TEST_SUBJECT_B], ['like'])
        imported_connectivity = TestFactory.get_entity(self.test_project, Connectivity(), filters)

        # check relationship between the imported connectivity and the reference
        self.assertTrue((reference_connectivity.centres == imported_connectivity.centres).all())
        self.assertTrue((reference_connectivity.orientations == imported_connectivity.orientations).all())

        self.assertEqual(reference_connectivity.number_of_regions, imported_connectivity.number_of_regions)
        self.assertTrue((reference_connectivity.region_labels == imported_connectivity.region_labels).all())

        self.assertFalse((reference_connectivity.weights == imported_connectivity.weights).all())
        self.assertFalse((reference_connectivity.tract_lengths == imported_connectivity.tract_lengths).all())



    def test_bad_reference(self):
        TestFactory.import_cff(test_user=self.test_user, test_project=self.test_project)
        field = FilterChain.datatype + '.subject'
        filters = FilterChain('', [field], [TEST_SUBJECT_A], ['!='])
        bad_reference_connectivity = TestFactory.get_entity(self.test_project, Connectivity(), filters)

        self.assertRaises(OperationException, self._import_csv_test_connectivity,
                          bad_reference_connectivity.gid, TEST_SUBJECT_A)

# 需要导入模块: from tvb.core.entities.file.files_helper import FilesHelper [as 别名]
# 或者: from tvb.core.entities.file.files_helper.FilesHelper import copy_file [as 别名]
class ImportService(object):
    """
    Service for importing TVB entities into system.
    It supports TVB exported H5 files as input, but it should also handle H5 files 
    generated outside of TVB, as long as they respect the same structure.
    """


    def __init__(self):
        self.logger = get_logger(__name__)
        self.user_id = None
        self.files_helper = FilesHelper()
        self.created_projects = []


    def _download_and_unpack_project_zip(self, uploaded, uq_file_name, temp_folder):

        if isinstance(uploaded, FieldStorage) or isinstance(uploaded, Part):
            if not uploaded.file:
                raise ProjectImportException("Please select the archive which contains the project structure.")
            with open(uq_file_name, 'wb') as file_obj:
                self.files_helper.copy_file(uploaded.file, file_obj)
        else:
            shutil.copy2(uploaded, uq_file_name)

        try:
            self.files_helper.unpack_zip(uq_file_name, temp_folder)
        except FileStructureException as excep:
            self.logger.exception(excep)
            raise ProjectImportException("Bad ZIP archive provided. A TVB exported project is expected!")


    @staticmethod
    def _compute_unpack_path():
        """
        :return: the name of the folder where to expand uploaded zip
        """
        now = datetime.now()
        date_str = "%d-%d-%d_%d-%d-%d_%d" % (now.year, now.month, now.day, now.hour,
                                             now.minute, now.second, now.microsecond)
        uq_name = "%s-ImportProject" % date_str
        return os.path.join(TvbProfile.current.TVB_TEMP_FOLDER, uq_name)


    @transactional
    def import_project_structure(self, uploaded, user_id):
        """
        Execute import operations:
         
        1. check if ZIP or folder 
        2. find all project nodes
        3. for each project node:
            - create project
            - create all operations
            - import all images
            - create all dataTypes
        """

        self.user_id = user_id
        self.created_projects = []

        # Now compute the name of the folder where to explode uploaded ZIP file
        temp_folder = self._compute_unpack_path()
        uq_file_name = temp_folder + ".zip"

        try:
            self._download_and_unpack_project_zip(uploaded, uq_file_name, temp_folder)
            self._import_projects_from_folder(temp_folder)

        except Exception as excep:
            self.logger.exception("Error encountered during import. Deleting projects created during this operation.")
            # Remove project folders created so far.
            # Note that using the project service to remove the projects will not work,
            # because we do not have support for nested transaction.
            # Removing from DB is not necessary because in transactional env a simple exception throw
            # will erase everything to be inserted.
            for project in self.created_projects:
                project_path = os.path.join(TvbProfile.current.TVB_STORAGE, FilesHelper.PROJECTS_FOLDER, project.name)
                shutil.rmtree(project_path)
            raise ProjectImportException(str(excep))

        finally:
            # Now delete uploaded file
            if os.path.exists(uq_file_name):
                os.remove(uq_file_name)
            # Now delete temporary folder where uploaded ZIP was exploded.
            if os.path.exists(temp_folder):
                shutil.rmtree(temp_folder)


    @staticmethod
    def _load_burst_info_from_json(project_path):
        bursts_dict = {}
        dt_mappings_dict = {}
        bursts_file = os.path.join(project_path, BURST_INFO_FILE)
        if os.path.isfile(bursts_file):
            with open(bursts_file) as f:
                bursts_info_dict = json.load(f)
            bursts_dict = bursts_info_dict[BURSTS_DICT_KEY]
            dt_mappings_dict = bursts_info_dict[DT_BURST_MAP]
#.........这里部分代码省略.........