Python utils.create_file函数代码示例

    def test_move_files(self, mock_client_class):
        """Tests moving files successfully"""

        s3_object_1a = MagicMock()
        s3_object_1b = MagicMock()
        s3_object_2a = MagicMock()
        s3_object_2b = MagicMock()
        mock_client = MagicMock(S3Client)
        mock_client.get_object.side_effect = [s3_object_1a, s3_object_1b, s3_object_2a, s3_object_2b]
        mock_client_class.return_value.__enter__ = Mock(return_value=mock_client)

        file_name_1 = 'my_file.txt'
        file_name_2 = 'my_file.json'
        old_workspace_path_1 = os.path.join('my_dir_1', file_name_1)
        old_workspace_path_2 = os.path.join('my_dir_2', file_name_2)
        new_workspace_path_1 = os.path.join('my_new_dir_1', file_name_1)
        new_workspace_path_2 = os.path.join('my_new_dir_2', file_name_2)

        file_1 = storage_test_utils.create_file(file_path=old_workspace_path_1)
        file_2 = storage_test_utils.create_file(file_path=old_workspace_path_2)
        file_1_mv = FileMove(file_1, new_workspace_path_1)
        file_2_mv = FileMove(file_2, new_workspace_path_2)

        # Call method to test
        self.broker.move_files(None, [file_1_mv, file_2_mv])

        # Check results
        self.assertTrue(s3_object_1b.copy_from.called)
        self.assertTrue(s3_object_1a.delete.called)
        self.assertTrue(s3_object_2b.copy_from.called)
        self.assertTrue(s3_object_2a.delete.called)
        self.assertEqual(file_1.file_path, new_workspace_path_1)
        self.assertEqual(file_2.file_path, new_workspace_path_2)

    def test_move_files(self, mock_conn_class):
        """Tests moving files successfully"""

        s3_key_1 = MagicMock(Key)
        s3_key_2 = MagicMock(Key)
        mock_conn = MagicMock(BrokerConnection)
        mock_conn.get_key.side_effect = [s3_key_1, s3_key_2]
        mock_conn_class.return_value.__enter__ = Mock(return_value=mock_conn)

        file_name_1 = 'my_file.txt'
        file_name_2 = 'my_file.json'
        old_workspace_path_1 = os.path.join('my_dir_1', file_name_1)
        old_workspace_path_2 = os.path.join('my_dir_2', file_name_2)
        new_workspace_path_1 = os.path.join('my_new_dir_1', file_name_1)
        new_workspace_path_2 = os.path.join('my_new_dir_2', file_name_2)

        file_1 = storage_test_utils.create_file(file_path=old_workspace_path_1)
        file_2 = storage_test_utils.create_file(file_path=old_workspace_path_2)
        file_1_mv = FileMove(file_1, new_workspace_path_1)
        file_2_mv = FileMove(file_2, new_workspace_path_2)

        # Call method to test
        self.broker.move_files(None, [file_1_mv, file_2_mv])

        # Check results
        self.assertTrue(s3_key_1.copy.called)
        self.assertTrue(s3_key_2.copy.called)
        self.assertEqual(file_1.file_path, new_workspace_path_1)
        self.assertEqual(file_2.file_path, new_workspace_path_2)

    def test_download_files(self, mock_client_class):
        """Tests downloading files successfully"""

        s3_object_1 = MagicMock()
        s3_object_2 = MagicMock()
        mock_client = MagicMock(S3Client)
        mock_client.get_object.side_effect = [s3_object_1, s3_object_2]
        mock_client_class.return_value.__enter__ = Mock(return_value=mock_client)

        file_name_1 = 'my_file.txt'
        file_name_2 = 'my_file.json'
        local_path_file_1 = os.path.join('my_dir_1', file_name_1)
        local_path_file_2 = os.path.join('my_dir_2', file_name_2)
        workspace_path_file_1 = os.path.join('my_wrk_dir_1', file_name_1)
        workspace_path_file_2 = os.path.join('my_wrk_dir_2', file_name_2)

        file_1 = storage_test_utils.create_file(file_path=workspace_path_file_1)
        file_2 = storage_test_utils.create_file(file_path=workspace_path_file_2)
        file_1_dl = FileDownload(file_1, local_path_file_1, False)
        file_2_dl = FileDownload(file_2, local_path_file_2, False)

        # Call method to test
        mo = mock_open()
        with patch('__builtin__.open', mo, create=True):
            self.broker.download_files(None, [file_1_dl, file_2_dl])

        # Check results
        self.assertTrue(s3_object_1.download_file.called)
        self.assertTrue(s3_object_2.download_file.called)

    def test_successfully(self, mock_copy, mock_chmod, mock_exists, mock_makedirs):
        """Tests calling HostBroker.upload_files() successfully"""

        def new_exists(path):
            return False
        mock_exists.side_effect = new_exists

        volume_path = os.path.join('the', 'volume', 'path')
        file_name_1 = 'my_file.txt'
        file_name_2 = 'my_file.json'
        local_path_file_1 = os.path.join('my_dir_1', file_name_1)
        local_path_file_2 = os.path.join('my_dir_2', file_name_2)
        workspace_path_file_1 = os.path.join('my_wrk_dir_1', file_name_1)
        workspace_path_file_2 = os.path.join('my_wrk_dir_2', file_name_2)
        full_workspace_path_file_1 = os.path.join(volume_path, workspace_path_file_1)
        full_workspace_path_file_2 = os.path.join(volume_path, workspace_path_file_2)

        file_1 = storage_test_utils.create_file(file_path=workspace_path_file_1)
        file_2 = storage_test_utils.create_file(file_path=workspace_path_file_2)
        file_1_up = FileUpload(file_1, local_path_file_1)
        file_2_up = FileUpload(file_2, local_path_file_2)

        # Call method to test
        self.broker.upload_files(volume_path, [file_1_up, file_2_up])

        # Check results
        two_calls = [call(os.path.dirname(full_workspace_path_file_1), mode=0755),
                     call(os.path.dirname(full_workspace_path_file_2), mode=0755)]
        mock_makedirs.assert_has_calls(two_calls)
        two_calls = [call(local_path_file_1, full_workspace_path_file_1),
                     call(local_path_file_2, full_workspace_path_file_2)]
        mock_copy.assert_has_calls(two_calls)
        two_calls = [call(full_workspace_path_file_1, 0644), call(full_workspace_path_file_2, 0644)]
        mock_chmod.assert_has_calls(two_calls)

    def test_successfully(self, mock_remove, mock_exists):
        """Tests calling NfsBroker.delete_files() successfully"""

        def new_exists(path):
            return True
        mock_exists.side_effect = new_exists

        volume_path = os.path.join('the', 'volume', 'path')
        file_path_1 = os.path.join('my_dir', 'my_file.txt')
        file_path_2 = os.path.join('my_dir', 'my_file.json')
        full_path_file_1 = os.path.join(volume_path, file_path_1)
        full_path_file_2 = os.path.join(volume_path, file_path_2)

        file_1 = storage_test_utils.create_file(file_path=file_path_1)
        file_2 = storage_test_utils.create_file(file_path=file_path_2)

        # Call method to test
        self.broker.delete_files(volume_path, [file_1, file_2])

        # Check results
        two_calls = [call(full_path_file_1), call(full_path_file_2)]
        mock_remove.assert_has_calls(two_calls)

        self.assertTrue(file_1.is_deleted)
        self.assertIsNotNone(file_1.deleted)
        self.assertTrue(file_2.is_deleted)
        self.assertIsNotNone(file_2.deleted)

    def test_deleted_file(self):
        '''Tests calling ScaleFileManager.move_files() with a deleted file'''

        work_dir = os.path.join('work', 'dir')

        workspace_1 = storage_test_utils.create_workspace()
        file_1 = storage_test_utils.create_file(file_name='my_file_1.txt', workspace=workspace_1)
        new_workspace_path_1 = os.path.join('my', 'new', 'path', '1', os.path.basename(file_1.file_path))
        file_2 = storage_test_utils.create_file(file_name='my_file_2.txt', workspace=workspace_1)
        file_2.is_deleted = True
        file_2.save()
        new_workspace_path_2 = os.path.join('my', 'new', 'path', '2', os.path.basename(file_2.file_path))
        workspace_1.move_files = MagicMock()
        workspace_1_work_dir = ScaleFile.objects._get_workspace_work_dir(work_dir, workspace_1)

        workspace_2 = storage_test_utils.create_workspace()
        workspace_2.is_active = False
        workspace_2.save()
        file_3 = storage_test_utils.create_file(file_name='my_file_3.txt', workspace=workspace_2)
        new_workspace_path_3 = os.path.join('my', 'new', 'path', '3', os.path.basename(file_3.file_path))
        file_4 = storage_test_utils.create_file(file_name='my_file_4.txt', workspace=workspace_2)
        new_workspace_path_4 = os.path.join('my', 'new', 'path', '4', os.path.basename(file_4.file_path))
        workspace_2.move_files = MagicMock()
        workspace_2_work_dir = ScaleFile.objects._get_workspace_work_dir(work_dir, workspace_2)

        files = [(file_1, new_workspace_path_1), (file_2, new_workspace_path_2), (file_3, new_workspace_path_3),
                 (file_4, new_workspace_path_4)]
        self.assertRaises(DeletedFile, ScaleFile.objects.move_files, work_dir, files)

    def test_host_link_files(self, mock_execute, mock_client_class):
        """Tests sym-linking files successfully"""

        volume_path = os.path.join('the', 'volume', 'path')
        file_name_1 = 'my_file.txt'
        file_name_2 = 'my_file.json'
        local_path_file_1 = os.path.join('my_dir_1', file_name_1)
        local_path_file_2 = os.path.join('my_dir_2', file_name_2)
        workspace_path_file_1 = os.path.join('my_wrk_dir_1', file_name_1)
        workspace_path_file_2 = os.path.join('my_wrk_dir_2', file_name_2)
        full_workspace_path_file_1 = os.path.join(volume_path, workspace_path_file_1)
        full_workspace_path_file_2 = os.path.join(volume_path, workspace_path_file_2)
        file_1 = storage_test_utils.create_file(file_path=workspace_path_file_1)
        file_2 = storage_test_utils.create_file(file_path=workspace_path_file_2)

        file_1_dl = FileDownload(file_1, local_path_file_1, True)
        file_2_dl = FileDownload(file_2, local_path_file_2, True)

        # Call method to test
        self.broker.download_files(volume_path, [file_1_dl, file_2_dl])

        # Check results
        two_calls = [call(['ln', '-s', full_workspace_path_file_1, local_path_file_1]),
                     call(['ln', '-s', full_workspace_path_file_2, local_path_file_2])]
        mock_execute.assert_has_calls(two_calls)

    def test_success(self):
        """Tests calling ScaleFileManager.download_files() successfully"""

        workspace_1 = storage_test_utils.create_workspace()
        file_1 = storage_test_utils.create_file(workspace=workspace_1)
        local_path_1 = '/my/local/path/file.txt'
        file_2 = storage_test_utils.create_file(workspace=workspace_1)
        local_path_2 = '/another/local/path/file.txt'
        file_3 = storage_test_utils.create_file(workspace=workspace_1)
        local_path_3 = '/another/local/path/file.json'
        workspace_1.setup_download_dir = MagicMock()
        workspace_1.download_files = MagicMock()

        workspace_2 = storage_test_utils.create_workspace()
        file_4 = storage_test_utils.create_file(workspace=workspace_2)
        local_path_4 = '/my/local/path/4/file.txt'
        file_5 = storage_test_utils.create_file(workspace=workspace_2)
        local_path_5 = '/another/local/path/5/file.txt'
        workspace_2.setup_download_dir = MagicMock()
        workspace_2.download_files = MagicMock()

        files = [FileDownload(file_1, local_path_1, False), FileDownload(file_2, local_path_2, False),
                 FileDownload(file_3, local_path_3, False), FileDownload(file_4, local_path_4, False),
                 FileDownload(file_5, local_path_5, False)]
        ScaleFile.objects.download_files(files)

        workspace_1.download_files.assert_called_once_with([FileDownload(file_1, local_path_1, False),
                                                            FileDownload(file_2, local_path_2, False),
                                                            FileDownload(file_3, local_path_3, False)])
        workspace_2.download_files.assert_called_once_with([FileDownload(file_4, local_path_4, False),
                                                            FileDownload(file_5, local_path_5, False)])

    def test_no_files(self):
        '''Tests calling ScaleFileManager.get_total_file_size() where no files match the file IDs'''

        storage_test_utils.create_file(file_size=100)

        file_size = ScaleFile.objects.get_total_file_size([4444444444, 555555555555, 666666666666])
        self.assertEqual(file_size, 0)

    def test_deleted_file(self):
        """Tests calling ScaleFileManager.download_files() with a deleted file"""

        workspace_1 = storage_test_utils.create_workspace()
        file_1 = storage_test_utils.create_file(workspace=workspace_1)
        local_path_1 = '/my/local/path/file.txt'
        file_2 = storage_test_utils.create_file(workspace=workspace_1)
        local_path_2 = '/another/local/path/file.txt'
        file_2.is_deleted = True
        file_2.save()
        file_3 = storage_test_utils.create_file(workspace=workspace_1)
        local_path_3 = '/another/local/path/file.json'
        workspace_1.download_files = MagicMock()

        workspace_2 = storage_test_utils.create_workspace()
        file_4 = storage_test_utils.create_file(workspace=workspace_2)
        local_path_4 = '/my/local/path/4/file.txt'
        file_5 = storage_test_utils.create_file(workspace=workspace_2)
        local_path_5 = '/another/local/path/5/file.txt'
        workspace_2.download_files = MagicMock()

        files = [FileDownload(file_1, local_path_1, False), FileDownload(file_2, local_path_2, False),
                 FileDownload(file_3, local_path_3, False), FileDownload(file_4, local_path_4, False),
                 FileDownload(file_5, local_path_5, False)]
        self.assertRaises(DeletedFile, ScaleFile.objects.download_files, files)

    def test_delete_files(self, mock_client_class):
        """Tests deleting files successfully"""

        s3_object_1 = MagicMock()
        s3_object_2 = MagicMock()
        mock_client = MagicMock(S3Client)
        mock_client.get_object.side_effect = [s3_object_1, s3_object_2]
        mock_client_class.return_value.__enter__ = Mock(return_value=mock_client)

        file_path_1 = os.path.join('my_dir', 'my_file.txt')
        file_path_2 = os.path.join('my_dir', 'my_file.json')

        file_1 = storage_test_utils.create_file(file_path=file_path_1)
        file_2 = storage_test_utils.create_file(file_path=file_path_2)

        # Call method to test
        self.broker.delete_files(None, [file_1, file_2])

        # Check results
        self.assertTrue(s3_object_1.delete.called)
        self.assertTrue(s3_object_2.delete.called)
        self.assertTrue(file_1.is_deleted)
        self.assertIsNotNone(file_1.deleted)
        self.assertTrue(file_2.is_deleted)
        self.assertIsNotNone(file_2.deleted)

    def test_upload_files(self, mock_client_class):
        """Tests uploading files successfully"""

        s3_object_1 = MagicMock()
        s3_object_2 = MagicMock()
        mock_client = MagicMock(S3Client)
        mock_client.get_object.side_effect = [s3_object_1, s3_object_2]
        mock_client_class.return_value.__enter__ = Mock(return_value=mock_client)

        file_name_1 = 'my_file.txt'
        file_name_2 = 'my_file.json'
        local_path_file_1 = os.path.join('my_dir_1', file_name_1)
        local_path_file_2 = os.path.join('my_dir_2', file_name_2)
        workspace_path_file_1 = os.path.join('my_wrk_dir_1', file_name_1)
        workspace_path_file_2 = os.path.join('my_wrk_dir_2', file_name_2)

        file_1 = storage_test_utils.create_file(file_path=workspace_path_file_1, media_type='text/plain')
        file_2 = storage_test_utils.create_file(file_path=workspace_path_file_2, media_type='application/json')
        file_1_up = FileUpload(file_1, local_path_file_1)
        file_2_up = FileUpload(file_2, local_path_file_2)

        # Call method to test
        mo = mock_open()
        with patch('__builtin__.open', mo, create=True):
            self.broker.upload_files(None, [file_1_up, file_2_up])

        # Check results
        self.assertTrue(s3_object_1.upload_file.called)
        self.assertTrue(s3_object_2.upload_file.called)
        self.assertEqual(s3_object_1.upload_file.call_args[0][1]['ContentType'], 'text/plain')
        self.assertEqual(s3_object_2.upload_file.call_args[0][1]['ContentType'], 'application/json')

    def test_success(self):
        '''Tests calling ScaleFileManager.get_total_file_size() successfully'''

        file_1 = storage_test_utils.create_file(file_size=100)
        file_2 = storage_test_utils.create_file(file_size=300)
        storage_test_utils.create_file(file_size=700)

        file_size = ScaleFile.objects.get_total_file_size([file_1.id, file_2.id])
        self.assertEqual(file_size, 400)

    def test_inactive_workspace(self):
        """Tests calling deleting files from an inactive workspace"""

        workspace_1 = storage_test_utils.create_workspace()
        workspace_1.download_files = MagicMock()
        file_1 = storage_test_utils.create_file(workspace=workspace_1)

        workspace_2 = storage_test_utils.create_workspace(is_active=False)
        file_2 = storage_test_utils.create_file(workspace=workspace_2)

        files = [file_1, file_2]
        self.assertRaises(ArchivedWorkspace, ScaleFile.objects.delete_files, files)

    def test_schedule_date_range_data_ended(self):
        """Tests calling BatchManager.schedule_recipes() for a batch with a data ended date range restriction"""
        file1 = storage_test_utils.create_file()
        file1.data_started = datetime.datetime(2016, 1, 1)
        file1.data_ended = datetime.datetime(2016, 1, 10)
        file1.save()
        data1 = {
            'version': '1.0',
            'input_data': [{
                'name': 'Recipe Input',
                'file_id': file1.id,
            }],
            'workspace_id': self.workspace.id,
        }
        recipe1 = Recipe.objects.create_recipe(recipe_type=self.recipe_type, data=RecipeData(data1),
                                               event=self.event).recipe

        file2 = storage_test_utils.create_file()
        file2.data_started = datetime.datetime(2016, 2, 1)
        file2.data_ended = datetime.datetime(2016, 2, 10)
        file2.save()
        data2 = {
            'version': '1.0',
            'input_data': [{
                'name': 'Recipe Input',
                'file_id': file2.id,
            }],
            'workspace_id': self.workspace.id,
        }
        Recipe.objects.create_recipe(recipe_type=self.recipe_type, data=RecipeData(data2), event=self.event)

        recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2)

        definition = {
            'date_range': {
                'type': 'data',
                'ended': '2016-01-15T00:00:00.000Z',
            },
        }
        batch = batch_test_utils.create_batch(recipe_type=self.recipe_type, definition=definition)

        Batch.objects.schedule_recipes(batch.id)

        batch = Batch.objects.get(pk=batch.id)
        self.assertEqual(batch.status, 'CREATED')
        self.assertEqual(batch.created_count, 1)
        self.assertEqual(batch.total_count, 1)

        batch_recipes = BatchRecipe.objects.all()
        self.assertEqual(len(batch_recipes), 1)
        self.assertEqual(batch_recipes[0].superseded_recipe, recipe1)