Python utils.create_source函数代码示例

    def setUp(self):
        django.setup()

        self.src_file_1 = source_test_utils.create_source()
        self.src_file_2 = source_test_utils.create_source()
        self.src_file_3 = source_test_utils.create_source()
        self.src_file_4 = source_test_utils.create_source()

        self.job_exe_1 = job_test_utils.create_job_exe()
        self.recipe_job_1 = recipe_test_utils.create_recipe_job(job=self.job_exe_1.job)
        self.product_1 = prod_test_utils.create_product(self.job_exe_1, has_been_published=True)
        self.product_2 = prod_test_utils.create_product(self.job_exe_1, has_been_published=True)
        FileAncestryLink.objects.create(ancestor=self.src_file_1, descendant=self.product_1, job_exe=self.job_exe_1,
                                        job=self.job_exe_1.job, recipe=self.recipe_job_1.recipe)
        FileAncestryLink.objects.create(ancestor=self.src_file_1, descendant=self.product_2, job_exe=self.job_exe_1,
                                        job=self.job_exe_1.job, recipe=self.recipe_job_1.recipe)
        FileAncestryLink.objects.create(ancestor=self.src_file_2, descendant=self.product_1, job_exe=self.job_exe_1,
                                        job=self.job_exe_1.job, recipe=self.recipe_job_1.recipe)
        FileAncestryLink.objects.create(ancestor=self.src_file_2, descendant=self.product_2, job_exe=self.job_exe_1,
                                        job=self.job_exe_1.job, recipe=self.recipe_job_1.recipe)

        self.job_exe_2 = job_test_utils.create_job_exe()
        self.recipe_job_2 = recipe_test_utils.create_recipe_job(job=self.job_exe_2.job)
        self.product_3 = prod_test_utils.create_product(self.job_exe_2, has_been_published=True)
        FileAncestryLink.objects.create(ancestor=self.src_file_3, descendant=self.product_3, job_exe=self.job_exe_2,
                                        job=self.job_exe_2.job, recipe=self.recipe_job_2.recipe)
        FileAncestryLink.objects.create(ancestor=self.src_file_4, descendant=self.product_3, job_exe=self.job_exe_2,
                                        job=self.job_exe_2.job, recipe=self.recipe_job_2.recipe)

    def test_no_conditions(self):
        """Tests calling IngestTriggerCondition.is_condition_met() with no conditions"""

        condition = IngestTriggerCondition(None, None)
        source_file = source_test_utils.create_source(media_type='text/plain')
        
        self.assertEqual(condition.is_condition_met(source_file), True)

    def test_media_type_mismatch(self):
        """Tests calling IngestTriggerCondition.is_condition_met() with a mismatched media type"""

        condition = IngestTriggerCondition('application/json', None)
        source_file = source_test_utils.create_source(media_type='text/plain')
        
        self.assertEqual(condition.is_condition_met(source_file), False)

    def test_media_type_match(self):
        """Tests calling IngestTriggerCondition.is_condition_met() with a matching media type"""

        condition = IngestTriggerCondition('text/plain', None)
        source_file = source_test_utils.create_source(media_type='text/plain')
        
        self.assertEqual(condition.is_condition_met(source_file), True)

def create_ingest(file_name='test.txt', status='TRANSFERRING', transfer_started=None, transfer_ended=None,
                  ingest_started=None, ingest_ended=None, data_started=None, data_ended=None, workspace=None,
                  strike=None, source_file=None):
    if not workspace:
        workspace = storage_test_utils.create_workspace()
    if not strike:
        strike = create_strike()
    if not source_file:
        source_file = source_test_utils.create_source(file_name=file_name, data_started=data_started,
                                                      data_ended=data_ended, workspace=workspace)
    if not transfer_started:
        transfer_started = timezone.now()
    if status not in ['QUEUED', 'TRANSFERRING'] and not ingest_started:
        ingest_started = timezone.now()
    if status not in ['QUEUED', 'TRANSFERRING', 'INGESTING'] and not ingest_ended:
        ingest_ended = timezone.now()

    try:
        job_type = Ingest.objects.get_ingest_job_type()
    except:
        job_type = job_utils.create_job_type()
    job = job_utils.create_job(job_type=job_type)
    job_utils.create_job_exe(job=job)

    return Ingest.objects.create(file_name=file_name, file_size=source_file.file_size, status=status, job=job,
                                 bytes_transferred=source_file.file_size, transfer_started=transfer_started,
                                 transfer_ended=transfer_ended, media_type='text/plain', ingest_started=ingest_started,
                                 ingest_ended=ingest_ended, workspace=workspace, strike=strike, source_file=source_file)

    def setUp(self):
        django.setup()

        self.source = source_test_utils.create_source()

        try:
            import ingest.test.utils as ingest_test_utils
            self.ingest = ingest_test_utils.create_ingest(source_file=self.source)
        except:
            self.ingest = None

        try:
            import product.test.utils as product_test_utils
            self.product1 = product_test_utils.create_product(is_superseded=True)

            product_test_utils.create_file_link(ancestor=self.source, descendant=self.product1)
        except:
            self.product1 = None

        try:
            import product.test.utils as product_test_utils
            self.product2 = product_test_utils.create_product()

            product_test_utils.create_file_link(ancestor=self.source, descendant=self.product2)
        except:
            self.product2 = None

    def setUp(self):
        django.setup()

        workspace = storage_test_utils.create_workspace()
        source_file = source_test_utils.create_source(workspace=workspace)
        self.event = trigger_test_utils.create_trigger_event()

        interface_1 = {
            "version": "1.0",
            "command": "test_command",
            "command_arguments": "test_arg",
            "input_data": [{"name": "Test Input 1", "type": "file", "media_types": ["text/plain"]}],
            "output_data": [{"name": "Test Output 1", "type": "files", "media_type": "image/png"}],
        }
        self.job_type_1 = job_test_utils.create_job_type(interface=interface_1)

        interface_2 = {
            "version": "1.0",
            "command": "test_command",
            "command_arguments": "test_arg",
            "input_data": [{"name": "Test Input 2", "type": "files", "media_types": ["image/png", "image/tiff"]}],
            "output_data": [{"name": "Test Output 2", "type": "file"}],
        }
        self.job_type_2 = job_test_utils.create_job_type(interface=interface_2)

        definition = {
            "version": "1.0",
            "input_data": [{"name": "Recipe Input", "type": "file", "media_types": ["text/plain"]}],
            "jobs": [
                {
                    "name": "Job 1",
                    "job_type": {"name": self.job_type_1.name, "version": self.job_type_1.version},
                    "recipe_inputs": [{"recipe_input": "Recipe Input", "job_input": "Test Input 1"}],
                },
                {
                    "name": "Job 2",
                    "job_type": {"name": self.job_type_2.name, "version": self.job_type_2.version},
                    "dependencies": [
                        {"name": "Job 1", "connections": [{"output": "Test Output 1", "input": "Test Input 2"}]}
                    ],
                },
            ],
        }

        recipe_definition = RecipeDefinition(definition)
        recipe_definition.validate_job_interfaces()

        self.recipe_type = recipe_test_utils.create_recipe_type(definition=definition)

        self.data = {
            "version": "1.0",
            "input_data": [{"name": "Recipe Input", "file_id": source_file.id}],
            "workspace_id": workspace.id,
        }

        # Register a fake processor
        self.mock_processor = MagicMock(QueueEventProcessor)
        Queue.objects.register_processor(lambda: self.mock_processor)

    def test_does_not_have_data_types(self):
        """Tests calling IngestTriggerCondition.is_condition_met() with a source file that does not have all required data types"""

        condition = IngestTriggerCondition(None, set(['A', 'B', 'C']))
        source_file = source_test_utils.create_source(media_type='text/plain')
        source_file.add_data_type_tag('A')
        source_file.add_data_type_tag('B')
        
        self.assertEqual(condition.is_condition_met(source_file), False)

    def test_calculate_stats(self):
        '''Tests calculating individual statistics for a metrics entry.'''
        strike = ingest_test_utils.create_strike()
        source_file = source_test_utils.create_source(file_size=200)
        ingest_test_utils.create_ingest(strike=strike, source_file=source_file, status='INGESTED',
                                        transfer_started=datetime.datetime(2015, 1, 1),
                                        transfer_ended=datetime.datetime(2015, 1, 1, 0, 10),
                                        ingest_started=datetime.datetime(2015, 1, 1),
                                        ingest_ended=datetime.datetime(2015, 1, 1, 1))
        ingest_test_utils.create_ingest(strike=strike, status='INGESTED',
                                        transfer_started=datetime.datetime(2015, 1, 1),
                                        transfer_ended=datetime.datetime(2015, 1, 1, 0, 20),
                                        ingest_started=datetime.datetime(2015, 1, 1),
                                        ingest_ended=datetime.datetime(2015, 1, 1, 2))
        ingest_test_utils.create_ingest(strike=strike, status='ERRORED',
                                        transfer_started=datetime.datetime(2015, 1, 1),
                                        transfer_ended=datetime.datetime(2015, 1, 1, 0, 30),
                                        ingest_started=datetime.datetime(2015, 1, 1),
                                        ingest_ended=datetime.datetime(2015, 1, 1, 3))
        ingest_test_utils.create_ingest(strike=strike, status='DEFERRED',
                                        transfer_started=datetime.datetime(2015, 1, 1),
                                        transfer_ended=datetime.datetime(2015, 1, 1, 0, 40),
                                        ingest_started=datetime.datetime(2015, 1, 1),
                                        ingest_ended=datetime.datetime(2015, 1, 1, 4))
        ingest_test_utils.create_ingest(strike=strike, status='DUPLICATE',
                                        transfer_started=datetime.datetime(2015, 1, 1),
                                        transfer_ended=datetime.datetime(2015, 1, 1, 0, 50),
                                        ingest_started=datetime.datetime(2015, 1, 1),
                                        ingest_ended=datetime.datetime(2015, 1, 1, 5))

        MetricsIngest.objects.calculate(datetime.date(2015, 1, 1))

        entries = MetricsIngest.objects.filter(occurred=datetime.date(2015, 1, 1))
        self.assertEqual(len(entries), 1)

        entry = entries.first()
        self.assertEqual(entry.occurred, datetime.date(2015, 1, 1))
        self.assertEqual(entry.deferred_count, 1)
        self.assertEqual(entry.ingested_count, 2)
        self.assertEqual(entry.errored_count, 1)
        self.assertEqual(entry.duplicate_count, 1)
        self.assertEqual(entry.total_count, 5)

        self.assertEqual(entry.file_size_sum, 600)
        self.assertEqual(entry.file_size_min, 100)
        self.assertEqual(entry.file_size_max, 200)
        self.assertEqual(entry.file_size_avg, 120)

        self.assertEqual(entry.transfer_time_sum, 9000)
        self.assertEqual(entry.transfer_time_min, 600)
        self.assertEqual(entry.transfer_time_max, 3000)
        self.assertEqual(entry.transfer_time_avg, 1800)

        self.assertEqual(entry.ingest_time_sum, 10800)
        self.assertEqual(entry.ingest_time_min, 3600)
        self.assertEqual(entry.ingest_time_max, 7200)
        self.assertEqual(entry.ingest_time_avg, 5400)

    def test_media_type_incorrect(self):
        '''Tests calling ParseTriggerCondition.is_condition_met() with a source file that only has the correct data types'''

        condition = ParseTriggerCondition('application/json', set(['A', 'B', 'C']))
        source_file = source_test_utils.create_source(media_type='text/plain')
        source_file.add_data_type_tag('A')
        source_file.add_data_type_tag('B')
        source_file.add_data_type_tag('C')
        
        self.assertEqual(condition.is_condition_met(source_file), False)

    def test_data_types_incorrect(self):
        """Tests calling IngestTriggerCondition.is_condition_met() with a source file that only has the correct media type"""

        condition = IngestTriggerCondition('text/plain', set(['A', 'B', 'C', 'D']))
        source_file = source_test_utils.create_source(media_type='text/plain')
        source_file.add_data_type_tag('A')
        source_file.add_data_type_tag('B')
        source_file.add_data_type_tag('C')
        
        self.assertEqual(condition.is_condition_met(source_file), False)

    def test_both_correct(self):
        """Tests calling IngestTriggerCondition.is_condition_met() with a source file that meets both criteria"""

        condition = IngestTriggerCondition('text/plain', set(['A', 'B', 'C']))
        source_file = source_test_utils.create_source(media_type='text/plain')
        source_file.add_data_type_tag('A')
        source_file.add_data_type_tag('B')
        source_file.add_data_type_tag('C')
        
        self.assertEqual(condition.is_condition_met(source_file), True)

    def test_has_data_types(self):
        '''Tests calling ParseTriggerCondition.is_condition_met() with a source file that has all required data types'''

        condition = ParseTriggerCondition(None, set(['A', 'B', 'C']))
        source_file = source_test_utils.create_source(media_type='text/plain')
        source_file.add_data_type_tag('A')
        source_file.add_data_type_tag('B')
        source_file.add_data_type_tag('C')
        source_file.add_data_type_tag('D')
        source_file.add_data_type_tag('E')
        
        self.assertEqual(condition.is_condition_met(source_file), True)

    def test_uuid_use_input_files(self):
        """Tests setting UUIDs on products with different source input files."""
        source_file2 = source_test_utils.create_source(file_name='input2.txt', workspace=self.workspace)

        products1 = ProductFile.objects.upload_files(self.files, [self.source_file.id], self.job_exe, self.workspace)
        products2 = ProductFile.objects.upload_files(self.files, [source_file2.id], self.job_exe, self.workspace)

        # Make sure the source files are taken into account
        self.assertIsNotNone(products1[0].uuid)
        self.assertIsNotNone(products1[1].uuid)
        self.assertNotEqual(products1[0].uuid, products2[0].uuid)
        self.assertNotEqual(products1[1].uuid, products2[1].uuid)

    def setUp(self):
        django.setup()

        self.source = source_test_utils.create_source()
        self.ancestor = product_test_utils.create_product(file_name='test_ancestor.txt')
        self.descendant = product_test_utils.create_product(file_name='test_descendant.txt')
        self.product = product_test_utils.create_product(file_name='test_product.txt')

        product_test_utils.create_file_link(ancestor=self.source, descendant=self.ancestor)
        product_test_utils.create_file_link(ancestor=self.source, descendant=self.product)
        product_test_utils.create_file_link(ancestor=self.source, descendant=self.descendant)
        product_test_utils.create_file_link(ancestor=self.ancestor, descendant=self.product)
        product_test_utils.create_file_link(ancestor=self.ancestor, descendant=self.descendant)
        product_test_utils.create_file_link(ancestor=self.product, descendant=self.descendant)