Python Location.get_location方法代码示例

# 需要导入模块: from tardis.tardis_portal.models import Location [as 别名]
# 或者: from tardis.tardis_portal.models.Location import get_location [as 别名]
    def testMigrateStoreWithSpaces(self):
        dest = Location.get_location('test')
        local = Location.get_location('local')

        datafile, replica = generate_datafile('1/1/Hi Mum', self.dataset,
                                              "Hi mum")
        datafile2, replica2 = generate_datafile('1/1/Hi Dad', self.dataset,
                                                "Hi dad")

        path = datafile.get_absolute_filepath()
        self.assertTrue(os.path.exists(path))
        path2 = datafile.get_absolute_filepath()
        self.assertTrue(os.path.exists(path2))

        # Migrate them
        migrate_replica(replica, dest)
        self.assertFalse(os.path.exists(path))
        migrate_replica(replica2, dest)
        self.assertFalse(os.path.exists(path2))

        # Bring them back
        migrate_replica(datafile.get_preferred_replica(), local)
        self.assertTrue(os.path.exists(path))
        migrate_replica(datafile2.get_preferred_replica(), local)
        self.assertTrue(os.path.exists(path2))

# 需要导入模块: from tardis.tardis_portal.models import Location [as 别名]
# 或者: from tardis.tardis_portal.models.Location import get_location [as 别名]
 def testReplicaVerify(self):
     from django.conf import settings
     saved = settings.REQUIRE_DATAFILE_CHECKSUMS
     try:
         Location.get_location('test')
         datafile, replica = generate_datafile("1/2/3", self.dataset,
                                               "Hi mum")
         settings.REQUIRE_DATAFILE_CHECKSUMS = True
         self.assertTrue(replica.verify(), 'Replica.verify() failed.')
         replica.datafile.sha512sum = ''
         replica.datafile.md5sum = ''
         self.assertFalse(
             replica.verify(),
             'Replica.verify() succeeded despite no checksum '
             '(settings.REQUIRE_DATAFILE_CHECKSUMS=True).')
         self.assertFalse(replica.verify(allowEmptyChecksums=False),
                          'Replica.verify() succeeded despite no checksum '
                          '(allowEmptyChecksums=False)')
         settings.REQUIRE_DATAFILE_CHECKSUMS = False
         datafile.sha512sum = None
         datafile.md5sum = None
         self.assertTrue(replica.verify(allowEmptyChecksums=True),
                         'Replica.verify() failed wrongly '
                         '(allowEmptyChecksums=True)')
         datafile.sha512sum = None
         datafile.md5sum = None
         self.assertTrue(replica.verify(),
                         'Replica.verify() failed wrongly')
     finally:
         settings.REQUIRE_DATAFILE_CHECKSUMS = saved

# 需要导入模块: from tardis.tardis_portal.models import Location [as 别名]
# 或者: from tardis.tardis_portal.models.Location import get_location [as 别名]
    def testRemoveExperimentData(self):
        # First with no sharing
        self._build()
        archive_location = Location.get_location('archtest')
        try:
            nos_experiments = Experiment.objects.count()
            nos_datasets = Dataset.objects.count()
            nos_datafiles = Dataset_File.objects.count()
            nos_replicas = Replica.objects.count()
            self.assertTrue(exists(self.replica.get_absolute_filepath()))
            remove_experiment_data(self.experiment, 
                                   'http://example.com/some.tar.gz',
                                   archive_location)
            self.assertEquals(nos_experiments, Experiment.objects.count())
            self.assertEquals(nos_datasets, Dataset.objects.count())
            self.assertEquals(nos_datafiles, Dataset_File.objects.count())
            self.assertEquals(nos_replicas, Replica.objects.count())
            new_replica = self.datafile.get_preferred_replica()
            self.assertTrue(self.replica.id != new_replica.id)
            self.assertFalse(new_replica.stay_remote)
            self.assertTrue(new_replica.verified)
            self.assertEqual(self.replica.protocol, new_replica.protocol)
            self.assertEqual(archive_location.id, new_replica.location.id)
            self.assertEqual('http://example.com/some.tar.gz#1/1/1',
                             new_replica.url)
            self.assertFalse(exists(self.replica.get_absolute_filepath()))
        finally:
            self._clear()

        # (Check that the deletes cascaded ... )
        self.assertEquals(0, Dataset_File.objects.count())
        self.assertEquals(0, Replica.objects.count())
        
        # Repeat, but with the first dataset in 2 experiments.
        self._build()
        self.dataset.experiments.add(self.experiment2)
        archive_location = Location.get_location('archtest')
        try:
            nos_experiments = Experiment.objects.count()
            nos_datasets = Dataset.objects.count()
            nos_datafiles = Dataset_File.objects.count()
            nos_replicas = Replica.objects.count()
            self.assertTrue(exists(self.replica.get_absolute_filepath()))
            remove_experiment_data(self.experiment, 
                                   'http://example.com/some.tar.gz',
                                   archive_location)
            self.assertEquals(nos_experiments, Experiment.objects.count())
            self.assertEquals(nos_datasets, Dataset.objects.count())
            self.assertEquals(nos_datafiles, Dataset_File.objects.count())
            self.assertEquals(nos_replicas, Replica.objects.count())
            new_replica = self.datafile.get_preferred_replica()
            self.assertTrue(self.replica.id == new_replica.id)
            self.assertTrue(exists(self.replica.get_absolute_filepath()))
            self.assertFalse(exists(self.replica2.get_absolute_filepath()))
        finally:
            self._clear()

# 需要导入模块: from tardis.tardis_portal.models import Location [as 别名]
# 或者: from tardis.tardis_portal.models.Location import get_location [as 别名]
 def test_location(self):
     from tardis.tardis_portal.models import Location
     self.assertEquals(Location.get_default_location().name,
                       'local')
     self.assertEquals(Location.get_location('staging').name,
                       'staging')
     self.assertEquals(len(Location.objects.all()), 6)

# 需要导入模块: from tardis.tardis_portal.models import Location [as 别名]
# 或者: from tardis.tardis_portal.models.Location import get_location [as 别名]
 def testScoring(self):
     self._setup()
     scorer = MigrationScorer(Location.get_location('local').id)
     self.assertEquals(2.0, scorer.datafile_score(self.df1))
     self.assertEquals(2, get_user_priority(self.user1))
     self.assertEquals(1, get_user_priority(self.user2))
     self.assertEquals(1.0, scorer.user_score(self.user1))
     self.assertEquals(2.0, scorer.user_score(self.user2))
     self.assertEquals(2.0, scorer.experiment_score(self.exp1))
     self.assertEquals(2.0, scorer.dataset_score(self.df1.dataset))
     self.assertEquals(4.0, scorer.score_datafile(self.df1))
     self.assertEquals([(self.df1, self.rep1, 4.0)], 
                       scorer.score_datafiles_in_dataset(self.ds1))
     self.assertEquals([(self.df5, self.rep5, 8.0), 
                        (self.df4, self.rep4, 6.0), 
                        (self.df1, self.rep1, 4.0)],
                       scorer.score_datafiles_in_experiment(self.exp1))
     self.assertEquals([(self.df5, self.rep5, 8.0), 
                        (self.df4, self.rep4, 6.0)],
                       scorer.score_datafiles_in_experiment(self.exp2))
     self.assertEquals([(self.df6, self.rep6, 5.0)],
                       scorer.score_datafiles_in_experiment(self.exp3))
     self.assertEquals([(self.df5, self.rep5, 8.0), 
                        (self.df4, self.rep4, 6.0), 
                        (self.df6, self.rep6, 5.0), 
                        (self.df1, self.rep1, 4.0), 
                        (self.df7, self.rep7, 0.0), 
                        (self.df8, self.rep8, 0.0)],
                       scorer.score_all_datafiles())
     self.assertEquals([(self.df7, self.rep7, 0.0), 
                        (self.df8, self.rep8, 0.0)], 
                       scorer.score_datafiles_in_dataset(self.ds4))

# 需要导入模块: from tardis.tardis_portal.models import Location [as 别名]
# 或者: from tardis.tardis_portal.models.Location import get_location [as 别名]
    def testScoringWithTimes(self):
        self._setup()
        scorer = MigrationScorer(
            Location.get_location('local').id, {
                'user_priority_weighting': [5.0, 2.0, 1.0, 0.5, 0.2],
                'file_size_weighting': 1.0,
                'file_access_weighting': 1.0,
                'file_age_weighting': 1.0,
                'file_size_threshold': 0,
                'file_access_threshold': 0,
                'file_age_threshold': 1})
        
        self.assertEquals(0.0, scorer.datafile_score(self.df1))
     
        f = tempfile.NamedTemporaryFile(dir=settings.FILE_STORE_PATH)
        f.write("Hi Mom!!\n")
        rep = Replica.objects.get(pk=self.rep1.pk)
        rep.url = f.name
        rep.save()

        self.assertEquals(2.0, scorer.datafile_score(self.df1))
        
        older = time.time() - (60 * 60 * 24 + 300)
        os.utime(f.name, (older, older))

        self.assertEquals(3.0, scorer.datafile_score(self.df1))

        older = time.time() - (60 * 60 * 24 * 2 + 300)
        os.utime(f.name, (older, older))

        self.assertEquals(5.0, scorer.datafile_score(self.df1))

        f.close()

# 需要导入模块: from tardis.tardis_portal.models import Location [as 别名]
# 或者: from tardis.tardis_portal.models.Location import get_location [as 别名]
def remove_experiment_data(exp, archive_url, archive_location):
    """Remove the online Replicas for an Experiment that are not shared with
    other Experiments.  When Replicas are removed, they are replaced with
    offline replicas whose 'url' consists of the archive_url, with the 
    archive pathname for the datafile as a url fragment id.
    """
    for ds in Dataset.objects.filter(experiments=exp):
        if ds.experiments.count() == 1:
            for df in Dataset_File.objects.filter(dataset=ds):
                replicas = Replica.objects.filter(datafile=df, 
                                                  location__type='online')
                if replicas.count() > 0:
                    for replica in replicas:
                        location = Location.get_location(replica.location.name)
                        location.provider.remove_file(replica)
                        if archive_url:
                            old_replica = replicas[0]
                            path_in_archive = '%s/%s/%s' % (
                                exp.id, ds.id, df.filename)
                            new_replica_url = '%s#%s' % (
                                archive_url, quote(path_in_archive))
                            new_replica = Replica(datafile=old_replica.datafile,
                                                  url=new_replica_url,
                                                  protocol=old_replica.protocol,
                                                  verified=True,
                                                  stay_remote=False,
                                                  location=archive_location)
                            new_replica.save()
                    replicas.delete()

# 需要导入模块: from tardis.tardis_portal.models import Location [as 别名]
# 或者: from tardis.tardis_portal.models.Location import get_location [as 别名]
    def testMigrateRestore(self):
        dest = Location.get_location('test')
        local = Location.get_location('local')
        datafile, replica = generate_datafile(None, self.dataset, "Hi mum",
                                              verify=False,
                                              verify_checksums_req=True)

        # Attempt to migrate without datafile hashes ... should
        # fail because we can't verify.
        with self.assertRaises(MigrationError):
            migrate_replica(replica, dest)

        # Verify sets hashes ...
        self.assertEquals(replica.verify(allowEmptyChecksums=True), True)
        replica = Replica.objects.get(pk=replica.pk)
        path = datafile.get_absolute_filepath()
        self.assertTrue(os.path.exists(path))
        self.assertTrue(migrate_replica(replica, dest))
        self.assertFalse(os.path.exists(path))

        # Bring it back
        new_replica = datafile.get_preferred_replica()
        url = new_replica.url
        self.assertTrue(migrate_replica(new_replica, local))
        self.assertTrue(os.path.exists(path))
        # Check it was deleted remotely
        with self.assertRaises(TransferError):
            dest.provider.get_length(new_replica)

        # Refresh the datafile object because it is now stale ...
        datafile = Dataset_File.objects.get(id=datafile.id)
        replica = datafile.get_preferred_replica()

        # Repeat the process with 'noRemove'
        self.assertTrue(migrate_replica(replica, dest, noRemove=True))
        new_replica = datafile.get_preferred_replica()
        self.assertTrue(os.path.exists(path))
        self.assertEquals(dest.provider.get_length(new_replica), 6)
        migrate_replica(new_replica, local, noRemove=True)
        newpath = datafile.get_absolute_filepath()
        replica = datafile.get_preferred_replica()
        self.assertTrue(os.path.exists(path))
        self.assertTrue(os.path.exists(newpath))
        self.assertNotEqual(path, newpath)
        self.assertEquals(dest.provider.get_length(new_replica), 6)

# 需要导入模块: from tardis.tardis_portal.models import Location [as 别名]
# 或者: from tardis.tardis_portal.models.Location import get_location [as 别名]
 def do_ext_provider(self, loc_name):
     # This test requires an external test server configured
     # as per the 'dest_name' destination.  We skip the test is the
     # server doesn't respond.
     loc = Location.get_location(loc_name)
     if loc.provider.alive():
         self.do_provider(loc)
     else:
         print "SKIPPING TEST - %s server on %s is not responding\n" % (loc_name, loc.url)

# 需要导入模块: from tardis.tardis_portal.models import Location [as 别名]
# 或者: from tardis.tardis_portal.models.Location import get_location [as 别名]
    def testArchiveExperiment(self):
        dataset = generate_dataset()
        experiment = generate_experiment([dataset], [self.dummy_user])
        datafile, _ = generate_datafile(None, dataset, "Hi grandpa")
        archtest = Location.get_location('archtest')

        # Dry run ...
        out = StringIO()
        try:
            call_command('archive', experiment.id, 
                         verbosity=1, stdout=out, dryRun=True)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(), 
                          'Would have archived experiment %s\n' % experiment.id)

        # Dry run ... all
        out = StringIO()
        try:
            call_command('archive', all=True,
                         verbosity=1, stdout=out, dryRun=True)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(), 
                          'Would have archived experiment %s\n' % experiment.id)

        # Do one ... to file
        out = StringIO()
        try:
            call_command('archive', experiment.id, directory='/tmp',
                         verbosity=1, stdout=out)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(
            out.read(), 
            'Archived experiment %s to /tmp/%s-archive.tar.gz\n' \
            'Archived 1 experiments with 0 errors\n' % \
                (experiment.id, experiment.id))

        # Do one ... to archtest
        out = StringIO()
        try:
            call_command('archive', experiment.id, location='archtest',
                         verbosity=1, stdout=out)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(
            out.read(), 
            'Archived experiment %s to %s%s-archive.tar.gz\n' \
            'Archived 1 experiments with 0 errors\n' % \
                (experiment.id, archtest.provider.base_url, 
                 experiment.id))

# 需要导入模块: from tardis.tardis_portal.models import Location [as 别名]
# 或者: from tardis.tardis_portal.models.Location import get_location [as 别名]
 def _get_destination(self, destName, default):
     if not destName:
         if not default:
             raise CommandError("No default destination configured")
         else:
             destName = default
     dest = Location.get_location(destName)
     if not dest:
         raise CommandError("Destination %s not known" % destName)
     return dest

# 需要导入模块: from tardis.tardis_portal.models import Location [as 别名]
# 或者: from tardis.tardis_portal.models.Location import get_location [as 别名]
    def testMigrationNoHashes(self):
        # Tweak the server to turn off the '?metadata' query
        self.server.server.allowQuery = False

        dest = Location.get_location('test')
        datafile, replica = generate_datafile("1/2/3", self.dataset, "Hi mum")
        self.assertEquals(replica.verify(allowEmptyChecksums=True), True)
        path = datafile.get_absolute_filepath()
        self.assertTrue(os.path.exists(path))
        migrate_replica(replica, dest)
        self.assertFalse(os.path.exists(path))

# 需要导入模块: from tardis.tardis_portal.models import Location [as 别名]
# 或者: from tardis.tardis_portal.models.Location import get_location [as 别名]
    def testProviderInstantiation(self):
        """
        Test that transfer_provider instantiation works
        """

        provider = Location.get_location("test").provider
        self.assertIsInstance(provider, TransferProvider)
        self.assertIsInstance(provider, SimpleHttpTransfer)
        self.assertEqual(provider.base_url, "http://127.0.0.1:4272/data/")

        provider = Location.get_location("test2").provider
        self.assertIsInstance(provider, TransferProvider)
        self.assertIsInstance(provider, WebDAVTransfer)
        self.assertFalse(401 in provider.opener.handle_error["http"])
        self.assertEqual(provider.base_url, "http://127.0.0.1/data2/")

        provider = Location.get_location("test3").provider
        self.assertIsInstance(provider, TransferProvider)
        self.assertIsInstance(provider, WebDAVTransfer)
        self.assertTrue(401 in provider.opener.handle_error["http"])
        self.assertEqual(provider.base_url, "http://127.0.0.1/data3/")

# 需要导入模块: from tardis.tardis_portal.models import Location [as 别名]
# 或者: from tardis.tardis_portal.models.Location import get_location [as 别名]
 def _get_destination(self, destName, default):
     if not destName:
         if not default:
             raise CommandError("No default destination configured")
         else:
             destName = default
     try:
         dest = Location.get_location(destName)
         if not dest:
             raise CommandError("Destination %s not known" % destName)
         return dest
     except MigrationError as e:
         raise CommandError("Migration error: %s" % e.args[0])

# 需要导入模块: from tardis.tardis_portal.models import Location [as 别名]
# 或者: from tardis.tardis_portal.models.Location import get_location [as 别名]
    def testProviderInstantiation(self):
        '''
        Test that transfer_provider instantiation works
        '''

        provider = Location.get_location('test').provider
        self.assertIsInstance(provider, TransferProvider)
        self.assertIsInstance(provider, SimpleHttpTransfer)
        self.assertEqual(provider.base_url, 'http://127.0.0.1:4272/data/')
        
        provider = Location.get_location('test2').provider
        self.assertIsInstance(provider, TransferProvider)
        self.assertIsInstance(provider, WebDAVTransfer)
        self.assertFalse(401 in provider.opener.handle_error['http'])
        self.assertEqual(provider.base_url, 'http://127.0.0.1/data2/')
        
        provider = Location.get_location('test3').provider
        self.assertIsInstance(provider, TransferProvider)
        self.assertIsInstance(provider, WebDAVTransfer)
        self.assertTrue(401 in provider.opener.handle_error['http'])
        self.assertEqual(provider.base_url, 'http://127.0.0.1/data3/')

        provider = Location.get_location('scptest').provider
        self.assertIsInstance(provider, ScpTransfer)