Python utils.id_of函数代码示例

def archive(evaluation, destination=None, name=None, query=None):
    """
    Archive the submissions for the given evaluation queue and store them in the destination synapse folder.

    :param evaluation: a synapse evaluation queue or its ID
    :param destination: a synapse folder or its ID
    :param query: a query that will return the desired submissions. At least the ID must be returned.
                  defaults to _select * from evaluation_[EVAL_ID] where status=="SCORED"_.
    """
    challenge = {'5877348':'FusionDetection','5952651':'IsoformQuantification'}
    if not query:
        query = 'select * from evaluation_%s where status=="VALIDATED"' % utils.id_of(evaluation)
    path = challenge[utils.id_of(evaluation)]
    ## for each submission, download it's associated file and write a line of metadata
    results = Query(query=query)
    if 'objectId' not in results.headers:
        raise ValueError("Can't find the required field \"objectId\" in the results of the query: \"{0}\"".format(query))
    for result in results:
        #Check if the folder has already been created in synapse 
        #(This is used as a tool to check submissions that have already been cached)
        submissionId = result[results.headers.index('objectId')]
        check = syn.query('select id,name from folder where parentId == "%s" and name == "%s"' % (destination,submissionId))
        if check['totalNumberOfResults']==0:
            os.mkdir(submissionId)
            submission_parent = syn.store(Folder(submissionId,parent=destination))
            submission = syn.getSubmission(submissionId, downloadLocation=submissionId)
            newFilePath = submission.filePath.replace(' ', '_')
            shutil.move(submission.filePath,newFilePath)
            #Store CWL file in bucket
            os.system('gsutil cp -R %s gs://smc-rna-cache/%s' % (submissionId,path))
            with open(newFilePath,"r") as cwlfile:
                docs = yaml.load(cwlfile)
                merged = docs['$graph']
                docker = []
                for tools in merged:
                    if tools['class'] == 'CommandLineTool':
                        if tools.get('requirements',None) is not None:
                            for i in tools['requirements']:
                                if i.get('dockerPull',None) is not None:
                                    docker.append(i['dockerPull'])
                    if tools['class'] == 'Workflow':
                        hints = tools.get("hints",None)
                        if hints is not None:
                            for i in tools['hints']:
                                if os.path.basename(i['class']) == "synData":
                                    temp = syn.get(i['entity'])
                                    #Store index files
                                    os.system('gsutil cp %s gs://smc-rna-cache/%s/%s' % (temp.path,path,submissionId))
            os.system('rm -rf ~/.synapseCache/*')
            #Pull, save, and store docker containers
            docker = set(docker)
            for i in docker:
                os.system('sudo docker pull %s' % i)
                os.system('sudo docker save %s' % i)
                os.system('sudo docker save -o %s.tar %s' %(os.path.basename(i),i))
                os.system('sudo chmod a+r %s.tar' % os.path.basename(i))
                os.system('gsutil cp %s.tar gs://smc-rna-cache/%s/%s' % (os.path.basename(i),path,submissionId))
                os.remove("%s.tar" % os.path.basename(i))
            os.system('rm -rf %s' % submissionId)

def tableUpdateWhere(tableSchema, whereClause, setDict):
    """ The UPDATE statement is used to update existing rows in a table.
    """
    from synapseclient.table import Table
    from synapseclient.utils import id_of
    import tempfile
    id = id_of(tableSchema)
    query = 'select %s from %s where %s' % (','.join(setDict.keys()), id, whereClause)
    df = syn.tableQuery(query).asDataFrame()
    for key, value in setDict.items():
        df[key] = value
    print(df)
    # df.to_csv('skit.csv')
    return syn.store(Table(id_of(tableSchema), 'skit.csv'))

def archive(evaluation, destination=None, name=None, query=None):
    """
    Archive the submissions for the given evaluation queue and store them in the destination synapse folder.

    :param evaluation: a synapse evaluation queue or its ID
    :param destination: a synapse folder or its ID
    :param query: a query that will return the desired submissions. At least the ID must be returned.
                  defaults to _select * from evaluation_[EVAL_ID] where status=="SCORED"_.
    """
    tempdir = tempfile.mkdtemp()
    archive_dirname = 'submissions_%s' % utils.id_of(evaluation)

    if not query:
        query = 'select * from evaluation_%s where status=="SCORED"' % utils.id_of(evaluation)

    ## for each submission, download it's associated file and write a line of metadata
    results = Query(query=query)
    if not name:
        name = 'submissions_%s.tgz' % utils.id_of(evaluation)
    tar_path = os.path.join(tempdir, name)
    metadata_file_path = os.path.join(tempdir, 'submission_metadata.csv')
    print "creating tar at:", tar_path

    ## for each submission, we add a file to the tar and a row
    ## to the metadata .csv file
    with tarfile.open(tar_path, mode='w:gz') as archive:
        with open(metadata_file_path, 'w') as f:

            ## write header row to .csv file
            header = ','.join(results.headers)
            print header
            f.write(header + '\n')

            ## add submissions to archive and write rows to .csv file 
            for result in results:
                ## retrieve file into cache and copy it to destination
                submission = syn.getSubmission(result[results.headers.index('objectId')])
                archive.add(submission.filePath, arcname=os.path.join(archive_dirname, submission.id + "_" + os.path.basename(submission.filePath)))
                line = (','.join(unicode(item) for item in result)).encode('utf-8')
                print line
                f.write(line + '\n')

        ## add metadata .csv file to the tar
        archive.add(
            name=metadata_file_path,
            arcname=os.path.join(archive_dirname, 'submission_metadata.csv'))

    entity = syn.store(File(tar_path, parent=destination), evaluation_id=utils.id_of(evaluation))
    print "created:", entity.id, entity.name
    return entity.id

def test_id_of():
    assert utils.id_of(1) == '1'
    assert utils.id_of('syn12345') == 'syn12345'
    assert utils.id_of({'foo':1, 'id':123}) == 123
    assert_raises(ValueError, utils.id_of, {'foo':1, 'idzz':123})
    assert utils.id_of({'properties':{'id':123}}) == 123
    assert_raises(ValueError, utils.id_of, {'properties':{'qq':123}})
    assert_raises(ValueError, utils.id_of, object())

    class Foo:
        def __init__(self, id):
            self.properties = {'id':id}

    foo = Foo(123)
    assert utils.id_of(foo) == 123

def test_id_of():
    assert utils.id_of(1) == "1"
    assert utils.id_of("syn12345") == "syn12345"
    assert utils.id_of({"foo": 1, "id": 123}) == 123
    assert_raises(ValueError, utils.id_of, {"foo": 1, "idzz": 123})
    assert utils.id_of({"properties": {"id": 123}}) == 123
    assert_raises(ValueError, utils.id_of, {"properties": {"qq": 123}})
    assert_raises(ValueError, utils.id_of, object())

    class Foo:
        def __init__(self, id):
            self.properties = {"id": id}

    foo = Foo(123)
    assert utils.id_of(foo) == 123

def list_evaluations(project):
    print '\n\nEvaluations for project: ', utils.id_of(project)
    print '-' * 60

    evaluations = syn.getEvaluationByContentSource(project)
    for evaluation in evaluations:
        print "Evaluation: %s" % evaluation.id, evaluation.name.encode('utf-8')

def test_id_of():
    assert_equals(utils.id_of(1), '1')
    assert_equals(utils.id_of('syn12345'), 'syn12345')
    assert_equals(utils.id_of({'foo': 1, 'id': 123}), '123')
    assert_raises(ValueError, utils.id_of, {'foo': 1, 'idzz': 123})
    assert_equals(utils.id_of({'properties': {'id': 123}}), '123')
    assert_raises(ValueError, utils.id_of, {'properties': {'qq': 123}})
    assert_raises(ValueError, utils.id_of, object())

    class Foo:
        def __init__(self, id_attr_name, id):
            self.properties = {id_attr_name: id}

    id_attr_names = ['id', 'ownerId', 'tableId']

    for attr_name in id_attr_names:
        foo = Foo(attr_name, 123)
        assert_equals(utils.id_of(foo), '123')

    def __init__(self, **kwargs):
        #Verify that the parameters are correct
        if not 'owner' in kwargs:
            sys.stderr.write('Wiki constructor must have an owner specified')
            raise ValueError

        super(Wiki, self).__init__(kwargs)
        self.ownerType=guess_object_type(self.owner)
        self.ownerId=id_of(self.owner)
        del self['owner']

 def removeColumn(self, column):
     """
     :param column: a column object or its ID
     """
     if isinstance(column, basestring) or isinstance(column, int) or hasattr(column, 'id'):
         self.properties.columnIds.remove(utils.id_of(column))
     elif isinstance(column, Column) and self.columns_to_store:
         self.columns_to_store.remove(column)
     else:
         ValueError("Can't remove column %s" + unicode(column))

def archive(evaluation, destination=None, name=None, query=None):
    """
    Archive the submissions for the given evaluation queue and store them in the destination synapse folder.

    :param evaluation: a synapse evaluation queue or its ID
    :param destination: a synapse folder or its ID
    :param query: a query that will return the desired submissions. At least the ID must be returned.
                  defaults to _select * from evaluation_[EVAL_ID] where status=="SCORED"_.
    """
    tempdir = tempfile.mkdtemp()
    archive_dirname = "submissions_%s" % utils.id_of(evaluation)

    if not query:
        query = 'select * from evaluation_%s where status=="SCORED"' % utils.id_of(evaluation)

    ## for each submission, download it's associated file and write a line of metadata
    results = Query(query=query)
    if "objectId" not in results.headers:
        raise ValueError('Can\'t find the required field "objectId" in the results of the query: "{0}"'.format(query))
    if not name:
        name = "submissions_%s.tgz" % utils.id_of(evaluation)
    tar_path = os.path.join(tempdir, name)
    print "creating tar at:", tar_path
    print results.headers
    with tarfile.open(tar_path, mode="w:gz") as archive:
        with open(os.path.join(tempdir, "submission_metadata.csv"), "w") as f:
            f.write((",".join(hdr for hdr in (results.headers + ["filename"])) + "\n").encode("utf-8"))
            for result in results:
                ## retrieve file into cache and copy it to destination
                submission = syn.getSubmission(result[results.headers.index("objectId")])
                prefixed_filename = submission.id + "_" + os.path.basename(submission.filePath)
                archive.add(submission.filePath, arcname=os.path.join(archive_dirname, prefixed_filename))
                line = (",".join(unicode(item) for item in (result + [prefixed_filename]))).encode("utf-8")
                print line
                f.write(line + "\n")
        archive.add(
            name=os.path.join(tempdir, "submission_metadata.csv"),
            arcname=os.path.join(archive_dirname, "submission_metadata.csv"),
        )

    entity = syn.store(File(tar_path, parent=destination), evaluation_id=utils.id_of(evaluation))
    print "created:", entity.id, entity.name
    return entity.id

    def __init__(self, properties=None, annotations=None, local_state=None, parent=None, **kwargs):

        if properties:
            if isinstance(properties, collections.Mapping):
                if "annotations" in properties and isinstance(properties["annotations"], collections.Mapping):
                    annotations.update(properties["annotations"])
                    del properties["annotations"]
                self.__dict__["properties"].update(properties)
            else:
                raise SynapseMalformedEntityError("Unknown argument type: properties is a %s" % str(type(properties)))

        if annotations:
            if isinstance(annotations, collections.Mapping):
                self.__dict__["annotations"].update(annotations)
            elif isinstance(annotations, str):
                self.properties["annotations"] = annotations
            else:
                raise SynapseMalformedEntityError("Unknown argument type: annotations is a %s" % str(type(annotations)))

        if local_state:
            if isinstance(local_state, collections.Mapping):
                self.local_state(local_state)
            else:
                raise SynapseMalformedEntityError("Unknown argument type: local_state is a %s" % str(type(local_state)))

        for key in self.__class__._local_keys:
            if key not in self.__dict__:
                self.__dict__[key] = None

        # Extract parentId from parent
        if "parentId" not in kwargs:
            if parent:
                try:
                    kwargs["parentId"] = id_of(parent)
                except Exception:
                    if isinstance(parent, Entity) and "id" not in parent:
                        raise SynapseMalformedEntityError(
                            "Couldn't find 'id' of parent.  Has it been stored in Synapse?"
                        )
                    else:
                        raise SynapseMalformedEntityError("Couldn't find 'id' of parent.")

        # Note: that this will work properly if derived classes declare their
        # internal state variable *before* invoking super(...).__init__(...)
        for key, value in six.iteritems(kwargs):
            self.__setitem__(key, value)

        if "concreteType" not in self:
            self["concreteType"] = self.__class__._synapse_entity_type

        ## Only project can be top-level. All other entity types require parentId
        ## don't enforce this for generic Entity
        if "parentId" not in self and not isinstance(self, Project) and not type(self) == Entity:
            raise SynapseMalformedEntityError("Entities of type %s must have a parentId." % type(self))

 def addColumn(self, column):
     """
     :param column: a column object or its ID
     """
     if isinstance(column, basestring) or isinstance(column, int) or hasattr(column, 'id'):
         self.properties.columnIds.append(utils.id_of(column))
     elif isinstance(column, Column):
         if not self.__dict__.get('columns_to_store', None):
             self.__dict__['columns_to_store'] = []
         self.__dict__['columns_to_store'].append(column)
     else:
         raise ValueError("Not a column? %s" % unicode(column))

def syncFromSynapse(syn, entity, path=None, ifcollision='overwrite.local', allFiles = None):
    """Synchronizes all the files in a folder (including subfolders) from Synapse.

    :param syn:    A synapse object as obtained with syn = synapseclient.login()

    :param entity:  A Synapse ID, a Synapse Entity object of type folder or project.

    :param path: An optional path where the file hierarchy will be
                 reproduced.  If not specified the files will by default
                 be placed in the synapseCache.

    :param ifcollision:   Determines how to handle file collisions.
                          May be "overwrite.local", "keep.local", or "keep.both".
                          Defaults to "overwrite.local".


    :returns: list of entities (files, tables, links)

    This function will crawl all subfolders of the project/folder
    specified by `id` and download all files that have not already
    been downloaded.  If there are newer files in Synapse (or a local
    file has been edited outside of the cache) since the last download
    then local the file will be replaced by the new file unless
    ifcollision is changed.

    Example::
    Download and print the paths of all downloaded files::

        entities = syncFromSynapse(syn, "syn1234")
        for f in entities:
            print(f.path)
    """
    if allFiles is None: allFiles = list()
    id = id_of(entity)
    results = syn.chunkedQuery("select id, name, nodeType from entity where entity.parentId=='%s'" %id)
    for result in results:
        if is_container(result):
            if path is not None:  #If we are downloading outside cache create directory.
                new_path = os.path.join(path, result['entity.name'])
                try:
                    os.mkdir(new_path)
                except OSError as err:
                    if err.errno!=errno.EEXIST:
                        raise
                print('making dir', new_path)
            else:
                new_path = None
            syncFromSynapse(syn, result['entity.id'], new_path, ifcollision, allFiles)
        else:
            ent = syn.get(result['entity.id'], downloadLocation = path, ifcollision = ifcollision)
            allFiles.append(ent)
    return allFiles

 def __init__(self, name=None, columns=None, parent=None, properties=None, annotations=None, local_state=None, **kwargs):
     self.properties.setdefault('columnIds',[])
     if name: kwargs['name'] = name
     if columns:
         for column in columns:
             if isinstance(column, basestring) or isinstance(column, int) or hasattr(column, 'id'):
                 kwargs.setdefault('columnIds',[]).append(utils.id_of(column))
             elif isinstance(column, Column):
                 kwargs.setdefault('columns_to_store',[]).append(column)
             else:
                 raise ValueError("Not a column? %s" % unicode(column))
     super(Schema, self).__init__(concreteType=Schema._synapse_entity_type, properties=properties, 
                                annotations=annotations, local_state=local_state, parent=parent, **kwargs)

    def setup(self):
        self.eval_id = '9090'
        self.contributors = None
        self.entity = {
            'versionNumber': 7,
            'id': 'syn1009',
            'etag': 'etag',
            'name': 'entity name'
        }
        self.eval = {
            'contentSource': self.entity['id'],
            'createdOn': '2013-11-06T06:04:26.789Z',
            'etag': '86485ea1-8c89-4f24-a0a4-2f63bc011091',
            'id': self.eval_id,
            'name': 'test evaluation',
            'ownerId': '1560252',
            'status': 'OPEN',
            'submissionReceiptMessage': 'Your submission has been received.!'
        }
        self.team = {
            'id': 5,
            'name': 'Team Blue'
        }
        self.submission = {
            'id': 123,
            'evaluationId': self.eval_id,
            'name': self.entity['name'],
            'entityId': self.entity['id'],
            'versionNumber': self.entity['versionNumber'],
            'teamId': id_of(self.team['id']),
            'contributors': self.contributors,
            'submitterAlias': self.team['name']
        }
        self.eligibility_hash = 23

        self.patch_private_submit = patch.object(syn, "_submit", return_value=self.submission)
        self.patch_getEvaluation = patch.object(syn, "getEvaluation", return_value=self.eval)
        self.patch_get = patch.object(syn, "get", return_value=self.entity)
        self.patch_getTeam = patch.object(syn, "getTeam", return_value= self.team)
        self.patch_get_contributors = patch.object(syn, "_get_contributors",
                                                   return_value=(self.contributors, self.eligibility_hash))

        self.mock_private_submit = self.patch_private_submit.start()
        self.mock_getEvaluation = self.patch_getEvaluation.start()
        self.mock_get = self.patch_get.start()
        self.mock_getTeam = self.patch_getTeam.start()
        self.mock_get_contributors = self.patch_get_contributors.start()