Python synapseclient.File类代码示例

def upload(args,syn):
	if args.dataType == "rnaseq":
		parentId = "syn6034916"
		pipeline = "syn6126122"
		dataType = "RNASeq"
	elif args.dataType == "dnaseq":
		parentId = "syn6034751"
		pipeline = "syn6126123"
		dataType = "TargDNASeq"
	elif args.dataType == "snparray":
		parentId = "syn6038475"
		pipeline = "syn6126121"
		dataType = "SNParray"
	elif args.dataType == "exparray":
		parentId = "syn6038915"
		pipeline = "syn6126120"
		dataType = "expression_microarray"
	elif args.dataType == "exome":
		parentId = "syn6115597"
		dataType = "exome"
		pipeline = ""
	else:
		raise ValueError("dataType needs to be rnaseq/dnaseq/snparray/exparray/exome")
	if args.workflow is not None:
		workflow = syn.get(pipeline,downloadFile=False)
		workflow.path = args.workflow
		workflow.name = os.path.basename(args.workflow)
		workflow = syn.store(workflow)
		pipeline = workflow.id
	fileEnt = File(args.input,parent=parentId)
	#fileEnt.annotations = temp.to_dict('index').values()[0]
	fileEnt.dataType = dataType
	fileEnt.sampleId = sampleId
	fileEnt = syn.store(fileEnt,used = pipeline)
	return(fileEnt.id)

def test_store_with_create_or_update_flag():
    project = create_project()

    filepath = utils.make_bogus_binary_file()
    bogus1 = File(filepath, name='Bogus Test File', parent=project)

    bogus1 = syn.store(bogus1, createOrUpdate=True)

    # Create a different file with the same name and parent
    new_filepath = utils.make_bogus_binary_file()
    bogus1.path = new_filepath

    # Expected behavior is that a new version of the first File will be created
    bogus2 = syn.store(bogus1, createOrUpdate=True)

    assert bogus2.id == bogus1.id
    assert bogus2.versionNumber == 2
    assert not filecmp.cmp(bogus2.path, filepath)

    bogus2a = syn.get(bogus2.id)
    assert bogus2a.id == bogus1.id
    assert bogus2a.versionNumber == 2
    assert filecmp.cmp(bogus2.path, bogus2a.path)

    # Create yet another file with the same name and parent
    newer_filepath = utils.make_bogus_binary_file()
    bogus3 = File(newer_filepath, name='Bogus Test File', parent=project)

    # Expected behavior is raising an exception with a 409 error
    assert_raises(requests.exceptions.HTTPError, syn.store, bogus3, createOrUpdate=False)

def test_multipart_upload_big_string():
    cities = ["Seattle", "Portland", "Vancouver", "Victoria",
              "San Francisco", "Los Angeles", "New York",
              "Oaxaca", "Cancún", "Curaçao", "जोधपुर",
              "অসম", "ལྷ་ས།", "ཐིམ་ཕུ་", "دبي", "አዲስ አበባ",
              "São Paulo", "Buenos Aires", "Cartagena",
              "Amsterdam", "Venice", "Rome", "Dubrovnik",
              "Sarajevo", "Madrid", "Barcelona", "Paris",
              "Αθήνα", "Ρόδος", "København", "Zürich",
              "金沢市", "서울", "แม่ฮ่องสอน", "Москва"]

    text = "Places I wanna go:\n"
    while len(text.encode('utf-8')) < multipart_upload_module.MIN_PART_SIZE:
        text += ", ".join( random.choice(cities) for i in range(5000) ) + "\n"

    fhid = multipart_upload_string(syn, text)
    print('FileHandle: {fhid}'.format(fhid=fhid))

    # Download the file and compare it with the original
    junk = File("message.txt", parent=project, dataFileHandleId=fhid)
    junk.properties.update(syn._createEntity(junk.properties))
    (tmp_f, tmp_path) = tempfile.mkstemp()
    schedule_for_cleanup(tmp_path)
    junk.update(syn._downloadFileEntity(junk, tmp_path))

    with open(junk.path, encoding='utf-8') as f:
        retrieved_text = f.read()

    assert retrieved_text == text

def test_round_trip():
    fh = None
    filepath = utils.make_bogus_binary_file(6*MB + 777771, verbose=True)
    print 'Made bogus file: ', filepath
    try:
        fh = syn._chunkedUploadFile(filepath, verbose=False)
        # print 'FileHandle:'
        # syn.printEntity(fh)

        # Download the file and compare it with the original
        junk = File(filepath, parent=project, dataFileHandleId=fh['id'])
        junk.properties.update(syn._createEntity(junk.properties))
        junk.update(syn._downloadFileEntity(junk, filepath))
        assert filecmp.cmp(filepath, junk.path)

    finally:
        try:
            if 'junk' in locals():
                syn.delete(junk)
        except Exception:
            print traceback.format_exc()
        try:
            os.remove(filepath)
        except Exception:
            print traceback.format_exc()
        if fh:
            # print 'Deleting fileHandle', fh['id']
            syn._deleteFileHandle(fh)

def uploadToSynapse(f):
    """Given a filepath extracts metadata and uploads to Synapse"""
    center, sample_id, workflow_name, date, call_type, dataType, fileType = ['']*7
    url = URLBASE+f
    if   'OICR_BL' in f: center = 'oicr_bl'
    elif 'CRG/clindel/somatic' in f:  center = 'crg_clindel'
    else: center = f.split('/')[4]
    filename =  f.split('/')[-1]
    if center in ('yale', 'wustl', 'LOHcomplete'):
        if filename =='bd829214-f230-4331-b234-def10bbe7938CNV.vcf.gz':
            sample_id, dataType, fileType='bd829214-f230-4331-b234-def10bbe7938', 'cnv', 'vcf'
        else:
            sample_id, dataType = filename.lower().split('.')[:2]
            fileType =  [i for i in filename.split('.')[2:] if i != 'gz'][-1]
    elif center in ('broad', 'BSC', 'oicr_sga', 'mda_kchen', 'MDA_HGSC', 'mcgill_popsv', 'sfu', 'UCSC', 'oicr_bl', 'Synteka_pgm21', 'crg_clindel'):
        sample_id, workflow_name, date, call_type, dataType  =  filename.replace('indels', 'indel', split('.')[:5])
        fileType =  [i for i in filename.split('.')[5:] if i != 'gz'][-1]
    else:
        print 'Not uploading:', f
        return 
    print center, workflow_name, date, call_type, dataType, fileType
    file = File(url, parentId=DIRS[center], synapseStore=False)
    file.center = center.lower()
    file.sample_id = sample_id
    file.workflow_name = workflow_name
    file.date = date
    file.call_type = call_type
    file.dataType = 'DNA'
    file.disease = 'Cancer'
    file.dataSubType = dataType
    file.fileType = fileType
    #file.analysis_id_tumor = ?????
    syn.store(file, forceVersion=False)

def test_round_trip():
    fhid = None
    filepath = utils.make_bogus_binary_file(multipart_upload_module.MIN_PART_SIZE + 777771)
    print('Made bogus file: ', filepath)
    try:
        fhid = multipart_upload(syn, filepath)
        print('FileHandle: {fhid}'.format(fhid=fhid))

        # Download the file and compare it with the original
        junk = File(filepath, parent=project, dataFileHandleId=fhid)
        junk.properties.update(syn._createEntity(junk.properties))
        (tmp_f, tmp_path) = tempfile.mkstemp()
        schedule_for_cleanup(tmp_path)
        junk.update(syn._downloadFileEntity(junk, tmp_path))
        assert filecmp.cmp(filepath, junk.path)

    finally:
        try:
            if 'junk' in locals():
                syn.delete(junk)
        except Exception:
            print(traceback.format_exc())
        try:
            os.remove(filepath)
        except Exception:
            print(traceback.format_exc())

def test_Entity():
    # Test CRUD on Entity objects, Project, Folder, File with createEntity/getEntity/updateEntity
    project_name = str(uuid.uuid4())
    project = Project(project_name, description='Bogus testing project')
    project = syn.createEntity(project)
    schedule_for_cleanup(project)

    folder = Folder('Test Folder', parent=project, description='A place to put my junk', foo=1000)
    folder = syn.createEntity(folder)

    path = utils.make_bogus_data_file()
    schedule_for_cleanup(path)
    a_file = File(path, parent=folder, description='Random data for testing', foo='An arbitrary value', bar=[33,44,55], bday=Datetime(2013,3,15))
    a_file = syn._createFileEntity(a_file)

    ## local object state should be preserved
    assert a_file.path == path

    ## check the project entity
    project = syn.getEntity(project)
    assert project.name == project_name

    ## check the folder entity
    folder = syn.getEntity(folder.id)
    assert folder.name == 'Test Folder'
    assert folder.parentId == project.id
    assert folder.foo[0] == 1000

    ## check the file entity
    a_file = syn.getEntity(a_file)
    assert a_file['foo'][0] == 'An arbitrary value'
    assert a_file['bar'] == [33,44,55]
    assert a_file['bday'][0] == Datetime(2013,3,15)

    ## make sure file comes back intact
    a_file = syn.downloadEntity(a_file)
    assert filecmp.cmp(path, a_file.path)

    #TODO We're forgotten the local file path
    a_file.path = path

    ## update the file entity
    a_file['foo'] = 'Another arbitrary chunk of text data'
    a_file['new_key'] = 'A newly created value'
    a_file = syn.updateEntity(a_file)
    assert a_file['foo'][0] == 'Another arbitrary chunk of text data'
    assert a_file['bar'] == [33,44,55]
    assert a_file['bday'][0] == Datetime(2013,3,15)
    assert a_file.new_key[0] == 'A newly created value'
    assert a_file.path == path

    ## upload a new file
    new_path = utils.make_bogus_data_file()
    schedule_for_cleanup(new_path)

    a_file = syn.uploadFile(a_file, new_path)

    ## make sure file comes back intact
    a_file = syn.downloadEntity(a_file)
    assert filecmp.cmp(new_path, a_file.path)

def test_getWithEntityBundle(*mocks):
    mocks = [item for item in mocks]
    is_loco_mock              = mocks.pop()
    cache_location_guess_mock = mocks.pop()
    download_file_mock        = mocks.pop()
    
    # -- Change downloadLocation but do not download more than once --
    is_loco_mock.return_value = False
    
    bundle = {"entity"     : {"name": "anonymous", 
                              "dataFileHandleId": "-1337", 
                              "concreteType": "org.sagebionetworks.repo.model.FileEntity",
                              "parentId": "syn12345"},
              "fileHandles": [{u'concreteType': u'org.sagebionetworks.repo.model.file.S3FileHandle',
                               u'fileName': u'anonymous',
                               u'contentMd5': u'1698d26000d60816caab15169efcd23a',
                               u'id': u'-1337'}],
              "annotations": {}}

    # Make the cache point to some temporary location
    cacheDir = synapseclient.cache.determine_cache_directory(bundle['entity'])
    
    # Pretend that the file is downloaded by the first call to syn._downloadFileEntity
    # The temp file should be added to the cache by the first syn._getWithEntityBundle() call
    f, cachedFile = tempfile.mkstemp()
    os.close(f)
    defaultLocation = os.path.join(cacheDir, bundle['entity']['name'])
    cache_location_guess_mock.return_value = (cacheDir, defaultLocation, cachedFile)
    
    # Make sure the Entity is updated with the cached file path
    def _downloadFileEntity(entity, path, submission):
        # We're disabling the download, but the given path should be within the cache
        assert path == defaultLocation
        return {"path": cachedFile}
    download_file_mock.side_effect = _downloadFileEntity

    # Make sure the cache does not already exist
    cacheMap = os.path.join(cacheDir, '.cacheMap')
    if os.path.exists(cacheMap):
        os.remove(cacheMap)
    
    syn._getWithEntityBundle(entityBundle=bundle, entity=None, downloadLocation=cacheDir, ifcollision="overwrite.local")
    syn._getWithEntityBundle(entityBundle=bundle, entity=None, ifcollision="overwrite.local")
    e = syn._getWithEntityBundle(entityBundle=bundle, entity=None, downloadLocation=cacheDir, ifcollision="overwrite.local")
    assert download_file_mock.call_count == 1

    assert e.name == bundle["entity"]["name"]
    assert e.parentId == bundle["entity"]["parentId"]
    assert e.cacheDir == cacheDir
    assert bundle['entity']['name'] in e.files
    assert e.path == os.path.join(cacheDir, bundle["entity"]["name"])

    ## test preservation of local state
    url = 'http://foo.com/secretstuff.txt'
    e = File(name='anonymous', parentId="syn12345", synapseStore=False, externalURL=url)
    e.local_state({'zap':'pow'})
    e = syn._getWithEntityBundle(entityBundle=bundle, entity=e)
    assert e.local_state()['zap'] == 'pow'
    assert e.synapseStore == False
    assert e.externalURL == url

def test_upload_string():
    ## This tests the utility that uploads  a _string_ rather than 
    ## a file on disk, to S3.
    
    fh = None
    content = "My dog has fleas.\n"
    f = tempfile.NamedTemporaryFile(suffix=".txt", delete=False)
    f.write(content)
    f.close()
    filepath=f.name
        
    print 'Made bogus file: ', filepath
    try:
        fh = syn._uploadStringToFile(content)
        # print 'FileHandle:'
        # syn.printEntity(fh)

        # Download the file and compare it with the original
        junk = File(filepath, parent=project, dataFileHandleId=fh['id'])
        junk.properties.update(syn._createEntity(junk.properties))
        junk.update(syn._downloadFileEntity(junk, filepath))
        assert filecmp.cmp(filepath, junk.path)

    finally:
        try:
            if 'junk' in locals():
                syn.delete(junk)
        except Exception:
            print traceback.format_exc()
        try:
            os.remove(filepath)
        except Exception:
            print traceback.format_exc()

def test_randomly_failing_parts():
    FAILURE_RATE = 1.0/3.0
    fhid = None
    multipart_upload_module.MIN_PART_SIZE = 5*MB
    multipart_upload_module.MAX_RETRIES = 20

    filepath = utils.make_bogus_binary_file(multipart_upload_module.MIN_PART_SIZE*2 + 777771)
    print('Made bogus file: ', filepath)

    normal_put_chunk = None

    def _put_chunk_or_fail_randomly(url, chunk, verbose=False):
        if random.random() < FAILURE_RATE:
            raise IOError("Ooops! Artificial upload failure for testing.")
        else:
            return normal_put_chunk(url, chunk, verbose)

    ## Mock _put_chunk to fail randomly
    normal_put_chunk = multipart_upload_module._put_chunk
    multipart_upload_module._put_chunk = _put_chunk_or_fail_randomly

    try:
        fhid = multipart_upload(syn, filepath)
        print('FileHandle: {fhid}'.format(fhid=fhid))

        # Download the file and compare it with the original
        junk = File(filepath, parent=project, dataFileHandleId=fhid)
        junk.properties.update(syn._createEntity(junk.properties))
        (tmp_f, tmp_path) = tempfile.mkstemp()
        schedule_for_cleanup(tmp_path)
        junk.update(syn._downloadFileEntity(junk, tmp_path))
        assert filecmp.cmp(filepath, junk.path)

    finally:
        ## Un-mock _put_chunk
        if normal_put_chunk:
            multipart_upload_module._put_chunk = normal_put_chunk

        try:
            if 'junk' in locals():
                syn.delete(junk)
        except Exception:
            print(traceback.format_exc())
        try:
            os.remove(filepath)
        except Exception:
            print(traceback.format_exc())

def test_ExternalFileHandle():
    # Tests shouldn't have external dependencies, but this is a pretty picture of Singapore
    singapore_url = 'http://upload.wikimedia.org/wikipedia/commons/thumb/3/3e/1_singapore_city_skyline_dusk_panorama_2011.jpg/1280px-1_singapore_city_skyline_dusk_panorama_2011.jpg'
    singapore = File(singapore_url, parent=project, synapseStore=False)
    singapore = syn.store(singapore)

    # Verify the file handle
    fileHandle = syn._getFileHandle(singapore.dataFileHandleId)
    assert fileHandle['concreteType'] == 'org.sagebionetworks.repo.model.file.ExternalFileHandle'
    assert fileHandle['externalURL']  == singapore_url

    # The download should occur only on the client side
    singapore = syn.get(singapore, downloadFile=True)
    assert singapore.path is not None
    assert singapore.externalURL == singapore_url
    assert os.path.exists(singapore.path)

    # Update external URL
    singapore_2_url = 'https://upload.wikimedia.org/wikipedia/commons/a/a2/Singapore_Panorama_v2.jpg'
    singapore.externalURL = singapore_2_url
    singapore = syn.store(singapore)
    s2 = syn.get(singapore, downloadFile=False)
    assert s2.externalURL == singapore_2_url

def test_download_file_false():
    RENAME_SUFFIX = 'blah'
    
    # Upload a file
    filepath = utils.make_bogus_binary_file()
    schedule_for_cleanup(filepath)
    schedule_for_cleanup(filepath + RENAME_SUFFIX)
    file = File(filepath, name='SYNR 619', parent=project)
    file = syn.store(file)
    
    # Now hide the file from the cache and download with downloadFile=False
    os.rename(filepath, filepath + RENAME_SUFFIX)
    file = syn.get(file.id, downloadFile=False)
    
    # Change something and reupload the file's metadata
    file.name = "Only change the name, not the file"
    reupload = syn.store(file)
    assert reupload.path is None, "Path field should be null: %s" % reupload.path
    
    # This should still get the correct file
    reupload = syn.get(reupload.id)
    assert filecmp.cmp(filepath + RENAME_SUFFIX, reupload.path)
    assert reupload.name == file.name

def test_round_trip():
    fh = None
    filepath = utils.make_bogus_binary_file(6*MB + 777771, verbose=True)
    print 'Made bogus file: ', filepath
    try:
        fh = syn._chunkedUploadFile(filepath, verbose=False)

        print '=' * 60
        print 'FileHandle:'
        syn.printEntity(fh)

        print 'creating project and file'
        project = create_project()
        junk = File(filepath, parent=project, dataFileHandleId=fh['id'])
        junk.properties.update(syn._createEntity(junk.properties))

        print 'downloading file'
        junk.update(syn._downloadFileEntity(junk, filepath))

        print 'comparing files'
        assert filecmp.cmp(filepath, junk.path)

        print 'ok!'

    finally:
        try:
            if 'junk' in locals():
                syn.delete(junk)
        except Exception as ex:
            print ex
        try:
            os.remove(filepath)
        except Exception as ex:
            print ex
        if fh:
            print 'Deleting fileHandle', fh['id']
            syn._deleteFileHandle(fh)

def test_download_file_URL_false():
    # Upload an external file handle
    fileThatExists = 'http://dev-versions.synapse.sagebase.org/synapsePythonClient'
    reupload = File(fileThatExists, synapseStore=False, parent=project)
    reupload = syn.store(reupload)
    reupload = syn.get(reupload, downloadFile=False)
    originalVersion = reupload.versionNumber
    
    # Reupload and check that the URL and version does not get mangled
    reupload = syn.store(reupload, forceVersion=False)
    assert reupload.path == fileThatExists, "Entity should still be pointing at a URL"
    assert originalVersion == reupload.versionNumber

    # Try a URL with an extra slash at the end
    fileThatDoesntExist = 'http://dev-versions.synapse.sagebase.org/synapsePythonClient/'
    reupload.synapseStore = False
    reupload.path = fileThatDoesntExist
    reupload = syn.store(reupload)
    reupload = syn.get(reupload, downloadFile=False)
    originalVersion = reupload.versionNumber
    
    reupload = syn.store(reupload, forceVersion=False)
    assert reupload.path == fileThatDoesntExist, "Entity should still be pointing at a URL"
    assert originalVersion == reupload.versionNumber

def _copyFile(syn, entity, destinationId, version=None, updateExisting=False, setProvenance="traceback",
              skipCopyAnnotations=False):
    """
    Copies most recent version of a file to a specified synapse ID.

    :param entity:              A synapse ID of a File entity

    :param destinationId:       Synapse ID of a folder/project that the file wants to be copied to

    :param version:             Can specify version of a file. 
                                Default to None

    :param updateExisting:      Can choose to update files that have the same name 
                                Default to False
    
    :param setProvenance:       Has three values to set the provenance of the copied entity:
                                    traceback: Sets to the source entity
                                    existing: Sets to source entity's original provenance (if it exists)
                                    None: No provenance is set
    :param skipCopyAnnotations: Skips copying the annotations
                                Default is False
    """
    ent = syn.get(entity, downloadFile=False, version=version, followLink=False)
    # CHECK: If File is in the same parent directory (throw an error) (Can choose to update files)
    if not updateExisting:
        existingEntity = syn.findEntityId(ent.name, parent=destinationId)
        if existingEntity is not None:
            raise ValueError('An entity named "%s" already exists in this location. File could not be copied'
                             % ent.name)
    profile = syn.getUserProfile()
    # get provenance earlier to prevent errors from being called in the end
    # If traceback, set activity to old entity
    if setProvenance == "traceback":
        act = Activity("Copied file", used=ent)
    # if existing, check if provenance exists
    elif setProvenance == "existing":
        try:
            act = syn.getProvenance(ent.id)
        except SynapseHTTPError as e:
            if e.response.status_code == 404:
                act = None
            else:
                raise e
    elif setProvenance is None or setProvenance.lower() == 'none':
        act = None
    else:
        raise ValueError('setProvenance must be one of None, existing, or traceback')
    # Grab entity bundle
    bundle = syn._getEntityBundle(ent.id, version=ent.versionNumber, bitFlags=0x800 | 0x1)
    fileHandle = synapseclient.utils.find_data_file_handle(bundle)
    createdBy = fileHandle['createdBy']
    # CHECK: If the user created the file, copy the file by using fileHandleId else copy the fileHandle
    if profile.ownerId == createdBy:
        newdataFileHandleId = ent.dataFileHandleId
    else:
        copiedFileHandle = copyFileHandles(syn, [fileHandle], ["FileEntity"], [bundle['entity']['id']],
                                           [fileHandle['contentType']], [fileHandle['fileName']])
        # Check if failurecodes exist
        copyResult = copiedFileHandle['copyResults'][0]
        if copyResult.get("failureCode") is not None:
            raise ValueError("%s dataFileHandleId: %s" % (copyResult["failureCode"],
                                                          copyResult['originalFileHandleId']))
        newdataFileHandleId = copyResult['newFileHandle']['id']

    new_ent = File(dataFileHandleId=newdataFileHandleId,  name=ent.name, parentId=destinationId)
    # Set annotations here
    if not skipCopyAnnotations:
        new_ent.annotations = ent.annotations
    # Store provenance if act is not None
    if act is not None:
        new_ent = syn.store(new_ent, activity=act)
    else:
        new_ent = syn.store(new_ent)
    # Leave this return statement for test
    return new_ent['id']