Python SearchConnection.new_context方法代码示例

# 需要导入模块: from pyesgf.search import SearchConnection [as 别名]
# 或者: from pyesgf.search.SearchConnection import new_context [as 别名]
    def get_esgf_cordex_info(self):
        
        res_dict = {}
        models = {}
        driving_models = {}
        
        conn = SearchConnection('http://esgf-data.dkrz.de/esg-search',distrib=True)
        ctx = conn.new_context(project='CORDEX',replica=False)
        #print ctx.hit_count
        
        domains = ctx.facet_counts['domain'].keys()
        #print domains

        for key in domains:
            ctx2 = conn.new_context(project='CORDEX',domain=key,replica=False)
            #print ctx2.hit_count 
            models[key] = ctx2.facet_counts['rcm_name'].keys()
            #print models[key]
            driving_models[key]={}
            for thismodel in models[key]:
              
                 ctx3 = conn.new_context(project='CORDEX',domain=key,rcm_name=thismodel,replica=False)
                 #print key,thismodel
                 #print ctx3.hit_count           
                 driving_models[key][thismodel] =  ctx3.facet_counts['driving_model'].keys()
    
        return driving_models

# 需要导入模块: from pyesgf.search import SearchConnection [as 别名]
# 或者: from pyesgf.search.SearchConnection import new_context [as 别名]
def test_context_facet_multivalue3():
    conn = SearchConnection(TEST_SERVICE)
    ctx = conn.new_context(project='CMIP5', query='humidity', experiment='rcp45')
    hits1 = ctx.hit_count
    assert hits1 > 0
    ctx2 = conn.new_context(project='CMIP5', query='humidity',
                           experiment=['rcp45','rcp85'])
    hits2 = ctx2.hit_count

    assert hits2 > hits1

# 需要导入模块: from pyesgf.search import SearchConnection [as 别名]
# 或者: from pyesgf.search.SearchConnection import new_context [as 别名]
def test_distrib():
    conn = SearchConnection(TEST_SERVICE, distrib=False)

    context = conn.new_context(project='CMIP5')
    count1 = context.hit_count

    conn2 = SearchConnection(TEST_SERVICE, distrib=True)
    context = conn2.new_context(project='CMIP5')
    count2 = context.hit_count

    assert count1 < count2

# 需要导入模块: from pyesgf.search import SearchConnection [as 别名]
# 或者: from pyesgf.search.SearchConnection import new_context [as 别名]
    def test_context_facet_multivalue3(self):
        conn = SearchConnection(self.test_service, cache=self.cache)
        ctx = conn.new_context(project='CMIP5', query='humidity',
                               experiment='rcp45')
        hits1 = ctx.hit_count
        assert hits1 > 0
        ctx2 = conn.new_context(project='CMIP5', query='humidity',
                                experiment=['rcp45', 'rcp85'])
        hits2 = ctx2.hit_count

        assert hits2 > hits1

# 需要导入模块: from pyesgf.search import SearchConnection [as 别名]
# 或者: from pyesgf.search.SearchConnection import new_context [as 别名]
def test_temporal_search_CORDEX():
    conn = SearchConnection("http://esgf-data.dkrz.de/esg-search", distrib=True)

    ctx1 = conn.new_context(project='CORDEX',
           from_timestamp="1990-01-01T12:00:00Z",
           to_timestamp="2100-12-31T12:00:00Z")

    ctx2 = conn.new_context(project='CORDEX',
           from_timestamp="2011-01-01T12:00:00Z",
           to_timestamp="2100-12-31T12:00:00Z")

    assert ctx2.hit_count < ctx1.hit_count

# 需要导入模块: from pyesgf.search import SearchConnection [as 别名]
# 或者: from pyesgf.search.SearchConnection import new_context [as 别名]
def test_temporal_search_CMIP5():
    conn = SearchConnection('http://esgf-index1.ceda.ac.uk/esg-search',
                        distrib=False)

    ctx1 = conn.new_context(project = "CMIP5", model = "HadGEM2-ES",
          time_frequency = "mon", realm = "atmos", ensemble = "r1i1p1", latest = True)

    ctx2 = conn.new_context(project = "CMIP5", model = "HadGEM2-ES",
          time_frequency = "mon", realm = "atmos", ensemble = "r1i1p1", latest = True,
          from_timestamp = "2100-12-30T23:23:59Z", to_timestamp = "2200-01-01T00:00:00Z")

    assert ctx2.hit_count < ctx1.hit_count 

# 需要导入模块: from pyesgf.search import SearchConnection [as 别名]
# 或者: from pyesgf.search.SearchConnection import new_context [as 别名]
    def test_temporal_search_CORDEX(self):
        conn = SearchConnection(self.test_service2, distrib=True)

        ctx1 = conn.new_context(project='CORDEX',
                                from_timestamp="1990-01-01T12:00:00Z",
                                to_timestamp="2100-12-31T12:00:00Z")

        ctx2 = conn.new_context(project='CORDEX',
                                from_timestamp="2011-01-01T12:00:00Z",
                                to_timestamp="2100-12-31T12:00:00Z")

        assert ctx2.hit_count < ctx1.hit_count

# 需要导入模块: from pyesgf.search import SearchConnection [as 别名]
# 或者: from pyesgf.search.SearchConnection import new_context [as 别名]
    def test_distrib(self):
        conn = SearchConnection(self.test_service, cache=self.cache,
                                distrib=False)

        context = conn.new_context(project='CMIP5')
        count1 = context.hit_count

        conn2 = SearchConnection(self.test_service, cache=self.cache,
                                 distrib=True)
        context = conn2.new_context(project='CMIP5')
        count2 = context.hit_count

        assert count1 < count2

# 需要导入模块: from pyesgf.search import SearchConnection [as 别名]
# 或者: from pyesgf.search.SearchConnection import new_context [as 别名]
    def test_replica(self):
        # Test that we can exclude replicas
        # This tests assumes the test dataset is replicated
        conn = SearchConnection(self.test_service, cache=self.cache)
        qry = 'id:cmip5.output1.MOHC.HadGEM2-ES.rcp45.mon.atmos.Amon.r1i1p1.*'
        version = '20111128'

        # Search for all replicas
        context = conn.new_context(query=qry, version=version)
        assert context.hit_count == 2

        # Search for only one replicant
        context = conn.new_context(query=qry, replica=False, version=version)
        assert context.hit_count == 1

# 需要导入模块: from pyesgf.search import SearchConnection [as 别名]
# 或者: from pyesgf.search.SearchConnection import new_context [as 别名]
def test_replica():
    # Test that we can exclude replicas
    # This tests assumes the test dataset is replicated
    conn = SearchConnection(TEST_SERVICE)

    context = conn.new_context(
        query='cmip5.output1.NIMR-KMA.HadGEM2-AO.rcp60.mon.atmos.Amon.r1i1p1')

    assert context.hit_count > 1

    context = conn.new_context(
        query='cmip5.output1.NIMR-KMA.HadGEM2-AO.rcp60.mon.atmos.Amon.r1i1p1',
        replica=False)

    assert context.hit_count == 1

# 需要导入模块: from pyesgf.search import SearchConnection [as 别名]
# 或者: from pyesgf.search.SearchConnection import new_context [as 别名]
def test_context_facets1():
    conn = SearchConnection(TEST_SERVICE)
    context = conn.new_context(project='CMIP5')

    context2 = context.constrain(model="IPSL-CM5A-LR")
    assert context2.facet_constraints['project'] == 'CMIP5'
    assert context2.facet_constraints['model'] == 'IPSL-CM5A-LR'

# 需要导入模块: from pyesgf.search import SearchConnection [as 别名]
# 或者: from pyesgf.search.SearchConnection import new_context [as 别名]
def get_esgf_data_and_nodes():
    PCMDI_SERVICE = 'http://pcmdi9.llnl.gov/esg-search/search'     
    conn = SearchConnection(PCMDI_SERVICE,distrib=True)
    ctx = conn.new_context(project='CMIP5',replica=False)
    #ctx = conn.new_context()
    #ctx = ctx.constrain(model='MPI-ESM-P',experiment='piControl',time_frequency='mon', data_node='aims3.llnl.gov')        
    
    my_result=[]    
    ctx = ctx.constrain(project='CMIP5',model='MPI-ESM-P',experiment='piControl', data_node='aims3.llnl.gov')  
    data_nodes = ctx.facet_counts['data_node'].keys()
    models = ctx.facet_counts['model'].keys()
    print "data nodes:", data_nodes
    print "models: ", models
        
    #constrain search to get to datasets
    
    #ctx = ctx.constrain(model='MPI-ESM-P',experiment='piControl',time_frequency='mon', data_node='aims3.llnl.gov')
    results = ctx.search()
    print 'Hits:', ctx.hit_count
    print 'Realms:', ctx.facet_counts['realm']
    print 'Ensembles:', ctx.facet_counts['ensemble']
    
    for i in range(0,ctx.hit_count):    
        file_ctx = results[i].file_context()
        files = file_ctx.search()
        size = files.batch_size
        
        for j in range(0,len(files)):
           my_result.append(files[j].file_id)
        print "files:", size
        
    return my_result

# 需要导入模块: from pyesgf.search import SearchConnection [as 别名]
# 或者: from pyesgf.search.SearchConnection import new_context [as 别名]
    def __init__(
            self,
            url='http://localhost:8081/esg-search',
            distrib=False,
            replica=False,
            latest=True,
            monitor=None):
        # replica is  boolean defining whether to return master records
        # or replicas, or None to return both.
        if replica is True:
            self.replica = None  # master + replica
        else:
            self.replica = False  # only master

        # latest: A boolean defining whether to return only latest versions
        #    or only non-latest versions, or None to return both.
        if latest is True:
            self.latest = True  # only latest versions
        else:
            self.latest = None  # all versions

        self.monitor = monitor

        from pyesgf.search import SearchConnection
        self.conn = SearchConnection(url, distrib=distrib)
        self.fields = 'id,instance_id,number_of_files,number_of_aggregations,size,url'
        # local context has *all* local datasets
        local_conn = SearchConnection(url, distrib=False)
        self.local_ctx = local_conn.new_context(fields=self.fields, replica=True, latest=None)

# 需要导入模块: from pyesgf.search import SearchConnection [as 别名]
# 或者: from pyesgf.search.SearchConnection import new_context [as 别名]
def fetch_cmip5(experiment, variable, time_frequency, models, ensembles):
    """ Download CMIP5 data for a specified set of facets
    
    Uses pyesgf module to query the ESGF nodes RESTful API, and generate a
    wget script which is then executed.
    
    Input parameters defining facets can be strings or lists of strings.
    """
    conn = SearchConnection('http://pcmdi9.llnl.gov/esg-search', distrib=True)
    ctx = conn.new_context(project='CMIP5', experiment=experiment, 
                           time_frequency=time_frequency, variable=variable, 
                           model=models, ensemble=ensembles, latest=True,
                           download_emptypath='unknown')

    a = ctx.get_download_script()
    
    # write the download script out
    with open('getc5.sh','w') as f:
        f.write(a)
        
    # run the download script
    subprocess.Popen(['chmod', 'u+x', 'getc5.sh']).wait()
    subprocess.Popen(['./getc5.sh']).wait()  
    
    # delete any empty files.
    os.system('find ./*.nc -type f -size 0 -delete') 

# 需要导入模块: from pyesgf.search import SearchConnection [as 别名]
# 或者: from pyesgf.search.SearchConnection import new_context [as 别名]
    def test_context_facets2(self):
        conn = SearchConnection(self.test_service, cache=self.cache)
        context = conn.new_context(project='CMIP5')

        context2 = context.constrain(model="IPSL-CM5A-LR")
        assert context2.facet_constraints['project'] == 'CMIP5'
        assert context2.facet_constraints['model'] == 'IPSL-CM5A-LR'