Python search.SearchConnection类代码示例

def get_esgf_data_and_nodes():
    PCMDI_SERVICE = 'http://pcmdi9.llnl.gov/esg-search/search'     
    conn = SearchConnection(PCMDI_SERVICE,distrib=True)
    ctx = conn.new_context(project='CMIP5',replica=False)
    #ctx = conn.new_context()
    #ctx = ctx.constrain(model='MPI-ESM-P',experiment='piControl',time_frequency='mon', data_node='aims3.llnl.gov')        
    
    my_result=[]    
    ctx = ctx.constrain(project='CMIP5',model='MPI-ESM-P',experiment='piControl', data_node='aims3.llnl.gov')  
    data_nodes = ctx.facet_counts['data_node'].keys()
    models = ctx.facet_counts['model'].keys()
    print "data nodes:", data_nodes
    print "models: ", models
        
    #constrain search to get to datasets
    
    #ctx = ctx.constrain(model='MPI-ESM-P',experiment='piControl',time_frequency='mon', data_node='aims3.llnl.gov')
    results = ctx.search()
    print 'Hits:', ctx.hit_count
    print 'Realms:', ctx.facet_counts['realm']
    print 'Ensembles:', ctx.facet_counts['ensemble']
    
    for i in range(0,ctx.hit_count):    
        file_ctx = results[i].file_context()
        files = file_ctx.search()
        size = files.batch_size
        
        for j in range(0,len(files)):
           my_result.append(files[j].file_id)
        print "files:", size
        
    return my_result

    def get_esgf_cordex_info(self):
        
        res_dict = {}
        models = {}
        driving_models = {}
        
        conn = SearchConnection('http://esgf-data.dkrz.de/esg-search',distrib=True)
        ctx = conn.new_context(project='CORDEX',replica=False)
        #print ctx.hit_count
        
        domains = ctx.facet_counts['domain'].keys()
        #print domains

        for key in domains:
            ctx2 = conn.new_context(project='CORDEX',domain=key,replica=False)
            #print ctx2.hit_count 
            models[key] = ctx2.facet_counts['rcm_name'].keys()
            #print models[key]
            driving_models[key]={}
            for thismodel in models[key]:
              
                 ctx3 = conn.new_context(project='CORDEX',domain=key,rcm_name=thismodel,replica=False)
                 #print key,thismodel
                 #print ctx3.hit_count           
                 driving_models[key][thismodel] =  ctx3.facet_counts['driving_model'].keys()
    
        return driving_models

    def __init__(
            self,
            url='http://localhost:8081/esg-search',
            distrib=False,
            replica=False,
            latest=True,
            monitor=None):
        # replica is  boolean defining whether to return master records
        # or replicas, or None to return both.
        if replica is True:
            self.replica = None  # master + replica
        else:
            self.replica = False  # only master

        # latest: A boolean defining whether to return only latest versions
        #    or only non-latest versions, or None to return both.
        if latest is True:
            self.latest = True  # only latest versions
        else:
            self.latest = None  # all versions

        self.monitor = monitor

        from pyesgf.search import SearchConnection
        self.conn = SearchConnection(url, distrib=distrib)
        self.fields = 'id,instance_id,number_of_files,number_of_aggregations,size,url'
        # local context has *all* local datasets
        local_conn = SearchConnection(url, distrib=False)
        self.local_ctx = local_conn.new_context(fields=self.fields, replica=True, latest=None)

def fetch_cmip5(experiment, variable, time_frequency, models, ensembles):
    """ Download CMIP5 data for a specified set of facets
    
    Uses pyesgf module to query the ESGF nodes RESTful API, and generate a
    wget script which is then executed.
    
    Input parameters defining facets can be strings or lists of strings.
    """
    conn = SearchConnection('http://pcmdi9.llnl.gov/esg-search', distrib=True)
    ctx = conn.new_context(project='CMIP5', experiment=experiment, 
                           time_frequency=time_frequency, variable=variable, 
                           model=models, ensemble=ensembles, latest=True,
                           download_emptypath='unknown')

    a = ctx.get_download_script()
    
    # write the download script out
    with open('getc5.sh','w') as f:
        f.write(a)
        
    # run the download script
    subprocess.Popen(['chmod', 'u+x', 'getc5.sh']).wait()
    subprocess.Popen(['./getc5.sh']).wait()  
    
    # delete any empty files.
    os.system('find ./*.nc -type f -size 0 -delete') 

    def test_context_facets2(self):
        conn = SearchConnection(self.test_service, cache=self.cache)
        context = conn.new_context(project='CMIP5')

        context2 = context.constrain(model="IPSL-CM5A-LR")
        assert context2.facet_constraints['project'] == 'CMIP5'
        assert context2.facet_constraints['model'] == 'IPSL-CM5A-LR'

def check_index(index_node, dataset_name, publish):
    """
    Check whether a dataset is published to the Solr Index or not.

    :param str index_node: The index node to check
    :param str dataset_name: The dataset name
    :param bool publish: True if check for existence, False otherwise
    :returns: True iff the dataset was successfully published or unpublished from Solr, otherwise returns False
    :rtype: *boolean*

    """
    if publish:
        hit_count_num = 1
    else:
        hit_count_num = 0

    conn = SearchConnection('http://%s/esg-search' % index_node, distrib=False)
    limit = 50
    i = 0
    while i < limit:
        i += 1
        ctx = conn.new_context(master_id=dataset_name, data_node=socket.gethostname())
        if ctx.hit_count == hit_count_num:
            return True
        elif i < limit:
            print '.',
            time.sleep(10)
    return False

def test_context_facets1():
    conn = SearchConnection(TEST_SERVICE)
    context = conn.new_context(project='CMIP5')

    context2 = context.constrain(model="IPSL-CM5A-LR")
    assert context2.facet_constraints['project'] == 'CMIP5'
    assert context2.facet_constraints['model'] == 'IPSL-CM5A-LR'

def node_info(request):
    if request.method == 'POST':
        ''' For demo purposes this is loading a local file '''
        try:
            from xml.etree.ElementTree import parse
            tree = parse('scripts/registration.xml')
            root = tree.getroot()
            name = json.loads(request.body)['node']

            response = {}
            for node in root:
                if node.attrib['shortName'] == name:
                    response['org'] = node.attrib['organization']
                    response['namespace'] = node.attrib['namespace']
                    response['email'] = node.attrib['supportEmail']
                    response['ip'] = node.attrib['ip']
                    response['longName'] = node.attrib['longName']
                    response['version'] = node.attrib['version']
                    response['shortName'] = name
                    response['adminPeer'] = node.attrib['adminPeer']
                    response['hostname'] = node.attrib['hostname']

                    for child in list(node):
                        if child.tag[-len('AuthorizationService'):] == "AuthorizationService":
                            response['authService'] = child.attrib["endpoint"]
                        if child.tag[-len('GeoLocation'):] == "GeoLocation":
                            response['location'] = child.attrib["city"]
                        if child.tag[-len('Metrics'):] == "Metrics":
                            for gchild in list(child):
                                if gchild.tag[-len('DownloadedData'):] == "DownloadedData":
                                    response['dataDownCount'] = gchild.attrib['count']
                                    response['dataDownSize'] = gchild.attrib['size']
                                    response['dataDownUsers'] = gchild.attrib['users']
                                if gchild.tag[-len('RegisteredUsers'):] == "RegisteredUsers":
                                    response['registeredUsers'] = gchild.attrib['count']

                    from pyesgf.search import SearchConnection
                    print 'attempting to connect to ' + 'http://' + response['hostname'] + 'esg-search/'
                    conn = SearchConnection('http://' + response['hostname'] + '/esg-search/', distrib=True)
                    try:
                        conn.get_shard_list()
                        response['status'] = 'up'
                    except Exception as e:
                        print repr(e)
                        response['status'] = 'down'

                    return HttpResponse(json.dumps(response))
        except Exception as e:
            import traceback
            print '1', e.__doc__
            print '2', sys.exc_info()
            print '3', sys.exc_info()[0]
            print '4', sys.exc_info()[1]
            print '5', traceback.tb_lineno(sys.exc_info()[2])
            ex_type, ex, tb = sys.exc_info()
            print '6', traceback.print_tb(tb)
            return HttpResponse(status=500)
    elif request.method == 'POST':
        print "Unexpected POST request"
        return HttpResponse(status=500)

def test_constrain_regression1():
    conn = SearchConnection(TEST_SERVICE)

    context = conn.new_context(project='CMIP5', model='IPSL-CM5A-LR')
    assert 'experiment' not in context.facet_constraints

    context2 = context.constrain(experiment='historical')
    assert 'experiment' not in context.facet_constraints

def test_constrain_freetext():
    conn = SearchConnection(TEST_SERVICE)

    context = conn.new_context(project='CMIP5', query='humidity')
    assert context.freetext_constraint == 'humidity'

    context = context.constrain(experiment='historical')
    assert context.freetext_constraint == 'humidity'

def test_context_facet_options():
    conn = SearchConnection(TEST_SERVICE)
    context = conn.new_context(project='CMIP5', model='IPSL-CM5A-LR',
                               ensemble='r1i1p1', experiment='rcp60',
                               realm='seaIce')

    assert context.get_facet_options().keys() == ['data_node', 'cf_standard_name', 'variable_long_name', 
                               'cmor_table', 'time_frequency', 'variable']

    def test_constrain_regression1(self):
        conn = SearchConnection(self.test_service, cache=self.cache)

        context = conn.new_context(project='CMIP5', model='IPSL-CM5A-LR')
        assert 'experiment' not in context.facet_constraints

        context2 = context.constrain(experiment='historical')
        assert 'experiment' in context2.facet_constraints

def test_context_facet_multivalue2():
    conn = SearchConnection(TEST_SERVICE)
    context = conn.new_context(project='CMIP5', model='IPSL-CM5A-MR')
    assert context.facet_constraints.getall('model') == ['IPSL-CM5A-MR']

    
    context2 = context.constrain(model=['IPSL-CM5A-MR', 'IPSL-CM5A-LR'])
    assert sorted(context2.facet_constraints.getall('model')) == ['IPSL-CM5A-LR', 'IPSL-CM5A-MR']

    def test_constrain_freetext(self):
        conn = SearchConnection(self.test_service, cache=self.cache)

        context = conn.new_context(project='CMIP5', query='humidity')
        assert context.freetext_constraint == 'humidity'

        context = context.constrain(experiment='historical')
        assert context.freetext_constraint == 'humidity'

def test_download_script():
    conn = SearchConnection(TEST_SERVICE, distrib=False)
    ctx = conn.new_context(project='CMIP5', ensemble='r1i1p1', model='IPSL-CM5A-LR', realm='seaIce',
                           experiment='historicalGHG')
    script = ctx.get_download_script()

    assert '# ESG Federation download script' in script
    assert '# Search URL: %s' % TEST_SERVICE in script