Python pycurl_manager.RequestHandler類代碼示例

def getSpec(request, reqSpecs=None):
    "Get request from workload cache"
    if reqSpecs and request['RequestName'] in reqSpecs:
        return reqSpecs[request['RequestName']]
    url = str('%s/%s/spec' % (reqmgrCacheUrl(), request['RequestName']))
    mgr = RequestHandler()
    data = mgr.getdata(url, params={}, cert=cert(), ckey=ckey())
    return pickle.loads(data)

def findParent(dataset):
    "Helper function to find a parent of the dataset"
    url = '%s/datasetparents' % dbsUrl()
    params = {'dataset': dataset}
    headers = {'Accept': 'application/json'}
    mgr = RequestHandler()
    data = mgr.getdata(url, params=params, headers=headers, cert=cert(), ckey=ckey())
    return [str(i['parent_dataset']) for i in json.loads(data)]

def getWorkflows(state):
    "Get list of workflows from ReqMgr2 data-service"
    url = '%s/data/request' % reqmgrUrl()
    headers = {'Accept': 'application/json'}
    params = {'status': state}
    mgr = RequestHandler()
    res = mgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert())
    data = json.loads(res)
    return data.get('result', [])

def getNodes(kind):
    "Get list of PhEDEx nodes"
    params = {}
    headers = {'Accept': 'application/json'}
    url = '%s/nodes' % phedexUrl()
    mgr = RequestHandler()
    data = mgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert())
    nodes = json.loads(data)['phedex']['node']
    return [node['name'] for node in nodes if node['kind'] == kind]

def getWorkflow(requestName):
    "Get list of workflow info from ReqMgr2 data-service for given request name"
    headers = {'Accept': 'application/json'}
    params = {}
    url = '%s/data/request/%s' % (reqmgrUrl(), requestName)
    mgr = RequestHandler()
    res = mgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert())
    data = json.loads(res)
    return data.get('result', [])

def getRequest(url, params):
    "Helper function to GET data from given URL"
    mgr = RequestHandler()
    headers = {'Accept': 'application/json'}
    verbose = 0
    if 'verbose' in params:
        verbose = params['verbose']
        del params['verbose']
    data = mgr.getdata(url, params, headers, ckey=ckey(), cert=cert(), verbose=verbose)
    return data

def getNodesForId(phedexid):
    "Helper function to get nodes for given phedex id"
    url = '%s/requestlist' % phedexUrl()
    params = {'request': str(phedexid)}
    headers = {'Accept': 'application/json'}
    mgr = RequestHandler()
    data = mgr.getdata(url, params, headers, ckey=ckey(), cert=cert())
    items = json.loads(data)['phedex']['request']
    nodes = [n['name'] for i in items for n in i['node']]
    return list(set(nodes))

def getNodeQueues():
    "Helper function to fetch nodes usage from PhEDEx data service"
    headers = {'Accept': 'application/json'}
    params = {}
    mgr = RequestHandler()
    url = '%s/nodeusagehistory' % phedexUrl()
    res = mgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert())
    data = json.loads(res)
    ret = defaultdict(int)
    for node in data['phedex']['node']:
        for usage in node['usage']:
            ret[node['name']] += int(usage['miss_bytes'] / 1023.**4) #in TB
    return ret

def workqueueRequests(state=None):
    "Helper functions to get requests from WorkQueue"
    url = workqueueView('jobsByRequest')
    if state:
        pass # we may need to use state when we'll query WorkQueue
    params = {}
    headers = {'Accept': 'application/json'}
    mgr = RequestHandler()
    data = mgr.getdata(url, params=params, headers=headers, cert=cert(), ckey=ckey())
    data = json.loads(data)
    rdict = {}
    for row in data.get('rows', []):
        rdict[row['key']] = row['value']
    return rdict

class MicroServiceTest(unittest.TestCase):
    "Unit test for MicroService module"
    def setUp(self):
        "Setup MicroService for testing"
        self.app = ServiceManager()
        config = TestConfig
        manager = 'WMCore_t.Services_t.MicroService_t.MicroService_t.ServiceManager'
        config.views.data.manager = manager
        config.manager = manager
        mount = '/microservice'
        self.mgr = RequestHandler()
        self.port = config.main.port
        self.url = 'http://localhost:%s%s/data' % (self.port, mount)
        cherrypy.config["server.socket_port"] = self.port
        self.server = RestApiHub(self.app, config, mount)
        cherrypy.tree.mount(self.server, mount)
        cherrypy.engine.start()

    def tearDown(self):
        "Tear down MicroService"
        cherrypy.engine.exit()
        cherrypy.engine.stop()

    def postRequest(self, params):
        "Perform POST request to our MicroService"
        headers = {'Content-type': 'application/json'}
        print("### post call %s params=%s headers=%s" % (self.url, params, headers))
        data = self.mgr.getdata(self.url, params=params, headers=headers, \
                verb='POST', cert=cert(), ckey=ckey())
        print("### post call data %s" % data)
        return json.loads(data)

    def test_getState(self):
        "Test function for getting state of the MicroService"
        url = '%s/status' % self.url
        data = self.mgr.getdata(url, params={})
        state = "bla"
        data = {"request":{"state": state}}
        self.postRequest(data)
        data = self.mgr.getdata(url, params={})
        data = json.loads(data)
        print("### url=%s, data=%s" % (url, data))
        for row in data['result']:
            if 'state' in row:
                self.assertEqual(state, row['state'])

    def setUp(self):
        "initialization"
        self.mgr = RequestHandler()
        self.ckey = os.path.join(os.environ['HOME'], '.globus/userkey.pem')
        self.cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')

        self.cricheader = 'Date: Tue, 06 Nov 2018 14:50:29 GMT\r\nServer: Apache/2.4.6 (CentOS) OpenSSL/1.0.2k-fips mod_wsgi/3.4 Python/2.7.5 mod_gridsite/2.3.4\r\nVary: Cookie\r\nX-Frame-Options: SAMEORIGIN\r\nSet-Cookie: sessionid=bc1xu8zi5rbbsd5fgjuklb2tk2r3f6tw; expires=Sun, 11-Nov-2018 14:50:29 GMT; httponly; Max-Age=432000; Path=/\r\nContent-Length: 32631\r\nContent-Type: application/json\r\n\r\n'
        self.dbsheader = 'Date: Tue, 06 Nov 2018 14:39:07 GMT\r\nServer: Apache\r\nCMS-Server-Time: D=1503 t=1541515147806112\r\nTransfer-Encoding: chunked\r\nContent-Type: text/html\r\n\r\n'
        self.HTTPheader = 'Date: Tue, 06 Nov 2018 14:50:29 GMT\r\nServer: Apache/2.4.6 (CentOS) OpenSSL/1.0.2k-fips mod_wsgi/3.4 Python/2.7.5 mod_gridsite/2.3.4\r\nVary: Cookie\r\nX-Frame-Options: SAMEORIGIN\r\nSet-Cookie: GRIDHTTP_PASSCODE=2c6da9c96efa2ad0farhda; domain=cms-cric.cern.ch; path=/; secure\r\nContent-Length: 32631\r\nContent-Type: application/json\r\n\r\n'

def alterSubscription(phedexid, decision, comments, nodes=None):
    "Helper function to alter subscriptions for given phedex id and nodes"
    mgr = RequestHandler()
    headers = {'Accept': 'application/json'}
    nodes = nodes if nodes else getNodesForId(phedexid)
    params = {
        'decision': decision,
        'request': phedexid,
        'node': ','.join(nodes),
        'comments': comments
        }
    url = '%s/updaterequest'
    data = mgr.getdata(url, params, headers, ckey=ckey(), cert=cert(), verb='POST')
    result = json.loads(data)
    if not result:
        return False
    if 'already' in result:
        return True
    return result

def getDataFromURL(url, proxyfilename = None):
    """
    Read the content of a URL and return it as a string.
    Type of content should not matter, it can be a json file or a tarball for example.

    url: the link you would like to retrieve
    proxyfilename: the x509 proxy certificate to be used in case auth is required

    Returns binary data encoded as a string, which can be later processed
    according to what kind of content it represents.
    """

    # Get rid of unicode which may cause problems in pycurl
    stringUrl = url.encode('ascii')

    reqHandler = RequestHandler()
    _, data = reqHandler.request(url=stringUrl, params={}, ckey=proxyfilename,
                                 cert=proxyfilename,
                                 capath=HTTPRequests.getCACertPath())

    return data

def agentsSites(url):
    "Return list of sites known in CMS WMAgents"
    sites_ready_in_agent = set()
    headers = {'Accept': 'application/json'}
    params = {}
    mgr = RequestHandler()
    res = mgr.getdata(url, params=params, headers=headers, ckey=ckey(), cert=cert())
    data = json.loads(res)
    agents = {}
    for r in [i['value'] for i in data['rows']]:
        team = r['agent_team']
        if team != 'production':
            continue
        agents.setdefault(team, []).append(r)
    for team, agents in agents.items():
        for agent in agents:
            if agent['status'] != 'ok':
                continue
            for site, sinfo in agent['WMBS_INFO']['thresholds'].iteritems():
                if sinfo['state'] in ['Normal']:
                    sites_ready_in_agent.add(site)
    return sites_ready_in_agent

 def setUp(self):
     "Setup MicroService for testing"
     self.app = ServiceManager()
     config = TestConfig
     manager = 'WMCore_t.Services_t.MicroService_t.MicroService_t.ServiceManager'
     config.views.data.manager = manager
     config.manager = manager
     mount = '/microservice'
     self.mgr = RequestHandler()
     self.port = config.main.port
     self.url = 'http://localhost:%s%s/data' % (self.port, mount)
     cherrypy.config["server.socket_port"] = self.port
     self.server = RestApiHub(self.app, config, mount)
     cherrypy.tree.mount(self.server, mount)
     cherrypy.engine.start()