本文整理汇总了Python中qds_sdk.qubole.Qubole类的典型用法代码示例。如果您正苦于以下问题:Python Qubole类的具体用法?Python Qubole怎么用?Python Qubole使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了Qubole类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: __init__
def __init__(self, *args, **kwargs):
conn = self.get_connection(kwargs['qubole_conn_id'])
Qubole.configure(api_token=conn.password, api_url=conn.host)
self.task_id = kwargs['task_id']
self.dag_id = kwargs['dag'].dag_id
self.kwargs = kwargs
self.cls = COMMAND_CLASSES[self.kwargs['command_type']]
self.cmd = None
示例2: _configure_qubole
def _configure_qubole(self):
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger('qds_connection')
logger.propagate = False
qdslog = logging.getLogger('qds')
if not self.config.API_TOKEN:
raise Exception("You didn't specify your QUBOLE_API_TOKEN in "
"your environment before running commands on "
"Qubole!\n. It can be found at http://api.qubole"
".com/users/edit")
Qubole.configure(api_token=self.config.API_TOKEN,
api_url=self.config.API_URL,
version=self.config.API_VERSION,
poll_interval=self.config.POLL_INTERVAL_SEC)
return qdslog
示例3: update
def update(cls, cluster_id_label, cluster_info):
"""
Update the cluster with id/label `cluster_id_label` using information provided in
`cluster_info`.
"""
conn = Qubole.agent(version="v2")
return conn.put(cls.element_path(cluster_id_label), data=cluster_info)
示例4: terminate
def terminate(cls, cluster_id_label):
"""
Terminate the cluster with id/label `cluster_id_label`.
"""
conn = Qubole.agent()
data = {"state": "terminate"}
return conn.put(cls.element_path(cluster_id_label) + "/state", data)
示例5: clone
def clone(cls, cluster_id_label, cluster_info):
"""
Update the cluster with id/label `cluster_id_label` using information provided in
`cluster_info`.
"""
conn = Qubole.agent()
return conn.post(cls.element_path(cluster_id_label) + '/clone', data=cluster_info)
示例6: find_by_name
def find_by_name(name):
conn = Qubole.agent()
if name is not None:
schedjson = conn.get(Scheduler.rest_entity_path, params={"name":name})
if schedjson["schedules"]:
return Scheduler(schedjson["schedules"][0])
return None
示例7: poke
def poke(self, context):
conn = BaseHook.get_connection(self.qubole_conn_id)
Qubole.configure(api_token=conn.password, api_url=conn.host)
this.log.info('Poking: %s', self.data)
status = False
try:
status = self.sensor_class.check(self.data)
except Exception as e:
logging.exception(e)
status = False
this.log.info('Status of this Poke: %s', status)
return status
示例8: add_node
def add_node(cls, cluster_id_label, parameters=None):
"""
Add a node to an existing cluster
"""
conn = Qubole.agent()
parameters = {} if not parameters else parameters
return conn.post(cls.element_path(cluster_id_label) + "/nodes", data={"parameters" : parameters})
示例9: index
def index(cls):
"""
Shows a list of all available reports by issuing a GET request to the
/reports endpoint.
"""
conn = Qubole.agent()
return conn.get(cls.rest_entity_path)
示例10: start
def start(cls, cluster_id_label):
"""
Start the cluster with id/label `cluster_id_label`.
"""
conn = Qubole.agent()
data = {"state": "start"}
return conn.put(cls.element_path(cluster_id_label) + "/state", data)
示例11: find
def find(cls, name="default", **kwargs):
if (name is None) or (name == "default"):
conn = Qubole.agent()
return cls(conn.get(cls.rest_entity_path))
else:
raise ParseError("Bad name %s" % name,
"Hadoop Clusters can only be named 'default' currently")
示例12: get_results
def get_results(self, fp=sys.stdout, inline=True, delim=None):
"""
Fetches the result for the command represented by this object
Args:
`fp`: a file object to write the results to directly
"""
result_path = self.meta_data['results_resource']
conn = Qubole.agent()
r = conn.get(result_path, {'inline': inline})
if r.get('inline'):
if sys.version_info < (3, 0, 0):
fp.write(r['results'].encode('utf8'))
else:
import io
if isinstance(fp, io.TextIOBase):
fp.buffer.write(r['results'].encode('utf8'))
elif isinstance(fp, io.BufferedIOBase) or isinstance(fp, io.RawIOBase):
fp.write(r['results'].encode('utf8'))
else:
# Can this happen? Don't know what's the right thing to do in this case.
pass
else:
acc = Account.find()
boto_conn = boto.connect_s3(aws_access_key_id=acc.storage_access_key,
aws_secret_access_key=acc.storage_secret_key)
log.info("Starting download from result locations: [%s]" % ",".join(r['result_location']))
#fetch latest value of num_result_dir
num_result_dir = Command.find(self.id).num_result_dir
for s3_path in r['result_location']:
# In Python 3, in this case, `fp` should always be binary mode.
_download_to_local(boto_conn, s3_path, fp, num_result_dir, delim=delim)
示例13: list
def list(cls, label=None, cluster_id=None, state=None):
"""
List existing clusters present in your account.
Kwargs:
`state`: list only those clusters which are in this state
Returns:
List of clusters satisfying the given criteria
"""
if cluster_id is not None:
return cls.show(cluster_id)
if label is not None:
return cls.show(label)
conn = Qubole.agent(version="v2")
cluster_list = conn.get(cls.rest_entity_path)
if state is None:
# return the complete list since state is None
return conn.get(cls.rest_entity_path)
# filter clusters based on state
result = []
if 'clusters' in cluster_list:
for cluster in cluster_list['clusters']:
if state.lower() == cluster['state'].lower():
result.append(cluster)
return result
示例14: update_node
def update_node(cls, cluster_id_label, command, private_dns, parameters=None):
"""
Add a node to an existing cluster
"""
conn = Qubole.agent()
parameters = {} if not parameters else parameters
data = {"command" : command, "private_dns" : private_dns, "parameters" : parameters}
return conn.put(cls.element_path(cluster_id_label) + "/nodes", data)
示例15: check
def check(cls, data):
"""
Method to call the sensors api with json payload
:param data: valid json object
:return: True or False
"""
conn = Qubole.agent()
return conn.post(cls.rest_entity_path, data=data)['status']