本文整理汇总了Python中qds_sdk.qubole.Qubole.configure方法的典型用法代码示例。如果您正苦于以下问题:Python Qubole.configure方法的具体用法?Python Qubole.configure怎么用?Python Qubole.configure使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类qds_sdk.qubole.Qubole
的用法示例。
在下文中一共展示了Qubole.configure方法的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: __init__
# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
def __init__(self, *args, **kwargs):
conn = self.get_connection(kwargs['qubole_conn_id'])
Qubole.configure(api_token=conn.password, api_url=conn.host)
self.task_id = kwargs['task_id']
self.dag_id = kwargs['dag'].dag_id
self.kwargs = kwargs
self.cls = COMMAND_CLASSES[self.kwargs['command_type']]
self.cmd = None
示例2: _configure_qubole
# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
def _configure_qubole(self):
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger('qds_connection')
logger.propagate = False
qdslog = logging.getLogger('qds')
if not self.config.API_TOKEN:
raise Exception("You didn't specify your QUBOLE_API_TOKEN in "
"your environment before running commands on "
"Qubole!\n. It can be found at http://api.qubole"
".com/users/edit")
Qubole.configure(api_token=self.config.API_TOKEN,
api_url=self.config.API_URL,
version=self.config.API_VERSION,
poll_interval=self.config.POLL_INTERVAL_SEC)
return qdslog
示例3: poke
# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
def poke(self, context):
conn = BaseHook.get_connection(self.qubole_conn_id)
Qubole.configure(api_token=conn.password, api_url=conn.host)
this.log.info('Poking: %s', self.data)
status = False
try:
status = self.sensor_class.check(self.data)
except Exception as e:
logging.exception(e)
status = False
this.log.info('Status of this Poke: %s', status)
return status
示例4: __init__
# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
def __init__(self, access=None, secret = None, testmode=False, db_parallelism=None, mode=None, db_table=None, db_where=None, db_columns=None, db_boundary_query=None, db_extract_query=None, db_split_column=None, hive_table=None, part_spec=None, db_user=None, db_passwd=None, db_host=None, db_port=None, db_type=None, db_name=None, api_token = None, api_url=None, fetch_size = None):
self.temp_location = "/tmp/sqoop/"+uuid.uuid1().hex
self.tmp_dir = tempfile.mkdtemp(prefix="/media/ephemeral0/logs"+"/sqoop")
logger.info("Temp Directory is:" + self.tmp_dir)
self.access = access
self.secret = secret
self.api_token = api_token
self.api_url = api_url
self.fetch_size = fetch_size
self.redshift_sink = False
self.__loadImportParamsFromCid(testmode, db_parallelism, mode, db_table, db_where, db_columns, db_boundary_query, db_extract_query, db_split_column, hive_table, part_spec, db_user, db_passwd, db_host, db_port, db_type, db_name)
self.sqoop_cmd=["/usr/lib/sqoop-h2/bin/sqoop"]
self.sqoop_cmd.extend(["import"])
self.__addBasicOptions()
self.__extendCmdSpecificOptions()
Qubole.configure(api_token=api_token, api_url=api_url)
self.cluster_label = Cluster.show(os.popen("cat /usr/lib/hustler/bin/nodeinfo_src.sh | grep cluster_id").read().split("=")[1].strip().replace('"',''))['cluster']['label'][0]
示例5: execute
# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
def execute(self):
logger.info("Running DbImportCommand " + str(self.sqoop_cmd))
if self.api_url is None:
Qubole.configure(api_token=self.api_token)
else:
Qubole.configure(api_token=self.api_token, api_url = self.api_url)
p = Popen(self.sqoop_cmd, cwd=self.tmp_dir)
retCode = p.wait()
a= os.popen("grep s3_default_db_location /usr/lib/hustler/bin/nodeinfo_src.sh").read()
print(self.temp_location)
print(self.get_s3_loc())
p = Popen(["hadoop", "dfs","-cp", self.temp_location, self.get_s3_loc() + self.temp_location])
retCode1 = p.wait()
if retCode != 0 or retCode1 != 0:
logger.warn("sqoop retCode = " + str(retCode))
self.__runCleanupScript()
self.__runDfsCleanup()
return(retCode or retCode1)
else:
logger.debug("sqoop retCode = " + str(retCode))
retCode = 1
if self.cmd_row['test_mode']:
logger.debug("Not running hive in test mode.")
retCode = 0
else:
logger.info("Running hive script.")
self.fixHiveQuery()
q = open(self.tmp_dir+"/hive_query.q").read()
logger.info("Query is: " + q)
cmd=HiveCommand.create(query=q, label=self.cluster_label)
while not Command.is_done(cmd.status):
time.sleep(5)
cmd = Command.find(cmd.id)
logger.info("Hive command id: " + str(cmd.id) + "status: "+ str(cmd.status))
logger.info(cmd.status)
if cmd.status == "done":
retCode = 0
if retCode != 0:
self.__runCleanupScript()
self.__runDfsCleanup()
return(retCode)
示例6: main
# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
def main():
logging.basicConfig(level=logging.INFO)
if (len(sys.argv) < 3):
usage()
if (len(sys.argv) >= 2 and sys.argv[1] == "-h"):
usage(0)
api_token = sys.argv[1]
output_path = sys.argv[2]
Qubole.configure(api_token=api_token)
args = HadoopCommand.parse(("streaming -files s3n://paid-qubole/HadoopAPIExamples/WordCountPython/mapper.py,s3n://paid-qubole/HadoopAPIExamples/WordCountPython/reducer.py -mapper mapper.py -reducer reducer.py -numReduceTasks 1 -input s3n://paid-qubole/default-datasets/gutenberg -output %s" % output_path).split())
cmd = HadoopCommand.run(**args)
print("Streaming Job run via command id: %s, finished with status %s"
% (cmd.id, cmd.status))
示例7: main
# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
def main():
root = logging.getLogger()
root.setLevel(logging.INFO)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(module)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
root.addHandler(ch)
# I am using this slightly complicated trick to pass config in the constructor of
# other packages. Better way to do this ?
config_parser, argparser = setup_parsers()
config_args, remaining_argv = config_parser.parse_known_args()
config = load_config(config_args)
args = argparser.parse_args(remaining_argv)
if args.debug:
ch.setLevel(logging.DEBUG)
root.setLevel(logging.DEBUG)
logging.debug("Debug is ON!")
if args.log_file is not None:
fh = logging.FileHandler(args.log_file, mode='w')
fh.setLevel(logging.DEBUG)
fh.setFormatter(formatter)
root.setLevel(logging.DEBUG)
root.addHandler(fh)
try:
Qubole.configure(
api_token=config.get("default", "auth_token"),
api_url=config.get("default", "api_url"),
skip_ssl_cert_check=True
)
args.func(config, args)
finally:
logging.debug("Cleaning up")
示例8: connect
# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
def connect(api_token=None, poll_interval=None):
# Try setting from environment variables
if api_token is None:
api_token = os.getenv('QDS_API_TOKEN')
if poll_interval is None:
poll_interval = os.getenv('QDS_POLL_INTERVAL')
api_url = os.getenv('QDS_API_URL')
api_version = os.getenv('QDS_API_VERSION')
# If they aren't set, resort to default values
if api_url is None:
api_url = "https://api.qubole.com/api/"
if api_token is None:
sys.stderr.write("No API Token provided\n")
if api_version is None:
api_version = "v1.2"
if poll_interval is None:
poll_interval = 5
Qubole.configure(api_token=api_token,
api_url=api_url,
version=api_version,
poll_interval=poll_interval,
skip_ssl_cert_check=False)
示例9: run_query
# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
def run_query(self, query, user):
qbol.configure(api_token=self.configuration['token'],
api_url='%s/api' % self.configuration['endpoint'])
try:
cls = PrestoCommand if(self.configuration['query_type'] == 'presto') else HiveCommand
cmd = cls.create(query=query, label=self.configuration['cluster'])
logging.info("Qubole command created with Id: %s and Status: %s", cmd.id, cmd.status)
while not Command.is_done(cmd.status):
time.sleep(qbol.poll_interval)
cmd = Command.find(cmd.id)
logging.info("Qubole command Id: %s and Status: %s", cmd.id, cmd.status)
rows = []
columns = []
error = None
if cmd.status == 'done':
fp = StringIO()
cmd.get_results(fp=fp, inline=True, delim='\t', fetch=False,
qlog=None, arguments=['true'])
results = fp.getvalue()
fp.close()
data = results.split('\r\n')
columns = self.fetch_columns([(i, TYPE_STRING) for i in data.pop(0).split('\t')])
rows = [dict(zip((c['name'] for c in columns), row.split('\t'))) for row in data]
json_data = json_dumps({'columns': columns, 'rows': rows})
except KeyboardInterrupt:
logging.info('Sending KILL signal to Qubole Command Id: %s', cmd.id)
cmd.cancel()
error = "Query cancelled by user."
json_data = None
return json_data, error
示例10: main
# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
def main():
optparser = OptionParser(usage=usage_str)
optparser.add_option("--token", dest="api_token",
default=os.getenv('QDS_API_TOKEN'),
help="api token for accessing Qubole. must be specified via command line or passed in via environment variable QDS_API_TOKEN")
optparser.add_option("--url", dest="api_url",
default=os.getenv('QDS_API_URL'),
help="base url for QDS REST API. defaults to https://api.qubole.com/api ")
optparser.add_option("--version", dest="api_version",
default=os.getenv('QDS_API_VERSION'),
help="version of REST API to access. defaults to v1.2")
optparser.add_option("--poll_interval", dest="poll_interval",
type=int,
default=os.getenv('QDS_POLL_INTERVAL'),
help="interval for polling API for completion and other events. defaults to 5s")
optparser.add_option("--skip_ssl_cert_check", dest="skip_ssl_cert_check", action="store_true",
default=False,
help="skip verification of server SSL certificate. Insecure: use with caution.")
optparser.add_option("-v", dest="verbose", action="store_true",
default=False,
help="verbose mode - info level logging")
optparser.add_option("--vv", dest="chatty", action="store_true",
default=False,
help="very verbose mode - debug level logging")
optparser.disable_interspersed_args()
(options, args) = optparser.parse_args()
if options.chatty:
logging.basicConfig(level=logging.DEBUG)
elif options.verbose:
logging.basicConfig(level=logging.INFO)
else:
logging.basicConfig(level=logging.WARN)
if options.api_token is None:
sys.stderr.write("No API Token provided\n")
usage(optparser)
if options.api_url is None:
options.api_url = "https://api.qubole.com/api/"
if options.api_version is None:
options.api_version = "v1.2"
if options.poll_interval is None:
options.poll_interval = 5
if options.skip_ssl_cert_check is None:
options.skip_ssl_cert_check = False
elif options.skip_ssl_cert_check:
log.warn("Insecure mode enabled: skipping SSL cert verification\n")
Qubole.configure(api_token=options.api_token,
api_url=options.api_url,
version=options.api_version,
poll_interval=options.poll_interval,
skip_ssl_cert_check=options.skip_ssl_cert_check)
if len(args) < 1:
sys.stderr.write("Missing first argument containing subcommand\n")
usage(optparser)
a0 = args.pop(0)
if a0 in CommandClasses:
return cmdmain(a0, args)
if a0 == "account":
return accountmain(args)
if a0 == "cluster":
api_version_number = float(options.api_version[1:])
return clustermain(args, api_version_number)
if a0 == "action":
return actionmain(args)
if a0 == "scheduler":
return schedulermain(args)
if a0 == "report":
return reportmain(args)
if a0 == "dbtap":
return dbtapmain(args)
if a0 == "group":
return groupmain(args)
if a0 == "role":
return rolemain(args)
if a0 == "app":
#.........这里部分代码省略.........
示例11: main
# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
def main():
optparser = OptionParser(usage=usage_str)
optparser.add_option("--token", dest="api_token",
default=os.getenv('QDS_API_TOKEN'),
help="api token for accessing Qubole. must be specified via command line or passed in via environment variable QDS_API_TOKEN")
optparser.add_option("--url", dest="api_url",
default=os.getenv('QDS_API_URL'),
help="base url for QDS REST API. defaults to https://api.qubole.com/api ")
optparser.add_option("--version", dest="api_version",
default=os.getenv('QDS_API_VERSION'),
help="version of REST API to access. defaults to v1.2")
optparser.add_option("--poll_interval", dest="poll_interval",
default=os.getenv('QDS_POLL_INTERVAL'),
help="interval for polling API for completion and other events. defaults to 5s")
optparser.add_option("--skip_ssl_cert_check", dest="skip_ssl_cert_check", action="store_true",
default=False,
help="skip verification of server SSL certificate. Insecure: use with caution.")
optparser.add_option("-v", dest="verbose", action="store_true",
default=False,
help="verbose mode - info level logging")
optparser.add_option("--vv", dest="chatty", action="store_true",
default=False,
help="very verbose mode - debug level logging")
optparser.disable_interspersed_args()
(options, args) = optparser.parse_args()
if options.chatty:
logging.basicConfig(level=logging.DEBUG)
elif options.verbose:
logging.basicConfig(level=logging.INFO)
else:
logging.basicConfig(level=logging.WARN)
pass
if options.api_token is None:
sys.stderr.write("No API Token provided\n")
usage(optparser)
if options.api_url is None:
options.api_url = "https://api.qubole.com/api/";
if options.api_version is None:
options.api_version = "v1.2";
if options.poll_interval is None:
options.poll_interval = 5;
if options.skip_ssl_cert_check is None:
options.skip_ssl_cert_check = False
elif options.skip_ssl_cert_check:
sys.stderr.write("[WARN] Insecure mode enabled: skipping SSL cert verification\n")
Qubole.configure(api_token=options.api_token,
api_url=options.api_url,
version=options.api_version,
poll_interval=options.poll_interval,
skip_ssl_cert_check=options.skip_ssl_cert_check)
if len(args) < 1:
sys.stderr.write("Missing first argument containing command type\n")
usage(optparser)
cmdsuffix = "cmd"
cmdset = set([x + cmdsuffix for x in ["hive", "pig", "hadoop", "shell", "dbexport", "presto"]])
a0 = args.pop(0)
if (a0 in cmdset):
return cmdmain(a0[:a0.find(cmdsuffix)], args)
if (a0 == "hadoop_cluster"):
return clustermain(a0, args)
sys.stderr.write("First command must be one of <%s>\n" %
"|".join(cmdset.union(["hadoop_cluster"])))
usage(optparser)
示例12: hivecommand_from_r
# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
def hivecommand_from_r(query = None, poll_interval = None , sample_size = None, macros = None , tags = None, cluster_label = None, notify = None, name = None, api_token = None ):
api_url = os.getenv('QDS_API_URL')
api_version = os.getenv('QDS_API_VERSION')
if poll_interval is None:
poll_interval = os.getenv('QDS_POLL_INTERVAL')
if api_token is None:
api_token = os.getenv('QDS_API_TOKEN')
chatty = False
verbose = False
skip_ssl_cert_check = None
api_url = None
api_version = None
queryString = ""
#reconstruct the queryString for to be parsed by hivecommand.parse function
if query is not None:
queryString += " --query '%s' "%str(query)
if macros is not None:
queryString += " --macros '%s' "%str(macros)
if tags is not None:
queryString += " --tags '%s' "%str(tags)
if sample_size is not None:
queryString += " --sample_size '%s' "%str(sample_size)
if cluster_label is not None:
queryString += " --cluster-label '%s' "%str(cluster_label)
if notify is not None:
queryString += " --notify '%s' "%str(notify)
if name is not None:
queryString += " --name '%s' "%str(name)
if chatty:
logging.basicConfig(level=logging.DEBUG)
elif verbose:
logging.basicConfig(level=logging.INFO)
else:
logging.basicConfig(level=logging.WARN)
if api_token is None:
sys.stderr.write("No API Token provided\n")
if api_url is None:
api_url = "https://api.qubole.com/api/"
if api_version is None:
api_version = "v1.2"
if poll_interval is None:
poll_interval = 5
if skip_ssl_cert_check is None:
skip_ssl_cert_check = False
elif skip_ssl_cert_check:
log.warn("Insecure mode enabled: skipping SSL cert verification\n")
Qubole.configure(api_token=api_token,
api_url=api_url,
version=api_version,
poll_interval=poll_interval,
skip_ssl_cert_check=skip_ssl_cert_check)
try:
return(hivecommand(queryString))
except qds_sdk.exception.Error as e:
sys.stderr.write("Error: Status code %s (%s) from url %s\n" %
(e.request.status_code, e.__class__.__name__,
e.request.url))
except qds_sdk.exception.ConfigError as e:
sys.stderr.write("Configuration error: %s\n" % str(e))
except qds_sdk.exception.ParseError as e:
sys.stderr.write("Error: %s\n" % str(e))
except Exception:
traceback.print_exc(file=sys.stderr)
示例13: main
# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
def main():
optparser = OptionParser(usage=usage_str)
optparser.add_option("--token", dest="api_token",
default=os.getenv('QDS_API_TOKEN'),
help="api token for accessing Qubole. must be specified via command line or passed in via environment variable QDS_API_TOKEN")
optparser.add_option("--url", dest="api_url",
default=os.getenv('QDS_API_URL'),
help="base url for QDS REST API. defaults to https://api.qubole.com/api ")
optparser.add_option("--version", dest="api_version",
default=os.getenv('QDS_API_VERSION'),
help="version of REST API to access. defaults to v1.2")
optparser.add_option("--poll_interval", dest="poll_interval",
default=os.getenv('QDS_POLL_INTERVAL'),
help="interval for polling API for completion and other events. defaults to 5s")
optparser.add_option("-v", dest="verbose", action="store_true",
default=False,
help="verbose mode - info level logging")
optparser.add_option("--vv", dest="chatty", action="store_true",
default=False,
help="very verbose mode - debug level logging")
optparser.disable_interspersed_args()
(options, args) = optparser.parse_args()
if options.chatty:
logging.basicConfig(level=logging.DEBUG)
elif options.verbose:
logging.basicConfig(level=logging.INFO)
else:
logging.basicConfig(level=logging.WARN)
pass
if options.api_token is None:
sys.stderr.write("No API Token provided\n")
usage(optparser)
if options.api_url is None:
options.api_url = "https://api.qubole.com/api/";
if options.api_version is None:
options.api_version = "v1.2";
if options.poll_interval is None:
options.poll_interval = 5;
Qubole.configure(api_token=options.api_token,
api_url=options.api_url,
version=options.api_version,
poll_interval=options.poll_interval)
if len(args) < 1:
sys.stderr.write("Missing first argument containing command type\n")
usage()
cmdset = set(["hive", "pig", "hadoop", "shell"])
cmdsuffix = "cmd"
cmd = args.pop(0)
if ((cmd.find(cmdsuffix) != len(cmd)-3) or
(cmd[:cmd.find(cmdsuffix)] not in cmdset)):
sys.stderr.write("First command must be one of <%s>\n" % "|".join(cmdset))
usage()
return cmdmain(cmd[:cmd.find(cmdsuffix)], args)
示例14:
# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
from qds_sdk.qubole import Qubole
Qubole.configure(api_token='BjnuEktAoiyFiHZCuLSZxdUFFsrzn15h3Hj9an3xjACBkwqiYCNJcNYygsaLvAFg',api_url="http://localhost:3000/api/")
from qds_sdk.commands import *
hc=HiveCommand.create(query='show tables',retry='4')
print "Id: %s, Status: %s" % (str(hc.id), hc.status)
示例15: backoff_poll_interval
# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
def backoff_poll_interval(self, multiple=2):
QDS.configure(QDS.api_token, poll_interval=QDS.poll_interval * multiple)