Python Qubole.configure方法代码示例

# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
 def __init__(self, *args, **kwargs):
     conn = self.get_connection(kwargs['qubole_conn_id'])
     Qubole.configure(api_token=conn.password, api_url=conn.host)
     self.task_id = kwargs['task_id']
     self.dag_id = kwargs['dag'].dag_id
     self.kwargs = kwargs
     self.cls = COMMAND_CLASSES[self.kwargs['command_type']]
     self.cmd = None

# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
 def _configure_qubole(self):
     logging.basicConfig(level=logging.INFO)
     logger = logging.getLogger('qds_connection')
     logger.propagate = False
     qdslog = logging.getLogger('qds')
     if not self.config.API_TOKEN:
         raise Exception("You didn't specify your QUBOLE_API_TOKEN in "
                         "your environment before running commands on "
                         "Qubole!\n. It can be found at http://api.qubole"
                         ".com/users/edit")
     Qubole.configure(api_token=self.config.API_TOKEN,
                      api_url=self.config.API_URL,
                      version=self.config.API_VERSION,
                      poll_interval=self.config.POLL_INTERVAL_SEC)
     return qdslog

# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
    def poke(self, context):
        conn = BaseHook.get_connection(self.qubole_conn_id)
        Qubole.configure(api_token=conn.password, api_url=conn.host)

        this.log.info('Poking: %s', self.data)

        status = False
        try:
            status = self.sensor_class.check(self.data)
        except Exception as e:
            logging.exception(e)
            status = False

        this.log.info('Status of this Poke: %s', status)

        return status

# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
 def __init__(self, access=None, secret = None, testmode=False, db_parallelism=None, mode=None, db_table=None, db_where=None, db_columns=None, db_boundary_query=None, db_extract_query=None, db_split_column=None, hive_table=None, part_spec=None, db_user=None, db_passwd=None, db_host=None, db_port=None, db_type=None, db_name=None, api_token = None, api_url=None, fetch_size = None):
   self.temp_location = "/tmp/sqoop/"+uuid.uuid1().hex
   self.tmp_dir = tempfile.mkdtemp(prefix="/media/ephemeral0/logs"+"/sqoop")
   logger.info("Temp Directory is:" + self.tmp_dir)
   self.access = access
   self.secret = secret
   self.api_token = api_token
   self.api_url = api_url
   self.fetch_size = fetch_size
   self.redshift_sink = False
   self.__loadImportParamsFromCid(testmode, db_parallelism, mode, db_table, db_where, db_columns, db_boundary_query, db_extract_query, db_split_column, hive_table, part_spec, db_user, db_passwd, db_host, db_port, db_type, db_name)
   self.sqoop_cmd=["/usr/lib/sqoop-h2/bin/sqoop"]
   self.sqoop_cmd.extend(["import"])
   self.__addBasicOptions()
   self.__extendCmdSpecificOptions()
   Qubole.configure(api_token=api_token, api_url=api_url)
   self.cluster_label = Cluster.show(os.popen("cat /usr/lib/hustler/bin/nodeinfo_src.sh | grep cluster_id").read().split("=")[1].strip().replace('"',''))['cluster']['label'][0]

# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
 def execute(self):
   logger.info("Running DbImportCommand " + str(self.sqoop_cmd))
   if self.api_url is None:
     Qubole.configure(api_token=self.api_token)
   else:
     Qubole.configure(api_token=self.api_token, api_url = self.api_url)
   p = Popen(self.sqoop_cmd, cwd=self.tmp_dir)
   retCode = p.wait()
   a= os.popen("grep s3_default_db_location /usr/lib/hustler/bin/nodeinfo_src.sh").read()
   print(self.temp_location)
   print(self.get_s3_loc())
   p = Popen(["hadoop", "dfs","-cp", self.temp_location, self.get_s3_loc() + self.temp_location])
   retCode1 = p.wait()
   if retCode != 0 or retCode1 != 0:
     logger.warn("sqoop retCode = " + str(retCode))
     self.__runCleanupScript() 
     self.__runDfsCleanup()
     return(retCode or retCode1)
   else:
     logger.debug("sqoop retCode = " + str(retCode))
   retCode = 1
   if self.cmd_row['test_mode']:
     logger.debug("Not running hive in test mode.")
     retCode = 0
   else:
     logger.info("Running hive script.")
     self.fixHiveQuery()
     q = open(self.tmp_dir+"/hive_query.q").read()
     logger.info("Query is: " + q)
     cmd=HiveCommand.create(query=q, label=self.cluster_label)
     while not Command.is_done(cmd.status):
       time.sleep(5)
       cmd = Command.find(cmd.id)
       logger.info("Hive command id: " + str(cmd.id) + "status: "+ str(cmd.status))
     logger.info(cmd.status)
     if cmd.status ==  "done":
       retCode = 0
   if retCode != 0:
     self.__runCleanupScript() 
           
   self.__runDfsCleanup()
   return(retCode)

# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
def main():

    logging.basicConfig(level=logging.INFO)

    if (len(sys.argv) < 3):
        usage()

    if (len(sys.argv) >= 2 and sys.argv[1] == "-h"):
        usage(0)

    api_token = sys.argv[1]
    output_path = sys.argv[2]
        
    Qubole.configure(api_token=api_token)

    args = HadoopCommand.parse(("streaming -files s3n://paid-qubole/HadoopAPIExamples/WordCountPython/mapper.py,s3n://paid-qubole/HadoopAPIExamples/WordCountPython/reducer.py -mapper mapper.py -reducer reducer.py -numReduceTasks 1 -input s3n://paid-qubole/default-datasets/gutenberg -output %s" % output_path).split())

    cmd = HadoopCommand.run(**args)
    
    print("Streaming Job run via command id: %s, finished with status %s" 
          % (cmd.id, cmd.status))

# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
def main():
    root = logging.getLogger()
    root.setLevel(logging.INFO)

    ch = logging.StreamHandler(sys.stdout)
    ch.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)s - %(module)s - %(levelname)s - %(message)s')
    ch.setFormatter(formatter)
    root.addHandler(ch)

    # I am using this slightly complicated trick to pass config in the constructor of
    # other packages. Better way to do this ?

    config_parser, argparser = setup_parsers()

    config_args, remaining_argv = config_parser.parse_known_args()
    config = load_config(config_args)

    args = argparser.parse_args(remaining_argv)

    if args.debug:
        ch.setLevel(logging.DEBUG)
        root.setLevel(logging.DEBUG)
        logging.debug("Debug is ON!")
    if args.log_file is not None:
        fh = logging.FileHandler(args.log_file, mode='w')
        fh.setLevel(logging.DEBUG)
        fh.setFormatter(formatter)

        root.setLevel(logging.DEBUG)
        root.addHandler(fh)
    try:
        Qubole.configure(
            api_token=config.get("default", "auth_token"),
            api_url=config.get("default", "api_url"),
            skip_ssl_cert_check=True
        )
        args.func(config, args)
    finally:
        logging.debug("Cleaning up")

# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
def connect(api_token=None, poll_interval=None):
    # Try setting from environment variables
    if api_token is None:
        api_token = os.getenv('QDS_API_TOKEN')
    if poll_interval is None:
        poll_interval = os.getenv('QDS_POLL_INTERVAL')
    api_url = os.getenv('QDS_API_URL')
    api_version = os.getenv('QDS_API_VERSION')
    # If they aren't set, resort to default values
    if api_url is None:
        api_url = "https://api.qubole.com/api/"
    if api_token is None:
        sys.stderr.write("No API Token provided\n")
    if api_version is None:
        api_version = "v1.2"
    if poll_interval is None:
        poll_interval = 5
    Qubole.configure(api_token=api_token,
                     api_url=api_url,
                     version=api_version,
                     poll_interval=poll_interval,
                     skip_ssl_cert_check=False)

# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
    def run_query(self, query, user):
        qbol.configure(api_token=self.configuration['token'],
                       api_url='%s/api' % self.configuration['endpoint'])

        try:
            cls = PrestoCommand if(self.configuration['query_type'] == 'presto') else HiveCommand
            cmd = cls.create(query=query, label=self.configuration['cluster'])
            logging.info("Qubole command created with Id: %s and Status: %s", cmd.id, cmd.status)

            while not Command.is_done(cmd.status):
                time.sleep(qbol.poll_interval)
                cmd = Command.find(cmd.id)
                logging.info("Qubole command Id: %s and Status: %s", cmd.id, cmd.status)

            rows = []
            columns = []
            error = None

            if cmd.status == 'done':
                fp = StringIO()
                cmd.get_results(fp=fp, inline=True, delim='\t', fetch=False,
                                qlog=None, arguments=['true'])

                results = fp.getvalue()
                fp.close()

                data = results.split('\r\n')
                columns = self.fetch_columns([(i, TYPE_STRING) for i in data.pop(0).split('\t')])
                rows = [dict(zip((c['name'] for c in columns), row.split('\t'))) for row in data]

            json_data = json_dumps({'columns': columns, 'rows': rows})
        except KeyboardInterrupt:
            logging.info('Sending KILL signal to Qubole Command Id: %s', cmd.id)
            cmd.cancel()
            error = "Query cancelled by user."
            json_data = None

        return json_data, error

# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
def main():

    optparser = OptionParser(usage=usage_str)
    optparser.add_option("--token", dest="api_token",
                         default=os.getenv('QDS_API_TOKEN'),
                         help="api token for accessing Qubole. must be specified via command line or passed in via environment variable QDS_API_TOKEN")

    optparser.add_option("--url", dest="api_url",
                         default=os.getenv('QDS_API_URL'),
                         help="base url for QDS REST API. defaults to https://api.qubole.com/api ")

    optparser.add_option("--version", dest="api_version",
                         default=os.getenv('QDS_API_VERSION'),
                         help="version of REST API to access. defaults to v1.2")

    optparser.add_option("--poll_interval", dest="poll_interval",
                         type=int,
                         default=os.getenv('QDS_POLL_INTERVAL'),
                         help="interval for polling API for completion and other events. defaults to 5s")

    optparser.add_option("--skip_ssl_cert_check", dest="skip_ssl_cert_check", action="store_true",
                         default=False,
                         help="skip verification of server SSL certificate. Insecure: use with caution.")

    optparser.add_option("-v", dest="verbose", action="store_true",
                         default=False,
                         help="verbose mode - info level logging")

    optparser.add_option("--vv", dest="chatty", action="store_true",
                         default=False,
                         help="very verbose mode - debug level logging")

    optparser.disable_interspersed_args()
    (options, args) = optparser.parse_args()

    if options.chatty:
        logging.basicConfig(level=logging.DEBUG)
    elif options.verbose:
        logging.basicConfig(level=logging.INFO)
    else:
        logging.basicConfig(level=logging.WARN)

    if options.api_token is None:
        sys.stderr.write("No API Token provided\n")
        usage(optparser)

    if options.api_url is None:
        options.api_url = "https://api.qubole.com/api/"

    if options.api_version is None:
        options.api_version = "v1.2"

    if options.poll_interval is None:
        options.poll_interval = 5

    if options.skip_ssl_cert_check is None:
        options.skip_ssl_cert_check = False
    elif options.skip_ssl_cert_check:
        log.warn("Insecure mode enabled: skipping SSL cert verification\n")

    Qubole.configure(api_token=options.api_token,
                     api_url=options.api_url,
                     version=options.api_version,
                     poll_interval=options.poll_interval,
                     skip_ssl_cert_check=options.skip_ssl_cert_check)

    if len(args) < 1:
        sys.stderr.write("Missing first argument containing subcommand\n")
        usage(optparser)

    a0 = args.pop(0)
    if a0 in CommandClasses:
        return cmdmain(a0, args)

    if a0 == "account":
        return accountmain(args)

    if a0 == "cluster":
        api_version_number = float(options.api_version[1:])
        return clustermain(args, api_version_number)

    if a0 == "action":
        return actionmain(args)

    if a0 == "scheduler":
        return schedulermain(args)

    if a0 == "report":
        return reportmain(args)

    if a0 == "dbtap":
        return dbtapmain(args)

    if a0 == "group":
        return groupmain(args)

    if a0 == "role":
        return rolemain(args)

    if a0 == "app":
#.........这里部分代码省略.........

# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
def main():

    optparser = OptionParser(usage=usage_str)
    optparser.add_option("--token", dest="api_token", 
                         default=os.getenv('QDS_API_TOKEN'),
                         help="api token for accessing Qubole. must be specified via command line or passed in via environment variable QDS_API_TOKEN")

    optparser.add_option("--url", dest="api_url", 
                         default=os.getenv('QDS_API_URL'),
                         help="base url for QDS REST API. defaults to https://api.qubole.com/api ")

    optparser.add_option("--version", dest="api_version", 
                         default=os.getenv('QDS_API_VERSION'),
                         help="version of REST API to access. defaults to v1.2")

    optparser.add_option("--poll_interval", dest="poll_interval", 
                         default=os.getenv('QDS_POLL_INTERVAL'),
                         help="interval for polling API for completion and other events. defaults to 5s")

    optparser.add_option("--skip_ssl_cert_check", dest="skip_ssl_cert_check", action="store_true",
                         default=False,
                         help="skip verification of server SSL certificate. Insecure: use with caution.")

    optparser.add_option("-v", dest="verbose", action="store_true",
                         default=False,
                         help="verbose mode - info level logging")

    optparser.add_option("--vv", dest="chatty", action="store_true",
                         default=False,
                         help="very verbose mode - debug level logging")


    optparser.disable_interspersed_args()
    (options, args) = optparser.parse_args()

    if options.chatty:
        logging.basicConfig(level=logging.DEBUG)
    elif options.verbose:
        logging.basicConfig(level=logging.INFO)
    else:
        logging.basicConfig(level=logging.WARN)
        pass

    if options.api_token is None:
        sys.stderr.write("No API Token provided\n")
        usage(optparser)

    if options.api_url is None:
        options.api_url = "https://api.qubole.com/api/";

    if options.api_version is None:
        options.api_version = "v1.2";

    if options.poll_interval is None:
        options.poll_interval = 5;

    if options.skip_ssl_cert_check is None:
        options.skip_ssl_cert_check = False
    elif options.skip_ssl_cert_check:
        sys.stderr.write("[WARN] Insecure mode enabled: skipping SSL cert verification\n")
        
    Qubole.configure(api_token=options.api_token,
                     api_url=options.api_url,
                     version=options.api_version,
                     poll_interval=options.poll_interval,
                     skip_ssl_cert_check=options.skip_ssl_cert_check)
                     

    if len(args) < 1:
        sys.stderr.write("Missing first argument containing command type\n")
        usage(optparser)

    cmdsuffix = "cmd"
    cmdset = set([x + cmdsuffix for x in ["hive", "pig", "hadoop", "shell", "dbexport", "presto"]])


    a0 = args.pop(0)

    if (a0 in cmdset):
        return cmdmain(a0[:a0.find(cmdsuffix)], args)

    if (a0 == "hadoop_cluster"):
        return clustermain(a0, args)

    sys.stderr.write("First command must be one of <%s>\n" % 
                     "|".join(cmdset.union(["hadoop_cluster"])))
    usage(optparser)

# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
def hivecommand_from_r(query = None, poll_interval = None , sample_size = None, macros = None , tags = None, cluster_label = None, notify = None, name = None, api_token = None ): 
    api_url = os.getenv('QDS_API_URL')
    api_version = os.getenv('QDS_API_VERSION')
    if poll_interval is None:
        poll_interval = os.getenv('QDS_POLL_INTERVAL')
    
    if api_token is None:
        api_token = os.getenv('QDS_API_TOKEN') 
    
    chatty = False
    verbose = False
    
    skip_ssl_cert_check = None
    api_url = None
    api_version = None

    queryString = ""
    #reconstruct the queryString for to be parsed by hivecommand.parse function
    if query is not None:
        queryString += " --query '%s' "%str(query)
    
    if macros is not None:
        queryString += " --macros '%s' "%str(macros)
    
    if tags is not None:
        queryString += " --tags '%s' "%str(tags)
    
    if sample_size is not None:
        queryString += " --sample_size '%s' "%str(sample_size)
    
    if cluster_label is not None:
        queryString += " --cluster-label '%s' "%str(cluster_label)
    
    if notify is not None:
        queryString += " --notify '%s' "%str(notify)
    
    if name is not None:
        queryString += " --name '%s' "%str(name)

    if chatty:
        logging.basicConfig(level=logging.DEBUG)
    elif verbose:
        logging.basicConfig(level=logging.INFO)
    else:
        logging.basicConfig(level=logging.WARN)
    
    if api_token is None:
        sys.stderr.write("No API Token provided\n")
    
    if api_url is None:
        api_url = "https://api.qubole.com/api/"
    
    if api_version is None:
        api_version = "v1.2"
    
    if poll_interval is None:
        poll_interval = 5
    
    if skip_ssl_cert_check is None:
        skip_ssl_cert_check = False
    elif skip_ssl_cert_check:
        log.warn("Insecure mode enabled: skipping SSL cert verification\n")
     
    Qubole.configure(api_token=api_token,
            api_url=api_url,
            version=api_version,
            poll_interval=poll_interval,
            skip_ssl_cert_check=skip_ssl_cert_check)


    try:
        return(hivecommand(queryString))
    except qds_sdk.exception.Error as e:
        sys.stderr.write("Error: Status code %s (%s) from url %s\n" %
                         (e.request.status_code, e.__class__.__name__,
                          e.request.url))
        
    except qds_sdk.exception.ConfigError as e:
        sys.stderr.write("Configuration error: %s\n" % str(e))
        
    except qds_sdk.exception.ParseError as e:
        sys.stderr.write("Error: %s\n" % str(e))
        
        
    except Exception:
        traceback.print_exc(file=sys.stderr)

# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
def main():

    optparser = OptionParser(usage=usage_str)
    optparser.add_option("--token", dest="api_token", 
                         default=os.getenv('QDS_API_TOKEN'),
                         help="api token for accessing Qubole. must be specified via command line or passed in via environment variable QDS_API_TOKEN")

    optparser.add_option("--url", dest="api_url", 
                         default=os.getenv('QDS_API_URL'),
                         help="base url for QDS REST API. defaults to https://api.qubole.com/api ")

    optparser.add_option("--version", dest="api_version", 
                         default=os.getenv('QDS_API_VERSION'),
                         help="version of REST API to access. defaults to v1.2")

    optparser.add_option("--poll_interval", dest="poll_interval", 
                         default=os.getenv('QDS_POLL_INTERVAL'),
                         help="interval for polling API for completion and other events. defaults to 5s")

    optparser.add_option("-v", dest="verbose", action="store_true",
                         default=False,
                         help="verbose mode - info level logging")

    optparser.add_option("--vv", dest="chatty", action="store_true",
                         default=False,
                         help="very verbose mode - debug level logging")


    optparser.disable_interspersed_args()
    (options, args) = optparser.parse_args()

    if options.chatty:
        logging.basicConfig(level=logging.DEBUG)
    elif options.verbose:
        logging.basicConfig(level=logging.INFO)
    else:
        logging.basicConfig(level=logging.WARN)
        pass

    if options.api_token is None:
        sys.stderr.write("No API Token provided\n")
        usage(optparser)

    if options.api_url is None:
        options.api_url = "https://api.qubole.com/api/";

    if options.api_version is None:
        options.api_version = "v1.2";

    if options.poll_interval is None:
        options.poll_interval = 5;

        
    Qubole.configure(api_token=options.api_token,
                     api_url=options.api_url,
                     version=options.api_version,
                     poll_interval=options.poll_interval)
                     

    if len(args) < 1:
        sys.stderr.write("Missing first argument containing command type\n")
        usage()

    cmdset = set(["hive", "pig", "hadoop", "shell"])
    cmdsuffix = "cmd"

    cmd = args.pop(0)

    if ((cmd.find(cmdsuffix) != len(cmd)-3) or
        (cmd[:cmd.find(cmdsuffix)] not in cmdset)):
        sys.stderr.write("First command must be one of <%s>\n" % "|".join(cmdset))
        usage()
        
    return cmdmain(cmd[:cmd.find(cmdsuffix)], args)

# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
from qds_sdk.qubole import Qubole

Qubole.configure(api_token='BjnuEktAoiyFiHZCuLSZxdUFFsrzn15h3Hj9an3xjACBkwqiYCNJcNYygsaLvAFg',api_url="http://localhost:3000/api/")

from qds_sdk.commands import *

hc=HiveCommand.create(query='show tables',retry='4')
print "Id: %s, Status: %s" % (str(hc.id), hc.status)

# 需要导入模块: from qds_sdk.qubole import Qubole [as 别名]
# 或者: from qds_sdk.qubole.Qubole import configure [as 别名]
 def backoff_poll_interval(self, multiple=2):
     QDS.configure(QDS.api_token, poll_interval=QDS.poll_interval * multiple)