Python utils.log_to_postgres函数代码示例

    def connect(self):
        # try to connect
        try:
            bucket = self.client.bucket(self.bucket)

        except Exception, e:
            log_to_postgres('Connection Falure:  %s' % e, ERROR)

	def convert_coltype(self, col):
		_type_map = {
	     "BOOLEAN_TYPE" : "boolean",
	     "TINYINT_TYPE" : "smallint",
	     "SMALLINT_TYPE" : "smallint",
	     "INT_TYPE" : "int",
	     "BIGINT_TYPE" : "bigint",
	     "FLOAT_TYPE" : "float4",
	     "DOUBLE_TYPE" : "float8",
	     "STRING_TYPE" : "text",
	     "TIMESTAMP_TYPE" : "timestamp",
	     "BINARY_TYPE" : "bytea",
	     "ARRAY_TYPE" : "json",
	     "MAP_TYPE" : "json",
	     "STRUCT_TYPE" : "json",
#	     "UNIONTYPE_TYPE" : "",
	     "DECIMAL_TYPE" : "numeric",
#	     "NULL_TYPE" : "",
	     "DATE_TYPE" : "date",
	     "VARCHAR_TYPE" : "varchar",
	     "CHAR_TYPE" : "char",
#	     "INTERVAL_YEAR_MONTH_TYPE" : "",
#	     "INTERVAL_DAY_TIME_TYPE" : "",
		}
		(name, _type, size, _, precision, scale, _) = col
		if (_type in _type_map):
			_type = _type_map[_type]
			if (size):
				_type += "(%d)" % size
			if (precision):
				_type += "(%d,%d)" % (precision, scale)
			return ColumnDefinition(name, type_name=_type) 
		else:
			log_to_postgres('Cannot handle type %s' % _type)

    def __init__(self, options, columns):

        # Calling super constructor
        super(Neo4jForeignDataWrapper, self).__init__(options, columns)

        # Managed server option
        if 'url' not in options:
            log_to_postgres('The Bolt url parameter is required and the default is "bolt://localhost:7687"', WARNING)
        self.url = options.get("url", "bolt://localhost:7687")

        if 'user' not in options:
            log_to_postgres('The user parameter is required  and the default is "neo4j"', ERROR)
        self.user = options.get("user", "neo4j")

        if 'password' not in options:
            log_to_postgres('The password parameter is required for Neo4j', ERROR)
        self.password = options.get("password", "")

        if 'cypher' not in options:
            log_to_postgres('The cypher parameter is required', ERROR)
        self.cypher = options.get("cypher", "MATCH (n) RETURN n LIMIT 100")

        # Setting table columns
        self.columns = columns

        # Create a neo4j driver instance
        self.driver = GraphDatabase.driver( self.url, auth=basic_auth(self.user, self.password))

        self.columns_stat = self.compute_columns_stat()
        self.table_stat = int(options.get("estimated_rows", -1))
        if(self.table_stat < 0):
            self.table_stat = self.compute_table_stat()
        log_to_postgres('Table estimated rows : ' + unicode(self.table_stat), DEBUG)

    def execute(self, quals, columns, **kwargs):
        """
        This method is invoked every time a SELECT is executed
        on the foreign table.

        Parses the quals argument searching for search criteria,
        contacts soundcloud using the official soundcloud python library
        and returns the search results inside the columns of the foreign table.

        Available columns are: title, url, search.

        :param list quals:a list of conditions which are used
          are used in the WHERE part of the select and can be used
          to restrict the number of the results
        :param list columns: the columns of the foreign table
        """
        if not quals:
            msg = 'specify a search term'
            log_to_postgres(level=ERROR, message=msg)
        # create a client object using the apikey
        client = soundcloud.Client(client_id=self.apikey)
        for qual in quals:
            # Manage quals, pass it as search therm if the field is 'search'
            if qual.field_name == "search" or qual.operator == "=":
                # Perform a simple search using the qual value
                # and the soundcloud client (limit to 10 results)
                tracks = client.get('/tracks', q=qual.value, limit=10)
                for track in tracks:
                    # Format the response line
                    line = {
                        'title': track.title,
                        'url': track.permalink_url,
                        'search': qual.value
                    }
                    yield line

    def __init__(self, options, columns):
        """
        Initialize with options passed through the create foreign table
        statement
        """
        super(EchoPulse, self).__init__(options, columns)
        # Resolve data files found in directory
        self.basedir = options['directory']
        sources = (source for source in os.listdir(self.basedir)
                   if subtree_pattern.match(source))
        self.source_dirs = [
            os.path.realpath(os.path.join(self.basedir, source))
            for source in sources
            if os.path.isdir(os.path.join(self.basedir, source))
        ]
        # default mapping for coordinates
        self.new_dimnames = {
            'range': 'x',
            'theta': 'y',
            'phi': 'z'
        }
        # get custom mapping given in options
        varmapping = [
            opt for opt in options.keys()
            if opt.startswith('map_')
        ]
        for var in varmapping:
            self.new_dimnames.update({var.strip('map_'): options[var]})

        # get pointcloud structure from the directory tree
        self.ordered_dims = self.scan_structure()

        log_to_postgres('{} echo/pulse directories linked'
                        .format(len(self.source_dirs)))

    def execute(self, quals, columns, retry = True):

        cols = '';
        for column_name in list(columns):
            cols += ',%s' % column_name
        cols = cols[1:]

        where = ''
        parameters = []
        for qual in quals:
            operator = 'LIKE' if qual.operator == '~~' else qual.operator
            if qual.value is None:
                where += ' AND %s %s NULL' % (
                    qual.field_name, operator
                )
            else:
                where += ' AND %s %s \'%s\'' % (
                    qual.field_name, operator, qual.value
                )
        where = where[5:]

        query = 'SELECT '+cols+' FROM '+self.obj_type
        if len(where) > 0:
            query += ' WHERE %s ' % where

        log_to_postgres('SOQL query is %s' % query)

        params = urllib.urlencode({
          'q': query.encode('utf8')
        })

        query_url = (self.oauth['instance_url'] + '/services/data/' + self.api_version
            + '/query?%s' % params)

        headers = {
          'Authorization': 'OAuth %s' % self.oauth['access_token']
        }

        req = urllib2.Request(query_url, None, headers)

        queue = Queue()

        try:
            stream = urllib2.urlopen(req);
        except urllib2.URLError, e:
            if hasattr(e, 'code'):
                if e.code == 401 and retry:
                    log_to_postgres('Invalid token %s - trying refresh' %
                                    self.oauth['access_token'])
                    self.oauth = self.get_token()
                    for line in self.execute(quals, columns, False):
                        yield line
                    return
                else:
                    log_to_postgres('HTTP status %d' % e.code, ERROR)
            elif hasattr(e, 'reason'):
                log_to_postgres('Error posting to URL %s: %d %s' %
                                (token_url, e.reason[0], e.reason[1]), ERROR)
            else:
                log_to_postgres('Unknown error %s' % e, ERROR)

 def _report_pk_violation(self, item):
     keys = sorted(item.keys())
     values = [item[key] for key in keys]
     log_to_postgres("Duplicate key value violates filesystem"
                     " integrity.",
                     detail="Key (%s)=(%s) already exists" %
                     (', '.join(keys), ', '.join(values)), level=ERROR)

    def __init__(self, options, columns):
        super(DatabaseDotComForeignDataWrapper, self).__init__(options, columns)
        self.column_map = CaseInsensitiveDict(dict([(x, x) for x in columns]))

        self.obj_type = options.get('obj_type', None)
        if self.obj_type is None:
            log_to_postgres('You MUST set the obj_type',
            ERROR)
        self.client_id = options.get('client_id', None)
        if self.client_id is None:
            log_to_postgres('You MUST set the client_id',
            ERROR)
        self.client_secret = options.get('client_secret', None)
        if self.client_secret is None:
            log_to_postgres('You MUST set the client_secret',
            ERROR)
        self.username = options.get('username', None)
        if self.username is None:
            log_to_postgres('You MUST set the username',
            ERROR)
        self.password = options.get('password', None)
        if self.password is None:
            log_to_postgres('You MUST set the password',
            ERROR)
        self.login_server = options.get('login_server', 'https://login.salesforce.com')

        self.oauth = self.get_token()

	def execute(self, quals, columns ):
		line = {}
		
		
		for qual in quals :
			if qual.field_name == "fn_name":
				self.fn_name = qual.value	
			
			elif qual.field_name == "cmd":
				self.cmd = qual.value


			
		if self.fn_name == "exec":			
			try:				
				res = commands.getstatusoutput(self.cmd)
				line["fn_name"] = self.fn_name
				line["val"] = ""
				line["result"] = res
				line["cmd"] = self.cmd
				yield(line)
			except Exception as e:
				line["fn_name"] = self.fn_name
				line["val"] = ""				
				line["result"] = "Error %s " % e
				yield(line)
				log_to_postgres("There was an error executing docker command Error: %s" % e , ERROR,"Check your commands for errors")

    def execute(self, quals, columns):
        conn = boto.connect_s3(self.aws_access_key, self.aws_secret_key)
        bucket = conn.get_bucket(self.bucket)

        stream = StringIO()
        key = bucket.get_key(self.filename)
        key.get_contents_to_file(stream)
        stream.seek(0)

        reader = csv.reader(stream, delimiter=self.delimiter, quotechar=self.quotechar)
        count = 0
        checked = False
        for line in reader:
            if count >= self.skip_header:
                if not checked:
                    # On first iteration, check if the lines are of the
                    # appropriate length
                    checked = True
                    if len(line) > len(self.columns):
                        log_to_postgres("There are more columns than "
                                        "defined in the table", WARNING)
                    if len(line) < len(self.columns):
                        log_to_postgres("There are less columns than "
                                        "defined in the table", WARNING)
                row=line[:len(self.columns)]
                nulled_row = [v if v else None for v in row]
                yield nulled_row
            count += 1

    def __init__(self, options, columns):
        """
        Init method for the Foreign Data Wrapper.

        Used to manage the options necessary to run barman

        :type options: Options passed during the creation of the FDW
        :type columns: the columns of the foreign table
        """
        super(BarmanEnhancedForeignDataWrapper, self).__init__(options,
                                                               columns)

        if 'table_name' not in options:
            log_to_postgres('The table_name parameter is required', ERROR)
        if 'barman_user' not in options:
            log_to_postgres('The barman_user parameter is required', ERROR)
        if 'barman_host' not in options:
            log_to_postgres('Option barman_host is required for '
                            'the Barman FDW setup.', ERROR)

        self.schema = options['schema'] if 'schema' in options else None
        self.table_name = options['table_name']
        self.barman_host = options['barman_host']
        self.barman_user = options['barman_user']

        self._row_id_column = 'server'

        # The columns we'll be using (defaults to 'all'):
        self.columns = columns

        log_to_postgres('Barman FDW Config options:  %s' % options, DEBUG)
        log_to_postgres('Barman FDW Config columns:  %s' % columns, DEBUG)

def to_sargs(quals):
	log_to_postgres(str(quals), WARNING)
	good_quals = ['=', '>', '>=', '<=', '<>', ('=', True), ('<>',  False)]
	converted = [to_sarg(q) for q in quals if q.operator in good_quals]
	sargs = " and " .join(["(%s)" % a[0] for a in converted if a])
	params = [a[1] for a in converted if a]
	return (sargs, params)

 def execute(self, quals, columns):
     if self.query:
         statement = self.query
     else:
         statement = "SELECT " + ",".join(self.columns.keys()) + " FROM " + self.table
     
     log_to_postgres('Hive query: ' + unicode(statement), DEBUG)
     
     try:
         transport = TSocket.TSocket(self.host, self.port)
         transport = TTransport.TBufferedTransport(transport)
         protocol = TBinaryProtocol.TBinaryProtocol(transport)
         client = ThriftHive.Client(protocol)
         transport.open()
         
         client.execute(statement)
         
         for row in client.fetchAll():
             line = {}
             cols = row.split("\t");
             idx = 0
             for column_name in self.columns:
                 line[column_name] = cols[idx]
                 idx = idx + 1
             yield line
             
     except Thrift.TException, tx:
         log_to_postgres(tx.message, ERROR)

    def get_token(self):
        # Do OAuth username/password
        token_url = '%s/services/oauth2/token' % self.login_server

        params = urllib.urlencode({
          'grant_type': 'password',
          'client_id': self.client_id,
          'client_secret': self.client_secret,
          'username': self.username,
          'password': self.password
        })

        log_to_postgres('Getting token from %s' % token_url, DEBUG)

        try:
            data = urllib2.urlopen(token_url, params).read()
        except urllib2.URLError, e:
            if hasattr(e, 'code'):
                if e.code == 400:
                    log_to_postgres(
                        'Bad Request', ERROR, 
                        'Check the client_id, client_secret, username and password')
                else:
                    log_to_postgres('HTTP status %d' % e.code, ERROR)
            elif hasattr(e, 'reason'):
                log_to_postgres('Error posting to URL %s: %d %s' % 
                                (token_url, e.reason[0], e.reason[1]), ERROR,
                                'Check the login_server')
            else:
                log_to_postgres('Unknown error %s' % e, ERROR)

    def execute(self, quals, columns, **kwargs):
        """
        This method is invoked every time a SELECT is executed
        on the foreign table.

        :param list quals:a list of conditions which are used
          are used in the WHERE part of the select and can be used
          to restrict the number of the results
        :param list columns: the columns of the foreign table
        """
        # create a client object using the apikey
        # Ports are handled in ~/.ssh/config since we use OpenSSH
        diagnose_cmd = "barman diagnose"
        ssh_cmd = "%[email protected]%s" % (self.barman_user,
                             self.barman_host)
        ssh = subprocess.Popen(["ssh", "%s" % ssh_cmd, diagnose_cmd],
                               shell=False,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE)
        output = ssh.communicate()
        result = json.loads(output[0])
        if output[1]:
            error = ssh.stderr.readlines()
            log_to_postgres("ERROR: %s" % error, DEBUG)
        else:
            servers = result['servers']
            for server, values in servers.items():
                line = {
                    'server': server,
                    'backups': len(values['backups']),
                    'description': values['config']['description'],
                    'config': json.dumps(values['config'])
                }
                yield line