Python Util.get_log方法代码示例

# 需要导入模块: from util import Util [as 别名]
# 或者: from util.Util import get_log [as 别名]
# -*- coding: utf-8 -*-

from util import Util
log = Util.get_log('factory')

class Factory:
    """
    Object and class Factory (Pattern).
    Based on: http://stackoverflow.com/questions/2226330/instantiate-a-python-class-from-a-name
    """

    def create_obj(self, configdict, section):
        class_obj_inst = None

        # Get value for 'class' property
        class_string = configdict.get(section, 'class')
        try:
            if not class_string:
                raise ValueError('Class name not defined in section %s.' % section)

            # class object from module.class name
            class_obj = self.class_forname(class_string)

            # class instance from class object with constructor args
            class_obj_inst = self.new_instance(class_obj, configdict, section)
        except Exception, e:
            log.error("cannot create object instance from class '%s' e=%s" % (class_string, str(e)))
            raise e

        return class_obj_inst

# 需要导入模块: from util import Util [as 别名]
# 或者: from util.Util import get_log [as 别名]
# -*- coding: utf-8 -*-
#
# Component base class for ETL.
#
# Author: Just van den Broecke
#
from util import Util, ConfigSection
from packet import FORMAT

log = Util.get_log('component')


class Component:
    """
    Abstract Base class for all Input, Filter and Output Components.

    """

    def __init__(self, configdict, section, consumes=None, produces=None):
        self.configdict = configdict
        self.cfg = ConfigSection(configdict.items(section))
        self.next = None
        self.output_format = produces
        self.input_format = consumes

    def process(self, packet):
        # Do something with the data
        packet = self.invoke(packet)

        # If there is a next component, let it process
        if self.next:

# 需要导入模块: from util import Util [as 别名]
# 或者: from util.Util import get_log [as 别名]
# -*- coding: utf-8 -*-
#
# Chain class: holds pipeline of components.
#
# Author: Just van den Broecke
#

from factory import factory
from packet import Packet
from util import Util

log = Util.get_log('chain')


class Chain:
    """
    Holder for single invokable pipeline of components
    A Chain is basically a singly linked list of Components
    Each Component executes a part of the total ETL.
    Data along the Chain is passed within a Packet object.
    The compatibility of input and output for linked
    Components is checked when adding a Component to the Chain.
    """

    def __init__(self, chain_str, config_dict):
        self.first_comp = None
        self.cur_comp = None
        self.config_dict = config_dict
        self.chain_str = chain_str

    def assemble(self):

# 需要导入模块: from util import Util [as 别名]
# 或者: from util.Util import get_log [as 别名]
# -*- coding: utf-8 -*-
#
# PostGIS support wrapper.
#
# Author: Just van den Broecke
#
from util import Util

log = Util.get_log("postgis")

try:
    import psycopg2
    import psycopg2.extensions
except ImportError:
    log.error("cannot find package psycopg2 for Postgres client support, please install psycopg2 first!")
    # sys.exit(-1)


class PostGIS:
    def __init__(self, config):
        # Lees de configuratie 
        self.config = config

    def initialiseer(self, bestand):
        log.info('Connecting...')
        self.connect(True)

        log.info('executing sql script...')
        try:
            script = open(bestand, 'r').read()
            self.cursor.execute(script)

# 需要导入模块: from util import Util [as 别名]
# 或者: from util.Util import get_log [as 别名]
# -*- coding: utf-8 -*-
#
# Main Stetl program.
#
# Author: Just van den Broecke
#
from etl import ETL
from factory import factory
from util import Util
from version import __version__
import argparse  # apt-get install python-argparse
import inspect
import os
import sys

log = Util.get_log('main')


def parse_args(args_list):
    log.info("Stetl version = %s" % __version__)

    argparser = argparse.ArgumentParser(description='Invoke Stetl')
    argparser.add_argument('-c ', '--config', type=str, help='ETL config file in .ini format', dest='config_file',
                           required=False)

    argparser.add_argument('-s ', '--section', type=str, help='Section in the config file to execute, default is [etl]',
                           dest='config_section', required=False)

    argparser.add_argument('-a ', '--args', type=str,
                           help='Arguments or .properties files to be substituted for symbolic {argN}s in Stetl config file,\
                                as -a "arg1=foo arg2=bar" and/or -a args.properties, multiple -a options are possible',

# 需要导入模块: from util import Util [as 别名]
# 或者: from util.Util import get_log [as 别名]
# -*- coding: utf-8 -*-
#
# Input classes for ETL.
#
# Author: Just van den Broecke
#
from util import Util
from component import Component

log = Util.get_log('input')


class Input(Component):
    """
    Abstract Base class for all Input Components.

    """

    def __init__(self, configdict, section, produces):
        Component.__init__(self, configdict, section, consumes=None, produces=produces)

        log.info("cfg = %s" % self.cfg.to_string())

    def invoke(self, packet):
        return self.read(packet)

    def read(self, packet):
        return packet

# 需要导入模块: from util import Util [as 别名]
# 或者: from util.Util import get_log [as 别名]
# -*- coding: utf-8 -*-
#
# Output base class for ETL.
#
# Author: Just van den Broecke
#
from component import Component
from util import Util

log = Util.get_log('output')


class Output(Component):
    """
    Abstract Base class for all Output Components.

    """

    def __init__(self, configdict, section, consumes):
        Component.__init__(self, configdict, section, consumes=consumes, produces=None)
        log.info("cfg = %s" % self.cfg.to_string())

    def invoke(self, packet):
        packet = self.write(packet)
        packet.consume()
        return packet

    def write(self, packet):
        return packet

# 需要导入模块: from util import Util [as 别名]
# 或者: from util.Util import get_log [as 别名]
# -*- coding: utf-8 -*-
#
# Output base class for ETL.
#
# Author: Just van den Broecke
#
from component import Component
from util import Util, etree

log = Util.get_log("output")


class Output(Component):
    """
    Abstract Base class for all Output Components.

    """

    def __init__(self, configdict, section, consumes):
        Component.__init__(self, configdict, section, consumes=consumes, produces=None)
        log.info("cfg = %s" % self.cfg.to_string())

    def invoke(self, packet):
        packet = self.write(packet)
        packet.consume()
        return packet

    def write(self, packet):
        return packet

# 需要导入模块: from util import Util [as 别名]
# 或者: from util.Util import get_log [as 别名]
# -*- coding: utf-8 -*-
#
# Merger Component base class for ETL.
#
# Author: Just van den Broecke
#

import random
from util import Util
from component import Component

log = Util.get_log('merger')


class Merger(Component):
    """
    Component that merges multiple Input Components into a single Component.
    Use this for example to combine multiple input streams like API endpoints.
    The Merger will embed Child Components to which actions are delegated.
    A Child Component may be a sub-Chain e.g. (Input|Filter|Filter..) sequence.
    Hence the "next" should be coupled to the last Component in that sub-Chain with
    the degenerate case where the sub-Chain is a single (Input) Component.
    NB this Component can only be used for Inputs.
    """

    def __init__(self, config_dict, child_list):
        # Assemble child list
        self.children = []
        section_name = ''
        for child in child_list:
            section_name += '-%s_%d' % (child.get_id(), random.randrange(0, 100000))

# 需要导入模块: from util import Util [as 别名]
# 或者: from util.Util import get_log [as 别名]
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Main Stetl program.
#
# Author: Just van den Broecke
#
from etl import ETL
from factory import factory
from util import Util
from version import __version__
import argparse  # apt-get install python-argparse
import inspect

log = Util.get_log("main")


def parse_args():
    log.info("Stetl version = %s" % __version__)

    argparser = argparse.ArgumentParser(description="Invoke Stetl")
    argparser.add_argument(
        "-c ", "--config", type=str, help="ETL config file in .ini format", dest="config_file", required=False
    )

    argparser.add_argument(
        "-s ",
        "--section",
        type=str,
        help="Section in the config file to execute, default is [etl]",
        dest="config_section",