Python importer.Importer类代码示例

def dispatch(request, *args, **kw):
    """
        Perform somes checks, call the importer, and returns.
        Arguments of the method MUST be passed as pickle in POST data.
        This method is not meant to be called directly with a web browser.
    """

    base = kw.pop('base')
    modules = kw.pop('modules')

    # Create Importer() if not already present in session.
    imp = Importer()

    full_path = base.replace('/', '.') + '.' + modules.replace('/', '.')
    mod, met = full_path.rsplit('.', 1)
    module = __import__(mod, {}, {}, [''])
    callee = getattr(module, met)
    if not hasattr(callee, '__exportable__'):
        logger.debug("Exporter: method not exportable: " + full_path)
        return (200, Exception("Method not exportable"))

    # GET request ?
    # Only method call without args or attributes. (from a webbrowser)
    if request.method == 'GET':
        try:
            return (200, imp.get(mod, met))
        except ImporterError, e:
            logger.debug("Exporter: Catched: " + e.traceback)
            return (500, {'msg': e.msg, 'traceback': e.traceback})

def get_discharge_page(uuid):
	data_importer = Importer()
	snapshot_data = data_importer.retrieve_snapshot_data(uuid)
	if snapshot_data == None:
		return render_template('patient_not_found.html')
	else:
		logging.debug(str(snapshot_data))
		return render_template('discharge_page.html', discharge_data=snapshot_data)

def import_data(request, key):
    """取り込み元の内容を取り込む"""

    importer = Importer()

    source = Source.get(db.Key(key))
    result = importer.import_data(source)
    return HttpResponseRedirect(reverse('admin.views.index'))

def start_import(data_dir, dry_run = False, tabular = True, variable = True, excel = True, delete = True):
    """
        Comienza el proceso de importación propiamende dicho. Hasta ahora todo eran preparaciones para
        tener los datos organizados y disponibles.
    """
    from importer import Importer
    importer = Importer(data_dir)

    if tabular:
        print u"Realizando la importacion tabular" 
        if not dry_run:
            importer.import_tabular_data()
    if variable:
        print u"Realizando la importacion variable"
        if not dry_run:
            importer.import_variable_data()
    if excel:
        print u"Realizando la importacion del datos_extra"
        if not dry_run:
            importer.import_excel(variable)
    print u"Eliminando los datos no marcados"
    # TODO: Crear gestionar_jornadas_continuas que meta las matriculas de 
    # los alumnos de los grupos en las asignaturas T.[INF|PRI]
    # de forma automatica, ahora mismo esto se hace si se importa del excel,
    # y si este no ha variado no se importa y elimina las matriculas
    # importer.gestionar_jornadas_continuas()
    importer.delete_non_used_data(not dry_run and delete)
    return importer.log_result

class MyFrame(wx.Frame):
    """ We simply derive a new class of Frame. """
    def __init__(self, parent, title, opts, sources):
        wx.Frame.__init__(self, parent, title=title, size=((800 if opts.verbosity < 2 else 1100), 400))
        self.lock = Lock()
        self.opts = opts
        self.control = wx.TextCtrl(self, style=wx.TE_MULTILINE)
        self.control.SetEditable(False)
        self.Show(True)
        self.twiddle_mode = 2
        self.twiddle_next = 0
        self.twiddle_me = '|/-\\'
        self.twiddle_size = len(self.twiddle_me)
        self.importer = Importer(self, sources, self.opts)
        self.Bind(wx.EVT_CLOSE, self.onClose)

    def onClose(self, event):
        self.importer.interrupt.set()
        while self.importer.isAlive():
            self.importer.join(1)
            wx.Yield()
        self.Destroy()

    def __append_text(self, s):
        with self.lock:
            self.control.AppendText(s)

    def logger(self, s):
        # print s
        if self.twiddle_mode != 2:
            re_twiddle = True
            self.twiddle(2)
        else:
            re_twiddle = False
        self.__append_text("%s\n" % (s,))
        if re_twiddle:
            self.twiddle(0)

    def twiddle(self, mode):
        # Mode (0, 1, 2) == (start (add first twiddle), advance, erase)
        self.twiddle_mode = mode
        if mode == 0:
            # append
            self.twiddle_next = 0
            self.__append_text(self.twiddle_me[self.twiddle_next])
        elif mode == 1:
            # replace
            lastPos = self.control.GetLastPosition()
            self.control.Replace(lastPos-1, lastPos, self.twiddle_me[self.twiddle_next])
        elif mode == 2:
            # erase
            lastPos = self.control.GetLastPosition()
            self.control.Remove(lastPos-1, lastPos)
        # Advance
        self.twiddle_next = (self.twiddle_next + 1) % self.twiddle_size

def start(path, username, password):

    rest = RestClient(username)
    webdav = WebDav(username, password)

    login_attempts = 0

    if not rest.login(username, password):
        login_attempts += 1
        print("Invalid username or password." + username)
        if login_attempts < 5:
            start()
        else:
            exec(1)

    log = logging.getLogger("lims_import")
    log.setLevel(logging.DEBUG)

    sh = logging.StreamHandler()
    sh.setLevel(logging.DEBUG)

    fh = logging.FileHandler("lims-import.log")
    fh.setLevel(logging.DEBUG)
    fh.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] [%(filename)s:%(lineno)d] \t %(message)s"))

    log.addHandler(sh)
    log.addHandler(fh)

    if not webdav.is_available():
        log.error("Unable to connect to Owncloud.")
        exit(1)

    for project in path:
        if not os.path.exists(project):
            log.error("{} not found.".format(project))
            exit(1)

    total = 0
    for project in path:
        total += 1
        for root, dirs, files in os.walk(project):
            total += len(files)
            total += len(dirs)

    bar = progressbar.ProgressBar(max_value=total, redirect_stdout=True)

    def update_progress():
        bar.update(bar.value + 1)

    importer = Importer(username, password)
    importer.on_update += update_progress

    for project in path:
        importer.start(project)

    def __init__(self, fileName, redisHost, redisPort, redisDB = 0):
        """
        Init a geonames cities importer
        @param fileName path to the geonames datafile
        @param redisConn redis connection
        """

        Importer.__init__(self, fileName ,redisHost, redisPort, redisDB)

        fileNames = fileName.split(',')
        self.fileName = fileNames[0]
        self.adminCodesFileName = fileNames[1] if len(fileNames) > 1 else None
        self._adminCodes = {}

def get_importer(config):
    importer = Importer()
    importer.load(
        dict(key='service_classes', module_names=config.SERVICES, class_name='Service'),
    )  # load all modules here

    services = []
    for service_class in importer.service_classes:
        srv = service_class()
        srv.name = service_class.__module__
        services.append(srv)
    importer.services = services

    return importer

 def __init__(self, fileName , redisHost, redisPort, redisDB, locationFileName=None, regionsFileName=None, countriesFileName=None):
     Importer.__init__(self, fileName, redisHost, redisPort, redisDB)
     if not locationFileName:
         dirname = os.path.dirname(fileName)
         locationFileName = os.path.join(dirname, 'GeoLiteCity-Location.csv')
     if not regionsFileName:
         dirname = os.path.dirname(fileName)
         regionsFileName = os.path.join(dirname, 'regions.csv')
     if not countriesFileName:
         dirname = os.path.dirname(fileName)
         countriesFileName = os.path.join(dirname, 'countries.csv')
     self.locationFileName = locationFileName
     self.regionsFileName = regionsFileName
     self.countriesFileName = countriesFileName

def main():
    try:
        url, key = sys.argv[1:]
    except (IndexError, ValueError):
        sys.stderr.write('Usage: %s <redmine_url> <auth_key>\n'
                         % sys.argv[0])
        sys.exit(1)

    transport = RedmineHttpTransport(url, key)
    api = Redmine(transport)
    importer = Importer(api)
    source = CsvSource.from_buffer(sys.stdin.readlines(),
                                   IssueModel)
    results = importer.import_from_source(source)
    print 'Created issue ids:', ', '.join(results)

 def setUp(self):
     """ Initialize test environment. """
     # TODO: clean up spvd database beforehand
     self.imp = Importer()
     self.imp['distant_url'] = 'https://192.168.2.81/exporter/'
     self.groups = []
     self.checks = []
     self.objects = []

    def create(chapter_id, kctype, file_kcids=None, debug=False, train_pct= 0.6, dev_pct=0.2, index=""):
        file_name =  IntermediateData.input_path + "homework_xref_" + str(chapter_id) + "_decompressed.csv"

        print "CLEANING DATA..."
        imp = Importer()
        df_imp = imp.get_data(file_name, kctype, debug)


        kc_col = "kc"
        if kctype == "tom":
            kc_definition = [kc_col]
        else:
            #TODO:  We should extracting features from exercises, but feature extracting does not support this
            kc_definition = [kc_col] #",exercise_id]"

        # Sort data
        df_imp = FeatureExtractor.sort(df_imp)

        for kc in df_imp[kc_col].unique():
            print kc_col, kc

            # GET SUBSET of data
            df = df_imp[ df_imp[kc_col] == kc]

            # "EXTRACTING FEATURES..."
            fe = FeatureExtractor(default_kc=kc_definition)
            df_features = fe.df_to_features(df, sort_data=False) # No need to sort data

            # "SUBSET OF DATA..."
            df_train, df_dev, df_test = split(df_features, train_pct, dev_pct)

            # "STORING"
            chapter_data = IntermediateData(chapter_id, kc, df_features, train_rows=df_train, dev_rows=df_dev, test_rows=df_test)

            filename = os.path.join(IntermediateData.output_path, index, IntermediateData.get_filename(kctype, kc))
            f_output = open( filename, "w")
            pickle.dump(chapter_data, f_output)
            f_output.close()

            #hy commented
            #df_features.to_csv(filename + ".csv")

            if file_kcids != None:
                file_kcids.write( str(chapter_id)  +","+  str(kc) + "," + index + "\n")
                file_kcids.flush()

    def post(self):
        """
            Does the import, and shows any errors.
        """

        self.msg = ""
        error = True
        importer = Importer(DataAccessor(self.addErrorMessage))

        try:
            target = self.request.POST.get("newFile").file.read()
            importer.parse(StringIO(target))

        except IOError:
            self.msg = "Please select a valid file to import"

        except Usage, err:
            self.msg = err.msg

    def __init__(self, options):
        self.importer = Importer()

        # try read all the files in the dir
        if options.dir_name is not None:
            self._files = [f for f in self.get_files(options.dir_name)]
        else:
            self._files = [options.file_name] # only one file
        self._algo_name = options.algorithm if (options.algorithm is not None) else 'bfs'
        self.options = options

 def __init__(self, parent, title, opts, sources):
     wx.Frame.__init__(self, parent, title=title, size=((800 if opts.verbosity < 2 else 1100), 400))
     self.lock = Lock()
     self.opts = opts
     self.control = wx.TextCtrl(self, style=wx.TE_MULTILINE)
     self.control.SetEditable(False)
     self.Show(True)
     self.twiddle_mode = 2
     self.twiddle_next = 0
     self.twiddle_me = '|/-\\'
     self.twiddle_size = len(self.twiddle_me)
     self.importer = Importer(self, sources, self.opts)
     self.Bind(wx.EVT_CLOSE, self.onClose)