Python preprocessor.Preprocessor类代码示例

 def process(self,file):
     Preprocessor.process(self,file)
     ir = InputReader(file)
     ir.read()
     cqpf = CQPFormat(ir.getText())
     pos = cqpf.getColumn(self.column)
     for i in range(2,len(pos)): # ignore first two pos ...
         uni =  (pos[i])[0:3]
         bi = (pos[i-1])[0:3] + "_" + uni
         tri = (pos[i-2])[0:3] + "_" + bi
               
         if uni not in self.unilexicon:
             self.unilexicon[uni] = 0
         self.unilexicon[uni] += 1
         
         if bi not in self.bilexicon:
             self.bilexicon[bi] = 0
         self.bilexicon[bi] += 1
         
         if tri not in self.trilexicon:
             self.trilexicon[tri] = 0
         self.trilexicon[tri] += 1
         
         
         self.count += 1

    def read(self, filename=None, preprocess=True, **defines):
        """Preprocess, read and parse itp file *filename*.

        Any keywords in *defines* are use to modify the default preprocessor
        variables (see
        :meth:`gromacs.fileformats.preprocessor.Preprocessor.parse` for
        details). Setting *preprocess* = ``False`` skips the preprocessing
        step.
        """
        self._init_filename(filename)

        if preprocess:
            kwargs = self.defines.copy()
            kwargs['commentchar'] = self.commentchar
            kwargs['clean'] = True
            ppitp = Preprocessor(self.real_filename, **kwargs)
            ppitp.parse(**defines)
            itp = ppitp.StringIO()
        else:
            itp = open(self.real_filename)

        try:
            stream = OneLineBuffer(itp.next)
            self.parse(stream)
        finally:
            itp.close()

 def __init__(self):
     self.unilexicon = {}
     self.bilexicon = {}
     self.trilexicon = {}
     
     self.count = 0
     self.column = 1
     Preprocessor.__init__(self)

    def _test_bg_subtraction2(self):
        p = Preprocessor(10)
        s = p.load_npy('./test.npy')
        generator = DataPreparator("", "", 512)
        samples1=len(s[0,:])
        
        snew,sbg = generator.bg_subtraction(s)
        samples2=len(snew[0,:])

        self.assertGreater(samples1, samples2)

 def process(self,file):
     Preprocessor.process(self,file)
     ir = InputReader(file)
     ir.read()
     cqpf = CQPFormat(ir.getText())
     for word in cqpf.getColumn(self.column):
         if word not in self.lexicon:
             self.lexicon[word] = 0
         self.lexicon[word] += 1
         self.count += 1

def voodooExpectHeader( input, output, pathToRemoveFromIdentifier, voodooDBFile, includes, defines, preIncludes, trace = False ):
	inputLines = _readLinesOfFile( input )
	perFileSettings = PerFileSettings( inputLines )
	preprocessor = Preprocessor( input, output, inputLines, pathToRemoveFromIdentifier )

	iterator = VoodooMultiplexerIterator( perFileSettings, voodooDBFile )
	iterator.process( input, includes = includes, defines = defines, preIncludes = preIncludes )

	out = preprocessor.headerOfHeader() + '\n'
        
	return out

def main(args):
    print("Athene Preprocessor v. 0.1")
    if (len(args) > 2):        
        source_filename, target_filename, include_filenames = parse_args(args)
        if ("" == source_filename or "" == target_filename):
            print("arguments error")
        preprocessor = Preprocessor(source_filename, target_filename, include_filenames)
        preprocessor.run()
        print("ok!")
    else:
        print("Usage:")
        print("\tathp -s source -t target [-i file file file]")    

    def track(self):
        print sys.argv
        cam = cv2.VideoCapture(int(sys.argv[1]))
        cam.set(cv2.cv.CV_CAP_PROP_FRAME_WIDTH, 640)
        cam.set(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT, 480)
        self._initialize_windows()

        p = Preprocessor()
        hs = HandSegment()

        positions = []
        self.count = 0
        self.skip_frames = 0
        x, y, w, h = 0, 0, 0, 0
        prev_x, prev_y, prev_w, prev_h = 0, 0, 0, 0
        while True:
            frame = self.get_frame(cam)
            if type(frame) == type(None):
                continue
            p.process(frame)


            hand = self.get_biggest_hand(frame, prev_x, prev_y, prev_w, prev_h)
            # print hand
            if not hand == []:
                x, y, w, h = hand
                prev_x, prev_y, prev_w, prev_h = hand
                centerx = x + w / 2
                centery = y + h / 2
                # Drawing rectangle around the hand
                cv2.rectangle(frame, (x, y), (x+w, y+w), (0, 0, 0), 1)

                # pointerx, pointery = hs.get_pointer(frame, x, y, w, h)

                # cv2.imshow("pointer", frame[max(y-h, 0):y+h, x:x+w+w/4])
            else:
                x, y, w, h = -1, -1, prev_w, prev_h
                centerx = -1
                centery = -1
            positions.append([centerx, centery])
            # Action
            skip_frames = self.motion(positions, w, h)
            # Drawing line of motion
            self._draw_motion(frame, positions)
            cv2.imshow("display", frame)


            ch = 0xFF & cv2.waitKey(1)
            if ch == 27:
                break

        cv2.destroyAllWindows()

def voodoo( input, output, pathToRemoveFromIdentifier, voodooDBFile, includes, defines, preIncludes, trace = False ):
	inputLines = _readLinesOfFile( input )
	perFileSettings = PerFileSettings( inputLines )
	preprocessor = Preprocessor( input, output, inputLines, pathToRemoveFromIdentifier )

	out = preprocessor.header()
	out += '#include <VoodooCommon/Common.h>\n\n'
	iterator = VoodooMultiplexerIterator( perFileSettings, voodooDBFile )
	iterator.process( input, includes = includes, defines = defines, preIncludes = preIncludes )
	out += iterator.iter()
	out += preprocessor.switchToExpectation()
	out += '#include "VoodooCommon/All.h"\n\n'
	out += iterator.expect()
	out += preprocessor.footer()
	return out

def main():

    # create lexer and parser instances:
    lexicon_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "lexicon")
    grammar_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "grammar")
    lexer = Lexer(lexicon_file, False)
    parser = Parser(grammar_file, lexer.lexicon_dict.keys())

    # run tests:
    for test in tests:

        # create preprocessor instance
        preprocessor_instance = Preprocessor(prefix + test)
        chunks = preprocessor_instance.get_chunks()
        ok = try_parse_program(chunks, lexer, parser)
        print("test " + test + " " + ("PASSED" if ok else "FAILED"))

 def process(self,file):
     Preprocessor.process(self,file)
     ir = InputReader(file)
     ir.read()
     cqpf = CQPFormat(ir.getText())
     pos = cqpf.getColumn(self.column)
     for i in range(2,len(pos)): # ignore first two pos ...
         uni =  (pos[i])[0:3]
         bi = (pos[i-1])[0:3] + "_" + uni
         tri = (pos[i-2])[0:3] + "_" + bi
         self.counts[self.posdict[uni]][self.filecount] += 1
         self.counts[self.posdict[bi]][self.filecount] += 1
         self.counts[self.posdict[tri]][self.filecount] += 1
         
         self.count += 1
     for x in self.posnames:
         self.counts[self.posdict[x]][self.filecount] /= float(len(pos)-3)
     self.filecount += 1

    def contains_preprocessor_constructs(self):
        """Check if file makes use of any preprocessor constructs.

        The test is done by running the file through the
        :class:`~gromacs.fileformats.preprocessor.Preprocessor` (while
        stripping all empty and lines starting with a comment character. This
        is compared to the original file, stripped in the same manner. If the
        two stripped files differ from each other then the preprocessor altered
        the file and preprocessor directives must have been involved and this
        function returns ``True``.

        .. versionadded: 0.3.1
        """
        from itertools import izip

        kwargs = self.defines.copy()
        kwargs['commentchar'] = self.commentchar
        kwargs['clean'] = True
        kwargs['strip'] = True
        ppitp = Preprocessor(self.real_filename, **kwargs)
        ppitp.parse()
        pp_lines = ppitp.StringIO().readlines()

        def strip_line(line):
            s = line.strip()
            return len(s) == 0 or s.startswith(self.commentchar)
        raw_lines = [line for line in open(self.real_filename) if not strip_line(line)]

        if len(pp_lines) != len(raw_lines):
            self.logger.debug("File %r is preprocessed (pp: %d vs raw %d lines (stripped))",
                              self.real_filename, len(pp_lines), len(raw_lines))
            return True
        for linenum, (raw, pp) in enumerate(izip(raw_lines, pp_lines)):
            if raw != pp:
                self.logger.debug("File %r is preprocessed. Difference at (stripped) line %d",
                                  self.real_filename, linenum)
                self.logger.debug("preprocessed: %s", pp)
                self.logger.debug("original:     %s", raw)
                return True
        self.logger.debug("File %r does not appear to contain recognized preprocessing directives",
                          self.real_filename)
        return False

    def get_important_vars(cfg, dat):
        '''
        This method does Feature Selection.
        '''

        # Balances the dataset
        idxs_pos = dat[cfg['target']] == 1
        pos = dat[idxs_pos]
        neg = dat[dat[cfg['target']] == 0][1:sum(idxs_pos)]

        # Concatenates pos and neg, it's already shuffled
        sub_dat = pos.append(neg, ignore_index = True)

        # Imputes the data and fills in the missing values
        sub_dat = Preprocessor.fill_nans(sub_dat)

        # Changes categorical vars to a numerical form
        X = pd.get_dummies(sub_dat)

        #### Correlation-based Feature Selection ####

        # Computes correlation between cfg['target'] and the predictors
        target_corr = X.corr()[cfg['target']].copy()
        target_corr.sort(ascending = False)

        # Sorts and picks the first x features
        # TODO: get optimal x value automatically
        tmp = abs(target_corr).copy()
        tmp.sort(ascending = False)
        important_vars = [tmp.index[0]]
        important_vars.extend(list(tmp.index[2:52])) # removes other target

        #### Variance-based Feature Selection ####

        #sel = VarianceThreshold(threshold = 0.005)
        #X_new = sel.fit_transform(X)

        #### Univariate Feature Selection ####

        #y = X.TARGET_B
        #X = X.drop("TARGET_B", axis = 1)

        #X_new = SelectKBest(chi2, k = 10).fit_transform(X.values, y.values)

        #### Tree-based Feature Selection ####

        #clf = ExtraTreesClassifier()
        #X_new = clf.fit(X.values, y.values).transform(X.values)

        #aux = dict(zip(X.columns, clf.feature_importances_))
        #important_vars = [i[0] for i in sorted(
        #    aux.items(), key = operator.itemgetter(0))]

        return important_vars

def externalVoodoo( input, output, linkTo, pathToRemoveFromIdentifier = "", trace = False ):
	inputLines = _readLinesOfFile( input )
	perFileSettings = PerFileSettings( inputLines )
	preprocessor = Preprocessor( linkTo, output, inputLines, pathToRemoveFromIdentifier )

	out = preprocessor.externalHeader()
	out += '#include "VoodooConfiguration.h"\n'
	out += '#include <VoodooCommon/Common.h>\n\n'
	out += "namespace External\n{\n\n"
	iterator = VoodooMultiplexerIterator( perFileSettings )
	iterator.process( input )
	out += iterator.iter()
	out += "\n}\n\n"
	out += preprocessor.externalSwitchToExpectation()
	out += '#include "VoodooCommon/All.h"\n\n'
	out += "namespace External\n{\n\n"
	out += iterator.expect()
	out += "\n}\n\n"
	out += preprocessor.externalFooter()
	return out

    def _createWidgets(self):
        self.SetBackgroundColour((60,60,60))
        self.SetForegroundColour((230,230,230))

        self.processSysIncCb = wx.CheckBox(self, -1, u"Process #include <...> files")
        self.processSysIncCb.SetBackgroundColour((100,100,100))

        sysIncDirs, appIncDirs = Preprocessor.getDefaultIncDirs()
        self._createSysIncWidgets(sysIncDirs)
        self._createAppIncWidgets(appIncDirs)
        self._createPredefMacroWidgets()
        self._createSaveOptionWidgets()