Python Preprocessor.process方法代码示例

# 需要导入模块: from preprocessor import Preprocessor [as 别名]
# 或者: from preprocessor.Preprocessor import process [as 别名]
 def process(self,file):
     Preprocessor.process(self,file)
     ir = InputReader(file)
     ir.read()
     cqpf = CQPFormat(ir.getText())
     pos = cqpf.getColumn(self.column)
     for i in range(2,len(pos)): # ignore first two pos ...
         uni =  (pos[i])[0:3]
         bi = (pos[i-1])[0:3] + "_" + uni
         tri = (pos[i-2])[0:3] + "_" + bi
               
         if uni not in self.unilexicon:
             self.unilexicon[uni] = 0
         self.unilexicon[uni] += 1
         
         if bi not in self.bilexicon:
             self.bilexicon[bi] = 0
         self.bilexicon[bi] += 1
         
         if tri not in self.trilexicon:
             self.trilexicon[tri] = 0
         self.trilexicon[tri] += 1
         
         
         self.count += 1

# 需要导入模块: from preprocessor import Preprocessor [as 别名]
# 或者: from preprocessor.Preprocessor import process [as 别名]
 def process(self,file):
     Preprocessor.process(self,file)
     ir = InputReader(file)
     ir.read()
     cqpf = CQPFormat(ir.getText())
     for word in cqpf.getColumn(self.column):
         if word not in self.lexicon:
             self.lexicon[word] = 0
         self.lexicon[word] += 1
         self.count += 1

# 需要导入模块: from preprocessor import Preprocessor [as 别名]
# 或者: from preprocessor.Preprocessor import process [as 别名]
    def track(self):
        print sys.argv
        cam = cv2.VideoCapture(int(sys.argv[1]))
        cam.set(cv2.cv.CV_CAP_PROP_FRAME_WIDTH, 640)
        cam.set(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT, 480)
        self._initialize_windows()

        p = Preprocessor()
        hs = HandSegment()

        positions = []
        self.count = 0
        self.skip_frames = 0
        x, y, w, h = 0, 0, 0, 0
        prev_x, prev_y, prev_w, prev_h = 0, 0, 0, 0
        while True:
            frame = self.get_frame(cam)
            if type(frame) == type(None):
                continue
            p.process(frame)


            hand = self.get_biggest_hand(frame, prev_x, prev_y, prev_w, prev_h)
            # print hand
            if not hand == []:
                x, y, w, h = hand
                prev_x, prev_y, prev_w, prev_h = hand
                centerx = x + w / 2
                centery = y + h / 2
                # Drawing rectangle around the hand
                cv2.rectangle(frame, (x, y), (x+w, y+w), (0, 0, 0), 1)

                # pointerx, pointery = hs.get_pointer(frame, x, y, w, h)

                # cv2.imshow("pointer", frame[max(y-h, 0):y+h, x:x+w+w/4])
            else:
                x, y, w, h = -1, -1, prev_w, prev_h
                centerx = -1
                centery = -1
            positions.append([centerx, centery])
            # Action
            skip_frames = self.motion(positions, w, h)
            # Drawing line of motion
            self._draw_motion(frame, positions)
            cv2.imshow("display", frame)


            ch = 0xFF & cv2.waitKey(1)
            if ch == 27:
                break

        cv2.destroyAllWindows()

# 需要导入模块: from preprocessor import Preprocessor [as 别名]
# 或者: from preprocessor.Preprocessor import process [as 别名]
 def process(self,file):
     Preprocessor.process(self,file)
     ir = InputReader(file)
     ir.read()
     cqpf = CQPFormat(ir.getText())
     pos = cqpf.getColumn(self.column)
     for i in range(2,len(pos)): # ignore first two pos ...
         uni =  (pos[i])[0:3]
         bi = (pos[i-1])[0:3] + "_" + uni
         tri = (pos[i-2])[0:3] + "_" + bi
         self.counts[self.posdict[uni]][self.filecount] += 1
         self.counts[self.posdict[bi]][self.filecount] += 1
         self.counts[self.posdict[tri]][self.filecount] += 1
         
         self.count += 1
     for x in self.posnames:
         self.counts[self.posdict[x]][self.filecount] /= float(len(pos)-3)
     self.filecount += 1

# 需要导入模块: from preprocessor import Preprocessor [as 别名]
# 或者: from preprocessor.Preprocessor import process [as 别名]
class ALE:
    actions = [np.uint8(0), np.uint8(1), np.uint8(3), np.uint8(4), np.uint8(11), np.uint8(12)]
    current_points = 0
    next_screen = ""
    game_over = False
    skip_frames = None
    display_screen = "true"
    game_ROM = None
    fin = ""
    fout = ""
    preprocessor = None
    
    def __init__(self, display_screen, skip_frames, game_ROM):
        """
        Initialize ALE class. Creates the FIFO pipes, launches ./ale and does the "handshake" phase of communication

        @param display_screen: bool, whether to show the game on screen or not
        @param skip_frames: int, number of frames to skip in the game emulator
        @param game_ROM: location of the game binary to launch with ./ale
        """

        self.display_screen = display_screen
        self.skip_frames = skip_frames
        self.game_ROM = game_ROM

        #: create FIFO pipes
        os.system("mkfifo ale_fifo_out")
        os.system("mkfifo ale_fifo_in")

        #: launch ALE with appropriate commands in the background
        command='./../libraries/ale/ale -max_num_episodes 0 -game_controller fifo_named -disable_colour_averaging true -run_length_encoding false -frame_skip '+str(self.skip_frames)+' -display_screen '+self.display_screen+" "+self.game_ROM+" &"
        os.system(command)

        #: open communication with pipes
        self.fin = open('ale_fifo_out')
        self.fout = open('ale_fifo_in', 'w')
        
        input = self.fin.readline()[:-1]
        size = input.split("-")  # saves the image sizes (160*210) for breakout

        #: first thing we send to ALE is the output options- we want to get only image data
        # and episode info(hence the zeros)
        self.fout.write("1,0,0,1\n")
        self.fout.flush()  # send the lines written to pipe

        #: initialize the variables that we will start receiving from ./ale
        self.next_image = []
        self.game_over = True
        self.current_points = 0

        #: initialise preprocessor
        self.preprocessor = Preprocessor()

    def new_game(self):
        """
        Start a new game when all lives are lost.
        """

        #: read from ALE:  game screen + episode info
        self.next_image, episode_info = self.fin.readline()[:-2].split(":")
        self.game_over = bool(int(episode_info.split(",")[0]))
        self.current_points = int(episode_info.split(",")[1])

        #: send the fist command
        #  first command has to be 1,0 or 1,1, because the game starts when you press "fire!",
        self.fout.write("1,0\n")
        self.fout.flush()
        self.fin.readline()

        #: preprocess the image and add the image to memory D using a special add function
        #self.memory.add_first(self.preprocessor.process(self.next_image))
        return self.preprocessor.process(self.next_image)

    def end_game(self):
        """
        When all lives are lost, end_game adds last frame to memory resets the system
        """
        #: tell the memory that we lost
        # self.memory.add_last() # this will be done in Main.py
        
        #: send reset command to ALE
        self.fout.write("45,45\n")
        self.fout.flush()
        self.game_over = False  # just in case, but new_game should do it anyway

    
    def move(self, action_index):
        """
        Sends action to ALE and reads responds
        @param action_index: int, the index of the chosen action in the list of available actions
        """
        #: Convert index to action
        action = self.actions[action_index]

	#: Generate a random number for the action of player B
        action_b = random.choice(range(255))


        #: Write and send to ALE stuff
        self.fout.write(str(action)+","+str(action_b)+"\n")
#.........这里部分代码省略.........

# 需要导入模块: from preprocessor import Preprocessor [as 别名]
# 或者: from preprocessor.Preprocessor import process [as 别名]
from scipy.cluster.hierarchy import linkage, fcluster
from dionysus import PairwiseDistances,ExplicitDistances
import numpy as np

def bench_cluster(X, y, pca_n_comp):
    n = len(np.unique(y))
    pca = PCA(pca_n_comp)
    X_ = pca.fit_transform(X)
    sc = SpectralClustering(n)
    km = KMeans(n)
    sc_pred = sc.fit_predict(X_)
    km_pred = km.fit_predict(X_)
    distances = PairwiseDistances(X_.tolist())
    distances = ExplicitDistances(distances)
    singlel_pred = fcluster(linkage(ssd.squareform(distances.distances)), n, criterion='maxclust')
    print "single-linkage clustering prediction:", singlel_pred
    print "single-linkage clustering score:", adjusted_rand_score(y, singlel_pred), mutual_info_score(y, singlel_pred)
    print "spectral clustering prediction:", sc_pred
    print "spectral clustering score:", adjusted_rand_score(y, sc_pred), mutual_info_score(y, sc_pred)
    print "kmeans clustering prediction", km_pred
    print "kmeans clustering score:", adjusted_rand_score(y, km_pred), mutual_info_score(y, km_pred)
    print "ground truth labels", y


if __name__ == "__main__":
    funcs = [word_lengths_funcs, sentence_lengths_funcs, ratio_most_n_common_words, ratio_length_of_words_texts,
            lambda text: ratio_length_of_words_texts(text, 8, ge)]
    pp = Preprocessor(Prepreprocessor, funcs, use_tfidf=20)
    X, y = pp.process(['../data/abstracts/', '../data/sports', '../data/reviews'])
    bench_cluster(X, y, 3)

# 需要导入模块: from preprocessor import Preprocessor [as 别名]
# 或者: from preprocessor.Preprocessor import process [as 别名]
from dionysus import PersistenceDiagram
from draw import draw_bar_code_graph,draw_persistance_diagram

cx_method = cech
dims = 3
random_split = True
use_default_R = True
n_intervals = None
funcs = []
prepre = StemLemPrepreprocessor

funcs = [word_lengths_funcs, sentence_lengths_funcs, ratio_most_n_common_words, ratio_length_of_words_texts,
            lambda text: ratio_length_of_words_texts(text, 8, ge)]
pp = Preprocessor(prepre, funcs, use_tfidf=50)
folder_names = ['abstracts', 'sports', 'reviews']
X, y = pp.process(['../data/' + fold_n for fold_n in folder_names])
Xs = flatten([halve_group(g, random_split=random_split) for g in get_groups(X, y)])
if cx_method == alpha_shapes:
    Xs = map(lambda X: reduce_n_columns(X, n=dims), Xs)
cxs = map(cx_method, Xs)
if use_default_R:
    Rs = map(get_max_dist, Xs)
else:
    Rs = map(lambda cx: max([sx.data for sx in cx]), cxs)
diagrams = [persistence_diagram(cx, R, X, n_intervals=n_intervals) for cx, R, X in zip(cxs, Rs, Xs)]
titles = flatten([[name + '_train', name + '_test'] for name in folder_names])
cluster_distances(map(lambda diagram: [PersistenceDiagram(d, diagram[d]) for d in range(dims)], diagrams),
                  labels=titles, name="main", ps=[0,1,2])
i = 0
for diagram in diagrams:
    print diagram

# 需要导入模块: from preprocessor import Preprocessor [as 别名]
# 或者: from preprocessor.Preprocessor import process [as 别名]
class Evaluator:
    __data = []
    __label = None
    __nb = None
    __prep = None
    __total_data = []
    __total_labels = []
    __verbose = False
    __level = 0

    def __init__(self, data, label, verbose=False, verbosity_level = 5):
        print('Preprocessing data...')
        self.__prep = Prep()
        for item in data:
            self.__data.append(self.__prep.process(item))
        self.__label = label
        self.__precision = []
        self.__verbose = verbose
        self.__level = verbosity_level

    def k_fold(self, k):
        assert k > 1
        print('Starting ' + str(k) + '-fold cross-validation.')
        input('Press Enter to continue...')
        for run in range(0, k):
            print('Run ' + str(run+1))
            nb = NBC()
            testing_data = []
            training_data = []
            for idx, d in enumerate(self.__data):
                subset_size = int(len(d)/k)
                testing_data.append(d[run*subset_size:(run+1)*subset_size])
                training_data.append(d[:run*subset_size] + d[(run+1)*subset_size:])

            nb.train(training_data, self.__label)

            hits = 0
            misses = 0
            for idx, item in enumerate(self.__label):
                for doc in testing_data[idx]:
                    if nb.predict(doc) == item:
                        hits += 1
                    else:
                        misses += 1

            total_length = 0
            for item in testing_data:
                total_length += len(item)

            self.__precision.append((hits/total_length, misses/total_length))

            if self.__verbose:
                nb.info(self.__level)
                input('Press Enter to continue...')

        total = 0
        for item in self.__precision:
            print('Precision: %.2d' % (item[0]*100))
            total += item[0]
        total /= len(self.__precision)
        print('Average precision: %.2d' % (total*100))