Python dataset.Dataset类代码示例

	def read_data(dirname, return_dataset=False):
		
		ds = Dataset(dirname, Reader.get_classes())
		emails, classes = [], []

		for sentences, email_type in ds.get_text():
			ds.build_vocabulary(sentences)
			emails.append(sentences)
			classes.append(email_type)

		
		# transform word to indices
		emails = [list(map(ds.get_word_indices().get, s)) for s in emails]

		# count how many times a word appear with the ith class
		counts = np.zeros((len(ds.vocabulary), len(set(classes))))
		for i, e in enumerate(emails):
			for w in e:
				counts[w, classes[i]] += 1 


		# emails = ds.bag_of_words(emails) # using bow we dont need counts

		if return_dataset:
			return np.array(emails), np.array(classes), counts, ds
		return np.array(emails), np.array(classes), counts

    def train(self, session, dataset: Dataset):
        word_vectors = []
        for tokens in dataset.get_tokens():
            word_vectors.append(self.encode_word_vector(tokens))

        slot_vectors = []
        for iob in dataset.get_iob():
            slot_vectors.append(self.encode_slot_vector(iob))

        cost_output = float('inf')
        for _ in range(self.__step_per_checkpoints):
            indexes = np.random.choice(len(word_vectors), self.__batch_size, replace=False)
            x = [word_vectors[index] for index in indexes]
            y = [slot_vectors[index] for index in indexes]

            self.__step += 1
            _, cost_output = session.run([self.__optimizer, self.__cost],
                                         feed_dict={self.__x: x,
                                                    self.__y: y,
                                                    self.__dropout: 0.5})

        checkpoint_path = os.path.join('./model', "slot_filling_model.ckpt")
        self.__saver.save(session, checkpoint_path, global_step=self.__step)

        return cost_output

	def load_dataset(self):
		d = Dataset(self._logger)
		self._logger.info("Loading dataset...")
		self.X, self.y = d.load_csvs_from_folder(CSV_DIR)
		self._logger.info("Done loading dataset")
		self._logger.debug(str(self.X.shape))
		self._logger.debug(str(self.y.shape))

    def learn(self, dataset_name=''):
        
        if not dataset_name:
            dataset = Dataset.create_ds(self.options, prefix='learn')
        else:               
            dataset = Dataset.get_ds(self.options, dataset_name)

        try:
            while True:
                controls = self.c.getUpdates()
                state = self.izzy.getState()
                self.update_gripper(controls)

                controls = self.controls2simple(controls)
                if not all(int(c) == 0 for c in controls):
                        frame = self.bc.read_frame(show=self.options.show, record=self.options.record, state=state)
                        dataset.stage(frame, controls, state)
                
                print "supervisor: " + str(controls)
                time.sleep(0.05)
                
        except KeyboardInterrupt:
            pass
        
        dataset.commit()
        if self.options.record:
            self.bc.save_recording()

    def dataset(self, mess, args):
        """send a csv dataset to your email address
           argument : date1(Y-m-d) date2(Y-m-d) sampling period (seconds)
           ex : dataset 2017-09-01 2017-09-02 600
                """
        knownUsers = self.unPickle("knownUsers")

        if len(args.split(' ')) == 3:
            dstart = args.split(' ')[0].strip().lower()
            dend = args.split(' ')[1].strip().lower()
            step = float(args.split(' ')[2].strip().lower())
        else:
            return 'not enough arguments'

        user = mess.getFrom().getNode()
        if user in knownUsers:
            try:
                dstart = dt.strptime(dstart, "%Y-%m-%d")
            except ValueError:
                return "ValueError : time data '%s'" % dstart + " does not match format '%Y-%m-%d'"

            try:
                dend = dt.strptime(dend, "%Y-%m-%d")
            except ValueError:
                return "ValueError : time data '%s'" % dend + " does not match format '%Y-%m-%d'"

            dataset = Dataset(self, mess.getFrom(), dstart.isoformat(), dend.isoformat(), step)
            dataset.start()
            return "Generating the dataset ..."

        else:
            return "Do I know you ? Send me your email address by using the command record "

def main():

  parser = argparse.ArgumentParser(description='Remove a dataset from the cache')
  parser.add_argument('dataset_name', action="store")

  result = parser.parse_args()
  ds = Dataset(result.dataset_name)
  ds.removeDataset()

def fullsim(name, nEvents, sigma, sigmaRelErr, filters, inputNames = None):
    if not inputNames:
        dataset = Dataset(name, [name], Dataset.FULLSIM, nEvents, sigma, sigmaRelErr, filters)
    else:
        dataset = Dataset(name, inputNames, Dataset.FULLSIM, nEvents, sigma, sigmaRelErr, filters)

    dataset.entryList = 'hardPhotonList'
    return dataset

 def test_dataset(self):
     cxn = yield connectAsync()
     context = yield cxn.context()
     dir = ['', 'Test']
     dataset = 1
     datasetName = 'Rabi Flopping'
     d = Dataset(cxn, context, dataset, dir, datasetName, None)
     d.openDataset()
     d.getData()

    def test_import_dataset(self):

        ## The Pixelman dataset object.
        pds = Dataset("testdata/ASCIIxyC/")

        # The tests.

        # The number of datafiles.
        self.assertEqual(pds.getNumberOfDataFiles(), 5)

	def __init__ (self, filename="93-15_top_9.npz"):
		data = np.load(filename)
		self.teams = data["teams"]
		Dataset.__init__(self, len(self.teams))
		self.pairwise_probs = data["probs"]
		print(len(self.teams))
		
		ranking = SE(len(self.teams), self.pairwise_probs)
		self.order = ranking.get_ranking()

def main():
    while True:
        data_set_name = input("Please provide the name of the data set you want to work with: ")

        # Load, Randomize, Normalize, Discretize Dataset
        data_set = Dataset()
        data_set.read_file_into_dataset("C:\\Users\\Grant\\Documents\\School\\Winter 2016\\CS 450\\Prove03\\" + data_set_name)
        data_set.randomize()
        data_set.data = normalize(data_set.data)
        data_set.discretize()

        data_set.set_missing_data()

        # Split Dataset
        split_percentage = 0.7
        data_sets    = split_dataset(data_set, split_percentage)
        training_set = data_sets['train']
        testing_set  = data_sets['test']

        # Create Custom Classifier, Train Dataset, Predict Target From Testing Set
        id3Classifier = ID3()
        id3Classifier.train(training_set)
        predictions = id3Classifier.predict(testing_set)

        id3Classifier.display_tree(0, id3Classifier.tree)

        # Check Results
        my_accuracy = get_accuracy(predictions, testing_set.target)
        print("Accuracy: " + str(my_accuracy) + "%")

        # Compare To Existing Implementations
        dtc = tree.DecisionTreeClassifier()
        dtc.fit(training_set.data, training_set.target)
        predictions = dtc.predict(testing_set.data)

        dtc_accuracy = get_accuracy(predictions, testing_set.target)
        print("DTC Accuracy: " + str(dtc_accuracy) + "%")

        # Do another or not
        toContinue = False

        while True:
            another = input("Do you want to examine another dataset? (y / n) ")

            if another != 'y' and another != 'n':
                print("Please provide you answer in a 'y' or 'n' format.")
            elif another == 'y':
                toContinue = True
                break
            else:
                toContinue = False
                break

        if not toContinue:
            break

	def log_preds(self, test_sentences=["hello", "how are you", "what is the meaning of life"]):
		d = Dataset(self._logger)

		for s in test_sentences:
			seed = np.zeros((TRAIN_BATCH_SIZE, (MAX_OUTPUT_TOKEN_LENGTH+1)*MSG_HISTORY_LEN, 29), dtype="bool")
			blahhh=d.sample({"Msg": s})
			for i in range(len(blahhh)):
				for j in range(len(blahhh[i])):
					seed[0][i][j]=blahhh[i][j]

			self._logger.info(self.predict_sentence(seed))

    def test_parse_iob(self):
        test = 'i would like to go from (Columbia University)[from_stop] to (Herald Square)[to_stop]'
        expected_iob = ['o', 'o', 'o', 'o', 'o', 'o', 'b-test.from_stop', 'i-test.from_stop', 'o', 'b-test.to_stop',
                        'i-test.to_stop']
        expected_tokens = ['i', 'would', 'like', 'to', 'go', 'from', 'columbia', 'university', 'to', 'herald', 'square']
        actual_iob, actual_tokens = Dataset.parse_iob('test', test)
        self.assertEqual(actual_iob, expected_iob)
        self.assertEqual(actual_tokens, expected_tokens)

        for slot in set(expected_iob):
            self.assertIn(slot, Dataset.get_slots())

	def __init__(self, data_path=None):

		self.data_path = data_path
		ds = Dataset(is_binary=True)
		ds.setup_dataset(data_path=self.data_path, train_split_scale=0.6)

		self.X = ds.Xtrain
		self.y = ds.Ytrain

		self.y = np.cast['uint8'](list(self.y))
		self.X = np.cast['float32'](list(self.X))

    def test_import_dataset(self):

        ## The Pixelman dataset object.
        pds = Dataset("data/sr/0-00_mm/ASCIIxyC/")

        # The tests.

        # The number of datafiles.
        self.assertEqual(pds.getNumberOfDataFiles(), 600)

        # The data format of the folder.
        self.assertEqual(pds.getFolderFormat(), "ASCII [x, y, C]")