Python dataset.Dataset方法代码示例

# 需要导入模块: import dataset [as 别名]
# 或者: from dataset import Dataset [as 别名]
def main():
    parser = argparse.ArgumentParser(description='test', formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument(
        '-a', '--attributes',
        nargs='+',
        type=str,
        help='Specify attribute name for training. \nAll attributes can be found in list_attr_celeba.txt'
    )
    parser.add_argument(
        '-g', '--gpu',
        default='0',
        type=str,
        help='Specify GPU id. \ndefault: %(default)s. \nUse comma to seperate several ids, for example: 0,1'
    )
    args = parser.parse_args()

    celebA = Dataset(args.attributes)
    DNA_GAN = Model(args.attributes, is_train=True)
    run(config, celebA, DNA_GAN, gpu=args.gpu) 

# 需要导入模块: import dataset [as 别名]
# 或者: from dataset import Dataset [as 别名]
def split_train_val_test(data_dir, img_size=256):
    df = pd.read_csv(
        join(data_dir, 'list_eval_partition.txt'),
        delim_whitespace=True, header=None
    )
    filenames, labels = df.values[:, 0], df.values[:, 1]

    train_filenames = filenames[labels == 0]
    valid_filenames = filenames[labels == 1]
    test_filenames  = filenames[labels == 2]

    train_set = Dataset(
        data_dir, train_filenames, input_transform_augment(178, img_size),
        target_transform(), target_transform_binary()
    )
    valid_set = Dataset(
        data_dir, valid_filenames, input_transform(178, img_size),
        target_transform(), target_transform_binary()
    )
    test_set = Dataset(
        data_dir, test_filenames, input_transform(178, img_size),
        target_transform(), target_transform_binary()
    )

    return train_set, valid_set, test_set 

# 需要导入模块: import dataset [as 别名]
# 或者: from dataset import Dataset [as 别名]
def main(unused_argv):
  # tf.logging.set_verbosity(tf.logging.INFO)

  # load two copies of the dataset
  print('Loading datasets...')
  dataset = [Dataset(bs=FLAGS.batch_size, filepattern=FLAGS.filepattern,
                     label=i) for i in range(10)]

  print('Computing Wasserstein distance(s)...')
  for i in range(10):
    for j in range(10):
      with tf.Graph().as_default():
        # compute Wasserstein distance between sets of labels i and j
        wasserstein = Wasserstein(dataset[i], dataset[j])
        loss = wasserstein.dist(C=.1, nsteps=FLAGS.loss_steps)
        with tf.Session() as sess:
          sess.run(tf.global_variables_initializer())
          res = sess.run(loss)
          print_flush('%f ' % res)
    print_flush('\n') 

# 需要导入模块: import dataset [as 别名]
# 或者: from dataset import Dataset [as 别名]
def main(unused_argv):
  # tf.logging.set_verbosity(tf.logging.INFO)

  # load two copies of the dataset
  print('Loading datasets...')
  subset1 = Dataset(bs=FLAGS.batch_size, filepattern=FLAGS.filepattern)
  subset2 = Dataset(bs=FLAGS.batch_size, filepattern=FLAGS.filepattern)

  print('Computing Wasserstein distance...')
  with tf.Graph().as_default():
    # compute Wasserstein distance between two sets of examples
    wasserstein = Wasserstein(subset1, subset2)
    loss = wasserstein.dist(C=.1, nsteps=FLAGS.loss_steps)
    with tf.Session() as sess:
      sess.run(tf.global_variables_initializer())
      res = sess.run(loss)
      print('result: %f\n' % res) 

# 需要导入模块: import dataset [as 别名]
# 或者: from dataset import Dataset [as 别名]
def extract_users(dataset_session: DatasetSession, log_metadata_obj: dict) -> List:
        ############## TESTS
        # get dataset
        log_file_dataset: Dataset = dataset_session.get_csv_dataset()
        # get core dataframe
        log_file_core_dataframe: CoreDataFrame = log_file_dataset.get_dataframe()
        # get data frame (.data)
        log_file_dataframe: pd.DataFrame = log_file_core_dataframe.data
        # test: get shape
        log_file_shape: Tuple = log_file_dataframe.shape
        logging.warning("execute(): dataframe shape: "+str(log_file_shape))
        ############

        logging.info("ExtractAllUsersCSV: extract_users log_file_data.columns: - "+str(log_file_dataframe.columns))
        logging.info("ExtractAllUsersCSV: extract_users log_metadata_obj: - "+str(log_metadata_obj))


        id_column: pd.Series = log_file_dataframe[ log_metadata_obj["id_feature"]]
        logging.info( "ExtractAllUsersCSV, extract_users, id_column, len of column: "+str(len(id_column)) )
        user_set: List = np.unique( log_file_dataframe[ log_metadata_obj["id_feature"] ].fillna("NA") )
        logging.info( "ExtractAllUsersCSV, extract_users, user_set len of column: "+str(len(user_set)) )
        logging.error(user_set)
        return user_set 

# 需要导入模块: import dataset [as 别名]
# 或者: from dataset import Dataset [as 别名]
def __init__(self, holo_env, name="session"):
        """
        Constructor for Holoclean session
        :param holo_env: Holoclean object
        :param name: Name for the Holoclean session
        """
        logging.basicConfig()

        # Initialize members
        self.name = name
        self.holo_env = holo_env
        self.Denial_constraints = []  # Denial Constraint strings
        self.dc_objects = {}  # Denial Constraint Objects
        self.featurizers = []
        self.error_detectors = []
        self.cv = None
        self.pruning = None
        self.dataset = Dataset()
        self.parser = ParserInterface(self)
        self.inferred_values = None
        self.feature_count = 0 

# 需要导入模块: import dataset [as 别名]
# 或者: from dataset import Dataset [as 别名]
def main():
    parser = argparse.ArgumentParser(description='test', formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument(
        '-a', '--attribute', 
        default='Smiling',
        type=str,
        help='Specify attribute name for training. \ndefault: %(default)s. \nAll attributes can be found in list_attr_celeba.txt'
    )
    parser.add_argument(
        '-g', '--gpu', 
        default='0',
        type=str,
        help='Specify GPU id. \ndefault: %(default)s. \nUse comma to seperate several ids, for example: 0,1'
    )
    args = parser.parse_args()

    celebA = Dataset(args.attribute)
    GeneGAN = Model(is_train=True)
    run(config, celebA, GeneGAN, gpu=args.gpu) 

# 需要导入模块: import dataset [as 别名]
# 或者: from dataset import Dataset [as 别名]
def create_dataset(exp_type, batch_size, split_name):
    hdf5_file_list_txt = os.path.join(g_shapenet_parts_dir,
            '{}_hdf5_file_list.txt'.format(split_name))
    assert(os.path.exists(hdf5_file_list_txt))

    with open(hdf5_file_list_txt, 'r') as f:
        hdf5_file_list = f.read().splitlines()

    point_clouds = []
    labels = []
    category_ids = []

    for i, hdf5_file in enumerate(hdf5_file_list):
        f = h5py.File(os.path.join(g_shapenet_parts_dir, hdf5_file))
        point_clouds.append(f['data'][:])
        labels.append(f['pid'][:])
        category_ids.append(f['label'][:])
        print("Loaded '{}'.".format(hdf5_file))

    point_clouds = np.concatenate(point_clouds)
    labels = np.concatenate(labels)
    category_ids = np.concatenate(category_ids)

    category_name_file = os.path.join(g_shapenet_parts_dir,
            'all_object_categories.txt')
    assert(os.path.exists(category_name_file))
    with open(category_name_file, 'r') as f:
        category_names = f.read().splitlines()
        for i, name in enumerate(category_names):
            category_names[i] = name.split('\t')[0]

    print(category_names)

    return Dataset('ShapeNetParts', exp_type, batch_size, point_clouds, labels,
            category_ids=category_ids, category_names=category_names) 

# 需要导入模块: import dataset [as 别名]
# 或者: from dataset import Dataset [as 别名]
def bad_cases():
    print("\nPredicting...\n")
    graph = tf.Graph()
    with graph.as_default():  # with tf.Graph().as_default() as g:
        sess = tf.Session()
        with sess.as_default():
            # Load the saved meta graph and restore variables
            # saver = tf.train.Saver(tf.global_variables())
            meta_file = os.path.abspath(os.path.join(FLAGS.model_dir, 'checkpoints/model-1000.meta'))
            new_saver = tf.train.import_meta_graph(meta_file)
            new_saver.restore(sess, tf.train.latest_checkpoint(os.path.join(FLAGS.model_dir, 'checkpoints')))
            # graph = tf.get_default_graph()

            # Get the placeholders from the graph by name
            # input_x1 = graph.get_operation_by_name("input_x1").outputs[0]
            input_x1 = graph.get_tensor_by_name("input_x1:0")  # Tensor("input_x1:0", shape=(?, 15), dtype=int32)
            input_x2 = graph.get_tensor_by_name("input_x2:0")
            dropout_keep_prob = graph.get_tensor_by_name("dropout_keep_prob:0")
            # Tensors we want to evaluate
            sim = graph.get_tensor_by_name("metrics/sim:0")
            y_pred = graph.get_tensor_by_name("metrics/y_pred:0")

            dev_sample = {}
            for line in open(FLAGS.data_file):
                line = line.strip().split('\t')
                dev_sample[line[0]] = line[1]

            # Generate batches for one epoch
            dataset = Dataset(data_file="data/pred.csv")
            x1, x2, y = dataset.process_data(sequence_length=FLAGS.max_document_length, is_training=False)
            with open("result/fp_file", 'w') as f_fp, open("result/fn_file", 'w') as f_fn:
                for lineno, x1_online, x2_online, y_online in enumerate(zip(x1, x2, y)):
                    sim, y_pred_ = sess.run(
                        [sim, y_pred], {input_x1: x1_online, input_x2: x2_online, dropout_keep_prob: 1.0})
                    if y_pred == 1 and y_online == 0:  # low precision
                        f_fp.write(dev_sample[lineno+1] + str(sim) + '\n')
                    elif y_pred == 0 and y_online == 1:  # low recall
                        f_fn.write(dev_sample[lineno + 1] + str(sim) + '\n') 

# 需要导入模块: import dataset [as 别名]
# 或者: from dataset import Dataset [as 别名]
def main(input_file, output_file):
    print("\nPredicting...\n")
    graph = tf.Graph()
    with graph.as_default():  # with tf.Graph().as_default() as g:
        sess = tf.Session()
        with sess.as_default():
            # Load the saved meta graph and restore variables
            # saver = tf.train.Saver(tf.global_variables())
            meta_file = os.path.abspath(os.path.join(FLAGS.model_dir, 'checkpoints/model-3400.meta'))
            new_saver = tf.train.import_meta_graph(meta_file)
            new_saver.restore(sess, tf.train.latest_checkpoint(os.path.join(FLAGS.model_dir, 'checkpoints')))
            # graph = tf.get_default_graph()

            # Get the placeholders from the graph by name
            # input_x1 = graph.get_operation_by_name("input_x1").outputs[0]
            input_x1 = graph.get_tensor_by_name("input_x1:0")  # Tensor("input_x1:0", shape=(?, 15), dtype=int32)
            input_x2 = graph.get_tensor_by_name("input_x2:0")
            dropout_keep_prob = graph.get_tensor_by_name("dropout_keep_prob:0")
            # Tensors we want to evaluate
            y_pred = graph.get_tensor_by_name("metrics/y_pred:0")
            # vars = tf.get_collection('vars')
            # for var in vars:
            #     print(var)

            e = graph.get_tensor_by_name("cosine:0")

            # Generate batches for one epoch
            dataset = Dataset(data_file=input_file, is_training=False)
            data = dataset.process_data(data_file=input_file, sequence_length=FLAGS.max_document_length)
            batches = dataset.batch_iter(data, FLAGS.batch_size, 1, shuffle=False)
            with open(output_file, 'w') as fo:
                lineno = 1
                for batch in batches:
                    x1_batch, x2_batch, _, _ = zip(*batch)
                    y_pred_ = sess.run([y_pred], {input_x1: x1_batch, input_x2: x2_batch, dropout_keep_prob: 1.0})
                    for pred in y_pred_[0]:
                        fo.write('{}\t{}\n'.format(lineno, pred))
                        lineno += 1 

# 需要导入模块: import dataset [as 别名]
# 或者: from dataset import Dataset [as 别名]
def __init__(self, env, name="session"):
        """
        Constructor for Holoclean session
        :param env: Holoclean environment
        :param name: Name for the Holoclean session
        """
        # use DEBUG logging level if verbose enabled
        if env['verbose']:
            root_logger.setLevel(logging.DEBUG)
            gensim_logger.setLevel(logging.DEBUG)

        logging.debug('initiating session with parameters: %s', env)

        # Initialize random seeds.
        random.seed(env['seed'])
        torch.manual_seed(env['seed'])
        np.random.seed(seed=env['seed'])

        # Initialize members
        self.name = name
        self.env = env
        self.ds = Dataset(name, env)
        self.dc_parser = Parser(env, self.ds)
        self.domain_engine = DomainEngine(env, self.ds)
        self.detect_engine = DetectEngine(env, self.ds)
        self.repair_engine = RepairEngine(env, self.ds)
        self.eval_engine = EvalEngine(env, self.ds) 

# 需要导入模块: import dataset [as 别名]
# 或者: from dataset import Dataset [as 别名]
def test_build_dataset(self):
		dirname = os.path.dirname(__file__)
		dataset_dir = os.path.join(dirname,'__tmp_dataset')
		posts_file = os.path.join(dirname,'posts.json.gz')

		app.build_dataset([dataset_dir,posts_file,'user_id','mentions'])

		# check if it's there
		ds = dataset.Dataset(dataset_dir)
		self.assertEquals(len(list(ds.post_iter())),15)

		os.system('rm -rf %s' % dataset_dir) 

# 需要导入模块: import dataset [as 别名]
# 或者: from dataset import Dataset [as 别名]
def main(args):
    with tf.Graph().as_default() as graph:
        # Create dataset
        logging.info('Create data flow from %s' % args.train)
        train_data = Dataset(directory=args.train, mean_path=args.mean, batch_size=args.batch_size, num_threads=2, capacity=10000)
    
        # Create initializer
        init = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
         
        # Config session
        config = get_config(args)
        
        # Setup summary
        check_summary_writer = tf.summary.FileWriter(os.path.join(args.log, 'check'), graph)

        check_op = tf.cast(train_data()['x_t_1'] * 255.0 + train_data()['mean'], tf.uint8)
 
        tf.summary.image('x_t_1_batch_restore', check_op, collections=['check'])
        check_summary_op = tf.summary.merge_all('check')

        # Start session
        with tf.Session(config=config) as sess:
            coord = tf.train.Coordinator()
            sess.run(init)
            threads = tf.train.start_queue_runners(sess=sess, coord=coord)
            for i in range(10):
                x_t_1_batch, summary = sess.run([check_op, check_summary_op])
                check_summary_writer.add_summary(summary, i)
            coord.request_stop()
            coord.join(threads) 

# 需要导入模块: import dataset [as 别名]
# 或者: from dataset import Dataset [as 别名]
def test(opt):
    # Load dataset
    dataset = Dataset(opt.data_dir, opt.train_txt, opt.test_txt, opt.bbox_txt)
    dataset.print_stats()

    # Load image transform
    test_transform = transforms.Compose([
        transforms.Resize((opt.image_width, opt.image_height)),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ])

    # Load data loader
    test_loader = mx.gluon.data.DataLoader(
        dataset=ImageData(dataset.test, test_transform),
        batch_size=opt.batch_size,
        num_workers=opt.num_workers
    )

    # Load model
    model = Model(opt)

    # Load evaluator
    evaluator = Evaluator(model, test_loader, opt.ctx)

    # Evaluate
    recalls = evaluator.evaluate(ranks=opt.recallk)
    for recallk, recall in zip(opt.recallk, recalls):
        print("R@{:4d}: {:.4f}".format(recallk, recall)) 

# 需要导入模块: import dataset [as 别名]
# 或者: from dataset import Dataset [as 别名]
def main(config):
    # For fast training.
    cudnn.benchmark = True

    # Create directories if not exist.
    if not os.path.exists(config.log_dir):
        os.makedirs(config.log_dir)
    if not os.path.exists(config.model_save_dir):
        os.makedirs(config.model_save_dir)

    imgdirs_train = ['data/afw/', 'data/helen/trainset/', 'data/lfpw/trainset/']
    imgdirs_test_commomset = ['data/helen/testset/','data/lfpw/testset/']

    # Dataset and Dataloader
    if config.phase == 'test':
        trainset=None
        train_loader = None
    else:
        trainset = Dataset(imgdirs_train, config.phase, 'train', config.rotFactor, config.res, config.gamma)
        train_loader = data.DataLoader(trainset,
                                       batch_size=config.batch_size,
                                       shuffle=True,
                                       num_workers=config.num_workers,
                                       pin_memory=True)
    testset = Dataset(imgdirs_test_commomset, 'test', config.attr, config.rotFactor, config.res, config.gamma)
    test_loader = data.DataLoader(testset,
                                  batch_size=config.batch_size,
                                  num_workers=config.num_workers,
                                  pin_memory=True)
    
    # Solver for training and testing.
    solver = Solver(train_loader, test_loader, config)
    if config.phase == 'train':
        solver.train()
    else:
        solver.load_state_dict(config.best_model)
        solver.test()