Python data.DataProvider类代码示例

 def init_subnet_data_provider(self):
   if self.output_method == 'disk':
     dp = DataProvider.get_by_name('intermediate')
     count = self.train_dumper.get_count()
     self.train_dp = dp(self.train_output_filename,  range(0, count), 'fc')
     count = self.test_dumper.get_count()
     self.test_dp = dp(self.test_output_filename, range(0, count), 'fc')
   elif self.output_method == 'memory':
     dp = DataProvider.get_by_name('memory')
     self.train_dp = dp(self.train_dumper)
     self.test_dp = dp(self.test_dumper)

 def init_data_providers(self):
     self.dp_params['convnet'] = self # dp aka dataprovider
     try:
         self.test_data_provider = DataProvider.get_instance(self.data_path, self.test_batch_range,
                                                             type=self.dp_type, dp_params=self.dp_params, test=True)
         self.train_data_provider = DataProvider.get_instance(self.data_path, self.train_batch_range,
                                                                  self.model_state["epoch"], self.model_state["batchnum"],
                                                                  type=self.dp_type, dp_params=self.dp_params, test=False)
     except DataProviderException, e:
         print "Unable to create data provider: %s" % e
         self.print_data_providers()
         sys.exit()

 def init_data_providers(self):
     self.dp_params['convnet'] = self
     self.dp_params['PCA_pixel_alter'] = self.PCA_pixel_alter        
     self.dp_params['regress_L_channel_only'] = self.regress_L_channel_only
     self.dp_params['use_local_context_ftr'] = self.use_local_context_ftr
     self.dp_params['use_local_context_color_ftr'] = self.use_local_context_color_ftr
     if hasattr(self,'use_position_ftr'):
         self.dp_params['use_position_ftr'] = self.use_position_ftr
     try:
         self.test_data_provider = DataProvider.get_instance(self.libmodel, self.data_path, self.test_batch_range,
                                                             type=self.dp_type, dp_params=self.dp_params, test=DataProvider.DP_TEST)
         self.train_data_provider = DataProvider.get_instance(self.libmodel, self.data_path, self.train_batch_range,
                                                                  self.model_state["epoch"], self.model_state["batch_idx"],
                                                                  self.model_state["epochBatchPerm"],
                                                                  type=self.dp_type, dp_params=self.dp_params, test=DataProvider.DP_TRAIN)
     except DataProviderException, e:
         print "Unable to create data provider: %s" % e
         self.print_data_providers()
         sys.exit()

    def init_data_providers(self):
        self.dp_params['convnet'] = self
        self.dp_params['imgprovider'] = self.img_provider_file
        try:
            self.test_data_provider = DataProvider.get_instance(self.data_path_test, self.test_batch_range,
                                                                type=self.dp_type_test, dp_params=self.dp_params, test=True)
            if not self.test_only:
                self.train_data_provider = DataProvider.get_instance(self.data_path_train, self.train_batch_range,
                                                                     self.model_state["epoch"], self.model_state["batchnum"],
                                                                     type=self.dp_type_train, dp_params=self.dp_params, test=False)

            self.test_batch_range = self.test_data_provider.batch_range
            print "Test data provider: ", len(self.test_batch_range), " batches "
            if not self.test_only:
                self.train_batch_range = self.train_data_provider.batch_range
                print "Training data provider: ", len(self.train_batch_range), " batches "

        except DataProviderException, e:
            print "Unable to create data provider: %s" % e
            self.print_data_providers()
            sys.exit()

 def init_data_providers(self):
     class Dummy:
         def advance_batch(self):
             pass
     if self.need_gpu:
         ConvNet.init_data_providers(self)
         if self.op.get_value("write_features_pred") or self.op.get_value("show_preds") == 2:
             self.pred_data_provider = DataProvider.get_instance(self.libmodel, self.data_path, self.pred_batch_range,
                                                                 type=self.dp_type, dp_params=self.dp_params, test=DataProvider.DP_PREDICT)
         
     else:
         self.train_data_provider = self.test_data_provider = Dummy()

 def init_data_providers(self):
     self.dp_params['convnet'] = self
     self.dp_params['imgprovider'] = self.img_provider_file
     try:
         if self.need_gpu:
             self.test_data_provider = DataProvider.get_instance(self.data_path_test, self.test_batch_range,
                                                                 type=self.dp_type_test, dp_params=self.dp_params, test=True)
             
             self.test_batch_range = self.test_data_provider.batch_range
     except Exception, e:
         print "Unable to create data provider: %s" % e
         self.print_data_providers()
         sys.exit()

 def set_num_group(self, n):
   dp = DataProvider.get_by_name(self.data_provider)
   self.train_dp = dp(self.data_dir, self.train_range, n)
   self.test_dp = dp(self.data_dir, self.test_range, n)

 def set_category_range(self, r):
   dp = DataProvider.get_by_name(self.data_provider)
   self.train_dp = dp(self.data_dir, self.train_range, category_range = range(r))
   self.test_dp = dp(self.data_dir, self.test_range, category_range = range(r))

 def init_subnet_data_provider(self):
   dp = DataProvider.get_by_name('intermediate')
   count = self.train_dumper.get_count()
   self.train_dp = dp(self.train_output_filename,  range(0, count), 'fc')
   count = self.test_dumper.get_count()
   self.test_dp = dp(self.test_output_filename, range(0, count), 'fc')

  #create a checkpoint dumper
  image_shape = (param_dict['image_color'], param_dict['image_size'], param_dict['image_size'], param_dict['batch_size'])
  param_dict['image_shape'] = image_shape
  cp_dumper = CheckpointDumper(param_dict['checkpoint_dir'], param_dict['test_id'])
  param_dict['checkpoint_dumper'] = cp_dumper

  #create the init_model
  init_model = cp_dumper.get_checkpoint()
  if init_model is None:
    init_model = parse_config_file(args.param_file)
  param_dict['init_model'] = init_model

  #create train dataprovider and test dataprovider
  dp_class = DataProvider.get_by_name(param_dict['data_provider'])
  train_dp = dp_class(param_dict['data_dir'], param_dict['train_range'])
  test_dp = dp_class(param_dict['data_dir'], param_dict['test_range'])
  param_dict['train_dp'] = train_dp
  param_dict['test_dp'] = test_dp


  #get all extra information
  param_dict['num_epoch'] = args.num_epoch
  num_batch = util.string_to_int_list(args.num_batch)
  if len(num_batch) == 1:
    param_dict['num_batch'] = num_batch[0]
  else:
    param_dict['num_batch'] = num_batch

  param_dict['num_group_list']  = util.string_to_int_list(args.num_group_list)

 def init_data_provider(self):
   self.train_dp = DataProvider(self.batch_size, self.data_dir, self.train_range)
   self.test_dp = DataProvider(self.batch_size, self.data_dir, self.test_range)

class Trainer:
  CHECKPOINT_REGEX = None
  def __init__(self, test_id, data_dir, checkpoint_dir, train_range, test_range, test_freq,
      save_freq, batch_size, num_epoch, image_size, image_color, learning_rate, n_out,
      autoInit=True, adjust_freq = 1, factor = 1.0):
    self.test_id = test_id
    self.data_dir = data_dir
    self.checkpoint_dir = checkpoint_dir
    self.train_range = train_range
    self.test_range = test_range
    self.test_freq = test_freq
    self.save_freq = save_freq
    self.batch_size = batch_size
    self.num_epoch = num_epoch
    self.image_size = image_size
    self.image_color = image_color
    self.learning_rate = learning_rate
    self.n_out = n_out
    self.factor = factor
    self.adjust_freq = adjust_freq
    self.regex = re.compile('^test%d-(\d+)\.(\d+)$' % self.test_id)

    self.init_data_provider()
    self.image_shape = (self.batch_size, self.image_color, self.image_size, self.image_size)
    self.train_outputs = []
    self.test_outputs = []
    self.net = FastNet(self.learning_rate, self.image_shape, self.n_out, autoAdd = autoInit)

    self.num_batch = self.curr_epoch = self.curr_batch = 0
    self.train_data = None
    self.test_data = None

    self.num_train_minibatch = 0
    self.num_test_minibatch = 0
    self.checkpoint_file = ''

  def init_data_provider(self):
    self.train_dp = DataProvider(self.batch_size, self.data_dir, self.train_range)
    self.test_dp = DataProvider(self.batch_size, self.data_dir, self.test_range)


  def get_next_minibatch(self, i, train = TRAIN):
    if train == TRAIN:
      num = self.num_train_minibatch
      data = self.train_data
    else:
      num = self.num_test_minibatch
      data = self.test_data

    batch_data = data['data']
    batch_label = data['labels']
    batch_size = self.batch_size

    if i == num -1:
      input = batch_data[:, i * batch_size: -1]
      label = batch_label[i* batch_size : -1]
    else:
      input = batch_data[:, i * batch_size: (i +1)* batch_size]
      label = batch_label[i * batch_size: (i + 1) * batch_size]

    return input, label


  def save_checkpoint(self):
    model = {}
    model['batchnum'] = self.train_dp.get_batch_num()
    model['epoch'] = self.num_epoch + 1
    model['layers'] = self.net.get_dumped_layers()
    model['train_outputs'] = self.train_outputs
    model['test_outputs'] = self.test_outputs

    dic = {'model_state': model, 'op':None}
    saved_filename = [f for f in os.listdir(self.checkpoint_dir) if self.regex.match(f)]
    for f in saved_filename:
      os.remove(os.path.join(self.checkpoint_dir, f))
    checkpoint_filename = "test%d-%d.%d" % (self.test_id, self.curr_epoch, self.curr_batch)
    checkpoint_file_path = os.path.join(self.checkpoint_dir, checkpoint_filename)
    self.checkpoint_file = checkpoint_file_path
    print checkpoint_file_path
    with open(checkpoint_file_path, 'w') as f:
      cPickle.dump(dic, f)

  def get_test_error(self):
    start = time.time()
    _, _, self.test_data = self.test_dp.get_next_batch()

    self.num_test_minibatch = ceil(self.test_data['data'].shape[1], self.batch_size)
    for i in range(self.num_test_minibatch):
      input, label = self.get_next_minibatch(i, TEST)
      label = np.array(label).astype(np.float32)
      label.shape = (label.size, 1)
      self.net.train_batch(input, label, TEST)
    cost , correct, numCase, = self.net.get_batch_information()
    self.test_outputs += [({'logprob': [cost, 1-correct]}, numCase, time.time() - start)]
    print 'error: %f logreg: %f time: %f' % (1-correct, cost, time.time() -
      start)

  def check_continue_trainning(self):
    return self.curr_epoch <= self.num_epoch

#.........这里部分代码省略.........

    if os.path.exists(_CUR_LOG_FILE_NAME):
        os.rename(_CUR_LOG_FILE_NAME, _PRED_LOG_FILE_NAME)
    log.basicConfig( level=log.INFO)
#    log.basicConfig(filename=_CUR_LOG_FILE_NAME, level=log.INFO)
    log.info('start')

if __name__ == '__main__':
#    try:
        FILE_NAME = "data.dat"
        INIT_DATA_FILE_NAME = 'init.dat'
        TEST_FILE_NAME = "test_data.dat"
        TEST_INIT_DATA_FILE_NAME = 'test_init.dat'
#        binder.bind(str, annotated_with="data_file_name", to_instance = FILE_NAME)
#        binder.bind(str, annotated_with="init_file_name", to_instance = INIT_DATA_FILE_NAME)
        _init_log()
        dataProvider = DataProvider(FILE_NAME, INIT_DATA_FILE_NAME)
#        test_data = DataProvider(TEST_FILE_NAME, TEST_INIT_DATA_FILE_NAME).get_data()
#        assert(calc_class_re())
        app = QApplication(sys.argv)
        data = dataProvider.get_data()
#        print data
        classifier = c45(data, max_repeat_var=10)
        form = MainWindow(data, classifier)

##        print data
#        classifier = c45(data, max_repeat_var=10)
#        pos_sum = 0
#        for row, target in zip(data.data, data.target):
#            pos = 0
#            for l, c in classifier.get_labels_count(row).items():
#                pos += 1

os.environ['THEANO_FLAGS'] = 'device=gpu'

import os
os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"   # see issue #152
os.environ["CUDA_VISIBLE_DEVICES"]="1"

config = tf.ConfigProto(log_device_placement=True, allow_soft_placement=True)
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
K.set_session(session)

conf = Config(flag, args[2], int(args[3]))
print(flag)

# get data
dp = DataProvider(conf)
n_terms = len(dp.idx2word)
word_embed_data = np.array(dp.word_embed)

item_embed_data = np.random.rand(dp.get_item_size(), conf.dim_word)
print("finish data processing")

# define model
word_input = Input(shape=(1,), dtype ="int32", name ="word_idx")
item_pos_input = Input(shape=(1,), dtype ="int32", name ="item_pos_idx")
item_neg_input = Input(shape=(1,), dtype ="int32", name ="item_neg_idx")

word_embed = Embedding(output_dim=conf.dim_word, input_dim=n_terms, input_length=1, name="word_embed",
                       weights=[word_embed_data], trainable=False)
item_embed = Embedding(output_dim=conf.dim_word, input_dim=dp.get_item_size(), input_length=1, name="item_embed",
                       weights=[item_embed_data], trainable=True)

from config import Config
from data import DataProvider
from gensim.models.word2vec import Word2Vec
import numpy as np
import os

flag = "tag"
conf = Config(flag, "tag" , 300)

if not os.path.exists(conf.path_word_w2c) and not os.path.exists(conf.path_doc_w2c):
    doc_embed = np.load(conf.path_doc_npy + ".npy")[0]
    dp = DataProvider(conf)

    # generate doc embedding file
    f = open(conf.path_doc_w2c,"w")
    f.write(str(len(dp.idx2prod)))
    f.write(" ")
    f.write(str(conf.dim_item))
    f.write("\n")
    idx = 0
    batch = ""
    for word in dp.idx2prod:
        batch = "".join([batch, word])
        batch = "".join([batch, " "])

        for i in range(conf.dim_item):
            batch = "".join([batch, str(doc_embed[idx][i])])
            batch = "".join([batch, " "])

        batch = "".join([batch, "\n"])
        idx += 1