本文整理汇总了Python中tensorflow.decode_csv函数的典型用法代码示例。如果您正苦于以下问题:Python decode_csv函数的具体用法?Python decode_csv怎么用?Python decode_csv使用的例子?那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了decode_csv函数的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: read_csv_examples
def read_csv_examples(image_dir, label_dir, batch_size=100, num_epochs=None, task_index=None, num_workers=None):
print_log(worker_num, "num_epochs: {0}".format(num_epochs))
# Setup queue of csv image filenames
tf_record_pattern = os.path.join(image_dir, 'part-*')
images = tf.gfile.Glob(tf_record_pattern)
print_log(worker_num, "images: {0}".format(images))
image_queue = tf.train.string_input_producer(images, shuffle=False, capacity=1000, num_epochs=num_epochs, name="image_queue")
# Setup queue of csv label filenames
tf_record_pattern = os.path.join(label_dir, 'part-*')
labels = tf.gfile.Glob(tf_record_pattern)
print_log(worker_num, "labels: {0}".format(labels))
label_queue = tf.train.string_input_producer(labels, shuffle=False, capacity=1000, num_epochs=num_epochs, name="label_queue")
# Setup reader for image queue
img_reader = tf.TextLineReader(name="img_reader")
_, img_csv = img_reader.read(image_queue)
image_defaults = [ [1.0] for col in range(784) ]
img = tf.pack(tf.decode_csv(img_csv, image_defaults))
# Normalize values to [0,1]
norm = tf.constant(255, dtype=tf.float32, shape=(784,))
image = tf.div(img, norm)
print_log(worker_num, "image: {0}".format(image))
# Setup reader for label queue
label_reader = tf.TextLineReader(name="label_reader")
_, label_csv = label_reader.read(label_queue)
label_defaults = [ [1.0] for col in range(10) ]
label = tf.pack(tf.decode_csv(label_csv, label_defaults))
print_log(worker_num, "label: {0}".format(label))
# Return a batch of examples
return tf.train.batch([image,label], batch_size, num_threads=args.readers, name="batch_csv")
示例2: read_fer2013
def read_fer2013(eval_data):
"""
Read and parse the examples from the FER2013 data file
Args:
eval_data: boolean indicating whether we are using training or evaluation data
Returns:
A single example contained in an object with fields:
height: number of rows
width: number of columns
depth: number of colour channels
key: filename and record number for the example
label: an int32 Tensor with the label in the range 0..6
image: a [height, width, depth] int32 Tensor with the image data
"""
class FER2013Record(object):
pass
result = FER2013Record()
# Dataset dimensions
result.height = 48
result.width = 48
result.depth = 1
# Set up the reader
filename = tf.train.string_input_producer(["FER2013 data/fer2013/fer2013.csv"])
# read from the data file
# training data starts on line 2 (single header line)
# test data starts after the training data
skip_lines = 1
if eval_data:
skip_lines = NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN
reader = tf.TextLineReader(skip_header_lines=skip_lines)
# Read a line corresponding to an example
result.key, value = reader.read(filename)
# Decode the line according to its formatting
def1 = [[0],["Empty"],["Empty"]]
result.label, image, result.testOrTrain = tf.decode_csv(value, def1)
# The middle column corresponds to the image data of 48x48 = 2304
# The data is space separated hence field_delim=' '
def2 = [[0]]*(result.height*result.width)
image = tf.decode_csv(image, def2, field_delim=' ')
image = tf.reshape(image, [result.height, result.width, -1])
result.image = tf.cast(image, tf.uint8)
return result
示例3: parse_csv
def parse_csv(csv_row, is_serving=False):
"""Takes the string input tensor (csv) and returns a dict of rank-2 tensors.
Takes a rank-1 tensor and converts it into rank-2 tensor, with respect to
its data type (inferred from the metadata).
Args:
csv_row: rank-2 tensor of type string (csv).
is_serving: boolean to indicate whether this function is called during
serving or training, since the csv_row serving input is different than
the training input (i.e., no target column).
Returns:
rank-2 tensor of the correct data type.
"""
if is_serving:
column_names = metadata.SERVING_COLUMN_NAMES
defaults = []
# create the defaults for the serving columns.
for serving_feature in metadata.SERVING_COLUMN_NAMES:
feature_index = metadata.COLUMN_NAMES.index(serving_feature)
defaults.append(metadata.DEFAULTS[feature_index])
else:
column_names = metadata.COLUMN_NAMES
defaults = metadata.DEFAULTS
columns = tf.decode_csv(csv_row, record_defaults=defaults)
features = dict(zip(column_names, columns))
return features
示例4: _decode
def _decode(example_batch):
"""Decode a batch of CSV lines into a feature map."""
if FLAGS.is_predicting:
record_defaults = [[0.0], [""], [0.0], [""], [0.0], [""], [""], [""],
[""], [""], [0.0], [0.0], [0.0], [""]]
else:
record_defaults = [[0.0], [""], [0.0], [""], [0.0], [""], [""], [""],
[""], [""], [0.0], [0.0], [0.0], [""], [""]]
fields = tf.decode_csv(example_batch, record_defaults, field_delim=',')
if FLAGS.is_predicting:
data = {LABEL: tf.constant("")}
else:
data = {LABEL: fields[14]}
data["age"] = fields[0]
data["workclass"] = fields[1]
data["fnlwgt"] = fields[2]
data["education"] = fields[3]
data["education-num"] = fields[4]
data["marital-status"] = fields[5]
data["occupation"] = fields[6]
data["relationship"] = fields[7]
data["race"] = fields[8]
data["sex"] = fields[9]
data["capital-gain"] = fields[10]
data["capital-loss"] = fields[11]
data["hours-per-week"] = fields[12]
data["native-country"] = fields[13]
return data
示例5: _decode_csv
def _decode_csv(line):
"""Takes the string input tensor and parses it to feature dict and target.
All the columns except the first one are treated as feature column. The
first column is expected to be the target.
Only returns target for if with_target is True.
Args:
line: csv rows in tensor format.
Returns:
features: A dictionary of features with key as "column_names" from
self._column_header.
target: tensor of target values which is the first column of the file.
This will only be returned if with_target==True.
"""
column_header = column_names if with_target else column_names[:4]
record_defaults = [[0.] for _ in xrange(len(column_names) - 1)]
# Pass label as integer.
if with_target:
record_defaults.append([0])
columns = tf.decode_csv(line, record_defaults=record_defaults)
features = dict(zip(column_header, columns))
target = features.pop(column_names[4]) if with_target else None
return features, target
示例6: parse_csv
def parse_csv(value):
tf.logging.info('Parsing {}'.format(data_file))
columns = tf.decode_csv(value, record_defaults=_CSV_COLUMN_DEFAULTS)
features = dict(zip(_CSV_COLUMNS, columns))
labels = features.pop('income_bracket')
classes = tf.equal(labels, '>50K') # binary classification
return features, classes
示例7: test_inputs
def test_inputs(self, csv, batch_size):
print("input csv file path: %s, batch size: %d" % (csv, batch_size))
filename_queue = tf.train.string_input_producer([csv], shuffle=False)
reader = tf.TextLineReader()
_, serialized_example = reader.read(filename_queue)
filename, label = tf.decode_csv(serialized_example, [["path"], [0]])
label = tf.cast(label, tf.int32)
jpg = tf.read_file(filename)
image = tf.image.decode_jpeg(jpg, channels=3)
image = tf.cast(image, tf.float32)
print "original image shape:"
print image.get_shape()
# resize to distort
dist = tf.image.resize_images(image, FLAGS.scale_h, FLAGS.scale_w)
# random crop
dist = tf.image.resize_image_with_crop_or_pad(dist, FLAGS.input_h, FLAGS.input_w)
min_fraction_of_examples_in_queue = 0.4
min_queue_examples = int(FLAGS.num_examples_per_epoch_for_train * min_fraction_of_examples_in_queue)
print (
'filling queue with %d train images before starting to train. This will take a few minutes.' % min_queue_examples)
return self._generate_image_and_label_batch(dist, label, min_queue_examples, batch_size, shuffle=False)
示例8: read
def read(filename_queue):
class Record(object):
pass
result = Record()
reader = tf.TextLineReader()
result.key, line = reader.read(filename_queue)
#sess = tf.Session()
#print(line[0].eval(session=sess), line[1].eval(session=sess))
#sess.close()
#print(line.get_shape())
record_defaults = [[0] for _ in xrange(2305)]
columns = tf.decode_csv(line, record_defaults=record_defaults)
#print("PRINT: " , len(columns))
x = tf.pack(columns[1:])
cls = columns[0]
result.height = 48
result.width = 48
result.label = tf.cast(cls, tf.int32)
depth_major = tf.reshape(x, [result.height, result.width, 1])
three_chann = tf.concat(2, [depth_major, depth_major, depth_major])
print(three_chann.get_shape())
result.image = three_chann
return result
示例9: _input_fn
def _input_fn():
num_epochs = 100 if mode == tf.contrib.learn.ModeKeys.TRAIN else 1
# could be a path to one file or a file pattern.
input_file_names = tf.train.match_filenames_once(filename)
filename_queue = tf.train.string_input_producer(
input_file_names, num_epochs=num_epochs, shuffle=True)
reader = tf.TextLineReader()
_, value = reader.read_up_to(filename_queue, num_records=BATCH_SIZE)
value_column = tf.expand_dims(value, -1)
print 'readcsv={}'.format(value_column)
# all_data is a list of tensors
all_data = tf.decode_csv(value_column, record_defaults=DEFAULTS)
inputs = all_data[:len(all_data)-N_OUTPUTS] # first few values
label = all_data[len(all_data)-N_OUTPUTS : ] # last few values
# from list of tensors to tensor with one more dimension
inputs = tf.concat(inputs, axis=1)
label = tf.concat(label, axis=1)
print 'inputs={}'.format(inputs)
return {TIMESERIES_COL: inputs}, label # dict of features, label
示例10: multi_reader_multi_example
def multi_reader_multi_example():
# create a FIFO queue
filenames = ['a.csv', 'b.csv', 'c.csv']
filename_queue = tf.train.string_input_producer(filenames, shuffle=False)
# create reader
reader = tf.TextLineReader()
key, value = reader.read(filename_queue)
record_defaults = [['null'], ['null']]
example_list = [tf.decode_csv(value, record_defaults=record_defaults) for _ in range(2)]
example_batch, label_batch = tf.train.batch_join(example_list, batch_size=5)
# run graph
with tf.Session() as sess:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
try:
while not coord.should_stop():
print(example_batch.eval())
except tf.errors.OutOfRangeError:
print('epoches completed!')
finally:
coord.request_stop()
coord.join(threads)
示例11: read_image_unlabeled
def read_image_unlabeled(filename_queue, raw_img):
class StatefarmRecord(object):
pass
result = StatefarmRecord()
# Read a record, getting filenames from the filename_queue.
result.key, _ = tf.decode_csv(filename_queue.dequeue(), [[""], [""]], " ")
# Extract raw JPG data as a string
# raw_contents = tf.read_file(result.key)
# raw_contents = raw_img
# Decode raw data as a PNG. Defaults to uint8 encoding.
# result.uint8image = tf.image.decode_png(raw_contents)
result.uint8image = raw_img.astype('uint8')
# TENSORFLOW BUG: image shape not statically determined, so force
# it to have correct CIFAR100 dimensions
# result.uint8image.set_shape((32, 32, 3))
# Kind of hacky, but set a label so we can use the same structure
# THIS SHOULD ALWAYS BE IGNORED DURING COMPUTATION, since we are
# dealing with unlabaled data
result.label = tf.cast(tf.string_to_number("0"), tf.int32)
return result
示例12: raw_training_input_fn
def raw_training_input_fn():
"""Training input function that reads raw data and applies transforms."""
if isinstance(raw_data_file_pattern, six.string_types):
filepath_list = [raw_data_file_pattern]
else:
filepath_list = raw_data_file_pattern
files = []
for path in filepath_list:
files.extend(file_io.get_matching_files(path))
filename_queue = tf.train.string_input_producer(
files, num_epochs=num_epochs, shuffle=randomize_input)
csv_id, csv_lines = tf.TextLineReader().read_up_to(filename_queue, training_batch_size)
queue_capacity = (reader_num_threads + 3) * training_batch_size + min_after_dequeue
if randomize_input:
_, batch_csv_lines = tf.train.shuffle_batch(
tensors=[csv_id, csv_lines],
batch_size=training_batch_size,
capacity=queue_capacity,
min_after_dequeue=min_after_dequeue,
enqueue_many=True,
num_threads=reader_num_threads,
allow_smaller_final_batch=allow_smaller_final_batch)
else:
_, batch_csv_lines = tf.train.batch(
tensors=[csv_id, csv_lines],
batch_size=training_batch_size,
capacity=queue_capacity,
enqueue_many=True,
num_threads=reader_num_threads,
allow_smaller_final_batch=allow_smaller_final_batch)
csv_header, record_defaults = csv_header_and_defaults(features, schema, stats, keep_target=True)
parsed_tensors = tf.decode_csv(batch_csv_lines, record_defaults, name='csv_to_tensors')
raw_features = dict(zip(csv_header, parsed_tensors))
transform_fn = make_preprocessing_fn(analysis_output_dir, features, keep_target=True)
transformed_tensors = transform_fn(raw_features)
# Expand the dims of non-sparse tensors. This is needed by tf.learn.
transformed_features = {}
for k, v in six.iteritems(transformed_tensors):
if isinstance(v, tf.Tensor) and v.get_shape().ndims == 1:
transformed_features[k] = tf.expand_dims(v, -1)
else:
transformed_features[k] = v
# Remove the target tensor, and return it directly
target_name = get_target_name(features)
if not target_name or target_name not in transformed_features:
raise ValueError('Cannot find target transform in features')
transformed_target = transformed_features.pop(target_name)
return transformed_features, transformed_target
示例13: parse_example_tensor
def parse_example_tensor(examples, train_config, keep_target):
"""Read the csv files.
Args:
examples: string tensor
train_config: training config
keep_target: if true, the target column is expected to exist and it is
returned in the features dict.
Returns:
Dict of feature_name to tensor. Target feature is in the dict.
"""
csv_header = []
if keep_target:
csv_header = train_config['csv_header']
else:
csv_header = [name for name in train_config['csv_header']
if name != train_config['target_column']]
# record_defaults are used by tf.decode_csv to insert defaults, and to infer
# the datatype.
record_defaults = [[train_config['csv_defaults'][name]]
for name in csv_header]
tensors = tf.decode_csv(examples, record_defaults, name='csv_to_tensors')
# I'm not really sure why expand_dims needs to be called. If using regression
# models, it errors without it.
tensors = [tf.expand_dims(x, axis=1) for x in tensors]
tensor_dict = dict(zip(csv_header, tensors))
return tensor_dict
示例14: decode_csv
def decode_csv(line):
parsed_line = tf.decode_csv(line, [[0.], [0.], [0.], [0.], [0]])
label = parsed_line[-1:] # Last element is the label
del parsed_line[-1] # Delete last element
features = parsed_line # Everything but last elements are the features
d = dict(zip(feature_names, features)), label
return d
示例15: filequeue_to_batch_data
def filequeue_to_batch_data(filename_queue, line_reader, batch_size = BATCH_SIZE):
# The text file format should be Query Image, Trieve Image, Query label,
# Trieve Label, Triplet loss Label( 0/1 )
key, next_line = line_reader.read(filename_queue)
query_image_name, retrieve_image_name, label_1, label_2, label_3 = tf.decode_csv(
next_line, [tf.constant([], dtype=tf.string),tf.constant([], dtype=tf.string),
tf.constant([], dtype = tf.int32), tf.constant([], dtype = tf.int32), tf.constant([], dtype = tf.int32)], field_delim=" ")
# batch_query_image, batch_label = tf.train.batch(
# [query_image_name, label], batch_size=batch_size)
reverse_channel = True # for pre-trained purpose
query_tensor = image_io.read_image(query_image_name, reverse_channel,
FEATURE_ROW, FEATURE_COL)
retrieve_tensor = image_io.read_image(retrieve_image_name, reverse_channel,
FEATURE_ROW, FEATURE_COL)
if SHUFFLE_DATA:
min_after_dequeue = 100
capacity = min_after_dequeue + 3 * batch_size
batch_query_tensor, batch_retrieve_tensor, batch_label_1, batch_label_2, batch_label_3 = tf.train.shuffle_batch(
[query_tensor, retrieve_tensor, label_1, label_2, label_3], batch_size = batch_size, capacity=capacity,
min_after_dequeue=min_after_dequeue)
else:
batch_query_tensor,batch_retrieve_tensor, batch_label_1, batch_label_2, batch_label_3 = tf.train.batch(
[query_tensor, retrieve_tensor, label_1, label_2, label_3], batch_size=batch_size)
batch_tensor = tf.concat(0, [batch_query_tensor, batch_retrieve_tensor])
batch_label = tf.concat(0, [batch_label_1, batch_label_2])
return batch_tensor, batch_label, batch_label_3