Python Dataset.load方法代码示例

# 需要导入模块: from Dataset import Dataset [as 别名]
# 或者: from Dataset.Dataset import load [as 别名]
def runner(
        PATH_DATA,
        RATIO_TEST_DATA,
        RATIO_SPECIFICITY,
        RATIO_CONFIDENCE,
        EXPERIMENTS,
        fe,
        setting_name
    ):

    results = []
    errors = Counter()
    qtypes = QuestionTypes()
    for e in range(1, EXPERIMENTS + 1):

        start = time.time()
        dataset = Dataset(PATH_DATA)
        dataset.load()

        invprob = InverseProbabilities(dataset)
        index = Index(invprob)

        train = [
    #         (bow(fe, label, RATIO_SPECIFICITY, prob_filter=invprob) + bow(fe, text, prob_filter=invprob), label, mark)
            (bow(fe, text, RATIO_SPECIFICITY, prob_filter=invprob), label, mark)
            for text, label, mark in dataset.train()
        ]
        train = train * 4

        test = [
            (bow(fe, label, RATIO_SPECIFICITY, prob_filter=invprob), label, mark)
#             (bow(fe, text, RATIO_SPECIFICITY, prob_filter=invprob), label, mark)
            for text, label, mark in dataset.test()
            if mark
        ][:int(len(train) * RATIO_TEST_DATA)]

        test += [
            (bow(fe, label, RATIO_SPECIFICITY, prob_filter=invprob), label, mark)
#             (bow(fe, text, RATIO_SPECIFICITY, prob_filter=invprob), label, mark)
            for text, label, mark in dataset.test()
            if not mark
        ][:len(test)]

        for tbow, label, mark in train:
            index.update(tbow)
            index.add(label)

        tp, tn, fp, fn, prec, rec, f, duration = 0, 0, 0, 0, 0.0, 0.0, 0.0, 0.0
        marked = sum([1 for _, _, mark in test if mark])
        for tbow, label, mark in test:
            qtypes.increment(label)
            expectation = sum([
                invprob[w]
                for w in set(bow(fe, label, RATIO_SPECIFICITY, prob_filter=invprob))
            ])
            matches = index(tbow)

            if not matches and not mark:
                tn += 1
                continue
            elif not matches and mark:
                fn += 1
                errors[('fn', '', label)] += 1
                qtypes.update('fn', None, label)
                continue

            best_match = matches[0]
            guess = best_match[2]
            sim = best_match[0]
            ratio = sim / (expectation + 0.1)

            if ratio <= RATIO_CONFIDENCE:
                if not mark:
                    tn += 1
                    continue
                else:
                    fn += 1
                    errors[('fn', '', label)] += 1
                    qtypes.update('fn', None, label)
                    continue
            else:
                if mark and guess == label:
                    tp += 1
                else:
                    fp += 1
                    _qtype = '_'.join(guess.lower().split()[:2])
                    errors[('fp', guess, label)] += 1
                    qtypes.update('fp', guess, label)

            duration = time.time() - start
            if tp:
                prec = tp / float(tp + fp)
                rec = tp / float(tp + fn)
                f = f1(prec, rec)
            else:
                prec, rec, f = 0.0, 0.0, 0.0

        vector = (e, _r(tp), _r(tn), _r(fp), _r(fn),
                  _r(prec), _r(rec), _r(f), _r(duration))
        results.append(vector)
#.........这里部分代码省略.........

# 需要导入模块: from Dataset import Dataset [as 别名]
# 或者: from Dataset.Dataset import load [as 别名]
                        print (log)
        
            end = time.time()   
            print ('Training took %.2f sec\n' % (end - start))
            
            final_embeddings = normalized_embeddings.eval()
    
    return final_embeddings
    


if __name__ == '__main__':
    print ('Loading the dataset... ')
    start = time.time()
    data = Dataset('Text8', reformatted=True, verbose=True)
    data, count, dictionary, reverse_dictionary = data.load()
    end = time.time() 
    print ('Loading the dataset took %.2f sec.\n' % (end - start))
        
    print ('Most common words (+UNK)', count[:5])
    print ('Sample data', data[:10])
    
    print('data:', [reverse_dictionary[di] for di in data[:8]])
        
    
    embedding_size = 128 # Dimension of the embedding vector.
    skip_window = 1 # How many words to consider left and right.
    num_skips = 2 # How many times to reuse an input to generate a label.    
    valid_size = 16 # Random set of words to evaluate similarity on.
    valid_window = 100 # Only pick dev samples in the head of the distribution.
    num_sampled = 64 # Number of negative examples to sample.

# 需要导入模块: from Dataset import Dataset [as 别名]
# 或者: from Dataset.Dataset import load [as 别名]
# Test code for detect heart region
import detectHeartRegion as dhr

from matplotlib import pyplot
from matplotlib import cm
from Dataset import Dataset


d = Dataset("C:\\Kaggle\\train\\27", "27");
d.load();
(num_slices, num_times, width,height) = d.images.shape

rois,circles = dhr.detect_heart_region(d.images);


#plot roi in each slice at time 0
numSlicesToDisplay = 10;
pyplot.figure(1);
pyplot.subplots_adjust(left=0.1,hspace=0.1,wspace=0);

numslicesPerRow = 2;
numRows = numSlicesToDisplay/numslicesPerRow;
index = 1;
for slice in range(numSlicesToDisplay):
    pyplot.subplot(numRows,2 * numslicesPerRow, index );
    pyplot.imshow(d.images[slice][0],cmap=cm.Greys_r);
    index = index + 1;
    
    pyplot.subplot(numRows,2 * numslicesPerRow, index) ;
    pyplot.imshow(rois[slice],cmap=cm.Greys_r);
    index = index + 1;

# 需要导入模块: from Dataset import Dataset [as 别名]
# 或者: from Dataset.Dataset import load [as 别名]
            print ('Batch size: ', batch_size)
            print ('Fully connected layer 1 size: ', full_layer_1)
                
            if show_plot:
                plt.plot(error_hist)
                plt.ylabel('Error rates')
                plt.show()
            
            
    
                
if __name__ == '__main__':
    print ('Loading the dataset... ')
    start = time.time()
    data = Dataset('notMNIST', reformatted=True, verbose=True)
    train_dataset, train_labels, \
            valid_dataset, valid_labels, \
            test_dataset, test_labels = data.load()
    test_dataset_alt, test_labels_alt = data.load_test()
    end = time.time() 
    print ('Loading the dataset took %.2f sec.\n' % (end - start))
    
    tensorflow_conv2(train_dataset, train_labels, valid_dataset, valid_labels
                    ,test_dataset, test_labels, test_dataset_alt, test_labels_alt
                    ,batch_size=16, patch_size=5, depth=16, save_summary=False
                    ,learn_rate=0.002, keep_probability=0.5, reg_param=0.000#1
                    ,num_steps=501, full_layer_1=50)
        
    print ('Dataset: ', data.name)