Python datamodel.DataModel类代码示例

def test_data_size_vs_diff(dm, given_dict, infer_dict):
    #Read all data from data model
    dm.read_data(normalize_data=False)   
    #attr_list = [U_UNIVERSITY_CODE, PROGRAM_CODE, UNIVERSITY, MAJOR_CODE, TERM]
    attr_list = [U_UNIVERSITY_CODE, PROGRAM_CODE, UNIVERSITY]
    #attr_list = [MAJOR_CODE, PROGRAM_CODE, TERM]
    
    #Size of data
    data_size = len(dm.data)

    #Step size = 10 steps 
    step_size = data_size//10

    #Get experiment data in a dict
    size = []
    accuracy = []

    for i in xrange(step_size, data_size, step_size):
        dm_test = DataModel("")
        dm_test.set_data(dm.data[:i])
        exp_test = Experimenter(dm_test, attr_list)
        actual = exp_test.get_actual_result(given_dict, infer_dict)
        estimation = exp_test.generic_get_estimated_result(given_dict, infer_dict)
        size.append(i)
        accuracy.append(abs(estimation - actual))
        print("Step:%d--->Actual:%f--->Estimate:%f" %(i, actual, estimation))
        print "-------------------------------------------------------------"
    plt.figure()
    plt.plot(size, accuracy)
    plt.title("Data Size vs Accuracy")
    plt.show()

def main(args):
    dm = DataModel(args.gig_file, args.chat_file)
    dm.read_data()
    exp = Experimenter(dm)
    if args.classify is True:
        scores = exp.classify_gigs()
    if args.feature_values is True:
        scores = exp.evaluate_feature_values()
    return dm

def main(argv):
  train_count = -1
  if(len(argv)>0):
    train_count = int(argv[0])
  dm = DataModel()
  dm.get_data(train_count)
#  (training, feature_names) = get_rich_featured_training(dm,lines)
  print(len(dm.data.keys()))
  print(len(dm.train))
  print(len(dm.test))

 def perform_datasize_vs_efficiency(self, given_dict, infer_dict, max_datasize=None, steps=10):
     sizes, est_times, acc_times = [], [], []
     if max_datasize is None:
         max_datasize = len(self.dm.data)
     data_step = max_datasize / steps
     for i in range(steps):
         cur_datasize = (i+1) * data_step
         data = self.dm.data
         while len(data) < cur_datasize:
             data.extend(self.dm.data)
         cur_data = data[:cur_datasize]
         cur_dm = DataModel("")
         cur_dm.set_data(cur_data)
         cur_exp = Experimenter(cur_dm, self.attr_list)
         (cur_est, cur_acc) = cur_exp.time_n_queries(given_dict, infer_dict)
         sizes.append(cur_datasize)
         est_times.append(float(sum(cur_est))/len(cur_est))
         acc_times.append(float(sum(cur_acc))/len(cur_acc))
     return (sizes, est_times, acc_times)

def export(items, tempdir):
    """Export a list of items
    
    Arguments
    items -- list of items to export
    tempdir -- directory to use for the export operation
    
    """
    initdir = spm.spmanager.getFirstPath([spm.ExportFolder, 
                                          spm.ImportFolder, 
                                          spm.MostRecentFolder])
    
    filenamepath = tkFileDialog.asksaveasfilename(initialdir = initdir, 
                                                  filetypes = ff.dlgExportFormats, 
                                                  defaultextension = ff.dlgDefaultExportExt)
        
    if(len(filenamepath) < 1):
        return
    
    spm.spmanager.setPath(spm.ExportFolder, os.path.dirname(filenamepath))
    
    #Create export dir and datamodel
    dmdir = os.path.join(tempdir, _exportdir)
    
    if(os.path.exists(dmdir)):
       shutil.rmtree(dmdir)
       
    os.makedirs(dmdir)
    
    dm = DataModel(dmdir)
    
    #Add all slideshows
    for item in items:
        if(not dm.addSlideshow(item, True)):
            showerror(lang[lng.txtExportError], lang[lng.txtCouldNotExport] + item.title)
            shutil.rmtree(dmdir)
            return

    #Save and zip
    dm.saveToFile()
    pack(dmdir, filenamepath)
    shutil.rmtree(dmdir)

def main(args):
    dm = DataModel(args.data_file)
    dm.read_data(to_read_count=10000)
    exp = Experimenter(dm, \
            process_datamodel=True, \
            serialise=False)
    t1 = time.time()
    exp.perform_multiclass_experiment(
            pred_mode=INDEPENDENT,
            use_exclusion=True,
            need_to_extract_features=True,
            prediction_file='../results/predictions_multiclass_independent_englishonly_legibleonly_wordunibigram_chartrigram_10000.csv',
            result_file='../results/results_multiclass_independent_englishonly_legibleonly_wordunibigram_chartrigram_10000.txt',
            english_only=True,
            legible_only=True)
    t2 = time.time()
    timeused = t2 - t1
    logging.getLogger(LOGGER).info('Time used in experiment (hour:min:sec): %d:%d:%d' % \
            (timeused/3600, timeused/60, timeused%60))
    return exp

    def OnOpenImage(self, evt=None):
        # 1) Get the image key
        # Start with the table_id if there is one
        tblNum = None
        if p.table_id:
            dlg = wx.TextEntryDialog(self, p.table_id + ":", "Enter " + p.table_id)
            dlg.SetValue("0")
            if dlg.ShowModal() == wx.ID_OK:
                try:
                    tblNum = int(dlg.GetValue())
                except ValueError:
                    errdlg = wx.MessageDialog(
                        self, "Invalid value for %s!" % (p.table_id), "Invalid value", wx.OK | wx.ICON_EXCLAMATION
                    )
                    errdlg.ShowModal()
                    return
                dlg.Destroy()
            else:
                dlg.Destroy()
                return
        # Then get the image_id
        dlg = wx.TextEntryDialog(self, p.image_id + ":", "Enter " + p.image_id)
        dlg.SetValue("")
        if dlg.ShowModal() == wx.ID_OK:
            try:
                imgNum = int(dlg.GetValue())
            except ValueError:
                errdlg = wx.MessageDialog(
                    self, "Invalid value for %s!" % (p.image_id), "Invalid value", wx.OK | wx.ICON_EXCLAMATION
                )
                errdlg.ShowModal()
                return
            dlg.Destroy()
        else:
            dlg.Destroy()
            return
        # Build the imkey
        if p.table_id:
            imkey = (tblNum, imgNum)
        else:
            imkey = (imgNum,)

        dm = DataModel.getInstance()
        if imkey not in dm.GetAllImageKeys():
            errdlg = wx.MessageDialog(
                self, "There is no image with that key.", "Couldn't find image", wx.OK | wx.ICON_EXCLAMATION
            )
            errdlg.ShowModal()
            self.Destroy()
        else:
            # load the image
            self.img_key = imkey
            self.SetImage(imagetools.FetchImage(imkey), p.image_channel_colors)
            self.DoLayout()

 def perform_datasize_vs_accuracy(self, given_dict, infer_dict, max_datasize=None, steps=10):
     #Get experiment data in a dict
     size = []
     accuracy = []
     if max_datasize is None:
         max_datasize = len(self.dm.data)
     data_step = max_datasize / steps
     
     for i in range(steps):
         cur_datasize = (i+1) * data_step
         data = self.dm.data
         while len(data) < cur_datasize:
             data.extend(self.dm.data)
         cur_data = data[:cur_datasize]
         cur_dm = DataModel("")
         cur_dm.set_data(cur_data)
         cur_exp = Experimenter(cur_dm, self.attr_list)
         actual = cur_exp.get_actual_result(given_dict, infer_dict)
         estimation = cur_exp.generic_get_estimated_result(given_dict, infer_dict)
         size.append(cur_datasize)
         accuracy.append(abs(estimation - actual))
     return (size, accuracy)

 def get_image_keys_at_row(self, row):
     '''Returns a list of image keys at the given row or None if the column 
     names can't be found in col_labels
     '''
     if self.key_indices is None or self.grouping is None:
         return None
     else:
         if self.grouping.lower() == 'image':     
             return [tuple(self.data[self.row_order,:][row, self.key_indices])]
         elif self.grouping.lower() == 'object': 
             return [tuple(self.data[self.row_order,:][row, self.key_indices[:-1]])]
         else:
             dm = DataModel.getInstance()
             return dm.GetImagesInGroup(self.grouping, self.get_row_key(row))

    def PerImageCounts(self, filter_name=None, cb=None):
        # Clear the current perClassObjects storage
        for bin in self.classBins:
            self.perClassObjects[bin.label] = []

        # Retrieve a data model instance
        dm = DataModel.getInstance()

        # Retrieve image keys and initialize variables
        imageKeys = dm.GetAllImageKeys(filter_name)
        imageAmount = float(len(imageKeys))
        perImageData = []

        # Process all images
        for k_index, imKey in enumerate(imageKeys):
            try:
                # Retrieve the keys of the objects in the current image
                obKeys = dm.GetObjectsFromImage(imKey)
            except:
                raise ValueError('No such image: %s' % (imKey,))

            # Calculate the amount of hits for each of the classes in the current image
            classHits = {}
            objectCount = [imKey[0]]
            if obKeys:
                classObjects = self.FilterObjectsFromClassN(keys = [imKey])
                for clNum, bin in enumerate(self.classBins):
                    # Get the objects from the image which belong to the selected class
                    classHits[bin.label] = classObjects[float(clNum+1)]

                    # Store the total object count of this class for the current image
                    nrHits = len(classHits[bin.label])
                    objectCount.append(nrHits)

                    # Store the objects for the current class and image grouped
                    # by class if any are found for this class in the selected image
                    if nrHits > 0:
                        self.perClassObjects[bin.label] += classHits[bin.label]
            else:
                # If there are objects in the image, add zeros for all bins
                [objectCount.append(0) for bin in self.classBins]

            # Store the results for the current image and update the callback
            # function if available
            perImageData.append(objectCount)
            if cb:
                cb(min(1, k_index/imageAmount))

        return perImageData

 def get_object_keys_at_row(self, row):
     '''Returns a list of object keys at the given row or None if the column
     names can't be found in col_labels
     '''
     if self.key_indices is None or self.grouping is None:
         return None
     else:
         dm = DataModel.getInstance()
         # If the key index for the row is an object key, just return that key
         if self.grouping.lower() == 'object': 
             return [tuple(self.data[self.row_order,:][row, self.key_indices])]
         else: # Otherwise, return all object keys in the image
             imkeys = self.get_image_keys_at_row(row) 
             obkeys = []
             for imkey in imkeys:
                 obs = dm.GetObjectCountFromImage(imkey)
                 obkeys += [tuple(list(imkey)+[i]) for i in range(1,obs+1)]
             return obkeys

    def __init__(self):
        self._dm = DataModel()              # Primary data model.

        self._topics    = []                # All topics for which judgments have been loaded.
        self._documents = []                # Documents for which we have a judgment for the
                                            # currently selected topic.

        self._selected_topic    = None      # Currently selected topic.
        self._selected_document = None      # Currently selected document.
        self._rationales        = []        # Rationales for the currently selected document.

        self._display_text = None           # Text of document being manipulated.

        super(CWR, self).__init__()

        self.init_UI()

        # For testing WebView.
        ''' 

 def on_dclick_label(self, evt):
     '''Handle display of images and objects'''
     if evt.Row >= 0:
         obkeys = self.grid.Table.get_object_keys_at_row(evt.Row)
         if self.grid.Table.grouping is None:
             # We need to know how the table is grouped to know what to do
             logging.warn('CPA does not know how to link this table to your images. Can\'t launch ImageViewer.')
             return
         elif self.grid.Table.grouping.lower() == 'object':
             # For per-object grouping, show the objects in the image
             imview = imagetools.ShowImage(obkeys[0][:-1], 
                                               p.image_channel_colors,
                                               parent=self.Parent)
             if obkeys is not None:
                 for obkey in obkeys:
                     imview.SelectObject(obkey)
         elif self.grid.Table.grouping.lower() == 'image':
             # For per-image grouping just show the images.
             # If there is only one object, then highlight it
             if obkeys is not None and len(obkeys) == 1:
                 imview = imagetools.ShowImage(obkeys[0][:-1], 
                                               p.image_channel_colors,
                                               parent=self.Parent)
                 imview.SelectObject(obkeys[0])
             else:
                 imkeys = self.grid.Table.get_image_keys_at_row(evt.Row)
                 if imkeys:
                     #XXX: warn if there are a lot
                     for imkey in imkeys:
                         imagetools.ShowImage(imkey, p.image_channel_colors,
                                              parent=self.Parent)
         else:
             key_cols = self.grid.Table.get_row_key(evt.Row)
             if key_cols:
                 dm = DataModel.getInstance()
                 imkeys = dm.GetImagesInGroup(self.grid.Table.grouping, key_cols)
                 for imkey in imkeys:
                     imagetools.ShowImage(imkey, p.image_channel_colors,
                                          parent=self.Parent)

    def FilterObjectsFromClassN(self, classN = None, keys = None):
        '''
    	Filter the input objects to output the keys of those in classN, 
    	using a defined SVM model classifier.
    	'''
        # Retrieve instance of the database connection
        db = dbconnect.DBConnect.getInstance()
        object_data = {}
        if isinstance(keys, str):
            object_data[0] = db.GetCellDataForClassifier(keys)
        elif keys != []:
            if len(keys) == len(dbconnect.image_key_columns()):
                # Retrieve instance of the data model and retrieve objects in the requested image
                dm = DataModel.getInstance()
                obKeys = dm.GetObjectsFromImage(keys[0])
            else:
                obKeys = keys
            for key in obKeys:
                object_data[key] = db.GetCellDataForClassifier(key)

        sorted_keys = sorted(object_data.keys())
        values_array = np.array([object_data[key] for key in sorted_keys])
        scaled_values = self.ScaleData(values_array)
        pred_labels = self.model.predict(scaled_values)

        # Group the object keys per class
        classObjects = {}
        for index in range(1, len(self.classBins)+1):
            classObjects[float(index)] = []
        for index, label in enumerate(pred_labels):
            classObjects[np.int(label)+1].append(sorted_keys[index])

        # Return either a summary of all classes and their corresponding objects
        # or just the objects for a specific class
        if classN is None:
            return classObjects
        else:
            return classObjects[classN]

def FormatPlateMapData(keys_and_vals, categorical=False):
    '''
    keys_and_vals -- a list of lists of well-keys and values
                     eg: [['p1', 'A01', 0.2], 
                          ['p1', 'A02', 0.9], ...]
    returns a 2-tuple containing:
       -an array in the shape of the plate containing the given values with 
        NaNs filling empty slots. If multiple sites per-well are given, then
        the array will be shaped (rows, cols, sites)
       -an array in the shape of the plate containing the given keys with 
        (UnknownPlate, UnknownWell) filling empty slots
    '''
    from itertools import groupby
    keys_and_vals = np.array(keys_and_vals)
    nkeycols = len(dbconnect.well_key_columns())
    shape = list(p.plate_shape)
    if p.plate_type == '5600': 
        well_keys = keys_and_vals[:,:-1] # first column(s) are keys
        data = keys_and_vals[:,-1]       # last column is data
        assert data.ndim == 1
        if len(data) < 5600: raise Exception(
            '''The measurement you chose to plot was missing for some spots. 
            Because CPA doesn't know the well labelling convention used by this
            microarray, we can't be sure how to plot the data. If you are 
            plotting an object measurement, you may have some spots with 0 
            objects and therefore no entry in the table.''')
        assert len(data) == 5600
        data = np.array(list(meander(data.reshape(shape)))).reshape(shape)
        sort_indices = np.array(list(meander(np.arange(np.prod(shape)).reshape(shape)))).reshape(shape)
        well_keys = np.array(list(meander(well_keys.reshape(shape + [nkeycols] )))).reshape(shape + [nkeycols])
        return data, well_keys, sort_indices

    # compute the number of sites-per-well as the max number of rows with the same well-key
    nsites = max([len(list(grp))
                  for k, grp in groupby(keys_and_vals, 
                                        lambda row: tuple(row[:nkeycols]))
                  ])
    if nsites > 1:
        # add a sites dimension to the array shape if there's >1 site per well
        shape += [nsites]
    data = np.ones(shape) * np.nan
    if categorical:
        data = data.astype('object')
    if p.plate_id:
        dummy_key = ('UnknownPlate', 'UnknownWell')
    else:
        dummy_key = ('UnknownWell',)
    well_keys = np.array([dummy_key] * np.prod(shape), 
                         dtype=object).reshape(shape + [nkeycols])
    sort_indices = np.ones(data.shape)*np.nan
    
    dm = DataModel.getInstance()
    ind = keys_and_vals.argsort(axis=0)
    for i, (k, well_grp) in enumerate(groupby(keys_and_vals[ind[:,len(dummy_key)-1],:], 
                                              lambda row: tuple(row[:len(dummy_key)]))):
        (row, col) = dm.get_well_position_from_name(k[-1])
        well_data = np.array(list(well_grp))[:,-1]
        if len(well_data) == 1:
            data[row, col] = well_data[0]
            sort_indices[row,col] = ind[:,len(dummy_key)-1][i]
        else:
            data[row, col] = well_data
            sort_indices[row,col] = ind[:,len(dummy_key)-1][i*nsites + np.array(range(nsites))] 
        well_keys[row, col] = k
        
    return data, well_keys, sort_indices