Python joblib.Parallel类代码示例

def auto_choose(actionfile, new_xyz, nparallel=-1):
    """
    @param demofile: h5py.File object
    @param new_xyz : new rope point-cloud
    @nparallel     : number of parallel jobs to run for tps cost calculaion.
                     If -1 only 1 job is used (no parallelization).
    
    @return          : return the name of the segment with the lowest warping cost.
    """
    if not nparallel == -1:
        from joblib import Parallel, delayed
        nparallel = min(nparallel, 8)

    demo_data = actionfile.items()

    if nparallel != -1:
        before = time.time()
        redprint("auto choose parallel with njobs = %d"%nparallel)
        costs  = Parallel(n_jobs=nparallel, verbose=0)(delayed(registration_cost)(ddata[1]['cloud_xyz'][:], new_xyz) for ddata in demo_data)
        after  = time.time()
        print "Parallel registration time in seconds =", after - before
    else:
        costs = []
        redprint("auto choose sequential..")
        for i, ddata in enumerate(demo_data):
            costs.append(registration_cost(ddata[1]['cloud_xyz'][:], new_xyz))
            print(("tps-cost completed %i/%i" % (i + 1, len(demo_data))))

    ibest = np.argmin(costs)
    redprint ("auto choose returning..")
    return demo_data[ibest][0]

	def __init__(self):
		global transport 

		variables = ['meridional_transport','psi']
		num_cores = 6

		data = np.ones((len(variables),len(scow.months),scow.latitude.shape[0],scow.longitude.shape[0]))*np.nan

		beta = c.beta.repeat(scow.longitude.shape[0]).reshape((c.beta.shape[0],scow.longitude.shape[0]))

		for i in xrange(scow.data.shape[1]):
			transport = scow.data[2,i,:,:]/beta

			psi = Parallel(n_jobs=num_cores)(delayed(integration)(lat) for lat in scow.latitude)
			psi = np.array(psi)

			D = np.array([transport.copy()/(c.rho*1.e+6),psi.copy()/(c.rho*1.e+6)])
			data[:,i,:,:] = D			

		del transport 

		# Here I could derive psi in y and get the zonal sverdrup transport (I think I won't need it)


		# "Isolating" the subtropical gyre 
		ibad = (np.abs(scow.latitude) <= 5) | (np.abs(scow.latitude) >= 50)
		data[:,:,ibad,:] = np.nan

		self.latitude = scow.latitude
		self.longitude = scow.longitude
		self.variables = variables
		self.data = data

def load_glm_inputs(study_dirs, hrf_model='canonical', drift_model='cosine',
                    img_ext='nii.gz', memory=Memory(None), n_jobs=1):
    """Returns data (almost) ready to be used for a GLM.
    """
    datasets, structural, functional, conditions, contrasts = \
        collect_openfmri(study_dirs, img_ext=img_ext, memory=memory, n_jobs=n_jobs)

    main = functional.merge(conditions)

    # computing design matrices
    print 'Computing models...'
    results = Parallel(n_jobs=n_jobs, pre_dispatch='n_jobs')(
        delayed(memory.cache(_make_design_matrix))(
            run_df, hrf_model, drift_model, orthogonalize=datasets[group_id[0]]['models'][group_id[2]]['orthogonalize'])
        for group_id, group_df in main.groupby(['study', 'subject', 'model'])
        for run_id, run_df in group_df.groupby(['task', 'run'])
        )

    # collect results
    print 'Collecting...'
    glm_inputs = {}
    for group_id, group_df in main.groupby(['study', 'subject', 'model']):
        study_id, subject_id, model_id = group_id
        for session_id, run_df in group_df.groupby(['task', 'run']):
            task_id, run_id = session_id
            bold_file, dm = results.pop(0)        
            glm_inputs.setdefault(group_id, {}).setdefault('bold', []).append(bold_file)
            glm_inputs.setdefault(group_id, {}).setdefault('design', []).append(dm)
        glm_inputs.setdefault(group_id, {}).setdefault(
            model_id, _make_contrasts(datasets, study_id, model_id, hrf_model, group_df))
        glm_inputs.setdefault(group_id, {}).setdefault(
            '%s_per_run' % model_id, _make_contrasts(
                datasets, study_id, model_id, hrf_model, group_df, per_run=True))
    return glm_inputs

def preprocess(file_in, file_out, test=False, n_jobs=6):
    """
    This function preprocesses raw data file.
    For each row and for each feature it extracts aggregations over TimeToEnd:
        From feature TimeToEnd it extracts total time ("time") and number of observations ("n_obs")
        From feature DistanceToRadar it extracts aggregations ('min', '50% quantile', 'mean', 'max')
        For any other features it calculates ('mean', 'std', 'min', '50% quantile', 'max')

        New features names follow the pattern: <feature name>_<aggregation function>

    Parameters
    ----------
    :param file_in: str
        csv-file name for data to be preprocessed
    :param file_out: str
        csv-file name for output data
    :param test: bool
        indicator for test data (data without label)
    :return:
    """
    # Load data to pandas.DataFrame
    data_raw = pd.read_csv(file_in, na_filter=False, chunksize=5000)

    # Apply transformations to data chunks in parallel
    start = time.time()
    data = Parallel(n_jobs=n_jobs, verbose=11)(delayed(foo)(x, transform, axis=1, test=test) for i, x in enumerate(data_raw))
    print "Preprocessing time: ", round((time.time() - start) / 60, 3)
    print "Records: ", len(data)

    # Join data chunks and save result to csv
    data = pd.concat(data)
    data.to_csv(file_out, index=False)

    print "File", file_in, "preprocessed to", file_out

def main():
    parser = argparse.ArgumentParser(description='Register & align images')
    parser.add_argument('filenames',nargs='+',help='List of target files to register. Images are aligned to first in list.')
    parser.add_argument('-odir',metavar='outdir',required=True,type=str,help='Output directory for files.')
    parser.add_argument('-m',metavar='method',choices=('point','extended'),default='extended',help='Specify alignment method (point or extended); default=extended.')
    parser.add_argument('-xy',nargs=2,type=float,default=None,help='Specify approximate "x y" pixel coordinate of object to centroid on.  Required for point mode; useful for extended mode (default=center of image).')
    parser.add_argument('-box',nargs=2,type=int,default=None,help='Specify box size (w h) to restrict alignment search.  Useful for both point & extended modes (default=full size of array).')
    parser.add_argument('--c',action='store_true',help='Clobber (overwrite) on output')
    parser.add_argument('-njobs',type=int,default=1,help='Process images in parallel. "-1" is all CPUs (default=1).')
    
    args = parser.parse_args()

    if args.m == 'point' and args.xy is None:
        parser.error("-m point requires -xy coordinate")

    # create output directory
    if args.odir not in ['','.']:
        makedirs(args.odir,exist_ok=True)

    # align all images to first filename
    ref = args.filenames[0]
    align = args.filenames[1:]

    imref = partial(register,ref=ref,outdir=args.odir,
                    method=args.m,center=args.xy,size=args.box,
                    overwrite=args.c)
    
    outfiles = Parallel(n_jobs=args.njobs,verbose=11)(delayed(imref)(toshift=a) for a in align)

    # Write ref to outdir
    refnew = os.path.join(args.odir,os.path.basename(ref))
    copy(ref,refnew)

    outfiles.append(refnew)
    print('Wrote %i files to %s' % (len(outfiles), args.odir))

def find_closest_auto(demofile, new_xyz):
    if args.parallel:
        from joblib import Parallel, delayed
    demo_clouds = [asarray(seg["cloud_xyz"]) for seg in demofile.values()]
    keys = demofile.keys()
    if args.parallel:
        costs = Parallel(n_jobs=3,verbose=100)(delayed(registration_cost)(demo_cloud, new_xyz) for demo_cloud in demo_clouds)
    else:
        costs = []
        for (i,ds_cloud) in enumerate(demo_clouds):
            costs.append(registration_cost(ds_cloud, new_xyz))
            print "completed %i/%i"%(i+1, len(demo_clouds))
    
    print "costs\n",costs
    if args.show_neighbors:
        nshow = min(5, len(keys))
        import cv2, rapprentice.cv_plot_utils as cpu
        sortinds = np.argsort(costs)[:nshow]
        near_rgbs = [asarray(demofile[keys[i]]["rgb"]) for i in sortinds]
        bigimg = cpu.tile_images(near_rgbs, 1, nshow)
        cv2.imshow("neighbors", bigimg)
        print "press any key to continue"
        cv2.waitKey()
        
    ibest = np.argmin(costs)
    return keys[ibest]

    def analysis(self, permute=False):
        """
        Classify based an iteratively increasing the number of features (electrodes) included in the model. Starts with
        the single best electrode (N=1) and increase until N = the number of electrodes.

        Note: permute is not used in this analysis, but kept to match the same signature as super.
        """
        if self.subject_data is None:
            print('%s: compute or load data first with .load_data()!' % self.subject)

        # Get recalled or not labels
        if self.recall_filter_func is None:
            print('%s classifier: please provide a .recall_filter_func function.' % self.subject)
        y = self.recall_filter_func(self.subject_data)

        # zscore the data by session
        x = self.zscore_data()

        # create the classifier
        classifier = LogisticRegression(C=self.C, penalty=self.norm, solver='liblinear')

        # create .num_rand_splits of cv_dicts
        cv_dicts = [self._make_cross_val_labels() for _ in range(self.num_rand_splits)]

        # run permutations with joblib
        f = _par_compute_and_run_split
        if self.use_joblib:
            aucs = Parallel(n_jobs=12, verbose=5)(delayed(f)(cv, classifier, x, y) for cv in cv_dicts)
        else:
            aucs = []
            for cv in tqdm(cv_dicts):
                aucs.append(f(cv, classifier, x, y))

        # store results
        self.res['auc_x_n'] = np.stack(aucs)

def best_classifier(X,Y,Xvs,Yvs):
    parameters = {'C':[3,13,67,330,1636,8103]}
    pg = ParameterGrid(parameters)
    clas = Parallel(n_jobs=4)(delayed(pfit)(p,X,Y,Xvs,Yvs) for p in pg)
    clas.sort(reverse=True)
    (sc,cla) = clas[0]
    print '-'*20
    print 'best is ',cla,sc
    print '-'*20
    return cla,sc

def retrieve_proposals(video_info, model, feature_filename,
                       feat_size=16, stride_intersection=0.1):
    """Retrieve proposals for a given video.
    
    Parameters
    ----------
    video_info : DataFrame
        DataFrame containing the 'video-name' and 'video-frames'.
    model : dict
        Dictionary containing the learned model.
        Keys: 
            'D': 2darray containing the sparse dictionary.
            'cost': Cost function at the last iteration.
            'durations': 1darray containing typical durations (n-frames)
                 in the training set.
            'type': Dictionary type.
    feature_filename : str
        String containing the path to the HDF5 file containing 
        the features for each video. The HDF5 file must contain 
        a group for each video where the id of the group is the name 
        of the video; and each group must contain a dataset containing
        the features.
    feat_size : int, optional
        Size of the temporal extension of the features.
    stride_intersection : float, optional
         Percentage of intersection between temporal windows.
    """
    feat_obj = FeatHelper(feature_filename, t_stride=1)
    candidate_df = generate_candidate_proposals(video_info, model['durations'],
                                                feat_size, stride_intersection)
    D = model['D']
    params = model['params']
    feat_obj.open_instance()
    feat_stack = feat_obj.read_feat(video_info['video-name'])
    feat_obj.close_instance()
    n_feats = feat_stack.shape[0]
    candidate_df = candidate_df[
        (candidate_df['f-init'] + candidate_df['n-frames']) <= n_feats]
    candidate_df = candidate_df.reset_index(drop=True)
    proposal_df = Parallel(n_jobs=-1)(delayed(wrapper_score_proposals)(this_df,
                                                                      D, 
                                                                     feat_stack,
                                                                       params,
                                                                     feat_size)
                                      for k, this_df in candidate_df.iterrows())
    proposal_df = pd.concat(proposal_df, axis=1).T
    proposal_df['score'] = (
        proposal_df['score'] - proposal_df['score'].min()) / (
            proposal_df['score'].max() - proposal_df['score'].min())
    proposal_df['score'] = np.abs(proposal_df['score'] - 1.0)
    proposal_df = proposal_df.loc[proposal_df['score'].argsort()[::-1]]
    proposal_df = proposal_df.rename(columns={'n-frames': 'f-end'})
    proposal_df['f-end'] = proposal_df['f-init'] + proposal_df['f-end'] - 1
    return proposal_df.reset_index(drop=True)

def basic_compute_loop(compute_function,looper,run_parallel=True,debug=False):
	"""Canonical form of the basic compute loop."""
	start = time.time()
	if run_parallel:
		incoming = Parallel(n_jobs=8,verbose=10 if debug else 0)(
			delayed(compute_function,has_shareable_memory)(**looper[ll]) 
			for ll in framelooper(len(looper),start=start))
	else: 
		incoming = []
		for ll in framelooper(len(looper)):
			incoming.append(compute_function(**looper[ll]))
	return incoming

def auto_choose(actionfile, new_xyz, softmin_k = 1, softmin_alpha = 1, nparallel=-1):
    """
    @param demofile  : h5py.File object
    @param new_xyz   : new rope point-cloud
    @param softmin   : use softmin distribution over first <softmin> demonstrations
                       set to 1 for nearest neighbor
    @param nparallel : number of parallel jobs to run for tps cost calculaion
                       set to -1 for no parallelization
    
    @return          : return the name of the segment with the lowest warping cost.
    """
    if not nparallel == -1:
        from joblib import Parallel, delayed
        nparallel = min(nparallel, 8)

    demo_data = actionfile.items()

    if nparallel != -1:
        before = time.time()
        redprint("auto choose parallel with njobs = %d"%nparallel)
        costs  = Parallel(n_jobs=nparallel, verbose=100)(delayed(registration_cost)(ddata[1]['cloud_xyz'][:], new_xyz) for ddata in demo_data)
        after  = time.time()
        print "Parallel registration time in seconds =", after - before
    else:
        costs = []
        redprint("auto choose sequential..")
        for i, ddata in enumerate(demo_data):
            costs.append(registration_cost(ddata[1]['cloud_xyz'][:], new_xyz))
            print(("tps-cost completed %i/%i" % (i + 1, len(demo_data))))
    
    # use a random draw from the softmin distribution
    demo_costs = zip(costs, demo_data)
    if softmin_k == 1:
        ibest = np.argmin(costs)
        return demo_data[ibest][0]
    best_k_demos = np.asarray(sorted(demo_costs)[:softmin_k])
    best_k_exps = np.exp(-1*softmin_alpha*float(best_k_demos[:, 0]))  #multiply by -1 b/c we're actually min-ing
    if len(best_k_exps) > 1:
        denom = sum(best_k_exps)
    else:
        denom = best_k_exps
    mass_fn = best_k_exps/denom

    draw = random.random()
    for i in range(best_k_demos):
        if draw <= mass_fn[i]:
            ret_val = demo_data[i][0]
            break
        draw -= mass_fn[i]
    
    redprint ("auto choose returning..")
    return ret_val

	def train(self):
		regressors = []
		if self.parallel:
			regressors = Parallel(n_jobs=-1)(delayed(trainBin)(self.params[b], np.atleast_2d(self.ind).T, self.dep[b],self.indWeights) for b in self.OD.bins)
		else:
			for b in self.OD.bins:
				regressors.append(trainBin(self.params[b],np.atleast_2d(self.ind).T, self.dep[b],self.indWeights))
				#self.svr[b] = SVR(cache_size=1000,kernel='rbf', C=self.params[b]['C'], gamma=self.params[b]['gamma'])
				#self.svr[b].fit(np.array([self.ind]).T,self.dep[b])
				
		
		for i,model in enumerate(regressors):
			self.svr[self.OD.bins[i]] = model

def extract_all_class_features(dataset, n_jobs=1, stride=5, patch_size=10):
    """Extract masked features from all dataset images, return features and labels"""
    cns = []
    labels = []
    for (label, cls) in enumerate(dataset.classes):
        print 'Extracting masked CNs from class {}'.format(cls)
        hists = Parallel(n_jobs=n_jobs)(delayed(extract_masked_cns)(imname, maskname) for (imname, maskname) in dataset.get_class_images(cls))
        hists = np.vstack(hists)
        labels.append(label * np.ones((len(hists),), dtype=np.float32))
        cns.append(hists.astype(np.float32))
    
    # Stack lists in numpy arrays.
    return (cns, labels)

def create_training_data():
  num_cores = 8

  # getting total number of trips
  list_of_files = [[folder, f.replace('.csv','')] for folder in os.listdir('drivers') if 'DS_Store' not in folder
                 for f in os.listdir('drivers/'+folder) if '.csv' in f]

  raw_data = Parallel( n_jobs=num_cores )(delayed(create_attributes)(i) for i in list_of_files)
  raw_data = pd.DataFrame(raw_data)
  raw_data.columns = ['driver_trip','trip_time','total_distance','skyway_distance','avg_speed','std_speed',
                      'avg_speed_up','avg_speed_down',
                      'avg_acc','std_acc','avg_turn','std_turn','standing_time','standing_speed']
  # save to file for later training
  raw_data.to_csv('training_set.csv', index=False)
  return raw_data

    def predict(self, test_set=True, location=None):
        Y, self.locations = self.data.get_y(location=location)
        t = self.data.observations['time'].values
        t = self._split_dataset(t, test_set=test_set)
        Y = self._split_dataset(Y, test_set=test_set)
        yhat_jobs = []
        ytrue =[]
        yoccur_jobs = []
        if not self.nearest_neighbor:
            X = self.data.get_X()
            X = self._split_dataset(X, test_set=test_set) 
            if self.xtransform is not None:
                X = self.xtrans.transform(X)
        for j, row in self.locations.iterrows():
            if self.nearest_neighbor:
                X = self.data.get_nearest_X(row[self.data.reanalysis_latdim],
                                   row[self.data.reanalysis_londim])

                X = self._split_dataset(X, test_set=test_set) 
                if self.xtransform is not None:
                    X = self.xtrans[j].transform(X)
            if self.conditional is not None:
                yoccur_jobs += [delayed(worker_predict_prob)(self.occurance_models[j], copy.deepcopy(X))]

            yhat_jobs += [delayed(worker_predict)(self.models[j], copy.deepcopy(X))]
            ytrue += [Y[:, j]]

        yhat = Parallel(n_jobs=self.num_proc)(yhat_jobs)
        if self.ytransform is not None:
            transform_jobs = [delayed(worker_invtrans)(self.ytrans[j], yhat[j]) for j in
                                                       range(len(yhat))]
            yhat = Parallel(n_jobs=self.num_proc)(transform_jobs)

        yhat = numpy.vstack(yhat).T
        ytrue = numpy.vstack(ytrue).T
        yhat = self.to_xarray(yhat, t).rename({"value": "projected"})
        ytrue = self.to_xarray(ytrue, t).rename({"value": "ground_truth"})
        if self.conditional is not None:
            yoccur = Parallel(n_jobs=self.num_proc)(yoccur_jobs)
            yoccur = numpy.vstack(yoccur).T > 0.5
            yoccur = self.to_xarray(yoccur, t).rename({"value": "occurance"})
            yhat['projected'] = yhat['projected']*yoccur['occurance']
            yhat = yhat.merge(yoccur)

        out = yhat.merge(ytrue) 
        out['error'] = out.projected - out.ground_truth
        return out