Python cython_code.State类代码示例

def generate_X_L_and_X_D(T, M_c, cols_to_views, row_to_clusters, seed=0):
    state = State.p_State(M_c, T, SEED=seed)
    X_L = state.get_X_L()

    # insert assigment into X_L (this is not a valid X_L because the counts and 
    # suffstats will be wrong)
    X_L['column_partition']['assignments'] = cols_to_views
    state = State.p_State(M_c, T, X_L=X_L, X_D=row_to_clusters, SEED=seed)

    X_L = state.get_X_L()
    X_D = state.get_X_D()

    return X_L, X_D

def GenerateRandomState(n_rows, n_cols, seed, mean_gen=0.0, std_gen=1.0, std_data=0.1, alpha_col=1.0, alpha_rows=1.0):

	# check the inputs 
	assert(type(n_rows) is int)
	assert(type(n_cols) is int)
	assert(type(mean_gen) is float)
	assert(type(std_gen) is float)
	assert(type(std_data) is float)
	assert(type(alpha_col) is float)
	assert(type(alpha_rows) is float)
	assert(n_rows > 0)
	assert(n_cols > 0)
	assert(std_gen > 0.0)
	assert(std_data > 0.0)
	assert(alpha_col > 0.0)
	assert(alpha_rows > 0.0)

	rng = np.random.RandomState(seed)

	# generate the partitioning
	part = GenerateRandomPartition(n_rows, n_cols, alpha_col, alpha_rows, seed=seed)

	# fill it with data
	T, M_r, M_c = GenDataFromPartitions(part['col_parts'], part['row_parts'], mean_gen, std_gen, std_data)

	# this part is kind of hacky:
	# generate a state from the prior 
	state = State.p_State(M_c, T, N_GRID=100)
	# get the X_L and X_D and implant part['col_parts'], part['row_parts'], then 
	# create a new state with the new X_L and X_D defined
	X_L = state.get_X_L()
	X_D = state.get_X_D()

	# this should be all we need to change for 
	# State.transform_latent_state_to_constructor_args(X_L, X_D) to be able
	# to construct the arguments to intialize a state
	X_L['column_partition']['assignments'] = part['col_parts'].tolist()
	X_D = part['row_parts'].tolist()

	# hack in the alpha values supplied (or not) by the user
	X_L['column_partition']['hypers']['alpha'] = alpha_col
	for i in range(len(X_L['view_state'])):
		X_L['view_state'][i]['row_partition_model']['hypers']['alpha'] = alpha_col
	for i in range(n_cols):
		X_L['column_hypers'][i]['alpha'] = alpha_rows

	# create a new state with the updated X_D and X_L
	state = State.p_State(M_c, T, X_L=X_L, X_D=X_D, N_GRID=100)

	return state, T, M_r, M_c

def gen_data_crosscat(mode, T):
    # edit transition list according to 
    
    all_transitions = []

    M_c = du.gen_M_c_from_T(T, cctypes=['continuous']*2)

    state = State.p_state(M_c, T)
    if mode == 'crp_mixture':
        # fix the views
        X_D = state.get_X_D();
        X_L = state.get_X_L();
        X_D = [X_D[0]]
        X_L['column_partition']['assignments'] = [1,1]
        state = State.p_state(M_c, T, X_L=X_L, X_D=X_D)

def get_generative_clustering(M_c, M_r, T,
                              data_inverse_permutation_indices,
                              num_clusters, num_views):
    # NOTE: this function only works because State.p_State doesn't use
    #       column_component_suffstats
    num_rows = len(T)
    num_cols = len(T[0])
    X_D_helper = numpy.repeat(range(num_clusters), (num_rows / num_clusters))
    gen_X_D = [
        X_D_helper[numpy.argsort(data_inverse_permutation_index)]
        for data_inverse_permutation_index in data_inverse_permutation_indices
        ]
    gen_X_L_assignments = numpy.repeat(range(num_views), (num_cols / num_views))
    # initialize to generate an X_L to manipulate
    local_engine = LE.LocalEngine()
    bad_X_L, bad_X_D = local_engine.initialize(M_c, M_r, T,
                                                         initialization='apart')
    bad_X_L['column_partition']['assignments'] = gen_X_L_assignments
    # manually constrcut state in in generative configuration
    state = State.p_State(M_c, T, bad_X_L, gen_X_D)
    gen_X_L = state.get_X_L()
    gen_X_D = state.get_X_D()
    # run inference on hyperparameters to leave them in a reasonable state
    kernel_list = (
        'row_partition_hyperparameters',
        'column_hyperparameters',
        'column_partition_hyperparameter',
        )
    gen_X_L, gen_X_D = local_engine.analyze(M_c, T, gen_X_L, gen_X_D, n_steps=1,
                                            kernel_list=kernel_list)
    #
    return gen_X_L, gen_X_D

def _do_analyze_with_diagnostic(
        SEED, X_L, X_D, M_c, T, kernel_list, n_steps, c, r, max_iterations,
        max_time, diagnostic_func_dict, every_N, ROW_CRP_ALPHA_GRID,
        COLUMN_CRP_ALPHA_GRID, S_GRID, MU_GRID, N_GRID, do_timing, CT_KERNEL,
        progress,):

    diagnostics_dict = collections.defaultdict(list)

    if diagnostic_func_dict is None:
        diagnostic_func_dict = dict()
        every_N = None

    p_State = State.p_State(
        M_c, T, X_L, X_D, SEED=SEED, ROW_CRP_ALPHA_GRID=ROW_CRP_ALPHA_GRID,
        COLUMN_CRP_ALPHA_GRID=COLUMN_CRP_ALPHA_GRID, S_GRID=S_GRID,
        MU_GRID=MU_GRID, N_GRID=N_GRID, CT_KERNEL=CT_KERNEL)

    with gu.Timer('all transitions', verbose=False) as timer:
        p_State.transition(
            kernel_list, n_steps, c, r, max_iterations, max_time,
            progress=progress,
            diagnostic_func_dict=diagnostic_func_dict,
            diagnostics_dict=diagnostics_dict,
            diagnostics_every_N=every_N)

    X_L_prime = p_State.get_X_L()
    X_D_prime = p_State.get_X_D()

    if do_timing:
        # Diagnostics and timing are exclusive.
        diagnostics_dict = timer.elapsed_secs

    return X_L_prime, X_D_prime, diagnostics_dict

def _do_analyze2((M_c, T, X_L, X_D, kernel_list, n_steps, c, r,
               max_iterations, max_time, SEED)):
    p_State = State.p_State(M_c, T, X_L, X_D, SEED=SEED)
    p_State.transition(kernel_list, n_steps, c, r,
                       max_iterations, max_time)
    X_L_prime = p_State.get_X_L()
    X_D_prime = p_State.get_X_D()
    return X_L_prime, X_D_prime

def GenerateStateFromPartitions(col_parts, row_parts, mean_gen=0.0, std_gen=1.0, std_data=0.1):
	
	T, M_r, M_c = GenDataFromPartitions(col_parts, row_parts, mean_gen=mean_gen, std_gen=std_gen, std_data=std_data)
	state = State.p_State(M_c, T, N_GRID=100)

	X_L = state.get_X_L()
	X_D = state.get_X_D()

	if type(col_parts) is not list:
		X_L['column_partition']['assignments'] = col_parts.tolist()

	if type(row_parts) is not list:
		X_D = row_parts.tolist()

	# create a new state with the updated X_D and X_L
	state = State.p_State(M_c, T, X_L=X_L, X_D=X_D, N_GRID=100)

	return state, T, M_c, M_r, X_L, X_D

 def _sample_and_insert(self, M_c, T, X_L, X_D, matching_row_indices):
     p_State = State.p_State(M_c, T, X_L, X_D)
     draws = []
     for matching_row_idx in matching_row_indices:
         random_seed = self.get_next_seed()
         draw = p_State.get_draw(matching_row_idx, random_seed)
         p_State.insert_row(draw, matching_row_idx)
         draws.append(draw)
         T.append(draw)
     X_L, X_D = p_State.get_X_L(), p_State.get_X_D()
     return draws, T, X_L, X_D

def _do_initialize(
        SEED, M_c, M_r, T, initialization, row_initialization,
         ROW_CRP_ALPHA_GRID, COLUMN_CRP_ALPHA_GRID, S_GRID, MU_GRID, N_GRID,):

    p_State = State.p_State(
        M_c, T, initialization=initialization,
        row_initialization=row_initialization, SEED=SEED,
        ROW_CRP_ALPHA_GRID=ROW_CRP_ALPHA_GRID,
        COLUMN_CRP_ALPHA_GRID=COLUMN_CRP_ALPHA_GRID, S_GRID=S_GRID,
        MU_GRID=MU_GRID, N_GRID=N_GRID,)

    X_L = p_State.get_X_L()
    X_D = p_State.get_X_D()
    return X_L, X_D

def _do_insert(M_c, T, X_L, X_D, new_rows, N_GRID, CT_KERNEL):
    p_State = State.p_State(M_c, T, X_L=X_L, X_D=X_D,
                            N_GRID=N_GRID,
                            CT_KERNEL=CT_KERNEL)

    row_idx = len(T)
    for row_data in new_rows:
        p_State.insert_row(row_data, row_idx)
        p_State.transition(which_transitions=['row_partition_assignments'], r=[row_idx])
        row_idx += 1

    X_L_prime = p_State.get_X_L()
    X_D_prime = p_State.get_X_D()
    return X_L_prime, X_D_prime

	def setUp(self):
		# generate a crosscat state and pull the metadata
		gen_seed = 0
		num_clusters = 2
		self.num_rows = 10
		self.num_cols = 2
		num_splits = 1

		self.T, self.M_r, self.M_c = du.gen_factorial_data_objects(gen_seed,
							   num_clusters, self.num_cols, 
							   self.num_rows, num_splits)

		state = State.p_State(self.M_c, self.T)
		self.X_L = state.get_X_L()
		self.X_D = state.get_X_D()

def _do_analyze(
        SEED, X_L, X_D, M_c, T, kernel_list, n_steps, c, r,
        max_iterations, max_time, ROW_CRP_ALPHA_GRID, COLUMN_CRP_ALPHA_GRID,
        S_GRID, MU_GRID, N_GRID, CT_KERNEL, progress):

    p_State = State.p_State(
        M_c, T, X_L, X_D, SEED=SEED, ROW_CRP_ALPHA_GRID=ROW_CRP_ALPHA_GRID,
        COLUMN_CRP_ALPHA_GRID=COLUMN_CRP_ALPHA_GRID, S_GRID=S_GRID,
        MU_GRID=MU_GRID, N_GRID=N_GRID, CT_KERNEL=CT_KERNEL)

    p_State.transition(
        kernel_list, n_steps, c, r, max_iterations, max_time, progress)

    X_L_prime = p_State.get_X_L()
    X_D_prime = p_State.get_X_D()
    return X_L_prime, X_D_prime

def _do_analyze_with_diagnostic(SEED, X_L, X_D, M_c, T, kernel_list, n_steps, c, r,
                                max_iterations, max_time, diagnostic_func_dict, every_N,
                                ROW_CRP_ALPHA_GRID, COLUMN_CRP_ALPHA_GRID,
                                S_GRID, MU_GRID,
                                N_GRID,
                                do_timing,
                                CT_KERNEL,
                                ):
    diagnostics_dict = collections.defaultdict(list)
    if diagnostic_func_dict is None:
        diagnostic_func_dict = dict()
        every_N = None
    child_n_steps_list = get_child_n_steps_list(n_steps, every_N)
    # import ipdb; ipdb.set_trace()
    p_State = State.p_State(M_c, T, X_L, X_D, SEED=SEED,
                            ROW_CRP_ALPHA_GRID=ROW_CRP_ALPHA_GRID,
                            COLUMN_CRP_ALPHA_GRID=COLUMN_CRP_ALPHA_GRID,
                            S_GRID=S_GRID,
                            MU_GRID=MU_GRID,
                            N_GRID=N_GRID,
                            CT_KERNEL=CT_KERNEL,
                            )
    with gu.Timer('all transitions', verbose=False) as timer:
        for child_n_steps in child_n_steps_list:
            p_State.transition(kernel_list, child_n_steps, c, r,
                               max_iterations, max_time)
            for diagnostic_name, diagnostic_func in six.iteritems(diagnostic_func_dict):
                diagnostic_value = diagnostic_func(p_State)
                diagnostics_dict[diagnostic_name].append(diagnostic_value)
                pass
            pass
        pass
    X_L_prime = p_State.get_X_L()
    X_D_prime = p_State.get_X_D()
    #
    if do_timing:
        # diagnostics and timing are exclusive
        diagnostics_dict = timer.elapsed_secs
        pass
    return X_L_prime, X_D_prime, diagnostics_dict

def test_one_feature_mixture(component_model_type, num_clusters=3, show_plot=False, seed=None):
    """

    """
    random.seed(seed)

    N = 1000
    separation = .9
    
    get_next_seed = lambda : random.randrange(2147483647)

    cluster_weights = [[1.0/float(num_clusters)]*num_clusters]

    cctype = component_model_type.cctype
    T, M_c, structure = sdg.gen_data([cctype], N, [0], cluster_weights,
                        [separation], seed=get_next_seed(),
                        distargs=[distargs[cctype]],
                        return_structure=True)

    T = numpy.array(T)
    T_list = T
    
    # create a crosscat state 
    M_c = du.gen_M_c_from_T(T_list, cctypes=[cctype])
    
    state = State.p_State(M_c, T_list)
    
    # transitions
    state.transition(n_steps=200)
    
    # get the sample
    X_L = state.get_X_L()
    X_D = state.get_X_D()
    
    # generate samples
    # kstest has doesn't compute the same answer with row and column vectors
    # so we flatten this column vector into a row vector.
    predictive_samples = sdg.predictive_columns(M_c, X_L, X_D, [0],
                            seed=get_next_seed()).flatten(1)
    
    # Get support over all component models
    discrete_support = qtu.get_mixture_support(cctype, component_model_type,
                         structure['component_params'][0], nbins=500)

    # calculate simple predictive probability for each point
    Q = [(N,0,x) for x in discrete_support]

    probabilities = su.simple_predictive_probability(M_c, X_L, X_D, []*len(Q), Q)
    
    # get histogram. Different behavior for discrete and continuous types. For some reason
    # the normed property isn't normalizing the multinomial histogram to 1.
    if is_discrete[component_model_type.model_type]:
        bins = range(len(discrete_support))
        T_hist = numpy.array(qtu.bincount(T, bins=bins))
        S_hist = numpy.array(qtu.bincount(predictive_samples, bins=bins))
        T_hist = T_hist/float(numpy.sum(T_hist))
        S_hist = S_hist/float(numpy.sum(S_hist))
        edges = numpy.array(discrete_support,dtype=float)
    else:
        T_hist, edges = numpy.histogram(T, bins=min(20,len(discrete_support)), normed=True)
        S_hist, _ =  numpy.histogram(predictive_samples, bins=edges, normed=True)
        edges = edges[0:-1]

    # Goodness-of-fit-tests
    if not is_discrete[component_model_type.model_type]:
        # do a KS tests if the distribution in continuous
        # cdf = lambda x: component_model_type.cdf(x, model_parameters)
        # stat, p = stats.kstest(predictive_samples, cdf)   # 1-sample test
        stat, p = stats.ks_2samp(predictive_samples, T[:,0]) # 2-sample test
        test_str = "KS"
    else:
        # Cressie-Read power divergence statistic and goodness of fit test.
        # This function gives a lot of flexibility in the method <lambda_> used.
        freq_obs = S_hist*N
        freq_exp = numpy.exp(probabilities)*N
        stat, p = stats.power_divergence(freq_obs, freq_exp, lambda_='pearson')
        test_str = "Chi-square"
    
    if show_plot:
        lpdf = qtu.get_mixture_pdf(discrete_support, component_model_type, 
                structure['component_params'][0], [1.0/num_clusters]*num_clusters)
        pylab.axes([0.1, 0.1, .8, .7])
        # bin widths
        width = (numpy.max(edges)-numpy.min(edges))/len(edges)
        pylab.bar(edges, T_hist, color='blue', alpha=.5, width=width, label='Original data', zorder=1)
        pylab.bar(edges, S_hist, color='red', alpha=.5, width=width, label='Predictive samples', zorder=2)

        # plot actual pdf of support given data params
        pylab.scatter(discrete_support, 
            numpy.exp(lpdf), 
            c="blue", 
            edgecolor="none",
            s=100, 
            label="true pdf", 
            alpha=1,
            zorder=3)
                
        # plot predictive probability of support points
        pylab.scatter(discrete_support, 
            numpy.exp(probabilities), 
#.........这里部分代码省略.........

def _do_initialize2((M_c, M_r, T, initialization, SEED)):
    p_State = State.p_State(M_c, T, initialization=initialization, SEED=SEED)
    X_L = p_State.get_X_L()
    X_D = p_State.get_X_D()
    return X_L, X_D