Python DataLoader.load_data方法代码示例

# 需要导入模块: import DataLoader [as 别名]
# 或者: from DataLoader import load_data [as 别名]
rc("savefig", dpi=92)
rc("axes", linewidth=0.5, labelsize=9.0, titlesize=9.0)
rc("legend", fontsize="small")
rc("xtick.major", width=0.3)
rc("xtick", labelsize="small")
rc("ytick.major", width=0.3)
rc("ytick", labelsize="small")

z      = np.loadtxt("../../inputs/universe_age.dat", dtype = np.str, usecols = (0,))
age    = np.loadtxt("../../inputs/universe_age.dat", usecols = (1,))
IDs    = np.loadtxt("../../inputs/photoz3/set3.counts", dtype = np.str, usecols = (0,))
counts = np.loadtxt("../../inputs/photoz3/set3.counts", dtype = np.int, usecols = (1,))
output = ["z0p50", "z1p00", "z1p50", "z2p00", "z2p50"]
#output = ["z1p50", "z2p00", "z2p50", "z3p00"]
mask   = np.array([True if ID.replace(".", "p") in output else False for ID in IDs], dtype = np.bool)
data_z = [dl.load_data(ID.replace(".", "p"), count, 50, 56) for ID, count in zip(IDs[mask], counts[mask])]
ages   = [age[i] for i in xrange(z.size) if "z" + z[i].replace(".", "p") in output]
lab    = [r"$\Delta M_\star/M_\star^\text{SSAG}$", r"$\Delta\,\left<\log{t_\star}\right>_M$", r"$\Delta\,\left<\log{t_\star}\right>_L$", r"$\Delta\,\left<\log{Z_\star/Z\odot}\right>_M$", r"$\Delta\,A_V$"]
lb     = "M log_t_M log_t_L log_Z_M Av".split()

fig, axs = plt.subplots(len(output), 5, sharex = True, figsize = (7, 5))

for i, j in product(xrange(len(output)), xrange(5)) :

  mask = data_z[i].physical["log_t_M_mod"] < np.log10(ages[i] * 1e9)

  med = np.median(data_z[i].residuals[lb[j]][mask])
  p16 = st.scoreatpercentile(data_z[i].residuals[lb[j]][mask], 16)
  p84 = st.scoreatpercentile(data_z[i].residuals[lb[j]][mask], 84)

  axs[i, j].hist(data_z[i].residuals[lb[j]][mask], 40, histtype = "stepfilled", alpha = 0.5, ec = "#0062FF", fc = "#0062FF", range = (-1.5, +1.5), lw = 2)

# 需要导入模块: import DataLoader [as 别名]
# 或者: from DataLoader import load_data [as 别名]
def run_batch(data):
    # Output
    date = str(time.asctime(time.localtime(time.time())))

    raw_output = [["Points gaussian;"],  # 0
                  ["Points linear;"],  # 1
                  ["Dec. margin;"],  # 2
                  ["C linear;"],  # 3
                  ["C gauss;"],  # 4
                  ["gamma gauss;"],  # 5
                  ["number SVs gauss;"],  # 6
                  ["time to fit;"],  # 7
                  ["      gauss;"],  # 8
                  ["      linear;"],  # 9
                  ["      overhead;"],  # 10
                  ["time to predict;"],  # 11
                  ["Error;"]  # 12
                  ]

    # Load the data
    x, x_test, y, y_test = DataLoader.load_data(data)
    k = 0
    c_lin = [10000000000, 10000000000, 10000000000, 10000000000, 10000000000, 10000000000, 10000000000, 10000000000,
             10000000000]
    c_gauss = [0, 800, 1, 1, 10, 10, 10, 10, 10]
    gamma = [0, 0.01, 200, 200, 0.001, 0.001, 0.001, 0.001, 0.001]
    Tools.write("Starting batch run, " + data)
    gridLinear = False
    gridGauss = False
    use_distance = True
    n = 0

    for j in range(4):  # Smaller steps from 0 to 20: 0, 5, 10, 15
        n = j
        Tools.write("Batch run " + str(j) + ", k = " + str(0.05 * j))
        # Load the classifier
        k = 0.05 * j
        clf = ds.DualSvm(use_distance=use_distance)
        clf.k = k

        # Parameter Tuning
        if j == 0 and gridLinear:  # In the first run, calculate best parameters for linear svm
            c_lin[n] = gridsearch_for_linear(x, y)
        else:
            clf.c_lin = c_lin[n]
            clf.fit_lin_svc(x,
                            y)  # Fit linear classifier beforehand. This is necessary for the get_points method to work correctly.
            x_gauss, y_gauss, margins = clf.get_points_close_to_hyperplane_by_count(x, y, k)
            if gridGauss:
                c_gauss[n], gamma[n] = gridsearch_for_gauss(x_gauss,
                                                            y_gauss)  # In the following runs, do the same for the gaussian svm, as the subset of points for the classifier is changing

        # Apply Parameters
        clf.c_gauss = c_gauss[n]
        clf.gamma = gamma[n]
        clf.c_lin = c_lin[n]

        timeStart = time.time()
        clf.fit(x, y)
        timeFit = time.time() - timeStart

        appendMiscStatsDualSvm(clf, raw_output)
        appendTimeStatistics(raw_output, "dualSvm", clf, timeFit, x_test, y_test)

    for i in range(5):  # Bigger steps from 20 to 100: 20, 40, 60, 80, 100
        n = 4 + i
        Tools.write("Batch run " + str(i + 4) + ", k = " + str(0.2 * (i + 1)))

        # Load the classifier
        k = 0.2 * (i + 1)
        clf = ds.DualSvm(use_distance=use_distance)
        clf.k = k

        if i == 0:
            clf.c_lin = c_lin[n]
            clf.fit_lin_svc(x, y)
            x_gauss, y_gauss, margins = clf.get_points_close_to_hyperplane_by_count(x, y, k)
            if gridGauss and 1 <= i < 3:
                c_gauss[n], gamma[n] = gridsearch_for_gauss(x_gauss, y_gauss)

        # Apply Parameters
        clf.c_gauss = c_gauss[n]
        clf.gamma = gamma[n]
        clf.c_lin = c_lin[n]

        timeStart = time.time()
        clf.fit(x, y)
        timeFit = time.time() - timeStart

        appendMiscStatsDualSvm(clf, raw_output)
        appendTimeStatistics(raw_output, "dualSvm", clf, timeFit, x_test, y_test)

    Tools.write("Batch run complete.")

    header = data + " " + date
    header = header.replace(" ", "_")
    header = header.replace(":", "_")
    try:
        file = 'master/output/' + header + ".csv"
        output = open(file, 'a')
#.........这里部分代码省略.........