Java PigServer.shutdown方法代码示例

import org.apache.pig.PigServer; //导入方法依赖的package包/类
@Test
public void testDump() throws Exception {
    PigServer pigServer = new PigServer(MODE);
    Data data = Storage.resetData(pigServer);
    data.set("input",
            tuple("test1"),
            tuple("test2"));

    pigServer.setBatchOn();
    pigServer.registerQuery("A = LOAD 'input' using mock.Storage;");
    Iterator<Tuple> result = pigServer.openIterator("A");

    List<Tuple> resultList = new ArrayList<Tuple>();
    while (result.hasNext()) {
        resultList.add(result.next());
    }

    assertEquals(Arrays.asList(tuple("test1"), tuple("test2")), resultList);
    pigServer.shutdown();
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
@Test()
public void testRegisteredJarVisibility() throws IOException {
    cluster.getFileSystem().copyFromLocalFile(
            new Path("file://" + INPUT_FILE.getAbsolutePath()), new Path(INPUT_FILE.getName()));
    PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());

    String query = "register " + jarFile.getAbsolutePath() + ";\n"
            + "a = load '" + INPUT_FILE.getName()
            + "' using org.apache.pig.test.RegisteredJarVisibilityLoader();\n"
            // register again to test classloader consistency
            + "register " +  jarFile.getAbsolutePath() + ";\n"
            + "b = load 'non_existent' "
            + "using org.apache.pig.test.RegisteredJarVisibilityLoader();";

    LOG.info("Running pig script:\n" + query);
    pigServer.registerScript(new ByteArrayInputStream(query.getBytes()));

    pigServer.openIterator("a");
    pigServer.shutdown();
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
@Test
public void testCountWithNoData() throws Exception {
    PigServer pigServer = new PigServer(ExecType.SPARK);
    Data data = Storage.resetData(pigServer);
    data.set("input");

    pigServer.registerQuery("A = LOAD 'input' using mock.Storage;");
    pigServer.registerQuery("B = GROUP A BY $0;");
    pigServer.registerQuery("C = FOREACH B GENERATE COUNT(A);");
    pigServer.registerQuery("STORE C INTO 'output' using mock.Storage;");

    assertEquals(
            Arrays.asList(),
            data.get("output"));

    pigServer.shutdown();
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
@Test
public void testInlineMacro() throws Throwable {
    PigServer pig = new PigServer(ExecType.LOCAL);

    Storage.Data data = resetData(pig);
    data.set("some_path", "(l:chararray)", tuple("first row"), tuple("second row"));

    pig.registerQuery("DEFINE row_count(X) RETURNS Z { Y = group $X all; $Z = foreach Y generate COUNT($X); };");
    pig.registerQuery("a = load 'some_path' USING mock.Storage();");
    pig.registerQuery("b = row_count(a);");
    Iterator<Tuple> iter = pig.openIterator("b");

    assertEquals(2L, ((Long)iter.next().get(0)).longValue());

    pig.shutdown();
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
@Test
public void testForEachFlatten() throws Exception {
    PigServer pigServer = newPigServer();
    Data data = Storage.resetData(pigServer);
    data.set("input",
            tuple(bag(tuple("1"), tuple("2"), tuple("3"))),
            tuple(bag(tuple("4"), tuple("5"), tuple("6"))));

    pigServer.registerQuery("A = LOAD 'input' using mock.Storage;");
    pigServer.registerQuery("B = FOREACH A GENERATE FLATTEN($0);");
    pigServer.registerQuery("STORE B INTO 'output' using mock.Storage;");

    assertEquals(
            Arrays.asList(tuple("1"), tuple("2"), tuple("3"), tuple("4"), tuple("5"), tuple("6")),
            data.get("output"));

    pigServer.shutdown();
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
@Test
public void testSimpleUDF() throws Exception {
    PigServer pigServer = new PigServer(ExecType.SPARK);
    Data data = Storage.resetData(pigServer);
    data.set("input",
            tuple("Foo"),
            tuple("BAR"),
            tuple("baT"));

    pigServer.registerQuery("A = LOAD 'input' using mock.Storage;");
    pigServer.registerQuery("B = FOREACH A GENERATE org.apache.pig.spark.LowercaseUDF($0);");
    pigServer.registerQuery("STORE B INTO 'output' using mock.Storage;");

    assertEquals(
            Arrays.asList(tuple("foo"), tuple("bar"), tuple("bat")),
            data.get("output"));

    pigServer.shutdown();
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
public static void initializeV(String prefix, String dataPath,
		char seperator, int vColIndex, String initialSchema, int L, int N,
		String vOutputPath) throws Exception {
	PigServer pigServer = new PigServer(ExecType.MAPREDUCE);
	String[] cols = initialSchema.split(",");
	String vColName = cols[vColIndex].split(":")[0];
	// pigServer.setDefaultParallel(k);
	pigServer.registerQuery("data = LOAD '" + prefix + dataPath
			+ "' USING PigStorage('" + seperator + "') AS ("
			+ initialSchema + ");");
	pigServer.registerQuery("V = group data by " + vColName + ";");

	StringBuilder randomClusterQuery = new StringBuilder(
			"V = foreach V generate group as iid");

	String randomValueGenCol = String.format(
			",(INT)(RANDOM() * %d) as itemSolId", (int) Math.pow(L, N));
	randomClusterQuery.append(randomValueGenCol);
	randomClusterQuery.append(";");
	pigServer.registerQuery(randomClusterQuery.toString());

	pigServer.store("V", prefix + vOutputPath);

	pigServer.shutdown();
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
@Test
public void testLimit() throws Exception {
    PigServer pigServer = newPigServer();
    Data data = Storage.resetData(pigServer);
    data.set("input",
            tuple("1"),
            tuple("2"),
            tuple("3"),
            tuple("4"));

    pigServer.registerQuery("A = LOAD 'input' using mock.Storage;");
    pigServer.registerQuery("B = LIMIT A 2;");
    pigServer.registerQuery("STORE B INTO 'output' using mock.Storage;");

    assertEquals(
            Arrays.asList(tuple("1"), tuple("2")),
            data.get("output"));

    pigServer.shutdown();
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
public static String generateFullJoinForPreprocessForAlphasLearning(int t,
		String dataPath, String dataInitialSchema, char seperator,
		String prefix, boolean isForRMSE) throws IOException {
	PigServer pigServer = new PigServer(ExecType.MAPREDUCE);
	pigServer.registerQuery("data = LOAD '" + prefix + dataPath
			+ "' USING PigStorage('" + seperator + "') AS ("
			+ dataInitialSchema + ");");
	pigServer.registerQuery("V = LOAD '" + prefix + "V_" + (t)
			+ "' USING PigStorage('\t') AS (iid:int, iSolNumber:int)  ;");
	pigServer.registerQuery("U = LOAD '" + prefix + "U_" + (t)
			+ "' USING PigStorage('\t') AS (uid:int, uSolNumber:int)  ;");
	pigServer.registerQuery("dataJoinV = JOIN data BY iid, V BY iid;");
	pigServer
	.registerQuery("dataJoinVJoinU = JOIN dataJoinV BY uid, U BY uid;");
	pigServer
	.registerQuery("fullJoin = foreach dataJoinVJoinU GENERATE data::uid AS uid, data::iid AS iid, U::uSolNumber AS uSolNumber , V::iSolNumber AS iSolNumber, data::rating as rating;");
	String outputPath = null;
	if (!isForRMSE){
		outputPath = "AlphasLearningFullJoin_" + t;
	}else {
		outputPath = "RMSEFullJoin_" + t;
	}
	pigServer.store("fullJoin", prefix + outputPath);
	pigServer.shutdown();
	return prefix + outputPath;
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
/**
 * 运行脚本
 *
 * @param scriptPath
 * @param storeId
 * @param storeFile
 */
public void script(InputStream scriptPath, String storeId, String storeFile) {
    PigServer pigServer = null;
    try {
        pigServer = new MumuPigConfiguration().mapreduce();
        pigServer.debugOn();
        pigServer.registerScript(scriptPath);
        if (storeId != null) {
            pigServer.store(storeId, storeFile);
        }
    } catch (IOException e) {
        e.printStackTrace();
        pigServer.shutdown();
    }
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
public void jar() {
    PigServer pigServer = null;
    try {
        pigServer = new MumuPigConfiguration().mapreduce();
        pigServer.registerJar("mumu-pig.jar");
        pigServer.registerQuery("nginxlog = load '/mapreduce/nginxlog/access/input' using com.lovecws.mumu.pig.loader.NginxLogLoader();");
        pigServer.dumpSchema("nginxlog");
    } catch (IOException e) {
        e.printStackTrace();
        pigServer.shutdown();
    }
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
@Override
public void execute(Query query) {
    String queryName = query.name;
    String queryValue = query.value;
    Map<String, String> queryMetadata = query.getMetadata();
    String execType = Query.getKey(queryMetadata, METADATA_EXEC_TYPE_KEY).orElse(defaultExecType);
    String alias = Query.getKey(queryMetadata, METADATA_ALIAS_KEY).orElse(defaultOutputAlias);
    log.info("Running {} for alias {}: {}", queryName, alias, queryValue);
    try {
        PigServer server = getPigServer(execType);
        server.registerScript(new ByteArrayInputStream(queryValue.getBytes()));
        Iterator<Tuple> queryResults = server.openIterator(alias);
        Result result = query.createResults();
        // dumpSchema will also, unfortunately, print the schema to stdout.
        List<FieldDetail> metadata = getFieldDetails(server.dumpSchema(alias));
        populateColumns(metadata, result);
        while (queryResults.hasNext()) {
            populateRow(queryResults.next(), metadata, result);
        }
        server.shutdown();
    } catch (IOException ioe) {
        log.error("Problem with Pig query: {}\n{}", queryValue, ioe);
        query.setFailure(ioe.toString());
    } catch (Exception e) {
        log.error("Error occurred while processing Pig query: {}\n{}", queryValue, e);
        query.setFailure(e.toString());
    }
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
public void saveToHadoop() throws IOException {
	PigServer pigServer = new PigServer(ExecType.MAPREDUCE);
	String pigQuery = "REGISTER 'WebContent/WEB-INF/lib/mongo-hadoop-core-1.3.2.jar';"
			+ "REGISTER 'WebContent/WEB-INF/lib/mongo-hadoop-pig-1.3.0.jar';"
			+ " A = LOAD 'mongodb://localhost:27017/chatroom.messageBackup'"
			+ " USING com.mongodb.hadoop.pig.MongoLoader('address, message, time')"
			+ " AS (address:chararray, message:chararray, time:datetime);";
	pigServer.registerQuery(pigQuery);
	pigServer.store("A", "/user/luffy/chatroom/" + this.time.toString().replaceAll(" ", "_").replaceAll(":", "-"));
	pigServer.shutdown();
	deleteBackupCollection();
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
@Test
public void testSuccessiveUserFuncs2() throws Exception {
    String dummyUDF = JiraPig1030.class.getName();
    String query = "a = load 'students.txt' as (c1,c2,c3,c4); " +
            "c = group a by c2; " +
            "f = foreach c generate COUNT(" + dummyUDF + "" +
            "(org.apache.pig.builtin.Distinct($1.$2)," + dummyUDF + "())); " +
            "store f into 'out';";
    PigServer pigServer = new PigServer(cluster.getExecType(), properties);
    PigContext pc = pigServer.getPigContext();
    assertTrue((Util.buildMRPlan(Util.buildPp(pigServer, query), pc).getRoots().get(0).combinePlan
            .isEmpty()));
    pigServer.shutdown();
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
@Test
public void testJoin() throws Exception {
    PigServer pigServer = newPigServer();
    Data data = Storage.resetData(pigServer);
    data.set("input1",
            tuple(1, "a"),
            tuple(2, "b"),
            tuple(3, "c"),
            tuple(1, "d"));
    data.set("input2",
            tuple(1, "e"),
            tuple(2, "f"),
            tuple(1, "g"));

    pigServer.registerQuery("A = LOAD 'input1' using mock.Storage;");
    pigServer.registerQuery("B = LOAD 'input2' using mock.Storage;");
    pigServer.registerQuery("C = JOIN A BY $0, B BY $0;");
    pigServer.registerQuery("STORE C INTO 'output' using mock.Storage;");
    
    List<Tuple> sortedOutput = data.get("output");
    Collections.sort(sortedOutput);

    assertEquals(
            Arrays.asList(
                    tuple(1, "a", 1, "e"),
                    tuple(1, "a", 1, "g"),
                    tuple(1, "d", 1, "e"),
                    tuple(1, "d", 1, "g"),
                    tuple(2, "b", 2, "f")
                    ),
                    sortedOutput);

    pigServer.shutdown();
}