本文整理汇总了Java中org.apache.pig.ExecType.LOCAL属性的典型用法代码示例。如果您正苦于以下问题：Java ExecType.LOCAL属性的具体用法？Java ExecType.LOCAL怎么用？Java ExecType.LOCAL使用的例子？那么, 这里精选的属性代码示例或许可以为您提供帮助。您也可以进一步了解该属性所在类org.apache.pig.ExecType的用法示例。
在下文中一共展示了ExecType.LOCAL属性的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: setup
private PigServer setup(String script, Configuration conf) throws Exception {
if (conf == null) {
conf = new HdfsConfiguration();
}
conf.setIfUnset(VespaConfiguration.DRYRUN, "true");
conf.setIfUnset(VespaConfiguration.ENDPOINT, "dummy-endpoint");
// Parameter substitutions - can also be set by configuration
Map<String, String> parameters = new HashMap<>();
parameters.put("ENDPOINT", "endpoint-does-not-matter-in-dryrun,another-endpoint-that-does-not-matter");
PigServer ps = new PigServer(ExecType.LOCAL, conf);
ps.setBatchOn();
ps.registerScript(script, parameters);
return ps;
}
示例2: createPig
protected PigServer createPig() throws ExecException {
HdpBootstrap.hackHadoopStagingOnWin();
Properties properties = HdpBootstrap.asProperties(QueryTestParams.provisionQueries(HdpBootstrap.hadoopConfig()));
String pigHost = properties.getProperty("pig");
// remote Pig instance
if (StringUtils.hasText(pigHost) && !"local".equals(pig)) {
LogFactory.getLog(PigWrapper.class).info("Executing Pig in Map/Reduce mode");
return new PigServer(ExecType.MAPREDUCE, properties);
}
// use local instance
LogFactory.getLog(PigWrapper.class).info("Executing Pig in local mode");
properties.put("mapred.job.tracker", "local");
return new PigServer(ExecType.LOCAL, properties);
}
示例3: setup
@Before
public void setup() throws IOException {
pig = new PigServer(ExecType.LOCAL);
Util.deleteDirectory(new File(dataDir));
try {
pig.mkdirs(dataDir);
Util.createLocalInputFile(dataDir + scalarInput,
new String[] {
"{ \"i\": 1, \"l\": 10, \"f\": 2.718, \"d\": 3.1415, \"b\": \"17\", \"c\": \"aardvark\" }",
"{ \"i\": 2, \"l\": 100, \"f\": 1.234, \"d\": 3.3333, \"b\": null, \"c\": \"17.0\" }"
});
Util.createLocalInputFile(dataDir + complexInput,
new String[] {
"{ \"tuple\": { \"a\": 1, \"b\": 2 }, \"nested_tuple\": { \"a\": 1, \"b\": { \"c\": 2, \"d\": 3 } }, \"bag\": [{ \"a\": 1, \"b\": 2 }, { \"a\": 3, \"b\": 4 }], \"nested_bag\": [{\"a\": 1, \"b\": [{ \"c\": 2, \"d\": 3 }, { \"c\": 4, \"d\": 5 }]}], \"map\": { \"a\": 1, \"b\": 2 }, \"nested_map\": { \"a\": { \"b\": 1, \"c\": 2 } } }",
"{ \"tuple\": { \"a\": 3, \"b\": 4 }, \"nested_tuple\": { \"a\": 4, \"b\": { \"c\": 5, \"d\": 6 } }, \"bag\": [{ \"a\": 5, \"b\": 6 }, { \"a\": 7, \"b\": 8 }], \"nested_bag\": [{\"a\": 6, \"b\": [{ \"c\": 7, \"d\": 8 }, { \"c\": 9, \"d\": 0 }]}], \"map\": { \"a\": 3, \"b\": 4 }, \"nested_map\": { \"a\": { \"b\": 3, \"c\": 4 } } }"
});
Util.createLocalInputFile(dataDir + nestedArrayInput,
new String[] {
"{ \"arr\": [1, 2, 3, 4], \"nested_arr\": [[1, 2], [3, 4]], \"nested_arr_2\": [[1, 2], [3, 4]], \"very_nested_arr\": [[[1, 2], [3, 4]], [[5, 6], [7, 6]]], \"i\": 9 }"
});
} catch (IOException e) {};
}
示例4: testMockStoreAndLoad
@Test
public void testMockStoreAndLoad() throws Exception {
PigServer pigServer = new PigServer(ExecType.LOCAL);
Data data = resetData(pigServer);
data.set("foo",
tuple("a"),
tuple("b"),
tuple("c")
);
pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage();");
pigServer.registerQuery("STORE A INTO 'bar' USING mock.Storage();");
List<Tuple> out = data.get("bar");
assertEquals(tuple("a"), out.get(0));
assertEquals(tuple("b"), out.get(1));
assertEquals(tuple("c"), out.get(2));
}
示例5: testSecondarySort
@Test
public void testSecondarySort() throws Exception {
PigServer pigServer = new PigServer(ExecType.LOCAL);
Data data = resetData(pigServer);
data.set("foo",
tuple("a", 1, "b"),
tuple("b", 2, "c"),
tuple("c", 3, "d")
);
pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage() AS (f1:chararray,f2:int,f3:chararray);");
pigServer.registerQuery("B = order A by f1,f2,f3 DESC;");
pigServer.registerQuery("STORE B INTO 'bar' USING mock.Storage();");
List<Tuple> out = data.get("bar");
assertEquals(tuple("a", 1, "b"), out.get(0));
assertEquals(tuple("b", 2, "c"), out.get(1));
assertEquals(tuple("c", 3, "d"), out.get(2));
}
示例6: testMissingCols2
@Test
public void testMissingCols2() throws Exception {
String inputFileName = "TestProject-testMissingCols2-input.txt";
String input[] = { "1\t(hello,world)", "2\t(good,bye)" };
Util.createLocalInputFile(inputFileName, input);
// in the script, PigStorage will return a null for the tuple field
// since it does not comply with the schema
String query = "a = load '" + inputFileName + "' as (i:int, " +
"t:tuple(s1:chararray, s2:chararray, s3:chararray));" +
"b = foreach a generate t.(s2,s3);";
PigServer ps = new PigServer(ExecType.LOCAL);
Util.registerMultiLineQuery(ps, query);
Iterator<Tuple> it = ps.openIterator("b");
Tuple[] expectedResults = new Tuple[] {
(Tuple)Util.getPigConstant("((null, null))"),
(Tuple)Util.getPigConstant("((null, null))")
};
int i = 0;
while (it.hasNext()) {
assertEquals(expectedResults[i++], it.next());
}
}
示例7: testScriptMissingLastNewLine
@Test
public void testScriptMissingLastNewLine() throws Throwable {
PigServer server = new PigServer(ExecType.LOCAL);
PigContext context = server.getPigContext();
String strCmd = "A = load 'bar';\nB = foreach A generate $0;";
BufferedReader reader = new BufferedReader(new StringReader(strCmd));
String substituted = context.doParamSubstitution(reader, null, null);
BufferedReader pigInput = new BufferedReader(new StringReader(substituted));
Grunt grunt = new Grunt(pigInput, context);
int results[] = grunt.exec();
for (int i=0; i<results.length; i++) {
assertEquals(0, results[i]);
}
}
示例8: testIllustrateScript4
@Test
public void testIllustrateScript4() throws Throwable {
// empty line/field test
PigServer server = new PigServer(ExecType.LOCAL, new Properties());
PigContext context = server.getPigContext();
String strCmd = "illustrate -script "
+ basedir + "/illustrate4.pig;";
ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
InputStreamReader reader = new InputStreamReader(cmd);
Grunt grunt = new Grunt(new BufferedReader(reader), context);
grunt.exec();
}
示例9: testParseUserAgentPigUDF_allFields
@Test
public void testParseUserAgentPigUDF_allFields() throws Exception {
PigServer pigServer = new PigServer(ExecType.LOCAL);
Storage.Data storageData = resetData(pigServer);
storageData.set("agents", "agent:chararray", tuple(testUserAgent));
pigServer.registerQuery("define ParseUserAgent nl.basjes.parse.useragent.pig.ParseUserAgent();");
pigServer.registerQuery("A = LOAD 'agents' USING mock.Storage();");
pigServer.registerQuery("B = FOREACH A GENERATE ParseUserAgent(agent);");
pigServer.registerQuery("STORE B INTO 'parsedAgents' USING mock.Storage();");
verifyStorageData(storageData);
}
示例10: testParseUserAgentPigUDF_Cache_allFields
@Test
public void testParseUserAgentPigUDF_Cache_allFields() throws Exception {
PigServer pigServer = new PigServer(ExecType.LOCAL);
Storage.Data storageData = resetData(pigServer);
storageData.set("agents", "agent:chararray", tuple(testUserAgent));
pigServer.registerQuery("define ParseUserAgent nl.basjes.parse.useragent.pig.ParseUserAgent('5');");
pigServer.registerQuery("A = LOAD 'agents' USING mock.Storage();");
pigServer.registerQuery("B = FOREACH A GENERATE ParseUserAgent(agent);");
pigServer.registerQuery("STORE B INTO 'parsedAgents' USING mock.Storage();");
verifyStorageData(storageData);
}
示例11: setup
private PigServer setup(String script, String endpoint) throws Exception {
Configuration conf = new HdfsConfiguration();
Map<String, String> parameters = new HashMap<>();
parameters.put("ENDPOINT", endpoint);
PigServer ps = new PigServer(ExecType.LOCAL, conf);
ps.setBatchOn();
ps.registerScript(script, parameters);
return ps;
}
示例12: setup
@Before
public void setup() throws IOException {
pig = new PigServer(ExecType.LOCAL);
Util.deleteDirectory(new File(dataDir));
try {
pig.mkdirs(dataDir);
Util.createLocalInputFile(dataDir + scalarInput,
new String[] {
"{ \"i\": 1, \"l\": 10, \"f\": 2.718, \"d\": 3.1415, \"b\": \"17\", \"c\": \"aardvark\" }",
"{ \"i\": 2, \"l\": 100, \"f\": 1.234, \"d\": 3.3333, \"b\": null, \"c\": \"17.0\" }"
});
Util.createLocalInputFile(dataDir + complexInput,
new String[] {
"{ \"tuple\": { \"a\": 1, \"b\": 2 }, \"nested_tuple\": { \"a\": 1, \"b\": { \"c\": 2, \"d\": 3 } }, \"bag\": [{ \"a\": 1, \"b\": 2 }, { \"a\": 3, \"b\": 4 }], \"nested_bag\": [{\"a\": 1, \"b\": [{ \"c\": 2, \"d\": 3 }, { \"c\": 4, \"d\": 5 }]}], \"map\": { \"a\": 1, \"b\": 2 }, \"nested_map\": { \"a\": { \"b\": 1, \"c\": 2 } } }",
"{ \"tuple\": { \"a\": 3, \"b\": 4 }, \"nested_tuple\": { \"a\": 4, \"b\": { \"c\": 5, \"d\": 6 } }, \"bag\": [{ \"a\": 5, \"b\": 6 }, { \"a\": 7, \"b\": 8 }], \"nested_bag\": [{\"a\": 6, \"b\": [{ \"c\": 7, \"d\": 8 }, { \"c\": 9, \"d\": 0 }]}], \"map\": { \"a\": 3, \"b\": 4 }, \"nested_map\": { \"a\": { \"b\": 3, \"c\": 4 } } }"
});
Util.createLocalInputFile(dataDir + nestedArrayInput,
new String[] {
"{ \"arr\": [1, 2, 3, 4], \"nested_arr\": [[1, 2], [3, 4]], \"nested_arr_2\": [[1, 2], [3, 4]], \"very_nested_arr\": [[[1, 2], [3, 4]], [[5, 6], [7, 6]]], \"i\": 9 }"
});
Util.createLocalInputFile(dataDir + unusualFieldNameInput,
new String[] {
"{\"f_1\": 1, \"__f2\": 2, \"f 3\": 3}",
"{\"f_1\": 4, \"__f2\": 5, \"f 3\": 6}"
});
} catch (IOException e) {};
}
示例13: setup
@Before
public void setup() throws IOException {
pig = new PigServer(ExecType.LOCAL);
Util.deleteDirectory(new File(dataDir));
try {
pig.mkdirs(dataDir);
Util.createLocalInputFile(dataDir + scalarInput,
new String[] {
"{ \"i\": 1, \"l\": 10, \"f\": 2.718, \"d\": 3.1415, \"b\": \"17\", \"c\": \"aardvark\", \"bl\": true}",
"{ \"i\": 2, \"l\": 100, \"f\": 1.234, \"d\": 3.3333, \"b\": null, \"c\": \"17.0\", \"bl\": false }"
});
Util.createLocalInputFile(dataDir + complexInput,
new String[] {
"{ \"tuple\": { \"a\": 1, \"b\": 2 }, \"nested_tuple\": { \"a\": 1, \"b\": { \"c\": 2, \"d\": 3 } }, \"bag\": [{ \"a\": 1, \"b\": 2 }, { \"a\": 3, \"b\": 4 }], \"nested_bag\": [{\"a\": 1, \"b\": [{ \"c\": 2, \"d\": 3 }, { \"c\": 4, \"d\": 5 }]}], \"map\": { \"a\": 1, \"b\": 2 }, \"nested_map\": { \"a\": { \"b\": 1, \"c\": 2 } } }",
"{ \"tuple\": { \"a\": 3, \"b\": 4 }, \"nested_tuple\": { \"a\": 4, \"b\": { \"c\": 5, \"d\": 6 } }, \"bag\": [{ \"a\": 5, \"b\": 6 }, { \"a\": 7, \"b\": 8 }], \"nested_bag\": [{\"a\": 6, \"b\": [{ \"c\": 7, \"d\": 8 }, { \"c\": 9, \"d\": 0 }]}], \"map\": { \"a\": 3, \"b\": 4 }, \"nested_map\": { \"a\": { \"b\": 3, \"c\": 4 } } }"
});
Util.createLocalInputFile(dataDir + nestedArrayInput,
new String[] {
"{ \"arr\": [1, 2, 3, 4], \"nested_arr\": [[1, 2], [3, 4]], \"nested_arr_2\": [[1, 2], [3, 4]], \"very_nested_arr\": [[[1, 2], [3, 4]], [[5, 6], [7, 6]]], \"i\": 9 }"
});
Util.createLocalInputFile(dataDir + unusualFieldNameInput,
new String[] {
"{\"f_1\": 1, \"__f2\": 2, \"f 3\": 3}",
"{\"f_1\": 4, \"__f2\": 5, \"f 3\": 6}"
});
Util.createLocalInputFile(dataDir + nullMapBagTuple,
new String[] {
"{ \"map\":null, \"bag\":null, \"tup\":null }"
});
} catch (IOException e) {};
}
示例14: createPigTest
public static PigTest createPigTest(String scriptFile, String[] args) throws PigException {
try {
PigServer pigServer = new PigServer(ExecType.LOCAL);
Cluster pigCluster = new Cluster(pigServer.getPigContext());
return new PigTest(scriptFile, args, pigServer, pigCluster);
} catch (Exception ex) {
throw new PigException("Failed to create PigTest instance. ", ex);
}
}
示例15: setUpBeforeClass
@BeforeClass
public static void setUpBeforeClass() throws Exception {
// Setup log4j for testing.
org.apache.log4j.BasicConfigurator.configure();
Properties properties = new Properties();
properties.setProperty("hive.metastore.uris", "thrift://localhost:9083");
server = new PigServer(ExecType.LOCAL, properties);
}