本文整理汇总了Java中org.apache.pig.test.Util.createLocalInputFile方法的典型用法代码示例。如果您正苦于以下问题：Java Util.createLocalInputFile方法的具体用法？Java Util.createLocalInputFile怎么用？Java Util.createLocalInputFile使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.pig.test.Util的用法示例。
在下文中一共展示了Util.createLocalInputFile方法的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: setup
import org.apache.pig.test.Util; //导入方法依赖的package包/类
@Before
public void setup() throws IOException {
pig = new PigServer(ExecType.LOCAL);
Util.deleteDirectory(new File(dataDir));
try {
pig.mkdirs(dataDir);
Util.createLocalInputFile(dataDir + scalarInput,
new String[] {
"{ \"i\": 1, \"l\": 10, \"f\": 2.718, \"d\": 3.1415, \"b\": \"17\", \"c\": \"aardvark\" }",
"{ \"i\": 2, \"l\": 100, \"f\": 1.234, \"d\": 3.3333, \"b\": null, \"c\": \"17.0\" }"
});
Util.createLocalInputFile(dataDir + complexInput,
new String[] {
"{ \"tuple\": { \"a\": 1, \"b\": 2 }, \"nested_tuple\": { \"a\": 1, \"b\": { \"c\": 2, \"d\": 3 } }, \"bag\": [{ \"a\": 1, \"b\": 2 }, { \"a\": 3, \"b\": 4 }], \"nested_bag\": [{\"a\": 1, \"b\": [{ \"c\": 2, \"d\": 3 }, { \"c\": 4, \"d\": 5 }]}], \"map\": { \"a\": 1, \"b\": 2 }, \"nested_map\": { \"a\": { \"b\": 1, \"c\": 2 } } }",
"{ \"tuple\": { \"a\": 3, \"b\": 4 }, \"nested_tuple\": { \"a\": 4, \"b\": { \"c\": 5, \"d\": 6 } }, \"bag\": [{ \"a\": 5, \"b\": 6 }, { \"a\": 7, \"b\": 8 }], \"nested_bag\": [{\"a\": 6, \"b\": [{ \"c\": 7, \"d\": 8 }, { \"c\": 9, \"d\": 0 }]}], \"map\": { \"a\": 3, \"b\": 4 }, \"nested_map\": { \"a\": { \"b\": 3, \"c\": 4 } } }"
});
Util.createLocalInputFile(dataDir + nestedArrayInput,
new String[] {
"{ \"arr\": [1, 2, 3, 4], \"nested_arr\": [[1, 2], [3, 4]], \"nested_arr_2\": [[1, 2], [3, 4]], \"very_nested_arr\": [[[1, 2], [3, 4]], [[5, 6], [7, 6]]], \"i\": 9 }"
});
} catch (IOException e) {};
}
示例2: testSchemaResetter
import org.apache.pig.test.Util; //导入方法依赖的package包/类
@Test
public void testSchemaResetter() throws IOException {
new File("build/test/tmp/").mkdirs();
Util.createLocalInputFile("build/test/tmp/TestSchemaResetter.pig", new String[] {
"A = LOAD 'foo' AS (group:tuple(uid, dst_id));",
"edges_both = FOREACH A GENERATE",
" group.uid AS src_id,",
" group.dst_id AS dst_id;",
"both_counts = GROUP edges_both BY src_id;",
"both_counts = FOREACH both_counts GENERATE",
" group AS src_id, SIZE(edges_both) AS size_both;",
"",
"edges_bq = FOREACH A GENERATE",
" group.uid AS src_id,",
" group.dst_id AS dst_id;",
"bq_counts = GROUP edges_bq BY src_id;",
"bq_counts = FOREACH bq_counts GENERATE",
" group AS src_id, SIZE(edges_bq) AS size_bq;",
"",
"per_user_set_sizes = JOIN bq_counts BY src_id LEFT OUTER, both_counts BY src_id;",
"store per_user_set_sizes into 'foo';"
});
assertEquals(0, PigRunner.run(new String[] {"-x", "local", "-c", "build/test/tmp/TestSchemaResetter.pig" } , null).getReturnCode());
}
示例3: setup
import org.apache.pig.test.Util; //导入方法依赖的package包/类
@Before
public void setup() throws IOException {
pig = new PigServer(ExecType.LOCAL);
Util.deleteDirectory(new File(dataDir));
try {
pig.mkdirs(dataDir);
Util.createLocalInputFile(dataDir + testFile,
new String[] {
" int long float double bit boolean datetime string string extra",
"12345 1234567890000 2.718 3.141593 0 true 2007-04-05T14:30:10Z avertwolowolo",
"12345 1234567890000 2.718 3.141593 1 false 2007-04-05T14:30:10Z avertwolowolo moose",
" 1234567890000 3.141593 true avert ",
" 1234567890000 3.141593 false",
" 1234567890000 cerulean true"
});
} catch (IOException e) {};
}
示例4: setup
import org.apache.pig.test.Util; //导入方法依赖的package包/类
@Before
public void setup() throws IOException {
pig = new PigServer(ExecType.LOCAL);
Util.deleteDirectory(new File(dataDir));
try {
pig.mkdirs(dataDir);
Util.createLocalInputFile(dataDir + scalarInput,
new String[] {
"{ \"i\": 1, \"l\": 10, \"f\": 2.718, \"d\": 3.1415, \"b\": \"17\", \"c\": \"aardvark\" }",
"{ \"i\": 2, \"l\": 100, \"f\": 1.234, \"d\": 3.3333, \"b\": null, \"c\": \"17.0\" }"
});
Util.createLocalInputFile(dataDir + complexInput,
new String[] {
"{ \"tuple\": { \"a\": 1, \"b\": 2 }, \"nested_tuple\": { \"a\": 1, \"b\": { \"c\": 2, \"d\": 3 } }, \"bag\": [{ \"a\": 1, \"b\": 2 }, { \"a\": 3, \"b\": 4 }], \"nested_bag\": [{\"a\": 1, \"b\": [{ \"c\": 2, \"d\": 3 }, { \"c\": 4, \"d\": 5 }]}], \"map\": { \"a\": 1, \"b\": 2 }, \"nested_map\": { \"a\": { \"b\": 1, \"c\": 2 } } }",
"{ \"tuple\": { \"a\": 3, \"b\": 4 }, \"nested_tuple\": { \"a\": 4, \"b\": { \"c\": 5, \"d\": 6 } }, \"bag\": [{ \"a\": 5, \"b\": 6 }, { \"a\": 7, \"b\": 8 }], \"nested_bag\": [{\"a\": 6, \"b\": [{ \"c\": 7, \"d\": 8 }, { \"c\": 9, \"d\": 0 }]}], \"map\": { \"a\": 3, \"b\": 4 }, \"nested_map\": { \"a\": { \"b\": 3, \"c\": 4 } } }"
});
Util.createLocalInputFile(dataDir + nestedArrayInput,
new String[] {
"{ \"arr\": [1, 2, 3, 4], \"nested_arr\": [[1, 2], [3, 4]], \"nested_arr_2\": [[1, 2], [3, 4]], \"very_nested_arr\": [[[1, 2], [3, 4]], [[5, 6], [7, 6]]], \"i\": 9 }"
});
Util.createLocalInputFile(dataDir + unusualFieldNameInput,
new String[] {
"{\"f_1\": 1, \"__f2\": 2, \"f 3\": 3}",
"{\"f_1\": 4, \"__f2\": 5, \"f 3\": 6}"
});
} catch (IOException e) {};
}
示例5: inexactSchema
import org.apache.pig.test.Util; //导入方法依赖的package包/类
@Test
public void inexactSchema() throws IOException, ParseException {
String input = "TestFromJsonWithSchema_inexact_schema";
String schema = "a: int, b: int, c: int, d: (i: int, j: int), e: int";
Util.createLocalInputFile(dataDir + input,
new String[] {
"{ \"a\": 1, \"b\": 2, \"c\": 3, \"d\": { \"i\": 4, \"j\": 5 }, \"e\": 6 }",
"{ \"a\": 1, \"c\": 3 }",
"{ \"a\": 1, \"b\": 2, \"d\": { \"i\": 4, \"j\": 5 }, \"e\": 6 }",
"{ \"a\": 1, \"extra\": \"extra\", \"b\": 2, \"d\": { \"i\": 4, \"j\": 5 } }",
"{ \"b\": 2, \"a\": 1, \"e\": 6, \"d\": { \"i\": 4, \"j\": 5}, \"c\": 3 }"
});
pig.registerQuery("DEFINE FromJson com.mortardata.pig.FromJsonWithSchema('" + schema + "');");
pig.registerQuery("data = load '" + dataDir + input + "' using TextLoader() as (text: chararray);");
pig.registerQuery("json = foreach data generate FLATTEN(FromJson(text));");
Iterator<Tuple> json = pig.openIterator("json");
String[] expected = {
"(1,2,3,(4,5),6)",
"(1,,3,,)",
"(1,2,,(4,5),6)",
"(1,2,,(4,5),)",
"(1,2,3,(4,5),6)"
};
Assert.assertEquals(StringUtils.join(expected, "\n"), StringUtils.join(json, "\n"));
}
示例6: malformedDocuments
import org.apache.pig.test.Util; //导入方法依赖的package包/类
@Test
public void malformedDocuments() throws IOException, ParseException {
String input = "TestFromJsonWithSchema_inexact_schema_malformed_documents";
String schema = "a: int, b: int";
Util.createLocalInputFile(dataDir + input,
new String[] {
"{ \"a\": 1, \"b\": 2 }",
"{ \"a\": 1, \"b\": \"2abc\" }",
"{ \"a\": 1, \"b\": abc2 }",
"{ \"a\": 1, \"b\": false }",
"{ \"a\": 1, \"b\": true }",
"{ \"a\": 1, b: }",
"{ \"a\": 1, ",
"\"b\": 2 }"
});
pig.registerQuery("DEFINE FromJson com.mortardata.pig.FromJsonWithSchema('" + schema + "');");
pig.registerQuery("data = load '" + dataDir + input + "' using TextLoader() as (text: chararray);");
pig.registerQuery("json = foreach data generate FLATTEN(FromJson(text));");
Iterator<Tuple> json = pig.openIterator("json");
String[] expected = {
"(1,2)",
"(1,)",
"(1,)",
"(1,)",
"(1,)",
"(1,)",
"(1,)",
"(,)"
};
Assert.assertEquals(StringUtils.join(expected, "\n"), StringUtils.join(json, "\n"));
}
示例7: setup
import org.apache.pig.test.Util; //导入方法依赖的package包/类
@Before
public void setup() throws IOException {
pig = new PigServer(ExecType.LOCAL);
Util.deleteDirectory(new File(dataDir));
try {
pig.mkdirs(dataDir);
Util.createLocalInputFile(dataDir + scalarInput,
new String[] {
"{ \"i\": 1, \"l\": 10, \"f\": 2.718, \"d\": 3.1415, \"b\": \"17\", \"c\": \"aardvark\", \"bl\": true}",
"{ \"i\": 2, \"l\": 100, \"f\": 1.234, \"d\": 3.3333, \"b\": null, \"c\": \"17.0\", \"bl\": false }"
});
Util.createLocalInputFile(dataDir + complexInput,
new String[] {
"{ \"tuple\": { \"a\": 1, \"b\": 2 }, \"nested_tuple\": { \"a\": 1, \"b\": { \"c\": 2, \"d\": 3 } }, \"bag\": [{ \"a\": 1, \"b\": 2 }, { \"a\": 3, \"b\": 4 }], \"nested_bag\": [{\"a\": 1, \"b\": [{ \"c\": 2, \"d\": 3 }, { \"c\": 4, \"d\": 5 }]}], \"map\": { \"a\": 1, \"b\": 2 }, \"nested_map\": { \"a\": { \"b\": 1, \"c\": 2 } } }",
"{ \"tuple\": { \"a\": 3, \"b\": 4 }, \"nested_tuple\": { \"a\": 4, \"b\": { \"c\": 5, \"d\": 6 } }, \"bag\": [{ \"a\": 5, \"b\": 6 }, { \"a\": 7, \"b\": 8 }], \"nested_bag\": [{\"a\": 6, \"b\": [{ \"c\": 7, \"d\": 8 }, { \"c\": 9, \"d\": 0 }]}], \"map\": { \"a\": 3, \"b\": 4 }, \"nested_map\": { \"a\": { \"b\": 3, \"c\": 4 } } }"
});
Util.createLocalInputFile(dataDir + nestedArrayInput,
new String[] {
"{ \"arr\": [1, 2, 3, 4], \"nested_arr\": [[1, 2], [3, 4]], \"nested_arr_2\": [[1, 2], [3, 4]], \"very_nested_arr\": [[[1, 2], [3, 4]], [[5, 6], [7, 6]]], \"i\": 9 }"
});
Util.createLocalInputFile(dataDir + unusualFieldNameInput,
new String[] {
"{\"f_1\": 1, \"__f2\": 2, \"f 3\": 3}",
"{\"f_1\": 4, \"__f2\": 5, \"f 3\": 6}"
});
Util.createLocalInputFile(dataDir + nullMapBagTuple,
new String[] {
"{ \"map\":null, \"bag\":null, \"tup\":null }"
});
} catch (IOException e) {};
}
示例8: inexactSchema
import org.apache.pig.test.Util; //导入方法依赖的package包/类
@Test
public void inexactSchema() throws IOException, ParseException {
String input = "TestJsonLoader_inexact_schema";
String schema = "a: int, b: int, c: int, d: (i: int, j: int), e: int";
Util.createLocalInputFile(dataDir + input,
new String[] {
"{ \"a\": 1, \"b\": 2, \"c\": 3, \"d\": { \"i\": 4, \"j\": 5 }, \"e\": 6 }",
"{ \"a\": 1, \"c\": 3 }",
"{ \"a\": 1, \"b\": 2, \"d\": { \"i\": 4, \"j\": 5 }, \"e\": 6 }",
"{ \"a\": 1, \"extra\": \"extra\", \"b\": 2, \"d\": { \"i\": 4, \"j\": 5 } }",
"{ \"b\": 2, \"a\": 1, \"e\": 6, \"d\": { \"i\": 4, \"j\": 5}, \"c\": 3 }"
});
pig.registerQuery(
"data = load '" + dataDir + input + "' " +
"using com.mortardata.pig.JsonLoader('" + schema + "');"
);
Iterator<Tuple> data = pig.openIterator("data");
String[] expected = {
"(1,2,3,(4,5),6)",
"(1,,3,,)",
"(1,2,,(4,5),6)",
"(1,2,,(4,5),)",
"(1,2,3,(4,5),6)"
};
Assert.assertEquals(StringUtils.join(expected, "\n"), StringUtils.join(data, "\n"));
}
示例9: malformedDocuments
import org.apache.pig.test.Util; //导入方法依赖的package包/类
@Test
public void malformedDocuments() throws IOException, ParseException {
String input = "TestJsonLoader_malformed_documents";
String schema = "a: int, b: int";
Util.createLocalInputFile(dataDir + input,
new String[] {
"{ \"a\": 1, \"b\": 2 }",
"{ \"a\": 1, \"b\": \"2abc\" }",
"{ \"a\": 1, \"b\": abc2 }",
"{ \"a\": 1, \"b\": false }",
"{ \"a\": 1, \"b\": true }",
"{ \"a\": 1, b: }",
"{ \"a\": 1, ",
"\"b\": 2 }"
});
pig.registerQuery(
"data = load '" + dataDir + input + "' " +
"using com.mortardata.pig.JsonLoader('" + schema + "');"
);
Iterator<Tuple> data = pig.openIterator("data");
String[] expected = {
"(1,2)",
"(1,)",
"(1,)",
"(1,)",
"(1,)",
"(1,)",
"(1,)",
"(,)"
};
Assert.assertEquals(StringUtils.join(expected, "\n"), StringUtils.join(data, "\n"));
}
示例10: test
import org.apache.pig.test.Util; //导入方法依赖的package包/类
@Test
public void test() throws Exception {
File input = File.createTempFile("test", "input");
input.deleteOnExit();
File output = File.createTempFile("test", "output");
output.delete();
Util.createLocalInputFile(input.getAbsolutePath(), new String[] {
"1,2,3",
"1,1,3",
"1,1,1",
"3,1,1",
"1,2,1",
});
PigServer pigServer = new PigServer(ExecType.LOCAL);
pigServer.setBatchOn();
pigServer.registerQuery(
"A = LOAD '" + input.getAbsolutePath() + "' using PigStorage();\n"
+ "B = GROUP A BY $0;\n"
+ "A = FOREACH B GENERATE COUNT(A);\n"
+ "STORE A INTO '" + output.getAbsolutePath() + "';");
ExecJob job = pigServer.executeBatch().get(0);
List<OriginalLocation> originalLocations = job.getPOStore().getOriginalLocations();
Assert.assertEquals(1, originalLocations.size());
OriginalLocation originalLocation = originalLocations.get(0);
Assert.assertEquals(4, originalLocation.getLine());
Assert.assertEquals(0, originalLocation.getOffset());
Assert.assertEquals("A", originalLocation.getAlias());
JobStats jStats = (JobStats)job.getStatistics().getJobGraph().getSinks().get(0);
Assert.assertEquals("M: A[1,4],A[3,4],B[2,4] C: A[3,4],B[2,4] R: A[3,4]", jStats.getAliasLocation());
}
示例11: createInputFile
import org.apache.pig.test.Util; //导入方法依赖的package包/类
private static void createInputFile(PigServer pigServer, String[] inputs, int id, File outputDir) throws IOException {
File input = File.createTempFile("tmp", "");
input.delete();
Util.createLocalInputFile(input.getAbsolutePath(), inputs);
pigServer.registerQuery("A = load '" + Util.encodeEscape(input.getAbsolutePath()) + "' as (a0:int, a1:int);");
File output = new File(outputDir, "/" + id);
pigServer.store("A", output.getAbsolutePath(), "org.apache.pig.piggybank.storage.IndexedStorage('\t','0,1')");
}
示例12: testSimpleCsv
import org.apache.pig.test.Util; //导入方法依赖的package包/类
@Test
public void testSimpleCsv() throws IOException {
String inputFileName = "TestCSVLoader-simple.txt";
Util.createLocalInputFile(inputFileName, new String[] {"foo,bar,baz", "fee,foe,fum"});
String script = "a = load '" + inputFileName + "' using org.apache.pig.piggybank.storage.CSVLoader() " +
" as (a:chararray, b:chararray, c:chararray); ";
Util.registerMultiLineQuery(pigServer, script);
Iterator<Tuple> it = pigServer.openIterator("a");
assertEquals(Util.createTuple(new String[] {"foo", "bar", "baz"}), it.next());
}
示例13: testQuotedQuotes
import org.apache.pig.test.Util; //导入方法依赖的package包/类
@Test
public void testQuotedQuotes() throws IOException {
String inputFileName = "TestCSVLoader-quotedquotes.txt";
Util.createLocalInputFile(inputFileName,
new String[] {"\"foo,\"\"bar\"\",baz\"", "\"\"\"\"\"\"\"\""});
String script = "a = load '" + inputFileName + "' using org.apache.pig.piggybank.storage.CSVLoader() " +
" as (a:chararray); ";
Util.registerMultiLineQuery(pigServer, script);
Iterator<Tuple> it = pigServer.openIterator("a");
assertEquals(Util.createTuple(new String[] {"foo,\"bar\",baz"}), it.next());
assertEquals(Util.createTuple(new String[] {"\"\"\""}), it.next());
}
示例14: storeScalarTypes
import org.apache.pig.test.Util; //导入方法依赖的package包/类
@Test
public void storeScalarTypes() throws IOException, ParseException {
String input = "pig_storage_scalar_data";
String schema = "i: int, l: long, f: float, d: double, " +
"b: boolean, dt: datetime, c_arr: chararray, b_arr: bytearray";
String output = "fixed_width_storage_scalar_data";
Util.createLocalInputFile(dataDir + input,
new String[] {
"1|10|2.718|3.14159|true|2007-04-05T14:30:10Z|aardvark|wooooolololo",
"1|100|1.234||false||cerulean skies|"
});
// Load input using PigStorage and store it using FixedWidthStorer
pig.registerQuery(
"data = load '" + dataDir + input + "' " +
"using PigStorage('|') as (" + schema + ");"
);
pig.store("data", dataDir + output,
"org.apache.pig.piggybank.storage.FixedWidthStorer('-5, 8-12, 15-19, 22-27, 29-33, 35-58, 62-69, 70-81', 'WRITE_HEADER')");
// Load the output and see if it is what it ought to be
pig.registerQuery(
"data = load '" + dataDir + output + "' " +
"using TextLoader() as (line: chararray);"
);
Iterator<Tuple> data = pig.openIterator("data");
String[] expected = {
// All columns right-aligned
"( i l f d b dt c_arr b_arr)", // Header written
"( 1 10 2.718 3.1416 true 2007-04-05T14:30:10.000Z aardvarkwooooolololo)", // 3.14159 rounded to fit in column
"( 1 100 1.234 false )" // "cerulean skies" does not fit, so a null is written (spaces)
};
Assert.assertEquals(StringUtils.join(expected, "\n"), StringUtils.join(data, "\n"));
}
示例15: setup
import org.apache.pig.test.Util; //导入方法依赖的package包/类
@Before
public void setup() throws IOException {
pig = new PigServer(ExecType.LOCAL);
pig.getPigContext().getProperties()
.setProperty("mapred.map.max.attempts", "1");
pig.getPigContext().getProperties()
.setProperty("mapred.reduce.max.attempts", "1");
pig.getPigContext().getProperties()
.setProperty("mapreduce.job.end-notification.retry.interval", "100");
Util.deleteDirectory(new File(dataDir));
pig.mkdirs(dataDir);
Util.createLocalInputFile(dataDir + testFile,
new String[] {
"int_field,long_field,float_field,double_field,chararray_field,bytearray_field",
"1,10,2.718,3.14159,qwerty,uiop",
"1,10,2.718,3.14159,,",
"1,10,,3.15159,,uiop",
"1,10,,3.15159,,uiop, moose",
"1,,\"2.718\",,\"qwerty\",\"uiop\"",
"1,,,,\"",
"qwe",
"rty\", uiop",
"1,,,,\"qwe,rty\",uiop",
"1,,,,\"q\"\"wert\"\"y\", uiop",
"1,,,,qwerty,\"u\"\"io\"\"p\""
});
Util.createLocalInputFile(testFileCommaName, testStrCommaArray);
Util.createLocalInputFile(testFileTabName, testStrTabArray);
}