本文整理汇总了Java中org.apache.pig.test.Util.copyFromLocalToCluster方法的典型用法代码示例。如果您正苦于以下问题:Java Util.copyFromLocalToCluster方法的具体用法?Java Util.copyFromLocalToCluster怎么用?Java Util.copyFromLocalToCluster使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.pig.test.Util
的用法示例。
在下文中一共展示了Util.copyFromLocalToCluster方法的6个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: createFile
import org.apache.pig.test.Util; //导入方法依赖的package包/类
private void createFile() throws IOException {
PrintWriter w = new PrintWriter(new FileWriter(INPUT_FILE));
w.println("100\tapple\taaa1");
w.println("200\torange\tbbb1");
w.println("300\tstrawberry\tccc1");
w.println("101\tapple\taaa2");
w.println("201\torange\tbbb2");
w.println("301\tstrawberry\tccc2");
w.println("102\tapple\taaa3");
w.println("202\torange\tbbb3");
w.println("302\tstrawberry\tccc3");
w.close();
Util.deleteFile(cluster, INPUT_FILE);
Util.copyFromLocalToCluster(cluster, INPUT_FILE, INPUT_FILE);
}
示例2: createFile
import org.apache.pig.test.Util; //导入方法依赖的package包/类
private void createFile() throws IOException {
PrintWriter w = new PrintWriter(new FileWriter(INPUT_FILE));
w = new PrintWriter(new FileWriter(INPUT_FILE));
w.println("100\tapple\t1.0\t2008-01-01");
w.println("100\torange\t2.0\t2008-02-01");
w.println("100\tbanana\t1.1\t2008-03-01");
w.close();
Util.copyFromLocalToCluster(cluster, INPUT_FILE, INPUT_FILE);
}
示例3: oneTimeSetup
import org.apache.pig.test.Util; //导入方法依赖的package包/类
@BeforeClass
public static void oneTimeSetup() throws Exception{
cluster = MiniGenericCluster.buildCluster();
Util.copyFromLocalToCluster(cluster, basedir + "orc-file-11-format.orc", basedir + "orc-file-11-format.orc");
Util.copyFromLocalToCluster(cluster, basedir + "charvarchar.orc", basedir + "charvarchar.orc");
createInputData();
if(Util.WINDOWS){
OUTPUT1 = OUTPUT1.replace("\\", "/");
OUTPUT2 = OUTPUT2.replace("\\", "/");
OUTPUT3 = OUTPUT3.replace("\\", "/");
OUTPUT4 = OUTPUT4.replace("\\", "/");
}
supportedOpTypes = new ArrayList<OpType>();
supportedOpTypes.add(OpType.OP_EQ);
supportedOpTypes.add(OpType.OP_NE);
supportedOpTypes.add(OpType.OP_GT);
supportedOpTypes.add(OpType.OP_GE);
supportedOpTypes.add(OpType.OP_LT);
supportedOpTypes.add(OpType.OP_LE);
supportedOpTypes.add(OpType.OP_IN);
supportedOpTypes.add(OpType.OP_BETWEEN);
supportedOpTypes.add(OpType.OP_NULL);
supportedOpTypes.add(OpType.OP_NOT);
supportedOpTypes.add(OpType.OP_AND);
supportedOpTypes.add(OpType.OP_OR);
Logger logger = Logger.getLogger(ColumnPruneVisitor.class);
logger.removeAllAppenders();
logger.setLevel(Level.INFO);
SimpleLayout layout = new SimpleLayout();
logFile = File.createTempFile("log", "");
FileAppender appender = new FileAppender(layout, logFile.toString(), false, false, 0);
logger.addAppender(appender);
}
示例4: testPythonUDF_onCluster
import org.apache.pig.test.Util; //导入方法依赖的package包/类
@Test
public void testPythonUDF_onCluster() throws Exception {
pigServerMapReduce = new PigServer(cluster.getExecType(), cluster.getProperties());
String[] pythonScript = {
"from pig_util import outputSchema",
"@outputSchema(\'c:chararray\')",
"def py_func(one,two):",
" return one + two"
};
Util.createLocalInputFile("pyfile_mr.py", pythonScript);
String[] input = {
"field10\tfield11",
"field20\tfield21"
};
Util.createLocalInputFile("input_mr", input);
Util.copyFromLocalToCluster(cluster, "input_mr", "input_mr");
pigServerMapReduce.registerQuery("REGISTER 'pyfile_mr.py' USING streaming_python AS pf;");
pigServerMapReduce.registerQuery("A = LOAD 'input_mr' as (c1:chararray, c2:chararray);");
pigServerMapReduce.registerQuery("B = FOREACH A generate pf.py_func(c1, c2);");
Iterator<Tuple> iter = pigServerMapReduce.openIterator("B");
assertTrue(iter.hasNext());
Tuple actual0 = iter.next();
assertTrue(iter.hasNext());
Tuple actual1 = iter.next();
Tuple expected0 = tf.newTuple("field10field11");
Tuple expected1 = tf.newTuple("field20field21");
assertEquals(expected0, actual0);
assertEquals(expected1, actual1);
}
示例5: createFile
import org.apache.pig.test.Util; //导入方法依赖的package包/类
private void createFile() throws IOException {
PrintWriter w = new PrintWriter(new FileWriter(INPUT_FILE));
w = new PrintWriter(new FileWriter(INPUT_FILE));
w.println("100\tapple\t1.0\t2008-01-01");
w.println("100\torange\t2.0\t2008-02-01");
w.println("100\tbanana\t1.1\t2008-03-01");
w.println("\t\t\t");
w.close();
Util.copyFromLocalToCluster(cluster, INPUT_FILE, INPUT_FILE);
}
示例6: createInputData
import org.apache.pig.test.Util; //导入方法依赖的package包/类
private static void createInputData() throws Exception {
pigServer = new PigServer(ExecType.LOCAL);
new File(inpbasedir).mkdirs();
new File(outbasedir).mkdirs();
String inputTxtFile = inpbasedir + File.separator + "input.txt";
BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(inputTxtFile), "UTF-8"));
long[] lVal = new long[] {100L, 200L, 300L};
float[] fVal = new float[] {50.0f, 100.0f, 200.0f, 300.0f};
double[] dVal = new double[] {1000.11, 2000.22, 3000.33};
StringBuilder sb = new StringBuilder();
for (int i=1; i <= 10000; i++) {
sb.append((i > 6500 && i <= 9000) ? true : false).append("\t"); //boolean
sb.append((i > 1000 && i < 3000) ? 1 : 5).append("\t"); //byte
sb.append((i > 2500 && i <= 4500) ? 100 : 200).append("\t"); //short
sb.append(i).append("\t"); //int
sb.append(lVal[i%3]).append("\t"); //long
sb.append(fVal[i%4]).append("\t"); //float
sb.append((i > 2500 && i < 3500) ? dVal[i%3] : dVal[i%1]).append("\t"); //double
sb.append((i%2 == 1 ? "" : RandomStringUtils.random(100).replaceAll("\t", " ")
.replaceAll("\n", " ").replaceAll("\r", " "))).append("\t"); //bytearray
sb.append((i%2 == 0 ? "" : RandomStringUtils.random(100).replaceAll("\t", " ")
.replaceAll("\n", " ").replaceAll("\r", " "))).append("\t"); //string
int year;
if (i > 5000 && i <= 8000) { //datetime
year = RandomUtils.nextInt(4)+2010;
} else {
year = RandomUtils.nextInt(10)+2000;
}
sb.append(new DateTime(year, RandomUtils.nextInt(12)+1,
RandomUtils.nextInt(28)+1, RandomUtils.nextInt(24), RandomUtils.nextInt(60),
DateTimeZone.UTC).toString()).append("\t"); // datetime
String bigString;
if (i>7500) {
bigString = RandomStringUtils.randomNumeric(9) + "." + RandomStringUtils.randomNumeric(5);
} else {
bigString = "1" + RandomStringUtils.randomNumeric(9) + "." + RandomStringUtils.randomNumeric(5);
}
sb.append(new BigDecimal(bigString)).append("\n"); //bigdecimal
bw.write(sb.toString());
sb.setLength(0);
}
bw.close();
// Store only 1000 rows in each row block (MIN_ROW_INDEX_STRIDE is 1000. So can't use less than that)
pigServer.registerQuery("A = load '" + Util.generateURI(inputTxtFile, pigServer.getPigContext()) + "' as (f1:boolean, f2:int, f3:int, f4:int, f5:long, f6:float, f7:double, f8:bytearray, f9:chararray, f10:datetime, f11:bigdecimal);");
pigServer.registerQuery("store A into '" + Util.generateURI(INPUT, pigServer.getPigContext()) +"' using OrcStorage('-r 1000 -s 100000');");
Util.copyFromLocalToCluster(cluster, INPUT, INPUT);
}