Java Util.copyFromLocalToCluster方法代码示例

本文整理汇总了Java中org.apache.pig.test.Util.copyFromLocalToCluster方法的典型用法代码示例。如果您正苦于以下问题：Java Util.copyFromLocalToCluster方法的具体用法？Java Util.copyFromLocalToCluster怎么用？Java Util.copyFromLocalToCluster使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.pig.test.Util的用法示例。

在下文中一共展示了Util.copyFromLocalToCluster方法的6个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: createFile

import org.apache.pig.test.Util; //导入方法依赖的package包/类
private void createFile() throws IOException {
  PrintWriter w = new PrintWriter(new FileWriter(INPUT_FILE));
  w.println("100\tapple\taaa1");
  w.println("200\torange\tbbb1");
  w.println("300\tstrawberry\tccc1");

  w.println("101\tapple\taaa2");
  w.println("201\torange\tbbb2");
  w.println("301\tstrawberry\tccc2");

  w.println("102\tapple\taaa3");
  w.println("202\torange\tbbb3");
  w.println("302\tstrawberry\tccc3");

  w.close();
  Util.deleteFile(cluster, INPUT_FILE);
  Util.copyFromLocalToCluster(cluster, INPUT_FILE, INPUT_FILE);
}

开发者ID:sigmoidanalytics，项目名称:spork-streaming，代码行数:19，代码来源:TestMultiStorage.java

示例2: createFile

import org.apache.pig.test.Util; //导入方法依赖的package包/类
private void createFile() throws IOException {
	PrintWriter w = new PrintWriter(new FileWriter(INPUT_FILE));
	w = new PrintWriter(new FileWriter(INPUT_FILE));
	w.println("100\tapple\t1.0\t2008-01-01");
	w.println("100\torange\t2.0\t2008-02-01");
	w.println("100\tbanana\t1.1\t2008-03-01");
	w.close();
	Util.copyFromLocalToCluster(cluster, INPUT_FILE, INPUT_FILE);
}

开发者ID:sigmoidanalytics，项目名称:spork-streaming，代码行数:10，代码来源:TestDBStorage.java

示例3: oneTimeSetup

import org.apache.pig.test.Util; //导入方法依赖的package包/类
@BeforeClass
public static void oneTimeSetup() throws Exception{
    cluster = MiniGenericCluster.buildCluster();
    Util.copyFromLocalToCluster(cluster, basedir + "orc-file-11-format.orc", basedir + "orc-file-11-format.orc");
    Util.copyFromLocalToCluster(cluster, basedir + "charvarchar.orc", basedir + "charvarchar.orc");
    createInputData();

    if(Util.WINDOWS){
        OUTPUT1 = OUTPUT1.replace("\\", "/");
        OUTPUT2 = OUTPUT2.replace("\\", "/");
        OUTPUT3 = OUTPUT3.replace("\\", "/");
        OUTPUT4 = OUTPUT4.replace("\\", "/");
    }

    supportedOpTypes = new ArrayList<OpType>();
    supportedOpTypes.add(OpType.OP_EQ);
    supportedOpTypes.add(OpType.OP_NE);
    supportedOpTypes.add(OpType.OP_GT);
    supportedOpTypes.add(OpType.OP_GE);
    supportedOpTypes.add(OpType.OP_LT);
    supportedOpTypes.add(OpType.OP_LE);
    supportedOpTypes.add(OpType.OP_IN);
    supportedOpTypes.add(OpType.OP_BETWEEN);
    supportedOpTypes.add(OpType.OP_NULL);
    supportedOpTypes.add(OpType.OP_NOT);
    supportedOpTypes.add(OpType.OP_AND);
    supportedOpTypes.add(OpType.OP_OR);

    Logger logger = Logger.getLogger(ColumnPruneVisitor.class);
    logger.removeAllAppenders();
    logger.setLevel(Level.INFO);
    SimpleLayout layout = new SimpleLayout();
    logFile = File.createTempFile("log", "");
    FileAppender appender = new FileAppender(layout, logFile.toString(), false, false, 0);
    logger.addAppender(appender);
}

开发者ID:sigmoidanalytics，项目名称:spork，代码行数:37，代码来源:TestOrcStoragePushdown.java

示例4: testPythonUDF_onCluster

import org.apache.pig.test.Util; //导入方法依赖的package包/类
@Test
public void testPythonUDF_onCluster() throws Exception {
    pigServerMapReduce = new PigServer(cluster.getExecType(), cluster.getProperties());

    String[] pythonScript = {
            "from pig_util import outputSchema",
            "@outputSchema(\'c:chararray\')",
            "def py_func(one,two):",
            "   return one + two"
    };
    
    Util.createLocalInputFile("pyfile_mr.py", pythonScript);

    String[] input = {
        "field10\tfield11",
        "field20\tfield21"
    };
    Util.createLocalInputFile("input_mr", input);
    Util.copyFromLocalToCluster(cluster, "input_mr", "input_mr");

    pigServerMapReduce.registerQuery("REGISTER 'pyfile_mr.py' USING streaming_python AS pf;");
    pigServerMapReduce.registerQuery("A = LOAD 'input_mr' as (c1:chararray, c2:chararray);");
    pigServerMapReduce.registerQuery("B = FOREACH A generate pf.py_func(c1, c2);");

    Iterator<Tuple> iter = pigServerMapReduce.openIterator("B");
    assertTrue(iter.hasNext());
    Tuple actual0 = iter.next();
    assertTrue(iter.hasNext());
    Tuple actual1 = iter.next();

    Tuple expected0 = tf.newTuple("field10field11");
    Tuple expected1 = tf.newTuple("field20field21");

    assertEquals(expected0, actual0);
    assertEquals(expected1, actual1);
}

开发者ID:sigmoidanalytics，项目名称:spork，代码行数:37，代码来源:TestStreamingUDF.java

示例5: createFile

import org.apache.pig.test.Util; //导入方法依赖的package包/类
private void createFile() throws IOException {
    PrintWriter w = new PrintWriter(new FileWriter(INPUT_FILE));
    w = new PrintWriter(new FileWriter(INPUT_FILE));
    w.println("100\tapple\t1.0\t2008-01-01");
    w.println("100\torange\t2.0\t2008-02-01");
    w.println("100\tbanana\t1.1\t2008-03-01");
    w.println("\t\t\t");
    w.close();
    Util.copyFromLocalToCluster(cluster, INPUT_FILE, INPUT_FILE);
}

开发者ID:sigmoidanalytics，项目名称:spork，代码行数:11，代码来源:TestDBStorage.java

示例6: createInputData

import org.apache.pig.test.Util; //导入方法依赖的package包/类
private static void createInputData() throws Exception {
    pigServer = new PigServer(ExecType.LOCAL);

    new File(inpbasedir).mkdirs();
    new File(outbasedir).mkdirs();
    String inputTxtFile = inpbasedir + File.separator + "input.txt";
    BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(inputTxtFile), "UTF-8"));
    long[] lVal = new long[] {100L, 200L, 300L};
    float[] fVal = new float[] {50.0f, 100.0f, 200.0f, 300.0f};
    double[] dVal = new double[] {1000.11, 2000.22, 3000.33};
    StringBuilder sb = new StringBuilder();
    for (int i=1; i <= 10000; i++) {
        sb.append((i > 6500 && i <= 9000) ? true : false).append("\t"); //boolean
        sb.append((i > 1000 && i < 3000) ? 1 : 5).append("\t"); //byte
        sb.append((i > 2500 && i <= 4500) ? 100 : 200).append("\t"); //short
        sb.append(i).append("\t"); //int
        sb.append(lVal[i%3]).append("\t"); //long
        sb.append(fVal[i%4]).append("\t"); //float
        sb.append((i > 2500 && i < 3500) ? dVal[i%3] : dVal[i%1]).append("\t"); //double
        sb.append((i%2 == 1 ? "" : RandomStringUtils.random(100).replaceAll("\t", " ")
                .replaceAll("\n", " ").replaceAll("\r", " "))).append("\t"); //bytearray
        sb.append((i%2 == 0 ? "" : RandomStringUtils.random(100).replaceAll("\t", " ")
                .replaceAll("\n", " ").replaceAll("\r", " "))).append("\t"); //string
        int year;
        if (i > 5000 && i <= 8000) { //datetime
            year = RandomUtils.nextInt(4)+2010;
        } else {
            year = RandomUtils.nextInt(10)+2000;
        }
        sb.append(new DateTime(year, RandomUtils.nextInt(12)+1,
                RandomUtils.nextInt(28)+1, RandomUtils.nextInt(24), RandomUtils.nextInt(60),
                DateTimeZone.UTC).toString()).append("\t"); // datetime
        String bigString;
        if (i>7500) {
            bigString = RandomStringUtils.randomNumeric(9) + "." + RandomStringUtils.randomNumeric(5);
        } else {
            bigString = "1" + RandomStringUtils.randomNumeric(9) + "." + RandomStringUtils.randomNumeric(5);
        }
        sb.append(new BigDecimal(bigString)).append("\n"); //bigdecimal
        bw.write(sb.toString());
        sb.setLength(0);
    }
    bw.close();

    // Store only 1000 rows in each row block (MIN_ROW_INDEX_STRIDE is 1000. So can't use less than that)
    pigServer.registerQuery("A = load '" + Util.generateURI(inputTxtFile, pigServer.getPigContext()) + "' as (f1:boolean, f2:int, f3:int, f4:int, f5:long, f6:float, f7:double, f8:bytearray, f9:chararray, f10:datetime, f11:bigdecimal);");
    pigServer.registerQuery("store A into '" + Util.generateURI(INPUT, pigServer.getPigContext()) +"' using OrcStorage('-r 1000 -s 100000');");
    Util.copyFromLocalToCluster(cluster, INPUT, INPUT);
}

开发者ID:sigmoidanalytics，项目名称:spork，代码行数:50，代码来源:TestOrcStoragePushdown.java

注：本文中的org.apache.pig.test.Util.copyFromLocalToCluster方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。