Java Utils.getSchemaFromString方法代码示例

import org.apache.pig.impl.util.Utils; //导入方法依赖的package包/类
public static Schema getEscapedSchemaFromString(String schemaStr) {
    schemaStr = schemaStr.replaceAll("[\\r\\n]", "");
    String[] fieldSchemaStrs = schemaStr.split(",");
    StringBuilder escapedSchemaBuilder = new StringBuilder();

    for (int i = 0; i < fieldSchemaStrs.length; i++) {
        escapedSchemaBuilder.append(escapeFieldSchemaStr(fieldSchemaStrs[i]));
        if (i != fieldSchemaStrs.length - 1)
            escapedSchemaBuilder.append(",");
    }

    try {
        return Utils.getSchemaFromString(escapedSchemaBuilder.toString());
    } catch (ParserException pe) {
        throw new IllegalArgumentException("Invalid schema format: " + pe.getMessage());
    }
}
 

import org.apache.pig.impl.util.Utils; //导入方法依赖的package包/类
@Override
public Schema outputSchema(Schema input) {
  if (null != this.schemaFunction) {
    try {
      Tuple t = TupleFactory.getInstance().newTuple(1);
      // Strip enclosing '{}' from schema
      t.set(0, input.toString().replaceAll("^\\{", "").replaceAll("\\}$", ""));
      return Utils.getSchemaFromString((String) this.schemaFunction.exec(t));
    } catch (ParserException pe) {
      throw new RuntimeException(pe);
    } catch (IOException ioe) {
      throw new RuntimeException(ioe);
    }
  } else {
    return this.schema;
  }
}
 

import org.apache.pig.impl.util.Utils; //导入方法依赖的package包/类
public void prepareToWrite(RecordWriter writer) {
    // Get the schema string from the UDFContext object.
    UDFContext udfc = UDFContext.getUDFContext();
    Properties p =
        udfc.getUDFProperties(this.getClass(), new String[]{ udfContextSignature });

    String strSchema = p.getProperty(SCHEMA_SIGNATURE);
    if (strSchema != null) {
        // Parse the schema from the string stored in the properties object.
        try {
            schema = new ResourceSchema(Utils.getSchemaFromString(strSchema));
        } catch (ParserException pex) {
            logger.warn("Could not parse schema for storing.");
        }
    }

    if (headerTreatment == Headers.DEFAULT) {
        headerTreatment = Headers.SKIP_OUTPUT_HEADER;
    }

    // PigStorage's prepareToWrite()
    super.prepareToWrite(writer);
}
 

import org.apache.pig.impl.util.Utils; //导入方法依赖的package包/类
@Test
public void testDereferenceTypeSet() throws IOException, ParserException {
    String query = "a = load 'a' as (i : int, j : int);"
    + " b = foreach a generate i, j/10.1 as jd;"
    + " c = group b by i;"
    + " d = foreach c generate MAX(b.jd) as mx;";

    PigServer pig = new PigServer(ExecType.LOCAL);
    Util.registerMultiLineQuery(pig, query);

    Schema expectedSch =
        Utils.getSchemaFromString("mx: double");
    Schema sch = pig.dumpSchema("d");
    assertEquals("Checking expected schema", expectedSch, sch);

}
 

import org.apache.pig.impl.util.Utils; //导入方法依赖的package包/类
public static Schema getPigSchema(String schemaString) {
    Schema schema = null;
    schemaString = schemaString.replace("/", "_");
    schemaString = "{(" + schemaString + ")}";
    try {
        schema = Utils.getSchemaFromString(schemaString);
    } catch (ParserException e) {
        e.printStackTrace();
    }
    return schema;
}
 

import org.apache.pig.impl.util.Utils; //导入方法依赖的package包/类
private ResourceSchema createSchema(String schema) {
    try {
        return new ResourceSchema(Utils.getSchemaFromString(schema));
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
}
 

import org.apache.pig.impl.util.Utils; //导入方法依赖的package包/类
@Override
public Schema outputSchema(Schema input) {
	try {
		return Utils
				.getSchemaFromString("tagged_sentence:tuple (tokens:bag {column:tuple (token:chararray)}, tags:bag {column:tuple (tag:chararray)})");

	} catch (Exception e) {
		return null;
	}

}
 

import org.apache.pig.impl.util.Utils; //导入方法依赖的package包/类
@Override
public Map exec(Tuple input) throws IOException {
    if (input == null || input.size() == 0) {
        return null;
    }
    try {
        String jsonStr = (String) input.get(0);
        String schemaStr = "object: map[]";

        ResourceSchema schema = new ResourceSchema(Utils.getSchemaFromString(schemaStr));
        ResourceFieldSchema[] fields = schema.getFields();

        JsonParser p = jsonFactory.createJsonParser(jsonStr);

        Tuple t = tupleFactory.newTuple(1);
        try {
            p.nextToken(); // move to start of object
            t.set(0, JsonLoader.readField(jsonStr, p, fields[0]));
        } catch (JsonParseException jpe) {
            log.error("Error parsing input: " + jsonStr + ": " + jpe.toString());
        }

        p.close();
        return (Map) t.get(0);
    } catch (ExecException e) {
        warn("Error reading input: " + e.getMessage(), PigWarning.UDF_WARNING_1);
        return null;
    }
}
 

import org.apache.pig.impl.util.Utils; //导入方法依赖的package包/类
@Override
public Schema outputSchema(Schema input) {
    try {
        return Utils.getSchemaFromString("m:map[chararray]");
    } catch (ParserException e) {
        return null;
    }
}
 

import org.apache.pig.impl.util.Utils; //导入方法依赖的package包/类
@Test
public void testMissingPrimaryKey() throws IOException, InterruptedException {
    String tableName = "mortar_test_foo_table";
    String awsAccessKeyId = "XXXXXXXXXXXXX";
    String awsSecretKey = "YYYYYYYYYYYYYY";
    ResourceSchema schema = 
            new ResourceSchema(Utils.getSchemaFromString("my_field:int"));
    
 // mock dynamo client
    AmazonDynamoDBClient dynamo = mock(AmazonDynamoDBClient.class);
    DescribeTableResult describeResult = new DescribeTableResult()
        .withTable(
            new TableDescription()
                .withProvisionedThroughput(
                    new ProvisionedThroughputDescription().withWriteCapacityUnits(50L))
                .withKeySchema(new KeySchema()
                    .withHashKeyElement(new KeySchemaElement()
                        .withAttributeName("not_the_key_you_will_find")
                        .withAttributeType(ScalarAttributeType.N))));
             
    when(dynamo.describeTable(any(DescribeTableRequest.class))).thenReturn(describeResult);
    DynamoDBStorage storage = 
            new DynamoDBStorage(tableName, awsAccessKeyId, awsSecretKey, dynamo, null);
    try {
        storage.checkSchema(schema);
        Assert.fail("Expected schema validation to fail");
    } catch(IOException e) {
        Assert.assertTrue("Expected " + e.getMessage() + " to contain hash msg", e.getMessage().contains("hash primary key"));
    }
}
 

import org.apache.pig.impl.util.Utils; //导入方法依赖的package包/类
@Test
public void testDescribeCogroup() throws Throwable {
    PigServer pig = new PigServer(cluster.getExecType(), properties);
    pig.registerQuery("a = load 'a' as (field1: int, field2: float, field3: chararray );") ;
    pig.registerQuery("b = load 'b' as (field4, field5: double, field6: chararray );") ;
    pig.registerQuery("c = cogroup a by field1, b by field4;") ;
    Schema dumpedSchema = pig.dumpSchema("c") ;
    Schema expectedSchema = Utils.getSchemaFromString("group:int,a:{(field1:int,field2:float,field3:chararray)},b:{(field4:bytearray,field5:double,field6:chararray)}");
    assertEquals(expectedSchema, dumpedSchema);
}
 

import org.apache.pig.impl.util.Utils; //导入方法依赖的package包/类
@Test
public void testDescribeFilter() throws Throwable {
    PigServer pig = new PigServer(cluster.getExecType(), properties);
    pig.registerQuery("a = load 'a' as (field1: int, field2: float, field3: chararray );") ;
    pig.registerQuery("b = filter a by field1 > 10;") ;
    Schema dumpedSchema = pig.dumpSchema("b") ;
    Schema expectedSchema = Utils.getSchemaFromString("field1: int,field2: float,field3: chararray");
    assertEquals(expectedSchema, dumpedSchema);
}
 

import org.apache.pig.impl.util.Utils; //导入方法依赖的package包/类
public JsFunction(String functionName) {
    this.jsScriptEngine = JsScriptEngine.getInstance();
    this.functionName = functionName;

    String outputSchemaDef;
    outputSchemaDef = jsScriptEngine.jsEval(this.getClass().getName()+"(String)", functionName+".outputSchema").toString();
    try {
        this.outputSchema = Utils.getSchemaFromString(outputSchemaDef);
    } catch (ParserException e) {
        throw new IllegalArgumentException(functionName+".outputSchema is not a valid schema: "+e.getMessage(), e); 
    }

}
 

import org.apache.pig.impl.util.Utils; //导入方法依赖的package包/类
/**
 * Test UNION ONSCHEMA where a common column has additional 'namespace' part
 *  in the column name in both the inputs
 * @throws IOException
 * @throws ParserException
 */
@Test
public void testUnionOnSchemaScopedColumnNameBothInp2() throws IOException, ParserException {
    PigServer pig = new PigServer(ExecType.LOCAL);
    String query =
        "   l1 = load '" + INP_FILE_2NUMS + "' as (i : int, j : int); " 
        + " l2 = load '" + INP_FILE_2NUMS + "' as (i : int, x : chararray); " 
        + " cg1 = cogroup l1 by i, l2 by i; "
        + " f1 = foreach cg1 generate group as gkey, flatten(l1), flatten(l2); "
        + " cg2 = cogroup l2 by i, l1 by i; "
        + " f2 = foreach cg1 generate group as gkey, flatten(l2), flatten(l1); "
        + "u = union onschema f1, f2; " ; 
    Util.registerMultiLineQuery(pig, query);
            
    Schema sch = pig.dumpSchema("u");
    Schema expectedSch = 
        Utils.getSchemaFromString("gkey: int, l1::i: int, l1::j: int, l2::i: int, l2::x: chararray");
    assertEquals("Checking expected schema",sch, expectedSch);

    Iterator<Tuple> it = pig.openIterator("u");
    List<Tuple> expectedRes = 
        Util.getTuplesFromConstantTupleStrings(
                new String[] {
                        "(1,1,2,1,'2')",
                        "(5,5,3,5,'3')",
                        "(1,1,2,1,'2')",
                        "(5,5,3,5,'3')",
                });
    Util.checkQueryOutputsAfterSort(it, expectedRes);
    
}
 

import org.apache.pig.impl.util.Utils; //导入方法依赖的package包/类
public StreamingUDF(String language,
                    String filePath, String funcName,
                    String outputSchemaString, String schemaLineNumber,
                    String execType, String isIllustrate)
                            throws StreamingUDFOutputSchemaException, ExecException {
    this.language = language;
    this.filePath = filePath;
    this.funcName = funcName;
    try {
        this.schema = Utils.getSchemaFromString(outputSchemaString);
        //ExecTypeProvider.fromString doesn't seem to load the ExecTypes in
        //mapreduce mode so we'll try to figure out the exec type ourselves.
        if (execType.equals("local")) {
            this.execType = ExecType.LOCAL;
        } else if (execType.equals("mapreduce")) {
            this.execType = ExecType.MAPREDUCE;
        } else {
            //Not sure what exec type - try to get it from the string.
            this.execType = ExecTypeProvider.fromString(execType);
        }
    } catch (ParserException pe) {
        throw new StreamingUDFOutputSchemaException(pe.getMessage(), Integer.valueOf(schemaLineNumber));
    } catch (IOException ioe) {
        String errorMessage = "Invalid exectype passed to StreamingUDF. Should be local or mapreduce";
        log.error(errorMessage, ioe);
        throw new ExecException(errorMessage, ioe);
    }
    this.isIllustrate = isIllustrate;
}