Java PrimitiveObjectInspectorFactory.javaIntObjectInspector方法代码示例

import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; //导入方法依赖的package包/类
@Nonnull
protected final ObjectInspector getFeatureOutputOI(@Nonnull final FeatureType featureType)
        throws UDFArgumentException {
    final PrimitiveObjectInspector outputOI;
    if (dense_model) {
        // TODO validation
        outputOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector; // see DenseModel (long/string is also parsed as int)
    } else {
        switch (featureType) {
            case STRING:
                outputOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
                break;
            case INT:
                outputOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
                break;
            case LONG:
                outputOI = PrimitiveObjectInspectorFactory.javaLongObjectInspector;
                break;
            default:
                throw new IllegalStateException("Unexpected feature type: " + featureType);
        }
    }
    return outputOI;
}
 

import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; //导入方法依赖的package包/类
@Test
public void testPA2EtaWithParameter() throws UDFArgumentException {
    PassiveAggressiveUDTF udtf = new PassiveAggressiveUDTF.PA2();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI = ObjectInspectorFactory.getStandardListObjectInspector(intOI);
    ObjectInspector param = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-c 3.0");

    /* do initialize() with aggressiveness parameter */
    udtf.initialize(new ObjectInspector[] {intListOI, intOI, param});
    float loss = 0.1f;

    PredictionResult margin1 = new PredictionResult(0.5f).squaredNorm(0.05f);
    float expectedLearningRate1 = 0.4615384f;
    assertEquals(expectedLearningRate1, udtf.eta(loss, margin1), 1e-5f);

    PredictionResult margin2 = new PredictionResult(0.5f).squaredNorm(0.01f);
    float expectedLearningRate2 = 0.5660377f;
    assertEquals(expectedLearningRate2, udtf.eta(loss, margin2), 1e-5f);
}
 

import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; //导入方法依赖的package包/类
@Test
public void test2Positive3NegativeSample() throws Exception {
    BinarizeLabelUDTF udtf = spy(new BinarizeLabelUDTF());
    ObjectInspector[] argOIs = new ObjectInspector[3];
    argOIs[0] = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    argOIs[1] = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    List<String> featureNames = Arrays.asList("positive", "negative", "features");
    argOIs[2] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, featureNames);

    doNothing().when(udtf, "forward", any());
    udtf.initialize(argOIs);

    Object[] arguments = new Object[3];
    arguments[0] = new Integer(2);
    arguments[1] = new Integer(3);
    arguments[2] = WritableUtils.val("a:1", "b:2");
    udtf.process(arguments);

    verifyPrivate(udtf, times(5)).invoke("forward", any(Object[].class));
    verifyPrivate(udtf, times(2)).invoke("forward",
        aryEq(new Object[] {WritableUtils.val("a:1", "b:2"), 1}));
    verifyPrivate(udtf, times(3)).invoke("forward",
        aryEq(new Object[] {WritableUtils.val("a:1", "b:2"), 0}));
}
 

import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; //导入方法依赖的package包/类
@Test
public void testPA1Eta() throws UDFArgumentException {
    PassiveAggressiveUDTF udtf = new PassiveAggressiveUDTF.PA1();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI = ObjectInspectorFactory.getStandardListObjectInspector(intOI);
    ObjectInspector param = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-c 3.0");

    /* do initialize() with aggressiveness parameter */
    udtf.initialize(new ObjectInspector[] {intListOI, intOI, param});
    float loss = 0.1f;

    PredictionResult margin1 = new PredictionResult(0.5f).squaredNorm(0.05f);
    float expectedLearningRate1 = 2.0f;
    assertEquals(expectedLearningRate1, udtf.eta(loss, margin1), 1e-5f);

    PredictionResult margin2 = new PredictionResult(0.5f).squaredNorm(0.01f);
    float expectedLearningRate2 = 3.0f;
    assertEquals(expectedLearningRate2, udtf.eta(loss, margin2), 1e-5f);
}
 

import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; //导入方法依赖的package包/类
@Test
public void test0Positive0NegativeSample() throws Exception {
    BinarizeLabelUDTF udtf = spy(new BinarizeLabelUDTF());
    ObjectInspector[] argOIs = new ObjectInspector[3];
    argOIs[0] = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    argOIs[1] = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    List<String> featureNames = Arrays.asList("positive", "negative", "features");
    argOIs[2] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, featureNames);

    doNothing().when(udtf, "forward", any());
    udtf.initialize(argOIs);

    Object[] arguments = new Object[3];
    arguments[0] = new Integer(0);
    arguments[1] = new Integer(0);
    arguments[2] = WritableUtils.val("a:1", "b:2");
    udtf.process(arguments);

    verifyPrivate(udtf, times(0)).invoke("forward", any(Object[].class));
}
 

import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; //导入方法依赖的package包/类
@Test
public void testInitialize() throws UDFArgumentException {
    PerceptronUDTF udtf = new PerceptronUDTF();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI = ObjectInspectorFactory.getStandardListObjectInspector(intOI);

    /* test for INT_TYPE_NAME feature */
    StructObjectInspector intListSOI = udtf.initialize(new ObjectInspector[] {intListOI, intOI});
    assertEquals("struct<feature:int,weight:float>", intListSOI.getTypeName());

    /* test for STRING_TYPE_NAME feature */
    ObjectInspector stringOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ListObjectInspector stringListOI = ObjectInspectorFactory.getStandardListObjectInspector(stringOI);
    StructObjectInspector stringListSOI = udtf.initialize(new ObjectInspector[] {stringListOI,
            intOI});
    assertEquals("struct<feature:string,weight:float>", stringListSOI.getTypeName());

    /* test for BIGINT_TYPE_NAME feature */
    ObjectInspector longOI = PrimitiveObjectInspectorFactory.javaLongObjectInspector;
    ListObjectInspector longListOI = ObjectInspectorFactory.getStandardListObjectInspector(longOI);
    StructObjectInspector longListSOI = udtf.initialize(new ObjectInspector[] {longListOI,
            intOI});
    assertEquals("struct<feature:bigint,weight:float>", longListSOI.getTypeName());
}
 

import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; //导入方法依赖的package包/类
@Test
public void testPA2EtaWithoutParameter() throws UDFArgumentException {
    PassiveAggressiveUDTF udtf = new PassiveAggressiveUDTF.PA2();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI = ObjectInspectorFactory.getStandardListObjectInspector(intOI);

    /* do initialize() with aggressiveness parameter */
    udtf.initialize(new ObjectInspector[] {intListOI, intOI});
    float loss = 0.1f;

    PredictionResult margin1 = new PredictionResult(0.5f).squaredNorm(0.05f);
    float expectedLearningRate1 = 0.1818181f;
    assertEquals(expectedLearningRate1, udtf.eta(loss, margin1), 1e-5f);

    PredictionResult margin2 = new PredictionResult(0.5f).squaredNorm(0.01f);
    float expectedLearningRate2 = 0.1960784f;
    assertEquals(expectedLearningRate2, udtf.eta(loss, margin2), 1e-5f);
}
 

import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; //导入方法依赖的package包/类
@Test
public void testRandInit() throws HiveException {
    println("--------------------------\n testRandInit()");
    OnlineMatrixFactorizationUDTF mf = new MatrixFactorizationSGDUDTF();

    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ObjectInspector floatOI = PrimitiveObjectInspectorFactory.javaFloatObjectInspector;
    ObjectInspector param = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, new String(
            "-factor 3 -rankinit random"));
    ObjectInspector[] argOIs = new ObjectInspector[] {intOI, intOI, floatOI, param};
    mf.initialize(argOIs);
    Assert.assertTrue(mf.rankInit == RankInitScheme.random);

    float[][] rating = { {5, 3, 0, 1}, {4, 0, 0, 1}, {1, 1, 0, 5}, {1, 0, 0, 4}, {0, 1, 5, 4}};
    Object[] args = new Object[3];
    final int num_iters = 100;
    for (int iter = 0; iter < num_iters; iter++) {
        for (int row = 0; row < rating.length; row++) {
            for (int col = 0, size = rating[row].length; col < size; col++) {
                args[0] = row;
                args[1] = col;
                args[2] = (float) rating[row][col];
                mf.process(args);
            }
        }
    }
    for (int row = 0; row < rating.length; row++) {
        for (int col = 0, size = rating[row].length; col < size; col++) {
            double predicted = mf.predict(row, col);
            print(rating[row][col] + "[" + predicted + "]\t");
            Assert.assertEquals(rating[row][col], predicted, 0.2d);
        }
        println();
    }
}
 

import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; //导入方法依赖的package包/类
@Test
public void testGaussianInit() throws HiveException {
    println("--------------------------\n testGaussianInit()");
    OnlineMatrixFactorizationUDTF mf = new MatrixFactorizationSGDUDTF();

    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ObjectInspector floatOI = PrimitiveObjectInspectorFactory.javaFloatObjectInspector;
    ObjectInspector param = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, new String(
            "-factor 3 -rankinit gaussian"));
    ObjectInspector[] argOIs = new ObjectInspector[] {intOI, intOI, floatOI, param};
    mf.initialize(argOIs);
    Assert.assertTrue(mf.rankInit == RankInitScheme.gaussian);

    float[][] rating = { {5, 3, 0, 1}, {4, 0, 0, 1}, {1, 1, 0, 5}, {1, 0, 0, 4}, {0, 1, 5, 4}};
    Object[] args = new Object[3];
    final int num_iters = 100;
    for (int iter = 0; iter < num_iters; iter++) {
        for (int row = 0; row < rating.length; row++) {
            for (int col = 0, size = rating[row].length; col < size; col++) {
                args[0] = row;
                args[1] = col;
                args[2] = (float) rating[row][col];
                mf.process(args);
            }
        }
    }
    for (int row = 0; row < rating.length; row++) {
        for (int col = 0, size = rating[row].length; col < size; col++) {
            double predicted = mf.predict(row, col);
            print(rating[row][col] + "[" + predicted + "]\t");
            Assert.assertEquals(rating[row][col], predicted, 0.2d);
        }
        println();
    }
}
 

import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; //导入方法依赖的package包/类
@Test
public void testPA1EtaDefaultParameter() throws UDFArgumentException {
    PassiveAggressiveUDTF udtf = new PassiveAggressiveUDTF.PA1();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ListObjectInspector intListOI = ObjectInspectorFactory.getStandardListObjectInspector(intOI);

    udtf.initialize(new ObjectInspector[] {intListOI, intOI});
    float loss = 0.1f;

    PredictionResult margin = new PredictionResult(0.5f).squaredNorm(0.05f);
    float expectedLearningRate = 1.0f;
    assertEquals(expectedLearningRate, udtf.eta(loss, margin), 1e-5f);
}
 

import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; //导入方法依赖的package包/类
public void testInsufficientLabelColumn() throws HiveException {
    BinarizeLabelUDTF udtf = new BinarizeLabelUDTF();
    ObjectInspector[] argOIs = new ObjectInspector[2];
    argOIs[0] = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    List<String> featureNames = Arrays.asList("positive", "features");
    argOIs[1] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, featureNames);

    udtf.initialize(argOIs);
}
 

import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; //导入方法依赖的package包/类
@Test
public void testStringInt() throws Exception {
    ObjectInspector featureOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ObjectInspector weightOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    udf.initialize(new ObjectInspector[] {featureOI, weightOI});

    Text ret = udf.evaluate(new GenericUDF.DeferredObject[] {new DeferredJavaObject("f1"),
            new DeferredJavaObject(2)});

    Assert.assertEquals("f1:2", ret.toString());
}
 

import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; //导入方法依赖的package包/类
@Test(expected = UDFArgumentException.class)
public void testUnsupportedLossFunction() throws Exception {
    GeneralClassifierUDTF udtf = new GeneralClassifierUDTF();
    ObjectInspector intOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ObjectInspector stringOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ListObjectInspector stringListOI = ObjectInspectorFactory.getStandardListObjectInspector(stringOI);
    ObjectInspector params = ObjectInspectorUtils.getConstantObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-loss UnsupportedLoss");

    udtf.initialize(new ObjectInspector[] {stringListOI, intOI, params});
}
 

import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; //导入方法依赖的package包/类
@Test
public void testIntInt() throws Exception {
    ObjectInspector featureOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ObjectInspector weightOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    udf.initialize(new ObjectInspector[] {featureOI, weightOI});

    Text ret = udf.evaluate(new GenericUDF.DeferredObject[] {new DeferredJavaObject(1),
            new DeferredJavaObject(2)});

    Assert.assertEquals("1:2", ret.toString());
}
 

import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; //导入方法依赖的package包/类
@Test
public void testIntDouble() throws Exception {
    ObjectInspector featureOI = PrimitiveObjectInspectorFactory.javaIntObjectInspector;
    ObjectInspector weightOI = PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
    udf.initialize(new ObjectInspector[] {featureOI, weightOI});

    Text ret = udf.evaluate(new GenericUDF.DeferredObject[] {new DeferredJavaObject(1),
            new DeferredJavaObject(2.5d)});

    Assert.assertEquals("1:2.5", ret.toString());
}