Java VIntWritable类代码示例

import org.apache.hadoop.io.VIntWritable; //导入依赖的package包/类
@Override
public void init() throws IOException {
  registerKey(NullWritable.class.getName(), NullWritableSerializer.class);
  registerKey(Text.class.getName(), TextSerializer.class);
  registerKey(LongWritable.class.getName(), LongWritableSerializer.class);
  registerKey(IntWritable.class.getName(), IntWritableSerializer.class);
  registerKey(Writable.class.getName(), DefaultSerializer.class);
  registerKey(BytesWritable.class.getName(), BytesWritableSerializer.class);
  registerKey(BooleanWritable.class.getName(), BoolWritableSerializer.class);
  registerKey(ByteWritable.class.getName(), ByteWritableSerializer.class);
  registerKey(FloatWritable.class.getName(), FloatWritableSerializer.class);
  registerKey(DoubleWritable.class.getName(), DoubleWritableSerializer.class);
  registerKey(VIntWritable.class.getName(), VIntWritableSerializer.class);
  registerKey(VLongWritable.class.getName(), VLongWritableSerializer.class);

  LOG.info("Hadoop platform inited");
}
 

import org.apache.hadoop.io.VIntWritable; //导入依赖的package包/类
@Test
public void tryOne() throws IOException, InterruptedException {
    Text inKey=new Text("qr");
    Iterable<VIntWritable> inValues=new RecyclingIterable(
        VIntWritable.class,
        new VIntWritable(76412),
        new VIntWritable(2),
        new VIntWritable(7)
    );

    reducer.reduce(inKey,inValues,context);

    Text outKey=new Text("1951-06 qr");
    Text outValue=new Text("3 76421");
    verify(context).write(outKey,outValue);
}
 

import org.apache.hadoop.io.VIntWritable; //导入依赖的package包/类
@Override
public Vertex<Text, VIntWritable, Text, Writable> getCurrentVertex() throws IOException, InterruptedException {
    Vertex<Text, VIntWritable, Text, Writable> vertex;
    String line = getRecordReader().getCurrentValue().toString();
    //Parse each line and create the vertex and edges
    String[] tokens = line.toString().trim().split("\t");
    if (tokens.length == 1) {
        //Null sender on the left hand side
        vertex = null;
        throw new IllegalArgumentException("bad data in line: " + line);
    }
    else if (tokens.length != 2) {
        throw new IllegalArgumentException("bad arguments in line: " + line);
    }
    else {
        vertex = getConf().createVertex();
        List<Edge<Text, Text>> edges = edgesFrom(tokens);
        Text vertexId = new Text(tokens[0]);
        VIntWritable vertexValue = new VIntWritable(0); 
        vertex.initialize(vertexId, vertexValue, edges);
    }

    return vertex;
}
 

import org.apache.hadoop.io.VIntWritable; //导入依赖的package包/类
/**
 * The actual algorithm. See class documentation for description.
 */
@Override
public void compute(Iterable<Text> messages) throws IOException {
	try {

		// Check to see if we received any messages from nodes notifying
		// that they have only a single edge
		for (Text incomingMessage : messages) {
			Text vertex = new Text(incomingMessage.toString().split(":")[0]);
			int value = Integer.parseInt(incomingMessage.toString().split(":")[1]);
			setValue(new VIntWritable(getValue().get() + 1 + value));

			// Remove the vertex and its corresponding edge
			removeVertexRequest(vertex);
			removeEdges(vertex);
			// System.err.println("Removing: " + vertex.toString());
		}

		// Broadcast the edges if we only have a single edge
		sendEdges();
	} catch (Exception e) {
		System.err.println(e.toString());
	}
}
 

import org.apache.hadoop.io.VIntWritable; //导入依赖的package包/类
public void write(Writable w) throws IOException {
  if (w instanceof TypedBytesWritable) {
    writeTypedBytes((TypedBytesWritable) w);
  } else if (w instanceof BytesWritable) {
    writeBytes((BytesWritable) w);
  } else if (w instanceof ByteWritable) {
    writeByte((ByteWritable) w);
  } else if (w instanceof BooleanWritable) {
    writeBoolean((BooleanWritable) w);
  } else if (w instanceof IntWritable) {
    writeInt((IntWritable) w);
  } else if (w instanceof VIntWritable) {
    writeVInt((VIntWritable) w);
  } else if (w instanceof LongWritable) {
    writeLong((LongWritable) w);
  } else if (w instanceof VLongWritable) {
    writeVLong((VLongWritable) w);
  } else if (w instanceof FloatWritable) {
    writeFloat((FloatWritable) w);
  } else if (w instanceof DoubleWritable) {
    writeDouble((DoubleWritable) w);
  } else if (w instanceof Text) {
    writeText((Text) w);
  } else if (w instanceof ArrayWritable) {
    writeArray((ArrayWritable) w);
  } else if (w instanceof MapWritable) {
    writeMap((MapWritable) w);
  } else if (w instanceof SortedMapWritable) {
    writeSortedMap((SortedMapWritable) w);
  } else if (w instanceof Record) {
    writeRecord((Record) w);
  } else {
    writeWritable(w); // last resort
  }
}
 

import org.apache.hadoop.io.VIntWritable; //导入依赖的package包/类
public Class<? extends Writable> readType() throws IOException {
  Type type = in.readType();
  if (type == null) {
    return null;
  }
  switch (type) {
  case BYTES:
    return BytesWritable.class;
  case BYTE:
    return ByteWritable.class;
  case BOOL:
    return BooleanWritable.class;
  case INT:
    return VIntWritable.class;
  case LONG:
    return VLongWritable.class;
  case FLOAT:
    return FloatWritable.class;
  case DOUBLE:
    return DoubleWritable.class;
  case STRING:
    return Text.class;
  case VECTOR:
    return ArrayWritable.class;
  case MAP:
    return MapWritable.class;
  case WRITABLE:
    return Writable.class;
  default:
    throw new RuntimeException("unknown type");
  }
}
 

import org.apache.hadoop.io.VIntWritable; //导入依赖的package包/类
public static void updateObject(Writable obj, byte[] seed) {
  if (obj instanceof IntWritable) {
    ((IntWritable)obj).set(Ints.fromByteArray(seed));
  } else if (obj instanceof FloatWritable) {
    ((FloatWritable)obj).set(r.nextFloat());
  } else if (obj instanceof DoubleWritable) {
    ((DoubleWritable)obj).set(r.nextDouble());
  } else if (obj instanceof LongWritable) {
    ((LongWritable)obj).set(Longs.fromByteArray(seed));
  } else if (obj instanceof VIntWritable) {
    ((VIntWritable)obj).set(Ints.fromByteArray(seed));
  } else if (obj instanceof VLongWritable) {
    ((VLongWritable)obj).set(Longs.fromByteArray(seed));
  } else if (obj instanceof BooleanWritable) {
    ((BooleanWritable)obj).set(seed[0] % 2 == 1 ? true : false);
  } else if (obj instanceof Text) {
    ((Text)obj).set(BytesUtil.toStringBinary(seed));
  } else if (obj instanceof ByteWritable) {
    ((ByteWritable)obj).set(seed.length > 0 ? seed[0] : 0);
  } else if (obj instanceof BytesWritable) {
    ((BytesWritable)obj).set(seed, 0, seed.length);
  } else if (obj instanceof UTF8) {
    ((UTF8)obj).set(BytesUtil.toStringBinary(seed));
  } else if (obj instanceof MockValueClass) {
    ((MockValueClass)obj).set(seed);
  } else {
    throw new IllegalArgumentException("unknown writable: " +
                                       obj.getClass().getName());
  }
}
 

import org.apache.hadoop.io.VIntWritable; //导入依赖的package包/类
public static <VTYPE> byte[] toBytes(VTYPE obj) {
  final String className = obj.getClass().getName();
  if (className.equals(IntWritable.class.getName())) {
    return Ints.toByteArray(((IntWritable) obj).get());
  } else if (className.equals(FloatWritable.class.getName())) {
    return BytesUtil.toBytes(((FloatWritable) obj).get());
  } else if (className.equals(DoubleWritable.class.getName())) {
    return BytesUtil.toBytes(((DoubleWritable) obj).get());
  } else if (className.equals(LongWritable.class.getName())) {
    return Longs.toByteArray(((LongWritable) obj).get());
  } else if (className.equals(VIntWritable.class.getName())) {
    return Ints.toByteArray(((VIntWritable) obj).get());
  } else if (className.equals(VLongWritable.class.getName())) {
    return Longs.toByteArray(((VLongWritable) obj).get());
  } else if (className.equals(BooleanWritable.class.getName())) {
    return BytesUtil.toBytes(((BooleanWritable) obj).get());
  } else if (className.equals(Text.class.getName())) {
    return ((Text)obj).copyBytes();
  } else if (className.equals(ByteWritable.class.getName())) {
    return Ints.toByteArray((int) ((ByteWritable) obj).get());
  } else if (className.equals(BytesWritable.class.getName())) {
    // TODO: copyBytes instead?
    return ((BytesWritable) obj).getBytes();
  } else {
    return new byte[0];
  }
}
 

import org.apache.hadoop.io.VIntWritable; //导入依赖的package包/类
public void write(Writable w) throws IOException {
  if (w instanceof TypedBytesWritable) {
    writeTypedBytes((TypedBytesWritable) w);
  } else if (w instanceof BytesWritable) {
    writeBytes((BytesWritable) w);
  } else if (w instanceof ByteWritable) {
    writeByte((ByteWritable) w);
  } else if (w instanceof BooleanWritable) {
    writeBoolean((BooleanWritable) w);
  } else if (w instanceof IntWritable) {
    writeInt((IntWritable) w);
  } else if (w instanceof VIntWritable) {
    writeVInt((VIntWritable) w);
  } else if (w instanceof LongWritable) {
    writeLong((LongWritable) w);
  } else if (w instanceof VLongWritable) {
    writeVLong((VLongWritable) w);
  } else if (w instanceof FloatWritable) {
    writeFloat((FloatWritable) w);
  } else if (w instanceof DoubleWritable) {
    writeDouble((DoubleWritable) w);
  } else if (w instanceof Text) {
    writeText((Text) w);
  } else if (w instanceof ArrayWritable) {
    writeArray((ArrayWritable) w);
  } else if (w instanceof MapWritable) {
    writeMap((MapWritable) w);
  } else if (w instanceof SortedMapWritable) {
    writeSortedMap((SortedMapWritable<?>) w);
  } else if (w instanceof Record) {
    writeRecord((Record) w);
  } else {
    writeWritable(w); // last resort
  }
}
 

import org.apache.hadoop.io.VIntWritable; //导入依赖的package包/类
/**
 * Create new XdmValue from value type and Writables.
 *  
 */
public static XdmValue newValue(ValueType valueType, Object value) {
    if (value instanceof Text) {
        return ValueFactory.newValue(valueType, ((Text)value).toString());
    } else if (value instanceof BytesWritable) {
        return ValueFactory.newValue(valueType, ((BytesWritable)value).getBytes());
    } else if (value instanceof IntWritable) {
        return ValueFactory.newValue(valueType, ((IntWritable)value).get());
    } else if (value instanceof LongWritable) {
        return ValueFactory.newValue(valueType, ((LongWritable)value).get());
    } else if (value instanceof VIntWritable) {
        return ValueFactory.newValue(valueType, ((VIntWritable)value).get());
    } else if (value instanceof VLongWritable) {
        return ValueFactory.newValue(valueType, ((VLongWritable)value).get());
    } else if (value instanceof BooleanWritable) {
        return ValueFactory.newValue(valueType, ((BooleanWritable)value).get());
    } else if (value instanceof FloatWritable) {
        return ValueFactory.newValue(valueType, ((FloatWritable)value).get());
    } else if (value instanceof DoubleWritable) {
        return ValueFactory.newValue(valueType, ((DoubleWritable)value).get());
    } else if (value instanceof MarkLogicNode) {
        return ValueFactory.newValue(valueType, ((MarkLogicNode)value).get());
    } else {
        throw new UnsupportedOperationException("Value " +  
                value.getClass().getName() + " is unsupported.");
    }
}
 

import org.apache.hadoop.io.VIntWritable; //导入依赖的package包/类
public static Writable toWritable(Object object) {
    if (object == null) {
        return null; //return NullWritable.get();
    }
    if (object instanceof Writable) {
        return (Writable) object;
    }
    if (object instanceof String) {
        return new Text((String) object);
    }
    if (object instanceof Long) {
        return new VLongWritable((Long) object);
    }
    if (object instanceof Integer) {
        return new VIntWritable((Integer) object);
    }
    if (object instanceof Byte) {
        return new ByteWritable((Byte) object);
    }
    if (object instanceof Double) {
        return new DoubleWritable((Double) object);
    }
    if (object instanceof Float) {
        return new FloatWritable((Float) object);
    }
    if (object instanceof Boolean) {
        return new BooleanWritable((Boolean) object);
    }
    if (object instanceof byte[]) {
        return new BytesWritable((byte[]) object);
    }
    return new BytesWritable(object.toString().getBytes());
}
 

import org.apache.hadoop.io.VIntWritable; //导入依赖的package包/类
@Override
protected void reduce(Text key, Iterable<VIntWritable> values, Context context) throws IOException, InterruptedException {
    int uriCount=0;
    long viewCount=0;

    for(VIntWritable views:values) {
        uriCount++;
        viewCount+=views.get();
    }

    Text outKey=new Text(yrmo+" "+key.toString());
    Text outValue=new Text(uriCount+" "+viewCount);
    context.write(outKey,outValue);
}
 

import org.apache.hadoop.io.VIntWritable; //导入依赖的package包/类
@Override
protected void map(Text key, Text value, Context context) throws IOException, InterruptedException {
    String keyString=key.toString();
    try {
        Iterator<String> parts=SPACE_SPLITTER.split(keyString).iterator();
        String siteId=parts.next();
        String uri=parts.next();
        int count=Integer.parseInt(value.toString());
        context.write(new Text(siteId),new VIntWritable(count));
    } catch(NoSuchElementException | NumberFormatException ex) {
        LOG.warn("Couldn't parse line ["+keyString+"]");
    }
}
 

import org.apache.hadoop.io.VIntWritable; //导入依赖的package包/类
@Test
public void tryOne() throws IOException, InterruptedException {
    Text inKey=new Text("zr PallaPalla");
    Text inValue=new Text("75312");

    mapper.map(inKey,inValue,context);
    verify(context).write(new Text("zr"),new VIntWritable(75312));
}
 

import org.apache.hadoop.io.VIntWritable; //导入依赖的package包/类
@Test
public void testPageId() throws IOException, InterruptedException {
    mapper.currentTag=new VIntWritable(1);
    Text line=new Text("<http://dbpedia.org/resource/AccessibleComputing> <http://dbpedia.org/ontology/wikiPageID> \"10\"^^<http://www.w3.org/2001/XMLSchema#integer>");
    mapper.map(new LongWritable(101),line,context);
    verify(context).write(
            new TaggedTextItem("<http://dbpedia.org/resource/AccessibleComputing>",1),
            new TaggedTextItem("",1)
    );
}