各种类型的Writable
各种类型的Writable(Text、ByteWritable、NullWritable、ObjectWritable、GenericWritable、ArrayWritable、MapWritable、SortedMapWritable)
@Test
public void testTextIndex(){
Text text=new Text("hadoop");
Assert.assertEquals(text.getLength(), 6);
Assert.assertEquals(text.getBytes().length, 6);
Assert.assertEquals(text.charAt(2),(int)'d');
Assert.assertEquals("Out of bounds",text.charAt(100),-1);
}
Text还有个find方法,类似String里indexOf方法
@Test
public void testTextFind() {
Text text = new Text("hadoop");
Assert.assertEquals("find a substring",text.find("do"),2);
Assert.assertEquals("Find first 'o'",text.find("o"),3);
Assert.assertEquals("Find 'o' from position 4 or later",text.find("o",4),4);
Assert.assertEquals("No match",text.find("pig"),-1);
}
Unicode的不同
@Test
public void string() throws UnsupportedEncodingException {
String str = "\u0041\u00DF\u6771\uD801\uDC00";
Assert.assertEquals(str.length(), 5);
Assert.assertEquals(str.getBytes("UTF-8").length, 10);
Assert.assertEquals(str.indexOf("\u0041"), 0);
Assert.assertEquals(str.indexOf("\u00DF"), 1);
Assert.assertEquals(str.indexOf("\u6771"), 2);
Assert.assertEquals(str.indexOf("\uD801\uDC00"), 3);
Assert.assertEquals(str.charAt(0), '\u0041');
Assert.assertEquals(str.charAt(1), '\u00DF');
Assert.assertEquals(str.charAt(2), '\u6771');
Assert.assertEquals(str.charAt(3), '\uD801');
Assert.assertEquals(str.charAt(4), '\uDC00');
Assert.assertEquals(str.codePointAt(0), 0x0041);
Assert.assertEquals(str.codePointAt(1), 0x00DF);
Assert.assertEquals(str.codePointAt(2), 0x6771);
Assert.assertEquals(str.codePointAt(3), 0x10400);
}
@Test
public void text() {
Text text = new Text("\u0041\u00DF\u6771\uD801\uDC00");
Assert.assertEquals(text.getLength(), 10);
Assert.assertEquals(text.find("\u0041"), 0);
Assert.assertEquals(text.find("\u00DF"), 1);
Assert.assertEquals(text.find("\u6771"), 3);
Assert.assertEquals(text.find("\uD801\uDC00"), 6);
Assert.assertEquals(text.charAt(0), 0x0041);
Assert.assertEquals(text.charAt(1), 0x00DF);
Assert.assertEquals(text.charAt(3), 0x6771);
Assert.assertEquals(text.charAt(6), 0x10400);
}
这样一比较就很明显了。
Text的迭代
package com.sweetop.styhadoop;
import org.apache.hadoop.io.Text;
import java.nio.ByteBuffer;
/**
* Created with IntelliJ IDEA.
* User: lastsweetop
* Date: 13-7-9
* Time: 下午5:00
* To change this template use File | Settings | File Templates.
*/
public class TextIterator {
public static void main(String[] args) {
Text text = new Text("\u0041\u00DF\u6771\uD801\udc00");
ByteBuffer buffer = ByteBuffer.wrap(text.getBytes(), 0, text.getLength());
int cp;
while (buffer.hasRemaining() && (cp = Text.bytesToCodePoint(buffer)) != -1) {
System.out.println(Integer.toHexString(cp));
}
}
}
Text的修改
@Test
public void testTextMutability() {
Text text = new Text("hadoop");
text.set("pig");
Assert.assertEquals(text.getLength(), 3);
Assert.assertEquals(text.getBytes().length, 3);
}
但要注意的就是,在某些情况下Text的getBytes方法返回的字节数组的长度和Text的getLength方法返回的长度不一致。因此,在调用getBytes()方法的同时最好也调用一下getLength方法,这样你就知道在字节数组里有多少有效的字符。
@Test
public void testTextMutability2() {
Text text = new Text("hadoop");
text.set(new Text("pig"));
Assert.assertEquals(text.getLength(),3);
Assert.assertEquals(text.getBytes().length,6);
}
BytesWritable类型
@Test
public void testByteWritableSerilizedFromat() throws IOException {
BytesWritable bytesWritable=new BytesWritable(new byte[]{3,5});
byte[] bytes=SerializeUtils.serialize(bytesWritable);
Assert.assertEquals(StringUtils.byteToHexString(bytes),"000000020305");
}
和Text一样,ByteWritable也可以通过set方法修改,getLength返回的大小是真实大小,而getBytes返回的大小确不是。
bytesWritable.setCapacity(11);
bytesWritable.setSize(4);
Assert.assertEquals(4,bytesWritable.getLength());
Assert.assertEquals(11,bytesWritable.getBytes().length);
NullWritable类型
package com.sweetop.styhadoop;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.util.StringUtils;
import java.io.IOException;
/**
* Created with IntelliJ IDEA.
* User: lastsweetop
* Date: 13-7-16
* Time: 下午9:23
* To change this template use File | Settings | File Templates.
*/
public class TestNullWritable {
public static void main(String[] args) throws IOException {
NullWritable nullWritable=NullWritable.get();
System.out.println(StringUtils.byteToHexString(SerializeUtils.serialize(nullWritable)));
}
}
ObjectWritable类型
ObjectWritable是其他类型的封装类,包括java原生类型,String,enum,Writable,null等,或者这些类型构成的数组。当你的一个field有多种类型时,ObjectWritable类型的用处就发挥出来了,不过有个不好的地方就是占用的空间太大,即使你存一个字母,因为它需要保存封装前的类型,我们来看下示例:
package com.sweetop.styhadoop;
import org.apache.hadoop.io.ObjectWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.util.StringUtils;
import java.io.IOException;
/**
* Created with IntelliJ IDEA.
* User: lastsweetop
* Date: 13-7-17
* Time: 上午9:14
* To change this template use File | Settings | File Templates.
*/
public class TestObjectWritable {
public static void main(String[] args) throws IOException {
Text text=new Text("\u0041");
ObjectWritable objectWritable=new ObjectWritable(text);
System.out.println(StringUtils.byteToHexString(SerializeUtils.serialize(objectWritable)));
}
}
仅仅是保存一个字母,那么看下它序列化后的结果是什么:
00196f72672e6170616368652e6861646f6f702e696f2e5465787400196f72672e6170616368652e6861646f6f702e696f2e546578740141
太浪费空间了,而且类型一般是已知的,也就那么几个,那么它的代替方法出现,看下一小节
GenericWritable类型
package com.sweetop.styhadoop;
import org.apache.hadoop.io.GenericWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
class MyWritable extends GenericWritable {
MyWritable(Writable writable) {
set(writable);
}
public static Class<? extends Writable>[] CLASSES=null;
static {
CLASSES= (Class<? extends Writable>[])new Class[]{
Text.class
};
}
@Override
protected Class<? extends Writable>[] getTypes() {
return CLASSES; //To change body of implemented methods use File | Settings | File Templates.
}
}
然后输出序列化后的结果
package com.sweetop.styhadoop;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.VIntWritable;
import org.apache.hadoop.util.StringUtils;
import java.io.IOException;
/**
* Created with IntelliJ IDEA.
* User: lastsweetop
* Date: 13-7-17
* Time: 上午9:51
* To change this template use File | Settings | File Templates.
*/
public class TestGenericWritable {
public static void main(String[] args) throws IOException {
Text text=new Text("\u0041\u0071");
MyWritable myWritable=new MyWritable(text);
System.out.println(StringUtils.byteToHexString(SerializeUtils.serialize(text)));
System.out.println(StringUtils.byteToHexString(SerializeUtils.serialize(myWritable)));
}
}
结果是:
024171
00024171
GenericWritable的序列化只是把类型在type数组里的索引放在了前面,这样就比ObjectWritable节省了很多空间,所以推荐大家使用GenericWritable
集合类型的Writable
package com.sweetop.styhadoop;
import org.apache.hadoop.io.ArrayWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.util.StringUtils;
import java.io.IOException;
/**
* Created with IntelliJ IDEA.
* User: lastsweetop
* Date: 13-7-17
* Time: 上午11:14
* To change this template use File | Settings | File Templates.
*/
public class TestArrayWritable {
public static void main(String[] args) throws IOException {
ArrayWritable arrayWritable=new ArrayWritable(Text.class);
arrayWritable.set(new Writable[]{new Text("\u0071"),new Text("\u0041")});
System.out.println(StringUtils.byteToHexString(SerializeUtils.serialize(arrayWritable)));
}
}
看下输出:
0000000201710141
可知,ArrayWritable以一个整型开始表示数组长度,然后数组里的元素一一排开。
MapWritable和SortedMapWritable
上一篇: 求一个整数存储在内存中的二进制中1的个数
下一篇: 二进制文件的读写操作