Apache Hive自定义函数-了解

概述

UDF函数其实就是一个简单的函数，执行过程就是在Hive转换成MapReduce程序后，执行java方法，类似于像MapReduce执行过程中加入一个插件，方便扩展。目前Hive除去一些内置的函数以外，还提供了一些内置的函数的扩扎接口：
UDF：针对单行数据操作，需要继承UDF
UDTF:操作一个数据行，产生多个数据行或者是多个列，需要用户继承GenericUDTF
UDAF:操作多个数据行，产生一个数据行,主要用以聚合统计，需要继承AbstractGenericUDAFResolver
目前由于Hive中已经提供了强大的聚合函数，本篇文章就UDF和UDTF实现给出以下两个案例:

<dependency>
    <groupId>org.apache.hive</groupId>
    <artifactId>hive-exec</artifactId>
    <version>1.2.2</version>
</dependency>
<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-common</artifactId>
    <version>2.9.2</version>
</dependency>

UDF

package com.jiangzz;

import org.apache.hadoop.hive.ql.exec.UDF;

public class DeptUDF extends UDF {
    public String evaluate(String value){
        return value+"部门";
    }
}

这里的evaluate方法的参数和返回值需要用户根据自己的需求定义

UDTF

package com.jiangzz;

import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

public class UserUDTF extends GenericUDTF {

    @Override
    public StructObjectInspector initialize(StructObjectInspector argOIs) throws UDFArgumentException {
        List<ObjectInspector> objectInspectors=new ArrayList<ObjectInspector>();

        objectInspectors.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
        objectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
        objectInspectors.add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);

        //返回的类型参数
        return ObjectInspectorFactory.getStandardStructObjectInspector(Arrays.asList("id","name","sex"),objectInspectors);
    }

    public void process(Object[] args) throws HiveException {

        String value = args[0].toString();
        String sep= args[1].toString();
        String[] tokens = value.split(sep);
        forward(Arrays.asList(Integer.parseInt(tokens[0]),tokens[1],Boolean.valueOf(tokens[2])));

    }

    public void close() throws HiveException {

    }
}

代码打包

将代码进行package打包 hive-function-1.0-SNAPSHOT.jar创建以下function.sql文件进行测试

create temporary function dept_fun as 'com.jiangzz.DeptUDF';
create temporary function user_fun as 'com.jiangzz.UserUDTF';
select dept_fun(20);
select user_fun('1,zhangsan,true',',');

[[email protected] ~]# hive -f function.sql --auxpath /root/hive-function-1.0-SNAPSHOT.jar