jstorm源码之TransactionalState
程序员文章站
2022-07-13 16:01:19
...
一、作用
主要是通过结合zookeeper,在zookeeper指定的目录下创建不同的自己 代表不同的transaction state,并赋予不同的内容;通过zookeeper完成对partitions的transaction的管理
二、源码分析
package storm.trident.topology.state;
import backtype.storm.Config;
import backtype.storm.utils.Utils;
import backtype.storm.utils.ZookeeperAuthInfo;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.api.ProtectACLCreateModePathAndBytesable;
import org.apache.curator.framework.api.PathAndBytesable;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Id;
import org.json.simple.JSONValue;
// 事务状态
// 通过zookeeper来进行完成操作
public class TransactionalState {
// zookeeper 的监护器
CuratorFramework _curator;
// zookeeper 访问控制列表
List<ACL> _zkAcls = null;
// 构建用户的transaction的状态
public static TransactionalState newUserState(Map conf, String id) {
return new TransactionalState(conf, id, "user");
}
// 构建协调器的transaction的状态
public static TransactionalState newCoordinatorState(Map conf, String id) {
return new TransactionalState(conf, id, "coordinator");
}
// 在zookeeper上构建对应的transaction的state的目录
protected TransactionalState(Map conf, String id, String subroot) {
try {
conf = new HashMap(conf);
// zookeeper的root目录
String transactionalRoot = (String)conf.get(Config.TRANSACTIONAL_ZOOKEEPER_ROOT);
// zookeeper的transaction的state的根目录
String rootDir = transactionalRoot + "/" + id + "/" + subroot;
// 根据配置信息中的transactional servers 和 zookeeper servers获取servers 优先使用transactional的servers 否则使用zookeeper servers
List<String> servers = (List<String>) getWithBackup(conf, Config.TRANSACTIONAL_ZOOKEEPER_SERVERS, Config.STORM_ZOOKEEPER_SERVERS);
// 根据配置信息中的transactional ports和zookeeper port设置对应的ports
Object port = getWithBackup(conf, Config.TRANSACTIONAL_ZOOKEEPER_PORT, Config.STORM_ZOOKEEPER_PORT);
// 设定zookeeper权限控制
ZookeeperAuthInfo auth = new ZookeeperAuthInfo(conf);
// 根据config、servers、port、auth完成zookeeper监护器初始化 并启动
CuratorFramework initter = Utils.newCuratorStarted(conf, servers, port, auth);
// 获取zookeeper可用列表
_zkAcls = Utils.getWorkerACL(conf);
try {
// 在zookeeper上创建对应的transaction 根目录node
TransactionalState.createNode(initter, transactionalRoot, null, null, null);
} catch (KeeperException.NodeExistsException e) {
}
try {
// 创建对应的transaction state的node的子根目录
TransactionalState.createNode(initter, rootDir, null, _zkAcls, null);
} catch (KeeperException.NodeExistsException e) {
}
// 关闭初始化监护器
initter.close();
// 开启全局监护器
_curator = Utils.newCuratorStarted(conf, servers, port, rootDir, auth);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
protected static String forPath(PathAndBytesable<String> builder,
String path, byte[] data) throws Exception {
return (data == null)
? builder.forPath(path)
: builder.forPath(path, data);
}
protected static void createNode(CuratorFramework curator, String path,
byte[] data, List<ACL> acls, CreateMode mode) throws Exception {
ProtectACLCreateModePathAndBytesable<String> builder =
curator.create().creatingParentsIfNeeded();
if (acls == null) {
if (mode == null ) {
TransactionalState.forPath(builder, path, data);
} else {
TransactionalState.forPath(builder.withMode(mode), path, data);
}
return;
}
TransactionalState.forPath(builder.withACL(acls), path, data);
}
// 指定path设置data
public void setData(String path, Object obj) {
path = "/" + path;
byte[] ser;
try {
ser = JSONValue.toJSONString(obj).getBytes("UTF-8");
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
try {
if(_curator.checkExists().forPath(path)!=null) {
_curator.setData().forPath(path, ser);
} else {
TransactionalState.createNode(_curator, path, ser, _zkAcls,
CreateMode.PERSISTENT);
}
} catch(Exception e) {
throw new RuntimeException(e);
}
}
// 删除指定path
public void delete(String path) {
path = "/" + path;
try {
_curator.delete().forPath(path);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
// 罗列出指定path下所有的子节点
public List<String> list(String path) {
path = "/" + path;
try {
if(_curator.checkExists().forPath(path)==null) {
return new ArrayList<String>();
} else {
return _curator.getChildren().forPath(path);
}
} catch(Exception e) {
throw new RuntimeException(e);
}
}
public void mkdir(String path) {
setData(path, 7);
}
// 获取指定路径的data
public Object getData(String path) {
path = "/" + path;
try {
if(_curator.checkExists().forPath(path)!=null) {
return JSONValue.parse(new String(_curator.getData().forPath(path), "UTF-8"));
} else {
return null;
}
} catch(Exception e) {
throw new RuntimeException(e);
}
}
// 关闭监护器
public void close() {
_curator.close();
}
// 根据指定内容返回对应的结果
private Object getWithBackup(Map amap, Object primary, Object backup) {
Object ret = amap.get(primary);
if(ret==null) return amap.get(backup);
return ret;
}
}
主要是通过结合zookeeper,在zookeeper指定的目录下创建不同的自己 代表不同的transaction state,并赋予不同的内容;通过zookeeper完成对partitions的transaction的管理
二、源码分析
package storm.trident.topology.state;
import backtype.storm.Config;
import backtype.storm.utils.Utils;
import backtype.storm.utils.ZookeeperAuthInfo;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.api.ProtectACLCreateModePathAndBytesable;
import org.apache.curator.framework.api.PathAndBytesable;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Id;
import org.json.simple.JSONValue;
// 事务状态
// 通过zookeeper来进行完成操作
public class TransactionalState {
// zookeeper 的监护器
CuratorFramework _curator;
// zookeeper 访问控制列表
List<ACL> _zkAcls = null;
// 构建用户的transaction的状态
public static TransactionalState newUserState(Map conf, String id) {
return new TransactionalState(conf, id, "user");
}
// 构建协调器的transaction的状态
public static TransactionalState newCoordinatorState(Map conf, String id) {
return new TransactionalState(conf, id, "coordinator");
}
// 在zookeeper上构建对应的transaction的state的目录
protected TransactionalState(Map conf, String id, String subroot) {
try {
conf = new HashMap(conf);
// zookeeper的root目录
String transactionalRoot = (String)conf.get(Config.TRANSACTIONAL_ZOOKEEPER_ROOT);
// zookeeper的transaction的state的根目录
String rootDir = transactionalRoot + "/" + id + "/" + subroot;
// 根据配置信息中的transactional servers 和 zookeeper servers获取servers 优先使用transactional的servers 否则使用zookeeper servers
List<String> servers = (List<String>) getWithBackup(conf, Config.TRANSACTIONAL_ZOOKEEPER_SERVERS, Config.STORM_ZOOKEEPER_SERVERS);
// 根据配置信息中的transactional ports和zookeeper port设置对应的ports
Object port = getWithBackup(conf, Config.TRANSACTIONAL_ZOOKEEPER_PORT, Config.STORM_ZOOKEEPER_PORT);
// 设定zookeeper权限控制
ZookeeperAuthInfo auth = new ZookeeperAuthInfo(conf);
// 根据config、servers、port、auth完成zookeeper监护器初始化 并启动
CuratorFramework initter = Utils.newCuratorStarted(conf, servers, port, auth);
// 获取zookeeper可用列表
_zkAcls = Utils.getWorkerACL(conf);
try {
// 在zookeeper上创建对应的transaction 根目录node
TransactionalState.createNode(initter, transactionalRoot, null, null, null);
} catch (KeeperException.NodeExistsException e) {
}
try {
// 创建对应的transaction state的node的子根目录
TransactionalState.createNode(initter, rootDir, null, _zkAcls, null);
} catch (KeeperException.NodeExistsException e) {
}
// 关闭初始化监护器
initter.close();
// 开启全局监护器
_curator = Utils.newCuratorStarted(conf, servers, port, rootDir, auth);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
protected static String forPath(PathAndBytesable<String> builder,
String path, byte[] data) throws Exception {
return (data == null)
? builder.forPath(path)
: builder.forPath(path, data);
}
protected static void createNode(CuratorFramework curator, String path,
byte[] data, List<ACL> acls, CreateMode mode) throws Exception {
ProtectACLCreateModePathAndBytesable<String> builder =
curator.create().creatingParentsIfNeeded();
if (acls == null) {
if (mode == null ) {
TransactionalState.forPath(builder, path, data);
} else {
TransactionalState.forPath(builder.withMode(mode), path, data);
}
return;
}
TransactionalState.forPath(builder.withACL(acls), path, data);
}
// 指定path设置data
public void setData(String path, Object obj) {
path = "/" + path;
byte[] ser;
try {
ser = JSONValue.toJSONString(obj).getBytes("UTF-8");
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
try {
if(_curator.checkExists().forPath(path)!=null) {
_curator.setData().forPath(path, ser);
} else {
TransactionalState.createNode(_curator, path, ser, _zkAcls,
CreateMode.PERSISTENT);
}
} catch(Exception e) {
throw new RuntimeException(e);
}
}
// 删除指定path
public void delete(String path) {
path = "/" + path;
try {
_curator.delete().forPath(path);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
// 罗列出指定path下所有的子节点
public List<String> list(String path) {
path = "/" + path;
try {
if(_curator.checkExists().forPath(path)==null) {
return new ArrayList<String>();
} else {
return _curator.getChildren().forPath(path);
}
} catch(Exception e) {
throw new RuntimeException(e);
}
}
public void mkdir(String path) {
setData(path, 7);
}
// 获取指定路径的data
public Object getData(String path) {
path = "/" + path;
try {
if(_curator.checkExists().forPath(path)!=null) {
return JSONValue.parse(new String(_curator.getData().forPath(path), "UTF-8"));
} else {
return null;
}
} catch(Exception e) {
throw new RuntimeException(e);
}
}
// 关闭监护器
public void close() {
_curator.close();
}
// 根据指定内容返回对应的结果
private Object getWithBackup(Map amap, Object primary, Object backup) {
Object ret = amap.get(primary);
if(ret==null) return amap.get(backup);
return ret;
}
}