欢迎您访问程序员文章站本站旨在为大家提供分享程序员计算机编程知识！
您现在的位置是： 首页

SpringBoot 基于Aspect实现多数据源，支持hive数据源（一）

程序员文章站 2022-04-11 10:22:46
...

项目背景：

       最近在摸索中接触大数据展示项目，提到大数据统计就不得不解决多数据源，动态数据源问题，当然这些问题可以通过其他方案替代，但响应的需要付出一定的代价。本笔记中主要探讨的是在一个简单的springboot 项目中引入多种数据源，用户根据业务自动切换响应的数据源进行业务统计，或者在一个API中实现多数据源联合统计。

工具：

      STS

施工：

新建项目工程:

SpringBoot 基于Aspect实现多数据源，支持hive数据源（一）

maven 配置

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<parent>
		<groupId>org.springframework.boot</groupId>
		<artifactId>spring-boot-starter-parent</artifactId>
		<version>2.2.1.RELEASE</version>
		<relativePath/> <!-- lookup parent from repository -->
	</parent>
	<groupId>com.example</groupId>
	<artifactId>dataSource</artifactId>
	<version>0.0.1-SNAPSHOT</version>
	<name>dataSource</name>
	<description>Demo project for Spring Boot</description>

	<properties>
		<java.version>1.8</java.version>
	</properties>

	<dependencies>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-data-jdbc</artifactId>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-data-jpa</artifactId>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-data-rest</artifactId>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-jdbc</artifactId>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-web</artifactId>
		</dependency>
		<dependency>
			<groupId>org.mybatis.spring.boot</groupId>
			<artifactId>mybatis-spring-boot-starter</artifactId>
			<version>2.1.1</version>
		</dependency>

		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-devtools</artifactId>
			<scope>runtime</scope>
			<optional>true</optional>
		</dependency>
		<dependency>
			<groupId>com.h2database</groupId>
			<artifactId>h2</artifactId>
			<scope>runtime</scope>
		</dependency>
		<dependency>
			<groupId>mysql</groupId>
			<artifactId>mysql-connector-java</artifactId>
			<scope>runtime</scope>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-test</artifactId>
			<scope>test</scope>
			<exclusions>
				<exclusion>
					<groupId>org.junit.vintage</groupId>
					<artifactId>junit-vintage-engine</artifactId>
				</exclusion>
			</exclusions>
		</dependency>
		<dependency>
			<groupId>org.springframework.data</groupId>
			<artifactId>spring-data-hadoop</artifactId>
			<version>2.5.0.RELEASE</version>
		</dependency>
        <!--引入hive依赖和jdbc驱动-->
        <dependency>
			<groupId>org.apache.hive</groupId>
			<artifactId>hive-jdbc</artifactId>
			<version>2.3.3</version>
			<exclusions>
				<exclusion>
					<groupId>org.eclipse.jetty.aggregate</groupId>
					<artifactId>*</artifactId>
				</exclusion>
			</exclusions>
		</dependency>
        <!--为了增强mybatis的部分能力，我们引入mybatis-plus增强框架-->
		<dependency>
		    <groupId>com.baomidou</groupId>
		    <artifactId>mybatis-plus-boot-starter</artifactId>
		    <version>2.2.0</version>
		</dependency>
		<dependency>
			<groupId>com.baomidou</groupId>
			<artifactId>mybatis-plus</artifactId>
			<version>2.3</version>
		</dependency>

	</dependencies>

	<build>
		<plugins>
			<plugin>
				<groupId>org.springframework.boot</groupId>
				<artifactId>spring-boot-maven-plugin</artifactId>
			</plugin>
		</plugins>
	</build>

</project>

数据源配置（application.properties）


spring.datasource.primary.jdbc-url=jdbc:mysql://ip:3306/XXX?useUnicode=true&characterEncoding=utf-8&serverTimezone=GMT&useSSL=false
spring.datasource.primary.username=XXX
spring.datasource.primary.password=XXX
spring.datasource.primary.driver-class=com.mysql.cj.jdbc.Driver

spring.datasource.secondary.jdbc-url=jdbc:mysql://xxx:3306/XXX?useUnicode=true&characterEncoding=utf-8&serverTimezone=GMT&useSSL=false
spring.datasource.secondary.username=XXX
spring.datasource.secondary.password=XXX
spring.datasource.secondary.driver-class=com.mysql.cj.jdbc.Driver

spring.datasource.third.jdbc-url=jdbc:hive2://XXX:10000/default
spring.datasource.third.username=XXX
spring.datasource.third.password=XXX
spring.datasource.third.driver-class=org.apache.hive.jdbc.HiveDriver
spring.datasource.third.type=com.zaxxer.hikari.HikariDataSource

 动态数据源

package com.example.demo.config;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

import javax.sql.DataSource;

import org.apache.ibatis.session.SqlSessionFactory;
import org.mybatis.spring.SqlSessionFactoryBean;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.jdbc.DataSourceBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.core.io.Resource;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import org.springframework.transaction.PlatformTransactionManager;

import com.baomidou.mybatisplus.spring.MybatisSqlSessionFactoryBean;

@Configuration
@MapperScan(basePackages = "com.example.demo.dao.mapper")
public class DataSourceConfig {
	
	@Bean("mysqlDataSource")
    @Primary
    @ConfigurationProperties(prefix = "spring.datasource.primary")
    public DataSource mysqlDataSource() {
        DataSource build = DataSourceBuilder.create().build();
        return build;
    }

    @Bean("oracleDataSource")
    @ConfigurationProperties(prefix = "spring.datasource.secondary")
    public DataSource oracleDataSource(){
        return DataSourceBuilder.create().build();
    }
    
    @Bean("hiveDataSource")
    @ConfigurationProperties(prefix = "spring.datasource.third")
    public DataSource hiveDataSource(){
        return DataSourceBuilder.create().build();
    }
    
    @Bean
    public DynamicDataSource dataSource(@Qualifier("mysqlDataSource") DataSource mysqlDataSource,
                                        @Qualifier("oracleDataSource") DataSource oracleDataSource,
                                        @Qualifier("hiveDataSource") DataSource hiveDataSource) {
        Map<Object, Object> map = new HashMap<>();
        map.put(DataSourceType.mysql, mysqlDataSource);
        map.put(DataSourceType.oracle, oracleDataSource);
        map.put(DataSourceType.hive, hiveDataSource);

        DynamicDataSource dynamicDataSource = new DynamicDataSource();
        dynamicDataSource.setTargetDataSources(map);
        dynamicDataSource.setDefaultTargetDataSource(mysqlDataSource);

        return dynamicDataSource;
    }
    
//    @Bean
//    public SqlSessionFactory sqlSessionFactory(DynamicDataSource dynamicDataSource) throws Exception {
//        SqlSessionFactoryBean factoryBean = new SqlSessionFactoryBean();
//        factoryBean.setDataSource(dynamicDataSource);
//        Resource[] resources = new PathMatchingResourcePatternResolver().getResources("classpath*:mapping/*.xml");
//        factoryBean.setMapperLocations(resources);
//        return factoryBean.getObject();
//    }
    //引入mybatis_plus增强包后必须通过MybatisSqlSessionFactoryBean进行session工厂建设，SqlSessionFactory不能满足，此处配置错误会报mapper找不到或无法实例化问题。
    @Bean
    public MybatisSqlSessionFactoryBean mybatisSqlSessionFactoryBean(DynamicDataSource dynamicDataSource) throws Exception {
        MybatisSqlSessionFactoryBean mybatisPlus = new MybatisSqlSessionFactoryBean();
        mybatisPlus.setDataSource(dynamicDataSource);
        Resource[] resources = new PathMatchingResourcePatternResolver().getResources("classpath*:mapping/*.xml");
        mybatisPlus.setMapperLocations(resources);
        return mybatisPlus;
    }

    
    @Bean
    public PlatformTransactionManager transactionManager(DynamicDataSource dynamicDataSource){
        return new DataSourceTransactionManager(dynamicDataSource);
    }

}

数据源枚举 

package com.example.demo.config;

public enum DataSourceType {
	mysql,
    oracle,
    hive
}

数据源注解 

package com.example.demo.config;

import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;

@Target({ElementType.METHOD, ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
public @interface DataSource {
    DataSourceType value() default DataSourceType.mysql;
}

数据源切面

package com.example.demo.config;


import org.aspectj.lang.annotation.After;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Before;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;


@Aspect
// 通过order指定数据源加载顺序一定是：先加载数据源再加载事务，否则会遇到事务丢失问题
@Order(-10)
@Component
public class DataSourceAspect {

        // 数据源切入点，切点就是注解DataSource，通过@annotation指定该切点的有效范围
	    @Before("@annotation(ds)")
	    public void beforeDataSource(DataSource ds) {
	        DataSourceType value = ds.value();
	        DataSourceContextHolder.setDataSource(value);
	        
	    }
	    @After("@annotation(ds)")
	    public void afterDataSource(DataSource ds){
	        DataSourceContextHolder.clearDataSource();
	    }
}

动态数据源主类

package com.example.demo.config;

import org.springframework.jdbc.datasource.lookup.AbstractRoutingDataSource;

public class DynamicDataSource extends AbstractRoutingDataSource {

	@Override
	protected Object determineCurrentLookupKey() {
		// TODO Auto-generated method stub
		return DataSourceContextHolder.getDataSource();
	}

}

UserInfoModel （PO） 

package com.example.demo.entry;

import com.baomidou.mybatisplus.annotations.TableField;
import com.baomidou.mybatisplus.annotations.TableId;
import com.baomidou.mybatisplus.annotations.TableName;
import com.baomidou.mybatisplus.enums.IdType;

@TableName(value="T_USER")
public class UserInfoModel {
	
	@TableId(value = "id",type = IdType.AUTO)
	private String id;
	
	@TableField(value = "name",exist = true)
	private String userName;
	
	@TableField(value = "password",exist = true)
	private String passWord;

	public String getUserName() {
		return userName;
	}

	public void setUserName(String userName) {
		this.userName = userName;
	}

	public String getPassWord() {
		return passWord;
	}

	public void setPassWord(String passWord) {
		this.passWord = passWord;
	}

}

DataSourceContextHolder

package com.example.demo.config;

public class DataSourceContextHolder {

	// 存放当前线程使用的数据源类型
    private static final ThreadLocal<DataSourceType> contextHolder = new ThreadLocal<>();
    
 // 设置数据源
    public static void setDataSource(DataSourceType type){
        contextHolder.set(type);
    }

    // 获取数据源
    public static DataSourceType getDataSource(){
        return contextHolder.get();
    }

    // 清除数据源
    public static void clearDataSource(){
        contextHolder.remove();
    }
}

UserInfoMapper（DAO）

package com.example.demo.dao.mapper;

import java.util.List;

import org.apache.ibatis.annotations.Select;
import org.springframework.stereotype.Repository;

import com.baomidou.mybatisplus.mapper.BaseMapper;
import com.example.demo.config.DataSource;
import com.example.demo.config.DataSourceType;
import com.example.demo.entry.UserInfoModel;

@Repository
public interface UserInfoMapper extends BaseMapper<UserInfoModel>{
    //dao 层切换数据源
	@Select(" select * from t_user ")
	@DataSource(value=DataSourceType.mysql)
    List<UserInfoModel> findAll();
	
	@Select(" select * from t_user ")
	@DataSource(value=DataSourceType.oracle)
    List<UserInfoModel> findAll2();
	
	@Select(" select t.name as userName,t.hobby as passWord from default.mytable1 t ")
	@DataSource(value=DataSourceType.hive)
    List<UserInfoModel> findAll3();
	
}

UserInfoServerImp（SERVER）

package com.example.demo.servers.imp;

import java.util.List;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import com.example.demo.config.DataSource;
import com.example.demo.config.DataSourceType;
import com.example.demo.dao.mapper.UserInfoMapper;
import com.example.demo.entry.UserInfoModel;
import com.example.demo.servers.IUserInfoServer;

@Service
public class UserInfoServerImp implements IUserInfoServer {
	
	@Autowired
	private UserInfoMapper usermapper;

	@Override
	public List<UserInfoModel> findAll() {
		// TODO Auto-generated method stub
		return usermapper.findAll();
	}

	@Override
	public List<UserInfoModel> findAll2() {
		// TODO Auto-generated method stub
		return usermapper.findAll2();
	}
	
	@Override
	public List<UserInfoModel> findAll3() {
		// TODO Auto-generated method stub
		List<UserInfoModel> list = usermapper.findAll3();
		return list;
	}
	
	@Override
	public List<UserInfoModel> findAll4() {
		// server 方法中调用dao 层不同数据源的数据，findAll3是hive数据源,findAll2是oracle数据源
		List<UserInfoModel> list = usermapper.findAll3();
		list.addAll(usermapper.findAll2());
		return list;
	}
	// server层方法中切换数据源.
	@DataSource(value=DataSourceType.mysql)
	public void insertDb(UserInfoModel model) {
		usermapper.insert(model);
	}
	
	@DataSource(value=DataSourceType.oracle)
	public void insertDb2(UserInfoModel model) {
		usermapper.insert(model);
	}

}

UserInfoControl (C)

package com.example.demo.control;

import java.util.List;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import com.example.demo.entry.UserInfoModel;
import com.example.demo.servers.imp.UserInfoServerImp;

@RestController
public class UserInfoControl {

	@Autowired
	private UserInfoServerImp userInfoServer;
	
	@RequestMapping("selectuser")
	public List<UserInfoModel> findAll(){
		return userInfoServer.findAll();
		
	} 
	
	@RequestMapping("selectuser2")
	public List<UserInfoModel> findAll2(){
		return userInfoServer.findAll2();
		
	} 
	
	@RequestMapping("selectuser3")
	public List<UserInfoModel> findAll3(){
		return userInfoServer.findAll3();
		
	} 
	
	@RequestMapping("selectuser4")
	public List<UserInfoModel> findAll4(){
		return userInfoServer.findAll4();
		
	} 
	
	@RequestMapping("insert")
	public void insert(){
		UserInfoModel model = new UserInfoModel();
		model.setPassWord("1111111");
		model.setUserName("mpftest");
		userInfoServer.insertDb(model);
		
	}
	
	@RequestMapping("insert2")
	public void insert2(){
		UserInfoModel model = new UserInfoModel();
		model.setPassWord("1111111");
		model.setUserName("mpftest");
		userInfoServer.insertDb2(model);
		userInfoServer.insertDb(model);
		
	}
}

总结

通过上面完整的例子，我们已经建立了一个支持3种数据源的springboot项目模型，能够满足自定义的SQL在server层和DAO层通过注解切换数据源，但针对mybatis_plus的增强API支持在server和dao层切换数据源进行增删改查，但需要注意，在服务层（server）不能实现多个不同数据源的接口在一个方法中调用，若有这个需求，可以考虑把整合多数据源统计的方法写在control层。

亮点：注解实现数据源切换可以实现在同一个server或dao层的不同方法使用不同的注解。

          通过后续的改造可以实现真正的动态数据源切换。

下篇预告

SpringBoot 基于分包分路径的方式实现多数据源，支持hive数据源（二）

SpringBoot基于Aspect实现动态数据源切换，支持hive数据源（三）

SpringBoot实现与HBase 数据库通过Phoenix 实现集成（四） 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

上一篇： virtualBox虚拟机网络配置

下一篇： 手机app自动化操作工具airtest之入门篇

推荐阅读