多线程实现大批量数据导入

2019独角兽企业重金招聘Python工程师标准>>>

【简述】

最近在做大批量数据导入问题，当数据量很大的时候（例如：20万条），程序在执行的时候，需要花很长的时间，为了提高程序的执行效率，现采用多线程的方法实现数据导入功能，具体实现如下：

【实现】

package com.mypack.thread;    
        
    public class Student {    
        
        private String name;    
        private String sex;    
        private int age;    
        
        public String getName() {    
            return name;    
        }    
        
        public void setName(String name) {    
            this.name = name;    
        }    
        
        public String getSex() {    
            return sex;    
        }    
        
        public void setSex(String sex) {    
            this.sex = sex;    
        }    
        
        public int getAge() {    
            return age;    
        }    
        
        public void setAge(int age) {    
            this.age = age;    
        }    
        
    }

package com.mypack.thread;    
        
    import java.util.ArrayList;    
    import java.util.List;    
        
    /**  
     * <ul>  
     * 学生线程  
     * </ul>  
     *   
     * @author liudong  
     *   
     */    
    public class StudentThread extends Thread {    
        
        private List<Student> students = new ArrayList<Student>();    
        
        public StudentThread(List<Student> students) {    
            this.students = students;    
        
        }    
        
        public void run() {    
        
            for (Student student : students) {    
                System.out.println("当前线程：" + Thread.currentThread().getName());    
                System.out.println("name = " + student.getName() + " age = "    
                        + student.getAge() + " sex = " + student.getSex());    
                // 导入学生信息到数据库中    
            }    
        }    
        
    }

package com.mypack.thread;    
        
    import java.util.ArrayList;    
    import java.util.List;    
        
    /**  
     *   
     * <ul>  
     * 测试类  
     * </ul>  
     *   
     * @author liudong  
     *   
     */    
    public class Test {    
        
        public static void main(String[] args) {    
        
            Test test = new Test();    
            // 查询所有的学生信息    
            List<Student> students = test.getAllStudents();    
            // 定义一个临时集合，用于存放学生信息    
            List<Student> tempStudents = new ArrayList<Student>();    
        
            for (int i = 0; i < students.size(); i++) {    
                tempStudents.add(students.get(i));    
                if (i != 0 && i % 50000 == 0) {    
                    test.createThread4ExportStudentInfo(tempStudents, i);    
                }    
            }    
        
            test.createThread4ExportStudentInfo(tempStudents, students.size());    
        
        }    
        
        /**  
         * <li>创建线程，导入学生信息</li>  
         *   
         * @param tempStudents  
         *            每个线程，需要导入的学生信息的数量  
         * @param i  
         */    
        public void createThread4ExportStudentInfo(List<Student> tempStudents, int i) {    
        
            List<Student> students = new ArrayList<Student>();    
        
            for (Student student : tempStudents) {    
                students.add(student);    
            }    
        
            StudentThread studentThread = new StudentThread(students);    
            // 设置线程名称    
            studentThread.setName("Thread-" + i);    
            // 启动线程    
            studentThread.start();    
        
            // 重新生成一个新的临时学生集合。    
            tempStudents = new ArrayList<Student>();    
        }    
        
        /**  
         * <li>查询所有的学生信息</li>  
         *   
         * @return  
         */    
        public List<Student> getAllStudents() {    
        
            List<Student> students = new ArrayList<Student>();    
        
            for (int i = 1; i <= 100000; i++) {    
                Student student = new Student();    
        
                student.setName("liudong" + i);    
                student.setAge(i);    
                student.setSex(i + "");    
        
                students.add(student);    
            }    
        
            return students;    
        }    
    }

转载于:https://my.oschina.net/bluesroot/blog/222530