欢迎您访问程序员文章站本站旨在为大家提供分享程序员计算机编程知识!
您现在的位置是: 首页

nio按行读文件

程序员文章站 2022-04-24 11:06:17
...

不废话,直接上代码

package com.syz.test.nio;

import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;


public class DataReader {
    /**
     * 按行读,行的分隔符必须是单个字符"\n"
     */
    public static void readByLine(String filePath,String encoding,int bufferSize) {
        byte lf = 10;//"\n"
        RandomAccessFile raf = null;
        FileChannel fc = null;
        try {
            raf = new RandomAccessFile(filePath, "r");
            fc = raf.getChannel();
            ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
            //保存换行后未处理的数据
            byte[] temp = new byte[0];
            while (fc.read(buffer)!= -1) {
                int position = buffer.position();// 读取结束后的位置,相当于读取的长度
                byte[] bs = new byte[position];// 用来存放读取的内容的数组
                buffer.rewind();// 将position设回0,所以你可以重读Buffer中的所有数据,此处如果不设置,无法使用下面的get方法
                buffer.get(bs);// 相当于buffer.get(bs,0,bs.length()):从position初始位置开始相对读,读bs.length个byte,并写入bs[0]到bs[bs.length-1]的区域
                buffer.clear();
                //要加上上次未处理的数据
                int readSize = position + temp.length;
                byte[] readByte = new byte[readSize];
                System.arraycopy(temp, 0, readByte, 0, temp.length);
                System.arraycopy(bs, 0, readByte, temp.length, position);
                int start = 0;
                int lineByteSize = 0;
                byte[] lineByte = new byte[0];
                for (int i = 0; i < readSize; i++) {
                    if (readByte[i] == lf) {
                        lineByte = new byte[lineByteSize];
                        System.arraycopy(readByte, start, lineByte, 0, lineByteSize);
                        String lineStr = new String(lineByte, encoding);
                        System.out.println(lineStr);
                        lineByteSize=0;
                        start = i+1;
                    } else {
                        lineByteSize++;
                    }

                }
                // 把剩余的放入临时数组中
                temp = new byte[lineByteSize];
                System.arraycopy(readByte, start, temp, 0, lineByteSize);
            }
            //处理临时数组中的数据
            if(temp.length>0){
                String lineStr = new String(temp, encoding);
                System.out.println(lineStr);
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (fc != null) {
                try {
                    fc.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (raf != null) {
                try {
                    raf.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }
    /**
     * 按条读
     * 举例:abc\ndef
     * 分隔符可以是多个字符,如:abc\n\rdedf
     * 如果设置成\n,就是按行读了
     * @param filePath 文件路径
     * @param encoding 字符集
     * @param bufferSize 每次读多少字节
     * @param itemSplit 第条的分隔符
     */
    public static void readByItem(String filePath,String encoding,int bufferSize,String itemSplit) {
        RandomAccessFile raf = null;
        FileChannel fc = null;
        try {
            raf = new RandomAccessFile(filePath, "r");
            fc = raf.getChannel();
            ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
            byte[] itemSplitByte = itemSplit.getBytes(encoding);
            int itemSplitLen = itemSplitByte.length;
            //保存未处理的数据
            byte[] temp = new byte[0];
            int itemIndex = 0;
            while (fc.read(buffer)!= -1) {
                int position = buffer.position();// 读取结束后的位置,相当于读取的长度
                byte[] bs = new byte[position];// 用来存放读取的内容的数组
                buffer.rewind();// 将position设回0,所以你可以重读Buffer中的所有数据,此处如果不设置,无法使用下面的get方法
                buffer.get(bs);// 相当于buffer.get(bs,0,bs.length()):从position初始位置开始相对读,读bs.length个byte,并写入bs[0]到bs[bs.length-1]的区域
                buffer.clear();
                //要加上上次未处理的数据
                int readSize = position + temp.length;
                byte[] readByte = new byte[readSize];
                System.arraycopy(temp, 0, readByte, 0, temp.length);
                System.arraycopy(bs, 0, readByte, temp.length, position);
                int start = 0;
                int columnByteSize = 0;
                byte[] columnByte = new byte[0];
                int itemSplitIndex = 0;
                for (int i = 0; i < readSize; i++) {
                    columnByteSize++;
                    if (readByte[i] == itemSplitByte[itemSplitIndex]) {
                        itemSplitIndex++;
                        if(itemSplitIndex==itemSplitLen){
                            columnByteSize = columnByteSize-itemSplitLen;
                            columnByte = new byte[columnByteSize];
                            System.arraycopy(readByte, start, columnByte, 0, columnByteSize);
                            String column = new String(columnByte, encoding);
                            System.out.println(itemIndex+"\t"+column);
                            itemIndex++;
                            start = i + 1;
                            columnByteSize = 0;
                            itemSplitIndex = 0;
                        }
                    } else {
                        itemSplitIndex=0;
                    }

                }
                // 把剩余的放入临时数组中
                temp = new byte[columnByteSize];
                System.arraycopy(readByte, start, temp, 0, columnByteSize);
            }
            //处理临时数组中的数据
            System.out.println("temp.length="+temp.length);
            if(temp.length>0){
                String column = new String(temp, encoding);
                System.out.println(itemIndex+"\t"+column);
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (fc != null) {
                try {
                    fc.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (raf != null) {
                try {
                    raf.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }
    /**
     * 按条读,每一条又有被分成若干个列
     * 举例:a|b|\nc|d\n
     * 分隔符可以是多个字符,如:a::b::c\n\rd::e::f\n\r
     * 严重声明:条分隔符与列分隔符不能有一样的符号,如columnSplit=|,itemSplit=||,它们都有|符号。
     * 因为这个是逐个读的,不太好区分读到||的第一个字符时要不要切分列
     * @param filePath 文件路径
     * @param encoding 字符集
     * @param bufferSize 每次读多少字节
     * @param itemSplit 第条的分隔符
     * @param columnSplit 每条中每列的分隔符
     */
    public static void readByItem(String filePath,String encoding,int bufferSize,String itemSplit,String columnSplit) {
        RandomAccessFile raf = null;
        FileChannel fc = null;
        try {
            raf = new RandomAccessFile(filePath, "r");
            fc = raf.getChannel();
            ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
            byte[] columnSplitByte = columnSplit.getBytes(encoding);
            int columnSplitLen = columnSplitByte.length;
            byte[] itemSplitByte = itemSplit.getBytes(encoding);
            int itemSplitLen = itemSplitByte.length;
            //保存未处理的数据
            byte[] temp = new byte[0];
            int columnIndex = 0;
            int itemIndex = 0;
            while (fc.read(buffer)!= -1) {
                int position = buffer.position();// 读取结束后的位置,相当于读取的长度
                byte[] bs = new byte[position];// 用来存放读取的内容的数组
                buffer.rewind();// 将position设回0,所以你可以重读Buffer中的所有数据,此处如果不设置,无法使用下面的get方法
                buffer.get(bs);// 相当于buffer.get(bs,0,bs.length()):从position初始位置开始相对读,读bs.length个byte,并写入bs[0]到bs[bs.length-1]的区域
                buffer.clear();
                //要加上上次未处理的数据
                int readSize = position + temp.length;
                byte[] readByte = new byte[readSize];
                System.arraycopy(temp, 0, readByte, 0, temp.length);
                System.arraycopy(bs, 0, readByte, temp.length, position);
                int start = 0;
                int columnByteSize = 0;
                byte[] columnByte = new byte[0];
                int columnSplitIndex = 0;
                int itemSplitIndex = 0;
                for (int i = 0; i < readSize; i++) {
                    columnByteSize++;
                    if (readByte[i] == columnSplitByte[columnSplitIndex]) {
                        columnSplitIndex++;
                        if(columnSplitIndex==columnSplitLen){
                            columnByteSize = columnByteSize-columnSplitLen;
                            columnByte = new byte[columnByteSize];
                            System.arraycopy(readByte, start, columnByte, 0, columnByteSize);
                            String column = new String(columnByte, encoding);
                            System.out.println(columnIndex+"\t"+column);
                            columnIndex++;
                            start = i + 1;
                            columnByteSize = 0;
                            columnSplitIndex = 0;
                        }
                    } else {
                        columnSplitIndex = 0;
                    }
                    if (readByte[i] == itemSplitByte[itemSplitIndex]) {
                        itemSplitIndex++;
                        if(itemSplitIndex==itemSplitLen){
                            columnByteSize = columnByteSize-itemSplitLen;
                            columnByte = new byte[columnByteSize];
                            System.arraycopy(readByte, start, columnByte, 0, columnByteSize);
                            String column = new String(columnByte, encoding);
                            System.out.println(columnIndex+"\t"+column);
                            System.out.println(itemIndex+"==========");
                            columnIndex=0;
                            itemIndex++;
                            start = i + 1;
                            columnByteSize = 0;
                            itemSplitIndex = 0;
                        }
                    } else {
                        itemSplitIndex=0;
                    }

                }
                // 把剩余的放入临时数组中
                temp = new byte[columnByteSize];
                System.arraycopy(readByte, start, temp, 0, columnByteSize);
            }
            //处理临时数组中的数据
            System.out.println("temp.length="+temp.length);
            if(temp.length>0){
                String column = new String(temp, encoding);
                System.out.println(columnIndex+"\t"+column);
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (fc != null) {
                try {
                    fc.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (raf != null) {
                try {
                    raf.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }

    public static void main(String[] args) {
        long t1 = System.currentTimeMillis();
        String columnSplit = "|";
        String itemSplit = "\n";
        int bufferSize = 48;
        String encoding = "GBK";
        String filePath = "d:/data/a1.txt";
        //readByLine(filePath,encoding,bufferSize);
        //readByItem(filePath,encoding,bufferSize,itemSplit);
        readByItem(filePath,encoding,bufferSize,itemSplit,columnSplit);
        long t2 = System.currentTimeMillis();
        System.out.println("time==="+(t2-t1));
    }
}

写文件

package com.syz.test.nio;

import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.text.SimpleDateFormat;
import java.util.Date;

public class DataWriter {
    public static void main(String[] args) {
        long t1 = System.currentTimeMillis();
        write();
        long t2 = System.currentTimeMillis();
        System.out.println("time==="+(t2-t1));
    }

    public static void write() {
        SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
        String date = sdf.format(new Date());
        StringBuffer sb = new StringBuffer();
        int bufferSize = 1024;
        String encoding = "UTF-8";
        String filePath = "d:/data/a_"+date+".txt";
        RandomAccessFile raf = null;
        FileChannel fc = null;
        try {
            raf = new RandomAccessFile(filePath, "rws");
            fc = raf.getChannel();
            ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
            int count = 0;
            for(int i=0;i<10;i++){
                sb = new StringBuffer();
                sb.append("20170410_"+i);
                sb.append("|");
                sb.append("127.0.0.1");
                sb.append("|");
                sb.append("测试中文");
                sb.append("|");
                sb.append("www.xxx.com"+date+"_"+i);
                sb.append("|");
                sb.append("");
                sb.append("|");
                sb.append("");
                sb.append("|");
                sb.append("测试中文222abc");
                sb.append("|");
                sb.append("测ha");
                sb.append("|");
                sb.append("");
                sb.append("|");
                sb.append("2017-03-02");
                sb.append("|");
                sb.append("2017-03-02");
                sb.append("|");
                sb.append("2017-03-02");
                sb.append("|");
                sb.append("22haa测试");
                sb.append("|");
                sb.append("abc123");
                sb.append("\n");
                String data = sb.toString();
                byte[] dataByte = data.getBytes(encoding);
                int dataLen = dataByte.length;
                for(int j=0;j<dataLen;j++){
                    if(count==bufferSize){
                        buffer.flip();
                        int a  = fc.write(buffer);
                        System.out.println("a="+a);
                        buffer.clear();
                        count=0;
                    }
                    buffer.put(dataByte[j]);
                    count ++;
                }
            }
            buffer.flip();
            //写入最后一点儿
            int b = fc.write(buffer);
            System.out.println("b="+b);
        } catch (Exception e) {
            e.printStackTrace();
        }finally{
            if(fc!=null){
                try {
                    fc.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if(raf!=null){
                try {
                    raf.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }
}