C#处理大批量数据

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
 
namespace ConsoleApi
{
    /// <summary>
    /// 多线程批量数据处理
    /// </summary>
    public class MultithreadBatchDataProcessing
    {
        //线程安全队列
        private ConcurrentQueue<ResponseModel> queue = new ConcurrentQueue<ResponseModel>();
 
        /// <summary>
        /// 模拟设置数据
        /// </summary>
        public void SetData()
        {
            Console.WriteLine($"开始数据设置，时间：{DateTime.Now}；");
            for (int i = 0; i < 10000; i++)
            {
                var model = new ResponseModel { Code=i, Msg=$"第{i+1}次循环", Data=$"产生随机数：{new Random().Next(1000,10000)}" };
                queue.Enqueue(model); //模拟数据入队
                Thread.Sleep(1);    //这里是随机数生成时需要
            }
            Console.WriteLine($"10000条数据设置完毕！时间：{DateTime.Now}；");
        }
 
        /// <summary>
        /// 多线程处理数据
        /// </summary>
        public void MultitDataProcessing()
        {
            int threadCount = 10; //开启10个线程
            for (int i = 0; i < threadCount; i++)
            {
                string fileName = $"task{i}.txt";
                //开启新线程
                Task.Factory.StartNew(() =>
                {
                    var sb = new StringBuilder();
                    int j = 0;
                    //数据循环出队
                    while (queue.TryDequeue(out ResponseModel model))
                    {
                        //处理数据
                        if (model != null)
                            sb.AppendLine($"==》Code={model.Code},Msg={model.Msg},Data={model.Data}");
 
                        if (j % 100 == 0 || (queue.Count.Equals(0) && j < 100))
                        {   
                            Console.WriteLine($"每100条输出一次控制台，并暂停100毫秒, 第{i}次文件：{fileName}");
                            Console.WriteLine(sb.ToString());
                            sb = new StringBuilder();
                            Thread.Sleep(100);
                        }
                        j++;
                    }
                });
            }
        }
    }
}