100万数据从两个集合寻找不同的值优化
using System;
using System.Collections.Generic;
using System.Data;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace ConsoleApplication3
{
class Program
{
static void Main(string[] args)
{
var org_data = "D:\\mesdata\\原始数据.csv";
var mes_dataList = OpenCSV(org_data);
var pis_data = "D:\\mesdata\\PIS数据2.csv";
var pis_dataList = OpenCSV(pis_data);
var dic = new Dictionary<string, string>();
var list = new List<string>();
foreach (DataRow item in pis_dataList.Rows)
{
list.Add(item[0].ToString());
}
DataTable dataTableColor = new DataTable();
dataTableColor.Columns.Add("SeriesNumber", typeof(string));
dataTableColor.Columns.Add("Color", typeof(string));
var mes_dic = new Dictionary<string, string>();
foreach (DataRow item in mes_dataList.Rows)
{
if (!mes_dic.Keys.Contains(item[0].ToString()))
{
mes_dic.Add(item[0].ToString(), item[1].ToString());
}
}
var result= mes_dic.Keys.Except(list);
var resultdic = new Dictionary<string, string>();
foreach (var item in result)
{
var color = string.Empty;
var flag= mes_dic.TryGetValue(item,out color);
resultdic.Add(item, color);
}
foreach (var item in resultdic)
{
DataRow dataRow = dataTableColor.NewRow();
dataRow["SeriesNumber"] = item.Key;
dataRow["Color"] = item.Value;
dataTableColor.Rows.Add(dataRow);
}
Console.WriteLine("end");
Console.ReadKey();
}
/// <summary>
/// 将CSV文件的数据读取到DataTable中
/// </summary>
/// <param name="fileName">CSV文件路径</param>
/// <returns>返回读取了CSV数据的DataTable</returns>
public static DataTable OpenCSV(string filePath)
{
Encoding encoding = GetType(filePath); //Encoding.ASCII;//
DataTable dt = new DataTable();
FileStream fs = new FileStream(filePath, System.IO.FileMode.Open, System.IO.FileAccess.Read);
//StreamReader sr = new StreamReader(fs, Encoding.UTF8);
StreamReader sr = new StreamReader(fs, encoding);
//string fileContent = sr.ReadToEnd();
//encoding = sr.CurrentEncoding;
//记录每次读取的一行记录
string strLine = "";
//记录每行记录中的各字段内容
string[] aryLine = null;
string[] tableHead = null;
//标示列数
int columnCount = 0;
//标示是否是读取的第一行
bool IsFirst = true;
//逐行读取CSV中的数据
while ((strLine = sr.ReadLine()) != null)
{
//strLine = Common.ConvertStringUTF8(strLine, encoding);
//strLine = Common.ConvertStringUTF8(strLine);
if (IsFirst == true)
{
tableHead = strLine.Split(',');
IsFirst = false;
columnCount = tableHead.Length;
//创建列
for (int i = 0; i < columnCount; i++)
{
DataColumn dc = new DataColumn(tableHead[i]);
dt.Columns.Add(dc);
}
}
else
{
aryLine = strLine.Split(',');
DataRow dr = dt.NewRow();
for (int j = 0; j < columnCount; j++)
{
dr[j] = aryLine[j];
}
dt.Rows.Add(dr);
}
}
if (aryLine != null && aryLine.Length > 0)
{
dt.DefaultView.Sort = tableHead[0] + " " + "asc";
}
sr.Close();
fs.Close();
return dt;
}
public static System.Text.Encoding GetType(string FILE_NAME)
{
System.IO.FileStream fs = new System.IO.FileStream(FILE_NAME, System.IO.FileMode.Open,
System.IO.FileAccess.Read);
System.Text.Encoding r = GetType(fs);
fs.Close();
return r;
}
public static System.Text.Encoding GetType(System.IO.FileStream fs)
{
byte[] Unicode = new byte[] { 0xFF, 0xFE, 0x41 };
byte[] UnicodeBIG = new byte[] { 0xFE, 0xFF, 0x00 };
byte[] UTF8 = new byte[] { 0xEF, 0xBB, 0xBF }; //带BOM
System.Text.Encoding reVal = System.Text.Encoding.Default;
System.IO.BinaryReader r = new System.IO.BinaryReader(fs, System.Text.Encoding.Default);
int i;
int.TryParse(fs.Length.ToString(), out i);
byte[] ss = r.ReadBytes(i);
if (IsUTF8Bytes(ss) || (ss[0] == 0xEF && ss[1] == 0xBB && ss[2] == 0xBF))
{
reVal = System.Text.Encoding.UTF8;
}
else if (ss[0] == 0xFE && ss[1] == 0xFF && ss[2] == 0x00)
{
reVal = System.Text.Encoding.BigEndianUnicode;
}
else if (ss[0] == 0xFF && ss[1] == 0xFE && ss[2] == 0x41)
{
reVal = System.Text.Encoding.Unicode;
}
r.Close();
return reVal;
}
private static bool IsUTF8Bytes(byte[] data)
{
int charByteCounter = 1; //计算当前正分析的字符应还有的字节数
byte curByte; //当前分析的字节.
for (int i = 0; i < data.Length; i++)
{
curByte = data[i];
if (charByteCounter == 1)
{
if (curByte >= 0x80)
{
//判断当前
while (((curByte <<= 1) & 0x80) != 0)
{
charByteCounter++;
}
//标记位首位若为非0 则至少以2个1开始 如:110XXXXX...........1111110X
if (charByteCounter == 1 || charByteCounter > 6)
{
return false;
}
}
}
else
{
//若是UTF-8 此时第一位必须为1
if ((curByte & 0xC0) != 0x80)
{
return false;
}
charByteCounter--;
}
}
if (charByteCounter > 1)
{
throw new Exception("非预期的byte格式");
}
return true;
}
}
}
上一篇: 百度发布“超级链”
下一篇: php使用正则表达式进行字符串搜索的方法