php简单去除大型文本重复

error_reporting(0);
@ini_set('memory_limit','-1');
set_time_limit(0);
echo" 去除文本重复工具"."\r\n\r\n";
echo"\n"."输入要整理的文件:"."\n";
$dic=trim(fgets(STDIN));
/*while (!feof($dic)){
$file[]=stream_get_line($fp,65535,"\r\n");
}
*/

$file=file($dic);
$array=preg_replace('/($\s*$)|(^\s*^)/m','',$file); //消除空行
$new_array=array_values(array_unique($array));//消除重复行
$new_filename="new_". basename($dic);

if(file_put_contents("$new_filename",join("\r\n",$new_array))){

$num=count($file);
$new_num=count($new_array);
$counts=$num-$new_num;
$files=dirname(__FILE__).DIRECTORY_SEPARATOR.$new_filename;

echo +----------------------------------------------+
| [+] 去除重复完毕! www.2cto.com |
| [+] 整理后的文件为:$files |
| [+] 原始字典数量:$num 行 |
| [+] 整理后为:$new_num 行 |
| [+] 共替换了$counts 行 |
+----------------------------------------------+
INFO;

}

else{
echo"------------------------------------------"."\r\n";
echo"[*] 错误!"."\r\n\r\n";
echo"[*] 找不到文件!请检查输入路径是否存在!"."\r\n";
echo"------------------------------------------"."\r\n";
exit();
}


?>