RMM分词算法类

RMM分词算法类

1. //RMM分词算法
2. class SplitWord{
3. var $TagDic = Array();
4. var $RankDic = Array();
5. var $SourceStr = '';
6. var $ResultStr = '';
7. var $SplitChar = ' '; //分隔符
8. var $SplitLen = 4; //保留词长度
9. var $MaxLen = 7; //词典最大中文字，这里的数值为字节数组的最大索引
10. var $MinLen = 3; //最小中文字，这里的数值为字节数组的最大索引
12. function SplitWord(){
13. $this->__construct();
14. }
16. function __construct(){
17. //高级分词，预先载入词典以提分词高速度
18. $dicfile = dirname(__FILE__)."/ppldic.csv";
19. $fp = fopen($dicfile,'r'); //读取词库中的词
20. while($line = fgets($fp,256)){
21. $ws = explode(' ',$line); //对词库中的词进行拆分
22. $this->TagDic[$ws[0]] = $ws[1];
23. $this->RankDic[strlen($ws[0])][$ws[0]] = $ws[2];
24. }
25. fclose($fp); //关闭词库文件
26. }
28. //析放资源
29. function Clear(){
30. @fclose($this->QuickDic);
31. }
33. //设置源字符串
34. function SetSource($str){
35. $this->SourceStr = $this->UpdateStr($str);
36. $this->ResultStr = "";
37. }
39. //检查字符串是否不存在中文
40. function NotGBK($str)
41. {
42. if($str=="") return "";
43. if( ord($str[0])>0x80 ) return false;
44. else return true;
45. }
47. //RMM分词算法
48. function SplitRMM($str=""){
49. if($str!="") $this->SetSource($str);
50. if($this->SourceStr=="") return "";
51. $this->SourceStr = $this->UpdateStr($this->SourceStr);
52. $spwords = explode(" ",$this->SourceStr);
53. $spLen = count($spwords);
54. $spc = $this->SplitChar;
55. for($i=($spLen-1);$i>=0;$i--){
56. if($spwords[$i]=="") continue;
57. if($this->NotGBK($spwords[$i])){
58. if(preg_match("/[^0-9\.\+\-]/",$spwords[$i]))
59. { $this->ResultStr = $spwords[$i].$spc.$this->ResultStr; }
60. else
61. {
62. $nextword = "";
63. @$nextword = substr($this->ResultStr,0,strpos($this->ResultStr,""));
64. }
65. }
66. else
67. {
68. $c = $spwords[$i][0].$spwords[$i][1];
69. $n = hexdec(bin2hex($c));
70. if(strlen($spwords[$i]) SplitLen)
71. {
72. }
73. else
74. {
75. $this->ResultStr = $this->RunRMM($spwords[$i]).$spc.$this->ResultStr;
76. }
77. }
78. }
79. return $this->ResultStr;
80. }
81. //对全中文字符串进行逆向匹配方式分解
82. function RunRMM($str){
83. $spc = $this->SplitChar;
84. $spLen = strlen($str);
85. $rsStr = "";
86. $okWord = "";
87. $tmpWord = "";
88. $WordArray = Array();
89. //逆向字典匹配
90. for($i=($spLen-1);$i>=0;){
91. //当i达到最小可能词的时候
92. if($iMinLen){
93. if($i==1){
94. $WordArray[] = substr($str,0,2);
95. }else
96. {
97. $w = substr($str,0,$this->MinLen+1);
98. if($this->IsWord($w)){
99. $WordArray[] = $w;
100. }else{
101. $WordArray[] = substr($str,2,2);
102. $WordArray[] = substr($str,0,2);
103. }
104. }
105. $i = -1; break;
106. }
107. //分析在最小词以上时的情况
108. if($i>=$this->MaxLen) $maxPos = $this->MaxLen;
109. else $maxPos = $i;
110. $isMatch = false;
111. for($j=$maxPos;$j>=0;$j=$j-2){
112. $w = substr($str,$i-$j,$j+1);
113. if($this->IsWord($w)){
114. $WordArray[] = $w;
115. $i = $i-$j-1;
116. $isMatch = true;
117. break;
118. }
119. }
120. }
121. $rsStr = $this->otherword($WordArray);
122. return $rsStr;
123. }
125. function otherword($WordArray){
126. $wlen = count($WordArray)-1; //计算数组的元素个数
127. $rsStr = ""; //初始化变量
128. $spc = $this->SplitChar;
129. for($i=$wlen;$i>=0;$i--)
130. {
131. $rsStr .= $spc.$WordArray[$i]."、"; //将数组为顿号进行拆分
132. }
133. //返回本段分词结果
134. $rsStr = preg_replace("/^".$spc."/","、",$rsStr);
135. return $rsStr;
136. }
138. //判断词典里是否存在某个词
139. function IsWord($okWord){
140. $slen = strlen($okWord);
141. if($slen > $this->MaxLen) return false;
142. else return isset($this->RankDic[$slen][$okWord]);
143. }
145. //整理字符串（对标点符号，中英文混排等初步处理）
146. function UpdateStr($str){
147. $spc = $this->SplitChar;
148. $slen = strlen($str);
149. if($slen==0) return '';
150. $okstr = '';
151. $prechar = 0; // 0-空白 1-英文 2-中文 3-符号
152. for($i=0;$i if(ord($str[$i]) //英文的空白符号
153. if(ord($str[$i]) if($prechar!=0&&$str[$i]!="\r"&&$str[$i]!="\n") $okstr .= $spc;
154. $prechar=0;
155. continue;
156. }else if(preg_match("/[^0-9a-zA-Z@\.%#:\\&_-]/",$str[$i])){
157. if($prechar==0){ $okstr .= $str[$i]; $prechar=3;}
158. else{ $okstr .= $spc.$str[$i]; $prechar=3;}
159. }else{
160. if($prechar==2||$prechar==3)
161. { $okstr .= $spc.$str[$i]; $prechar=1;}
162. else
163. {
164. if(preg_match("/@#%:/",$str[$i])){ $okstr .= $str[$i]; $prechar=3; }
165. else { $okstr .= $str[$i]; $prechar=1; }
166. }
167. }
168. }
169. else{
170. //如果上一个字符为非中文和非空格，则加一个空格
171. if($prechar!=0 && $prechar!=2) $okstr .= $spc;
172. //如果中文字符
173. if(isset($str[$i+1])){
174. $c = $str[$i].$str[$i+1];
176. $n = hexdec(bin2hex($c));
177. if($n 0xAA40){
178. if($prechar!=0) $okstr .= $spc.$c;
179. else $okstr .= $c;
180. $prechar = 3;
181. }
182. else{
183. $okstr .= $c;
184. $prechar = 2;
185. }
186. $i++;
187. }
188. }
189. }
190. return $okstr;
191. }
192. }
194. // 调用
195. $split=new SplitWord();
197. echo $split->SplitRMM("php搜索技术");
199. // 注意 ppldic.csv 词典的格式是 词语+空格+编号+n

复制代码