turbo_split.php 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493
  1. <?php
  2. require_once '../public/casesuf.inc';
  3. //require_once '../studio/dict_find_un.inc';
  4. //require_once '../studio/sandhi.php';
  5. require_once "../path.php";
  6. require_once "../public/_pdo.php";
  7. // open word part db
  8. global $dbh;
  9. $dns = "" . _FILE_DB_PART_;
  10. $dbh = new PDO($dns, "", "", array(PDO::ATTR_PERSISTENT => true));
  11. $dbh->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_WARNING);
  12. global $path;
  13. global $confidence;
  14. global $result;
  15. global $part;
  16. define("MAX_RESULT",100);
  17. $part = array();
  18. $path[] = array("", 0);
  19. $path[] = array("", 0);
  20. $path[] = array("", 0);
  21. $path[] = array("", 0);
  22. $path[] = array("", 0);
  23. $path[] = array("", 0);
  24. $path[] = array("", 0);
  25. $path[] = array("", 0);
  26. $path[] = array("", 0);
  27. $path[] = array("", 0);
  28. $path[] = array("", 0);
  29. $path[] = array("", 0);
  30. $path[] = array("", 0);
  31. $path[] = array("", 0);
  32. $path[] = array("", 0);
  33. $path[] = array("", 0);
  34. $path[] = array("", 0);
  35. global $sandhi;
  36. //sandhi rules table 语尾表
  37. $sandhi[] = array("a" => "", "b" => "", "c" => "", "len" => 0, "adj_len" => 0, "advance" => false);
  38. $sandhi[] = array("a" => "a", "b" => "a", "c" => "ā", "len" => 1, "adj_len" => 0, "advance" => false);
  39. $sandhi[] = array("a" => "ā", "b" => "ā", "c" => "ā", "len" => 1, "adj_len" => 0, "advance" => false);
  40. $sandhi[] = array("a" => "a", "b" => "ā", "c" => "ā", "len" => 1, "adj_len" => 0, "advance" => false);
  41. $sandhi[] = array("a" => "ā", "b" => "a", "c" => "ā", "len" => 1, "adj_len" => 0, "advance" => false);
  42. $sandhi[] = array("a" => "a", "b" => "e", "c" => "e", "len" => 1, "adj_len" => 0, "advance" => false);
  43. $sandhi[] = array("a" => "a", "b" => "i", "c" => "i", "len" => 1, "adj_len" => 0, "advance" => false);
  44. $sandhi[] = array("a" => "a", "b" => "o", "c" => "o", "len" => 1, "adj_len" => 0, "advance" => false);
  45. $sandhi[] = array("a" => "a", "b" => "u", "c" => "o", "len" => 1, "adj_len" => 0, "advance" => false);
  46. $sandhi[] = array("a" => "u", "b" => "a", "c" => "o", "len" => 1, "adj_len" => 0, "advance" => false);
  47. $sandhi[] = array("a" => "u", "b" => "u", "c" => "ū", "len" => 1, "adj_len" => 0, "advance" => false);
  48. $sandhi[] = array("a" => "a", "b" => "u", "c" => "u", "len" => 1, "adj_len" => 0, "advance" => false);
  49. $sandhi[] = array("a" => "a", "b" => "ī", "c" => "ī", "len" => 1, "adj_len" => 0, "advance" => false);
  50. $sandhi[] = array("a" => "a", "b" => "ū", "c" => "ū", "len" => 1, "adj_len" => 0, "advance" => false);
  51. $sandhi[] = array("a" => "a", "b" => "i", "c" => "e", "len" => 1, "adj_len" => 0, "advance" => false);
  52. $sandhi[] = array("a" => "e", "b" => "a", "c" => "e", "len" => 1, "adj_len" => 0, "advance" => false);
  53. $sandhi[] = array("a" => "i", "b" => "i", "c" => "ī", "len" => 1, "adj_len" => 0, "advance" => false);
  54. $sandhi[] = array("a" => "i", "b" => "e", "c" => "e", "len" => 1, "adj_len" => 0, "advance" => false);
  55. $sandhi[] = array("a" => "i", "b" => "a", "c" => "ya", "len" => 2, "adj_len" => 0, "advance" => false);
  56. $sandhi[] = array("a" => "a", "b" => "atth", "c" => "atth", "len" => 4, "adj_len" => 0, "advance" => false);
  57. $sandhi[] = array("a" => "taṃ", "b" => "n", "c" => "tann", "len" => 4, "adj_len" => 0, "advance" => false);
  58. $sandhi[] = array("a" => "[ṃ]", "b" => "api", "c" => "mpi", "len" => 3, "adj_len" => 0, "advance" => false);
  59. $sandhi[] = array("a" => "[ṃ]", "b" => "eva", "c" => "meva", "len" => 4, "adj_len" => 0, "advance" => false);
  60. $sandhi[] = array("a" => "[o]", "b" => "iva", "c" => "ova", "len" => 3, "adj_len" => 0, "advance" => false);
  61. $sandhi[] = array("a" => "o", "b" => "a", "c" => "o", "len" => 1, "adj_len" => 0, "advance" => false);
  62. $sandhi[] = array("a" => "a", "b" => "ādi", "c" => "ādi", "len" => 3, "adj_len" => 0, "advance" => false);
  63. $sandhi[] = array("a" => "a[ānaṃ]", "b" => "a", "c" => "ānama", "len" => 5, "adj_len" => 0, "advance" => false);
  64. $sandhi[] = array("a" => "a", "b" => "iti", "c" => "āti", "len" => 3, "adj_len" => 0, "advance" => false);
  65. $sandhi[] = array("a" => "[ṃ]", "b" => "ca", "c" => "ñca", "len" => 3, "adj_len" => 0, "advance" => false);
  66. $sandhi[] = array("a" => "[ṃ]", "b" => "iti", "c" => "nti", "len" => 3, "adj_len" => 0, "advance" => false);
  67. $sandhi[] = array("a" => "[ṃ]", "b" => "a", "c" => "ma", "len" => 2, "adj_len" => 0, "advance" => false);
  68. $sandhi[] = array("a" => "ṃ", "b" => "a", "c" => "m", "len" => 1, "adj_len" => 0, "advance" => false);
  69. $sandhi[] = array("a" => "[ṃ]", "b" => "ā", "c" => "mā", "len" => 2, "adj_len" => 0, "advance" => false);
  70. $sandhi[] = array("a" => "[ṃ]", "b" => "u", "c" => "mu", "len" => 2, "adj_len" => 0, "advance" => false);
  71. $sandhi[] = array("a" => "[ṃ]", "b" => "h", "c" => "ñh", "len" => 2, "adj_len" => 0, "advance" => false);
  72. $sandhi[] = array("a" => "ā", "b" => "[ṃ]", "c" => "am", "len" => 2, "adj_len" => 0, "advance" => false);
  73. $sandhi[] = array("a" => "ī", "b" => "[ṃ]", "c" => "im", "len" => 2, "adj_len" => 0, "advance" => false);
  74. $sandhi[] = array("a" => "ati", "b" => "tabba", "c" => "atabba", "len" => 6, "adj_len" => 0, "advance" => false);
  75. $sandhi[] = array("a" => "ati", "b" => "tabba", "c" => "itabba", "len" => 6, "adj_len" => 0, "advance" => false);
  76. $sandhi[] = array("a" => "iti", "b" => "a", "c" => "icca", "len" => 4, "adj_len" => 0, "advance" => false);
  77. $sandhi[] = array("a" => "uṃ", "b" => "a", "c" => "uma", "len" => 3, "adj_len" => 0, "advance" => false);
  78. $sandhi[] = array("a" => "u[ūnaṃ]", "b" => "a", "c" => "ūnama", "len" => 5, "adj_len" => 0, "advance" => false);
  79. $sandhi[] = array("a" => "ī[īnaṃ]", "b" => "a", "c" => "īnama", "len" => 5, "adj_len" => 0, "advance" => false);
  80. $sandhi[] = array("a" => "su", "b" => "a", "c" => "sva", "len" => 3, "adj_len" => 0, "advance" => false);
  81. #other sandhi rule. can be use but program will be slow down
  82. #其他连音规则,如果使用则会让程序运行变慢
  83. $sandhi[] = array("a" => "ā", "b" => "iti", "c" => "āti", "len" => 3, "adj_len" => 0, "advance" => false);
  84. $sandhi[] = array("a" => "a", "b" => "iti", "c" => "āti", "len" => 3, "adj_len" => 0, "advance" => false);
  85. $sandhi[] = array("a" => "e", "b" => "iti", "c" => "eti", "len" => 3, "adj_len" => 0, "advance" => false);
  86. $sandhi[] = array("a" => "ī", "b" => "iti", "c" => "īti", "len" => 3, "adj_len" => 0, "advance" => false);
  87. $sandhi[] = array("a" => "i", "b" => "iti", "c" => "īti", "len" => 3, "adj_len" => 0, "advance" => false);
  88. $sandhi[] = array("a" => "o", "b" => "iti", "c" => "oti", "len" => 3, "adj_len" => 0, "advance" => false);
  89. $sandhi[] = array("a" => "ū", "b" => "iti", "c" => "ūti", "len" => 3, "adj_len" => 0, "advance" => false);
  90. $sandhi[] = array("a" => "u", "b" => "iti", "c" => "ūti", "len" => 3, "adj_len" => 0, "advance" => false);
  91. $sandhi[] = array("a" => "ṃ", "b" => "iti", "c" => "nti", "len" => 3, "adj_len" => 0, "advance" => false);
  92. $sandhi[] = array("a" => "ṃ", "b" => "ca", "c" => "ñca", "len" => 3, "adj_len" => 0, "advance" => false);
  93. $sandhi[] = array("a" => "ṃ", "b" => "cāti", "c" => "ñcāti", "len" => 5, "adj_len" => 0, "advance" => false);
  94. $sandhi[] = array("a" => "ṃ", "b" => "cet", "c" => "ñcet", "len" => 4, "adj_len" => 0, "advance" => false);
  95. /*
  96. $sandhi[] = array("a" => "a", "b" => "eva", "c" => "eva", "len" => 3, "adj_len" => 0, "advance" => false);
  97. $sandhi[] = array("a" => "ā", "b" => "eva", "c" => "āyeva", "len" => 5, "adj_len" => 0, "advance" => false);
  98. $sandhi[] = array("a" => "e", "b" => "eva", "c" => "eva", "len" => 3, "adj_len" => 0, "advance" => false);
  99. $sandhi[] = array("a" => "i", "b" => "eva", "c" => "yeva", "len" => 4, "adj_len" => 0, "advance" => false);
  100. $sandhi[] = array("a" => "ī", "b" => "eva", "c" => "iyeva", "len" => 5, "adj_len" => 0, "advance" => false);
  101. $sandhi[] = array("a" => "ī", "b" => "eva", "c" => "īyeva", "len" => 5, "adj_len" => 0, "advance" => false);
  102. $sandhi[] = array("a" => "o", "b" => "eva", "c" => "ova", "len" => 3, "adj_len" => 0, "advance" => false);
  103. $sandhi[] = array("a" => "u", "b" => "eva", "c" => "veva", "len" => 3, "adj_len" => 0, "advance" => false);
  104. $sandhi[] = array("a" => "a", "b" => "eva", "c" => "evā", "len" => 3, "adj_len" => 0, "advance" => false);
  105. $sandhi[] = array("a" => "e", "b" => "eva", "c" => "evā", "len" => 3, "adj_len" => 0, "advance" => false);
  106. $sandhi[] = array("a" => "i", "b" => "eva", "c" => "yevā", "len" => 4, "adj_len" => 0, "advance" => false);
  107. $sandhi[] = array("a" => "ī", "b" => "eva", "c" => "yevā", "len" => 4, "adj_len" => 0, "advance" => false);
  108. $sandhi[] = array("a" => "ī", "b" => "eva", "c" => "iyevā", "len" => 4, "adj_len" => 0, "advance" => false);
  109. $sandhi[] = array("a" => "ī", "b" => "eva", "c" => "īyevā", "len" => 4, "adj_len" => 0, "advance" => false);
  110. $sandhi[] = array("a" => "o", "b" => "eva", "c" => "ovā", "len" => 4, "adj_len" => 0, "advance" => false);
  111. $sandhi[] = array("a" => "ā", "b" => "api", "c" => "āpi", "len" => 3, "adj_len" => 0, "advance" => false);
  112. $sandhi[] = array("a" => "a", "b" => "api", "c" => "āpi", "len" => 3, "adj_len" => 0, "advance" => false);
  113. $sandhi[] = array("a" => "e", "b" => "api", "c" => "epi", "len" => 3, "adj_len" => 0, "advance" => false);
  114. $sandhi[] = array("a" => "ī", "b" => "api", "c" => "īpi", "len" => 3, "adj_len" => 0, "advance" => false);
  115. $sandhi[] = array("a" => "i", "b" => "api", "c" => "īpi", "len" => 3, "adj_len" => 0, "advance" => false);
  116. $sandhi[] = array("a" => "o", "b" => "api", "c" => "opi", "len" => 3, "adj_len" => 0, "advance" => false);
  117. $sandhi[] = array("a" => "ū", "b" => "api", "c" => "ūpi", "len" => 3, "adj_len" => 0, "advance" => false);
  118. $sandhi[] = array("a" => "u", "b" => "api", "c" => "ūpi", "len" => 3, "adj_len" => 0, "advance" => false);
  119. $sandhi[] = array("a" => "u", "b" => "api", "c" => "upi", "len" => 3, "adj_len" => 0, "advance" => false);
  120. $sandhi[] = array("a" => "ṃ", "b" => "api", "c" => "mpi", "len" => 3, "adj_len" => 0, "advance" => false);
  121. */
  122. $sandhi[] = array("a" => "a", "b" => "a", "c" => "a", "len" => 1, "adj_len" => -1, "advance" => true);
  123. $sandhi[] = array("a" => "ī", "b" => "", "c" => "i", "len" => 1, "adj_len" => 0, "advance" => true);
  124. function split_diphthong($word)
  125. {
  126. //diphthong table双元音表
  127. $search = array('aa', 'ae', 'ai', 'ao', 'au', 'aā', 'aī', 'aū', 'ea', 'ee', 'ei', 'eo', 'eu', 'eā', 'eī', 'eū', 'ia', 'ie', 'ii', 'io', 'iu', 'iā', 'iī', 'iū', 'oa', 'oe', 'oi', 'oo', 'ou', 'oā', 'oī', 'oū', 'ua', 'ue', 'ui', 'uo', 'uu', 'uā', 'uī', 'uū', 'āa', 'āe', 'āi', 'āo', 'āu', 'āā', 'āī', 'āū', 'īa', 'īe', 'īi', 'īo', 'īu', 'īā', 'īī', 'īū', 'ūa', 'ūe', 'ūi', 'ūo', 'ūu', 'ūā', 'ūī', 'ūū');
  128. $replace = array('a-a', 'a-e', 'a-i', 'a-o', 'a-u', 'a-ā', 'a-ī', 'a-ū', 'e-a', 'e-e', 'e-i', 'e-o', 'e-u', 'e-ā', 'e-ī', 'e-ū', 'i-a', 'i-e', 'i-i', 'i-o', 'i-u', 'i-ā', 'i-ī', 'i-ū', 'o-a', 'o-e', 'o-i', 'o-o', 'o-u', 'o-ā', 'o-ī', 'o-ū', 'u-a', 'u-e', 'u-i', 'u-o', 'u-u', 'u-ā', 'u-ī', 'u-ū', 'ā-a', 'ā-e', 'ā-i', 'ā-o', 'ā-u', 'ā-ā', 'ā-ī', 'ā-ū', 'ī-a', 'ī-e', 'ī-i', 'ī-o', 'ī-u', 'ī-ā', 'ī-ī', 'ī-ū', 'ū-a', 'ū-e', 'ū-i', 'ū-o', 'ū-u', 'ū-ā', 'ū-ī', 'ū-ū');
  129. //将双元音拆开
  130. //step 1 : split at diphthong . ~aa~ -> ~a-a~
  131. $word1 = str_replace($search, $replace, $word);
  132. //按连字符拆开处理
  133. $arrword = str_getcsv($word1, "-");
  134. return $arrword;
  135. }
  136. /*
  137. 用于数组连接字符串
  138. */
  139. function myfunction($v1, $v2)
  140. {
  141. return $v1 . "+" . $v2;
  142. }
  143. function microtime_float()
  144. {
  145. list($usec, $sec) = explode(" ", microtime());
  146. return ((float) $usec + (float) $sec);
  147. }
  148. function dict_lookup($word)
  149. {
  150. if (strlen($word) <= 1) {
  151. return 0;
  152. }
  153. global $case;
  154. global $dbh;
  155. $str = strstr($word, "[");
  156. if ($str === false) {
  157. $search = $word;
  158. } else {
  159. $search = $str;
  160. }
  161. $query = "SELECT weight from part where word = ? ";
  162. $stmt = $dbh->prepare($query);
  163. $stmt->execute(array($search));
  164. $row = $stmt->fetch(PDO::FETCH_NUM);
  165. if ($row) {
  166. return array($row[0],0);
  167. } else {
  168. //去除尾查
  169. $newWord = array();
  170. for ($row = 0; $row < count($case); $row++) {
  171. $len = mb_strlen($case[$row][1], "UTF-8");
  172. $end = mb_substr($word, 0 - $len, null, "UTF-8");
  173. if ($end == $case[$row][1]) {
  174. $base = mb_substr($word, 0, mb_strlen($word, "UTF-8") - $len, "UTF-8") . $case[$row][0];
  175. if ($base != $word) {
  176. $newWord[$base] = mb_strlen($case[$row][1],"UTF-8");
  177. }
  178. }
  179. }
  180. #找到最高频的base
  181. $base_weight = 0;
  182. $len = 0;
  183. foreach ($newWord as $x => $x_value) {
  184. $query = "SELECT weight from part where word = ? ";
  185. $stmt = $dbh->prepare($query);
  186. $stmt->execute(array($x));
  187. $row = $stmt->fetch(PDO::FETCH_NUM);
  188. if ($row) {
  189. if ($row[0] > $base_weight) {
  190. $base_weight = $row[0];
  191. $len=$x_value;
  192. }
  193. }
  194. }
  195. return array($base_weight,$len);
  196. }
  197. }
  198. /*
  199. 查找某个单词是否在现有词典出现
  200. 返回信心指数
  201. look up single word in dictionary vocabulary
  202. return the confidence value
  203. */
  204. function isExsit($word, $adj_len = 0)
  205. {
  206. global $auto_split_times;
  207. global $result;
  208. global $part;
  209. global $confidence;
  210. $auto_split_times++;
  211. if (isset($_POST["debug"])) {
  212. echo "<div>正在查询:{$word}</div>";
  213. }
  214. $isFound = false;
  215. $count = 0;
  216. if (isset($part["{$word}"])) {
  217. $word_count = $part["{$word}"][0];
  218. $case_len = $part["{$word}"][1];
  219. if ($word_count > 0) {
  220. $isFound = true;
  221. $count = $word_count + 1;
  222. }
  223. } else {
  224. $db = dict_lookup($word);
  225. $word_count = $db[0];
  226. $case_len = $db[1];
  227. //加入查询缓存
  228. $part["{$word}"] = $db;
  229. if ($word_count > 0) {
  230. if (isset($_POST["debug"])) {
  231. echo "查到:{$word}:{$word_count}个\n";
  232. }
  233. $isFound = true;
  234. $count = $word_count + 1;
  235. }
  236. }
  237. //fomular of confidence value 信心值计算公式
  238. if ($isFound) {
  239. if (isset($confidence["{$word}"])) {
  240. $cf = $confidence["{$word}"];
  241. } else {
  242. $len = mb_strlen($word, "UTF-8") + $adj_len;
  243. $len_correct = 1.2;
  244. $count2 = 1.1 + pow($count, 1.18);
  245. $conf_num = pow(1 / $count2, pow(($len - 0.5), $len_correct));
  246. //$cf = round(1 / (1 + 640 * $conf_num), 9);
  247. $cf = round((1-0.02*$case_len) / (1 + 640 * $conf_num), 9);
  248. $confidence["{$word}"] = $cf;
  249. if (isset($_POST["debug"])) {
  250. echo "信心指数:{$word}:{$cf}\n";
  251. }
  252. }
  253. return ($cf);
  254. } else {
  255. return (-1);
  256. }
  257. }
  258. /*
  259. 核心拆分函数
  260. $strWord, word to be look up 要查询的词
  261. $deep, 当前递归深度
  262. $express=true, 快速查询
  263. $adj_len=0 长度校正系数
  264. $c_threshhold 信心指数阈值
  265. */
  266. function mySplit2($strWord, $deep = 0, $express = false, $adj_len = 0, $c_threshhold = 0.8, $w_threshhold = 0.8, $forward = true, $sandhi_advance = false)
  267. {
  268. global $path;
  269. global $result;
  270. global $sandhi;
  271. $output = array();
  272. //达到最大搜索深度,返回
  273. if ($deep >= 16) {
  274. $word = "";
  275. $cf = 1.0;
  276. for ($i = 0; $i < $deep; $i++) {
  277. if (!empty($path[$i][0])) {
  278. $word .= $path[$i][0] . "+";
  279. if (isset($_POST["debug"])) {
  280. $word .= "(" . $path[$i][1] . ")-";
  281. }
  282. $cf = $cf * $path[$i][1];
  283. }
  284. }
  285. $len = pow(mb_strlen($strWord, "UTF-8"), 3);
  286. $cf += (0 - $len) / ($len + 150);
  287. $word .= "{$strWord}";
  288. if ($forward == true) {
  289. $result[$word] = $cf;
  290. return;
  291. } else {
  292. $reverseWord = word_reverse($word);
  293. $result[$reverseWord] = $cf;
  294. return;
  295. }
  296. }
  297. //直接找到
  298. $confidence = isExsit($strWord, $adj_len);
  299. if ($confidence >= 0) {
  300. $output[] = array($strWord, "", $confidence);
  301. } else {
  302. $confidence = isExsit("[" . $strWord . "]");
  303. if ($confidence >= 0) {
  304. $output[] = array("[" . $strWord . "]", "", $confidence);
  305. }
  306. }
  307. //如果开头有双辅音,去掉第一个辅音。因为巴利语中没有以双辅音开头的单词。
  308. $doubleword = "kkggccjjṭṭḍḍttddppbb";
  309. if (mb_strlen($strWord, "UTF-8") > 2) {
  310. $left2 = mb_substr($strWord, 0, 2, "UTF-8");
  311. if (mb_strpos($doubleword, $left2, 0, "UTF-8") !== false) {
  312. $strWord = mb_substr($strWord, 1, null, "UTF-8");
  313. }
  314. }
  315. $len = mb_strlen($strWord, "UTF-8");
  316. if ($len > 2) {
  317. if ($forward) {
  318. #正向切
  319. for ($i = $len; $i > 1; $i--) {
  320. foreach ($sandhi as $key => $row) {
  321. if ($sandhi_advance == false && $row["advance"] == true) {
  322. continue;
  323. }
  324. if (mb_substr($strWord, $i - $row["len"], $row["len"], "UTF-8") == $row["c"]) {
  325. $str1 = mb_substr($strWord, 0, $i - $row["len"], "UTF-8") . $row["a"];
  326. $str2 = $row["b"] . mb_substr($strWord, $i, null, "UTF-8");
  327. $confidence = isExsit($str1, $adj_len);
  328. if ($row["advance"] == true) {
  329. $confidence = $confidence * 0.99;
  330. }
  331. if ($confidence > $c_threshhold) {
  332. $output[] = array($str1, $str2, $confidence, $row["adj_len"]);
  333. if (isset($_POST["debug"])) {
  334. echo "插入:{$str1}\n";
  335. }
  336. if ($express) {
  337. break;
  338. }
  339. }
  340. }
  341. }
  342. }
  343. } else {
  344. #反向切
  345. for ($i = 1; $i < $len - 1; $i++) {
  346. foreach ($sandhi as $key => $row) {
  347. if ($sandhi_advance == false && $row["advance"] == true) {
  348. continue;
  349. }
  350. if (mb_substr($strWord, $i, $row["len"], "UTF-8") == $row["c"]) {
  351. $str1 = mb_substr($strWord, 0, $i, "UTF-8") . $row["a"];
  352. $str2 = $row["b"] . mb_substr($strWord, $i + $row["len"], null, "UTF-8");
  353. $confidence = isExsit($str2, $adj_len);
  354. if ($row["advance"] == true) {
  355. $confidence = $confidence * 0.99;
  356. }
  357. if ($confidence > $c_threshhold) {
  358. $output[] = array($str2, $str1, $confidence, $row["adj_len"]);
  359. if (isset($_POST["debug"])) {
  360. echo "插入:{$str2}\n";
  361. }
  362. if ($express) {
  363. break;
  364. }
  365. }
  366. }
  367. }
  368. }
  369. }
  370. }
  371. $word = "";
  372. if (count($output) > 0) {
  373. foreach ($output as $part) {
  374. $checked = $part[0];
  375. $remainder = $part[1];
  376. $path[$deep][0] = $checked;
  377. $path[$deep][1] = $part[2];
  378. if (empty($remainder)) {
  379. #全切完了
  380. $word = "";
  381. $cf = 1.0;
  382. for ($i = 0; $i < $deep; $i++) {
  383. $word .= $path[$i][0];
  384. if (isset($_POST["debug"])) {
  385. $word .= "(" . $path[$i][1] . ")";
  386. }
  387. $word .= "+";
  388. $cf = $cf * $path[$i][1];
  389. }
  390. if (isset($_POST["debug"])) {
  391. $word .= $checked . "({$part[2]})";
  392. } else {
  393. $word .= $checked;
  394. }
  395. $cf = $cf * $part[2];
  396. if ($cf > $w_threshhold) {
  397. if ($forward == true) {
  398. $result[$word] = $cf;
  399. return;
  400. } else {
  401. $reverseWord = word_reverse($word);
  402. $result[$reverseWord] = $cf;
  403. return;
  404. }
  405. }
  406. } else {
  407. #接着切
  408. mySplit2($remainder, ($deep + 1), $express, $adj_len, $c_threshhold, $w_threshhold, $forward, $sandhi_advance);
  409. }
  410. }
  411. } else {
  412. #尾巴查不到了
  413. $word = "";
  414. $cf = 1.0;
  415. for ($i = 0; $i < $deep; $i++) {
  416. $word .= $path[$i][0];
  417. if (isset($_POST["debug"])) {
  418. $word .= "(" . $path[$i][1] . ")";
  419. }
  420. $word .= "+";
  421. $cf = $cf * $path[$i][1];
  422. }
  423. $len = pow(mb_strlen($strWord, "UTF-8"), 3);
  424. if ($forward) {
  425. $cf =(1-$cf) * $len / ($len + 150);
  426. } else
  427. $cf =(1-$cf) * $len / ($len + 5);
  428. }
  429. if (isset($_POST["debug"])) {
  430. $word = $word.$strWord . "(0)";
  431. } else {
  432. $word = $word .$strWord;
  433. }
  434. if ($cf > $w_threshhold) {
  435. if ($forward == true) {
  436. $result[$word] = $cf;
  437. return;
  438. } else {
  439. $reverseWord = word_reverse($word);
  440. $result[$reverseWord] = $cf;
  441. return;
  442. }
  443. }
  444. }
  445. function word_reverse($word)
  446. {
  447. $reverse = array();
  448. $newword = explode("+", $word);
  449. $len = count($newword);
  450. if ($len > 0) {
  451. for ($i = $len - 1; $i >= 0; $i--) {
  452. # code...
  453. $reverse[] = $newword[$i];
  454. }
  455. $output = implode("+", $reverse);
  456. return $output;
  457. } else {
  458. return $word;
  459. }
  460. }