turbo_split.php 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480
  1. <?php
  2. require_once '../public/casesuf.inc';
  3. //require_once '../studio/dict_find_un.inc';
  4. //require_once '../studio/sandhi.php';
  5. require_once "../path.php";
  6. require_once "../public/_pdo.php";
  7. // open word part db
  8. global $dbh;
  9. $dns = "sqlite:" . _FILE_DB_PART_;
  10. $dbh = new PDO($dns, "", "", array(PDO::ATTR_PERSISTENT => true));
  11. $dbh->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_WARNING);
  12. global $path;
  13. global $confidence;
  14. global $result;
  15. global $part;
  16. $part = array();
  17. $path[] = array("", 0);
  18. $path[] = array("", 0);
  19. $path[] = array("", 0);
  20. $path[] = array("", 0);
  21. $path[] = array("", 0);
  22. $path[] = array("", 0);
  23. $path[] = array("", 0);
  24. $path[] = array("", 0);
  25. $path[] = array("", 0);
  26. $path[] = array("", 0);
  27. $path[] = array("", 0);
  28. $path[] = array("", 0);
  29. $path[] = array("", 0);
  30. $path[] = array("", 0);
  31. $path[] = array("", 0);
  32. $path[] = array("", 0);
  33. $path[] = array("", 0);
  34. global $sandhi;
  35. //sandhi rules table 语尾表
  36. $sandhi[] = array("a" => "", "b" => "", "c" => "", "len" => 0, "adj_len" => 0, "advance" => false);
  37. $sandhi[] = array("a" => "a", "b" => "a", "c" => "ā", "len" => 1, "adj_len" => 0, "advance" => false);
  38. $sandhi[] = array("a" => "ā", "b" => "ā", "c" => "ā", "len" => 1, "adj_len" => 0, "advance" => false);
  39. $sandhi[] = array("a" => "a", "b" => "ā", "c" => "ā", "len" => 1, "adj_len" => 0, "advance" => false);
  40. $sandhi[] = array("a" => "ā", "b" => "a", "c" => "ā", "len" => 1, "adj_len" => 0, "advance" => false);
  41. $sandhi[] = array("a" => "a", "b" => "e", "c" => "e", "len" => 1, "adj_len" => 0, "advance" => false);
  42. $sandhi[] = array("a" => "a", "b" => "i", "c" => "i", "len" => 1, "adj_len" => 0, "advance" => false);
  43. $sandhi[] = array("a" => "a", "b" => "o", "c" => "o", "len" => 1, "adj_len" => 0, "advance" => false);
  44. $sandhi[] = array("a" => "a", "b" => "u", "c" => "o", "len" => 1, "adj_len" => 0, "advance" => false);
  45. $sandhi[] = array("a" => "u", "b" => "a", "c" => "o", "len" => 1, "adj_len" => 0, "advance" => false);
  46. $sandhi[] = array("a" => "u", "b" => "u", "c" => "ū", "len" => 1, "adj_len" => 0, "advance" => false);
  47. $sandhi[] = array("a" => "a", "b" => "u", "c" => "u", "len" => 1, "adj_len" => 0, "advance" => false);
  48. $sandhi[] = array("a" => "a", "b" => "ī", "c" => "ī", "len" => 1, "adj_len" => 0, "advance" => false);
  49. $sandhi[] = array("a" => "a", "b" => "ū", "c" => "ū", "len" => 1, "adj_len" => 0, "advance" => false);
  50. $sandhi[] = array("a" => "a", "b" => "i", "c" => "e", "len" => 1, "adj_len" => 0, "advance" => false);
  51. $sandhi[] = array("a" => "e", "b" => "a", "c" => "e", "len" => 1, "adj_len" => 0, "advance" => false);
  52. $sandhi[] = array("a" => "i", "b" => "i", "c" => "ī", "len" => 1, "adj_len" => 0, "advance" => false);
  53. $sandhi[] = array("a" => "i", "b" => "e", "c" => "e", "len" => 1, "adj_len" => 0, "advance" => false);
  54. $sandhi[] = array("a" => "i", "b" => "a", "c" => "ya", "len" => 2, "adj_len" => 0, "advance" => false);
  55. $sandhi[] = array("a" => "a", "b" => "atth", "c" => "atth", "len" => 4, "adj_len" => 0, "advance" => false);
  56. $sandhi[] = array("a" => "taṃ", "b" => "n", "c" => "tann", "len" => 4, "adj_len" => 0, "advance" => false);
  57. $sandhi[] = array("a" => "[ṃ]", "b" => "api", "c" => "mpi", "len" => 3, "adj_len" => 0, "advance" => false);
  58. $sandhi[] = array("a" => "[ṃ]", "b" => "eva", "c" => "meva", "len" => 4, "adj_len" => 0, "advance" => false);
  59. $sandhi[] = array("a" => "[o]", "b" => "iva", "c" => "ova", "len" => 3, "adj_len" => 0, "advance" => false);
  60. $sandhi[] = array("a" => "o", "b" => "a", "c" => "o", "len" => 1, "adj_len" => 0, "advance" => false);
  61. $sandhi[] = array("a" => "a", "b" => "ādi", "c" => "ādi", "len" => 3, "adj_len" => 0, "advance" => false);
  62. $sandhi[] = array("a" => "a[ānaṃ]", "b" => "a", "c" => "ānama", "len" => 5, "adj_len" => 0, "advance" => false);
  63. $sandhi[] = array("a" => "a", "b" => "iti", "c" => "āti", "len" => 3, "adj_len" => 0, "advance" => false);
  64. $sandhi[] = array("a" => "[ṃ]", "b" => "ca", "c" => "ñca", "len" => 3, "adj_len" => 0, "advance" => false);
  65. $sandhi[] = array("a" => "[ṃ]", "b" => "iti", "c" => "nti", "len" => 3, "adj_len" => 0, "advance" => false);
  66. $sandhi[] = array("a" => "[ṃ]", "b" => "a", "c" => "ma", "len" => 2, "adj_len" => 0, "advance" => false);
  67. $sandhi[] = array("a" => "ṃ", "b" => "a", "c" => "m", "len" => 1, "adj_len" => 0, "advance" => false);
  68. $sandhi[] = array("a" => "[ṃ]", "b" => "ā", "c" => "mā", "len" => 2, "adj_len" => 0, "advance" => false);
  69. $sandhi[] = array("a" => "[ṃ]", "b" => "u", "c" => "mu", "len" => 2, "adj_len" => 0, "advance" => false);
  70. $sandhi[] = array("a" => "[ṃ]", "b" => "h", "c" => "ñh", "len" => 2, "adj_len" => 0, "advance" => false);
  71. $sandhi[] = array("a" => "ā", "b" => "[ṃ]", "c" => "am", "len" => 2, "adj_len" => 0, "advance" => false);
  72. $sandhi[] = array("a" => "ī", "b" => "[ṃ]", "c" => "im", "len" => 2, "adj_len" => 0, "advance" => false);
  73. $sandhi[] = array("a" => "ati", "b" => "tabba", "c" => "atabba", "len" => 6, "adj_len" => 0, "advance" => false);
  74. $sandhi[] = array("a" => "ati", "b" => "tabba", "c" => "itabba", "len" => 6, "adj_len" => 0, "advance" => false);
  75. $sandhi[] = array("a" => "iti", "b" => "a", "c" => "icca", "len" => 4, "adj_len" => 0, "advance" => false);
  76. $sandhi[] = array("a" => "uṃ", "b" => "a", "c" => "uma", "len" => 3, "adj_len" => 0, "advance" => false);
  77. $sandhi[] = array("a" => "u[ūnaṃ]", "b" => "a", "c" => "ūnama", "len" => 5, "adj_len" => 0, "advance" => false);
  78. $sandhi[] = array("a" => "ī[īnaṃ]", "b" => "a", "c" => "īnama", "len" => 5, "adj_len" => 0, "advance" => false);
  79. $sandhi[] = array("a" => "su", "b" => "a", "c" => "sva", "len" => 3, "adj_len" => 0, "advance" => false);
  80. #other sandhi rule. can be use but program will be slow down
  81. #其他连音规则,如果使用则会让程序运行变慢
  82. $sandhi[] = array("a" => "ā", "b" => "iti", "c" => "āti", "len" => 3, "adj_len" => 0, "advance" => false);
  83. $sandhi[] = array("a" => "a", "b" => "iti", "c" => "āti", "len" => 3, "adj_len" => 0, "advance" => false);
  84. $sandhi[] = array("a" => "e", "b" => "iti", "c" => "eti", "len" => 3, "adj_len" => 0, "advance" => false);
  85. $sandhi[] = array("a" => "ī", "b" => "iti", "c" => "īti", "len" => 3, "adj_len" => 0, "advance" => false);
  86. $sandhi[] = array("a" => "i", "b" => "iti", "c" => "īti", "len" => 3, "adj_len" => 0, "advance" => false);
  87. $sandhi[] = array("a" => "o", "b" => "iti", "c" => "oti", "len" => 3, "adj_len" => 0, "advance" => false);
  88. $sandhi[] = array("a" => "ū", "b" => "iti", "c" => "ūti", "len" => 3, "adj_len" => 0, "advance" => false);
  89. $sandhi[] = array("a" => "u", "b" => "iti", "c" => "ūti", "len" => 3, "adj_len" => 0, "advance" => false);
  90. $sandhi[] = array("a" => "ṃ", "b" => "iti", "c" => "nti", "len" => 3, "adj_len" => 0, "advance" => false);
  91. $sandhi[] = array("a" => "ṃ", "b" => "ca", "c" => "ñca", "len" => 3, "adj_len" => 0, "advance" => false);
  92. $sandhi[] = array("a" => "ṃ", "b" => "cāti", "c" => "ñcāti", "len" => 5, "adj_len" => 0, "advance" => false);
  93. $sandhi[] = array("a" => "ṃ", "b" => "cet", "c" => "ñcet", "len" => 4, "adj_len" => 0, "advance" => false);
  94. /*
  95. $sandhi[] = array("a" => "a", "b" => "eva", "c" => "eva", "len" => 3, "adj_len" => 0, "advance" => false);
  96. $sandhi[] = array("a" => "ā", "b" => "eva", "c" => "āyeva", "len" => 5, "adj_len" => 0, "advance" => false);
  97. $sandhi[] = array("a" => "e", "b" => "eva", "c" => "eva", "len" => 3, "adj_len" => 0, "advance" => false);
  98. $sandhi[] = array("a" => "i", "b" => "eva", "c" => "yeva", "len" => 4, "adj_len" => 0, "advance" => false);
  99. $sandhi[] = array("a" => "ī", "b" => "eva", "c" => "iyeva", "len" => 5, "adj_len" => 0, "advance" => false);
  100. $sandhi[] = array("a" => "ī", "b" => "eva", "c" => "īyeva", "len" => 5, "adj_len" => 0, "advance" => false);
  101. $sandhi[] = array("a" => "o", "b" => "eva", "c" => "ova", "len" => 3, "adj_len" => 0, "advance" => false);
  102. $sandhi[] = array("a" => "u", "b" => "eva", "c" => "veva", "len" => 3, "adj_len" => 0, "advance" => false);
  103. $sandhi[] = array("a" => "a", "b" => "eva", "c" => "evā", "len" => 3, "adj_len" => 0, "advance" => false);
  104. $sandhi[] = array("a" => "e", "b" => "eva", "c" => "evā", "len" => 3, "adj_len" => 0, "advance" => false);
  105. $sandhi[] = array("a" => "i", "b" => "eva", "c" => "yevā", "len" => 4, "adj_len" => 0, "advance" => false);
  106. $sandhi[] = array("a" => "ī", "b" => "eva", "c" => "yevā", "len" => 4, "adj_len" => 0, "advance" => false);
  107. $sandhi[] = array("a" => "ī", "b" => "eva", "c" => "iyevā", "len" => 4, "adj_len" => 0, "advance" => false);
  108. $sandhi[] = array("a" => "ī", "b" => "eva", "c" => "īyevā", "len" => 4, "adj_len" => 0, "advance" => false);
  109. $sandhi[] = array("a" => "o", "b" => "eva", "c" => "ovā", "len" => 4, "adj_len" => 0, "advance" => false);
  110. $sandhi[] = array("a" => "ā", "b" => "api", "c" => "āpi", "len" => 3, "adj_len" => 0, "advance" => false);
  111. $sandhi[] = array("a" => "a", "b" => "api", "c" => "āpi", "len" => 3, "adj_len" => 0, "advance" => false);
  112. $sandhi[] = array("a" => "e", "b" => "api", "c" => "epi", "len" => 3, "adj_len" => 0, "advance" => false);
  113. $sandhi[] = array("a" => "ī", "b" => "api", "c" => "īpi", "len" => 3, "adj_len" => 0, "advance" => false);
  114. $sandhi[] = array("a" => "i", "b" => "api", "c" => "īpi", "len" => 3, "adj_len" => 0, "advance" => false);
  115. $sandhi[] = array("a" => "o", "b" => "api", "c" => "opi", "len" => 3, "adj_len" => 0, "advance" => false);
  116. $sandhi[] = array("a" => "ū", "b" => "api", "c" => "ūpi", "len" => 3, "adj_len" => 0, "advance" => false);
  117. $sandhi[] = array("a" => "u", "b" => "api", "c" => "ūpi", "len" => 3, "adj_len" => 0, "advance" => false);
  118. $sandhi[] = array("a" => "u", "b" => "api", "c" => "upi", "len" => 3, "adj_len" => 0, "advance" => false);
  119. $sandhi[] = array("a" => "ṃ", "b" => "api", "c" => "mpi", "len" => 3, "adj_len" => 0, "advance" => false);
  120. */
  121. $sandhi[] = array("a" => "a", "b" => "a", "c" => "a", "len" => 1, "adj_len" => -1, "advance" => true);
  122. $sandhi[] = array("a" => "ī", "b" => "", "c" => "i", "len" => 1, "adj_len" => 0, "advance" => true);
  123. function split_diphthong($word)
  124. {
  125. //diphthong table双元音表
  126. $search = array('aa', 'ae', 'ai', 'ao', 'au', 'aā', 'aī', 'aū', 'ea', 'ee', 'ei', 'eo', 'eu', 'eā', 'eī', 'eū', 'ia', 'ie', 'ii', 'io', 'iu', 'iā', 'iī', 'iū', 'oa', 'oe', 'oi', 'oo', 'ou', 'oā', 'oī', 'oū', 'ua', 'ue', 'ui', 'uo', 'uu', 'uā', 'uī', 'uū', 'āa', 'āe', 'āi', 'āo', 'āu', 'āā', 'āī', 'āū', 'īa', 'īe', 'īi', 'īo', 'īu', 'īā', 'īī', 'īū', 'ūa', 'ūe', 'ūi', 'ūo', 'ūu', 'ūā', 'ūī', 'ūū');
  127. $replace = array('a-a', 'a-e', 'a-i', 'a-o', 'a-u', 'a-ā', 'a-ī', 'a-ū', 'e-a', 'e-e', 'e-i', 'e-o', 'e-u', 'e-ā', 'e-ī', 'e-ū', 'i-a', 'i-e', 'i-i', 'i-o', 'i-u', 'i-ā', 'i-ī', 'i-ū', 'o-a', 'o-e', 'o-i', 'o-o', 'o-u', 'o-ā', 'o-ī', 'o-ū', 'u-a', 'u-e', 'u-i', 'u-o', 'u-u', 'u-ā', 'u-ī', 'u-ū', 'ā-a', 'ā-e', 'ā-i', 'ā-o', 'ā-u', 'ā-ā', 'ā-ī', 'ā-ū', 'ī-a', 'ī-e', 'ī-i', 'ī-o', 'ī-u', 'ī-ā', 'ī-ī', 'ī-ū', 'ū-a', 'ū-e', 'ū-i', 'ū-o', 'ū-u', 'ū-ā', 'ū-ī', 'ū-ū');
  128. //将双元音拆开
  129. //step 1 : split at diphthong . ~aa~ -> ~a-a~
  130. $word1 = str_replace($search, $replace, $word);
  131. //按连字符拆开处理
  132. $arrword = str_getcsv($word1, "-");
  133. return $arrword;
  134. }
  135. /*
  136. 用于数组连接字符串
  137. */
  138. function myfunction($v1, $v2)
  139. {
  140. return $v1 . "+" . $v2;
  141. }
  142. function microtime_float()
  143. {
  144. list($usec, $sec) = explode(" ", microtime());
  145. return ((float) $usec + (float) $sec);
  146. }
  147. function dict_lookup($word)
  148. {
  149. if (strlen($word) <= 1) {
  150. return 0;
  151. }
  152. global $case;
  153. global $dbh;
  154. $str = strstr($word, "[");
  155. if ($str === false) {
  156. $search = $word;
  157. } else {
  158. $search = $str;
  159. }
  160. $query = "SELECT weight from part where word = ? ";
  161. $stmt = $dbh->prepare($query);
  162. $stmt->execute(array($search));
  163. $row = $stmt->fetch(PDO::FETCH_NUM);
  164. if ($row) {
  165. return $row[0];
  166. } else {
  167. //去除尾查
  168. $newWord = array();
  169. for ($row = 0; $row < count($case); $row++) {
  170. $len = mb_strlen($case[$row][1], "UTF-8");
  171. $end = mb_substr($word, 0 - $len, null, "UTF-8");
  172. if ($end == $case[$row][1]) {
  173. $base = mb_substr($word, 0, mb_strlen($word, "UTF-8") - $len, "UTF-8") . $case[$row][0];
  174. if ($base != $word) {
  175. $newWord[$base] = 1;
  176. }
  177. }
  178. }
  179. #找到最高频的base
  180. $base_weight = 0;
  181. foreach ($newWord as $x => $x_value) {
  182. $query = "SELECT weight from part where word = ? ";
  183. $stmt = $dbh->prepare($query);
  184. $stmt->execute(array($x));
  185. $row = $stmt->fetch(PDO::FETCH_NUM);
  186. if ($row) {
  187. if ($row[0] > $base_weight) {
  188. $base_weight = $row[0];
  189. }
  190. }
  191. }
  192. return $base_weight;
  193. }
  194. }
  195. /*
  196. 查找某个单词是否在现有词典出现
  197. 返回信心指数
  198. look up single word in dictionary vocabulary
  199. return the confidence value
  200. */
  201. function isExsit($word, $adj_len = 0)
  202. {
  203. global $auto_split_times;
  204. global $result;
  205. global $part;
  206. global $confidence;
  207. $auto_split_times++;
  208. if (isset($_POST["debug"])) {
  209. echo "<div>正在查询:{$word}</div>";
  210. }
  211. $isFound = false;
  212. $count = 0;
  213. if (isset($part["{$word}"])) {
  214. if ($part["{$word}"] > 0) {
  215. $isFound = true;
  216. $count = $part["{$word}"] + 1;
  217. }
  218. } else {
  219. $db = dict_lookup($word);
  220. //加入查询缓存
  221. $part["{$word}"] = $db;
  222. if ($db > 0) {
  223. if (isset($_POST["debug"])) {
  224. echo "查到:{$word}:{$db}个\n";
  225. }
  226. $isFound = true;
  227. $count = $db + 1;
  228. }
  229. }
  230. //fomular of confidence value 信心值计算公式
  231. if ($isFound) {
  232. if (isset($confidence["{$word}"])) {
  233. $cf = $confidence["{$word}"];
  234. } else {
  235. $len = mb_strlen($word, "UTF-8") + $adj_len;
  236. $len_correct = 1.2;
  237. $count2 = 1.1 + pow($count, 1.18);
  238. $conf_num = pow(1 / $count2, pow(($len - 0.5), $len_correct));
  239. $cf = round(1 / (1 + 640 * $conf_num), 9);
  240. $confidence["{$word}"] = $cf;
  241. if (isset($_POST["debug"])) {
  242. echo "信心指数:{$word}:{$cf}\n";
  243. }
  244. }
  245. return ($cf);
  246. } else {
  247. return (-1);
  248. }
  249. }
  250. /*
  251. 核心拆分函数
  252. $strWord, word to be look up 要查询的词
  253. $deep, 当前递归深度
  254. $express=true, 快速查询
  255. $adj_len=0 长度校正系数
  256. $c_threshhold 信心指数阈值
  257. */
  258. function mySplit2($strWord, $deep = 0, $express = false, $adj_len = 0, $c_threshhold = 0.8, $w_threshhold = 0.8, $forward = true, $sandhi_advance = false)
  259. {
  260. global $path;
  261. global $result;
  262. global $sandhi;
  263. $output = array();
  264. //达到最大搜索深度,返回
  265. if ($deep >= 16) {
  266. $word = "";
  267. $cf = 1.0;
  268. for ($i = 0; $i < $deep; $i++) {
  269. if (!empty($path[$i][0])) {
  270. $word .= $path[$i][0] . "+";
  271. if (isset($_POST["debug"])) {
  272. $word .= "(" . $path[$i][1] . ")-";
  273. }
  274. $cf = $cf * $path[$i][1];
  275. }
  276. }
  277. $len = pow(mb_strlen($strWord, "UTF-8"), 3);
  278. $cf += (0 - $len) / ($len + 150);
  279. $word .= "{$strWord}";
  280. if ($forward == true) {
  281. $result[$word] = $cf;
  282. } else {
  283. $reverseWord = word_reverse($word);
  284. $result[$reverseWord] = $cf;
  285. }
  286. return;
  287. }
  288. //直接找到
  289. $confidence = isExsit($strWord, $adj_len);
  290. if ($confidence >= 0) {
  291. $output[] = array($strWord, "", $confidence);
  292. } else {
  293. $confidence = isExsit("[" . $strWord . "]");
  294. if ($confidence >= 0) {
  295. $output[] = array("[" . $strWord . "]", "", $confidence);
  296. }
  297. }
  298. //如果开头有双辅音,去掉第一个辅音。因为巴利语中没有以双辅音开头的单词。
  299. $doubleword = "kkggccjjṭṭḍḍttddppbb";
  300. if (mb_strlen($strWord, "UTF-8") > 2) {
  301. $left2 = mb_substr($strWord, 0, 2, "UTF-8");
  302. if (mb_strpos($doubleword, $left2, 0, "UTF-8") !== false) {
  303. $strWord = mb_substr($strWord, 1, null, "UTF-8");
  304. }
  305. }
  306. $len = mb_strlen($strWord, "UTF-8");
  307. if ($len > 2) {
  308. if ($forward) {
  309. #正向切
  310. for ($i = $len; $i > 1; $i--) {
  311. foreach ($sandhi as $key => $row) {
  312. if ($sandhi_advance == false && $row["advance"] == true) {
  313. continue;
  314. }
  315. if (mb_substr($strWord, $i - $row["len"], $row["len"], "UTF-8") == $row["c"]) {
  316. $str1 = mb_substr($strWord, 0, $i - $row["len"], "UTF-8") . $row["a"];
  317. $str2 = $row["b"] . mb_substr($strWord, $i, null, "UTF-8");
  318. $confidence = isExsit($str1, $adj_len);
  319. if ($row["advance"] == true) {
  320. $confidence = $confidence * 0.99;
  321. }
  322. if ($confidence > $c_threshhold) {
  323. $output[] = array($str1, $str2, $confidence, $row["adj_len"]);
  324. if (isset($_POST["debug"])) {
  325. echo "插入:{$str1}\n";
  326. }
  327. if ($express) {
  328. break;
  329. }
  330. }
  331. }
  332. }
  333. }
  334. } else {
  335. #反向切
  336. for ($i = 1; $i < $len - 1; $i++) {
  337. foreach ($sandhi as $key => $row) {
  338. if ($sandhi_advance == false && $row["advance"] == true) {
  339. continue;
  340. }
  341. if (mb_substr($strWord, $i, $row["len"], "UTF-8") == $row["c"]) {
  342. $str1 = mb_substr($strWord, 0, $i, "UTF-8") . $row["a"];
  343. $str2 = $row["b"] . mb_substr($strWord, $i + $row["len"], null, "UTF-8");
  344. $confidence = isExsit($str2, $adj_len);
  345. if ($row["advance"] == true) {
  346. $confidence = $confidence * 0.99;
  347. }
  348. if ($confidence > $c_threshhold) {
  349. $output[] = array($str2, $str1, $confidence, $row["adj_len"]);
  350. if (isset($_POST["debug"])) {
  351. echo "插入:{$str2}\n";
  352. }
  353. if ($express) {
  354. break;
  355. }
  356. }
  357. }
  358. }
  359. }
  360. }
  361. }
  362. if (count($output) > 0) {
  363. foreach ($output as $part) {
  364. $checked = $part[0];
  365. $remainder = $part[1];
  366. $path[$deep][0] = $checked;
  367. $path[$deep][1] = $part[2];
  368. if (empty($remainder)) {
  369. #全切完了
  370. $word = "";
  371. $cf = 1.0;
  372. for ($i = 0; $i < $deep; $i++) {
  373. $word .= $path[$i][0];
  374. if (isset($_POST["debug"])) {
  375. $word .= "(" . $path[$i][1] . ")";
  376. }
  377. $word .= "+";
  378. $cf = $cf * $path[$i][1];
  379. }
  380. if (isset($_POST["debug"])) {
  381. $word .= $checked . "({$part[2]})";
  382. } else {
  383. $word .= $checked;
  384. }
  385. $cf = $cf + $part[2] * 0.1;
  386. if ($cf > $w_threshhold) {
  387. if ($forward == true) {
  388. $result[$word] = $cf;
  389. } else {
  390. $reverseWord = word_reverse($word);
  391. $result[$reverseWord] = $cf;
  392. }
  393. }
  394. } else {
  395. #接着切
  396. mySplit2($remainder, ($deep + 1), $express, $adj_len, $c_threshhold, $w_threshhold, $forward, $sandhi_advance);
  397. }
  398. }
  399. } else {
  400. #尾巴查不到了
  401. $word = "";
  402. $cf = 1.0;
  403. for ($i = 0; $i < $deep; $i++) {
  404. $word .= $path[$i][0];
  405. if (isset($_POST["debug"])) {
  406. $word .= "(" . $path[$i][1] . ")";
  407. }
  408. $word .= "+";
  409. $cf = $cf * $path[$i][1];
  410. }
  411. $len = pow(mb_strlen($strWord, "UTF-8"), 3);
  412. if ($forward) {
  413. $cf += (0 - $len) / ($len + 150);
  414. } else {
  415. $cf += (0 - $len) / ($len + 5);
  416. }
  417. if (isset($_POST["debug"])) {
  418. $word .= $strWord . "(0)";
  419. } else {
  420. $word .= $strWord;
  421. }
  422. if ($cf > $w_threshhold) {
  423. if ($forward == true) {
  424. $result[$word] = $cf;
  425. } else {
  426. $reverseWord = word_reverse($word);
  427. $result[$reverseWord] = $cf;
  428. }
  429. }
  430. }
  431. }
  432. function word_reverse($word)
  433. {
  434. $reverse = array();
  435. $newword = explode("+", $word);
  436. $len = count($newword);
  437. if ($len > 0) {
  438. for ($i = $len - 1; $i >= 0; $i--) {
  439. # code...
  440. $reverse[] = $newword[$i];
  441. }
  442. $output = implode("+", $reverse);
  443. return $output;
  444. } else {
  445. return $word;
  446. }
  447. }