mint/app/pali_sent/sim_sent.php

<?php
//计算句子相似度
require_once "../path.php";
require_once "../public/_pdo.php";

global $PDO;
PDO_Connect("" . _FILE_DB_PALI_SENTENCE_);
PDO_Execute("PRAGMA synchronous = OFF");
PDO_Execute("PRAGMA journal_mode = WAL");
PDO_Execute("PRAGMA foreign_keys = ON");
PDO_Execute("PRAGMA busy_timeout = 5000");

// 输入一个句子，输出整个句子的单词 array
function words_of_sentence(string $sent)
{
    $words = preg_split("/[ \.\[\]\{\}\-,';‘’–0123456789]+/", $sent); // 去除标点、数字
    $words = array_filter($words); // 去除空词
    $words = array_filter($words, function ($item) {
        if ($item != 'ca' && $item != 'vā' && $item != 'na') {
            return $item;
        }
    }); // 去除 ca, vā 和 na
    return $words;
}

// 采用 jaccard 相似度，考虑到圣典中的相似句单词、句式都是非常接近的
function jaccard_similarity($words_of_sent1, $words_of_sent2)
{
    $intersect = count(array_intersect($words_of_sent1, $words_of_sent2));
    $union = count($words_of_sent1) + count($words_of_sent2) - $intersect;
    if ($union) {
        return $intersect / $union;
    } else {
        return 0;
    }
}

// 带顺序的 jaccard 算法，当前效果一般，TODO: 切片相同时加入得分
function ordered_jaccard_similarity($words_of_sent1, $words_of_sent2)
{
    $score = 0;
    $k = min(count($words_of_sent1), count($words_of_sent2));
    for ($i = 1; $i < $k; $i++) {
        $score += jaccard_similarity(
            array_slice($words_of_sent1, 0, $i),
            array_slice($words_of_sent2, 0, $i));
    }
    return $score / $k;
}

// 定义一个链表节点，方便 sim_sent_id 按照相似度插入
class sim_sent_node
{
    public $id;
    public $jaccard_score;
    public $next;

    public function __construct($id = null, $jaccard_score = null, $next = null)
    {
        $this->id = $id;
        $this->jaccard_score = $jaccard_score;
        $this->next = $next;
    }
}

// 定义链表
class sim_sent_list
{
    public $head; // 头节点，默认一个虚头节点
    public $size;

    public function __construct()
    {
        $this->head = new sim_sent_node();
        $this->size = 0;
    }

    // 按照 jaccard_score 相似度插入
    public function jaccard_add($id, $jaccard_score)
    {
        $prev = $this->head;
        while ($prev->next != null && $prev->next->jaccard_score > $jaccard_score) {
            $prev = $prev->next;
        }
        $prev->next = new sim_sent_node($id, $jaccard_score, $prev->next);
        $this->size++;
    }

    public function get_text_list()
    {
        $prev = $this->head;
        if ($this->size == 0) {
            return;
        }

        $ids = "";
        while ($prev->next != null) {
            $ids = $ids . "," . $prev->next->id;
            $prev = $prev->next;
        }
        $ids = substr($ids, 1); // 去掉第一个逗号
        return $ids;
    }

    public function print_list()
    {
        $prev = $this->head;
        if ($this->size == 0) {
            return;
        }
        while ($prev->next != null) {
            print($prev->next->id . "\t" . $prev->next->jaccard_score . "\n");
            $prev = $prev->next;
        }
    }
}

// 将相似句列表存入数据库
function insert_similar_sent_list_into_sqlite($current_id, $text_list)
{
    /* 使用这部分代码先为数据库添加一个 sim_sents 字段
    $add_column = "ALTER TABLE pali_sent ADD COLUMN sim_sents TEXT";
    $Action_add = PDO_Execute($add_column);
    $query = "PRAGMA TABLE_INFO (pali_sent)";
    $Fetch = PDO_FetchALL($query);
    print_r($Fetch);
     */
    global $PDO;
    $Update = "UPDATE pali_sent SET sim_sents = " . $PDO->quote($text_list) . " WHERE id = " . $current_id;
    $Result = PDO_Execute($Update);
    return;
}

// 预计算，存入数据库
function similar_sent_matrix($begin, $end, $begin_id = 0)
{
    // 按照 count = 18, 8, ..., 255 依次获得查询结果 (i-3,i+3)
    //          count = 17,16,...,7                                       (i-2,i+2)
    for ($current_count = $begin; $current_count <= $end; $current_count++) {
        print("单词数：" . $current_count . "\n");
        $current_query = "select id,text from pali_sent where count=" . $current_count;
        $Current = PDO_FetchAll($current_query);
        if (count($Current)) {
            foreach ($Current as $current_row) {
                $current_id = $current_row['id'];
                if ($begin_id > 0 && $current_count == $begin) {
                    if ($current_id < $begin_id) {
                        continue;
                    }
                }
                $current_sent = $current_row['text'];
                $current_words = words_of_sentence($current_sent);

                // 按照 count > $current_count-3 and count <$current_count+3 查询希望比较的语句
                if ($current_count > 17) {
                    $section = 3;
                } else if ($current_count > 6) {
                    $section = 2;
                } else if ($current_count > 3) {
                    $section = 1;
                } else {
                    $section = 0;
                }
                $compare_query = "select id,text from pali_sent where count>" . ($current_count - $section) . " and count<" . ($current_count + $section);
                $Compare = PDO_FetchALL($compare_query);

                $current_sim_sent_list = new sim_sent_list(); // 新建相似句链表
                foreach ($Compare as $compare_row) {
                    if ($current_row != $compare_row) {
                        $compare_id = $compare_row['id'];
                        $compare_sent = $compare_row['text'];
                        $compare_words = words_of_sentence($compare_sent);

                        if ($current_count > 3) {
                            $jaccard_score = jaccard_similarity($current_words, $compare_words);
                        } else if ($current_count == 3) {
                            if ($current_words == $compare_words) {
                                $jaccard_score = 1;
                            }
                        } else if ($current_count == 2) {
                            if ($current_words == $compare_words) {
                                $jaccard_score = 1;
                            }
                        } else {
                            $jaccard_score = 0;
                        }
                        if ($jaccard_score > 0.3) {
                            $current_sim_sent_list->jaccard_add($compare_id, $jaccard_score);
                        }
                    }
                } // end of foreach $compare_row

                if ($current_sim_sent_list->size != 0) {
                    print("update $current_count - " . $current_id . "\n");
                    $text_list = $current_sim_sent_list->get_text_list();
                    insert_similar_sent_list_into_sqlite($current_id, $text_list);
                }
            } // end of foreach $current_row
        }
    }
    return;
}

// 实时计算相似句
function sents_similar_to_id($id)
{
    $query = "SELECT count,text FROM pali_sent WHERE id=" . $id;
    $Current = PDO_FetchALL($query);
    if (count($Current)) {
        foreach ($Current as $current_row) {
            $current_count = $current_row['count'];
            $current_sent = $current_row['text'];
            $current_words = words_of_sentence($current_sent);
            print("current text: \n" . $current_sent . "\n");

            if ($current_count <= 5) {
                print("[-] too short.\n");
                return;
            }

            // 只和单词数大于 5 的比较
            $compare_query = "SELECT id,text FROM pali_sent WHERE count>5";
            $Compare = PDO_FetchALL($compare_query);

            $current_sim_sent_list = new sim_sent_list(); // 新建相似句链表
            foreach ($Compare as $compare_row) {
                if ($current_row != $compare_row) {
                    $compare_id = $compare_row['id'];
                    $compare_sent = $compare_row['text'];
                    $compare_words = words_of_sentence($compare_sent);

                    $jaccard_score = jaccard_similarity($current_words, $compare_words);
                    if ($jaccard_score > 0.3) {
                        print("Jaccard similarity: " . $jaccard_score . "\tSentence id:" . $compare_id . "\n");
                        print("Text: \n" . $compare_sent . "\n");
                        $current_sim_sent_list->jaccard_add($compare_id, $jaccard_score);
                    }
                }
            } // end of foreach $compare_row

            if ($current_sim_sent_list->size != 0) {
                // $current_sim_sent_list->print_list();
            } else {
                print("[-]not found.\n");
            }

        } // end of foreach($Current)
    } // end of if (count($Current))
}

//$id = $argv[1];
//sents_similar_to_id($id);

if ($argc < 3) {
    echo "无效的参数 ";
    exit;
}
$from = (int) $argv[1];
$to = (int) $argv[2];
if ($argc > 3) {
    $from_id = (int) $argv[3];
} else {
    $from_id = 0;
}
if ($from < 2) {
    $from = 2;
}
if ($to > 255) {
    $to = 255;
}
similar_sent_matrix($from, $to, $from_id);
echo "\n all done";