Merge pull request #1709 from visuddhinanda/agile

tulip从api获取搜索内容

rpc/tulip/tulip/README.md

@@ -1,4 +1,4 @@
-# 全文搜索表更新流程
+# 全文搜索字典更新
 
 1. 下载字典文件
 
@@ -11,3 +11,7 @@ php dict_maker.php
 2. 复制字典文件到 postgresql 目录
 3. 运行 `php dict_update.php` 令字典文件生效
 4. 运行 `php content_update.php` 重新生成索引
+
+# 全文搜索内容数据更新
+
+运行 `php content_download.php` 从 api 获取巴利语全文搜索数据。每次运行时间约 1 小时。

rpc/tulip/tulip/content_download.php

@@ -0,0 +1,108 @@
+<?php
+require dirname(__FILE__) . '/vendor/autoload.php';
+require dirname(__FILE__) . '/config.php';
+require dirname(__FILE__) . '/logger.php';
+require dirname(__FILE__) . '/pdo.php';
+
+logger('debug','download full test search content start');
+
+$PDO = new PdoHelper;
+
+$PDO->connectDb();
+logger('debug','connect database finished');
+
+$client = new GuzzleHttp\Client();
+
+$pageSize = 1000;
+    $urlBase = Config['api_server'] . '/v2/pali-search-data';
+    logger('debug','url='.$urlBase);
+
+    for ($book=1; $book < 218; $book++) { 
+        $currPage = 1;
+        $url = $urlBase . "?book={$book}";
+        logger('debug','fetch book='.$book);
+        do {
+            $goNext = false;
+            $url = $url . "&start={$currPage}&page_size={$pageSize}";
+            logger('debug','url='.$url);
+            $res = $client->request('GET', $url);
+            $status = $res->getStatusCode();
+            if($status === 200){
+                $json = json_decode($res->getBody());
+                if($json->ok){
+                    $content = $json->data->rows;
+                    foreach ($json->data->rows as $row) {
+                        $book = $row->book;
+                        $paragraph = $row->paragraph;
+                        logger('debug',"update start book={$book} para={$paragraph} ");
+                        $now = date("Y-m-d H:i:s");
+                        //查询是否存在
+                        $query = 'SELECT id from fts_texts where book=? and paragraph = ?';
+                        $result = $PDO->dbSelect($query, [$book,$paragraph]);
+                        if(count($result) >0 ){
+                            //存在 update
+                            $query = 'UPDATE fts_texts set 
+                                                "bold_single"=?,
+                                                "bold_double"=?,
+                                                "bold_multiple"=?,
+                                                "content"=?,
+                                                "pcd_book_id"=?,
+                                                "updated_at"=?  where id=? ';
+                            $update = $PDO->dbSelect($query, [
+                                                $row->bold1,
+                                                $row->bold2,
+                                                $row->bold3,
+                                                $row->content,
+                                                $row->pcd_book_id,
+                                                $now,
+                                                $result[0]['id']
+                                                    ]);
+                        }else{
+                            // new
+                            $query = "INSERT INTO fts_texts (
+                                        book,
+                                        paragraph,
+                                        bold_single,
+                                        bold_double,
+                                        bold_multiple,
+                                        \"content\",
+                                        created_at,
+                                        updated_at,
+                                        pcd_book_id) VALUES
+                                            (?,?,?,?,?,?,?,?,? )";
+                            $insert = $PDO->dbSelect($query, 
+                                            [
+                                                $row->book,
+                                                $row->paragraph,
+                                                $row->bold1,
+                                                $row->bold2,
+                                                $row->bold3,
+                                                $row->content,
+                                                $now,
+                                                $now,
+                                                $row->pcd_book_id,
+                                            ]);
+                        }
+                    }
+                    logger('debug',"update done book={$book} para={$paragraph} ");
+                    $maxPage = $json->data->count;
+                    logger('debug','max page ='.$maxPage);
+                    if($currPage+$pageSize<$maxPage){
+                        $goNext = true;
+                    }else{
+                        logger('debug',"book {$book} is done");
+                        $goNext = false;
+                    }
+                }else{
+                    logger('error');
+                }
+            }else{
+                logger('error', 'status='.$status);
+            }
+            $currPage += $pageSize;
+        } while ($goNext);        
+    }
+
+logger('debug','all done') ;
+
+