Merge pull request #1684 from visuddhinanda/laravel

添加章节导出

app/Console/Commands/ExportChapter.php

@@ -12,16 +12,21 @@ use App\Http\Api\ChannelApi;
 use App\Http\Api\MdRender;
 use App\Tools\Export;
 use Illuminate\Support\Facades\Log;
+use App\Tools\RedisClusters;
 
 class ExportChapter extends Command
 {
+    protected $exportStatusKey = 'export/status';
+    protected $exportStatusExpiry = 3600;
+    protected $currExportStatusKey = '';
     /**
      * The name and signature of the console command.
-     * php artisan export:chapter 213 1913 a19eaf75-c63f-4b84-8125-1bce18311e23 --format=html
-     * php artisan export:chapter 213 3  a19eaf75-c63f-4b84-8125-1bce18311e23 --format=html
+     * php artisan export:chapter 213 1913 a19eaf75-c63f-4b84-8125-1bce18311e23 213-1913 --format=html
+     * php artisan export:chapter 168 915 19f53a65-81db-4b7d-8144-ac33f1217d34 168-915.html --format=html
+     * php artisan export:chapter 168 915 19f53a65-81db-4b7d-8144-ac33f1217d34 168-915.html --format=html --origin=true
      * @var string
      */
-    protected $signature = 'export:chapter {book} {para} {channel} {--debug} {--format=tex}';
+    protected $signature = 'export:chapter {book} {para} {channel} {filename} {--origin=false} {--translation=true} {--debug} {--format=tex} ';
 
     /**
      * The console command description.
@@ -40,6 +45,23 @@ class ExportChapter extends Command
         parent::__construct();
     }
 
+    /**
+     * progress: 0-1, error -1
+     * message: string
+     */
+    protected function setStatus($progress,$message=''){
+        RedisClusters::put($this->currExportStatusKey,
+                            [
+                                'progress'=>$progress,
+                                'message'=>$message,
+                            ],
+                            $this->exportStatusExpiry);
+        $percent = (int)($progress * 100);
+        $this->info("[{$percent}%]".$message);
+    }
+    public function getStatus($filename){
+        return RedisClusters::get($this->exportStatusKey.'/'.$filename);
+    }
     /**
      * Execute the console command.
      *
@@ -47,10 +69,22 @@ class ExportChapter extends Command
      */
     public function handle()
     {
-        Log::debug('task export offline chapter-table start');
+        $this->info('task export chapter start');
+        Log::debug('task export chapter start');
         if(\App\Tools\Tools::isStop()){
             return 0;
         }
+        $m = new \Mustache_Engine(array('entity_flags'=>ENT_QUOTES,
+                                        'delimiters' => '[[ ]]',
+                                        'escape'=>function ($value){
+                                            return $value;
+                                        }));
+        $tplParagraph = file_get_contents(resource_path("mustache/".$this->option('format')."/paragraph.".$this->option('format')));
+
+        MdRender::init();
+
+        $this->currExportStatusKey = $this->exportStatusKey . '/' . $this->argument('filename');
+
         switch ($this->option('format')) {
             case 'md':
                 $renderFormat='markdown';
@@ -64,126 +98,200 @@ class ExportChapter extends Command
         }
         $book = $this->argument('book');
         $para = $this->argument('para');
-        $channelId = $this->argument('channel');
-        $channel = ChannelApi::getById($channelId);
-        $chapter = PaliText::where('book',$book)->where('paragraph',$para)->first();
+
+        //获取原文channel
+        $orgChannelId = ChannelApi::getSysChannel('_System_Pali_VRI_');
+
+        $tranChannelsId = explode('_',$this->argument('channel'));
+
+        $channelsId = array_merge([$orgChannelId],$tranChannelsId);
+
+        $channels = array();
+        $channelsIndex = array();
+        foreach ($channelsId as $key => $id) {
+            $channels[] = ChannelApi::getById($id);
+            $channelsIndex[$id] = ChannelApi::getById($id);
+        }
+
+        $bookMeta = array();
+        $bookMeta['book_author'] = "";
+        foreach ($channels as $key => $channel) {
+            $bookMeta['book_author'] .= $channel['name'] . ' ';
+        }
+
+        $chapter = PaliText::where('book',$book)
+                           ->where('paragraph',$para)->first();
         if(!$chapter){
             return $this->error("no data");
         }
-        $bookMeta = array();
+
+        $currProgress = 0;
+        $this->setStatus($currProgress,'start');
+
+
         if(empty($chapter->toc)){
             $bookMeta['title'] = "unknown";
         }else{
-            $title = ProgressChapter::where('book',$book)->where('para',$para)
-                                    ->where('channel_id',$channelId)
-                                    ->value('title');
-            $bookMeta['book_title'] = $title?$title:$chapter->toc;
+            $bookMeta['book_title'] = '';
+            foreach ($channelsId as $key => $id) {
+                $title = ProgressChapter::where('book',$book)->where('para',$para)
+                        ->where('channel_id',$id)
+                        ->value('title');
+                $bookMeta['book_title'] .= $title;
+            }
             $bookMeta['sub_title'] = $chapter->toc;
         }
-        if($channel){
-            $bookMeta['book_author'] = $channel['name'];
-        }
+
         $subChapter = PaliText::where('book',$book)->where('parent',$para)
                               ->where('level','<',8)
                               ->orderBy('paragraph')
                               ->get();
+        if(count($subChapter) === 0){
+            //没有子章节
+            $subChapter = PaliText::where('book',$book)->where('paragraph',$para)
+                              ->where('level','<',8)
+                              ->orderBy('paragraph')
+                              ->get();
+        }
+
+        $chapterParagraph = PaliText::where('book',$book)->where('paragraph',$para)->value('chapter_len');
+        if($chapterParagraph >0 ){
+            $step = 0.9 / $chapterParagraph;
+        }else{
+            $step = 0.9;
+            Log::error('段落长度不能为0',['book'=>$book,'para'=>$para]);
+        }
+
+        $outputChannelsId = [];
+        if($this->option('origin') === 'true'){
+            $outputChannelsId[] = $orgChannelId;
+        }
+        if($this->option('translation') === 'true'){
+            $outputChannelsId = array_merge($outputChannelsId,$tranChannelsId);
+        }
+
         $sections = array();
         foreach ($subChapter as $key => $sub) {
-            # code...
-            $chapter = ProgressChapter::where('book',$book)->where('para',$sub->paragraph)
-                                    ->where('channel_id',$channelId)
-                                    ->first();
-            if($chapter){
-                $filename = "{$sub->paragraph}.".$this->option('format');
-                $bookMeta['sections'][] = ['filename'=>$filename];
-                $paliTitle = PaliText::where('book',$book)->where('paragraph',$sub->paragraph)->value('toc');
-                $title = $chapter->title?$chapter->title:$paliTitle;
-                $content = array();
-
-                $chapterStart = $sub->paragraph+1;
-                $chapterEnd = $sub->paragraph + $sub->chapter_len;
-                $chapterBody = PaliText::where('book',$book)
-                                          ->whereBetween('paragraph',[$chapterStart,$chapterEnd])
-                                          ->orderBy('paragraph')->get();
-                foreach ($chapterBody as $body) {
-                    # code...
+            # 看这个章节是否存在译文
+            $hasChapter = false;
+            if($this->option('origin') === 'true'){
+                $hasChapter = true;
+            }
+            if($this->option('translation') === 'true'){
+                foreach ($tranChannelsId as $id) {
+                    if(ProgressChapter::where('book',$book)->where('para',$sub->paragraph)
+                        ->where('channel_id',$id)
+                        ->exists()){
+                            $hasChapter = true;
+                    }
+                }
+            }
+            if(!$hasChapter){
+                //不存在需要导出的数据
+                continue;
+            }
+            $filename = "{$sub->paragraph}.".$this->option('format');
+            $bookMeta['sections'][] = ['filename'=>$filename];
+            $paliTitle = PaliText::where('book',$book)
+                                 ->where('paragraph',$sub->paragraph)
+                                 ->value('toc');
+            $sectionTitle = $paliTitle;
+            if($this->option('translation') === 'true'){
+                $chapter = ProgressChapter::where('book',$book)->where('para',$sub->paragraph)
+                                        ->where('channel_id',$tranChannelsId[0])
+                                        ->first();
+                if($chapter && !empty($chapter->title)){
+                    $sectionTitle = $chapter->title;
+                }
+            }
+
+
+            $content = array();
+
+            $chapterStart = $sub->paragraph+1;
+            $chapterEnd = $sub->paragraph + $sub->chapter_len;
+            $chapterBody = PaliText::where('book',$book)
+                                    ->whereBetween('paragraph',[$chapterStart,$chapterEnd])
+                                    ->orderBy('paragraph')->get();
+
+
+
+            foreach ($chapterBody as $body) {
+                $currProgress += $step;
+                $this->setStatus($currProgress,'export chapter '.$body->paragraph);
+                $paraData = array();
+                $paraData['translations'] = array();
+                foreach ($outputChannelsId as $key => $channelId) {
                     $translationData = Sentence::where('book_id',$book)
-                                    ->where('paragraph',$body->paragraph)
-                                    ->where('channel_uid',$channelId)
-                                    ->orderBy('word_start')->get();
+                                        ->where('paragraph',$body->paragraph)
+                                        ->where('channel_uid',$channelId)
+                                        ->orderBy('word_start')->get();
                     $sentContent = array();
                     foreach ($translationData as $sent) {
                         $texText = MdRender::render($sent->content,
-                                                        [$sent->channel_uid],
-                                                        null,
-                                                        'read',
-                                                        $channel['type'],
-                                                        $sent->content_type,
-                                                        $renderFormat
-                                                        );
+                                                    [$sent->channel_uid],
+                                                    null,
+                                                    'read',
+                                                    $channelsIndex[$channelId]['type'],
+                                                    $sent->content_type,
+                                                    $renderFormat
+                                                    );
                         $sentContent[] = trim($texText);
                     }
                     $paraContent = implode(' ',$sentContent);
-                    if($body->level > 7){
-                        switch ($this->option('format')) {
-                            case 'tex':
-                                $content[] = '\par '.$paraContent;
-                                break;
-                            case 'html':
-                                $content[] = '<p>'.$paraContent.'</p>';
-                                break;
-                            case 'md':
-                                $content[] = "\n\n".$paraContent;
-                                break;
-                        }
-
+                    if($channelsIndex[$channelId]['type'] === 'original'){
+                        $paraData['origin'] = $paraContent;
                     }else{
-                        $currLevel = $body->level - $sub->level;
-                        if($currLevel>0){
-                            if(empty($paraContent)){
-                                $subSessionTitle = PaliText::where('book',$book)
-                                                     ->where('paragraph',$body->paragraph)
-                                                     ->value('toc');
-                            }else{
-                                $subSessionTitle = $paraContent;
-                            }
-                            switch ($this->option('format')) {
-                                case 'tex':
-                                    $subStr = array_fill(0,$currLevel,'sub');
-                                    $content[] = '\\'. implode('',$subStr) . "section{".$subSessionTitle.'}';
-                                    break;
-                                case 'md':
-                                    $subStr = array_fill(0,$currLevel,'#');
-                                    $content[] = implode('',$subStr) . " ".$subSessionTitle;
-                                    break;
-                                case 'html':
-                                    $level = $currLevel+2;
-                                    $content[] = "<h{$currLevel}".$subSessionTitle."</h{$currLevel}";
-                                    break;
-                            }
-
-                        }else{
-                            $content[] = '\par '.$paraContent;
-                        }
+                        $paraData['translations'][] = ['content'=>$paraContent];
                     }
-                    $content[] = "\n\n";
                 }
+                if($body->level > 7){
+                    $content[] = $m->render($tplParagraph,$paraData);
+                }else{
+                    $currLevel = $body->level - $sub->level;
+                    if($currLevel<=0){
+                        $currLevel = 1;
+                    }
 
-                $sections[] = [
-                                'name'=>$filename,
-                                'body'=>[
-                                    'title'=>$title,
-                                    'content'=>implode('',$content)
-                                ]
-                            ];
+                    if(count($paraData['translations'])===0){
+                        $subSessionTitle = PaliText::where('book',$book)
+                                            ->where('paragraph',$body->paragraph)
+                                            ->value('toc');
+                    }else{
+                        $subSessionTitle = $paraData['translations'][0]['content'];
+                    }
+                    switch ($this->option('format')) {
+                        case 'tex':
+                            $subStr = array_fill(0,$currLevel,'sub');
+                            $content[] = '\\'. implode('',$subStr) . "section{".$subSessionTitle.'}';
+                            break;
+                        case 'md':
+                            $subStr = array_fill(0,$currLevel,'#');
+                            $content[] = implode('',$subStr) . " ".$subSessionTitle;
+                            break;
+                        case 'html':
+                            $level = $currLevel+2;
+                            $content[] = "<h{$currLevel}".$subSessionTitle."</h{$currLevel}";
+                            break;
+                    }
+                }
+                $content[] = "\n\n";
             }
+
+            $sections[] = [
+                    'name'=>$filename,
+                    'body'=>[
+                        'title'=>$sectionTitle,
+                        'content'=>implode('',$content)
+                    ]
+                ];
         }
+
+        $this->setStatus(0.9,'export content done');
+        Log::debug('导出结束');
+
         $tex = array();
-        $m = new \Mustache_Engine(array('entity_flags'=>ENT_QUOTES,
-                                        'delimiters' => '[[ ]]',
-                                        'escape'=>function ($value){
-                                            return $value;
-                                        }));
+
         $tpl = file_get_contents(resource_path("mustache/".$this->option('format')."/main.".$this->option('format')));
         $texContent = $m->render($tpl,$bookMeta);
         $tex[] = ['name'=>'main.'.$this->option('format'),
@@ -197,7 +305,8 @@ class ExportChapter extends Command
                  ];
         }
 
-        //脚注
+        Log::debug('footnote start');
+        //footnote
         $tplFile = resource_path("mustache/".$this->option('format')."/footnote.".$this->option('format'));
         if(isset($GLOBALS['note']) &&
             is_array($GLOBALS['note']) &&
@@ -215,26 +324,37 @@ class ExportChapter extends Command
                 Storage::disk('local')->put($dir.$section['name'], $section['content']);
             }
         }
+
+        Log::debug('footnote finished');
+        Log::debug('upload start');
+        $this->setStatus(0.95,'export content done');
+
+        //upload
+        $fileDate = '';
         switch ($this->option('format')) {
             case 'tex':
                 $data = Export::ToPdf($tex);
                 if($data['ok']){
-                    $filename = "export/{$book}-{$para}-{$channelId}.pdf";
                     $this->info($data['content-type']);
-                    Storage::disk('local')->put($filename, $data['data']);
+                    $fileDate = $data['data'];
                 }else{
                     $this->error($data['code'].'-'.$data['message']);
                 }
                 break;
             case 'html':
-                $fHtml = "export/".$this->option('format')."/{$book}-{$para}-{$channelId}.html";
-                Storage::disk('local')->put($fHtml, '');
+                $file = array();
                 foreach ($tex as $key => $section) {
-                    Storage::disk('local')->append($fHtml, $section['content']);
+                    $file[] = $section['content'];
                 }
+                $fileDate = implode('',$file);
                 break;
         }
+        $filename = $this->argument('filename');
+        $bucket = config('mint.attachments.bucket_name.temporary');
+        $tmpFile =  $bucket.'/'. $filename ;
+        Storage::put($tmpFile, $fileDate);
 
+        $this->setStatus(1,'export done filename='.$filename);
         Log::debug('task export offline chapter-table finished');
         return 0;
     }

app/Console/Commands/MqExport.php

@@ -0,0 +1,71 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\Http\Api\Mq;
+use Illuminate\Support\Facades\Log;
+
+class MqExport extends Command
+{
+    /**
+     * The name and signature of the console command.
+     * php artisan mq:export
+     * @var string
+     */
+    protected $signature = 'mq:export';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = '导出功能用的消息队列';
+
+    /**
+     * Create a new command instance.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        parent::__construct();
+    }
+
+    /**
+     * Execute the console command.
+     *
+     * @return int
+     */
+    public function handle()
+    {
+        $exchange = 'router';
+        $queue = 'export';
+        $this->info(" [*] Waiting for {$queue}. To exit press CTRL+C");
+        Log::debug("mq:progress start.");
+        Mq::worker($exchange,$queue,function ($message){
+            $data = [
+                        'book'=>$message->book,
+                        'para'=>$message->para,
+                        'channel'=>$message->channel,
+                        '--format'=>$message->format,
+                        'filename'=>$message->filename,
+                    ];
+            if(isset($message->origin) && is_string($message->origin)){
+                $data['--origin'] = $message->origin;
+            }
+            if(isset($message->translation) && is_string($message->translation)){
+                $data['--translation'] = $message->translation;
+            }
+            $ok = $this->call('export:chapter',$data);
+            if($ok !== 0){
+                Log::error('mq:progress upgrade:progress fail',$data);
+            }else{
+                $this->info("Received book=".$message->book.' result='.$ok);
+                Log::debug("mq:export: done ",$data);
+                return $ok;
+            }
+        });
+        return 0;
+    }
+}

app/Http/Api/Mq.php

@@ -116,7 +116,7 @@ class Mq
                     }
                 } catch (\Exception $e) {
                     // push to issues
-                    Log::error('mq worker exception', $e);
+                    Log::error('mq worker exception', ['exception'=>$e] );
                     $channelName = 'issues';
                     $channelIssues = $connection->channel();
                     $channelIssues->queue_declare($channelName, false, true, false, false);

app/Http/Controllers/ExportController.php

@@ -0,0 +1,110 @@
+<?php
+
+namespace App\Http\Controllers;
+
+use Illuminate\Http\Request;
+use Illuminate\Support\Str;
+use App\Http\Api\Mq;
+use Illuminate\Support\Facades\Log;
+use App\Tools\RedisClusters;
+use Illuminate\Support\Facades\App;
+use Illuminate\Support\Facades\Storage;
+use App\Console\Commands\ExportChapter;
+
+class ExportController extends Controller
+{
+    /**
+     * Display a listing of the resource.
+     *
+     * @return \Illuminate\Http\Response
+     */
+    public function index(Request $request)
+    {
+        //
+        $filename = $request->get('book').'-'.
+                    $request->get('par').'-'.
+                    Str::uuid().'.'.$request->get('format');
+
+
+        Mq::publish('export',[
+            'book'=>$request->get('book'),
+            'para'=>$request->get('par'),
+            'channel'=>$request->get('channel'),
+            'format'=>$request->get('format'),
+            'origin'=>$request->get('origin'),
+            'translation'=>$request->get('translation'),
+            'filename'=>$filename,
+        ]);
+        return $this->ok($filename);
+    }
+
+    /**
+     * Store a newly created resource in storage.
+     *
+     * @param  \Illuminate\Http\Request  $request
+     * @return \Illuminate\Http\Response
+     */
+    public function store(Request $request)
+    {
+        //
+    }
+
+    /**
+     * Display the specified resource.
+     *
+     * @param  string  $id
+     * @return \Illuminate\Http\Response
+     */
+    public function show($filename)
+    {
+        //
+        $exportChapter = new ExportChapter();
+        $exportStatus = $exportChapter->getStatus($filename);
+        if(empty($exportStatus)){
+            return $this->error('no file',200,200);
+        };
+
+        $output = array();
+        $output['status'] = $exportStatus;
+        if($exportStatus['progress']===1){
+            $bucket = config('mint.attachments.bucket_name.temporary');
+            $tmpFile =  $bucket.'/'. $filename ;
+            if (App::environment('local')) {
+                $s3Link = Storage::url($tmpFile);
+            }else{
+                try{
+                    $s3Link = Storage::temporaryUrl($tmpFile, now()->addDays(7));
+                }catch(\Exception $e){
+                    Log::error('export {Exception}',['exception'=>$e]);
+                    return $this->error('temporaryUrl fail',404,404);
+                }
+            }
+            $output['url'] = $s3Link;
+        }
+        return $this->ok($output);
+
+    }
+
+    /**
+     * Update the specified resource in storage.
+     *
+     * @param  \Illuminate\Http\Request  $request
+     * @param  int  $id
+     * @return \Illuminate\Http\Response
+     */
+    public function update(Request $request, $id)
+    {
+        //
+    }
+
+    /**
+     * Remove the specified resource from storage.
+     *
+     * @param  int  $id
+     * @return \Illuminate\Http\Response
+     */
+    public function destroy($id)
+    {
+        //
+    }
+}

resources/mustache/html/glossary.html

@@ -0,0 +1,9 @@
+<h2>glossary</h2>
+<div class="glossary">
+[[#pali]]
+<div class="item pali"><span class="head">[[pali]]</span><span class="content">[[meaning]]</span></div>
+[[/pali]]
+[[#meaning]]
+<div class="item meaning"><span class="head">[[meaning]]</span><span  class="content">[[pali]]</span></div>
+[[/meaning]]
+</div>

resources/mustache/html/main.html

@@ -11,6 +11,21 @@
         a {
             text-decoration: none;
         }
+        .paragraph {
+            margin-bottom: 1em;
+        }
+
+        .row {
+            display: flex;
+        }
+        @media screen and (max-width: 960px){
+            .row {
+                flex-direction: column;
+            }
+        }
+        .col {
+            flex: 5;
+        }
     </style>
 </head>
 <body>

resources/mustache/html/paragraph.html

@@ -0,0 +1,6 @@
+<div class="paragraph row [[layout]]">
+    <div class="col origin">[[origin]]</div>
+    [[#translations]]
+    <div class="col translation">[[content]]</div>
+    [[/translations]]
+</div>

resources/mustache/html/sentence.html

@@ -0,0 +1,6 @@
+<div class="row [[layout]]">
+    <div class="col origin">[[origin]]</div>
+    [[#translations]]
+    <div class="col translation">[[content]]</div>
+    [[/translations]]
+</div>

resources/mustache/tex/paragraph.tex

@@ -0,0 +1,6 @@
+\par [[origin]]
+
+[[#translations]]
+\par [[content]]
+[[/translations]]
+

resources/mustache/tex/sentence.tex

@@ -0,0 +1,5 @@
+\newline [[origin]]</div>
+[[#translations]]
+\newline [[content]]
+[[/translations]]
+

routes/api.php

@@ -69,6 +69,7 @@ use App\Http\Controllers\TransferController;
 use App\Http\Controllers\HealthCheckController;
 use App\Http\Controllers\OfflineIndexController;
 use App\Http\Controllers\TaskController;
+use App\Http\Controllers\ExportController;
 
 /*
 |--------------------------------------------------------------------------
@@ -190,7 +191,8 @@ Route::group(['prefix' => 'v2'],function(){
     Route::apiResource('health-check',HealthCheckController::class);
     Route::apiResource('offline-index',OfflineIndexController::class);
     Route::apiResource('task',TaskController::class);
-    
+    Route::apiResource('export',ExportController::class);
+
     Route::get('download/{type1}/{type2}/{uuid}/{filename}', function ($type1,$type2,$uuid,$filename) {
         header("Content-Type: {$type1}/{$type1}");
         header("Content-Disposition: attachment; filename=\"{$filename}\"");