Merge pull request #1662 from visuddhinanda/laravel

导出文件保存到s3

app/Console/Commands/ExportOffline.php

@@ -87,10 +87,10 @@ class ExportOffline extends Command
         $this->call('export:sentence',['--type'=>'original']);
 
         $this->info('zip');
-
+        Log::debug('export offline: db写入完毕 开始压缩');
         $exportPath = 'app/public/export/offline';
         $exportFile = 'wikipali-offline-'.date("Y-m-d").'.db3';
-        Log::debug('zip file {filename} {format}',
+        Log::debug('export offline: zip file {filename} {format}',
                     [
                         'filename'=>$exportFile,
                         'format'=>$this->argument('format')
@@ -119,10 +119,29 @@ class ExportOffline extends Command
             $command = "gzip -k -q --best -c {$exportFullFileName} > {$zipFullFileName}";
         }
         $this->info($command);
+        Log::debug('export offline: zip command:'.$command);
         shell_exec($command);
-
+        Log::debug('zip file {filename} in {format} saved.',
+                    [
+                        'filename'=>$exportFile,
+                        'format'=>$this->argument('format')
+                    ]);
         $info = array();
+        $url = array();
+        $url[] = [
+                'link'=>'https://www.wikipali.cc/downloads/'.$zipFile,
+                'hostname'=>'阿里云·中国',
+            ];
+        //s3
+        Storage::put($zipFile, file_get_contents($zipFullFileName));
+        $s3Link = Storage::url($zipFile);
+        Log::info('export offline: link='.$s3Link);
+        $url[] = [
+            'link'=>$s3Link,
+            'hostname'=>'Amazon cloud hongkong',
+        ];
         $info[] = ['filename'=>$zipFile,
+                    'url' => $url,
                    'create_at'=>date("Y-m-d H:i:s"),
                    'chapter'=>RedisClusters::get("/export/chapter/count"),
                    'filesize'=>filesize($zipFullFileName),
@@ -132,11 +151,7 @@ class ExportOffline extends Command
         unlink($exportStop);
         unlink($exportFullFileName);
 
-        Log::debug('zip file {filename} in {format} saved.',
-                    [
-                        'filename'=>$exportFile,
-                        'format'=>$this->argument('format')
-                    ]);
+
         return 0;
     }
 }

app/Console/Commands/MqDiscussion.php

@@ -52,7 +52,7 @@ class MqDiscussion extends Command
         $exchange = 'router';
         $queue = 'discussion';
         $this->info(" [*] Waiting for {$queue}. To exit press CTRL+C");
-        Log::info("discussion worker start . Waiting for {$queue}. ");
+        Log::info("discussion worker start .");
         Mq::worker($exchange,$queue,function ($message){
             Log::info('mq discussion start {message}',['message'=>json_encode($message,JSON_UNESCAPED_UNICODE)]);
             $result = 0;
@@ -97,7 +97,7 @@ class MqDiscussion extends Command
                                   ->where('title',$articleTitle)
                                   ->value('content');
                     if(empty($tpl)){
-                        Log::error('模版不能为空',['tpl_title'=>$articleTitle]);
+                        Log::error('mq:pr 模版不能为空',['tpl_title'=>$articleTitle]);
                         return 1;
                     }
                     $m = new \Mustache_Engine(array('entity_flags'=>ENT_QUOTES,
@@ -139,9 +139,11 @@ class MqDiscussion extends Command
                             $this->error($logMsg);
                         }
 
-                        if($ok===0){
+                        if($ok === 0){
+                            Log::debug('mq:pr: success {url}',['url'=>$hook->url]);
                             WebHook::where('id',$hook->id)->increment('success');
                         }else{
+                            Log::error('mq:pr: success {url}',['url'=>$hook->url]);
                             WebHook::where('id',$hook->id)->increment('fail');
                         }
                     }

app/Console/Commands/MqPr.php

@@ -9,6 +9,7 @@ use App\Models\WebHook;
 use App\Models\PaliSentence;
 use App\Tools\WebHook as WebHookSend;
 use App\Http\Api\MdRender;
+use Illuminate\Support\Facades\Log;
 
 class MqPr extends Command
 {
@@ -49,6 +50,7 @@ class MqPr extends Command
         $exchange = 'router';
         $queue = 'suggestion';
         $this->info(" [*] Waiting for {$queue}. To exit press CTRL+C");
+        Log::debug("mq:pr start.");
         Mq::worker($exchange,$queue,function ($message){
             /**生成消息内容 */
             $msgTitle = '修改建议';
@@ -93,9 +95,11 @@ class MqPr extends Command
                 }
                 $this->info("{$command}  ok={$ok}");
                 $result+=$ok;
-                if($ok===0){
+                if($ok === 0){
+                    Log::debug('mq:pr: send success {url}',['url'=>$hook->url]);
                     WebHook::where('id',$hook->id)->increment('success');
                 }else{
+                    Log::error('mq:pr: send fail {url}',['url'=>$hook->url]);
                     WebHook::where('id',$hook->id)->increment('fail');
                 }
             }

app/Console/Commands/MqProgress.php

@@ -4,6 +4,7 @@ namespace App\Console\Commands;
 
 use Illuminate\Console\Command;
 use App\Http\Api\Mq;
+use Illuminate\Support\Facades\Log;
 
 class MqProgress extends Command
 {
@@ -44,16 +45,22 @@ class MqProgress extends Command
         $exchange = 'router';
         $queue = 'progress';
         $this->info(" [*] Waiting for {$queue}. To exit press CTRL+C");
+        Log::debug("mq:progress start.");
         Mq::worker($exchange,$queue,function ($message){
-            $ok1 = $this->call('upgrade:progress',['--book'=>$message->book,
-                                            '--para'=>$message->para,
-                                            '--channel'=>$message->channel,
-                                            ]);
-            $ok2 = $this->call('upgrade:progress.chapter',['--book'=>$message->book,
-                                                '--para'=>$message->para,
-                                                '--channel'=>$message->channel,
-                                                ]);
+            $data = ['--book'=>$message->book,
+                        '--para'=>$message->para,
+                        '--channel'=>$message->channel,
+                    ];
+            $ok1 = $this->call('upgrade:progress',$data);
+            if($ok1 !== 0){
+                Log::error('mq:progress upgrade:progress fail',$data);
+            }
+            $ok2 = $this->call('upgrade:progress.chapter',$data);
+            if($ok2 !== 0){
+                Log::error('mq:progress upgrade:progress.chapter fail',$data);
+            }
             $this->info("Received book=".$message->book.' progress='.$ok1.' chapter='.$ok2);
+            Log::debug("mq:progress: done book=".$message->book.' progress='.$ok1.' chapter='.$ok2);
             return $ok1+$ok2;
         });
         return 0;

app/Console/Commands/MqWbwAnalyses.php

@@ -4,6 +4,7 @@ namespace App\Console\Commands;
 
 use Illuminate\Console\Command;
 use App\Http\Api\Mq;
+use Illuminate\Support\Facades\Log;
 
 class MqWbwAnalyses extends Command
 {
@@ -44,9 +45,16 @@ class MqWbwAnalyses extends Command
         $exchange = 'router';
         $queue = 'wbw-analyses';
         $this->info(" [*] Waiting for {$queue}. To exit press CTRL+C");
+        Log::debug("mq:wbw.analyses start.");
         Mq::worker($exchange,$queue,function ($message){
-            $ok = $this->call('upgrade:wbw.analyses',['id'=>implode(',',$message)]);
-            $this->info("Received count=".count($message).' ok='.$ok);
+            $data = ['id'=>implode(',',$message)];
+            $ok = $this->call('upgrade:wbw.analyses',$data);
+            if($ok === 0){
+                $this->info("Received count=".count($message).' ok='.$ok);
+                Log::debug('mq:wbw.analyses done count='.count($message));
+            }else{
+                Log::error('mq:wbw.analyses',$data);
+            }
             return $ok;
         });
 

app/Console/Commands/UpgradeProgressChapter.php

@@ -167,8 +167,8 @@ class UpgradeProgressChapter extends Command
                     $chapterData->all_trans = $final->cp_len/$chapter_strlen;
                     $chapterData->public = $final->cp_len/$chapter_strlen;
                     $chapterData->progress = $final->cp_len/$chapter_strlen;
-                    $chapterData->title = mb_substr($title,0,255,"UTF-8");
-                    $chapterData->summary = mb_substr($summaryText,0,255,"UTF-8");
+                    $chapterData->title = $title? mb_substr($title,0,255,"UTF-8"):"";
+                    $chapterData->summary = $summaryText? mb_substr($summaryText,0,255,"UTF-8"):"";
                     $chapterData->created_at = $finalAt;
                     $chapterData->updated_at = $updateAt;
                     $chapterData->save();

app/Http/Api/Mq.php

@@ -66,13 +66,6 @@ class Mq{
         $vhost = config("queue.connections.rabbitmq.virtual_host");
         $connection = new AMQPStreamConnection($host,$port,$user,$password,$vhost);
 
-    /*
-        $connection = new AMQPStreamConnection(env("RABBITMQ_HOST"),
-                                            env("RABBITMQ_PORT"),
-                                            env("RABBITMQ_USER"),
-                                            env("RABBITMQ_PASSWORD"),
-                                            env("RABBITMQ_VIRTUAL_HOST"));
-*/
         $channel = $connection->channel();
 
  /*