sent pr 迁移测试完成

app/Models/SentPr.php

@@ -0,0 +1,11 @@
+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Model;
+
+class SentPr extends Model
+{
+    use HasFactory;
+}

database/migrations/2022_01_27_132319_create_sent_prs_table.php

@@ -0,0 +1,58 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class CreateSentPrsTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+
+    public function up()
+    {
+        Schema::create('sent_prs', function (Blueprint $table) {
+			#使用雪花id
+            $table->bigInteger('id')->primary();
+            //用作数据迁移时的相同数据比对
+			$table->integer('old_id')->uniqid();
+
+			$table->integer('book_id');
+			$table->integer('paragraph');
+            $table->integer('word_start');
+			$table->integer('word_end');
+			$table->string('channel_uid',36);
+
+			$table->string('author',512)->nullable();
+			$table->string('editor_uid',36);
+
+			$table->text('content');
+			$table->string('language',16);
+			$table->integer('status')->default(10);
+			$table->integer('strlen')->default(0);
+
+			$table->bigInteger('create_time')->index();
+            $table->bigInteger('modify_time')->index();
+
+			$table->timestamp('created_at')->useCurrent();
+			$table->timestamp('updated_at')->useCurrent()->useCurrentOnUpdate();
+			$table->timestamp('deleted_at')->nullable();
+
+			$table->index(['book_id','paragraph','word_start','word_end','channel_uid']);
+
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::dropIfExists('sent_prs');
+    }
+}

v1/scripts/migrations/20211207164600_sent_pr_copy.php

@@ -5,28 +5,43 @@
 插入时用uuid判断是否曾经插入
 曾经插入就不插入了
 */
-require_once __DIR__."/../../app/config.php";
+require_once __DIR__."/../../../public/app/config.php";
+require_once __DIR__."/../../../public/app/public/snowflakeid.php";
+
+set_exception_handler(function($e){
+	fwrite(STDERR,"error-msg:".$e->getMessage().PHP_EOL);
+	fwrite(STDERR,"error-file:".$e->getFile().PHP_EOL);
+	fwrite(STDERR,"error-line:".$e->getLine().PHP_EOL);
+	exit;
+});
+
+$fpError = fopen(__DIR__.'/log/'.basename($_SERVER['PHP_SELF'],'.php').".err.data.csv",'w');
+
+$start = time();
+# 雪花id
+$snowflake = new SnowFlakeId();
 
 # 更新索引表
-$src_db=_SRC_DB_SENTENCE_;#源数据库
-$src_table=_TABLE_SRC_SENTENCE_PR_;#源表名
-$dest_db=_FILE_DB_SENTENCE_;#目标数据库
-$dest_table=_TABLE_SENTENCE_PR_;#目标表名
+$src_db=_SQLITE_DB_SENTENCE_;#源数据库
+$src_table=_SQLITE_TABLE_SENTENCE_PR_;#源表名
+$dest_db=_PG_DB_SENTENCE_;#目标数据库
+$dest_table=_PG_TABLE_SENTENCE_PR_;#目标表名
 
 fwrite(STDOUT,"migarate sent_pr".PHP_EOL);
 #打开源数据库
 $PDO_SRC = new PDO($src_db,_DB_USERNAME_,_DB_PASSWORD_,array(PDO::ATTR_PERSISTENT=>true));
-$PDO_SRC->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_WARNING);
+$PDO_SRC->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
 fwrite(STDOUT,"open src table".PHP_EOL);
 
 #打开目标数据库
 $PDO_DEST = new PDO($dest_db,_DB_USERNAME_,_DB_PASSWORD_,array(PDO::ATTR_PERSISTENT=>true));
-$PDO_DEST->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_WARNING);
+$PDO_DEST->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
 fwrite(STDOUT,"open dest table".PHP_EOL);
 
 $queryInsert = "INSERT INTO ".$dest_table." 
 								(
 									id, 
+                                    old_id,
 									book_id,
 									paragraph,
 									word_start,
@@ -42,7 +57,7 @@ $queryInsert = "INSERT INTO ".$dest_table."
 									modify_time,
 									created_at,
 									updated_at) 
-									VALUES ( ? , ? , ? , ? ,? ,? ,? ,? ,? ,? ,?,?,?,?,to_timestamp(?),to_timestamp(?))";
+									VALUES ( ? , ? , ? , ? , ? ,? ,? ,? ,? ,? ,? ,?,?,?,?,to_timestamp(?),to_timestamp(?))";
 
 
 $commitData = [];
@@ -64,29 +79,70 @@ while($srcData = $stmtSrc->fetch(PDO::FETCH_ASSOC)){
 	{
 		$uuid = $srcData["id"];
 		#查询目标表中是否有相同数据
-		$queryExsit = "SELECT id  FROM ".$dest_table." WHERE id = ? ";
+		$queryExsit = "SELECT id  FROM ".$dest_table." WHERE old_id = ? ";
 		$getExist = $PDO_DEST->prepare($queryExsit);
 		$getExist->execute(array($uuid));
 		$exist = $getExist->fetch(PDO::FETCH_ASSOC);
 		if(!$exist){
 			#没有相同数据
-			if(strlen($srcData["editor"])>36){
-				fwrite(STDERR,"error: {$uuid} editor {$srcData["editor"]} is too long".PHP_EOL);
-				continue;
-			}
+		if(strlen($srcData["editor"])>36){
+			fwrite(STDERR,"error: {$uuid} editor {$srcData["editor"]} too long".PHP_EOL);
+			continue;
+		}
+        if(empty($srcData["language"]) ){
+            $srcData["language"]="zh-hans";
+        }
+        if(empty($srcData["strlen"]) ){
+            $srcData["strlen"]=mb_strlen($srcData["strlen"],"UTF-8");
+        }
+        if(empty($srcData["status"]) ){
+            $srcData["status"]=10;
+        }
+        if(empty($srcData["book"]) || !is_numeric($srcData["book"])){
+            fwrite(STDERR,"book is error id=".$uuid.PHP_EOL);
+            fputcsv($fpError,$srcData);
+            continue;
+        }
+        if(empty($srcData["paragraph"]) || !is_numeric($srcData["paragraph"])){
+            fwrite(STDERR,"paragraph is error id=".$uuid.PHP_EOL);
+            fputcsv($fpError,$srcData);
+            continue;
+        }
+        if(empty($srcData["begin"]) || !is_numeric($srcData["begin"]) ){
+            fwrite(STDERR,"begin is error id=".$uuid.PHP_EOL);
+            fputcsv($fpError,$srcData);
+            continue;
+        }
+        if(empty($srcData["end"]) || !is_numeric($srcData["end"]) ){
+            fwrite(STDERR,"end is error id=".$uuid.PHP_EOL);
+            fputcsv($fpError,$srcData);
+            continue;
+        }
+        if(empty($srcData["channel"]) ){
+            fwrite(STDERR,"end is error id=".$uuid.PHP_EOL);
+            fputcsv($fpError,$srcData);
+            continue;
+        }
+        if(empty($srcData["create_time"]) || $srcData["create_time"]<1532590551000){
+            $srcData["create_time"]=1532590551000;
+        }
+        if(empty($srcData["modify_time"]) || $srcData["modify_time"]<1532590551000){
+            $srcData["modify_time"]=1532590551000;
+        }
 			$commitData[] = array(
+                    $snowflake->id(),
 					$uuid,
-					(int)$srcData["book"],
-					(int)$srcData["paragraph"],
-					(int)$srcData["begin"],
-					(int)$srcData["end"],
+					$srcData["book"],
+					$srcData["paragraph"],
+					$srcData["begin"],
+					$srcData["end"],
 					$srcData["channel"],
 					$srcData["author"],
 					$srcData["editor"],
 					$srcData["text"],
 					$srcData["language"],
-					(int)$srcData["status"],
-					(int)$srcData["strlen"],
+					$srcData["status"],
+					$srcData["strlen"],
 					$srcData["create_time"],
 					$srcData["modify_time"],
 					$srcData["create_time"]/1000,
@@ -102,19 +158,23 @@ while($srcData = $stmtSrc->fetch(PDO::FETCH_ASSOC)){
 			$PDO_DEST->beginTransaction();
 			$stmtDEST = $PDO_DEST->prepare($queryInsert);
 			foreach ($commitData as $key => $value) {
-				$stmtDEST->execute($value);
-				if (!$stmtDEST || ($stmtDEST && $stmtDEST->errorCode() != 0)) {
-					$error = $PDO_DEST->errorInfo();
-					echo "error - $error[2] ";
-					exit;
-				}
+                try{
+                    $stmtDEST->execute($value);	
+                }catch(PDOException $e){
+                    fwrite(STDERR,$e->getMessage().PHP_EOL);
+                    fwrite(STDERR,implode(',',$value).PHP_EOL);
+                    exit;
+                }
 			}
 			// 提交更改
 			$PDO_DEST->commit();
 			$commitData = [];
-			echo "finished $count".PHP_EOL;
+			fwrite(STDOUT, "finished $count".PHP_EOL);
 			$count=0;
 		}	
+        if($allSrcCount % 10000 ==0){
+		    echo "find from src table $allSrcCount / $allInsertCount is new.".PHP_EOL;
+	    }
 	}
 }
 if($count>0){
@@ -122,24 +182,25 @@ if($count>0){
 	$PDO_DEST->beginTransaction();
 	$stmtDEST = $PDO_DEST->prepare($queryInsert);
 	foreach ($commitData as $key => $value) {
-		$stmtDEST->execute($value);
-		if (!$stmtDEST || ($stmtDEST && $stmtDEST->errorCode() != 0)) {
-			$error = $PDO_DEST->errorInfo();
-			echo "error - $error[2] ";
-			exit;
-		}
+        try{
+            $stmtDEST->execute($value);	
+        }catch(PDOException $e){
+            fwrite(STDERR,$e->getMessage().PHP_EOL);
+            fwrite(STDERR,implode(',',$value).PHP_EOL);
+            exit;
+        }
 	}
 	// 提交更改
 	$PDO_DEST->commit();
 	$commitData = [];
-	echo "finished $count".PHP_EOL;
+	fwrite(STDOUT, "finished $count".PHP_EOL);
 }
 
-echo "insert done $allInsertCount in $allSrcCount ".PHP_EOL;
-
-echo "all done".PHP_EOL;
+fwrite(STDOUT, "insert done $allInsertCount in $allSrcCount ".PHP_EOL);
 
+fwrite(STDOUT, "all done in ".(time()-$start)."s".PHP_EOL);
 
+fclose($fpError);