:construction: migrate user-dict

app/Models/UserDict.php

@@ -0,0 +1,11 @@
+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Model;
+
+class UserDict extends Model
+{
+    use HasFactory;
+}

database/migrations/2022_01_28_081954_create_user_dicts_table.php

@@ -0,0 +1,76 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class CreateUserDictsTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+     /*
+    id SERIAL PRIMARY KEY,
+	word VARCHAR(1024)  NOT NULL , 
+	type VARCHAR(128), 
+	gramma VARCHAR(128), 
+	stem VARCHAR(1024), 
+	mean TEXT, 
+	note TEXT, 
+	factors VARCHAR(1024), 
+	factormean TEXT, 
+	status INTEGER  NOT NULL DEFAULT (1),
+	source VARCHAR(256), 
+	language VARCHAR(16)  NOT NULL DEFAULT ('en'), 
+	confidence INTEGER  NOT NULL DEFAULT (100), 
+	creator_id INTEGER   NOT NULL , 
+	ref_counter INTEGER DEFAULT (1),
+	create_time INTEGER  NOT NULL ,
+	created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+	updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
+     */
+    public function up()
+    {
+        Schema::create('user_dicts', function (Blueprint $table) {
+			#使用雪花id
+            $table->bigInteger('id')->primary();
+
+			$table->string('word',1024)->index();
+			$table->string('type',128)->nullable();
+			$table->string('gramma',128)->nullable();
+			$table->string('parent',1024)->nullable();
+			$table->text('mean')->nullable();
+			$table->text('note')->nullable();
+			$table->string('factors',1024)->nullable();
+			$table->string('factormean',1024)->nullable();
+		    $table->integer('status')->default(10);
+			$table->string('source',1024)->nullable();
+
+			$table->string('language',16)->default('en-US');
+		    $table->integer('confidence')->default(100);
+		    $table->integer('exp')->default(0);
+            $table->bigInteger('creator_id')->index();
+            $table->bigInteger('ref_counter')->default(1);
+
+			$table->bigInteger('create_time')->index();
+
+			$table->timestamp('created_at')->useCurrent();
+			$table->timestamp('updated_at')->useCurrent()->useCurrentOnUpdate();
+			$table->timestamp('deleted_at')->nullable();
+
+			$table->index(['word','creator_id']);
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::dropIfExists('user_dicts');
+    }
+}

v1/scripts/migrations/20211210160700_user_dict_copy.php

@@ -4,26 +4,39 @@
 插入时用uuid判断是否曾经插入
 曾经插入就不插入了
 */
-require_once __DIR__."/../../app/config.php";
+require_once __DIR__."/../../../public/app/config.php";
+require_once __DIR__."/../../../public/app/public/snowflakeid.php";
+
+set_exception_handler(function($e){
+	fwrite(STDERR,"error-msg:".$e->getMessage().PHP_EOL);
+	fwrite(STDERR,"error-file:".$e->getFile().PHP_EOL);
+	fwrite(STDERR,"error-line:".$e->getLine().PHP_EOL);
+	exit;
+});
+$start = time();
+# 雪花id
+$snowflake = new SnowFlakeId();
+
+$fpError = fopen(__DIR__.'/log/'.basename($_SERVER['PHP_SELF'],'.php').".err.data.csv",'w');
 
 # 更新索引表
-$src_db=_FILE_SRC_WBW_;#源数据库
-$src_table=_TABLE_SRC_DICT_WBW_;#源表名
-$dest_db=_FILE_DB_WBW_;#目标数据库
-$dest_table=_TABLE_DICT_WBW_;#目标表名
+$src_db=_SQLITE_DB_WBW_;#源数据库
+$src_table=_SQLITE_TABLE_DICT_WBW_;#源表名
+$dest_db=_PG_DB_WBW_;#目标数据库
+$dest_table=_PG_TABLE_DICT_WBW_;#目标表名
 
 echo "migarate user dict".PHP_EOL;
 #打开源数据库
 $PDO_SRC = new PDO($src_db,_DB_USERNAME_,_DB_PASSWORD_,array(PDO::ATTR_PERSISTENT=>true));
-$PDO_SRC->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_WARNING);
+$PDO_SRC->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
 echo "open src".PHP_EOL;
 
 #打开目标数据库
 $PDO_DEST = new PDO($dest_db,_DB_USERNAME_,_DB_PASSWORD_,array(PDO::ATTR_PERSISTENT=>true));
-$PDO_DEST->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_WARNING);
+$PDO_DEST->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
 echo "open dest".PHP_EOL;
 
-#删除源数据表中全部数据
+#删除目标数据表中全部数据
 fwrite(STDOUT,"delete dest".PHP_EOL);
 
 $query = "delete from $dest_table where true;";
@@ -32,6 +45,7 @@ $stmtDest->execute();
 
 $queryInsert = "INSERT INTO ".$dest_table." 
 								(
+                                    id,
 									word , 
 									type, 
 									gramma,
@@ -50,7 +64,7 @@ $queryInsert = "INSERT INTO ".$dest_table."
 									created_at,
 									updated_at
 								) 
-									VALUES ( ? , ? , ? ,? ,? ,? ,? ,? ,? ,? , ?,?,?,?,?,to_timestamp(?),to_timestamp(?))";
+									VALUES (? , ? , ? , ? ,? ,? ,? ,? ,? ,? ,? , ?,?,?,?,?,to_timestamp(?),to_timestamp(?))";
 
 echo "read from orginal".PHP_EOL;
 $commitData = [];
@@ -65,46 +79,52 @@ $stmtSrc->execute();
 while($srcData = $stmtSrc->fetch(PDO::FETCH_ASSOC)){
 	$allSrcCount++;
 	#插入目标表
-	
-	//if(strlen($srcData["id"])>10 && strlen($srcData["owner"])>30)
-	{
-
-		{
-			#没有相同数据
-			$commitData[] = array(
-					$srcData["pali"],
-					$srcData["type"],
-					$srcData["gramma"],
-					$srcData["parent"],
-					$srcData["mean"],
-					$srcData["note"],
-					$srcData["factors"],
-					$srcData["factormean"],
-					(int)$srcData["status"],
-					"_SYS_USER_WBW_",
-					$srcData["language"],
-					(int)$srcData["confidence"],
-					(int)$srcData["creator"],
-					$srcData["ref_counter"],
-					$srcData["time"],
-					$srcData["time"],
-					$srcData["time"]
-				);	
+        if(empty($srcData["pali"]) ){
+            fwrite(STDERR,"pali is null id=".$srcData["id"].PHP_EOL);
+            fputcsv($fpError,$srcData);
+            continue;
+        }
+        if(empty($srcData["creator"]) ){
+            fwrite(STDERR,"creator is null id=".$srcData["id"].PHP_EOL);
+            fputcsv($fpError,$srcData);
+            continue;
+        }
+        $commitData[] = array(
+                $snowflake->id(),
+                $srcData["pali"],
+                $srcData["type"],
+                $srcData["gramma"],
+                $srcData["parent"],
+                $srcData["mean"],
+                $srcData["note"],
+                $srcData["factors"],
+                $srcData["factormean"],
+                $srcData["status"],
+                "_SYS_USER_WBW_",
+                $srcData["language"],
+                $srcData["confidence"],
+                $srcData["creator"],
+                $srcData["ref_counter"],
+                $srcData["time"],
+                $srcData["time"],
+                $srcData["time"]
+            );	
 			$count++;	
 			$allInsertCount++;
-		}
+		
 
 		if($count ==10000){
 			#10000行插入一次
 			$PDO_DEST->beginTransaction();
 			$stmtDEST = $PDO_DEST->prepare($queryInsert);
 			foreach ($commitData as $key => $value) {
-				$stmtDEST->execute($value);
-				if (!$stmtDEST || ($stmtDEST && $stmtDEST->errorCode() != 0)) {
-					$error = $PDO_DEST->errorInfo();
-					echo "error - $error[2] ";
-					exit;
-				}
+                try{
+                    $stmtDEST->execute($value);	
+                }catch(PDOException $e){
+                    fwrite(STDERR,$e->getMessage().PHP_EOL);
+                    fwrite(STDERR,implode(',',$value).PHP_EOL);
+                    exit;
+                }
 			}
 			// 提交更改
 			$PDO_DEST->commit();
@@ -112,7 +132,9 @@ while($srcData = $stmtSrc->fetch(PDO::FETCH_ASSOC)){
 			echo "finished $count".PHP_EOL;
 			$count=0;
 		}	
-	}
+		if($allSrcCount % 10000 ==0){
+            echo "find from src table $allSrcCount / $allInsertCount is new.".PHP_EOL;
+        }
 }
 if($count>0){
 	#最后的没有到10000的数据插入
@@ -120,11 +142,7 @@ if($count>0){
 	$stmtDEST = $PDO_DEST->prepare($queryInsert);
 	foreach ($commitData as $key => $value) {
 		$stmtDEST->execute($value);
-		if (!$stmtDEST || ($stmtDEST && $stmtDEST->errorCode() != 0)) {
-			$error = $PDO_DEST->errorInfo();
-			echo "error - $error[2] ";
-			exit;
-		}
+
 	}
 	// 提交更改
 	$PDO_DEST->commit();
@@ -136,33 +154,11 @@ echo "insert done $allInsertCount in $allSrcCount ".PHP_EOL;
 
 # 更新索引表
 
-$src_table=_TABLE_SRC_DICT_WBW_INDEX_;#源表名
-$src_word_table=_TABLE_SRC_DICT_WBW_;#源word表名
+$src_table=_SQLITE_TABLE_DICT_WBW_INDEX_;#源表名
+$src_word_table=_SQLITE_TABLE_DICT_WBW_;#源word表名
 
 echo "migarating usr dict index ".PHP_EOL;
 
-$queryInsert = "INSERT INTO ".$dest_table." 
-								(
-									word , 
-									type, 
-									gramma,
-									parent,
-									mean,
-									note,
-									factors,
-									factormean,
-									status,
-									source,
-									language,
-									confidence,
-									creator_id,
-									ref_counter,
-									create_time,
-									created_at,
-									updated_at
-								) 
-									VALUES ( ? , ? , ? ,? ,? ,? ,? ,? ,? ,? , ?,?,?,?,?, to_timestamp(?), to_timestamp(?))";
-
 
 // 开始一个事务，关闭自动提交
 $commitData = [];
@@ -170,7 +166,7 @@ $allInsertCount = 0;
 $allSrcCount = 0;
 $count = 0;
 #从源数据表中读取
-$query = "SELECT *  FROM ".$src_table." WHERE true ";
+$query = "SELECT *  FROM ".$src_table;
 $stmtSrc = $PDO_SRC->prepare($query);
 $stmtSrc->execute();
 
@@ -184,7 +180,21 @@ while($srcData = $stmtSrc->fetch(PDO::FETCH_ASSOC)){
 	$getWord->execute(array($wordIndex));
 	$exist = $getWord->fetch(PDO::FETCH_ASSOC);
 	if($exist){
+        if(empty($srcData["user_id"]) ){
+            fwrite(STDERR,"index user_id is null id=".$srcData["id"].PHP_EOL);
+            fputcsv($fpError,$srcData);
+            continue;
+        }
+        if(empty($exist["pali"]) ){
+            fwrite(STDERR,"pali is null id=".$srcData["id"].PHP_EOL);
+            continue;
+        }
+        if(empty($exist["creator"]) ){
+            fwrite(STDERR,"creator is null id=".$srcData["id"].PHP_EOL);
+            continue;
+        }
 		$commitData[] = array(
+            $snowflake->id(),
 			$exist["pali"],
 			$exist["type"],
 			$exist["gramma"],
@@ -196,8 +206,8 @@ while($srcData = $stmtSrc->fetch(PDO::FETCH_ASSOC)){
 			(int)$exist["status"],
 			"_USER_WBW_",
 			$exist["language"],
-			(int)$exist["confidence"],
-			(int)$srcData["user_id"],
+			$exist["confidence"],
+		    $srcData["user_id"],
 			$exist["ref_counter"],
 			$exist["time"],
 			$exist["time"],
@@ -211,14 +221,10 @@ while($srcData = $stmtSrc->fetch(PDO::FETCH_ASSOC)){
 			foreach ($commitData as $key => $value) {
 				# code...
 				$stmtDEST->execute($value);
-				if (!$stmtDEST || ($stmtDEST && $stmtDEST->errorCode() != 0)) {
-					$error = $PDO_DEST->errorInfo();
-					echo "error - $error[2] ";
-					exit;
-				}	
 			}
 			// 提交更改
 			$PDO_DEST->commit();
+            $commitData = [];
 			echo "finished $count".PHP_EOL;
 			$count = 0;
 		}
@@ -236,20 +242,16 @@ if($count>0){
 	foreach ($commitData as $key => $value) {
 		# code...
 		$stmtDEST->execute($value);
-		if (!$stmtDEST || ($stmtDEST && $stmtDEST->errorCode() != 0)) {
-			$error = $PDO_DEST->errorInfo();
-			fwrite(STDOUT,"error - $error[2] ");
-			exit;
-		}	
 	}
 	// 提交更改
 	$PDO_DEST->commit();
+    $commitData = [];
 	fwrite(STDOUT,"finished $count".PHP_EOL);
 }
 
 fwrite(STDOUT,"insert done $allInsertCount in $allSrcCount ".PHP_EOL);
 
-fwrite(STDOUT,"all done".PHP_EOL);
+fwrite(STDOUT, "all done in ".(time()-$start)."s".PHP_EOL);