:wrench: merge deployment scripts v2.2

deploy/mint.yml

@@ -23,3 +23,9 @@
   hosts: all:!localhost
   roles:
     - mint-v2.2
+
+- name: Setup nginx
+  hosts:
+    - web
+  roles:
+    - mint-nginx

deploy/roles/mint-nginx/tasks/main.yml

@@ -10,8 +10,9 @@
   become: true
   ansible.builtin.template:
     src: fpm.conf.j2
-    dest: /etc/nginx/sites-enabled/{{ app_domain }}-fpm.conf
+    dest: /etc/nginx/sites-enabled/php-fpm-{{ app_php_fpm_port }}.conf
     mode: "0644"
+    force: false
 
 - name: Upload nginx.conf
   become: true
@@ -19,3 +20,8 @@
     src: nginx.conf.j2
     dest: /etc/nginx/sites-enabled/{{ app_domain }}.conf
     mode: "0644"
+    force: false
+
+- name: Reload nginx.conf
+  become: true
+  ansible.builtin.shell: nginx -s reload

deploy/roles/mint-nginx/templates/fpm.conf.j2

@@ -1,5 +1,5 @@
 upstream php_fpm_{{ app_php_fpm_port }} {
-   least_conn;
+   # least_conn;
 
    # server 192.168.0.11:{{ app_php_fpm_port }} weight=10;   
 

deploy/roles/mint-nginx/templates/nginx.conf.j2

@@ -3,7 +3,7 @@
 server {
     listen 80;
     server_name {{ app_domain }};
-    root {{ app_deploy_target | dirname }}/current/api-v8/public;
+    root {{ app_deploy_target }}/api-v8/public;
  
     add_header X-Frame-Options "SAMEORIGIN";
     add_header X-Content-Type-Options "nosniff";
@@ -23,7 +23,7 @@ server {
     # error_log  syslog:server=unix:/dev/log,tag={{ app_domain }},nohostname,severity=error;
 
     location {{ app_dashboard_base_path }}/ {
-            alias {{ app_deploy_target | dirname }}/current/dashboard-v4/dashboard/dist/;
+            alias {{ app_deploy_target }}/dashboard-v4/dashboard/dist/;
             try_files $uri $uri/ {{ app_dashboard_base_path }}/index.html;
 
             location ~* \.(css|js|png|jpg|jpeg|gif|gz|svg|mp4|ogg|ogv|webm|htc|xml|woff)$ {

deploy/roles/mint-v2.2/tasks/clove.yml

@@ -0,0 +1,27 @@
+- name: Create clove folder
+  become: true
+  ansible.builtin.file:
+    path: "{{ app_deploy_target | dirname }}/clove"
+    state: directory
+    owner: "{{ ansible_user }}"
+    mode: "0755"
+
+- name: Upload clove data
+  ansible.builtin.unarchive:
+    src: "{{ playbook_dir }}/tmp/clove.tar.xz"
+    dest: "{{ app_deploy_target | dirname }}/clove"
+    creates: "{{ app_deploy_target | dirname }}/clove/dict_text"
+
+- name: Setup clove resources for v8
+  ansible.builtin.file:
+    src: "{{ app_deploy_target | dirname }}/clove"
+    dest: "{{ app_deploy_target }}/api-v8/storage/resources"
+    state: link
+    force: true
+
+- name: Setup clove resources for v12
+  ansible.builtin.file:
+    src: "{{ app_deploy_target | dirname }}/clove"
+    dest: "{{ app_deploy_target }}/api-v12/storage/resources"
+    state: link
+    force: true

deploy/roles/mint-v2.2/tasks/laravel.yml

@@ -0,0 +1,27 @@
+- name: Upload .env(v2)
+  ansible.builtin.template:
+    src: v2/env.j2
+    dest: "{{ app_deploy_target }}/api-v8/.env"
+    mode: "0444"
+
+- name: Upload config.php(v1)
+  ansible.builtin.template:
+    src: v1/config.php.j2
+    dest: "{{ app_deploy_target }}/api-v8/public/app/config.php"
+    mode: "0444"
+
+- name: Upload config.js(v1)
+  ansible.builtin.template:
+    src: v1/config.js.j2
+    dest: "{{ app_deploy_target }}/api-v8/public/app/config.js"
+    mode: "0444"
+
+- name: "Setup configuration"
+  ansible.builtin.shell:
+    cmd: "docker run --rm -it --hostname=mint --network host -v /srv/{{ app_domain }}/clove:/srv/{{ app_domain }}/clove:z -v {{ app_deploy_target }}/api-v8:{{ app_deploy_target }}/api-v8:z {{ app_docker_image_name }} {{ app_deploy_target }}/scripts/run.sh setup"
+
+- name: Create a current link
+  ansible.builtin.file:
+    src: "{{ app_deploy_target }}"
+    dest: "{{ app_deploy_target | dirname }}/current"
+    state: link

deploy/roles/mint-v2.2/tasks/main.yml

@@ -0,0 +1,75 @@
+- name: Create mint folder
+  become: true
+  ansible.builtin.file:
+    path: "{{ app_deploy_target }}"
+    state: directory
+    owner: "{{ ansible_user }}"
+    mode: "0755"
+
+- name: Upload mint data
+  ansible.builtin.unarchive:
+    src: "{{ playbook_dir }}/tmp/{{ app_domain }}-mint-{{ mint_version }}.tar.xz"
+    dest: "{{ app_deploy_target }}"
+    creates: "{{ app_deploy_target }}/api-v12"
+
+- name: Setup clove
+  ansible.builtin.import_tasks: clove.yml
+
+# -----------------------------------------------------------------------------
+
+- name: Create scripts folder
+  ansible.builtin.file:
+    path: "{{ app_deploy_target }}/scripts"
+    state: directory
+    mode: "0755"
+
+- name: Upload setup.sh
+  ansible.builtin.template:
+    src: containers/shell.sh.j2
+    dest: "{{ app_deploy_target }}/scripts/shell.sh"
+    mode: "0555"
+
+- name: Upload run.sh
+  ansible.builtin.template:
+    src: containers/run.sh.j2
+    dest: "{{ app_deploy_target }}/scripts/run.sh"
+    mode: "0555"
+
+# -----------------------------------------------------------------------------
+
+- name: Setup laravel
+  ansible.builtin.import_tasks: laravel.yml
+
+# -----------------------------------------------------------------------------
+
+- name: Upload config for ai.translate
+  ansible.builtin.template:
+    src: v2/ai-translate.toml.j2
+    dest: "{{ app_deploy_target }}/ai-translate/config.toml"
+    mode: "0444"
+
+- name: Upload config for openai-proxy
+  ansible.builtin.template:
+    src: v2/openai-proxy.json.j2
+    dest: "{{ app_deploy_target }}/open-ai-server/config.json"
+    mode: "0444"
+
+# -----------------------------------------------------------------------------
+
+- name: Upload version.txt(api-v8)
+  ansible.builtin.template:
+    src: version.txt.j2
+    dest: "{{ app_deploy_target }}/api-v8/public/version.txt"
+    mode: "0555"
+
+- name: Upload version.txt(api-v12)
+  ansible.builtin.template:
+    src: version.txt.j2
+    dest: "{{ app_deploy_target }}/api-v12/public/version.txt"
+    mode: "0555"
+
+- name: Upload version.txt(dashboard-v4)
+  ansible.builtin.template:
+    src: version.txt.j2
+    dest: "{{ app_deploy_target }}/dashboard-v4/dashboard/dist/version.txt"
+    mode: "0555"

deploy/roles/mint-v2.2/templates/containers/run.sh.j2

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -e
+
+if [ "$#" -ne 1 ]; then
+    echo "USAGE: $0 COMMAND"
+    exit 1
+fi
+
+export WORK_DIR="{{ app_deploy_target }}"
+
+
+if [[ "$1" == "mq:discussion" || "$1" == "mq:pr" || "$1" == "mq:progress" || "$1" == "mq:wbw.analyses" || "$1" == "mq:export.pali.chapter" || "$1" == "mq:export.article" || "$1" == "schedule:run" ]]; then
+    cd $WORK_DIR/api-v8/    
+    php -d memory_limit={{ app_php_memory_limit }} artisan $1
+elif [ "$1" = "ai-translate" ]; then
+    cd $WORK_DIR/ai-translate/
+    source $PWD/python3/bin/activate
+    # https://github.com/iapt-platform/mint/blob/df8e1cf7ade16d17add360e7a869540c1ddaf1b9/api-v8/config/mint.php#L129
+    python3 -m ai_translate -c config.toml -n ai.translate -q ai_translate_v2
+elif [ "$1" = "openai-proxy" ]; then
+    cd $WORK_DIR/openai-proxy/
+    source "$HOME/.nvm/nvm.sh"    
+    node dist/main.*.js config.json
+elif [ "$1" = "php-fpm" ]; then
+    sed -i 's/^listen = .*/listen = 9000/g' /etc/php/{{ app_php_version }}/fpm/pool.d/www.conf
+    /usr/sbin/php-fpm{{ app_php_version }} --nodaemonize --fpm-config /etc/php/{{ app_php_version }}/fpm/php-fpm.conf
+elif [ "$1" = "expected_value" ]; then
+    cd $WORK_DIR/api-v8/
+    echo "caching configuration "
+    php artisan config:cache
+    echo "caching events"
+    php artisan event:cache
+    echo "caching routes"
+    php artisan route:cache
+    echo "caching views"
+    php artisan view:cache
+
+    echo "check file permissions"
+    cd $WORK_DIR/api-v8/
+    chown -R www-data:www-data bootstrap/cache storage
+else
+    echo "unknown command $1"
+    exit 1
+fi
+
+echo 'done.'
+exit 0

deploy/roles/mint-v2.2/templates/containers/schedule-run.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Runs Mint {{ app_domain }} scheduler every minute
+
+[Timer]
+OnBootSec=15minutes
+OnUnitActiveSec=1minute
+Unit={{ app_container_prefix }}-{{ app_domain }}-worker-schedule-run.service
+
+[Install]
+WantedBy=timers.target

deploy/roles/mint-v2.2/templates/containers/shell.sh.j2

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+docker run --rm -it --hostname=mint --network host \
+    -v /srv/{{ app_deploy_target | dirname }}/clove:/srv/{{ app_deploy_target | dirname }}/clove:z \
+    -v {{ app_deploy_target }}/api-v8:{{ app_deploy_target }}/api-v8:z \
+    {{ app_docker_image_name }} 

deploy/roles/mint-v2.2/templates/v1/config.js.j2

@@ -0,0 +1,18 @@
+var ICP_CODE = "{{ app_icp_code }}"
+var GRPC_WEB_SERVER = "{{ app_grpc_web_server }}";
+var DOCUMENTS_SERVER = "{{ app_documents_server }}";
+var REACT_APP_QUESTIONNAIRE_LINK = "{{ app_questionnaire_link }}";
+var DASHBOARD_BASE_PATH = "{{ app_dashboard_base_path }}";
+
+/*
+  |---------------
+  |网站资源文件，非用户的图片，音频，视频
+  |---------------
+  |对应/public/tmp/ 目录 开发线可以设置为 http://127.0.0.1:8000/tmp
+  |所有文件存储在 https://drive.google.com/drive/folders/1-4dn4juD-0-lsKndDui2W9nT9wcS_Y33?usp=sharing
+  |开发线可自行下载放到/public/tmp/
+  |或直接引用离您最近的assets server
+  |------------------------
+*/
+
+var ASSETS_SERVER = "{{ app_assets_server }}";

deploy/roles/mint-v2.2/templates/v1/config.php.j2

@@ -0,0 +1,67 @@
+<?php
+
+#域名设置
+define("GRPC_WEB_SERVER","{{ app_grpc_web_server}}");
+define("ASSETS_SERVER","{{ app_assets_server }}");
+define("DOCUMENTS_SERVER","{{ app_documents_server }}");
+define('APP_KEY','{{ app_secret_key }}');
+define('APP_ENV','{{ app_env }}');
+define('CORS_ALLOWED_ORIGINS', '{{ app_cors_allowed_origins }}');
+define('DASHBOARD_BASE_PATH', "{{ app_dashboard_base_path }}");
+
+/*
+电子邮件设置
+PHPMailer
+*/
+define("Email", [
+                                 "Host"=>"{{ app_smtp_host }}",//Set the SMTP server to send through
+                                 "SMTPAuth"=>true,//Enable SMTP authentication
+                                 "Username"=>'{{ app_smtp_user }}',//SMTP username
+                                 "Password"=>'{{ app_smtp_password }}',//SMTP password
+                                 "Port"=>{{ app_smtp_port }},//TCP port to connect to 465; use 587 if you have set `SMTPSecure = PHPMailer::ENCRYPTION_STARTTLS`
+                                 "From"=>"{{ app_smtp_user }}",
+                                 "Sender"=>"webmaster"
+                                 ]);
+
+/*
+数据库设置
+*/
+define("Database",[
+        "type"=>"pgsql",
+        "server"=>"{{ app_postgresql_host }}",
+        "port"=>{{ app_postgresql_port }},
+        "name"=>"{{ app_postgresql_dbname }}",
+        "sslmode" => "disable",
+        "user" => "{{ app_postgresql_user }}",
+        "password" => "{{ app_postgresql_password }}"
+]);
+
+
+/*
+Redis 设置，
+*/
+define("Redis",[        
+        "host" => "{{ app_redis_host }}",
+        "port" => {{ app_redis_port }},
+        "password" => "",
+        "prefix"=>"{{ app_domain }}://v1/"
+]);
+
+
+# 雪花id
+define("SnowFlake",[
+        "DatacenterId"=>{{ app_snowflake_data_center_id }},
+        "WorkerId"=>{{ app_snowflake_worker_id }}
+]);
+
+#目录设置，不能更改
+require_once __DIR__."/config.dir.php";
+
+/*
+数据表
+*/
+#表设置，此行不能更改
+require_once __DIR__."/config.table.php";
+
+
+?>

deploy/roles/mint-v2.2/templates/v2/ai-translate.toml.j2

@@ -0,0 +1,16 @@
+[rabbitmq]
+host = '{{ app_rabbitmq_host }}'
+port = {{ app_rabbitmq_port }}
+user = '{{ app_rabbitmq_user }}'
+password = '{{ app_rabbitmq_password }}'
+virtual-host = '{{ app_rabbitmq_virtual_host }}'
+customer-timeout = 3600
+
+[redis]
+namespace = '{{ app_domain }}://'
+host = '{{ app_redis_host }}'
+port = {{ app_redis_port }}
+
+[app]
+api-url = '{{ app_api_server }}/api'
+openai-proxy-url = '{{ app_openai_proxy_server }}/api/openai'

deploy/roles/mint-v2.2/templates/v2/env.j2

@@ -0,0 +1,97 @@
+BASE_DIR="{{ app_deploy_target }}/.env.global"
+CACHE_DIR="${BASE_DIR}/cache"
+TMP_DIR="${BASE_DIR}/tmp"
+
+APP_NAME="wikipali"
+APP_ENV={{ app_env }}
+APP_KEY={{ app_secret_key }}
+APP_DEBUG={{ app_debug }}
+APP_URL="https://{{ app_domain }}"
+
+LOG_CHANNEL=daily
+LOG_DEPRECATIONS_CHANNEL=null
+LOG_LEVEL=debug
+
+DB_CONNECTION=pgsql
+DB_HOST={{ app_postgresql_host }}
+DB_PORT={{ app_postgresql_port }}
+DB_DATABASE={{ app_postgresql_dbname }}
+DB_USERNAME={{ app_postgresql_user }}
+DB_PASSWORD="{{ app_postgresql_password }}"
+
+BROADCAST_DRIVER=log
+CACHE_DRIVER=redis
+FILESYSTEM_DRIVER=s3
+QUEUE_CONNECTION=sync
+SESSION_DRIVER=cookie
+SESSION_LIFETIME=120
+
+AWS_ACCESS_KEY_ID={{ app_aws_access_key }}
+AWS_SECRET_ACCESS_KEY={{ app_aws_secret_key }}
+AWS_DEFAULT_REGION={{ app_aws_region }}
+AWS_BUCKET=assets
+AWS_URL=
+AWS_ENDPOINT=https://{{ app_aws_endpoint_domain }}
+
+REDIS_HOST={{ app_redis_host }}
+REDIS_PORT={{ app_redis_port }}
+REDIS_PASSWORD=
+REDIS_DB=0
+REDIS_PREFIX="{{ app_domain }}://v20250323/"
+
+REDIS_CACHE_HOST={{ app_redis_cache_host }}
+REDIS_CACHE_PORT={{ app_redis_cache_port }}
+REDIS_CACHE_PASSWORD=
+REDIS_CACHE_DB={{ app_redis_cache_db }}
+REDIS_CACHE_PREFIX="{{ app_domain }}://v20250323/"
+
+
+MAIL_MAILER=smtp
+MAIL_HOST={{ app_smtp_host }}
+MAIL_PORT={{ app_smtp_port }}
+MAIL_USERNAME="{{ app_smtp_user }}"
+MAIL_PASSWORD="{{ app_smtp_password }}"
+MAIL_ENCRYPTION=ssl
+MAIL_FROM_ADDRESS="{{ app_smtp_user }}"
+MAIL_FROM_NAME="wikipali"
+
+PUSHER_APP_ID=
+PUSHER_APP_KEY=
+PUSHER_APP_SECRET=
+PUSHER_APP_CLUSTER=mt1
+
+MIX_PUSHER_APP_KEY="${PUSHER_APP_KEY}"
+MIX_PUSHER_APP_CLUSTER="${PUSHER_APP_CLUSTER}"
+
+DASHBOARD_BASE_PATH="{{ app_dashboard_base_path }}"
+ASSETS_SERVER="{{ app_assets_server }}"
+GRPC_WEB_SERVER="{{ app_grpc_web_server }}"
+DOCUMENTS_SERVER="{{ app_documents_server }}"
+CORS_ALLOWED_ORIGINS="{{ app_cors_allowed_origins }}"
+
+MORUS_GRPC_HOST="{{ app_morus_grpc_host }}"
+MORUS_GRPC_PORT="{{ app_morus_listen_port }}"
+LILY_GRPC_HOST="{{ app_lily_grpc_host }}"
+LILY_GRPC_PORT="{{ app_lily_listen_port }}"
+TULIP_GRPC_HOST="{{ app_tulip_grpc_host }}"
+TULIP_GRPC_PORT="{{ app_tulip_listen_port }}"
+
+SNOWFLAKE_DATA_CENTER_ID={{ app_snowflake_data_center_id }}
+SNOWFLAKE_WORKER_ID={{ app_snowflake_worker_id }}
+
+RABBITMQ_HOST="{{ app_rabbitmq_host }}"
+RABBITMQ_PORT={{ app_rabbitmq_port }}
+RABBITMQ_VIRTUAL_HOST="{{ app_rabbitmq_virtual_host }}"
+RABBITMQ_USER="{{ app_rabbitmq_user }}"
+RABBITMQ_PASSWORD="{{ app_rabbitmq_password }}"
+
+CDN_URLS={{ app_cdn_urls }}
+ATTACHMENTS_TEMPORARY_BUCKET_NAME={{ app_attachments_bucket_name }}-t
+ATTACHMENTS_PERMANTENT_BUCKET_NAME={{ app_attachments_bucket_name }}-p
+
+FTP_HOST="{{ app_ftp_host }}"
+
+APP_ICP_CODE="{{ app_icp_code }}"
+APP_MPS_CODE="{{ app_mps_code }}"
+
+MQ_LOOP_LIMIT_AI_TRANSLATE="{{ app_consumer_loop_limit_ai_translate }}"

deploy/roles/mint-v2.2/templates/v2/openai-proxy.json.j2

@@ -0,0 +1,5 @@
+{ 
+    "port": 8080,
+    "debug": false, 
+    "api-url": "{{ app_api_server }}/api" 
+}

deploy/roles/mint-v2.2/templates/version.txt.j2

@@ -0,0 +1,2 @@
+git version: {{ mint_version }}
+deployed at: {{ ansible_date_time.iso8601 }}