:wrench: merge deployments

.gitignore

@@ -0,0 +1 @@
+/tmp/

deploy/.gitignore

@@ -0,0 +1,4 @@
+/clients/
+/tmp/
+/roles/mint/files/dashboard/
+*.log

deploy/README.md

@@ -0,0 +1,88 @@
+# Deployment
+
+## Setup a cluster
+
+```bash
+# create cluster
+mkdir -p clients/CLUSTER/.ssh
+cd clients/CLUSTER
+# append your cluster hosts
+touch hosts
+# generate ssh key
+ssh-keygen -t ed25519 -f .ssh/id_ed25519
+# upload the ssh public key to target host
+ssh-copy-id -i .ssh/id_ed25519 USER@HOST
+```
+
+## Deploy
+
+```bash
+# test ssh connections
+peony -i clients/CLUSTER JOB.yml
+# run on all hosts
+peony -i clients/CLUSTER JOB.yml
+# run on only group
+peony -i clients/CLUSTER JOB.yml -l GROUP
+```
+
+## System image
+
+- [Raspberry Pi OS Lite](https://www.raspberrypi.com/software/operating-systems/)
+- [Armbian](https://www.armbian.com/download/)
+- [wiringPi for Orange Pi](https://github.com/orangepi-xunlong/WiringOP)
+
+## Import Database Data
+
+### on deploy a new server
+
+```bash
+php ../../public/app/install/db_insert_templet_cli.php 1 217
+php ../../public/app/install/db_update_toc_cli.php 1 217 pali
+php ../../public/app/install/db_update_toc_cli.php 1 217 zh-hans
+php ../../public/app/install/db_update_toc_cli.php 1 217 zh-hant
+php ../../public/app/install/db_insert_palitext_cli.php 1 217
+php ../../public/app/install/db_update_palitext_cli.php 1 217
+php ../../public/app/install/db_insert_bookword_from_csv_cli.php 1 217
+php ../../public/app/install/db_insert_word_from_csv_cli.php 1 217
+php ../../public/app/install/db_insert_wordindex_from_csv_cli.php
+
+php ./migrations/20211202084900_init_pali_serieses.php
+php ./migrations/20211125155600_word_statistics.php
+php ./migrations/20211125155700_pali_sent_org.php
+php ./migrations/20211125165700-pali_sent-upgrade.php
+php ./migrations/20211126220400-pali_sent_index-upgrade.php
+php ./migrations/20211127214800_sent_sim.php
+php ./migrations/20211127214900-sent_sim_index.php
+
+php ../../public/app/fts/sql.php
+
+php ../../public/app/admin/word_index_weight_refresh.php 1 217
+```
+
+### on update
+
+```bash
+# public/pali_title目录下文件*_pali.csv改变时触发
+php ../../public/app/install/db_update_palitext_cli.php 1 217
+
+# public/pali_title目录下文件其他改变时触发
+php ../../public/app/install/db_update_toc_cli.php 1 217 pali
+php ../../public/app/install/db_update_toc_cli.php 1 217 zh-hans
+php ../../public/app/install/db_update_toc_cli.php 1 217 zh-hant
+
+# public/dependence/pali_sentence/data 目录下文件其他改变时触发
+# TODO 导入pali_sent使用上述目录csv文件。目前用的是sqlite db文件
+php ./migrations/20211125165700-pali_sent-upgrade.php
+php ./migrations/20211126220400-pali_sent_index-upgrade.php
+
+```
+
+## Crontab
+
+### Daily
+
+1. upgrade_pali_toc.php
+
+```bash
+/public/app/upgrade/upgrade_pali_toc.php
+```

deploy/ec2.yml

@@ -0,0 +1,12 @@
+- name: Testing ssh connections
+  hosts:
+    - postgresql
+    - rabbitmq
+    - redis
+    - logging
+    - web
+    - task
+  roles:
+    - os
+    - ubuntu
+    - reset

deploy/group_vars/all.yml

@@ -0,0 +1,8 @@
+ansible_user: "deploy"
+ansible_python_interpreter: /usr/bin/python3
+ansible_ssh_private_key_file: "{{ inventory_dir }}/.ssh/id_ed25519"
+
+app_deploy_target: "/var/www/{{ inventory_hostname }}"
+app_downloads: "{{ ansible_env.HOME }}/downloads"
+app_dashboard_base_path: "/pcd"
+app_postgresql_version: "15"

deploy/ping.yml

@@ -0,0 +1,8 @@
+- name: Testing ssh connections
+  hosts: all
+  tasks:
+    - name: Test ssh connection
+      ansible.builtin.ping:
+    - name: Show facts available on the system
+      ansible.builtin.debug:
+        var: ansible_facts

deploy/roles/os/tasks/main.yml

@@ -0,0 +1,22 @@
+- import_tasks: sshd.yml
+- import_tasks: ulimits.yml
+
+- name: Set timezone
+  become: true
+  shell: timedatectl set-timezone UTC
+
+- name: Setup journald storage
+  become: true
+  lineinfile:
+    path: /etc/systemd/journald.conf
+    regexp: "^#Storage="
+    line: Storage=persistent
+
+# https://www.linode.com/docs/quick-answers/linux/how-to-use-journalctl/
+- name: Setup journald storage keep-free
+  become: true
+  lineinfile:
+    path: /etc/systemd/journald.conf
+    state: present
+    regexp: "^#SystemKeepFree="
+    line: SystemKeepFree=6%

deploy/roles/os/tasks/sshd.yml

@@ -0,0 +1,15 @@
+- name: Disable dns for sshd
+  become: true
+  ansible.builtin.lineinfile:
+    path: /etc/ssh/sshd_config
+    regexp: "^UseDNS "
+    line: UseDNS no
+    backup: true
+
+- name: Disable GSS api auth for sshd
+  become: true
+  ansible.builtin.lineinfile:
+    path: /etc/ssh/sshd_config
+    regexp: "^GSSAPIAuthentication "
+    line: GSSAPIAuthentication no
+    backup: true

deploy/roles/os/tasks/ulimits.yml

@@ -0,0 +1,62 @@
+- name: Setup nofile for system
+  become: true
+  ansible.builtin.lineinfile:
+    path: /etc/systemd/system.conf
+    regexp: "^DefaultLimitNOFILE="
+    line: DefaultLimitNOFILE=2097152
+    backup: true
+
+- name: Setup nproc for system
+  become: true
+  ansible.builtin.lineinfile:
+    path: /etc/systemd/system.conf
+    regexp: "^DefaultLimitNPROC"
+    line: DefaultLimitNPROC=524288
+    backup: true
+
+- name: Setup nofile for user
+  become: true
+  ansible.builtin.lineinfile:
+    path: /etc/systemd/user.conf
+    regexp: "^DefaultLimitNOFILE="
+    line: DefaultLimitNOFILE=1048576
+    backup: true
+
+- name: Setup nproc for user
+  become: true
+  ansible.builtin.lineinfile:
+    path: /etc/systemd/user.conf
+    regexp: "^DefaultLimitNPROC"
+    line: DefaultLimitNPROC=262144
+    backup: true
+
+- name: Set user level ppen file limits for root
+  become: true
+  ansible.builtin.lineinfile:
+    path: /etc/security/limits.conf
+    line: root        soft nofile 10240
+    backup: true
+
+- name: Set user level open file limits for {{ ansible_user }}
+  become: true
+  lineinfile:
+    path: /etc/security/limits.conf
+    line: "{{ ansible_user }}        soft nofile 10240"
+    backup: true
+
+# https://docs.oracle.com/en/database/oracle/oracle-database/12.2/ladbi/changing-kernel-parameter-values.html#GUID-FB0CC366-61C9-4AA2-9BE7-233EB6810A31
+- name: Setup file max
+  become: true
+  lineinfile:
+    path: "/etc/sysctl.d/100-fs.conf"
+    state: present
+    line: fs.file-max = 6815744
+    create: true
+
+- name: Setup file max
+  become: true
+  lineinfile:
+    path: "/etc/sysctl.d/100-fs.conf"
+    state: present
+    line: fs.inotify.max_user_watches = 512000
+    create: true

deploy/roles/reset/tasks/main.yml

@@ -0,0 +1,8 @@
+- name: Reset root password
+  become: true
+  shell: echo "root:$(pwgen 32 1)" | chpasswd
+
+- name: Reset {{ ansible_user }} password
+  become: true
+  shell: echo "{{ ansible_user }}:$(pwgen 32 1)" | chpasswd
+  when: ansible_user != 'root'

deploy/roles/ubuntu/tasks/clean.yml

@@ -0,0 +1,14 @@
+- name: Remove useless packages from the cache
+  become: true
+  apt:
+    autoclean: yes
+
+- name: Remove dependencies that are no longer required
+  become: true
+  apt:
+    autoremove: yes
+
+- name: Force systemd to reread configs
+  become: true
+  systemd:
+    daemon_reload: yes

deploy/roles/ubuntu/tasks/init.yml

@@ -0,0 +1,6 @@
+- name: create downloads folder
+  become: true
+  ansible.builtin.file:
+    path: "{{ app_downloads }}"
+    state: directory
+    owner: "{{ ansible_user }}"

deploy/roles/ubuntu/tasks/locales.yml

@@ -0,0 +1,31 @@
+- name: Enable en-US locale
+  become: true
+  ansible.builtin.lineinfile:
+    path: /etc/locale.gen
+    state: present
+    line: en_US.UTF-8 UTF-8
+    backup: true
+
+- name: Enable zh-CN locale
+  become: true
+  ansible.builtin.lineinfile:
+    path: /etc/locale.gen
+    state: present
+    line: zh_CN.UTF-8 UTF-8
+    backup: true
+
+- name: Enable zh-TW locale
+  become: true
+  ansible.builtin.lineinfile:
+    path: /etc/locale.gen
+    state: present
+    line: zh_TW.UTF-8 UTF-8
+    backup: true
+
+- name: Generate locales
+  become: true
+  ansible.builtin.shell: locale-gen
+
+- name: Use en_US as default locale
+  become: true
+  ansible.builtin.shell: update-locale LANG=en_US.UTF-8

deploy/roles/ubuntu/tasks/main.yml

@@ -0,0 +1,161 @@
+- import_tasks: init.yml
+
+- name: add PPA for Ubuntu Toolchain
+  become: true
+  ansible.builtin.apt_repository:
+    repo: ppa:ubuntu-toolchain-r/test
+  when: ansible_distribution == 'Ubuntu'
+
+# https://classic.yarnpkg.com/lang/en/docs/install/#debian-stable
+- name: Add an yarn signing key
+  become: true
+  ansible.builtin.apt_key:
+    url: https://dl.yarnpkg.com/debian/pubkey.gpg
+    state: present
+  when: ansible_distribution == 'Ubuntu'
+
+- name: Add an yarn package repository
+  become: true
+  ansible.builtin.lineinfile:
+    path: /etc/apt/sources.list.d/yarn.list
+    line: "deb https://dl.yarnpkg.com/debian/ stable main"
+    create: true
+  when: ansible_distribution == 'Ubuntu'
+
+- name: Install nodejs
+  become: true
+  community.general.snap:
+    name: node
+    classic: true
+
+- name: Update system
+  become: true
+  apt:
+    upgrade: yes
+    update_cache: yes
+    cache_valid_time: 3600
+
+- name: Install dependicy packages
+  become: true
+  apt:
+    pkg:
+      - apt-transport-https
+      - software-properties-common
+      - gnupg
+      - openssh-server
+      - openssh-client
+      - sshpass
+      - wpasupplicant
+      - rsync
+      - at
+      - sysstat
+      - libtool
+      - ethtool
+      - dnsutils
+      - dnsmasq
+      - uuid-runtime
+      - lshw
+      - tcpdump
+      - lm-sensors
+      - dmidecode
+      - net-tools
+      - iputils-arping
+      - iputils-ping
+      - telnet
+      - vim
+      - git
+      - pwgen
+      - locales
+      - ntpdate
+      - imagemagick
+      - mpg123
+      - ffmpeg
+      - sqlite3
+      - tree
+      - alsa-utils
+      - pulseaudio
+      - zsh
+      - wget
+      - curl
+      - zip
+      - unzip
+      - nginx
+      - libnginx-mod-http-upstream-fair
+      - certbot
+      - python3-certbot-nginx
+      - openvpn
+      - snmpd
+      - mutt
+      - systemd-cron
+      - screen
+      - tmux
+      - hugo
+      - yarn
+      - python3
+      - python3-pip
+      - python3-distutils
+      - python3-dev
+      - libssl-dev
+      - libpq-dev
+      - libmysqlclient-dev
+      - libevent-dev
+      - crun
+      - podman
+      - buildah
+      - fuse-overlayfs
+
+- name: Install dependicy packages(>bionic)
+  become: true
+  apt:
+    pkg:
+      - systemd-timesyncd
+  # ansible_facts['distribution'] == "Ubuntu"
+  when: ansible_facts['distribution_major_version'] | int >= 20
+
+- import_tasks: locales.yml
+
+- name: Set default editor to vim
+  become: true
+  shell: update-alternatives --set editor /usr/bin/vim.basic
+
+- name: Set timezone
+  become: true
+  shell: timedatectl set-timezone UTC
+
+- name: Set git rebase mode
+  become: true
+  shell: git config --global pull.rebase false
+
+- import_tasks: zsh.yml
+
+# ---------------------------------------------------
+
+- name: enable nginx service
+  become: true
+  ansible.builtin.systemd:
+    name: nginx
+    enabled: yes
+    masked: no
+
+- name: enable cron service
+  become: true
+  ansible.builtin.systemd:
+    name: cron
+    enabled: yes
+    masked: no
+
+- name: enable ssh service
+  become: true
+  ansible.builtin.systemd:
+    name: ssh
+    enabled: yes
+    masked: no
+
+- name: enable systemd-timesyncd service
+  become: true
+  ansible.builtin.systemd:
+    name: systemd-timesyncd
+    enabled: yes
+    masked: no
+
+- import_tasks: clean.yml

deploy/roles/ubuntu/tasks/zsh.yml

@@ -0,0 +1,37 @@
+- name: Clone oh-my-zsh
+  git:
+    repo: https://github.com/robbyrussell/oh-my-zsh.git
+    dest: "{{ansible_env.HOME}}/.oh-my-zsh"
+
+# - name: Extract ohmyzsh
+#   unarchive:
+#     src: ohmyzsh-master.zip
+#     dest: "{{ansible_env.HOME}}/"
+
+# - name: Rename ohmyzsh
+#   command: mv ohmyzsh-master .oh-my-zsh
+#   args:
+#     chdir: "{{ansible_env.HOME}}"
+#     creates: .oh-my-zsh
+
+- name: Setup .zshrc
+  copy:
+    src: "{{ansible_env.HOME}}/.oh-my-zsh/templates/zshrc.zsh-template"
+    dest: "{{ansible_env.HOME}}/.zshrc"
+    remote_src: true
+    mode: 0600
+
+- name: Enable $HOME/.local
+  ansible.builtin.lineinfile:
+    path: "{{ansible_env.HOME}}/.zshrc"
+    line: 'export PATH=$HOME/.local/bin:$PATH'
+
+
+- name: Setup EDITOR
+  ansible.builtin.lineinfile:
+    path: "{{ansible_env.HOME}}/.zshrc"
+    line: 'export EDITOR=vim'
+
+- name: Use zsh
+  become: true
+  shell: chsh -s /bin/zsh {{ansible_user}}

deploy/scripts/assets.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -e
+
+# rclone copy --drive-shared-with-me $1:assets assets
+
+export WORKSPACE=$PWD
+
+function build_book(){
+    local target=$WORKSPACE/tmp/$1/$2
+    local dist=$WORKSPACE/roles/mint-assets/files/public/$1/$2
+    if [ ! -d $target ]
+    then
+        git clone -b $2 "https://github.com/iapt-platform/$1.git" $target
+    fi
+    cd $target
+    git pull
+    if [ -d $dist ]
+    then
+        rm -r $dist
+    fi
+    mkdir -p $dist
+    $HOME/.local/bin/mdbook build --dest-dir $dist
+}
+
+declare -a languages=(
+    "zh-Hans"
+)
+
+declare -a books=(
+    "pali-handbook"
+    "help"
+)
+
+for b in "${books[@]}"
+do
+    for l in "${languages[@]}"
+    do
+        build_book $b $l
+    done
+done
+
+echo 'done.'
+exit 0

deploy/scripts/dashboard.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+
+export WORKSPACE=$PWD
+
+cd $WORKSPACE/dashboard
+if [ ! -d node_modules ]
+then
+    yarn install
+fi
+
+# GENERATE_SOURCEMAP=false 
+BUILD_PATH=$WORKSPACE/deploy/roles/mint/files/dashboard PUBLIC_URL=/pcd yarn build
+
+echo 'done.'
+
+exit 0

deploy/scripts/handbooks.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -e
+
+
+if [ $# -ne 1 ]
+then
+	echo "Usage: $0 FOLDER"
+	exit 1
+fi
+
+
+function build_book(){
+    local target=$HOME/tmp/$2/$3
+    local dist=$1/public/$2/$3
+    if [ ! -d $target ]
+    then
+        git clone -b $3 "https://github.com/iapt-platform/$2.git" $target
+    fi
+    cd $target
+    git pull
+    if [ -d $dist ]
+    then
+        rm -r $dist
+    fi
+    mkdir -p $dist
+    $HOME/.local/bin/mdbook build --dest-dir $dist
+    echo "done($dist)."
+}
+
+declare -a languages=(
+    "zh-Hans"
+)
+
+declare -a books=(
+    "pali-handbook"
+    "help"
+)
+
+for b in "${books[@]}"
+do
+    for l in "${languages[@]}"
+    do
+        build_book $1 $b $l
+    done
+done
+
+exit 0

deploy/scripts/laravel-react.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+set -e
+
+
+if [ "$#" -ne 1 ]
+then
+    echo "USAGE: $0 DOMAIN"
+    exit 1
+fi
+
+echo "check $1.conf"
+if [ ! -d /var/www/$1/logs ]
+then
+  mkdir -p /var/www/$1/logs
+  chown -R www-data:www-data /var/www/$1/logs
+fi
+
+if [ ! -f /etc/nginx/sites-enabled/$1.conf ]
+then
+    # https://laravel.com/docs/10.x/deployment
+    cat > /etc/nginx/sites-enabled/$1.conf <<EOF
+server {
+
+  server_name $1;
+  root /var/www/$1/htdocs/public;
+  access_log /var/www/$1/logs/access.log;
+  error_log  /var/www/$1/logs/error.log;
+  
+  add_header X-Frame-Options "SAMEORIGIN";
+  add_header X-Content-Type-Options "nosniff";
+
+  index index.php;
+  charset utf-8;
+
+  gzip on;
+  gzip_comp_level 9;
+  gzip_min_length 1k;
+  gzip_types text/plain text/css application/xml application/javascript;
+  gzip_vary on;
+  client_max_body_size 128M;
+  
+
+  location /pcd/ {
+    alias /var/www/$1/dashboard/;
+    try_files \$uri \$uri/ /pcd/index.html;
+    
+    location ~* \\.(css|js|png|jpg|jpeg|gif|gz|svg|mp4|ogg|ogv|webm|htc|xml|woff)\$ {
+      access_log off;
+      expires max;
+    }
+  }
+  
+  location / {
+    try_files \$uri \$uri/ /index.php?\$query_string;
+  }
+
+  location = /favicon.ico { access_log off; log_not_found off; }
+  location = /robots.txt  { access_log off; log_not_found off; }
+  error_page 404 /index.php;
+
+  location ~ \.php\$ {
+    fastcgi_pass unix:/var/run/php/php8.1-fpm.sock;
+    fastcgi_param SCRIPT_FILENAME \$realpath_root\$fastcgi_script_name;
+    include fastcgi_params;
+  }
+ 
+  location ~ /\.(?!well-known).* {
+      deny all;
+  }
+
+}
+EOF
+
+    chmod 644 /etc/nginx/sites-enabled/$1.conf
+fi
+
+echo "done($1)."
+exit 0

deploy/scripts/postgresql/2023-10-21-global-fts-setup.sql

@@ -0,0 +1,15 @@
+-- FROM https://github.com/iapt-platform/mint/blob/laravel/database/migrations/2021_12_30_053602_add_func_to_fts_texts_table.php
+
+CREATE TEXT SEARCH CONFIGURATION pali ( parser = pg_catalog.default );
+CREATE TEXT SEARCH CONFIGURATION pali_unaccent ( parser = pg_catalog.default );
+CREATE TEXT SEARCH DICTIONARY pali_stem ( TEMPLATE = synonym, SYNONYMS = pali );
+CREATE TEXT SEARCH DICTIONARY pali_stopwords ( TEMPLATE = pg_catalog.simple, STOPWORDS = pali, ACCEPT = true);
+
+ALTER TEXT SEARCH CONFIGURATION pali
+ADD MAPPING FOR asciiword, word, hword_part, hword_asciipart
+WITH pali_stem, pali_stopwords;
+
+CREATE EXTENSION IF NOT EXISTS "unaccent";
+ALTER TEXT SEARCH CONFIGURATION pali_unaccent
+ADD MAPPING FOR asciiword, word, hword_part, hword_asciipart
+WITH unaccent, pali_stem, pali_stopwords;

deploy/scripts/sim_sent.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+set -e
+
+export SIM=$(pgrep -f "php sim_sent")
+
+echo "find pid $SIM"
+renice +19 $SIM
+ionice -c 2 -n 7 -p $SIM
+
+echo "done."
+
+exit 0

deploy/scripts/sqlite_fix.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -e
+if [ $# -ne 1 ]
+then
+	echo "Usage: $0 DB"
+	exit 1
+fi
+
+if [ ! -f $1 ]
+then
+	echo "$1 not exists"
+	exit 1
+fi
+
+echo '.dump'|sqlite3 $1|sqlite3 $1_repaired
+mv -v $1 $1_corrupt
+mv -v $1_repaired $1
+
+exit 0

deploy/staging/.gitignore

@@ -0,0 +1 @@
+/.ssh/