Merge branch 'iapt-platform:development' into development

ai-translate/ai_translate/__init__.py

@@ -13,31 +13,34 @@ logger = logging.getLogger(__name__)
 
 
 def open_redis_cluster(config):
+    logger.debug("open redis cluster tcp://%s:%s/%s",
+                 config['host'], config['port'], config['namespace'])
     cli = RedisCluster(host=config['host'], port=config['port'])
     logger.debug("%s", cli.get_nodes())
     return (cli, config['namespace'])
 
 
-def start_consumer(context, name, queue, config):
-    mq_config = config['rabbitmq']
+def start_consumer(context, name, config, queue, callback):
+    logger.debug("open rabbitmq %s@%s:%d/%s with timeout %ds",
+                 config['user'], config['host'], config['port'], config['virtual-host'], config['customer-timeout'])
     connection = pika.BlockingConnection(
         pika.ConnectionParameters(
-            host=mq_config['host'], port=mq_config['port'],
+            host=config['host'], port=config['port'],
             credentials=pika.PlainCredentials(
-                mq_config['user'], mq_config['password']),
-            virtual_host=mq_config['virtual-host']))
+                config['user'], config['password']),
+            virtual_host=config['virtual-host']))
     channel = connection.channel()
 
-    def callback(ch, method, properties, body):
+    def _callback(ch, method, properties, body):
         logger.info("received message(%s,%s)",
                     properties.message_id, properties.content_type)
         handle_message(context, ch, method, properties.message_id,
                        properties.content_type, json.loads(
                            body, object_hook=SimpleNamespace),
-                       config['app']['api-url'], config['rabbitmq']['customer-timeout'])
+                       callback, ['customer-timeout'])
 
     channel.basic_consume(
-        queue=queue, on_message_callback=callback, auto_ack=False)
+        queue=queue, on_message_callback=_callback, auto_ack=False)
 
     logger.info('start a consumer(%s) for queue(%s)', name, queue)
     channel.start_consuming()
@@ -47,8 +50,7 @@ def launch(name, queue, config_file):
     logger.debug('load configuration from %s', config_file)
     with open(config_file, "rb") as config_fd:
         config = tomllib.load(config_fd)
+        logger.debug('api-url:(%s)', config['app']['api-url'])
         redis_cli = open_redis_cluster(config['redis'])
-        logger.info('api-url:(%s)', config['app']['api-url'])
-        logger.info('customer-timeout:(%s)',
-                    config['rabbitmq']['customer-timeout'])
-        start_consumer(redis_cli, name, queue, config)
+        start_consumer(redis_cli, name,
+                       config['rabbitmq'], queue, config['app']['api-url'])

ai-translate/ai_translate/__main__.py

@@ -23,7 +23,7 @@ def main():
     parser.add_argument('-d', '--debug',
                         action='store_true', help='run on debug mode')
     parser.add_argument('-v', '--version',
-                        action='version', version='%(prog)s v2025.6.11')
+                        action='version', version='%(prog)s v2025.6.27')
     args = parser.parse_args()
 
     if args.debug:

ai-translate/docker/.gitignore

@@ -0,0 +1,2 @@
+*.tar
+*.md5

ai-translate/docker/Dockerfile

@@ -0,0 +1,44 @@
+FROM ubuntu:latest
+LABEL maintainer="Kassapa"
+
+ENV DEBIAN_FRONTEND noninteractive
+# https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa
+ARG PYTHON_VERSION=3.13
+
+RUN apt update
+RUN apt -y install lsb-release apt-utils \
+    debian-keyring debian-archive-keyring apt-transport-https software-properties-common curl wget gnupg
+RUN add-apt-repository -y ppa:deadsnakes/ppa
+RUN apt -y upgrade
+RUN apt -y install git vim sudo locales locales-all tzdata build-essential \
+    python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv
+RUN apt -y autoremove
+RUN apt -y clean
+
+RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen
+RUN locale-gen
+RUN update-locale LANG=en_US.UTF-8
+RUN update-alternatives --set editor /usr/bin/vim.basic
+
+RUN useradd -s /bin/bash -m deploy
+RUN passwd -l deploy
+RUN echo 'deploy ALL=(ALL:ALL) NOPASSWD: ALL' > /etc/sudoers.d/101-deploy
+USER deploy
+
+RUN python${PYTHON_VERSION} -m venv $HOME/python3
+RUN echo 'source $HOME/python3/bin/activate' >> $HOME/.bashrc
+
+# https://pip.pypa.io/en/stable/installation/#get-pip-py
+ADD --chown=deploy https://bootstrap.pypa.io/get-pip.py /opt/
+RUN bash -c "source $HOME/python3/bin/activate && python3 /opt/get-pip.py"
+
+# COPY --chown=deploy README.md pyproject.toml /opt/ai-translate/
+# COPY --chown=deploy ai_translate /opt/ai-translate/ai_translate
+# RUN bash -c "source $HOME/python3/bin/activate && python3 -m pip install -e /opt/ai-translate"
+
+RUN echo "$(date -u +%4Y%m%d%H%M%S)" | sudo tee /VERSION
+
+VOLUME /srv
+WORKDIR /srv
+
+CMD ["/bin/bash", "-l"]

ai-translate/docker/build.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+
+if [ "$#" -ne 1 ]; then
+    echo "USAGE: $0 PYTHON_VERSION"
+    exit 1
+fi
+
+export VERSION=$(date "+%4Y%m%d%H%M%S")
+export CODE="mint-python$1"
+export TAR="$CODE-$VERSION-$(uname -m)"
+
+podman pull ubuntu:latest
+podman build --build-arg PYTHON_VERSION=$1 -t $CODE .
+podman save --format=oci-archive -o $TAR.tar $CODE
+md5sum $TAR.tar >$TAR.md5
+
+echo "done($TAR.tar)."
+
+exit 0

ai-translate/docker/run.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [ "$#" -ne 1 ]; then
+    echo "USAGE: $0 PYTHON_VERSION"
+    exit 1
+fi
+
+podman run --rm -it --events-backend=file --hostname=palm --network host -v $PWD:/srv:z "mint-python$1"

ai-translate/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ai_translate"
-version = "2025.6.11"
+version = "2025.6.27"
 requires-python = ">= 3.13"
 description = "An OpenAI consumer process"
 readme = "README.md"
@@ -14,3 +14,6 @@ dependencies = ["pika", "requests", "redis[hiredis]", "openai"]
 
 [project.scripts]
 mint-ai-translate-consumer = "ai_translate.__main__:main"
+
+[tool.setuptools]
+py-modules = []

ai-translate/run.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [ "$#" -lt 2 ]; then
+    echo "USAGE: $0 PYTHON_VERSION ARGS"
+    exit 1
+fi
+
+podman run --rm -it --events-backend=file --hostname=palm --network host -v $PWD:/srv:z "mint-python$1" bash -c "source ~/python3/bin/activate && mint-ai-translate-consumer ${@:2}"

api-v12/docker/.gitignore

@@ -0,0 +1,2 @@
+*.tar
+*.md5

api-v12/docker/Dockerfile

@@ -0,0 +1,31 @@
+FROM ubuntu:latest
+LABEL maintainer="Kassapa"
+
+ENV DEBIAN_FRONTEND noninteractive
+# https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa
+ARG _VERSION=3.13
+
+RUN apt update
+RUN apt -y install lsb-release apt-utils \
+    debian-keyring debian-archive-keyring apt-transport-https software-properties-common curl wget gnupg
+RUN add-apt-repository -y ppa:ondrej/php
+RUN apt -y upgrade
+RUN apt -y install git vim locales locales-all tzdata build-essential \
+    php${PHP_VERSION}-cli php${PHP_VERSION}-fpm \
+    php${PHP_VERSION}-xml php${PHP_VERSION}-imap php${PHP_VERSION}-intl php${PHP_VERSION}-mbstring php${PHP_VERSION}-bcmath \
+    php${PHP_VERSION}-bz2 php${PHP_VERSION}-zip php${PHP_VERSION}-curl php${PHP_VERSION}-gd php${PHP_VERSION}-imagick \
+    php${PHP_VERSION}-pgsql php${PHP_VERSION}-mysql php${PHP_VERSION}-sqlite3 php${PHP_VERSION}-redis php${PHP_VERSION}-amqp
+RUN apt -y autoremove
+RUN apt -y clean
+
+RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen
+RUN locale-gen
+RUN update-locale LANG=en_US.UTF-8
+RUN update-alternatives --set editor /usr/bin/vim.basic
+
+RUN echo "$(date -u +%4Y%m%d%H%M%S)" | tee /VERSION
+
+VOLUME /srv
+WORKDIR /srv
+
+CMD ["/bin/bash", "-l"]

api-v12/docker/build.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+
+if [ "$#" -ne 1 ]; then
+    echo "USAGE: $0 PYTHON_VERSION"
+    exit 1
+fi
+
+export VERSION=$(date "+%4Y%m%d%H%M%S")
+export CODE="mint-php$1"
+export TAR="$CODE-$VERSION-$(uname -m)"
+
+podman pull ubuntu:latest
+podman build --build-arg PHP_VERSION=$1 -t $CODE .
+podman save --format=oci-archive -o $TAR.tar $CODE
+md5sum $TAR.tar >$TAR.md5
+
+echo "done($TAR.tar)."
+
+exit 0

api-v12/docker/run.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [ "$#" -ne 1 ]; then
+    echo "USAGE: $0 PHP_VERSION"
+    exit 1
+fi
+
+podman run --rm -it --events-backend=file --hostname=palm --network host -v $PWD:/srv:z "mint-php$1"

deploy/.gitignore

@@ -1,5 +1,4 @@
 /clients/
-/python/
 /shared/
 /tmp/
 *.log

deploy/group_vars/all.yml

@@ -9,6 +9,7 @@ app_debug: false
 app_dashboard_base_path: "/pcd"
 app_postgresql_version: "16"
 app_open_search_version: "2.19.1"
+app_python_version: "3.13"
 app_php_version: "8.1"
 app_php_memory_limit: "128M"
 app_container_prefix: "mint"

deploy/mint.yml

@@ -56,7 +56,6 @@
         - wbw.analyses
         - export.pali.chapter
         - export.article
-        - ai.translate
 
 - name: Start mint php-fpm
   hosts:
@@ -73,11 +72,11 @@
   hosts:
     - ai_translate
   tasks:
-    - name: Start ai-translate service
+    - name: Disable php ai-translate service
       ansible.builtin.systemd_service:
         name: "{{ app_container_prefix }}-{{ app_domain }}-worker-mq-ai.translate"
-        enabled: true
-        state: restarted
+        enabled: false
+        state: stopped
         scope: user
 
 - name: Setup nginx

deploy/roles/docker/tasks/kubernetes.yml

@@ -0,0 +1,28 @@
+# https://minikube.sigs.k8s.io/docs/start/
+- name: Install minikube
+  become: true
+  ansible.builtin.get_url:
+    url: https://github.com/kubernetes/minikube/releases/latest/download/minikube-linux-amd64
+    dest: /usr/local/bin/minikube
+    mode: "0755"
+  when: ansible_architecture == "x86_64"
+
+- name: Install minikube
+  become: true
+  ansible.builtin.get_url:
+    url: https://github.com/kubernetes/minikube/releases/latest/download/minikube-linux-arm64
+    dest: /usr/local/bin/minikube
+    mode: "0755"
+  when: ansible_architecture == "aarch64"
+
+# https://minikube.sigs.k8s.io/docs/handbook/kubectl/
+- name: Install Kubectl
+  become: true
+  ansible.builtin.file:
+    src: /usr/local/bin/minikube
+    dest: /usr/local/bin/kubectl
+    state: link
+
+- name: Setup Kubectl
+  ansible.builtin.shell:
+    cmd: kubectl help

deploy/roles/docker/tasks/main.yml

@@ -67,3 +67,6 @@
     line: "net.ipv4.ip_forward = 1"
     create: true
     mode: "0644"
+
+- name: Setup kubernetes
+  ansible.builtin.import_tasks: kubernetes.yml

deploy/roles/mint-v2.1/tasks/clove.yml

@@ -0,0 +1,30 @@
+- name: Clone source codes
+  ansible.builtin.git:
+    repo: "https://github.com/iapt-platform/clove.git"
+    dest: "{{ app_deploy_target | dirname }}/clove"
+
+# - name: Clean resources for v8
+#   ansible.builtin.file:
+#     path: "{{ app_deploy_target }}/api-v8/storage/resources"
+#     state: absent
+
+- name: Setup resources for v8
+  become: true
+  ansible.builtin.file:
+    src: "{{ app_deploy_target | dirname }}/clove"
+    dest: "{{ app_deploy_target }}/api-v8/storage/resources"
+    state: link
+    force: true
+
+# - name: Clean resources for v12
+#   ansible.builtin.file:
+#     path: "{{ app_deploy_target }}/api-v12/storage/resources"
+#     state: absent
+
+- name: Setup resources for v12
+  become: true
+  ansible.builtin.file:
+    src: "{{ app_deploy_target | dirname }}/clove"
+    dest: "{{ app_deploy_target }}/api-v12/storage/resources"
+    state: link
+    force: true

deploy/roles/mint-v2.1/tasks/laravel.yml

@@ -43,10 +43,13 @@
     - wbw.analyses
     - export.pali.chapter
     - export.article
-    - ai.translate
+    # - ai.translate
   loop_control:
     loop_var: worker_name
 
+- name: Setup clove resources
+  ansible.builtin.import_tasks: clove.yml
+
 - name: Setup schedule run
   ansible.builtin.import_tasks: schedule-run.yml
 

deploy/roles/mint-v2.1/tasks/main.yml

@@ -6,24 +6,41 @@
     owner: "{{ ansible_user }}"
     mode: "0755"
 
-# - name: Download source code
-#   ansible.builtin.unarchive:
-#     src: https://github.com/iapt-platform/mint/archive/{{ mint_version }}.zip
-#     dest: "{{ app_deploy_target | dirname }}"
-#     remote_src: true
-#     creates: "{{ app_deploy_target }}"
-
-- name: Download source codes
-  ansible.builtin.git:
-    repo: "https://github.com/iapt-platform/mint.git"
-    dest: "{{ app_deploy_target | dirname }}/repo"
-
-- name: Clone to spec version
-  ansible.builtin.git:
-    repo: "{{ app_deploy_target | dirname }}/repo"
-    dest: "{{ app_deploy_target }}"
-    depth: 1
-    single_branch: true
+# ---------------------------------------------------------
+
+- name: Download source code
+  ansible.builtin.unarchive:
+    src: https://github.com/iapt-platform/mint/archive/{{ mint_version }}.zip
+    dest: "{{ app_deploy_target | dirname }}"
+    remote_src: true
+    creates: "{{ app_deploy_target }}"
+
+# ---------------------------------------------------------
+
+# - name: Clone source codes directly
+#   ansible.builtin.git:
+#     repo: "https://github.com/iapt-platform/mint.git"
+#     dest: "{{ app_deploy_target }}"
+#     version: "{{ mint_version }}"
+
+# ---------------------------------------------------------
+
+# - name: Clone source codes to repo
+#   ansible.builtin.git:
+#     repo: "https://github.com/iapt-platform/mint.git"
+#     dest: "{{ app_deploy_target | dirname }}/repo"
+#     update: true
+#     version: "development"
+
+# - name: Clone to from local repo
+#   ansible.builtin.git:
+#     repo: "{{ app_deploy_target | dirname }}/repo"
+#     dest: "{{ app_deploy_target }}"
+#     version: "{{ mint_version }}"
+#     # depth: 1
+#     # single_branch: true
+
+# ---------------------------------------------------------
 
 - name: Upload dashboard-v4 dist
   ansible.posix.synchronize:
@@ -48,3 +65,21 @@
   ansible.builtin.systemd:
     daemon_reload: true
     scope: user
+
+- name: Upload version.txt(api-v8)
+  ansible.builtin.template:
+    src: version.txt.j2
+    dest: "{{ app_deploy_target }}/api-v8/public/version.txt"
+    mode: "0555"
+
+- name: Upload version.txt(api-v12)
+  ansible.builtin.template:
+    src: version.txt.j2
+    dest: "{{ app_deploy_target }}/api-v12/public/version.txt"
+    mode: "0555"
+
+- name: Upload version.txt(dashboard-v4)
+  ansible.builtin.template:
+    src: version.txt.j2
+    dest: "{{ app_deploy_target }}/dashboard-v4/dashboard/dist/version.txt"
+    mode: "0555"

deploy/roles/mint-v2.1/templates/version.txt.j2

@@ -0,0 +1,2 @@
+git version: {{ mint_version }}
+deployed at: {{ ansible_date_time.iso8601 }}

deploy/roles/python3/tasks/main.yml

@@ -0,0 +1,20 @@
+# https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa
+
+- name: Add python3 stable repository from PPA and install its signing key on Ubuntu target
+  become: true
+  ansible.builtin.apt_repository:
+    repo: ppa:deadsnakes/ppa
+
+- name: Update apt cache
+  become: true
+  ansible.builtin.apt:
+    update_cache: true
+    # cache_valid_time: 3600
+
+- name: Install python3 packages
+  become: true
+  ansible.builtin.apt:
+    pkg:
+      - python{{ app_python_version }}-dev
+      - python{{ app_python_version }}-venv
+      - python{{ app_python_version }}-distutils

docker/mint/Dockerfile

@@ -2,6 +2,7 @@ FROM ubuntu:latest
 LABEL maintainer="Jeremy Zheng"
 
 ENV DEBIAN_FRONTEND noninteractive
+# https://launchpad.net/~ondrej/+archive/ubuntu/php
 ARG PHP_VERSION=8.4
 
 RUN apt update