Merge branch 'development' into development

ai-translate/ai_translate/__init__.py

@@ -1,6 +1,8 @@
 import logging
 import tomllib
 import json
+import socket
+import os
 
 import pika
 from redis.cluster import RedisCluster
@@ -44,6 +46,7 @@ def start_consumer(context, name, config, queue, callback, proxy):
     channel.basic_consume(
         queue=queue, on_message_callback=_callback, auto_ack=False)
 
+    name = "%s.%s.%d" % (name, socket.gethostname(), os.getpid())
     logger.info('start a consumer(%s) for queue(%s)', name, queue)
     channel.start_consuming()
 

ai-translate/config.orig.toml

@@ -13,4 +13,4 @@ port = 6371
 
 [app]
 api-url = 'http://127.0.0.1:8000/api'
-openai-proxy = 'http://localhost:4000/api/openai'
+openai-proxy-url = 'http://localhost:4000/api/openai'

ai-translate/docker/Dockerfile

@@ -20,16 +20,17 @@ RUN locale-gen
 RUN update-locale LANG=en_US.UTF-8
 RUN update-alternatives --set editor /usr/bin/vim.basic
 
-RUN useradd -s /bin/bash -m deploy
-RUN passwd -l deploy
-RUN echo 'deploy ALL=(ALL:ALL) NOPASSWD: ALL' > /etc/sudoers.d/101-deploy
-USER deploy
+# RUN useradd -s /bin/bash -m deploy
+# RUN passwd -l deploy
+# RUN echo 'deploy ALL=(ALL:ALL) NOPASSWD: ALL' > /etc/sudoers.d/101-deploy
+# USER deploy
+# ADD --chown=deploy https://bootstrap.pypa.io/get-pip.py /opt/
 
 RUN python${PYTHON_VERSION} -m venv $HOME/python3
 RUN echo 'source $HOME/python3/bin/activate' >> $HOME/.bashrc
 
 # https://pip.pypa.io/en/stable/installation/#get-pip-py
-ADD --chown=deploy https://bootstrap.pypa.io/get-pip.py /opt/
+ADD https://bootstrap.pypa.io/get-pip.py /opt/
 RUN bash -c ". $HOME/python3/bin/activate && python3 /opt/get-pip.py"
 RUN bash -c ". $HOME/python3/bin/activate && pip install pika requests redis[hiredis] openai"
 

deploy/mint.yml

@@ -10,6 +10,7 @@
     - task
     - fort
     - ai_translate
+    - openai_proxy
   roles:
     - mint-v2.1
 
@@ -68,15 +69,26 @@
         state: restarted
         scope: user
 
-- name: Start mint php-fpm
+- name: Start mint ai-translate worker
   hosts:
     - ai_translate
   tasks:
-    - name: Disable php ai-translate service
+    - name: Enable php ai-translate service
       ansible.builtin.systemd_service:
         name: "{{ app_container_prefix }}-{{ app_domain }}-worker-mq-ai.translate"
-        enabled: false
-        state: stopped
+        enabled: true
+        state: started
+        scope: user
+
+- name: Start mint openai-proxy worker
+  hosts:
+    - openai_proxy
+  tasks:
+    - name: Enable openai-proxy service
+      ansible.builtin.systemd_service:
+        name: "{{ app_container_prefix }}-{{ app_domain }}-worker-mq-openai.proxy"
+        enabled: true
+        state: started
         scope: user
 
 - name: Setup nginx

deploy/roles/mint-v2.1/tasks/ai-translate.yml

@@ -0,0 +1,35 @@
+- name: Upload script for ai.translate
+  ansible.builtin.template:
+    src: containers/ai-translate.sh.j2
+    dest: "{{ app_deploy_target }}/scripts/worker-mq-ai.translate.sh"
+    mode: "0555"
+
+- name: Upload config for ai.translate
+  ansible.builtin.template:
+    src: v2/ai-translate.toml.j2
+    dest: "{{ app_deploy_target }}/ai-translate/config.toml"
+    mode: "0555"
+
+- name: Stop ai.translate
+  containers.podman.podman_container:
+    name: "{{ app_domain }}-worker-mq-ai.translate"
+    state: absent
+
+- name: Create ai.translate container
+  containers.podman.podman_container:
+    name: "{{ app_domain }}-worker-mq-ai.translate"
+    image: "mint-python-3.13"
+    command: "{{ app_deploy_target }}/scripts/worker-mq-ai.translate.sh"
+    volumes:
+      - "{{ app_deploy_target }}/ai-translate:{{ app_deploy_target }}:z"
+    workdir: "{{ app_deploy_target }}"
+    state: present
+    auto_remove: true
+    generate_systemd:
+      path: "{{ ansible_env.HOME }}/.config/systemd/user"
+      container_prefix: "{{ app_container_prefix }}"
+      new: true
+      names: true
+      restart_policy: always
+      restart_sec: 10
+      stop_timeout: 5

deploy/roles/mint-v2.1/tasks/workers.yml → deploy/roles/mint-v2.1/tasks/laravel-workers.yml

@@ -1,6 +1,6 @@
 - name: Upload script for worker-{{ zone_name }}-{{ worker_name }}
   ansible.builtin.template:
-    src: containers/worker.sh.j2
+    src: containers/laravel-worker.sh.j2
     dest: "{{ app_deploy_target }}/scripts/worker-{{ zone_name }}-{{ worker_name }}.sh"
     mode: "0555"
 
@@ -16,6 +16,7 @@
     command: "{{ app_deploy_target }}/scripts/worker-{{ zone_name }}-{{ worker_name }}.sh"
     volumes:
       - "{{ app_deploy_target }}:{{ app_deploy_target }}:z"
+      - "/srv/{{ app_domain }}/clove:/srv/{{ app_domain }}/clove:z"
     workdir: "{{ app_deploy_target }}"
     state: present
     auto_remove: true

deploy/roles/mint-v2.1/tasks/laravel.yml

@@ -32,8 +32,8 @@
     workdir: "{{ app_deploy_target }}"
     auto_remove: true
 
-- name: Setup background worker
-  ansible.builtin.include_tasks: workers.yml
+- name: Setup background php worker
+  ansible.builtin.include_tasks: laravel-workers.yml
   vars:
     zone_name: mq
   loop:
@@ -43,7 +43,6 @@
     - wbw.analyses
     - export.pali.chapter
     - export.article
-    # - ai.translate
   loop_control:
     loop_var: worker_name
 

deploy/roles/mint-v2.1/tasks/openai-proxy.yml

@@ -0,0 +1,35 @@
+- name: Upload script for openai-proxy
+  ansible.builtin.template:
+    src: containers/openai-proxy.sh.j2
+    dest: "{{ app_deploy_target }}/scripts/openai-proxy.sh"
+    mode: "0555"
+
+- name: Upload config for openai-proxy
+  ansible.builtin.template:
+    src: v2/openai-proxy.json.j2
+    dest: "{{ app_deploy_target }}/open-ai-server/config.json"
+    mode: "0555"
+
+- name: Stop openai-proxy server
+  containers.podman.podman_container:
+    name: "{{ app_domain }}-openai.proxy"
+    state: absent
+
+- name: Create openai-proxy container
+  containers.podman.podman_container:
+    name: "{{ app_domain }}-openai.proxy"
+    image: "mint-nodejs-jod"
+    command: "{{ app_deploy_target }}/scripts/openai-proxy.sh"
+    volumes:
+      - "{{ app_deploy_target }}/open-ai-server:{{ app_deploy_target }}:z"
+    workdir: "{{ app_deploy_target }}"
+    state: present
+    auto_remove: true
+    generate_systemd:
+      path: "{{ ansible_env.HOME }}/.config/systemd/user"
+      container_prefix: "{{ app_container_prefix }}"
+      new: true
+      names: true
+      restart_policy: always
+      restart_sec: 10
+      stop_timeout: 5

deploy/roles/mint-v2.1/templates/containers/ai-translate.sh.j2

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+export WORK_DIR="{{ app_deploy_target }}"
+
+source $HOME/python3/bin/activate
+
+cd $WORK_DIR/
+# https://github.com/iapt-platform/mint/blob/df8e1cf7ade16d17add360e7a869540c1ddaf1b9/api-v8/config/mint.php#L129
+python3 -m ai_translate -c config.toml -n ai.translate -q ai_translate_v2
+
+ai_translate_v2
+exit 0

deploy/roles/mint-v2.1/templates/containers/openai-proxy.sh.j2

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -e
+
+export WORK_DIR="{{ app_deploy_target }}"
+
+export NVM_DIR="$HOME/.nvm"
+source "$NVM_DIR/nvm.sh"
+
+cd $WORK_DIR/
+node dist/main-*.js config.json
+
+exit 0

deploy/roles/mint-v2.1/templates/containers/shell.sh.j2

@@ -1,3 +1,3 @@
 #!/bin/sh
 
-podman run --rm -it --events-backend=file --hostname=mint --network host -w {{ app_deploy_target }} -v {{ app_deploy_target }}:{{ app_deploy_target }}:z {{ app_mint_image_name }} /bin/bash -l
+podman run --rm -it --events-backend=file --hostname=mint --network host -w {{ app_deploy_target }} -v /srv/{{ app_domain }}/clove:/srv/{{ app_domain }}/clove:z -v {{ app_deploy_target }}:{{ app_deploy_target }}:z {{ app_mint_image_name }} /bin/bash -l

deploy/roles/mint-v2.1/templates/v2/ai-translate.toml.j2

@@ -0,0 +1,16 @@
+[rabbitmq]
+host = '{{ app_rabbitmq_host }}'
+port = {{ app_rabbitmq_port }}
+user = '{{ app_rabbitmq_user }}'
+password = '{{ app_rabbitmq_password }}'
+virtual-host = '{{ app_rabbitmq_virtual_host }}'
+customer-timeout = 3600
+
+[redis]
+namespace = '{{ app_domain }}://'
+host = '{{ app_redis_host }}'
+port = {{ app_redis_port }}
+
+[app]
+api-url = '{{ app_openai_proxy_server }}/api'
+openai-proxy-url = '{{ app_openai_proxy_server }}/api/openai'

deploy/roles/mint-v2.1/templates/v2/openai-proxy.json.j2

@@ -0,0 +1,5 @@
+{ 
+    "port": 4000, 
+    "debug": false, 
+    "api-server": "{{ app_api_server }}" 
+}

open-ai-server/config.orig.json

@@ -1,5 +1,5 @@
 {
   "port": 4000,
   "debug": true,
-  "api_server": "http://staging.wikipali.org/api"
+  "api-server": "https://staging.wikipali.org"
 }