:building_construction: add lily rpc server

lily/.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+/tmp/

lily/README.md

@@ -0,0 +1,31 @@
+# LILY
+
+## Setup
+
+```bash
+$ sudo apt install python3-full imagemagick ffmpeg fonts-dejavu-extra texlive-full
+$ python3 -m venv $HOME/local/python3
+$ source $HOME/local/python3/bin/activate
+$ pip install psycopg pika matplotlib ebooklib \
+    grpcio protobuf grpcio-health-checking \
+    pandas openpyxl xlrd pyxlsb
+```
+
+## Start
+
+- create config.toml
+
+  ```toml
+  [rpc]
+  port = 9999
+  workers = 8
+  ```
+
+- run `python lily -d`
+
+## Documents
+
+- [https://matplotlib.org/stable/gallery/index.html](Matplotlib)
+- [https://graphviz.org/](Graphviz)
+- [EbookLib](https://github.com/aerkalov/ebooklib)
+- [Excel files](https://pandas.pydata.org/docs/user_guide/io.html#excel-files)

lily/__main__.py

@@ -0,0 +1,37 @@
+import logging
+import argparse
+import sys
+import tomllib
+import pika
+
+from palm import VERSION, start_server
+
+NAME = 'lily'
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        prog=NAME,
+        description='Background worker for palm',
+        epilog='https://github.com/saturn-xiv/palm')
+    parser.add_argument('-c', '--config',
+                        type=argparse.FileType(mode='rb'),
+                        default='config.toml',
+                        help='load configuration(toml)')
+    parser.add_argument('-d', '--debug',
+                        action='store_true',
+                        help='run on debug mode')
+    parser.add_argument('-v', '--version',
+                        action='store_true',
+                        help=('print %s version' % NAME))
+    args = parser.parse_args()
+    if args.version:
+        print(VERSION)
+        sys.exit()
+    logging.basicConfig(level=(logging.DEBUG if args.debug else logging.INFO))
+    if args.debug:
+        logging.debug('run on debug mode with %s', args)
+    logging.info('load configuration from %s', args.config.name)
+
+    config = tomllib.load(args.config)
+    rpc_params = config['rpc']
+    start_server('0.0.0.0:%d' % (rpc_params['port']), rpc_params['workers'])

lily/palm/__init__.py

@@ -0,0 +1,77 @@
+
+import logging
+from time import sleep
+from concurrent import futures
+import threading
+
+import psycopg
+import pika
+import grpc
+from grpc_health.v1 import health_pb2, health, health_pb2_grpc
+
+from . import lily_pb2_grpc, excel, tex
+
+
+VERSION = '2023.9.29'
+
+
+def _health_checker(servicer, name):
+    while True:
+        servicer.set(name, health_pb2.HealthCheckResponse.SERVING)
+        sleep(5)
+
+
+def _setup_health_thread(server):
+    servicer = health.HealthServicer(
+        experimental_non_blocking=True,
+        experimental_thread_pool=futures.ThreadPoolExecutor(max_workers=2)
+    )
+    health_pb2_grpc.add_HealthServicer_to_server(servicer, server)
+    health_checker_thread = threading.Thread(
+        target=_health_checker,
+        args=(servicer, 'palm.lily'),
+        daemon=True
+    )
+    health_checker_thread.start()
+
+
+def start_server(addr, workers):
+    server = grpc.server(futures.ThreadPoolExecutor(max_workers=workers))
+    lily_pb2_grpc.add_ExcelServicer_to_server(excel.Service(), server)
+    lily_pb2_grpc.add_TexServicer_to_server(tex.Service(), server)
+    _setup_health_thread(server)
+    server.add_insecure_port(addr)
+    server.start()
+    logging.info(
+        "Lily gRPC server started, listening on %s with %d threads", addr, workers)
+    try:
+        server.wait_for_termination()
+    except KeyboardInterrupt:
+        logging.warn('exited...')
+        server.stop(0)
+
+
+# https://pika.readthedocs.io/en/stable/modules/parameters.html
+def rabbitmq_parameters(config):
+    credentials = pika.PlainCredentials(config['user'], config['password'])
+    parameters = pika.ConnectionParameters(
+        config['host'],
+        config['port'],
+        config['virtual-host'],
+        credentials)
+    return parameters
+
+
+# https://www.postgresql.org/docs/current/libpq-connect.html
+def postgresql_url(config):
+    logging.debug('open postgresql://%s@%s:%d/%s',
+                  config['user'], config['host'], config['port'], config['name'])
+    url = 'host=%s port=%d user=%s password=%s dbname=%s sslmode=disable' % (
+        config['host'], config['port'], config['user'], config['password'], config['name'])
+    with psycopg.connect(url) as db:
+        cur = db.cursor()
+        cur.execute('SELECT VERSION(), CURRENT_TIMESTAMP')
+        row = cur.fetchone()
+        if row:
+            logging.debug("%s %s", row[0], row[1])
+    return url

lily/palm/excel.py

@@ -0,0 +1,42 @@
+import logging
+import io
+
+import pandas
+
+from . import lily_pb2, lily_pb2_grpc
+
+
+class Service(lily_pb2_grpc.ExcelServicer):
+    def Parse(self, request, context):
+        logging.info("parse excel %s" % request.content_type)
+        response = lily_pb2.ExcelParseResponse()
+
+        file = io.BytesIO(request.payload)
+        doc = pandas.read_excel(file, sheet_name=None)
+        for name, sheet in doc:
+            logging.info("find sheet %s", name)
+            sht = response.sheets.add()
+            sht.name = name
+            for row, item in sheet.to_dict().items():
+                for col, val in item.items():
+                    logging.debug('find (%s, %s, %s)', row, col, val)
+                    cell = sht.cells.add()
+                    cell.row = row
+                    cell.col = col
+                    cell.val = val
+        return response
+
+    def Generate(self, request, context):
+        logging.info("parse excel %s" % request.content_type)
+        bio = io.BytesIO()
+        writer = pandas.ExcelWriter(bio, engine="xlsxwriter")
+        for name, sheet in request.sheets:
+            # https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html
+            df = pandas.DataFrame(data={'col1': [1, 2], 'col2': [3, 4]})
+            df.to_excel(writer, scheet_name=name)
+        writer.save()
+
+        bio.seek(0)
+        response = lily_pb2.File()
+        response.payload = bio.read()
+        return response

lily/palm/lily_pb2.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: lily.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\nlily.proto\x12\x0cpalm.lily.v1\"C\n\x04\x46ile\x12\x19\n\x0c\x63ontent_type\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x07payload\x18\x02 \x01(\x0c\x42\x0f\n\r_content_type\"\xb6\x01\n\nExcelModel\x12.\n\x06sheets\x18\x01 \x03(\x0b\x32\x1e.palm.lily.v1.ExcelModel.Sheet\x1ax\n\x05Sheet\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x32\n\x05\x63\x65lls\x18\x02 \x03(\x0b\x32#.palm.lily.v1.ExcelModel.Sheet.Cell\x1a-\n\x04\x43\x65ll\x12\x0b\n\x03row\x18\x01 \x01(\r\x12\x0b\n\x03\x63ol\x18\x02 \x01(\r\x12\x0b\n\x03val\x18\x03 \x01(\t\"r\n\x0cTexToRequest\x12\x34\n\x05\x66iles\x18\x01 \x03(\x0b\x32%.palm.lily.v1.TexToRequest.FilesEntry\x1a,\n\nFilesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01\"\x12\n\x10\x45pubBuildRequest2|\n\x05\x45xcel\x12\x37\n\x05Parse\x12\x12.palm.lily.v1.File\x1a\x18.palm.lily.v1.ExcelModel\"\x00\x12:\n\x08Generate\x12\x18.palm.lily.v1.ExcelModel\x1a\x12.palm.lily.v1.File\"\x00\x32|\n\x03Tex\x12\x39\n\x05ToPdf\x12\x1a.palm.lily.v1.TexToRequest\x1a\x12.palm.lily.v1.File\"\x00\x12:\n\x06ToWord\x12\x1a.palm.lily.v1.TexToRequest\x1a\x12.palm.lily.v1.File\"\x00\x32\x45\n\x04\x45pub\x12=\n\x05\x42uild\x12\x1e.palm.lily.v1.EpubBuildRequest\x1a\x12.palm.lily.v1.File\"\x00\x42.\n*com.github.saturn_xiv.palm.plugins.lily.v1P\x01\x62\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'lily_pb2', _globals)
+if _descriptor._USE_C_DESCRIPTORS == False:
+
+  DESCRIPTOR._options = None
+  DESCRIPTOR._serialized_options = b'\n*com.github.saturn_xiv.palm.plugins.lily.v1P\001'
+  _TEXTOREQUEST_FILESENTRY._options = None
+  _TEXTOREQUEST_FILESENTRY._serialized_options = b'8\001'
+  _globals['_FILE']._serialized_start=28
+  _globals['_FILE']._serialized_end=95
+  _globals['_EXCELMODEL']._serialized_start=98
+  _globals['_EXCELMODEL']._serialized_end=280
+  _globals['_EXCELMODEL_SHEET']._serialized_start=160
+  _globals['_EXCELMODEL_SHEET']._serialized_end=280
+  _globals['_EXCELMODEL_SHEET_CELL']._serialized_start=235
+  _globals['_EXCELMODEL_SHEET_CELL']._serialized_end=280
+  _globals['_TEXTOREQUEST']._serialized_start=282
+  _globals['_TEXTOREQUEST']._serialized_end=396
+  _globals['_TEXTOREQUEST_FILESENTRY']._serialized_start=352
+  _globals['_TEXTOREQUEST_FILESENTRY']._serialized_end=396
+  _globals['_EPUBBUILDREQUEST']._serialized_start=398
+  _globals['_EPUBBUILDREQUEST']._serialized_end=416
+  _globals['_EXCEL']._serialized_start=418
+  _globals['_EXCEL']._serialized_end=542
+  _globals['_TEX']._serialized_start=544
+  _globals['_TEX']._serialized_end=668
+  _globals['_EPUB']._serialized_start=670
+  _globals['_EPUB']._serialized_end=739
+# @@protoc_insertion_point(module_scope)

lily/palm/lily_pb2_grpc.py

@@ -0,0 +1,272 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+
+from . import lily_pb2 as lily__pb2
+
+
+class ExcelStub(object):
+    """----------------------------------------------------------------------------
+
+    """
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.Parse = channel.unary_unary(
+                '/palm.lily.v1.Excel/Parse',
+                request_serializer=lily__pb2.File.SerializeToString,
+                response_deserializer=lily__pb2.ExcelModel.FromString,
+                )
+        self.Generate = channel.unary_unary(
+                '/palm.lily.v1.Excel/Generate',
+                request_serializer=lily__pb2.ExcelModel.SerializeToString,
+                response_deserializer=lily__pb2.File.FromString,
+                )
+
+
+class ExcelServicer(object):
+    """----------------------------------------------------------------------------
+
+    """
+
+    def Parse(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def Generate(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_ExcelServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'Parse': grpc.unary_unary_rpc_method_handler(
+                    servicer.Parse,
+                    request_deserializer=lily__pb2.File.FromString,
+                    response_serializer=lily__pb2.ExcelModel.SerializeToString,
+            ),
+            'Generate': grpc.unary_unary_rpc_method_handler(
+                    servicer.Generate,
+                    request_deserializer=lily__pb2.ExcelModel.FromString,
+                    response_serializer=lily__pb2.File.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'palm.lily.v1.Excel', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+ # This class is part of an EXPERIMENTAL API.
+class Excel(object):
+    """----------------------------------------------------------------------------
+
+    """
+
+    @staticmethod
+    def Parse(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/palm.lily.v1.Excel/Parse',
+            lily__pb2.File.SerializeToString,
+            lily__pb2.ExcelModel.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def Generate(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/palm.lily.v1.Excel/Generate',
+            lily__pb2.ExcelModel.SerializeToString,
+            lily__pb2.File.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+
+class TexStub(object):
+    """----------------------------------------------------------------------------
+
+    """
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.ToPdf = channel.unary_unary(
+                '/palm.lily.v1.Tex/ToPdf',
+                request_serializer=lily__pb2.TexToRequest.SerializeToString,
+                response_deserializer=lily__pb2.File.FromString,
+                )
+        self.ToWord = channel.unary_unary(
+                '/palm.lily.v1.Tex/ToWord',
+                request_serializer=lily__pb2.TexToRequest.SerializeToString,
+                response_deserializer=lily__pb2.File.FromString,
+                )
+
+
+class TexServicer(object):
+    """----------------------------------------------------------------------------
+
+    """
+
+    def ToPdf(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def ToWord(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_TexServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'ToPdf': grpc.unary_unary_rpc_method_handler(
+                    servicer.ToPdf,
+                    request_deserializer=lily__pb2.TexToRequest.FromString,
+                    response_serializer=lily__pb2.File.SerializeToString,
+            ),
+            'ToWord': grpc.unary_unary_rpc_method_handler(
+                    servicer.ToWord,
+                    request_deserializer=lily__pb2.TexToRequest.FromString,
+                    response_serializer=lily__pb2.File.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'palm.lily.v1.Tex', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+ # This class is part of an EXPERIMENTAL API.
+class Tex(object):
+    """----------------------------------------------------------------------------
+
+    """
+
+    @staticmethod
+    def ToPdf(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/palm.lily.v1.Tex/ToPdf',
+            lily__pb2.TexToRequest.SerializeToString,
+            lily__pb2.File.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def ToWord(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/palm.lily.v1.Tex/ToWord',
+            lily__pb2.TexToRequest.SerializeToString,
+            lily__pb2.File.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+
+class EpubStub(object):
+    """----------------------------------------------------------------------------
+
+    """
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.Build = channel.unary_unary(
+                '/palm.lily.v1.Epub/Build',
+                request_serializer=lily__pb2.EpubBuildRequest.SerializeToString,
+                response_deserializer=lily__pb2.File.FromString,
+                )
+
+
+class EpubServicer(object):
+    """----------------------------------------------------------------------------
+
+    """
+
+    def Build(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_EpubServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'Build': grpc.unary_unary_rpc_method_handler(
+                    servicer.Build,
+                    request_deserializer=lily__pb2.EpubBuildRequest.FromString,
+                    response_serializer=lily__pb2.File.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'palm.lily.v1.Epub', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+ # This class is part of an EXPERIMENTAL API.
+class Epub(object):
+    """----------------------------------------------------------------------------
+
+    """
+
+    @staticmethod
+    def Build(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/palm.lily.v1.Epub/Build',
+            lily__pb2.EpubBuildRequest.SerializeToString,
+            lily__pb2.File.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

lily/palm/tex.py

@@ -0,0 +1,33 @@
+import logging
+import tempfile
+import os.path
+import subprocess
+
+from . import lily_pb2, lily_pb2_grpc
+
+
+def _tmp_root():
+    return os.path.join(tempfile.tempdir())
+
+
+class Service(lily_pb2_grpc.TexServicer):
+    def ToPdf(self, request, context):
+        logging.info("convert tex to pdf(%d)" % len(request.files))
+        with tempfile.TemporaryDirectory(prefix='tex-') as root:
+            for name in request.files:
+                with open(os.path.join(root, name), mode='wb') as fd:
+                    logging.debug("generate file %s/%s", root, name)
+                    fd.write(request.files[name])
+            for x in range(2):
+                subprocess.run(
+                    ['xelatex', '-halt-on-error', 'main.tex'], cwd=root)
+            with open(os.path.join(root, 'main.pdf'), mode="rb") as fd:
+                response = lily_pb2.File()
+                response.content_type = 'application/pdf'
+                response.payload = fd.read()
+                return response
+
+    def ToWord(self, request, context):
+        logging.info("convert tex to word %s" % request.content_type)
+        response = lily_pb2.File()
+        return response