elg - npm Package Compare versions

Comparing version 0.4.20 to 0.4.21

elg/model/request/ImageRequest.py

+from collections.abc import AsyncIterable, Iterable
+from pathlib import Path
+from typing import Any, Dict
+
+from pydantic import validator
+
+from .. import Request
+
+
+class ImageRequest(Request):
+    """
+    Request representing a piece of an image - the actual image data may be sent as a separate request.
+    Subclass of :class:`elg.model.base.Request.Request`
+    """
+
+    type: str = "image"
+    """*(required)* the type of request must be \"image\""""
+
+    content: bytes = None
+    """*(optional)* image itself, if not being sent as separate stream"""
+
+    generator: Any = None
+    """*(optional)* generator that provide the audio itself"""
+
+    format: str = "png"
+    """*(required)* format of image, e.g BMP, PNG, JPG. Default is \"png\""""
+
+    features: Dict = None
+    """*(optional)* arbitrary json metadata about content"""
+
+    @validator("format")
+    def format_must_be_valid(cls, format_value):
+        """
+        *(validator)* ensures the format field of the image request is either None or one of the currently accepted 5
+        """
+        acceptable_formats = ["tiff", "bmp", "png", "jpeg", "gif"]
+        if format_value.lower() in acceptable_formats:
+            return format_value
+        raise ValueError("This image format is not supported")
+
+    @validator("generator")
+    def generator_must_be_iterable(cls, v):
+        """
+        *(validator)* ensures the iterator field of the image request is either None or an Iterable
+        """
+        if v is None:
+            return v
+        if isinstance(v, (AsyncIterable, Iterable)):
+            return v
+        raise ValueError(f"The generator musts be None or an Iterable, not {type(v)}")
+
+    @staticmethod
+    def generator_from_file(filename, blocksize=1024):
+        with open(filename, "rb") as file:
+            byte = file.read(blocksize)
+            while byte:
+                yield byte
+                byte = file.read(blocksize)
+
+    @classmethod
+    def from_file(
+        cls,
+        filename,
+        format: str = None,
+        features: Dict = None,
+        params: dict = None,
+        streaming: bool = False,
+        blocksize: int = 1024,
+    ):
+        """
+        allows you to generate image request from file
+        """
+        filename = Path(filename)
+        if not filename.is_file():
+            raise ValueError(f"{filename} musts be the path to a file.")
+        if streaming:
+            generator = cls.generator_from_file(filename=filename, blocksize=blocksize)
+            content = None
+        else:
+            with open(filename, "rb") as f:
+                content = f.read()
+            generator = None
+        if format is None:
+            format = filename.suffix[1 : len(filename.suffix)]
+        return cls(
+            content=content,
+            generator=generator,
+            format=format,
+            features=features,
+            params=params,
+        )
+
+    def __str__(self):
+        return " - ".join(
+            [f"{k}: {v}" for k, v in self.dict(exclude={"content", "generator"}).items() if v is not None]
+        ) + ((" - content " + str(len(self.content))) if self.content else (" - content generator"))

elg.egg-info/PKG-INFO

 Metadata-Version: 2.1
 Name: elg
-Version: 0.4.20
+Version: 0.4.21
 Summary: Use the European Language Grid in your Python projects
@@ -5,0 +5,0 @@ Home-page: https://gitlab.com/european-language-grid/platform/python-client

elg.egg-info/SOURCES.txt

@@ -45,2 +45,3 @@ LICENSE
 elg/model/request/AudioRequest.py
+elg/model/request/ImageRequest.py
 elg/model/request/StructuredTextRequest.py
@@ -47,0 +48,0 @@ elg/model/request/TextRequest.py

elg/__init__.py

@@ -1,2 +0,2 @@
-__version__ = "0.4.20"
+__version__ = "0.4.21"
 
@@ -3,0 +3,0 @@ import importlib.util

elg/cli/docker_create.py

@@ -33,3 +33,8 @@ import sys
         info_parser.add_argument(
-            "-n", "--classname", type=str, default=None, required=True, help="Name of the Service Class"
+            "-n",
+            "--classname",
+            type=str,
+            default=None,
+            required=True,
+            help="Name of the Service Class",
         )
@@ -48,3 +53,3 @@ info_parser.add_argument(
             type=str,
-            default="python:slim",
+            default="python:3.8-slim",
             required=None,
@@ -98,5 +103,5 @@ help="Name of the base Docker image used in the Dockerfile",
             type=str,
-            default="flask",
+            default="auto",
             required=None,
-            help="Type of service used. Can be 'flask' or 'quart'.",
+            help="Type of service used. Can be 'auto' (the service type is discovered), 'flask', or 'quart'.",
             choices=["flask", "quart"],
@@ -168,2 +173,18 @@ )
     def run(self):
+        if self._service_type == "auto":
+            with open(self._path) as f:
+                script = f.read()
+            if "FlaskService" in script:
+                logger.info(
+                    "flask has been found as the service_type. Please use `--service_type quart` to overwrite it."
+                )
+                self._service_type = "flask"
+            elif "QuartService" in script:
+                logger.info(
+                    "quart has been found as the service_type. Please use `--service_type flask` to overwrite it."
+                )
+                self._service_type = "quart"
+            else:
+                raise ValueError("Neither FlaskService nor QuartService were found in the Python script.")
+
         if self._service_type == "flask":
@@ -170,0 +191,0 @@ from ..flask_service import FlaskService

elg/cli/local_installation_docker.py

@@ -16,2 +16,3 @@ import sys
         sidecar_image=args.sidecar_image,
+        image_envvars=args.image_envvars,
         name=args.name,
@@ -25,2 +26,3 @@ full_name=args.full_name,
         expose_port=args.expose_port,
+        helpers=args.helper,
     )
@@ -49,2 +51,10 @@
         local_installation_parser.add_argument(
+            "--image_envvars",
+            type=str,
+            default=[],
+            required=False,
+            nargs="*",
+            help='environment variables to be set in the docker image. format "key=value"',
+        )
+        local_installation_parser.add_argument(
             "--sidecar_image",
@@ -112,2 +122,11 @@ type=str,
         )
+        local_installation_parser.add_argument(
+            "--helper",
+            type=str,
+            required=False,
+            default=[],
+            action="append",
+            choices=["temp-storage"],
+            help="helper services to include in the compose stack",
+        )
 
@@ -125,2 +144,3 @@ local_installation_parser.set_defaults(func=local_installation_command_factory)
         name: str = None,
+        image_envvars: List[str] = [],
         full_name: str = None,
@@ -130,2 +150,3 @@ gui_image: str = "registry.gitlab.com/european-language-grid/usfd/gui-ie:latest",
         gui_path: str = "",
+        helpers: List[str] = None,
         record: Any = {},
@@ -140,2 +161,3 @@ ):
         self._name = name
+        self._image_envvars = image_envvars
         self._full_name = full_name
@@ -145,2 +167,3 @@ self._gui_image = gui_image
         self._gui_path = gui_path
+        self._helpers = helpers
         self._record = record
@@ -158,2 +181,3 @@
                 name=self._name,
+                image_envvars=self._image_envvars,
                 full_name=self._full_name,
@@ -166,3 +190,3 @@ gui=self._gui,
             )
-            LocalInstallation(ltservices=[ltservice]).create_docker_compose(
+            LocalInstallation(ltservices=[ltservice], helpers=self._helpers).create_docker_compose(
                 expose_port=self._expose_port,
@@ -169,0 +193,0 @@ path=self._folder,

elg/cli/local_installation_ids.py

@@ -22,2 +22,3 @@ import sys
         gui_ports=args.gui_ports,
+        helpers=args.helper,
     )
@@ -97,2 +98,11 @@
         )
+        local_installation_parser.add_argument(
+            "--helper",
+            type=str,
+            required=False,
+            default=[],
+            action="append",
+            choices=["temp-storage"],
+            help="helper services to include in the compose stack",
+        )
 
@@ -112,2 +122,3 @@ local_installation_parser.set_defaults(func=local_installation_command_factory)
         gui_ports: int = 80,
+        helpers: List[str] = None,
     ):
@@ -123,2 +134,3 @@ self._ids = ids
         self._gui_ports = gui_ports
+        self._helpers = helpers
 
@@ -134,2 +146,3 @@ def run(self):
                 gui_ports=self._gui_ports,
+                helpers=self._helpers,
                 domain=self._domain,
@@ -136,0 +149,0 @@ use_cache=self._use_cache,

elg/flask_service.py

@@ -12,3 +12,3 @@ import inspect
     import docker
-    from flask import Flask
+    from flask import Flask, g
     from flask import request as flask_request
@@ -22,4 +22,5 @@ from requests_toolbelt import MultipartDecoder
 
-from .model import (AudioRequest, Failure, Progress, Request, ResponseObject,
-                    StandardMessages, StructuredTextRequest, TextRequest)
+from .model import (AudioRequest, Failure, ImageRequest, Progress, Request,
+                    ResponseObject, StandardMessages, StructuredTextRequest,
+                    TextRequest)
 from .utils.docker import COPY_FOLDER, DOCKERFILE, ENTRYPOINT_FLASK, ENV_FLASK
@@ -37,3 +38,3 @@ from .utils.json_encoder import json_encoder
 
-    def __init__(self, name: str):
+    def __init__(self, name: str = "My ELG Service", path: str = "/process"):
         """
@@ -43,3 +44,6 @@ Init function of the FlaskService
         Args:
-            name (str): Name of the service. It doesn't have any importance.
+            name (str, optional): Name of the service. It doesn't have any importance.
+            path (str, optional): Path of the endpoint to expose. It can contain parameters, e.g., "/translate/<src>/<target>"
+                later accessible in the process_* methods using the `self.url_param` method, e.g.,
+                self.url_param('src'). Default to "/process".
         """
@@ -53,3 +57,3 @@ self.name = name
         self.app.json_encoder = json_encoder(self)
-        self.app.add_url_rule("/process", "process", self.process, methods=["POST"])
+        self.app.add_url_rule(path, "process", self.process, methods=["POST"])
 
@@ -95,7 +99,14 @@ def to_json(self, obj):
 
-    def process(self):
+    def url_param(self, name: str):
         """
+        Method to get give access to url parameters
+        """
+        return g._elg_args[name]
+
+    def process(self, **kwargs):
+        """
         Main request processing logic - accepts a JSON request and returns a JSON response.
         """
         logger.info("Process request")
+        g._elg_args = kwargs
         even_stream = True if "text/event-stream" in flask_request.accept_mimetypes else False
@@ -114,3 +125,3 @@ logger.debug("Accept MimeTypes: {mimetypes}", mimetypes=flask_request.accept_mimetypes)
                         data[k] = v
-                elif "audio" in headers["Content-Type"]:
+                elif "audio" in headers["Content-Type"] or "image" in headers["Content-Type"]:
                     data["content"] = part.content
@@ -124,6 +135,8 @@ else:
         logger.debug("Data type: {request_type}", request_type=request_type)
-        if request_type in ["audio", "text", "structuredText"]:
+        if request_type in ["audio", "image", "text", "structuredText"]:
             try:
                 if request_type == "audio":
                     request = AudioRequest(**data)
+                elif request_type == "image":
+                    request = ImageRequest(**data)
                 elif request_type == "text":
@@ -192,2 +205,5 @@ request = TextRequest(**data)
             return self.process_structured_text(request)
+        elif request.type == "image":
+            logger.debug("Process image request")
+            return self.process_image(request)
         elif request.type == "audio":
@@ -225,2 +241,11 @@ logger.debug("Process audio request")
 
+    def process_image(self, request: ImageRequest):
+        """
+        Method to implement if the service takes image as input.
+
+        Args:
+            request (ImageRequest): ImageRequest object.
+        """
+        raise NotImplementedError()
+
     def generator_mapping(self, generator):
@@ -297,3 +322,3 @@ end = False
         commands: List[str] = [],
-        base_image: str = "python:slim",
+        base_image: str = "python:3.8-slim",
         path: str = None,
@@ -311,3 +336,3 @@ log_level: str = "INFO",
             commands (List[str], optional): List off additional commands to run in the Dockerfile. Defaults to [].
-            base_image (str, optional): Name of the base Docker image used in the Dockerfile. Defaults to 'python:slim'.
+            base_image (str, optional): Name of the base Docker image used in the Dockerfile. Defaults to 'python:3.8-slim'.
             path (str, optional): Path where to generate the file. Defaults to None.
@@ -342,3 +367,6 @@ log_level (str, optional): The minimum severity level from which logged messages should be displayed. Defaults to 'INFO'.
                     env=ENV_FLASK.format(
-                        workers=workers, timeout=timeout, worker_class=worker_class, log_level=log_level
+                        workers=workers,
+                        timeout=timeout,
+                        worker_class=worker_class,
+                        log_level=log_level,
                     ),
@@ -345,0 +373,0 @@ )

elg/local_installation.py

@@ -15,14 +15,15 @@ import json
                                        EXPOSE_PORT_CONFIG, FRONTEND, GUI,
-                                       GUI_CONF_TEMPLATE,
+                                       GUI_CONF_TEMPLATE, HELPERS,
+                                       HELPERS_TEMPLATE,
                                        HTML_INDEX_HTML_TEMPLATE,
                                        HTML_INDEX_IFRAME, HTML_INDEX_SCRIPT,
-                                       LTSERVICE, LTSERVICE_URL,
-                                       LTSERVICE_WITH_SIDECAR)
+                                       I18N, I18N_CONF_TEMPLATE, LTSERVICE,
+                                       LTSERVICE_SIDECAR, LTSERVICE_URL)
 
 
-def name_from_image(image):
+def _name_from_image(image):
     return re.sub("[^0-9a-zA-Z]+", "_", image)[-60:]
 
 
-def random_name(length: int = 10):
+def _random_name(length: int = 10):
     return "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(length))
@@ -32,2 +33,6 @@
 class LTServiceLocalInstallation:
+    """
+    Class that contains all the information to deploy an ELG-compatible service locally
+    """
+
     def __init__(
@@ -39,2 +44,3 @@ self,
         name: str,
+        image_envvars: List[str],
         full_name: str,
@@ -49,2 +55,20 @@ port: int,
     ):
+        """
+        Initialize a LTServiceLocalInstallation object
+
+        Args:
+            id (int): id of the LT service
+            image (str): docker image of the LT service
+            sidecar_image (str): docker image of the sidecar container
+            name (str): short name of the LT service used in the docker-compose file
+            image_envvars (List[str]): environment variables to pass to the LT service docker container
+            full_name (str): long name of the LT service used in the GUI
+            port (int): port exposed by the LT service docker container
+            path (str): path to the LT service endpoint
+            gui (bool): boolean to indicate if yes or no the GUI should be deployed
+            gui_image (str): docker image of the GUI
+            gui_port (int): port exposed by the GUI docker container
+            gui_path (str): path to the GUI endpoint
+            record (Any): metadata record of the LT service
+        """
         self.id = id
@@ -54,2 +78,3 @@ self.image = image
         self.name = name
+        self.image_envvars = image_envvars
         self.full_name = full_name
@@ -64,11 +89,14 @@ self.port = port
 
-        self.ltservice = (
-            LTSERVICE.format(LTSERVICE_NAME=self.name, LTSERVICE_IMAGE=self.image)
-            if self.sidecar_image is None or self.sidecar_image == ""
-            else LTSERVICE_WITH_SIDECAR.format(
-                LTSERVICE_NAME=self.name,
-                LTSERVICE_IMAGE=self.image,
-                SIDECAR_IMAGE=self.sidecar_image,
-            )
+        self.ltservice = LTSERVICE.format(
+            LTSERVICE_NAME=self.name,
+            LTSERVICE_IMAGE=self.image,
+            LTSERVICE_ENVVARS=str(self.image_envvars),
         )
+
+        self.sidecarservice = (
+            LTSERVICE_SIDECAR.format(LTSERVICE_NAME=self.name, SIDECAR_IMAGE=self.sidecar_image)
+            if self.sidecar_image
+            else ""
+        )
+
         self.url = LTSERVICE_URL.format(
@@ -87,3 +115,3 @@ LTSERVICE_NAME=self.name,
                 LTSERVICE_NAME=self.name,
-                GUI_NAME=name_from_image(self.gui_image),
+                GUI_NAME=_name_from_image(self.gui_image),
                 GUI_PATH=self.gui_path,
@@ -99,2 +127,3 @@ )
         gui_port: int = 80,
+        image_envvars: List[str] = [],
         domain: str = "live",
@@ -104,2 +133,18 @@ use_cache: bool = True,
     ):
+        """
+        Class method to init a LTServiceLocalInstallation object from the id of an LT service deployed in the ELG cluster
+
+        Args:
+            id (int): id of the LT service in the ELG cluster
+            gui (bool, optional): boolean to indicate if yes or no the GUI should be deployed. Defaults to True.
+            gui_image (_type_, optional): docker image of the GUI. Defaults to "registry.gitlab.com/european-language-grid/usfd/gui-ie:latest".
+            gui_port (int, optional): port exposed by the GUI docker container. Defaults to 80.
+            image_envvars (List[str], optional): environment variables to pass to the LT service docker container. Defaults to [].
+            domain (str, optional): domain of the ELG cluster. Defaults to "live".
+            use_cache (bool, optional): True if you want to use cached files. Defaults to True.
+            cache_dir (str, optional): path to the cache_dir. Set it to None to not store any cached files. Defaults to "~/.cache/elg".
+
+        Returns:
+            LTServiceLocalInstallation: the LTServiceLocalInstallation object created
+        """
         entity = Entity.from_id(id=id, domain=domain, use_cache=use_cache, cache_dir=cache_dir)
@@ -116,5 +161,3 @@ software_distribution = get_information(
             )
-            # bypass for UDPipe English to make some tests
-            if entity.id != 423:
-                return None
+            return None
         sidecar_image = software_distribution.get("service_adapter_download_location")
@@ -140,2 +183,3 @@ image = software_distribution.get("docker_download_location")
             name=name,
+            image_envvars=image_envvars,
             full_name=full_name,
@@ -158,2 +202,3 @@ port=port,
         name: str = None,
+        image_envvars: List[str] = [],
         full_name: str = None,
@@ -166,3 +211,22 @@ gui: bool = False,
     ):
-        name = name if name else random_name()
+        """
+        Class method to init a LTServiceLocalInstallation object from a docker image
+
+        Args:
+            image (str): docker image of the LT service
+            execution_location (str): execution location of the LT service in the LT service docker container (e.g. http://localhost:8000/process)
+            sidecar_image (str, optional): docker image of the sidecar. Defaults to "".
+            name (str, optional): short name of the LT service used in the docker-compose. Defaults to None.
+            image_envvars (List[str], optional): environment variables to pass to the LT service docker container. Defaults to [].
+            full_name (str, optional): long name of the LT service used in the GUI. Defaults to None.
+            gui (bool, optional): boolean to indicate if yes or no the GUI should be deplo. Defaults to False.
+            gui_image (str, optional): docker image of the GUI. Defaults to "registry.gitlab.com/european-language-grid/usfd/gui-ie:latest".
+            gui_port (int, optional): port exposed by the GUI docker container. Defaults to 80.
+            gui_path (str, optional): path to the GUI endpoint. Defaults to "".
+            record (Any, optional): metadata record of the LT service. Defaults to {}.
+
+        Returns:
+            LTServiceLocalInstallation: the LTServiceLocalInstallation object created
+        """
+        name = name if name else _random_name()
         full_name = full_name if full_name else f"ELG Service from Docker {name}"
@@ -178,2 +242,3 @@ execution_location = urlparse(execution_location)
             name=name,
+            image_envvars=image_envvars,
             full_name=full_name,
@@ -191,4 +256,16 @@ port=port,
 class LocalInstallation:
-    def __init__(self, ltservices: List[LTServiceLocalInstallation]):
+    """
+    Class that contains all the information to deploy ELG-compatible services with a small part of the ELG infrastructure locally
+    """
+
+    def __init__(self, ltservices: List[LTServiceLocalInstallation], helpers: List[str] = None):
+        """
+        Initialize a LocalInstallation object
+
+        Args:
+            ltservices (List[LTServiceLocalInstallation]): list of the LT services local installation to deploy
+            helpers (List[str], optional): list of helpers to deploy. Currently only "temp-storage" is a valid helper and can be used to deploy a temporary storage needed for some services. Defaults to None.
+        """
         self.ltservices = ltservices
+        self.helpers = helpers
 
@@ -202,2 +279,4 @@ @classmethod
         gui_ports: Union[int, List[int]] = 80,
+        helpers: List[str] = None,
+        images_envvars: List[List[str]] = None,
         domain: str = "live",
@@ -207,2 +286,19 @@ use_cache: bool = True,
     ):
+        """
+        Class method to init a LocalInstallation object from multiple ids of LT services deployed in the ELG cluster
+
+        Args:
+            ids (List[int]): list of ids od LT services in the ELG cluster
+            gui (bool, optional): boolean to indicate if yes or no the GUI should be deployed. Defaults to True.
+            gui_images (Union[str, List[str], optional): docker images of the GUI. If string, the same GUI docker image will be used for all the services. Otherwise, the number of GUI docker images needs to be the same as the number of LT services deployed. Defaults to "registry.gitlab.com/european-language-grid/usfd/gui-ie:latest".
+            gui_ports (Union[int, List[int]], optional): port exposed by the GUI docker container. If integer, the same port will be used for all the services. Otherwise, the number of port needs to be the same as the number of LT services deployed. Defaults to 80.
+            helpers (List[str], optional): list of helpers to deploy. Currently only "temp-storage" is a valid helper and can be used to deploy a temporary storage needed for some services. Defaults to None.
+            images_envvars (List[List[str]], optional): environment variables to pass to the LT services docker container. Defaults to None.
+            domain (str, optional): domain of the ELG cluster. Defaults to "live".
+            use_cache (bool, optional): True if you want to use cached files. Defaults to True.
+            cache_dir (str, optional): path to the cache_dir. Set it to None to not store any cached files. Defaults to "~/.cache/elg".
+
+        Returns:
+            LocalInstallation: the LocalInstallation object created
+        """
         if isinstance(gui_images, list) and len(gui_images) == 1:
@@ -232,4 +328,15 @@ gui_images = gui_images[0]
 
+        if images_envvars is None:
+            images_envvars = [[] for _ in range(len(ids))]
+        if isinstance(images_envvars, list):
+            if len(images_envvars) != len(ids):
+                raise ValueError(
+                    f"You provided {len(images_envvars)} images env variables and {len(ids)} service ids. These two numbers must be equal."
+                )
+        else:
+            raise ValueError("images_envvars must be a list of the same length of the number of service ids.")
+
         assert len(ids) == len(gui_images)
         assert len(ids) == len(gui_ports)
+        assert len(ids) == len(images_envvars)
 
@@ -242,2 +349,3 @@ ltservices = [
                 gui_port=gui_port,
+                image_envvars=image_envvars,
                 domain=domain,
@@ -247,6 +355,6 @@ use_cache=use_cache,
             )
-            for ltservice_id, gui_image, gui_port in zip(ids, gui_images, gui_ports)
+            for ltservice_id, gui_image, gui_port, image_envvars in zip(ids, gui_images, gui_ports, images_envvars)
         ]
         ltservices = [ltservice for ltservice in ltservices if ltservice]
-        return cls(ltservices=ltservices)
+        return cls(ltservices=ltservices, helpers=helpers)
 
@@ -258,2 +366,9 @@ def create_docker_compose(
     ):
+        """
+        Method to generate the docker compose file and all the configuration files to deploy the LocalInstallation
+
+        Args:
+            expose_port (int, optional): port used to expose the GUI or the LT service execution server. Defaults to 8080.
+            path (str, optional): path where to store the configuration files. Defaults to "./elg_local_installation/".
+        """
         if self.ltservices == []:
@@ -266,13 +381,39 @@ logger.warning("None of the services can be deployed locally. Therefore, no files will be created.")
         gui = len(guis_image_port) > 0
-        if gui:
-            guis = [
-                GUI.format(GUI_NAME=name_from_image(gui_image), GUI_IMAGE=gui_image)
-                for gui_image, _ in guis_image_port
-            ]
+        has_helpers = self.helpers and len(self.helpers) > 0
+        if gui or has_helpers:
+            guis = (
+                (
+                    [
+                        GUI.format(GUI_NAME=_name_from_image(gui_image), GUI_IMAGE=gui_image)
+                        for gui_image, _ in guis_image_port
+                    ]
+                    + [I18N]
+                )
+                if gui
+                else []
+            )
+            helpers = []
+            helpers_conf = []
+            if has_helpers:
+                for h in self.helpers:
+                    if h in HELPERS:
+                        helpers.append(HELPERS[h].format(EXPOSE_PORT=expose_port))
+                        helpers_conf.append(HELPERS_TEMPLATE[h])
+                    else:
+                        logger.warning(f"Unknown helper {h}, ignored")
             frontend = FRONTEND.format(EXPOSE_PORT=expose_port)
-            gui_conf_templates = [
-                GUI_CONF_TEMPLATE.format(GUI_NAME=name_from_image(gui_image), GUI_PORT=gui_port)
-                for gui_image, gui_port in guis_image_port
-            ]
-            default_conf_template = DEFAULT_CONF_TEMPLATE.format(GUIS="\n\n".join(gui_conf_templates))
+            gui_conf_templates = (
+                (
+                    [
+                        GUI_CONF_TEMPLATE.format(GUI_NAME=_name_from_image(gui_image), GUI_PORT=gui_port)
+                        for gui_image, gui_port in guis_image_port
+                    ]
+                    + [I18N_CONF_TEMPLATE]
+                )
+                if gui
+                else []
+            )
+            default_conf_template = DEFAULT_CONF_TEMPLATE.format(
+                GUIS="\n\n".join(gui_conf_templates), HELPERS="\n\n".join(helpers_conf)
+            )
             html_index_html_template = HTML_INDEX_HTML_TEMPLATE.format(
@@ -282,10 +423,16 @@ IFRAMES="\n".join([ltservice.iframe for ltservice in self.ltservices if ltservice.gui]),
             )
+
         docker_compose = DOCKER_COMPOSE.format(
             LTSERVICES="\n\n".join([ltservice.ltservice for ltservice in self.ltservices]),
+            LTSERVICES_SIDECAR="\n".join(
+                [ltservice.sidecarservice for ltservice in self.ltservices if ltservice.sidecarservice]
+            ),
             LTSERVICES_URL="\n".join([ltservice.url for ltservice in self.ltservices]),
             EXPOSE_PORT=expose_port,
-            EXPOSE_PORT_CONFIG=EXPOSE_PORT_CONFIG.format(EXPOSE_PORT=expose_port) if not gui else "",
-            EXECUTION_PATH="" if not gui else "/execution",
+            EXPOSE_PORT_CONFIG=EXPOSE_PORT_CONFIG.format(EXPOSE_PORT=expose_port)
+            if (not gui and not has_helpers)
+            else "",
             GUIS="\n".join(guis) if gui else "",
-            FRONTEND=frontend if gui else "",
+            HELPERS="\n".join(helpers) if has_helpers else "",
+            FRONTEND=frontend if gui or has_helpers else "",
         )
@@ -292,0 +439,0 @@ path = Path(path)

elg/model/__init__.py

 from .base import (Annotation, Failure, Progress, Request, ResponseObject,
                    StandardMessages, StatusMessage)
-from .request import AudioRequest, StructuredTextRequest, TextRequest
+from .request import (AudioRequest, ImageRequest, StructuredTextRequest,
+                      TextRequest)
 from .response import (AnnotationsResponse, AudioResponse, ClassesResponse,
                        ClassificationResponse, TextsResponse,
                        TextsResponseObject, get_response)

elg/model/base/Annotation.py

 from numbers import Number
+from typing import Any
 
@@ -30,1 +31,14 @@ from pydantic import BaseModel
         arbitrary_types_allowed = True
+
+    def json(self, **kwargs: Any) -> str:
+        if "exclude_none" not in kwargs.keys():
+            kwargs["exclude_none"] = True
+        return super().json(**kwargs)
+
+    def __getitem__(self, key):
+        return getattr(self, key)
+
+    def get(self, key, value: Any = None):
+        if hasattr(self, key):
+            return getattr(self, key)
+        return value

elg/model/base/Failure.py

@@ -24,1 +24,9 @@ from typing import Any, List
         return super().json(**kwargs)
+
+    def __getitem__(self, key):
+        return getattr(self, key)
+
+    def get(self, key, value: Any = None):
+        if hasattr(self, key):
+            return getattr(self, key)
+        return value

elg/model/base/Progress.py

@@ -27,1 +27,9 @@ from typing import Any
         return super().json(**kwargs)
+
+    def __getitem__(self, key):
+        return getattr(self, key)
+
+    def get(self, key, value: Any = None):
+        if hasattr(self, key):
+            return getattr(self, key)
+        return value

elg/model/base/Request.py

@@ -36,1 +36,9 @@ from typing import Any
         return super().json(**kwargs)
+
+    def __getitem__(self, key):
+        return getattr(self, key)
+
+    def get(self, key, value: Any = None):
+        if hasattr(self, key):
+            return getattr(self, key)
+        return value

elg/model/base/ResponseObject.py

@@ -35,1 +35,9 @@ from typing import Any, List
         return super().json(**kwargs)
+
+    def __getitem__(self, key):
+        return getattr(self, key)
+
+    def get(self, key, value: Any = None):
+        if hasattr(self, key):
+            return getattr(self, key)
+        return value

elg/model/base/StatusMessage.py

@@ -37,1 +37,9 @@ from typing import Any, Dict, List
         return super().json(**kwargs)
+
+    def __getitem__(self, key):
+        return getattr(self, key)
+
+    def get(self, key, value: Any = None):
+        if hasattr(self, key):
+            return getattr(self, key)
+        return value

elg/model/request/__init__.py

 from .AudioRequest import AudioRequest
+from .ImageRequest import ImageRequest
 from .StructuredTextRequest import StructuredTextRequest
 from .TextRequest import TextRequest

elg/model/request/StructuredTextRequest.py

@@ -1,2 +0,2 @@
-from typing import Dict, List
+from typing import Any, Dict, List
 
@@ -30,4 +30,4 @@ from pydantic import BaseModel, root_validator
 
-    texts: List = None
-    """*(optional)* recursive, same structure (should be List[Text] but postponed annotations introduced post python 3.6)"""
+    texts: "List[Text]" = None
+    """*(optional)* recursive, same structure"""
 
@@ -51,3 +51,19 @@ class Config:
 
+    def json(self, **kwargs: Any) -> str:
+        if "exclude_none" not in kwargs.keys():
+            kwargs["exclude_none"] = True
+        return super().json(**kwargs)
 
+    def __getitem__(self, key):
+        return getattr(self, key)
+
+    def get(self, key, value: Any = None):
+        if hasattr(self, key):
+            return getattr(self, key)
+        return value
+
+
+Text.update_forward_refs()
+
+
 class StructuredTextRequest(Request):
@@ -54,0 +70,0 @@ """

elg/model/response/ClassificationResponse.py

@@ -1,2 +0,2 @@
-from typing import List
+from typing import Any, List
 
@@ -24,3 +24,16 @@ from pydantic import BaseModel, Field
 
+    def json(self, **kwargs: Any) -> str:
+        if "exclude_none" not in kwargs.keys():
+            kwargs["exclude_none"] = True
+        return super().json(**kwargs)
 
+    def __getitem__(self, key):
+        return getattr(self, key)
+
+    def get(self, key, value: Any = None):
+        if hasattr(self, key):
+            return getattr(self, key)
+        return value
+
+
 class ClassificationResponse(ResponseObject):
@@ -27,0 +40,0 @@ """

elg/model/response/TextsResponse.py

 from numbers import Number
-from typing import Dict, List
+from typing import Any, Dict, List
 
@@ -21,3 +21,3 @@ from pydantic import BaseModel, root_validator
 
-    texts: List = None
+    texts: "List[TextsResponseObject]" = None
     """*(optional)* list of same structures, recursive"""
@@ -54,2 +54,15 @@
 
+    def json(self, **kwargs: Any) -> str:
+        if "exclude_none" not in kwargs.keys():
+            kwargs["exclude_none"] = True
+        return super().json(**kwargs)
+
+    def __getitem__(self, key):
+        return getattr(self, key)
+
+    def get(self, key, value: Any = None):
+        if hasattr(self, key):
+            return getattr(self, key)
+        return value
+
     def auto_content(self):
@@ -71,2 +84,5 @@ if self.features is not None and self.annotations is not None:
 
+TextsResponseObject.update_forward_refs()
+
+
 class TextsResponse(ResponseObject):
@@ -73,0 +89,0 @@ """

elg/quart_service.py

@@ -12,3 +12,3 @@ import inspect
     import docker
-    from quart import Quart, current_app, make_response
+    from quart import Quart, current_app, g, make_response
     from quart import request as input_request
@@ -25,4 +25,5 @@ from requests_toolbelt import MultipartDecoder
 
-from .model import (AudioRequest, Failure, Progress, ResponseObject,
-                    StandardMessages, StructuredTextRequest, TextRequest)
+from .model import (AudioRequest, Failure, ImageRequest, Progress,
+                    ResponseObject, StandardMessages, StructuredTextRequest,
+                    TextRequest)
 from .utils.docker import COPY_FOLDER, DOCKERFILE, ENTRYPOINT_QUART, ENV_QUART
@@ -39,3 +40,6 @@ from .utils.json_encoder import json_encoder
     def InternalError(text, detail={}):
-        return ProcessingError(500, StandardMessages.generate_elg_service_internalerror(params=[text], detail=detail))
+        return ProcessingError(
+            500,
+            StandardMessages.generate_elg_service_internalerror(params=[text], detail=detail),
+        )
 
@@ -140,3 +144,8 @@ @staticmethod
 
-    def __init__(self, name: str, request_size_limit: int = None):
+    def __init__(
+        self,
+        name: str = "My ELG Service",
+        path: str = "/process",
+        request_size_limit: int = None,
+    ):
         """
@@ -146,3 +155,6 @@ Init function of the QuartService
         Args:
-            name (str): Name of the service. It doesn't have any importance.
+            name (str, optional): Name of the service. It doesn't have any importance.
+            path (str, optional): Path of the endpoint to expose. It can contain parameters, e.g., "/translate/<src>/<target>"
+                later accessible in the process_* methods using the `self.url_param` method, e.g.,
+                self.url_param('src'). Default to "/process".
         """
@@ -172,3 +184,3 @@ self.name = name
         self.app.add_url_rule("/health", "health", self.health, methods=["GET"])
-        self.app.add_url_rule("/process", "process", self.process, methods=["POST"])
+        self.app.add_url_rule(path, "process", self.process, methods=["POST"])
 
@@ -253,2 +265,6 @@ def to_json(self, obj):
     @staticmethod
+    async def parse_plain_image(image_format):
+        return {"type": "image", "format": image_format}, input_request.body
+
+    @staticmethod
     async def parse_multipart():
@@ -263,5 +279,5 @@ boundary = input_request.mimetype_params.get("boundary", "").encode("ascii")
             "request" containing JSON, and second a "file upload" part named
-            "content" containing the audio.  This generator fully parses the JSON
+            "content" containing the audio or image. This generator fully parses the JSON
             part and yields that as a dict, then subsequently yields chunks of the
-            audio data until they run out.  We create the generator and consume its
+            audio/image data until they run out.  We create the generator and consume its
             first yield (the parsed JSON), then return the active generator so the
@@ -323,7 +339,14 @@ rest of the binary chunks can be consumed by the caller in an async for.
 
-    async def process(self):
+    def url_param(self, name: str):
         """
+        Method to get give access to url parameters
+        """
+        return g._elg_args[name]
+
+    async def process(self, **kwargs):
+        """
         Main request processing logic - accepts a JSON request and returns a JSON response.
         """
         logger.info("Process request")
+        g._elg_args = kwargs
         even_stream = True if "text/event-stream" in input_request.accept_mimetypes else False
@@ -344,2 +367,6 @@ logger.debug("Accept MimeTypes: {mimetypes}", mimetypes=input_request.accept_mimetypes)
             data["content"] = content
+        elif input_request.mimetype.startswith("image/"):
+            mime_type = input_request.mimetype.split("/")[1]
+            data, content = await self.parse_plain_image(mime_type)
+            data["content"] = content
         elif input_request.mimetype == "application/json":
@@ -356,2 +383,8 @@ data = await input_request.get_json()
                 raise ProcessingError.InvalidRequest()
+        elif data.get("type") == "image":
+            try:
+                logger.info(data)
+                request = ImageRequest(**data)
+            except Exception as e:
+                raise ProcessingError.InvalidRequest()
         elif data.get("type") == "text":
@@ -436,2 +469,5 @@ try:
             return await self.process_audio(request)
+        if request.type == "image":
+            logger.debug("Process image request")
+            return await self.process_image(request)
         raise ProcessingError.InvalidRequest()
@@ -466,2 +502,11 @@
 
+    async def process_image(self, request: ImageRequest):
+        """
+        Method to implement if the service takes an image as input. This method must be implemented as async.
+
+        Args:
+            request (ImageRequest): ImageRequest object.
+        """
+        raise ProcessingError.UnsupportedType()
+
     async def generator_mapping(self, generator):
@@ -538,3 +583,3 @@ end = False
         commands: List[str] = [],
-        base_image: str = "python:slim",
+        base_image: str = "python:3.8-slim",
         path: str = None,
@@ -549,3 +594,3 @@ log_level: str = "INFO",
             commands (List[str], optional): List off additional commands to run in the Dockerfile. Defaults to [].
-            base_image (str, optional): Name of the base Docker image used in the Dockerfile. Defaults to 'python:slim'.
+            base_image (str, optional): Name of the base Docker image used in the Dockerfile. Defaults to 'python:3.8-slim'.
             path (str, optional): Path where to generate the file. Defaults to None.
@@ -552,0 +597,0 @@ log_level (str, optional): The minimum severity level from which logged messages should be displayed. Defaults to 'INFO'.

elg/service.py

@@ -20,4 +20,4 @@ import hashlib
 from .entity import Entity, MetadataRecordObj
-from .model import (AudioRequest, Request, StructuredTextRequest, TextRequest,
-                    get_response)
+from .model import (AudioRequest, ImageRequest, Request, StructuredTextRequest,
+                    TextRequest, get_response)
 from .utils import (MIME, get_argument_from_json, get_domain, get_information,
@@ -328,2 +328,3 @@ get_metadatarecord, map_metadatarecord_to_result)
         }
+
         parameters = {
@@ -417,2 +418,6 @@ "id": 0,
                     request = AudioRequest.from_file(request_input, streaming=True)
+                elif request_type == "image":
+                    request = ImageRequest.from_file(request_input)
+                elif request_type == "imageStream":
+                    request = ImageRequest.from_file(request_input, streaming=True)
                 else:
@@ -453,5 +458,4 @@ raise ValueError(
         post_kwargs = {}
-        if (
-            isinstance(request, AudioRequest) and self.id == 0
-        ):  # self.id == 0 corresponds to a service initialized from a docker image
+        # self.id == 0 corresponds to a service initialized from a docker image
+        if isinstance(request, AudioRequest) and self.id == 0:
             files = {
@@ -483,2 +487,6 @@ "request": (
             headers["Content-Type"] = "audio/x-wav" if request.format == "LINEAR16" else "audio/mpeg"
+        elif isinstance(request, ImageRequest):
+            data = request.content if request.content else request.generator
+            post_kwargs["params"] = request.params
+            headers["Content-Type"] = "image/" + request.format
         elif isinstance(request, TextRequest) or isinstance(request, StructuredTextRequest):
@@ -485,0 +493,0 @@ data = json.dumps(request.dict())

elg/utils/local_installation.py

@@ -8,8 +8,11 @@ DOCKER_COMPOSE = """\
 
+{LTSERVICES_SIDECAR}
+
   restserver:
     image: registry.gitlab.com/european-language-grid/ilsp/elg-lt-service-execution-all:production-reactive
     command:
+    - "--spring.webflux.base-path=/execution"
     - "--logging.level.elg.ltserviceexecution.api=WARN"
 {LTSERVICES_URL}
-    - "--elg.base.url=http://localhost:{EXPOSE_PORT}{EXECUTION_PATH}"
+    - "--elg.base.url=http://localhost:{EXPOSE_PORT}/execution"
 {EXPOSE_PORT_CONFIG}
@@ -20,2 +23,4 @@ restart: always
 
+{HELPERS}
+
 {FRONTEND}
@@ -33,11 +38,8 @@
     image: "{LTSERVICE_IMAGE}"
+    environment: {LTSERVICE_ENVVARS}
     restart: always\
 """
 
-LTSERVICE_WITH_SIDECAR = """\
-  {LTSERVICE_NAME}:
-    image: "{LTSERVICE_IMAGE}"
-    restart: always
-
-  sidecar:
+LTSERVICE_SIDECAR = """\
+  {LTSERVICE_NAME}_sidecar:
     image: "{SIDECAR_IMAGE}"
@@ -53,2 +55,19 @@ network_mode: "service:{LTSERVICE_NAME}"
 
+HELPERS = {
+    "temp-storage": """\
+  tempstore:
+    image: registry.gitlab.com/european-language-grid/platform/temporary-storage:latest
+    command:
+    - "--base-dir=/mnt"
+    - "--upload-address=:80"
+    - "--url-base=http://localhost:{EXPOSE_PORT}/storage/retrieve/"
+    - "--cleanup-interval=10m"
+    restart: always
+    networks:
+      default:
+        aliases:
+        - "storage.elg"
+""",
+}
+
 GUI = """\
@@ -60,2 +79,8 @@ {GUI_NAME}:
 
+I18N = """\
+  i18n:
+    image: registry.gitlab.com/european-language-grid/platform/i18n-service:latest
+    restart: always
+"""
+
 FRONTEND = """\
@@ -89,9 +114,5 @@ frontend:
 
-    location /i18n/ {{
-        proxy_pass https://live.european-language-grid.eu/i18n/;
-    }}
-
     location /execution/ {{
         access_log /dev/stdout upstream_logging;
-        proxy_pass http://restserver:8080/;
+        proxy_pass http://restserver:8080/execution/;
     }}
@@ -101,2 +122,4 @@
 
+{HELPERS}
+
     # redirect server error pages to the static page /50x.html
@@ -118,2 +141,17 @@ #
 
+I18N_CONF_TEMPLATE = """\
+    location /i18n/ {
+        proxy_pass http://i18n:8080/;
+    }
+"""
+
+HELPERS_TEMPLATE = {
+    "temp-storage": """\
+    location /storage/ {
+        access_log /dev/stdout upstream_logging;
+        proxy_pass http://tempstore:8081/;
+    }
+""",
+}
+
 HTML_INDEX_HTML_TEMPLATE = """\
@@ -173,11 +211,17 @@ <!DOCTYPE html>
 HTML_INDEX_SCRIPT = """\
-window.addEventListener('message', function (e) {{
-  document.getElementById('{LTSERVICE_NAME}').contentWindow.postMessage(JSON.stringify({{
-    "StyleCss":" ",
-    "ServiceUrl":window.location.origin+"/execution/async/process/{LTSERVICE_NAME}",
-    "ApiRecordUrl":window.location.origin+"/{LTSERVICE_NAME}.json"
-  }}), window.location.origin);
-}}, false);
+(function() {{
+  var iframe = document.getElementById('{LTSERVICE_NAME}');
+  window.addEventListener('message', function (e) {{
+    if(iframe && iframe.contentWindow && e.source === iframe.contentWindow) {{
+      // this is a configuration request from {LTSERVICE_NAME}
+      iframe.contentWindow.postMessage(JSON.stringify({{
+        "StyleCss":" ",
+        "ServiceUrl":window.location.origin+"/execution/async/process/{LTSERVICE_NAME}",
+        "ApiRecordUrl":window.location.origin+"/{LTSERVICE_NAME}.json"
+      }}), window.location.origin);
+    }}
+  }}, false);
 
-document.getElementById('{LTSERVICE_NAME}').src = '/{GUI_NAME}/{GUI_PATH}';
+  iframe.src = '/{GUI_NAME}/{GUI_PATH}';
+}})();
 """

PKG-INFO

 Metadata-Version: 2.1
 Name: elg
-Version: 0.4.20
+Version: 0.4.21
 Summary: Use the European Language Grid in your Python projects
@@ -5,0 +5,0 @@ Home-page: https://gitlab.com/european-language-grid/platform/python-client

setup.py

@@ -5,3 +5,3 @@ from setuptools import find_packages, setup
     name="elg",
-    version="0.4.20",
+    version="0.4.21",
     author="ELG Technical Team",
@@ -8,0 +8,0 @@ url="https://gitlab.com/european-language-grid/platform/python-client",

Alert delta unavailable

Currently unable to show alert delta for PyPI packages.

Improved metrics

Total package byte prevSize

320080

6.9%

Number of package files

64

1.59%

Lines of code

7284

6.73%