Implementación del servicio Text To Speech (#63)

* Initial project layout

* Matame camion

* Minor modifications, readme and docker-compose for tts

* Estoy flipando

* Toma esa git :)

Author: alejgh

release/docker-compose.yml

@@ -71,6 +71,18 @@ services:
     networks:
       - postgres
 
+  texttospeech:
+    build: ../src/texttospeechservice/
+    container_name: texttospeechservice
+    ports:
+      - "5003:5000"
+    depends_on:
+      - kafka
+    networks:
+      - event_bus
+    environment: 
+      KAFKA_ENDPOINT: "kafka:9092"
+  
   authenticationservice:
     build: ../src/authenticationservice/
     container_name: authservice

src/PostsService/.dockerignore src/postsservice/.dockerignore

@@ -0,0 +1,11 @@
+FROM python:3.8.7-slim
+
+EXPOSE 5000
+
+RUN mkdir app
+COPY . app
+WORKDIR app
+RUN pip install -r requirements.txt
+ENV FLASK_CONFIG=production
+ENV GOOGLE_APPLICATION_CREDENTIALS=keys.json
+ENTRYPOINT ["python", "wsgi.py"]

src/PostsService/.gitignore src/postsservice/.gitignore

@@ -0,0 +1,35 @@
+""" Configuration module for the flask app.
+
+Each class represents a configuration to use by the flask app
+created in src/__init__.py.
+"""
+
+import os
+
+def _try_get_config_from_env(config_key, default):
+    if config_key not in os.environ:
+        return default
+    return os.environ[config_key]
+
+class BaseConfig():
+    DEBUG = False
+    TESTING = False
+    # these are our environment variables...
+    # if they are not found, a default value is provided for local tests
+    KAFKA_ENDPOINT = _try_get_config_from_env('KAFKA_ENDPOINT', 'localhost:9092')
+    KAFKA_LOGGING_TOPIC = _try_get_config_from_env('KAFKA_LOGGING_TOPIC', 'service_logs')
+    SERVICE_KEY = _try_get_config_from_env('SERVICE_KEY', 'tts_service')
+    LOCALE_MAPPINGS_FILE = _try_get_config_from_env('MAPPINGS_FILE', 'lang2locale.csv')
+    DEPLOY_PORT = int(_try_get_config_from_env('DEPLOY_PORT', '5000'))
+
+class DevelopmentConfig(BaseConfig):
+    DEBUG = True
+    TESTING = True
+    ENV = 'development'
+
+class ProductionConfig(BaseConfig):
+    ENV = 'production'
+
+class TestingConfig(BaseConfig):
+    TESTING = True
+    ENV = 'testing'

src/PostsService/Controllers/PostsController.cs src/postsservice/Controllers/PostsController.cs

@@ -0,0 +1,12 @@
+{
+  "type": "service_account",
+  "project_id": "translate-3e8bb",
+  "private_key_id": "fedd58a3dd62b7453f7d7917d23ecdd66a11bb1c",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC3eiMaoxj9x9T/\noMvEew/7omPK1BYmss2pbBFWi3q4RTJC1JUY2ae+YLZKByyqwdyMc8/GW2JOfzZc\nYb4B6eZd2/v4hToCFB6qNN1e0BAEZbhO2ZlJoZkOr4yse8cuQ/IohX7YxFrCOmdh\nQFSdTxpWEFmTZYUri2QBS7TrbiVObVzC3AoHEaPg0qdlVIRjlKkkS53BFgPdCl+9\nbG3RkDKex5JHpUGUc6g6MlrV4/JEB5vsu+SqLHFR37LcQ9M6zmWkJFimNFUx6OPx\n/QM3Zcvii+tPvwFzS/yhyNMIHCssbVbWis1xHRyV45kC/cxABMWl6zxQ746odCiX\nNAeGnTU3AgMBAAECggEAAcuLuGD/VGXpUZVZj2yQAkUOd0rBhLnPHujUKpkAWsX8\nJE2lv8tK9tpFx+ei3XDUHy1Efr5TvrND7UqpHz+rzCAXRDeZeUrus3k2iUAj4ZPu\nzIBLtz3oT3QSpKvGJ4LtEmefKOPGHfLDiPhy9RVz2ByE1wpXos6hH3O4+DmCdLsp\nucwMRjZIpTnFWbFnOc6yQdJWNJ759ZAnvooiII5d6WlgMvSG00Vqh6li8ZV3vFhu\nkfc83+HF7J23psdES0DO5YmQsPRwGqL6ikmcs44QeTyhpaueil1Mbx9PRxycZWKk\nYWATPURtgZq3HDCpjahvIJcCnqXwGjMWS1AOq+v9kQKBgQDvQsxW0zoico/sPK8i\ni2yCtbPA32c4dJtah8jBPxvhIBVf+AoX5DWniQze4BA+YO5vGCoYFUqsimo5f8g/\noi9znRJDsMxKycTcQ0QK526qANjABkZh3YJn6Mb+kRp7rcHYi0NcoFjzaCPy2qIg\nn5y0NlpS8cZQ0sVEzzvGadZBfQKBgQDEUD3TMtINhjWQu5bNUY9zXE9ZEM05tJCs\nniHkCEad65OwgIEPQkWWijZ9fT+czhxSriiKVn/zGLrWRMzBYeH1uPPOpHdtoYUj\nH+OxvOUTJhbyys1XFmkfVZ2ymYvdAs6PscjfRXsHTzKCBfQgjynzUpr4CnT1mFzE\nKRZIyU8PwwKBgQCQmKcTpg5ROTk/xSD22JoYmKVs39bq6JXm7X7nQzOfJ5ujZyz6\naWH/wTT1ESbf/Aa9PzZZXazGf9RYsaAczPCuh3O9UwD0BeKiV0is6lcYCPD1hBVP\nGeaw055HxPvjWQx4yRlJxmJboElxgK5Q1wWGZ/7Id/OpbufngPKYI+hnpQKBgAdw\nDteYnlK62f6wzMbcpzW0sqDqPQxJpg2UNC6CjcJf8YmHZNxiI2bPt77LZwSW3oXX\naVvMaS6ZqkKB+sv83GXF6x7SJmA19WheRe8u+/Lcx5PNUThSdgsk4EOrA5yNBax2\nDOlApaeiPYSlmxk8s1cvswVgyAuiCSm5cMmzLtanAoGBANjC/fbqaslQFn/ufsM/\nR+2toel1toDRObBN3EDP/1/3ct4tNpJYO1KtoZIS0n+ZVM4BXTzRWpb+xy4zb7vh\niXJCVGozL8fSpuSUoJQUUtQAx0Swia3+2Caqi55OVAj39AAmD5mkh9hYbmjXRFIt\nMF1xw6QkhGS64am7ZdQAMn9k\n-----END PRIVATE KEY-----\n",
+  "client_email": "[email protected]",
+  "client_id": "115250184755501138688",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/servicios-web%40translate-3e8bb.iam.gserviceaccount.com"
+}

src/PostsService/Data/DataContext.cs src/postsservice/Data/DataContext.cs

@@ -0,0 +1,37 @@
+language,locale
+ar,ar-XA
+bn,bn-IN
+cmn,cmn-CN
+cs,cs-CZ
+da,da-DK
+de,de-DE
+el,el-GR
+en,en-US
+es,es-ES
+fi,fi-FI
+fil,fil-PH
+fr,fr-FR
+gu,gu-IN
+hi,hi-IN
+hu,hu-HU
+id, id-ID
+it,it-IT
+ja,ja-JP
+kn,kn-IN
+ko,ko-KR
+ml,ml-IN
+nb,nb-NO
+nl,nl-NL
+pl,pl-PL
+pt,pt-PT
+ro,ro-RO
+ru,ru-RU
+sk,sk-SK
+sv,sv-SE
+ta,ta-IN
+te,te-IN
+th,th-TH
+tr,tr-TR
+uk,uk-UA
+vi,vi-VN
+yue,yue-HK

src/PostsService/Dockerfile src/postsservice/Dockerfile

@@ -0,0 +1,5 @@
+Flask==1.1.2
+google-cloud-texttospeech
+kafka-python==1.4.6
+waitress
+zeep==4.0.0

src/PostsService/Kafka/Consumers/LanguageDetectionConsumer.cs src/postsservice/Kafka/Consumers/LanguageDetectionConsumer.cs

@@ -0,0 +1,48 @@
+""" src package __init__ file
+"""
+
+import logging
+import os
+
+from flask import Flask
+
+from .util.logging_handler import KafkaLoggingHandler
+
+
+CONFIG = {
+    "base": "config.BaseConfig",
+    "development": "config.DevelopmentConfig",
+    "production": "config.ProductionConfig",
+    "testing": "config.TestingConfig"
+}
+
+def create_app():
+    """ Factory to create the flask application and load the config.
+    By default config.BaseConfig is used. In order to change this config
+    the FLASK_CONFIG environment variable must be changed to match one of
+    the keys in the CONFIG dict.
+    
+    Returns
+    -------
+    app: Flask.app object
+    """
+    app = Flask(__name__, instance_relative_config=True)
+    config_name = os.getenv('FLASK_CONFIG', 'base')
+    app.config.from_object(CONFIG[config_name])
+    _setup_logging(app.config['KAFKA_ENDPOINT'], app.config['KAFKA_LOGGING_TOPIC'], app.config['SERVICE_KEY'])
+    with app.app_context():
+        from .controller import tts
+    return app
+
+def _setup_logging(kafka_endpoint, topic, service_key):
+    """ Setup of logging to a kafka endpoint """
+    logger = logging.getLogger(service_key)
+    logger.setLevel(logging.DEBUG)
+
+    if len(logger.handlers) == 0:
+        logger.addHandler(logging.StreamHandler())
+
+    logger.debug(f"Starting kafka logging with endpoint '{kafka_endpoint}' and topic '{topic}'")
+    kh = KafkaLoggingHandler(kafka_endpoint, key="text_to_speech", topic=topic)
+    logger.addHandler(kh)
+    logger.debug("Logging system started")

src/PostsService/Kafka/Consumers/OffensiveTextDetectionConsumer.cs src/postsservice/Kafka/Consumers/OffensiveTextDetectionConsumer.cs

@@ -0,0 +1,2 @@
+from .gcloud_client import GCloudTTSClient
+from .soap_client import SoapClient

src/PostsService/Kafka/Consumers/TopicsConsumer.cs src/postsservice/Kafka/Consumers/TopicsConsumer.cs

@@ -0,0 +1,31 @@
+import logging
+
+from flask import current_app as app
+from google.cloud import texttospeech
+
+
+logger = logging.getLogger(app.config['SERVICE_KEY'])
+logger.setLevel(logging.DEBUG)
+
+
+class GCloudTTSClient():
+    """ Wrapper around the Google Cloud Text-To-Speech Service.
+    """
+
+    def __init__(self):
+        logger.debug("Initializing google cloud client")
+        self.client = texttospeech.TextToSpeechClient()
+        self.audio_config = texttospeech.AudioConfig(
+            audio_encoding=texttospeech.AudioEncoding.MP3
+        )
+    
+    def tts(self, text, locale):
+        """ Transforms text to audio spoken in the given locale.
+        """
+        logger.debug(f"Calling GCloud tts with params[text='{text}', locale={locale}]")
+        synthesis_input = texttospeech.SynthesisInput(text=text)
+        voice = texttospeech.VoiceSelectionParams(language_code=locale,
+            ssml_gender=texttospeech.SsmlVoiceGender.NEUTRAL)
+        response = self.client.synthesize_speech(input=synthesis_input, voice=voice,
+            audio_config=self.audio_config)
+        return response.audio_content

src/PostsService/Kafka/Deserializers/ArrayDeserializer.cs src/postsservice/Kafka/Deserializers/ArrayDeserializer.cs

@@ -0,0 +1,48 @@
+import logging
+import zeep
+
+from flask import current_app as app
+
+
+logger = logging.getLogger(app.config['SERVICE_KEY'])
+logger.setLevel(logging.DEBUG)
+
+
+class SoapResult:
+    def __init__(self, successful, result=None):
+        self.successful = successful
+        self.result = result
+
+
+class SoapClient:
+    """ Allows making request to a given SOAP endpoint.
+
+    This class uses the ZEEP Python library (https://docs.python-zeep.org/en/master/index.html#)
+    to allow users making requests to a SOAP endpoint.
+
+    When the zeep client is created, it parses the WSDL given as an argument and creates the
+    appropiate functions to call the service in the client class. This wrapper allows users
+    to access those functions with introspection.
+
+    Examples
+    --------
+    # this sample service provides a see_cars method that returns a list of cars
+    >>> client = SoapClient('http://localhost:8080/mysoapws/cars.wsdl')
+    >>> cars = client.see_cars() # cars will be a list of dicts (car 'objects')
+    """
+
+    def __init__(self, wsdl_endpoint):
+        settings = zeep.Settings(strict=False, xml_huge_tree=True)
+        self.client = zeep.Client(wsdl=wsdl_endpoint, settings=settings)
+    
+    def __getattribute__(self, attr_name, *args, **kwargs):
+        logger.debug(f"Calling method of SOAP Client: {attr_name}")
+        logger.debug(f"Args: {args} - Kwargs: {kwargs}")
+        try:
+            res = getattr(client.service, attr_name)(*args, **kwargs)
+            logger.info(f"SOAP request was successful. Result: {res}")
+            return SoapResult(True, res)
+        except Exception as e:
+            logger.error(f"There was an error calling the SOAP client: {e}")
+            logger.debug("Returning unsuccessful SOAP result")
+            return SoapResult(False)

src/PostsService/Kafka/Deserializers/BooleanDeserializer.cs src/postsservice/Kafka/Deserializers/BooleanDeserializer.cs

@@ -0,0 +1,84 @@
+""" Controllers of the text to speech service.
+"""
+
+import base64
+import logging
+
+from flask import current_app as app
+from flask import jsonify, request, send_file
+
+from .clients import GCloudTTSClient, SoapClient
+from .util.error import TooManyRequestsError
+from .util.locale_manager import LocaleManager
+
+
+logger = logging.getLogger(app.config['SERVICE_KEY'])
+logger.setLevel(logging.DEBUG)
+
+gcloud_tts_client = GCloudTTSClient()
+#statistics_client = SoapClient()
+locale_manager = LocaleManager(app.config['LOCALE_MAPPINGS_FILE'])
+
+
+@app.errorhandler(404)
+def page_not_found(e):
+    # override default html response
+    return jsonify({
+        'response': 'Page not found'
+    }), 404
+
+@app.route('/api/TextsToSpeechs', methods=['POST'])
+def tts():
+    """ Transforms the given text to an audio.
+
+    The returned audio is encoded as a base64 string.
+    """
+    logger.debug("POST to TextsToSpeechs received")
+    body_data = request.get_json()
+    logger.debug(f"Data: {body_data}")
+
+    if 'language' not in body_data or 'text' not in body_data:
+        logger.debug("Some of the required information is not present, returning bad request.")
+        return jsonify({
+            'response': 'Not all required parameters are present in body'
+        }), 400  # BadRequest
+    
+    try:
+        language = body_data['language']
+        logger.debug(f"Transforming language '{language}' to valid locale")
+        locale = language if locale_manager.is_locale(language) \
+                          else locale_manager.try_get_locale(language, 'en-US')
+        logger.debug(f"Locale after transformation: '{locale}'")
+
+        logger.debug("Calling Google Cloud client to convert text to speech")
+        audio = gcloud_tts_client.tts(body_data['text'], locale)
+        logger.debug("Audio file retrieved from client")
+        
+        # Ok, this took a while to figure out... GCloud returns the audio in a base64 encoded string,
+        # but the python library auto decodes it for us and we get the audio bytes as a result.
+        # In order to send it again we have to encode the bytes as base64 and decode it
+        # so it can be sent in json (no bytes allowed there)
+        # A user can then use it with base64.b64decode(audio_str)
+        logger.debug("Encoding audio file to base64 and decoding to string...")
+        audio_str = base64.b64encode(audio).decode()
+
+        # TODO: call statistics service to update number of tts of user
+        #statistics_client.update_
+
+        return jsonify({
+            'locale': locale,
+            'result': audio_str
+        }), 201  # Created
+    except TooManyRequestsError as e:
+        # Either we have sent too many requests to gcloud...
+        # ...or we run out of money! :(
+        logger.error(f"There was an error calling google cloud: {e}")
+        return jsonify({
+            'response': 'Too many requests'
+        }), 429
+    except Exception as e:
+        # Internal server error
+        logger.error(f"An error was raised: {e}")
+        return jsonify({
+            'response': 'Internal server error'
+        }), 500