Merge remote-tracking branch 'origin/developer_ca' into developer_ev

# Conflicts:
#	app/main/engine/enum/CodeResponseEnum.py
#	app/main/engine/service/Process.py
#	scripts/match-and-exclude-records-actions_v1.py

app/main/engine/database/Database.py

@@ -44,6 +44,15 @@ class Database:
         except Exception as e:
             self.app.logger.error(f"Error cerrando básica conexión. {e}")
 
+    def get_dialect(self) -> str:
+        dialect = ""
+        try:
+            dialect = self.factory.get_dialect()
+        except Exception as e:
+            self.app.logger.error(f"Error obteniendo dialect. {e}")
+        finally:
+            return dialect
+
     def create_engine(self) -> None:
         try:
             if isinstance(self.engine, type(None)):

app/main/engine/database/Mysql.py

@@ -23,6 +23,7 @@ class Mysql:
         self.params = params
         self.engine = None
         self.connection = None
+        self.dialect = None
 
     def create_spark_connection(self):
         params = {}
@@ -46,10 +47,17 @@ class Mysql:
         finally:
             return self.connection
 
-    def create_engine(self) -> None:
+    def get_dialect(self) -> str:
         try:
             dialect = DatabaseDialectEnum.MYSQL.value
-            url = f"{dialect}://{self.user}:{self.password}@{self.host}:{str(self.port)}/{self.database}?charset=utf8mb4"
+            self.dialect = f"{dialect}://{self.user}:{self.password}@{self.host}:{str(self.port)}/{self.database}?charset=utf8mb4"
+        except Exception as e:
+            self.app.logger.error(f"Error obteniendo dialect de Mysql. {e}")
+        return self.dialect
+
+    def create_engine(self) -> None:
+        try:
+            url = self.get_dialect()
             self.engine = create_engine(url, pool_recycle=3600, pool_pre_ping=True, **self.params)
         except Exception as e:
             self.app.logger.error(f"Error creando engine de Mysql. {e}")

app/main/engine/enum/CodeResponseEnum.py

@@ -11,4 +11,4 @@ class CodeResponseEnum(Enum):
     OUTPUT_ERROR = 606
     EMPTY_DATASET = 607
     ERROR = 609
-    TIMEOUT_ERROR = 800
+    TIMEOUT = 610

app/main/engine/enum/StatusEnum.py

@@ -4,3 +4,4 @@ from enum import Enum
 class StatusEnum(Enum):
     OK = 200
     ERROR = 609
+    TIMEOUT = 610

app/main/engine/service/Process.py

@@ -2,14 +2,15 @@ from typing import Dict, Any
 import time
 import traceback as traceback_lib
 import importlib
-from threading import Timer
+
 from config import Config as cfg
 from app.main.engine.util.Timezone import Timezone
 from app.main.engine.util.Utils import Utils
 from app.main.engine.enum.StatusEnum import StatusEnum
 from app.main.engine.enum.CodeResponseEnum import CodeResponseEnum
 from app.main.engine.database.Database import Database
-from wrapt_timeout_decorator import *
+
+
 class Process:
     def __init__(self, app, descriptor: Dict[str, Any]) -> None:
         self.app = app
@@ -21,7 +22,6 @@ class Process:
     def run(self) -> Dict[str, Any]:
         status, status_description = StatusEnum.OK, ""
         try:
-
             # Obteniendo la conexión a la BD
             db_params = cfg.db_params
             source = Database(self.app, db_params)
@@ -44,39 +44,22 @@ class Process:
             obj_script = globals()[relation](self.app)
 
             obj_script.parser(self.descriptor)
-            tiempo_limite = obj_script.timeout
-            if tiempo_limite is not None:
-                @timeout(tiempo_limite)
-                def procesamiento():
-                    try:
-                        self.app.logger.info(f"Iniciando procesamiento de script")
-                        obj_script.process(source)
 
-                        # Guardando resultado
-                        self.app.logger.info(f"Generado y guardando resultado")
-                        response = obj_script.response()
-                        # response.show()
-                        result = self.utils.create_result(response, self.descriptor)
-                        save = self.utils.save_result(result, self.descriptor, db_session)
-                        if save["status"] == StatusEnum.ERROR.name:
-                            raise InterruptedError(save["message"])
+            # Iniciando process
+            self.app.logger.info(f"Iniciando procesamiento de script")
+            obj_script.process(source)
 
-                    except Exception as e:
-                        raise TimeoutError(f"Tiempo límite de ejecución superado{e}")
-                procesamiento()
-
-            else:
-                # Iniciando process
-                self.app.logger.info(f"Iniciando procesamiento de script")
-                obj_script.process(source)
-                # Guardando resultado
-                self.app.logger.info(f"Generado y guardando resultado")
-                response = obj_script.response()
-                # response.show()
-                result = self.utils.create_result(response, self.descriptor)
-                save = self.utils.save_result(result, self.descriptor, db_session)
-                if save["status"] == StatusEnum.ERROR.name:
-                    raise InterruptedError(save["message"])
+            # Guardando resultado
+            self.app.logger.info(f"Generado y guardando resultado")
+            response = obj_script.response()
+            # response.show()
+            result = self.utils.create_result(response, self.descriptor)
+            save = self.utils.save_result(result, self.descriptor, db_session)
+            if save["status"] == StatusEnum.ERROR.name:
+                raise InterruptedError(save["message"])
+        except TimeoutError as e:
+            self.app.logger.error(f"Error de Timeout. Error: {e}")
+            status, status_description = CodeResponseEnum.TIMEOUT, str(e)
         except IndexError as e:
             self.app.logger.error(f"Error extrayendo insumos. Vacío. Error: {e}")
             status, status_description = CodeResponseEnum.EMPTY_DATASET, str(e)
@@ -95,9 +78,6 @@ class Process:
         except ReferenceError as e:
             self.app.logger.error(f"Error validando parámetros del descriptor. {e}")
             status, status_description = CodeResponseEnum.PARAMETERS_ERROR, str(e)
-        except TimeoutError as e:
-            self.app.logger.error(f"Error validando parámetros del descriptor. {e}")
-            status, status_description = CodeResponseEnum.TIMEOUT_ERROR, str(e)
         except Exception as e:
             traceback_lib.print_exc()
             self.app.logger.error(f"Error procesando engine. {e}")

app/main/engine/util/Utils.py

@@ -5,6 +5,8 @@ import shutil
 from enum import Enum
 # from pyspark.sql import SparkSession
 import json
+
+from app.main.engine.enum.CodeResponseEnum import CodeResponseEnum
 from app.main.engine.util.Timezone import Timezone
 
 # from config import Config as cfg
@@ -52,8 +54,11 @@ class Utils:
         if codeEnum.value == StatusEnum.OK.value:
             response.update({'status': StatusEnum.OK.name, 'detail': detail})
         else:
+            error = StatusEnum.ERROR.name
+            if codeEnum.value == CodeResponseEnum.TIMEOUT.value:
+                error = StatusEnum.TIMEOUT.name
             description = DescResponseEnum[codeEnum.name].value
-            response.update({'status': StatusEnum.ERROR.name, 'message': description,
+            response.update({'status': error, 'message': description,
                              'detail': detail})
         return response
 
@@ -65,6 +70,14 @@ class Utils:
             pivot_params = descriptor["params-input"]["pivot-config"]
             ctp_params = descriptor["params-input"]["counterpart-config"]
 
+            for key_p, key_c in zip(pivot_params.keys(), ctp_params.keys()):
+                if isinstance(pivot_params[key_p], str):
+                    pivot_params[key_p] = "PIVOT_" + pivot_params[key_p]
+                    ctp_params[key_c] = "COUNTERPART_" + ctp_params[key_c]
+                else:
+                    pivot_params[key_p] = ["PIVOT_" + column for column in pivot_params[key_p]]
+                    ctp_params[key_c] = ["COUNTERPART_" + column for column in ctp_params[key_c]]
+
             group_pivot_match = pivot_params["columns-group"]
             transaction_pivot_match = pivot_params["columns-transaction"]
 
@@ -73,7 +86,7 @@ class Utils:
 
             used_list = transaction_counterpart_match if exclude_pivot else transaction_pivot_match
 
-            if data.empty:
+            if data is None or data.empty:
                 self.app.logger.info(f"El dataframe resultado esta vacio")
             else:
                 for idx, i in data.iterrows():

conf.yml

@@ -23,16 +23,16 @@ app:
   timezone: 'GMT-5'
   time_pattern: '%Y-%m-%d %H:%M:%S'
   logging: 'INFO'
-  max_engine_threads: 2  # threads (maximum)
+  max_engine_threads: 50  # threads (maximum)
 
   # Make the service in a production state
   # Manage connections to the REST Service published. Allow workers to receive the connections.
   # https://docs.gunicorn.org/en/stable/
   gunicorn:
-    bind: '0.0.0.0:7500'
+    bind: '0.0.0.0:8000'
     worker_class: 'gthread'
-    threads: 8
-    worker_connections: 50
+    threads: 51
+    worker_connections: 51
     loglevel: 'debug'
     accesslog: '-'
     capture_output: True
\ No newline at end of file