Commit 26bf8a24 authored by Cristian Aguirre's avatar Cristian Aguirre

Update 03-08-23. Update README

parent e5ddb0fc
File added
...@@ -51,7 +51,6 @@ NOTA: *Detalle de cada variable de entorno usada por los POD y Airflow:* ...@@ -51,7 +51,6 @@ NOTA: *Detalle de cada variable de entorno usada por los POD y Airflow:*
AIRFLOW__LOGGING__LOGGING_LEVEL: INFO # Nivel de log de Airflow (webserver, scheduler y workers) AIRFLOW__LOGGING__LOGGING_LEVEL: INFO # Nivel de log de Airflow (webserver, scheduler y workers)
AIRFLOW__WEBSERVER__DEFAULT_UI_TIMEZONE: America/Lima # Timezone de la Web de Airflow AIRFLOW__WEBSERVER__DEFAULT_UI_TIMEZONE: America/Lima # Timezone de la Web de Airflow
AIRFLOW__CORE__DEFAULT_TIMEZONE: America/Lima # Timezone del Scheduler de Airflow AIRFLOW__CORE__DEFAULT_TIMEZONE: America/Lima # Timezone del Scheduler de Airflow
AIRFLOW__KUBERNETES__KUBE_CLIENT_REQUEST_ARGS: '{"_request_timeout": [60,60]}' # Tiempo de espera de respuesta de Kubernetes API
AIRFLOW__KUBERNETES__WORKER_CONTAINER_REPOSITORY: cristianfernando/airflow_custom # Ruta de imagen a usar para crear el worker POD AIRFLOW__KUBERNETES__WORKER_CONTAINER_REPOSITORY: cristianfernando/airflow_custom # Ruta de imagen a usar para crear el worker POD
AIRFLOW__KUBERNETES__WORKER_CONTAINER_TAG: "0.0.1" # Tag a usar para crear el worker POD AIRFLOW__KUBERNETES__WORKER_CONTAINER_TAG: "0.0.1" # Tag a usar para crear el worker POD
AIRFLOW__KUBERNETES__LOGS_VOLUME_CLAIM: airflow-logs-pvc # Nombre del volumen usado para almacenar los logs AIRFLOW__KUBERNETES__LOGS_VOLUME_CLAIM: airflow-logs-pvc # Nombre del volumen usado para almacenar los logs
......
...@@ -33,7 +33,7 @@ class Mysql: ...@@ -33,7 +33,7 @@ class Mysql:
self.connection = pymysql.connect(host=self.host, port=self.port, user=self.user, self.connection = pymysql.connect(host=self.host, port=self.port, user=self.user,
password=self.password, db=self.database) password=self.password, db=self.database)
except Exception as e: except Exception as e:
logger.error(f"Error obteniendo conexion básica de Oracle. {e}") logger.error(f"Error obteniendo conexion básica de Mysql. {e}")
finally: finally:
return self.connection return self.connection
......
...@@ -34,7 +34,7 @@ class Postgres: ...@@ -34,7 +34,7 @@ class Postgres:
password=self.password, database=self.database, password=self.password, database=self.database,
options="-c search_path="+self.schema) options="-c search_path="+self.schema)
except Exception as e: except Exception as e:
logger.error(f"Error obteniendo conexion básica de Oracle. {e}") logger.error(f"Error obteniendo conexion básica de Postgres. {e}")
finally: finally:
return self.connection return self.connection
......
...@@ -107,7 +107,7 @@ def extract_from_source(command, source_conn, intern_conn, chunksize: int, **kwa ...@@ -107,7 +107,7 @@ def extract_from_source(command, source_conn, intern_conn, chunksize: int, **kwa
indexes = [] indexes = []
if "indexes" in procedure.keys(): if "indexes" in procedure.keys():
indexes = procedure["indexes"] indexes = procedure["indexes"]
model = generateModel(tablename, procedure["fields"], indexes) model = generateModel(tablename, procedure["fields"], indexes, intern_conn.db_type)
columns_name = [field["name"] for field in procedure["fields"]] columns_name = [field["name"] for field in procedure["fields"]]
if isinstance(model, type(None)): if isinstance(model, type(None)):
raise AssertionError(f"Definición del extracción para {tablename} en el json-descriptor no encontraddo") raise AssertionError(f"Definición del extracción para {tablename} en el json-descriptor no encontraddo")
......
...@@ -9,7 +9,8 @@ from enums.CatalogConfigurationEnum import CatalogConfigurationEnum ...@@ -9,7 +9,8 @@ from enums.CatalogConfigurationEnum import CatalogConfigurationEnum
from enums.FileTypeEnum import FileTypeEnum from enums.FileTypeEnum import FileTypeEnum
from enums.DataTypeEnum import DataTypeEnum from enums.DataTypeEnum import DataTypeEnum
from enums.DataTypeOrmEnum import DataTypeOrmEnum from enums.DataTypeOrmEnum import DataTypeOrmEnum
from components.Model.InsumoModel import InsumoModel from enums.DatabaseTypeEnum import DatabaseTypeEnum
from components.Model.InsumoModel import InsumoModel, InsumoModelOracle
from enums.CommentsScriptEnum import CommentsScriptEnum from enums.CommentsScriptEnum import CommentsScriptEnum
from components.Timezone import datetime_by_tzone from components.Timezone import datetime_by_tzone
...@@ -148,14 +149,14 @@ def select_multiple(command: str) -> Dict[str, Any]: ...@@ -148,14 +149,14 @@ def select_multiple(command: str) -> Dict[str, Any]:
if command.lower().replace(" ", "").find(no_procedure_init) != -1: if command.lower().replace(" ", "").find(no_procedure_init) != -1:
response["is_multiple"] = True response["is_multiple"] = True
tablename = command[:command.index("|")].strip() tablename = command[:command.index("|")].strip()
init_index = command.lower().find("from") # init_index = command.lower().find("from")
if init_index == -1: # if init_index == -1:
raise AssertionError("Query malformed") # raise AssertionError("Query malformed")
else: # else:
from_command = command[init_index + 4:] # from_command = command[init_index + 4:]
tablename_base = from_command.strip().split(" ") # tablename_base = from_command.strip().split(" ")
if len(tablename_base) > 0 and tablename == "": # if len(tablename_base) > 0 and tablename == "":
tablename = tablename_base[0] # tablename = tablename_base[0]
response["tablename"] = tablename response["tablename"] = tablename
except Exception as e: except Exception as e:
raise AssertionError(f"Error validando si es múltiple select y nombre de tabla. {e}") raise AssertionError(f"Error validando si es múltiple select y nombre de tabla. {e}")
...@@ -198,12 +199,17 @@ def delete_temp_dir(module_name: str) -> bool: ...@@ -198,12 +199,17 @@ def delete_temp_dir(module_name: str) -> bool:
return drop return drop
def generateModel(tablename: str, attributes: List[Dict[str, Any]], indexes: List[str], modelName: str = "TableModel"): def generateModel(tablename: str, attributes: List[Dict[str, Any]], indexes: List[str], db_target: str,
modelName: str = "TableModel"):
default_precision = 8 default_precision = 8
model = type(modelName, (InsumoModel,), { model_args = {
'__tablename__': tablename, '__tablename__': tablename,
'__table_args__': {'extend_existing': True} '__table_args__': {'extend_existing': True}
}) }
if db_target == DatabaseTypeEnum.ORACLE.value:
model = type(modelName, (InsumoModelOracle,), model_args)
else:
model = type(modelName, (InsumoModel,), model_args)
try: try:
for attribute in attributes: for attribute in attributes:
index = False index = False
......
...@@ -93,7 +93,7 @@ data: ...@@ -93,7 +93,7 @@ data:
_AIRFLOW_WWW_USER_USERNAME: admin _AIRFLOW_WWW_USER_USERNAME: admin
_AIRFLOW_WWW_USER_PASSWORD: admin _AIRFLOW_WWW_USER_PASSWORD: admin
S3_DAGS_DIR: 's3://prueba1234568/dags' S3_DAGS_DIR: 's3://prueba1234568/dags'
GCS_DAGS_DIR: 'gs://prueba-rsync3/carpeta' GCS_DAGS_DIR: 'gs://prueba-rsync2/carpeta'
SYNCHRONYZE_DAG_DIR: '30' SYNCHRONYZE_DAG_DIR: '30'
MINIO_SERVER: 'http://192.168.49.2:9000' MINIO_SERVER: 'http://192.168.49.2:9000'
MINIO_DAGS_DIR: '/prueba-ca/dags' MINIO_DAGS_DIR: '/prueba-ca/dags'
\ No newline at end of file
...@@ -10,7 +10,7 @@ spec: ...@@ -10,7 +10,7 @@ spec:
- ReadWriteMany - ReadWriteMany
storageClassName: airflow-dags storageClassName: airflow-dags
nfs: nfs:
server: 10.115.7.82 server: 10.216.137.186
path: "/volume1/nfs_share" path: "/volume1/nfs_share"
--- ---
...@@ -27,7 +27,7 @@ spec: ...@@ -27,7 +27,7 @@ spec:
- ReadWriteMany - ReadWriteMany
storageClassName: airflow-postgres storageClassName: airflow-postgres
nfs: nfs:
server: 10.115.7.82 server: 10.216.137.186
path: "/volume1/nfs_postgres" path: "/volume1/nfs_postgres"
--- ---
...@@ -44,7 +44,7 @@ spec: ...@@ -44,7 +44,7 @@ spec:
- ReadWriteMany - ReadWriteMany
storageClassName: airflow-logs storageClassName: airflow-logs
nfs: nfs:
server: 10.115.7.82 server: 10.216.137.186
path: "/volume1/nfs_logs" path: "/volume1/nfs_logs"
--- ---
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment