Skip to content

B
bcom-tp-etl-transformation-pipelines
Commit ad679d5e authored by Erly Villaroel's avatar Erly Villaroel
Browse files
Options

Cambios en Oracle.py

parent 8ebc05c1
Hide whitespace changes
Inline Side-by-side
Showing with 44 additions and 150 deletions
dags/components/DatabaseOperation/DatabaseTransformation.py
View file @ ad679d5e
...@@ -18,7 +18,7 @@ def execute_transformations(commands: List[str], engine): ...@@ -18,7 +18,7 @@ def execute_transformations(commands: List[str], engine):
def delete_table(tablename: str, engine) -> bool: def delete_table(tablename: str, engine) -> bool:
delete = False delete = False
try: try:
command = f'DROP TABLE IF EXISTS {tablename}' command = f'DROP TABLE {tablename}'
start_time = time.time() start_time = time.time()
with engine.connect() as conn: with engine.connect() as conn:
try: try:
...@@ -36,7 +36,7 @@ def delete_table(tablename: str, engine) -> bool: ...@@ -36,7 +36,7 @@ def delete_table(tablename: str, engine) -> bool:
def delete_procedure(procedure: str, engine) -> bool: def delete_procedure(procedure: str, engine) -> bool:
delete = False delete = False
try: try:
command = f"DROP PROCEDURE IF EXISTS {procedure}" command = f"DROP PROCEDURE {procedure}"
start_time = time.time() start_time = time.time()
with engine.connect() as conn: with engine.connect() as conn:
try: try:
......
dags/components/Databases/Oracle.py
View file @ ad679d5e
...@@ -91,7 +91,7 @@ class Oracle: ...@@ -91,7 +91,7 @@ class Oracle:
response = "" response = ""
try: try:
command = command.replace(reserved_word, "").replace(";", "") command = command.replace(reserved_word, "").replace(";", "")
response = f"begin {command}; end;" response = f"BEGIN {command}; END;"
logger.debug("COMANDO ORACLE:", response) logger.debug("COMANDO ORACLE:", response)
except Exception as e: except Exception as e:
logger.error(f"Error generando comando sql para procedure Oracle. Comando: {command}. {e}") logger.error(f"Error generando comando sql para procedure Oracle. Comando: {command}. {e}")
...@@ -113,7 +113,7 @@ class Oracle: ...@@ -113,7 +113,7 @@ class Oracle:
def check_procedure(self, procedure_name, connection) -> bool: def check_procedure(self, procedure_name, connection) -> bool:
exists = False exists = False
try: try:
check_query = f"SELECT text FROM all_source WHERE name = '{procedure_name}'" check_query = f"SELECT object_name FROM all_procedures WHERE object_name = '{procedure_name}'"
result = connection.execute(check_query) result = connection.execute(check_query)
exists = bool(result.fetchone()) exists = bool(result.fetchone())
except Exception as e: except Exception as e:
...@@ -124,7 +124,7 @@ class Oracle: ...@@ -124,7 +124,7 @@ class Oracle:
def check_table(self, table_name, connection) -> bool: def check_table(self, table_name, connection) -> bool:
exists = False exists = False
try: try:
check_query = f"DESCRIBE {table_name}" check_query = f"SELECT table_name FROM all_tables WHERE table_name = '{table_name}'"
result = connection.execute(check_query) result = connection.execute(check_query)
exists = bool(result.fetchone()) exists = bool(result.fetchone())
except Exception as e: except Exception as e:
......
dags/dag_conf.yml
View file @ ad679d5e
...@@ -5,23 +5,23 @@ app: ...@@ -5,23 +5,23 @@ app:
database: database:
sources: sources:
source1: source1:
type: mysql type: oracle
host: 192.168.1.13 host: 192.168.27.22
port: 13306 port: 21521
username: root username: PRUEBABCOM2
password: root password: admin
database: prueba database:
service: service: ORCLPDB1
schema: sources schema:
transformation: transformation:
type: mysql type: oracle
host: 192.168.1.13 host: 192.168.27.22
port: 13306 port: 21521
username: root username: RLQA_AIR
password: root password: RLQA_AIR99
database: prueba_ca database:
service: service: ORCLPDB1
schema: intern_db schema:
chunksize: 4000 chunksize: 4000
label_multiple_select: TABLENAME label_multiple_select: TABLENAME
label_transform_procedure: STORE label_transform_procedure: STORE
...@@ -55,14 +55,14 @@ app: ...@@ -55,14 +55,14 @@ app:
delimiter: '|' delimiter: '|'
tmp_path: /tmp tmp_path: /tmp
s3_params: s3_params:
tabla4: tabla1:
bucket: pruebairflow bucket: prueba-id
prefix: bcom_results prefix: bcom_results
connection_id: prueba_af connection_id: conn_script
tabla5: tabla5:
bucket: pruebairflow bucket: prueba-id
prefix: bcom_results prefix: bcom_results
connection_id: prueba_af connection_id: conn_script
report: report:
s3_params: s3_params:
bucket: prueba-id bucket: prueba-id
......
dags/procedure_prueba.json
View file @ ad679d5e
[ [
{ {
"identifier": "ModificarTabla4", "identifier": "SP_ACTUALIZAR_TABLA_5",
"transformation_store_procedure": true "transformation_store_procedure": true
}, },
{ {
"identifier" : "UnionYInsert", "identifier": "tabla1",
"transformation_store_procedure": true
},
{
"identifier": "TempTabla",
"temp_table" : true
},
{
"identifier": "tabla11",
"fields": [ "fields": [
{ {
"name": "id", "name": "CD_FOLIO",
"datatype": "NUMBER",
"decimal_precision": 0
},
{
"name": "nombre",
"datatype": "TEXT", "datatype": "TEXT",
"maxLength": 50 "maxLength": 100
}, },
{ {
"name": "fecha_nacimiento", "name": "CD_CUENTA",
"datatype": "DATE"
}
],
"indexes": [
{
"name" : "indice1",
"index_fields": [
"id"
]
}
],
"save_output" : false
},
{
"identifier": "tabla2",
"fields": [
{
"name": "id",
"datatype": "NUMBER",
"decimal_precision": 0
},
{
"name": "apellido",
"datatype": "TEXT", "datatype": "TEXT",
"maxLength": 50 "maxLength": 100
},
{
"name": "fecha_registro",
"datatype": "DATETIME"
}
],
"indexes": [
{
"name" : "indice1",
"index_fields": [
"id"
]
}
],
"save_output" : false
},
{
"identifier": "tabla3",
"fields": [
{
"name": "id",
"datatype": "NUMBER",
"decimal_precision": 0
},
{
"name": "edad",
"datatype": "NUMBER",
"decimal_precision": 0
}, },
{ {
"name": "fecha_actualizacion", "name": "CD_PAQUETE",
"datatype": "DATETIME"
}
],
"indexes": [
{
"name" : "indice1",
"index_fields": [
"id"
]
}
],
"save_output" : false
},
{
"identifier": "tabla4",
"fields": [
{
"name": "id",
"datatype": "NUMBER",
"decimal_precision": 0
},
{
"name": "nombre",
"datatype": "TEXT", "datatype": "TEXT",
"maxLength": 50 "maxLength": 100
}, },
{ {
"name": "fecha", "name": "NB_PAQUETE",
"datatype": "DATE" "datatype": "TEXT",
}, "maxLength": 200
{
"name": "fecha_2",
"datatype": "DATE"
},
{
"name": "fecha_3",
"datatype": "DATETIME"
} }
], ],
"indexes": [ "indexes": [
{ {
"name" : "indice1", "name" : "indice1",
"index_fields": [ "index_fields": [
"id" "CD_FOLIO"
] ]
} }
], ],
...@@ -135,36 +41,24 @@ ...@@ -135,36 +41,24 @@
"identifier": "tabla5", "identifier": "tabla5",
"fields": [ "fields": [
{ {
"name": "id", "name": "NOMBRE_PRODUCTO",
"datatype": "NUMBER", "datatype": "TEXT",
"decimal_precision": 0 "maxLength": 100
}, },
{ {
"name": "nombre", "name": "CD_PAQUETE",
"datatype": "TEXT", "datatype": "TEXT",
"maxLength": 50 "maxLength": 50
},
{
"name": "fecha",
"datatype": "DATE"
},
{
"name": "fecha_2",
"datatype": "DATETIME"
},
{
"name": "fecha_3",
"datatype": "DATETIME"
} }
], ],
"indexes": [ "indexes": [
{ {
"name" : "indice1", "name" : "indice1",
"index_fields": [ "index_fields": [
"id" "NOMBRE_PRODUCTO"
] ]
} }
], ],
"save_output" : true "save_output" : false
} }
] ]
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment