Skip to content

B
bcom-tp-etl-transformation-pipelines
Commit c4edffad authored by Cristian Aguirre's avatar Cristian Aguirre
Browse files
Options

Fix bugs

parent 3170e104
Show whitespace changes
Inline Side-by-side
Showing with 441 additions and 78 deletions
dags/components/Extractor.py
View file @ c4edffad
......@@ -163,13 +163,13 @@ def extract_from_source(command, source_conn, intern_conn, chunksize: int, timez
data.append(row)
if len(data) == chunksize:
dataframe = pd.DataFrame(data, columns=columns_name)
save = save_from_dataframe(dataframe, tablename, intern_conn.engine)
save = save_from_dataframe(dataframe, tablename, intern_conn)
if save:
logger.debug(f"Guardado correctamente dataframe. Procesando más bloques")
data.clear()
if len(data) > 0:
dataframe = pd.DataFrame(data, columns=columns_name)
save = save_from_dataframe(dataframe, tablename, intern_conn.engine)
save = save_from_dataframe(dataframe, tablename, intern_conn)
if save:
logger.debug(f"Migrado correctamente todos los datos")
extract_tables.append(tablename)
......@@ -183,13 +183,13 @@ def extract_from_source(command, source_conn, intern_conn, chunksize: int, timez
data.append(row)
if len(data) == chunksize:
dataframe = pd.DataFrame(data, columns=columns_name)
save = save_from_dataframe(dataframe, tablename, intern_conn.engine)
save = save_from_dataframe(dataframe, tablename, intern_conn)
if save:
logger.debug(f"Guardado correctamente dataframe. Procesando más bloques")
data.clear()
if len(data) > 0:
dataframe = pd.DataFrame(data, columns=columns_name)
save = save_from_dataframe(dataframe, tablename, intern_conn.engine)
save = save_from_dataframe(dataframe, tablename, intern_conn)
if save:
logger.debug(f"Migrado correctamente todos los datos")
......
dags/procedure_prueba.json
View file @ c4edffad
[
{
"identifier": "ModificarTabla4",
"transformation_store_procedure": true
"transformation_store_procedure" : true
},
{
"identifier" : "UnionYInsert",
"transformation_store_procedure": true
"identifier": "SP_ACTUALIZAR_TABLA_5",
"transformation_store_procedure" : true
},
{
"identifier": "TempTabla",
"identifier": "REGLA7_1",
"temp_table" : true
},
{
"identifier": "tabla11",
"identifier": "REGLA7_2",
"temp_table" : true
},
{
"identifier": "RANGO_VENTAS_CON_PROMOCION",
"fields": [
{
"name": "id",
"name": "CD_PAQUETE",
"datatype": "TEXT",
"maxLength": 100
},
{
"name": "NU_ADDON",
"datatype": "TEXT",
"maxLength": 15
},
{
"name": "CD_CLIENTE",
"datatype": "TEXT",
"maxLength": 50
}
]
},
{
"identifier": "TACOMVENTAS",
"fields": [
{
"name": "CD_EMPRESA",
"datatype": "NUMBER",
"decimal_precision": 0
},
{
"name": "nombre",
"name": "CD_FOLIO",
"datatype": "TEXT",
"maxLength": 100
},
{
"name": "CD_CUENTA",
"datatype": "TEXT",
"maxLength": 100
},
{
"name": "NU_VENDEDOR",
"datatype": "TEXT",
"maxLength": 100
},
{
"name": "CD_PAQUETE",
"datatype": "TEXT",
"maxLength": 100
},
{
"name": "NU_ADDON",
"datatype": "TEXT",
"maxLength": 5
},
{
"name": "NB_PAQUETE",
"datatype": "TEXT",
"maxLength": 200
},
{
"name": "CD_CLIENTE",
"datatype": "TEXT",
"maxLength": 50
},
{
"name": "fecha_nacimiento",
"name": "NB_CLIENTE",
"datatype": "TEXT",
"maxLength": 200
},
{
"name": "FH_CONTRATACION",
"datatype": "DATE"
},
{
"name": "FH_ACTIVACION",
"datatype": "DATETIME"
},
{
"name": "FH_OPERACION",
"datatype": "DATETIME"
},
{
"name": "TP_SERVICIO",
"datatype": "NUMBER",
"decimal_precision": 0
},
{
"name": "ST_CLIENTE",
"datatype": "NUMBER",
"decimal_precision": 0
},
{
"name": "TP_PAGO",
"datatype": "TEXT",
"maxLength": 10
},
{
"name": "NB_USUACARGA",
"datatype": "TEXT",
"maxLength": 50
},
{
"name": "FH_CARGA",
"datatype": "DATETIME"
},
{
"name": "NU_ANIO",
"datatype": "NUMBER",
"decimal_precision": 0
},
{
"name": "NU_MES",
"datatype": "NUMBER",
"decimal_precision": 0
},
{
"name": "NU_SEMANA",
"datatype": "NUMBER",
"decimal_precision": 0
},
{
"name": "NU_COMISION",
"datatype": "NUMBER",
"decimal_precision": 0
},
{
"name": "TP_PAGOANT",
"datatype": "NUMBER",
"decimal_precision": 0
},
{
"name": "REGLA_APLICADA",
"datatype": "NUMBER",
"decimal_precision": 0
},
{
"name": "AUX",
"datatype": "TEXT",
"maxLength": 50
},
{
"name": "PROMOCION",
"datatype": "TEXT",
"maxLength": 80
},
{
"name": "EMPLEADOEMBAJADOR__C",
"datatype": "TEXT",
"maxLength": 100
},
{
"name": "CANAL_DE_VENTA__C",
"datatype": "TEXT",
"maxLength": 50
},
{
"name": "CANALREFERIDO__C",
"datatype": "TEXT",
"maxLength": 50
}
],
"indexes": [
{
"name" : "indice1",
"name": "indice1",
"index_fields": [
"id"
"CD_FOLIO","CD_PAQUETE"
]
},
{
"name": "indice2",
"index_fields": [
"CD_PAQUETE"
]
},
{
"name": "indice3",
"index_fields": [
"CD_CLIENTE","PROMOCION","NB_PAQUETE"
]
},
{
"name": "indice4",
"index_fields": [
"NU_ADDON"
]
}
],
"save_output" : false
"save_output": true
},
{
"identifier": "tabla2",
"identifier": "PROMOCIONES_RESIDENCIAL",
"fields": [
{
"name": "id",
"name": "EMPRESA",
"datatype": "TEXT",
"maxLength": 250
},
{
"name": "CUENTA",
"datatype": "TEXT",
"maxLength": 250
},
{
"name": "PLAN",
"datatype": "TEXT",
"maxLength": 250
},
{
"name": "NOMBRE_PRODUCTO",
"datatype": "TEXT",
"maxLength": 250
},
{
"name": "DESCR",
"datatype": "TEXT",
"maxLength": 250
},
{
"name": "TIPO_ADICION",
"datatype": "TEXT",
"maxLength": 250
},
{
"name": "RENTACONIMPUESTOS",
"datatype": "NUMBER"
},
{
"name": "RENTASINIMPUESTOS",
"datatype": "NUMBER"
},
{
"name": "QUANTITY",
"datatype": "NUMBER",
"decimal_precision": 0
},
{
"name": "apellido",
"name": "CREACION_PRODUCTO",
"datatype": "DATE"
},
{
"name": "INICIO_COBRO",
"datatype": "DATE"
},
{
"name": "FIN_COBRO",
"datatype": "DATE"
},
{
"name": "FIN_COMPRA",
"datatype": "DATE"
},
{
"name": "SERV_STATUS",
"datatype": "TEXT",
"maxLength": 20
},
{
"name": "POID_TYPE",
"datatype": "TEXT",
"maxLength": 200
},
{
"name": "POID_PRODUCT",
"datatype": "TEXT",
"maxLength": 50
},
{
"name": "fecha_registro",
"datatype": "DATETIME"
"name": "STATUS_PRODUCTO",
"datatype": "TEXT",
"decimal_precision": null,
"maxLength": 200
},
{
"name": "SERVICIO",
"datatype": "TEXT",
"maxLength": 200
},
{
"name": "CD_PAQUETE",
"datatype": "TEXT",
"maxLength": 100
}
],
"indexes": [
{
"name" : "indice1",
"index_fields": [
"id"
"name": "indice1",
"index_fields" : [
"CD_PAQUETE"
]
}
],
"save_output" : false
]
},
{
"identifier": "tabla3",
"identifier": "TABLA_PROMOCIONES_2",
"fields": [
{
"name": "id",
"datatype": "NUMBER",
"decimal_precision": 0
"name": "PLAN",
"datatype": "TEXT",
"maxLength": 250
},
{
"name": "edad",
"datatype": "NUMBER",
"decimal_precision": 0
"name": "NOMBRE_PRODUCTO",
"datatype": "TEXT",
"maxLength": 250
},
{
"name": "fecha_actualizacion",
"datatype": "DATETIME"
"name": "SUMA_RENTA",
"datatype": "NUMBER"
}
],
"indexes": [
{
"name" : "indice1",
"index_fields": [
"id"
"name": "indice1",
"index_fields" : [
"PLAN"
]
}
],
"save_output" : false
]
},
{
"identifier": "tabla4",
"identifier": "CATALOGO_PROMOCIONES",
"fields": [
{
"name": "id",
"datatype": "NUMBER",
"decimal_precision": 0
"name": "NOMBRE_PRODUCTO",
"datatype": "TEXT",
"maxLength": 100
},
{
"name": "nombre",
"name": "CD_PAQUETE",
"datatype": "TEXT",
"maxLength": 50
}
]
},
{
"name": "fecha",
"datatype": "DATE"
"identifier": "TEMP_PROMO",
"fields": [
{
"name": "TRESP",
"datatype": "TEXT",
"maxLength": 50
},
{
"name": "fecha_2",
"datatype": "DATE"
"name": "DOSP",
"datatype": "TEXT",
"maxLength": 50
}
]
},
{
"name": "fecha_3",
"datatype": "DATETIME"
"identifier": "RELACION_POID_PAQ",
"fields": [
{
"name": "POID_PRODUCT",
"datatype": "TEXT",
"maxLength": 50
},
{
"name": "CD_PAQUETE",
"datatype": "TEXT",
"maxLength": 50
}
],
"indexes": [
]
},
{
"name" : "indice1",
"index_fields": [
"id"
"identifier": "RELACION_PAQINI_PAQFIN",
"fields": [
{
"name": "COD_PAQ_INI",
"datatype": "TEXT",
"maxLength": 50
},
{
"name": "COD_PAQ_FIN",
"datatype": "TEXT",
"maxLength": 50
}
]
},
{
"identifier": "ADDONS_UNICO",
"fields": [
{
"name": "COD_PAQ",
"datatype": "TEXT",
"maxLength": 50
}
]
},
{
"identifier": "procedure1",
"fields": [
{
"name": "CD_FOLIO",
"datatype": "TEXT",
"maxLength": 100
},
{
"name": "CD_CUENTA",
"datatype": "TEXT",
"maxLength": 100
},
{
"name": "CD_PAQUETE",
"datatype": "TEXT",
"maxLength": 100
},
{
"name": "NB_PAQUETE",
"datatype": "TEXT",
"maxLength": 100
}
],
"save_output" : true
"save_output": true
},
{
"identifier": "tabla5",
"identifier": "procedure2",
"fields": [
{
"name": "id",
"datatype": "NUMBER",
"decimal_precision": 0
"name": "PLAN",
"datatype": "TEXT",
"maxLength": 250
},
{
"name": "nombre",
"name": "NOMBRE_PRODUCTO",
"datatype": "TEXT",
"maxLength": 50
"maxLength": 250
}
]
},
{
"name": "fecha",
"datatype": "DATE"
"identifier": "procedure3",
"fields": [
{
"name": "NU_VENDEDOR",
"datatype": "TEXT",
"maxLength": 100
},
{
"name": "fecha_2",
"datatype": "DATETIME"
"name": "NU_ADDON",
"datatype": "TEXT",
"maxLength": 5
}
]
},
{
"name": "fecha_3",
"datatype": "DATETIME"
"identifier": "TablaJOIN_1",
"fields": [
{
"name": "NOMBRE_PRODUCTO",
"datatype": "TEXT",
"maxLength": 100
},
{
"name": "NU_ANIO",
"datatype": "NUMBER"
},
{
"name": "TP_SERVICIO",
"datatype": "NUMBER"
},
{
"name": "SUMA",
"datatype": "NUMBER"
}
],
"indexes": [
]
},
{
"name" : "indice1",
"index_fields": [
"id"
"identifier": "TablaJOIN_2",
"fields": [
{
"name": "CD_PAQUETE",
"datatype": "TEXT",
"maxLength": 500
},
{
"name": "NB_PAQUETE",
"datatype": "TEXT",
"maxLength": 500
},
{
"name": "NB_USUAMODIFICA",
"datatype": "TEXT",
"maxLength": 500
}
]
},
{
"identifier": "TablaJOIN_3",
"fields": [
{
"name": "NOMBRE_PRODUCTO",
"datatype": "TEXT",
"maxLength": 500
},
{
"name": "NB_PAQUETE",
"datatype": "TEXT",
"maxLength": 500
},
{
"name": "CT_RENTASIN",
"datatype": "DECIMAL",
"decimal_precision": 2
},
{
"name": "NU_COMISION",
"datatype": "DECIMAL",
"decimal_precision": 1
},
{
"name": "NU_TIPOESQUEMA",
"datatype": "NUMBER"
}
],
"save_output" : true
]
}
]
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment