Skip to content

B
bcom-tp-etl-transformation-pipelines
Commit c4edffad authored by Cristian Aguirre's avatar Cristian Aguirre
Browse files
Options

Fix bugs

parent 3170e104
Hide whitespace changes
Inline Side-by-side
Showing with 441 additions and 78 deletions
dags/components/Extractor.py
View file @ c4edffad
...@@ -163,13 +163,13 @@ def extract_from_source(command, source_conn, intern_conn, chunksize: int, timez ...@@ -163,13 +163,13 @@ def extract_from_source(command, source_conn, intern_conn, chunksize: int, timez
data.append(row) data.append(row)
if len(data) == chunksize: if len(data) == chunksize:
dataframe = pd.DataFrame(data, columns=columns_name) dataframe = pd.DataFrame(data, columns=columns_name)
save = save_from_dataframe(dataframe, tablename, intern_conn.engine) save = save_from_dataframe(dataframe, tablename, intern_conn)
if save: if save:
logger.debug(f"Guardado correctamente dataframe. Procesando más bloques") logger.debug(f"Guardado correctamente dataframe. Procesando más bloques")
data.clear() data.clear()
if len(data) > 0: if len(data) > 0:
dataframe = pd.DataFrame(data, columns=columns_name) dataframe = pd.DataFrame(data, columns=columns_name)
save = save_from_dataframe(dataframe, tablename, intern_conn.engine) save = save_from_dataframe(dataframe, tablename, intern_conn)
if save: if save:
logger.debug(f"Migrado correctamente todos los datos") logger.debug(f"Migrado correctamente todos los datos")
extract_tables.append(tablename) extract_tables.append(tablename)
...@@ -183,13 +183,13 @@ def extract_from_source(command, source_conn, intern_conn, chunksize: int, timez ...@@ -183,13 +183,13 @@ def extract_from_source(command, source_conn, intern_conn, chunksize: int, timez
data.append(row) data.append(row)
if len(data) == chunksize: if len(data) == chunksize:
dataframe = pd.DataFrame(data, columns=columns_name) dataframe = pd.DataFrame(data, columns=columns_name)
save = save_from_dataframe(dataframe, tablename, intern_conn.engine) save = save_from_dataframe(dataframe, tablename, intern_conn)
if save: if save:
logger.debug(f"Guardado correctamente dataframe. Procesando más bloques") logger.debug(f"Guardado correctamente dataframe. Procesando más bloques")
data.clear() data.clear()
if len(data) > 0: if len(data) > 0:
dataframe = pd.DataFrame(data, columns=columns_name) dataframe = pd.DataFrame(data, columns=columns_name)
save = save_from_dataframe(dataframe, tablename, intern_conn.engine) save = save_from_dataframe(dataframe, tablename, intern_conn)
if save: if save:
logger.debug(f"Migrado correctamente todos los datos") logger.debug(f"Migrado correctamente todos los datos")
......
dags/procedure_prueba.json
View file @ c4edffad
[ [
{ {
"identifier": "ModificarTabla4", "identifier": "ModificarTabla4",
"transformation_store_procedure": true "transformation_store_procedure" : true
}, },
{ {
"identifier" : "UnionYInsert", "identifier": "SP_ACTUALIZAR_TABLA_5",
"transformation_store_procedure": true "transformation_store_procedure" : true
}, },
{ {
"identifier": "TempTabla", "identifier": "REGLA7_1",
"temp_table" : true "temp_table" : true
}, },
{ {
"identifier": "tabla11", "identifier": "REGLA7_2",
"temp_table" : true
},
{
"identifier": "RANGO_VENTAS_CON_PROMOCION",
"fields": [
{
"name": "CD_PAQUETE",
"datatype": "TEXT",
"maxLength": 100
},
{
"name": "NU_ADDON",
"datatype": "TEXT",
"maxLength": 15
},
{
"name": "CD_CLIENTE",
"datatype": "TEXT",
"maxLength": 50
}
]
},
{
"identifier": "TACOMVENTAS",
"fields": [ "fields": [
{ {
"name": "id", "name": "CD_EMPRESA",
"datatype": "NUMBER", "datatype": "NUMBER",
"decimal_precision": 0 "decimal_precision": 0
}, },
{ {
"name": "nombre", "name": "CD_FOLIO",
"datatype": "TEXT",
"maxLength": 100
},
{
"name": "CD_CUENTA",
"datatype": "TEXT",
"maxLength": 100
},
{
"name": "NU_VENDEDOR",
"datatype": "TEXT",
"maxLength": 100
},
{
"name": "CD_PAQUETE",
"datatype": "TEXT",
"maxLength": 100
},
{
"name": "NU_ADDON",
"datatype": "TEXT",
"maxLength": 5
},
{
"name": "NB_PAQUETE",
"datatype": "TEXT",
"maxLength": 200
},
{
"name": "CD_CLIENTE",
"datatype": "TEXT", "datatype": "TEXT",
"maxLength": 50 "maxLength": 50
}, },
{ {
"name": "fecha_nacimiento", "name": "NB_CLIENTE",
"datatype": "TEXT",
"maxLength": 200
},
{
"name": "FH_CONTRATACION",
"datatype": "DATE" "datatype": "DATE"
},
{
"name": "FH_ACTIVACION",
"datatype": "DATETIME"
},
{
"name": "FH_OPERACION",
"datatype": "DATETIME"
},
{
"name": "TP_SERVICIO",
"datatype": "NUMBER",
"decimal_precision": 0
},
{
"name": "ST_CLIENTE",
"datatype": "NUMBER",
"decimal_precision": 0
},
{
"name": "TP_PAGO",
"datatype": "TEXT",
"maxLength": 10
},
{
"name": "NB_USUACARGA",
"datatype": "TEXT",
"maxLength": 50
},
{
"name": "FH_CARGA",
"datatype": "DATETIME"
},
{
"name": "NU_ANIO",
"datatype": "NUMBER",
"decimal_precision": 0
},
{
"name": "NU_MES",
"datatype": "NUMBER",
"decimal_precision": 0
},
{
"name": "NU_SEMANA",
"datatype": "NUMBER",
"decimal_precision": 0
},
{
"name": "NU_COMISION",
"datatype": "NUMBER",
"decimal_precision": 0
},
{
"name": "TP_PAGOANT",
"datatype": "NUMBER",
"decimal_precision": 0
},
{
"name": "REGLA_APLICADA",
"datatype": "NUMBER",
"decimal_precision": 0
},
{
"name": "AUX",
"datatype": "TEXT",
"maxLength": 50
},
{
"name": "PROMOCION",
"datatype": "TEXT",
"maxLength": 80
},
{
"name": "EMPLEADOEMBAJADOR__C",
"datatype": "TEXT",
"maxLength": 100
},
{
"name": "CANAL_DE_VENTA__C",
"datatype": "TEXT",
"maxLength": 50
},
{
"name": "CANALREFERIDO__C",
"datatype": "TEXT",
"maxLength": 50
} }
], ],
"indexes": [ "indexes": [
{ {
"name" : "indice1", "name": "indice1",
"index_fields": [
"CD_FOLIO","CD_PAQUETE"
]
},
{
"name": "indice2",
"index_fields": [ "index_fields": [
"id" "CD_PAQUETE"
]
},
{
"name": "indice3",
"index_fields": [
"CD_CLIENTE","PROMOCION","NB_PAQUETE"
]
},
{
"name": "indice4",
"index_fields": [
"NU_ADDON"
] ]
} }
], ],
"save_output" : false "save_output": true
}, },
{ {
"identifier": "tabla2", "identifier": "PROMOCIONES_RESIDENCIAL",
"fields": [ "fields": [
{ {
"name": "id", "name": "EMPRESA",
"datatype": "TEXT",
"maxLength": 250
},
{
"name": "CUENTA",
"datatype": "TEXT",
"maxLength": 250
},
{
"name": "PLAN",
"datatype": "TEXT",
"maxLength": 250
},
{
"name": "NOMBRE_PRODUCTO",
"datatype": "TEXT",
"maxLength": 250
},
{
"name": "DESCR",
"datatype": "TEXT",
"maxLength": 250
},
{
"name": "TIPO_ADICION",
"datatype": "TEXT",
"maxLength": 250
},
{
"name": "RENTACONIMPUESTOS",
"datatype": "NUMBER"
},
{
"name": "RENTASINIMPUESTOS",
"datatype": "NUMBER"
},
{
"name": "QUANTITY",
"datatype": "NUMBER", "datatype": "NUMBER",
"decimal_precision": 0 "decimal_precision": 0
}, },
{ {
"name": "apellido", "name": "CREACION_PRODUCTO",
"datatype": "DATE"
},
{
"name": "INICIO_COBRO",
"datatype": "DATE"
},
{
"name": "FIN_COBRO",
"datatype": "DATE"
},
{
"name": "FIN_COMPRA",
"datatype": "DATE"
},
{
"name": "SERV_STATUS",
"datatype": "TEXT",
"maxLength": 20
},
{
"name": "POID_TYPE",
"datatype": "TEXT",
"maxLength": 200
},
{
"name": "POID_PRODUCT",
"datatype": "TEXT", "datatype": "TEXT",
"maxLength": 50 "maxLength": 50
}, },
{ {
"name": "fecha_registro", "name": "STATUS_PRODUCTO",
"datatype": "DATETIME" "datatype": "TEXT",
"decimal_precision": null,
"maxLength": 200
},
{
"name": "SERVICIO",
"datatype": "TEXT",
"maxLength": 200
},
{
"name": "CD_PAQUETE",
"datatype": "TEXT",
"maxLength": 100
} }
], ],
"indexes": [ "indexes": [
{ {
"name" : "indice1", "name": "indice1",
"index_fields": [ "index_fields" : [
"id" "CD_PAQUETE"
] ]
} }
], ]
"save_output" : false
}, },
{ {
"identifier": "tabla3", "identifier": "TABLA_PROMOCIONES_2",
"fields": [ "fields": [
{ {
"name": "id", "name": "PLAN",
"datatype": "NUMBER", "datatype": "TEXT",
"decimal_precision": 0 "maxLength": 250
}, },
{ {
"name": "edad", "name": "NOMBRE_PRODUCTO",
"datatype": "NUMBER", "datatype": "TEXT",
"decimal_precision": 0 "maxLength": 250
}, },
{ {
"name": "fecha_actualizacion", "name": "SUMA_RENTA",
"datatype": "DATETIME" "datatype": "NUMBER"
} }
], ],
"indexes": [ "indexes": [
{ {
"name" : "indice1", "name": "indice1",
"index_fields": [ "index_fields" : [
"id" "PLAN"
] ]
} }
], ]
"save_output" : false
}, },
{ {
"identifier": "tabla4", "identifier": "CATALOGO_PROMOCIONES",
"fields": [ "fields": [
{ {
"name": "id", "name": "NOMBRE_PRODUCTO",
"datatype": "NUMBER", "datatype": "TEXT",
"decimal_precision": 0 "maxLength": 100
}, },
{ {
"name": "nombre", "name": "CD_PAQUETE",
"datatype": "TEXT",
"maxLength": 50
}
]
},
{
"identifier": "TEMP_PROMO",
"fields": [
{
"name": "TRESP",
"datatype": "TEXT", "datatype": "TEXT",
"maxLength": 50 "maxLength": 50
}, },
{ {
"name": "fecha", "name": "DOSP",
"datatype": "DATE" "datatype": "TEXT",
"maxLength": 50
}
]
},
{
"identifier": "RELACION_POID_PAQ",
"fields": [
{
"name": "POID_PRODUCT",
"datatype": "TEXT",
"maxLength": 50
}, },
{ {
"name": "fecha_2", "name": "CD_PAQUETE",
"datatype": "DATE" "datatype": "TEXT",
"maxLength": 50
}
]
},
{
"identifier": "RELACION_PAQINI_PAQFIN",
"fields": [
{
"name": "COD_PAQ_INI",
"datatype": "TEXT",
"maxLength": 50
}, },
{ {
"name": "fecha_3", "name": "COD_PAQ_FIN",
"datatype": "DATETIME" "datatype": "TEXT",
"maxLength": 50
} }
], ]
"indexes": [ },
{
"identifier": "ADDONS_UNICO",
"fields": [
{ {
"name" : "indice1", "name": "COD_PAQ",
"index_fields": [ "datatype": "TEXT",
"id" "maxLength": 50
] }
]
},
{
"identifier": "procedure1",
"fields": [
{
"name": "CD_FOLIO",
"datatype": "TEXT",
"maxLength": 100
},
{
"name": "CD_CUENTA",
"datatype": "TEXT",
"maxLength": 100
},
{
"name": "CD_PAQUETE",
"datatype": "TEXT",
"maxLength": 100
},
{
"name": "NB_PAQUETE",
"datatype": "TEXT",
"maxLength": 100
} }
], ],
"save_output" : true "save_output": true
}, },
{ {
"identifier": "tabla5", "identifier": "procedure2",
"fields": [ "fields": [
{ {
"name": "id", "name": "PLAN",
"datatype": "NUMBER", "datatype": "TEXT",
"decimal_precision": 0 "maxLength": 250
}, },
{ {
"name": "nombre", "name": "NOMBRE_PRODUCTO",
"datatype": "TEXT", "datatype": "TEXT",
"maxLength": 50 "maxLength": 250
}
]
},
{
"identifier": "procedure3",
"fields": [
{
"name": "NU_VENDEDOR",
"datatype": "TEXT",
"maxLength": 100
}, },
{ {
"name": "fecha", "name": "NU_ADDON",
"datatype": "DATE" "datatype": "TEXT",
"maxLength": 5
}
]
},
{
"identifier": "TablaJOIN_1",
"fields": [
{
"name": "NOMBRE_PRODUCTO",
"datatype": "TEXT",
"maxLength": 100
}, },
{ {
"name": "fecha_2", "name": "NU_ANIO",
"datatype": "DATETIME" "datatype": "NUMBER"
}, },
{ {
"name": "fecha_3", "name": "TP_SERVICIO",
"datatype": "DATETIME" "datatype": "NUMBER"
},
{
"name": "SUMA",
"datatype": "NUMBER"
} }
], ]
"indexes": [ },
{
"identifier": "TablaJOIN_2",
"fields": [
{ {
"name" : "indice1", "name": "CD_PAQUETE",
"index_fields": [ "datatype": "TEXT",
"id" "maxLength": 500
] },
{
"name": "NB_PAQUETE",
"datatype": "TEXT",
"maxLength": 500
},
{
"name": "NB_USUAMODIFICA",
"datatype": "TEXT",
"maxLength": 500
} }
], ]
"save_output" : true },
{
"identifier": "TablaJOIN_3",
"fields": [
{
"name": "NOMBRE_PRODUCTO",
"datatype": "TEXT",
"maxLength": 500
},
{
"name": "NB_PAQUETE",
"datatype": "TEXT",
"maxLength": 500
},
{
"name": "CT_RENTASIN",
"datatype": "DECIMAL",
"decimal_precision": 2
},
{
"name": "NU_COMISION",
"datatype": "DECIMAL",
"decimal_precision": 1
},
{
"name": "NU_TIPOESQUEMA",
"datatype": "NUMBER"
}
]
} }
] ]
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment