Update 01-08-23. Update deploy

dags/dag_conf.yml

@@ -5,41 +5,41 @@ app:
   database:
     sources:
       source1:
-        type: postgres
-        host: airflow
-        port: 5432
-        username: airflow
-        password: airflow
-        database: postgres
+        type: mysql
+        host: database-11.cluster-ro-cvsz4ey9eiec.us-east-1.rds.amazonaws.com
+        port: 3306
+        username: admin
+        password: adminadmin
+        database: prueba_bcom
         service: ORCLPDB1
         schema: sources
     transformation:
-      type: postgres
-      host: airflow
-      port: 5432
-      username: airflow
-      password: airflow
-      database: postgres
+      type: mysql
+      host: database-11.cluster-ro-cvsz4ey9eiec.us-east-1.rds.amazonaws.com
+      port: 3306
+      username: admin
+      password: adminadmin
+      database: prueba_bcom2
       service:
       schema: intern_db
   chunksize: 8000
-  label_multiple_select: TABLENAME
+  label_multiple_select: TABLE
   source_mask: select  # Sufijo (S)
   procedure_mask: procedure  # S
   transformation_mask: transform  # S
   prefix_order_delimiter: .
-  cloud_provider: aws
+  cloud_provider: google
   scripts:
     s3_params:
-      bucket: prueba1234568
+      bucket: prueba-airflow3
       prefix: bcom_scripts
       connection_id: conn_script
   control:
     s3_params:
       connection_id: conn_script
-      bucket: prueba1234568
+      bucket: prueba-airflow3
       prefix: bcom_control
-      filename: control_example.json
+      filename: control_<period>.json
   timezone: 'GMT-5'
   outputs:
     filename_mask: <source_name>_<datetime>
@@ -48,12 +48,12 @@ app:
     delimiter: '|'
     tmp_path: /tmp
     s3_params:
-      bucket: prueba1234568
+      bucket: prueba-airflow3
       prefix: bcom_results
       connection_id: conn_script
   report:
     s3_params:
-      bucket: prueba1234568
+      bucket: prueba-airflow3
       prefix: bcom_report
       connection_id: conn_script
       filename: report_<datetime>.xlsx

dags/procedure_definition2.json

 [
     {
-  "identifier": "tabla2",
+  "identifier": "TABLA1",
   "fields": [
     {
-      "name": "column1",
+      "name": "CD_CUENTA",
       "datatype": "TEXT",
       "maxLength": 100
     },
     {
-      "name": "column2",
+      "name": "DPP_FLAG",
       "datatype": "NUMBER"
     },
     {
-      "name": "column3",
-      "datatype": "NUMBER"
+      "name": "FECHA_FACTURA",
+      "datatype": "DATE"
+    },
+    {
+      "name": "FECHA_INICIAL_DPP",
+      "datatype": "DATE"
     },
     {
-      "name": "column4",
+      "name": "FECHA_LIMITE_DPP",
       "datatype": "DATE"
     },
     {
-      "name": "column5",
-      "datatype": "DECIMAL",
-      "decimal_precision": 5
+      "name": "FH_CARGA",
+      "datatype": "DATE"
+    },
+    {
+      "name": "ID_FACTURA",
+      "datatype": "TEXT",
+      "maxLength": 100
+    },
+    {
+      "name": "MONTO_AJUSTADO",
+      "datatype": "DECIMAL"
+    },
+    {
+      "name": "MONTO_FACTURA",
+      "datatype": "DECIMAL"
+    },
+    {
+      "name": "MONTO_PAGO",
+      "datatype": "DECIMAL"
+    },
+    {
+      "name": "SALDO_FACTURA",
+      "datatype": "DECIMAL"
     }
   ]
 }

deploy-k8/requirements.txt

@@ -3,4 +3,6 @@ XlsxWriter==3.1.2
 pymysql==1.1.0
 oracledb==1.3.2
 apache-airflow-providers-google
-apache-airflow-providers-amazon
\ No newline at end of file
+apache-airflow-providers-amazon
+apache-airflow-providers-postgres
+apache-airflow-providers-oracle
\ No newline at end of file