Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
B
bcom-tp-etl-transformation-pipelines
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
general
bcom-tp-etl-transformation-pipelines
Commits
a1345ded
Commit
a1345ded
authored
Aug 08, 2023
by
Cristian Aguirre
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Update 07-08-23. Update Generation.py
parent
331c3000
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
4 additions
and
10 deletions
+4
-10
Generation.py
dags/components/Generation.py
+4
-10
No files found.
dags/components/Generation.py
View file @
a1345ded
...
@@ -137,13 +137,10 @@ def get_generate_from_xcom(**kwargs):
...
@@ -137,13 +137,10 @@ def get_generate_from_xcom(**kwargs):
tasks_with_save
=
[]
tasks_with_save
=
[]
definition
=
task
.
xcom_pull
(
task_ids
=
"SCRIPTS-EXTRACTOR"
,
key
=
"EXTRACTION-DEFINITION-JSON"
)
definition
=
task
.
xcom_pull
(
task_ids
=
"SCRIPTS-EXTRACTOR"
,
key
=
"EXTRACTION-DEFINITION-JSON"
)
#LOGICA - TABLAS QUE SI TIENEN EL PARAMETRO SAVE_OUTPUT
# LOGICA - TABLAS QUE SI TIENEN EL PARAMETRO SAVE_OUTPUT
for
i
in
range
(
len
(
definition
)):
for
item
in
definition
:
if
definition
[
i
][
"save_output"
]
==
True
:
if
"save_output"
in
item
.
keys
()
and
item
[
"save_output"
]:
tasks_with_save
.
append
(
definition
[
i
][
"identifier"
])
tasks_with_save
.
append
(
item
[
"identifier"
])
#print(tasks_with_save)
success_tasks
=
tasks
[
"tasks"
]
success_tasks
=
tasks
[
"tasks"
]
success_tasks
=
[
item
[
1
]
for
item
in
success_tasks
]
success_tasks
=
[
item
[
1
]
for
item
in
success_tasks
]
logger
.
info
(
f
"GENERADORES QUE FUERON EXITOSOS (TABLAS): {success_tasks}"
)
logger
.
info
(
f
"GENERADORES QUE FUERON EXITOSOS (TABLAS): {success_tasks}"
)
...
@@ -153,14 +150,11 @@ def get_generate_from_xcom(**kwargs):
...
@@ -153,14 +150,11 @@ def get_generate_from_xcom(**kwargs):
if
not
key
.
startswith
(
OperationTypeEnum
.
SELECT
.
value
)
and
not
key
.
startswith
(
OperationTypeEnum
.
PROCEDURE
.
value
):
if
not
key
.
startswith
(
OperationTypeEnum
.
SELECT
.
value
)
and
not
key
.
startswith
(
OperationTypeEnum
.
PROCEDURE
.
value
):
continue
continue
xcom_outputs
=
task
.
xcom_pull
(
task_ids
=
"SCRIPTS-EXTRACTOR"
,
key
=
key
)
xcom_outputs
=
task
.
xcom_pull
(
task_ids
=
"SCRIPTS-EXTRACTOR"
,
key
=
key
)
logger
.
info
(
f
"Trayendo tablas {xcom_outputs}"
)
logger
.
info
(
f
"Trayendo tablas {xcom_outputs}"
)
for
select
in
xcom_outputs
:
for
select
in
xcom_outputs
:
tablename
=
select_multiple
(
select
)[
"tablename"
]
tablename
=
select_multiple
(
select
)[
"tablename"
]
#print("IMPRIMIR", tablename)
if
(
tasks
[
"reset"
]
or
tasks
[
"status"
]
==
ProcessStatusEnum
.
SUCCESS
.
value
or
select
not
in
success_tasks
)
and
tablename
in
tasks_with_save
:
if
(
tasks
[
"reset"
]
or
tasks
[
"status"
]
==
ProcessStatusEnum
.
SUCCESS
.
value
or
select
not
in
success_tasks
)
and
tablename
in
tasks_with_save
:
final_outputs
.
append
(
select
)
final_outputs
.
append
(
select
)
#print(final_outputs)
logger
.
info
(
f
"Final outputs: {final_outputs}"
)
logger
.
info
(
f
"Final outputs: {final_outputs}"
)
Variable
.
set
(
key
=
'GENERATES'
,
value
=
final_outputs
,
serialize_json
=
True
)
Variable
.
set
(
key
=
'GENERATES'
,
value
=
final_outputs
,
serialize_json
=
True
)
if
len
(
final_outputs
)
>
0
:
if
len
(
final_outputs
)
>
0
:
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment