Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
s3_bk_files-git
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
general
s3_bk_files-git
Commits
16cc8915
Commit
16cc8915
authored
Jun 27, 2021
by
criq
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
rename files
parent
06058ae4
Changes
8
Hide whitespace changes
Inline
Side-by-side
Showing
8 changed files
with
82 additions
and
1 deletion
+82
-1
README.md
README.md
+1
-1
__init__.py
s3_bk_files/__init__.py
+0
-0
__main__.py
s3_bk_files/__main__.py
+0
-0
app.py
s3_bk_files/app.py
+81
-0
config.py
s3_bk_files/config.py
+0
-0
util_registry.py
s3_bk_files/src/util_registry/util_registry.py
+0
-0
s3_helper.py
s3_bk_files/src/utils_aws_s3/s3_helper.py
+0
-0
local_helper.py
s3_bk_files/src/utils_local_os/local_helper.py
+0
-0
No files found.
README.md
View file @
16cc8915
...
...
@@ -26,5 +26,5 @@ pip3 install -r requirements.txt
## Ejecución
```
sh
python3
-m
s3_
local_manager
python3
-m
s3_
bk_files
```
s3_
local_manager
/__init__.py
→
s3_
bk_files
/__init__.py
View file @
16cc8915
File moved
s3_
local_manager
/__main__.py
→
s3_
bk_files
/__main__.py
View file @
16cc8915
File moved
s3_
local_manager
/app.py
→
s3_
bk_files
/app.py
View file @
16cc8915
...
...
@@ -33,46 +33,46 @@ def run():
tic
=
time
.
time
()
try
:
s3Help
=
AwsS3Helper
(
aws_access_key_id
,
aws_secret_access_key
,
aws_region
)
# Obtener archivos del bucket ordenado
# por defecto el primer elemento es el más antiguo
lista_s3
=
s3Help
.
sorted_list_keys
(
config
.
S3_BUCKET
)
print
(
lista_s3
)
# Obtener la fecha del ultimo archivo procesado
last_datetime
=
rg
.
get_last_datetime_file_process
()
logger
.
info
(
' Fecha del último archivo procesado : '
+
str
(
last_datetime
))
s3Help
=
AwsS3Helper
(
aws_access_key_id
,
aws_secret_access_key
,
aws_region
)
# Obtener archivos del bucket ordenado
# por defecto el primer elemento es el más antiguo
lista_s3
=
s3Help
.
sorted_list_keys
(
config
.
S3_BUCKET
)
print
(
lista_s3
)
# Obtener la fecha del ultimo archivo procesado
last_datetime
=
rg
.
get_last_datetime_file_process
()
logger
.
info
(
' Fecha del último archivo procesado : '
+
str
(
last_datetime
))
## Filtrar lista por la fecha anterior
filter_list_s3
=
s3Help
.
filter_by_datetime
(
lista_s3
,
last_datetime
)
if
len
(
filter_list_s3
)
==
0
and
last_datetime
!=
None
:
logger
.
warning
(
' No hay nuevos archivos a procesar ... '
)
elif
last_datetime
==
None
:
filter_list_s3
=
lista_s3
## Filtrar lista por la fecha anterior
filter_list_s3
=
s3Help
.
filter_by_datetime
(
lista_s3
,
last_datetime
)
if
len
(
filter_list_s3
)
==
0
and
last_datetime
!=
None
:
logger
.
warning
(
' No hay nuevos archivos a procesar ... '
)
elif
last_datetime
==
None
:
filter_list_s3
=
lista_s3
print
(
'**** '
)
# Clase para gestionar carpetas locales
locaTemplHelper
=
LocalFilesHelper
(
config
.
local_folder_temp
)
locaDefinitivelHelper
=
LocalFilesHelper
(
config
.
local_folder_definitive
)
print
(
'**** '
)
# Clase para gestionar carpetas locales
locaTemplHelper
=
LocalFilesHelper
(
config
.
local_folder_temp
)
locaDefinitivelHelper
=
LocalFilesHelper
(
config
.
local_folder_definitive
)
# iterar archivos
# guardar el último
last_file
=
None
for
s3_file
in
filter_list_s3
:
local_dest_file
=
locaTemplHelper
.
create_pathfile_for_dict
(
s3_file
)
if
local_dest_file
:
# Retorna True si ya esta registrado
is_registred
=
rg
.
registry_file_dict
(
s3_file
)
if
not
is_registred
:
s3Help
.
download_file_from_dict
(
s3_file
,
local_dest_file
)
print
(
local_dest_file
)
# guardar en el otro directorio tambien
locaDefinitivelHelper
.
move_file
(
local_dest_file
)
last_file
=
s3_file
print
(
'_'
*
60
)
# iterar archivos
# guardar el último
last_file
=
None
for
s3_file
in
filter_list_s3
:
local_dest_file
=
locaTemplHelper
.
create_pathfile_for_dict
(
s3_file
)
if
local_dest_file
:
# Retorna True si ya esta registrado
is_registred
=
rg
.
registry_file_dict
(
s3_file
)
if
not
is_registred
:
s3Help
.
download_file_from_dict
(
s3_file
,
local_dest_file
)
print
(
local_dest_file
)
# guardar en el otro directorio tambien
locaDefinitivelHelper
.
move_file
(
local_dest_file
)
last_file
=
s3_file
print
(
'_'
*
60
)
toc
=
time
.
time
()
rg
.
save_process_time
(
start_time
,
tic
,
toc
,
last_file
)
toc
=
time
.
time
()
rg
.
save_process_time
(
start_time
,
tic
,
toc
,
last_file
)
except
Exception
as
e
:
logger
.
error
(
'Existe un problema en el proceso:
%
s'
,
e
)
...
...
s3_
local_manager
/config.py
→
s3_
bk_files
/config.py
View file @
16cc8915
File moved
s3_
local_manager
/src/util_registry/util_registry.py
→
s3_
bk_files
/src/util_registry/util_registry.py
View file @
16cc8915
File moved
s3_
local_manager
/src/utils_aws_s3/s3_helper.py
→
s3_
bk_files
/src/utils_aws_s3/s3_helper.py
View file @
16cc8915
File moved
s3_
local_manager
/src/utils_local_os/local_helper.py
→
s3_
bk_files
/src/utils_local_os/local_helper.py
View file @
16cc8915
File moved
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment