Skip to content

E
ejercicio2-framework-back
Commit 53a752db authored by Roberto Loayza's avatar Roberto Loayza
Browse files
Options

Despliegue de base de conocimiento.

parent a0d2f14c
Hide whitespace changes
Inline Side-by-side
Showing with 193 additions and 32 deletions
src/main/java/com/bytesw/bytebot/controller/AgentController.java
View file @ 53a752db
......@@ -175,8 +175,8 @@ public class AgentController {
return new ResponseEntity<>(gson.toJson(response), HttpStatus.OK);
} catch (Exception e) {
e.printStackTrace();
return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR);
}
return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR);
}
@PutMapping("/file-upload/{uuid}")
......
src/main/java/com/bytesw/bytebot/etl/batch/service/ScheduleService.java
View file @ 53a752db
......@@ -65,7 +65,7 @@ public class ScheduleService implements SchedulingConfigurer {
@Qualifier("schedulerLockTaskScheduler")
TaskScheduler poolScheduler;
@Value("${application.byte-bot.batch.cron: 0 0/1 * * * * }")
@Value("${application.byte-bot.batch.cron}")
private String cronExpression;
@Value("${application.byte-bot.batch.chunk:500}")
......@@ -135,10 +135,19 @@ public class ScheduleService implements SchedulingConfigurer {
@Autowired
private CalendarExceptionFullRepository calendarExceptionFullRepository;
@Autowired
private BusinessParameterRepository businessParameterRepository;
private boolean schedulerFlag = false;
private List<String> keys = new ArrayList<>();
private String tenant;
private ScheduledTaskRegistrar taskRegister;
private TimeZone timeZone;
@Override
public void configureTasks(ScheduledTaskRegistrar taskRegistrar) {
if (scheduledTaskRegistrar == null) {
......@@ -168,7 +177,17 @@ public class ScheduleService implements SchedulingConfigurer {
}
private void configureTask(String tenantIdentifier, String identifier, ScheduledTaskRegistrar taskRegistrar) {
Trigger trigger = new CronTrigger(cronExpression, TimeZone.getDefault());
tenant = tenantIdentifier;
taskRegister = taskRegistrar;
Optional<BusinessParameter> zone = businessParameterRepository.findByKey("TimeZone");
if (zone.isPresent()) {
timeZone = TimeZone.getTimeZone(zone.get().getDefaultValue());
} else {
timeZone = TimeZone.getDefault();
}
System.out.println(timeZone);
Trigger trigger = new CronTrigger(cronExpression, timeZone);
/* ETL de Dashboard */
String key = String.format("%s-%s", tenantIdentifier, identifier);
futureMap.put(key, taskRegistrar.getScheduler().schedule(() -> scheduleCron(createJob(tenantIdentifier), tenantIdentifier), trigger));
......@@ -179,12 +198,30 @@ public class ScheduleService implements SchedulingConfigurer {
/* Busca las listas de SchedulerTask */
List<SchedulerTask> listSchedulerTask = schedulerTaskRepository.findByEtlId(processDelete.getId());
for (SchedulerTask schedulerTask : listSchedulerTask) {
Trigger triggerDelete = new CronTrigger(schedulerTask.getCronExpression(), TimeZone.getDefault());
String keyScheduler = String.format("%s-%s", tenantIdentifier, schedulerTask.getCalendarID());
Trigger triggerDelete = new CronTrigger(schedulerTask.getCronExpression(), timeZone);
String keyScheduler = String.format("%s-%s-%s", tenantIdentifier, schedulerTask.getCalendarID().trim(), schedulerTask.getId());
futureMap.put(keyScheduler, taskRegistrar.getScheduler()
.schedule(() -> processDeleteData(tenantIdentifier, taskRegistrar, schedulerTask.getCalendarID().trim()), triggerDelete));
}
}
public void restartTask(SchedulerTask model) {
String key = String.format("%s-%s-%s", tenant, model.getCalendarID().trim(), model.getId());
Trigger trigger = new CronTrigger(model.getCronExpression(), timeZone);
if (futureMap.containsKey(key)) {
cancelTask(model);
}
futureMap.put(key, taskRegister.getScheduler()
.schedule(() -> processDeleteData(tenant, taskRegister, model.getCalendarID().trim()), trigger));
}
public void cancelTask(SchedulerTask model) {
String key = String.format("%s-%s-%s", tenant, model.getCalendarID().trim(), model.getId());
if (futureMap.containsKey(key)){
ScheduledFuture future = futureMap.get(key);
future.cancel(true);
futureMap.remove(key);
}
}
/*ETL eliminacion de data sensible*/
......@@ -202,11 +239,10 @@ public class ScheduleService implements SchedulingConfigurer {
} else {
schedulerFlag = false;
}
}
if (schedulerFlag) {
Trigger trigger = new CronTrigger(cronExpression, TimeZone.getDefault());
Trigger trigger = new CronTrigger(cronExpression, timeZone);
List<DeleteDataSensBean> deleteDataSensBeans = deleteDataSensJDBCRepository
.getListAgentChannel(AgentStatusEnum.DEPLOYED.getName(), AgentParameterEnum.ACCESS_TWILIO.getName());
String keyDataSens = "";
......@@ -221,7 +257,11 @@ public class ScheduleService implements SchedulingConfigurer {
}
} else {
keys.forEach(x -> {
futureMap.remove(x);
if (futureMap.containsKey(x)) {
ScheduledFuture future = futureMap.get(x);
future.cancel(true);
futureMap.remove(x);
}
});
}
}
......@@ -284,7 +324,7 @@ public class ScheduleService implements SchedulingConfigurer {
private ItemReader getReader(String tenantIdentifier) {
if (properties == null) {
throw new RuntimeException("Properties not found");
throw new RuntimeException("Prope rties not found");
}
Optional<TenantBatchBean> tenantFound = findTenant(tenantIdentifier);
......@@ -303,8 +343,9 @@ public class ScheduleService implements SchedulingConfigurer {
/* Métodos ETL de data sensible */
private Job createJobDataSens(String tenantIdentifier, DeleteDataSensBean data) {
System.out.println("ETL de eliminacion");
ThreadLocalStorage.setTenantName(tenantIdentifier);
return jobBuilderFactory.get( String.format("processDataSensible%d",+data.getAgenId()))
return jobBuilderFactory.get( String.format("processDataSensible-%d",+data.getAgenId()))
.incrementer(new RunIdIncrementer()).listener(listener)
.flow(createStepDataSens(tenantIdentifier, data)).end().build();
}
......
src/main/java/com/bytesw/bytebot/http/enums/ValidationStatusEnum.java
View file @ 53a752db
......@@ -12,6 +12,8 @@ import java.util.NoSuchElementException;
public enum ValidationStatusEnum {
OK("OK"),
ERROR("ERROR"),
DATA_INCOMPLETE ("DI"),
FORMAT_INCORRECT("FI"),
INCOMPATIBLE_EXTENSION("IE"),
HEADER_ERROR ("HE"),
CONTENT_ERROR("CE");
......
src/main/java/com/bytesw/bytebot/service/FileManagementService.java
View file @ 53a752db
......@@ -5,7 +5,6 @@
*/
package com.bytesw.bytebot.service;
import com.bytesw.bytebot.etl.model.GoalForActions;
import com.bytesw.bytebot.http.FileValidationResponse;
import com.bytesw.bytebot.http.FileValidationResult;
import com.bytesw.bytebot.http.enums.ValidationStatusEnum;
......@@ -13,8 +12,8 @@ import com.bytesw.bytebot.model.QuestionFile;
import com.bytesw.bytebot.model.enums.FrequentQuestionStatusEnum;
import com.bytesw.bytebot.repository.QuestionFileRepository;
import com.bytesw.bytebot.utils.Utilities;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.*;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
......@@ -46,20 +45,17 @@ public class FileManagementService {
@Autowired
private QuestionFileRepository questionFileRepository;
@Transactional(propagation = Propagation.REQUIRED)
public FileValidationResponse validateAndSaveFile(String uuid, MultipartFile file) throws IOException {
public FileValidationResponse saveFile(String uuid, MultipartFile file) throws IOException {
FileValidationResponse response = new FileValidationResponse();
FileValidationResult result = new FileValidationResult();
//@Validaciones: Tipo de Extension
boolean acceptedExtension = false;
String fileExtension = FilenameUtils.getExtension(file.getOriginalFilename());
for (String validExension : validExtensions) {
if (validExension.equalsIgnoreCase(fileExtension.trim())) {
acceptedExtension = true;
continue;//Break?
break;
}
}
......@@ -69,6 +65,24 @@ public class FileManagementService {
response.setFileValidationResult(result);
return response;
}
switch (fileExtension) {
case "csv":
response = validateCsv(uuid, file);
break;
case "xls":
case "xlsx":
response = validateAndSaveFile(uuid, file);
break;
}
return response;
}
@Transactional(propagation = Propagation.REQUIRED)
public FileValidationResponse validateAndSaveFile(String uuid, MultipartFile file) throws IOException {
FileValidationResponse response = new FileValidationResponse();
FileValidationResult result = new FileValidationResult();
//@Validaciones: De Estructura del Excel (Cabeceras)
byte[] fileData = file.getBytes();
......@@ -81,7 +95,19 @@ public class FileManagementService {
//@HU: Llenando los headers encontrados
Map<String, Integer> headersXPosition = new HashMap<>();
List<String> headersList = new ArrayList<>();
//Cabeceras vacias
if (headerRow == null) {
Map<String, String> headerErrorMap = new HashMap<>();
for (String key : fileHeaders) {
headerErrorMap.put(key, "HEADER.NOT.FOUND.FIRST.ROW");
}
result.setHeadersErrorMap(headerErrorMap);
result.setStatus(ValidationStatusEnum.HEADER_ERROR);
response.setFileValidationResult(result);
return response;
}
Iterator<Cell> cells = headerRow.cellIterator();
int position = 0;
while (cells.hasNext()) {
......@@ -165,7 +191,9 @@ public class FileManagementService {
Map<String, String> recordsErrorMap = validateRecordsXRow(recordsXRow);
if (!recordsErrorMap.isEmpty()) {
recordsRowErrorMap.put(rowNumber + 1, recordsErrorMap);
if (recordsErrorMap.size() != fileHeaders.length){
recordsRowErrorMap.put(rowNumber + 1, recordsErrorMap);
}
}
rowNumber++;
......@@ -210,6 +238,80 @@ public class FileManagementService {
return null;
}
public FileValidationResponse validateCsv(String uuid, MultipartFile file) throws IOException{
FileValidationResponse response = new FileValidationResponse();
FileValidationResult result = new FileValidationResult();
List<List<String>> rows = new ArrayList<>();
try {
String line;
InputStream is = file.getInputStream();
BufferedReader br = new BufferedReader(new InputStreamReader(is));
while ((line = br.readLine()) != null) {
List<String> row = new ArrayList<>();
String[] data = line.split(";");
if (data.length != fileHeaders.length) {
Map<String, String> dataError = new HashMap<>();
dataError.put("DATA", "DATA.INCOMPLETE");
result.setHeadersErrorMap(dataError);
result.setStatus(ValidationStatusEnum.ERROR);
response.setFileValidationResult(result);
return response;
}
for (String dato : data) {
if (validateData(dato)) {
Map<String, String> errorFormat = new HashMap<>();
errorFormat.put("DATA", "FORMAT.INCORRECT");
result.setHeadersErrorMap(errorFormat);
result.setStatus(ValidationStatusEnum.ERROR);
response.setFileValidationResult(result);
return response;
}
row.add(dato);
}
rows.add(row);
}
List<String> headersList = new ArrayList<>();
for (String header: rows.get(0)) {
headersList.add(header.replace('"', ' '));
}
Map<String, String> headerErrorMap = validateHeaders(headersList);
if (!headerErrorMap.isEmpty()) {
result.setHeadersErrorMap(headerErrorMap);
result.setStatus(ValidationStatusEnum.ERROR);
response.setFileValidationResult(result);
return response;
}
} catch (IOException e) {
System.err.println(e.getMessage());
}
result.setStatus(ValidationStatusEnum.OK);
QuestionFile questionFileBD = new QuestionFile();
questionFileBD.setUuid(uuid);
questionFileBD.setName(file.getOriginalFilename());
questionFileBD.setSize(file.getSize());
questionFileBD.setUploadDate(LocalDateTime.now());
questionFileBD.setData(file.getBytes());
questionFileBD = questionFileRepository.save(questionFileBD);
response.setId(questionFileBD.getId());
response.setUuid(questionFileBD.getUuid());
response.setFileName(questionFileBD.getName());
response.setStatus(FrequentQuestionStatusEnum.PENDING_SYNCHRONIZED.getName());
response.setUploadDate(getUploadDate(questionFileBD.getUploadDate()));
response.setFileValidationResult(result);
return response;
}
private boolean validateData (String data) {
if (data.charAt(0) == '"' && data.charAt(data.length()-1) == '"') {
return false;
}
return true;
}
private Map<String, String> validateHeaders(List<String> headersList) {
Map<String, Integer> headerOcurrencesMap = new HashMap<>();
......
src/main/java/com/bytesw/bytebot/service/KnowledgeService.java
View file @ 53a752db
......@@ -2,12 +2,10 @@ package com.bytesw.bytebot.service;
import com.bytesw.bytebot.etl.utils.JsonUtils;
import com.bytesw.bytebot.http.FileValidationResponse;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.log4j.Log4j2;
import org.apache.http.HttpEntity;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.ResponseHandler;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
......@@ -18,8 +16,6 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
@Service
@Log4j2
......
src/main/java/com/bytesw/bytebot/service/OrquestadorService.java
View file @ 53a752db
......@@ -4,10 +4,12 @@ import com.bytesw.bytebot.http.FileValidationResponse;
import com.bytesw.bytebot.http.FileValidationResult;
import com.bytesw.bytebot.http.enums.ValidationStatusEnum;
import com.bytesw.bytebot.model.BdcControl;
import com.bytesw.bytebot.model.BusinessParameter;
import com.bytesw.bytebot.model.QuestionFile;
import com.bytesw.bytebot.model.enums.FrequentQuestionStatusEnum;
import com.bytesw.bytebot.model.enums.StatusBcdEnum;
import com.bytesw.bytebot.repository.BdcControlRepository;
import com.bytesw.bytebot.repository.BusinessParameterRepository;
import com.bytesw.bytebot.repository.QuestionFileRepository;
import com.bytesw.xdf.exception.NotFoundException;
import lombok.extern.log4j.Log4j2;
......@@ -39,14 +41,17 @@ public class OrquestadorService {
@Autowired
private KnowledgeService knowledgeService;
@Autowired
private BusinessParameterRepository businessParameterRepository;
public FileValidationResponse executeGenerateBCD(String uuid, MultipartFile file) {
BdcControl model = new BdcControl();
FileValidationResponse response = new FileValidationResponse();
boolean step = false;
try{
//Valida y guarda en base de datos
response = fileManagementService.validateAndSaveFile(uuid,file);
response = fileManagementService.saveFile(uuid, file);
if(response.getFileValidationResult().getStatus().getName() != ValidationStatusEnum.OK.getName()) {
response.setStatus(FrequentQuestionStatusEnum.ERROR.getName());
return response;
......@@ -58,14 +63,21 @@ public class OrquestadorService {
model.setStatus(StatusBcdEnum.CARGADO.getName());
model = bdcControlRepository.save(model);
//Carga archivo a bucket
step = bucketService.generate(file.getBytes(), response);
if (!step) {
throw new NotFoundException("Error a cargar archivo.");
//Modo de despliegue
Optional<BusinessParameter> deployment = businessParameterRepository.findByKey("Despliegue");
if (!deployment.isPresent()) {
throw new NotFoundException("Modo de despliegue no encontrado");
}
if (deployment.get().getDefaultValue().equalsIgnoreCase("onCloud")) {
//Carga archivo a bucket
step = bucketService.generate(file.getBytes(), response);
if (!step) {
throw new NotFoundException("Error a cargar archivo.");
}
step = false;
model.setStatus(StatusBcdEnum.INDEXANDO.getName());
model = bdcControlRepository.save(model);
}
step = false;
model.setStatus(StatusBcdEnum.INDEXANDO.getName());
model = bdcControlRepository.save(model);
//Generacion del modelo
step = knowledgeService.generate(response);
......@@ -76,6 +88,7 @@ public class OrquestadorService {
model.setStatus(StatusBcdEnum.INDEXADO.getName());
model = bdcControlRepository.save(model);
}catch (Exception e) {
System.out.println(e.getMessage());
response.setStatus(FrequentQuestionStatusEnum.ERROR.getName());
}
return response;
......
src/main/java/com/bytesw/bytebot/service/SchedulerTaskService.java
View file @ 53a752db
......@@ -2,6 +2,7 @@ package com.bytesw.bytebot.service;
import com.bytesw.bytebot.bean.CalendarBean;
import com.bytesw.bytebot.bean.SchedulerTaskBean;
import com.bytesw.bytebot.etl.batch.service.ScheduleService;
import com.bytesw.bytebot.model.Calendar;
import com.bytesw.bytebot.model.SchedulerTask;
import com.bytesw.bytebot.repository.CalendarRepository;
......@@ -30,6 +31,9 @@ public class SchedulerTaskService extends XDFService<SchedulerTask, SchedulerTas
@Autowired
private SchedulerTaskRepository schedulerTaskRepository;
@Autowired
private ScheduleService scheduleService;
protected SchedulerTaskService(SchedulerTaskRepository repository) {
super(repository);
}
......@@ -83,6 +87,7 @@ public class SchedulerTaskService extends XDFService<SchedulerTask, SchedulerTas
SchedulerTask schedulerTask = new SchedulerTask();
schedulerTask = toModel(schedulerTask, schedulerTaskBean);
schedulerTask = schedulerTaskRepository.save(schedulerTask);
scheduleService.restartTask(schedulerTask);
System.out.println(schedulerTask);
return toBean(schedulerTask);
}
......@@ -96,6 +101,7 @@ public class SchedulerTaskService extends XDFService<SchedulerTask, SchedulerTas
SchedulerTask schedulerTask = new SchedulerTask();
schedulerTask = toModel(schedulerTask, bean);
schedulerTask = schedulerTaskRepository.save(schedulerTask);
scheduleService.restartTask(schedulerTask);
return toBean(schedulerTask);
}
......@@ -105,6 +111,7 @@ public class SchedulerTaskService extends XDFService<SchedulerTask, SchedulerTas
if (!model.isPresent()) {
throw new NotFoundException();
}
scheduleService.cancelTask(model.get());
schedulerTaskRepository.deleteById(id);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment