Base de conocimiento

pom.xml

@@ -246,6 +246,14 @@
             <artifactId>twilio</artifactId>
             <version>8.22.0</version>
         </dependency>
+
+        <!-- https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk -->
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk</artifactId>
+            <version>1.12.129</version>
+        </dependency>
+
     </dependencies>
 
     <build>

src/main/java/com/bytesw/bytebot/bean/BcdControlBean.java → src/main/java/com/bytesw/bytebot/bean/BdcControlBean.java

@@ -7,8 +7,12 @@ import lombok.Setter;
 import java.io.Serializable;
 
 @Getter @Setter
-public class BcdControlBean implements Serializable {
+public class BdcControlBean implements Serializable {
     @Expose private Long id;
-    @Expose private Long agentId;
+    @Expose private String uuid;
+    @Expose private String fileName;
     @Expose private String status;
+    @Expose private String user;
+    @Expose private String date;
+    @Expose private Long fileId;
 }

src/main/java/com/bytesw/bytebot/controller/AgentController.java

@@ -3,8 +3,7 @@ package com.bytesw.bytebot.controller;
 import com.bytesw.bytebot.bean.AgentBean;
 import com.bytesw.bytebot.controller.bean.ResponseController;
 import com.bytesw.bytebot.http.FileValidationResponse;
-import com.bytesw.bytebot.service.AgentService;
-import com.bytesw.bytebot.service.FileManagementService;
+import com.bytesw.bytebot.service.*;
 import com.bytesw.xdf.annotation.ProgramSecurity;
 import com.bytesw.xdf.sql.beans.Pagination;
 import com.google.gson.Gson;
@@ -38,6 +37,9 @@ public class AgentController {
     @Autowired
     private GsonBuilder gsonBuilder;
 
+    @Autowired
+    private OrquestadorService orquestadorService;
+
     @PostMapping(value = "/page")
     @PreAuthorize("hasPermission(this, 'view')")
     public ResponseEntity<String> paginationConversationalAgent(@RequestBody Pagination<AgentBean> pagination) {
@@ -168,7 +170,20 @@ public class AgentController {
         Gson gson = gsonBuilder.create();
         try {
             String uuid = UUID.randomUUID().toString();
-            FileValidationResponse response = fileManagementService.validateAndSaveFile(uuid, file);
+            FileValidationResponse response = orquestadorService.executeGenerateBCD(uuid, file);
+            response.setUser(principal != null ? principal.getName() : null);
+            return new ResponseEntity<>(gson.toJson(response), HttpStatus.OK);
+        } catch (Exception e) {
+            e.printStackTrace();
+            return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR);
+        }
+    }
+
+    @PutMapping("/file-upload/{uuid}")
+    public ResponseEntity<String> uploadFile(@PathVariable("uuid") String uuid, Principal principal) {
+        Gson gson = gsonBuilder.create();
+        try {
+            FileValidationResponse response = orquestadorService.retry(uuid);
             response.setUser(principal != null ? principal.getName() : null);
             return new ResponseEntity<>(gson.toJson(response), HttpStatus.OK);
         } catch (Exception e) {

src/main/java/com/bytesw/bytebot/controller/BcdControlController.java → src/main/java/com/bytesw/bytebot/controller/BdcControlController.java

 package com.bytesw.bytebot.controller;
 
-import com.bytesw.bytebot.bean.BcdControlBean;
+import com.bytesw.bytebot.bean.BdcControlBean;
 import com.bytesw.bytebot.service.BcdControlService;
 import com.bytesw.xdf.annotation.ProgramSecurity;
 import com.bytesw.xdf.controller.XDFController;
@@ -12,8 +12,8 @@ import org.springframework.web.bind.annotation.RestController;
 @RequestMapping("/service/settings/bcdControl")
 @ProgramSecurity("BCDCONTROL")
 @Log4j2
-public class BcdControlController extends XDFController<BcdControlBean, Long> {
-    public BcdControlController(BcdControlService service) {
+public class BdcControlController extends XDFController<BdcControlBean, Long> {
+    public BdcControlController(BcdControlService service) {
         super(service);
     }
 }

src/main/java/com/bytesw/bytebot/http/FileValidationResponse.java

@@ -8,8 +8,6 @@ package com.bytesw.bytebot.http;
 import lombok.Getter;
 import lombok.Setter;
 
-import java.time.LocalDate;
-
 @Getter @Setter
 public class FileValidationResponse {
     

src/main/java/com/bytesw/bytebot/http/enums/ValidationStatusEnum.java

@@ -11,6 +11,7 @@ import java.util.NoSuchElementException;
 
 public enum ValidationStatusEnum {
     OK("OK"),
+    ERROR("ERROR"),
     INCOMPATIBLE_EXTENSION("IE"),
     HEADER_ERROR ("HE"), 
     CONTENT_ERROR("CE");

src/main/java/com/bytesw/bytebot/model/BcdControl.java → src/main/java/com/bytesw/bytebot/model/BdcControl.java

@@ -10,8 +10,8 @@ import java.io.Serializable;
 @Setter
 @ToString
 @Table(name = "avb_bcd_control")
-@NamedQuery(name = "BcdControl.findByPK", query = "Select u from BcdControl u where u.id = ?1")
-public class BcdControl  implements Serializable {
+@NamedQuery(name = "BdcControl.findByPK", query = "Select u from BdcControl u where u.id = ?1")
+public class BdcControl implements Serializable {
 
     @Id
     @Column(name = "bcd_id")
@@ -19,10 +19,22 @@ public class BcdControl  implements Serializable {
     @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "AVB_BCD_CONTROL_GENERATOR")
     private Long id;
 
-    @Column(name = "agen_id")
-    private Long agentId;
+    @Column(name = "bcd_uuid")
+    private String uuid;
+
+    @Column(name = "bcd_file")
+    private String fileName;
+
+    @Column(name = "bcd_file_id")
+    private Long fileId;
 
     @Column(name = "bcd_status")
     private String status;
 
+    @Column(name = "bcd_user")
+    private String user;
+
+    @Column(name = "bcd_date")
+    private String date;
+
 }

src/main/java/com/bytesw/bytebot/model/enums/FrequentQuestionStatusEnum.java

@@ -21,7 +21,9 @@ public enum FrequentQuestionStatusEnum {
 
     PENDING_SYNCHRONIZED("PS"),
     UPLOADED("LO"),
-    DELETED("DE");
+    DELETED("DE"),
+    ERROR("ER"),
+    CORRECTO("OK");
 
     private static final Map<String, FrequentQuestionStatusEnum> map = new HashMap<>();
 

src/main/java/com/bytesw/bytebot/model/enums/StatusBcdEnum.java

@@ -8,10 +8,10 @@ import java.util.Map;
 @Getter
 public enum StatusBcdEnum {
 
-    CARGANDO("CG"),
-    CARGADO("CGD"),
-    INDEXANDO("IDX"),
-    INDEXADO("IDXD");
+    CARGANDO("Cargando"),
+    CARGADO("Cargado"),
+    INDEXANDO("Indexando"),
+    INDEXADO("Indexado");
 
 
     private static final Map<String, StatusBcdEnum> map = new HashMap<>();

src/main/java/com/bytesw/bytebot/repository/BcdControlRepository.java

-package com.bytesw.bytebot.repository;
-
-import com.bytesw.bytebot.model.BcdControl;
-import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
-import org.springframework.data.repository.CrudRepository;
-
-public interface BcdControlRepository extends CrudRepository<BcdControl, Long>, JpaSpecificationExecutor<BcdControl> {
-}

src/main/java/com/bytesw/bytebot/repository/BdcControlRepository.java

+package com.bytesw.bytebot.repository;
+
+import com.bytesw.bytebot.model.BdcControl;
+import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
+import org.springframework.data.jpa.repository.Query;
+import org.springframework.data.repository.CrudRepository;
+import org.springframework.data.repository.query.Param;
+
+import java.util.Optional;
+
+public interface BdcControlRepository extends CrudRepository<BdcControl, Long>, JpaSpecificationExecutor<BdcControl> {
+    @Query("select s from BdcControl s where s.uuid = :uuid")
+    Optional<BdcControl> findByUuid(@Param("uuid") String uuid);
+}

src/main/java/com/bytesw/bytebot/service/BcdControlService.java

 package com.bytesw.bytebot.service;
 
-import com.bytesw.bytebot.bean.BcdControlBean;
-import com.bytesw.bytebot.model.BcdControl;
-import com.bytesw.bytebot.repository.BcdControlRepository;
+import com.bytesw.bytebot.bean.BdcControlBean;
+import com.bytesw.bytebot.model.BdcControl;
+import com.bytesw.bytebot.repository.BdcControlRepository;
 import com.bytesw.xdf.service.XDFService;
 import lombok.extern.log4j.Log4j2;
 import org.springframework.beans.BeanUtils;
-import org.springframework.data.repository.CrudRepository;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
 @Service
 @Transactional
 @Log4j2
-public class BcdControlService extends XDFService<BcdControl, BcdControlBean, Long> {
-    protected BcdControlService(BcdControlRepository repository) {
+public class BcdControlService extends XDFService<BdcControl, BdcControlBean, Long> {
+    protected BcdControlService(BdcControlRepository repository) {
         super(repository);
     }
 
     @Override
-    protected BcdControl toModel(BcdControl model, BcdControlBean bean) {
+    protected BdcControl toModel(BdcControl model, BdcControlBean bean) {
         if (model == null) {
-            model = new BcdControl();
+            model = new BdcControl();
         }
         BeanUtils.copyProperties(bean, model);
         return model;
     }
 
     @Override
-    protected BcdControlBean toBean(BcdControl model) {
-        BcdControlBean bean = new BcdControlBean();
+    protected BdcControlBean toBean(BdcControl model) {
+        BdcControlBean bean = new BdcControlBean();
         BeanUtils.copyProperties(model, bean);
         return bean;
     }

src/main/java/com/bytesw/bytebot/service/BucketService.java

+package com.bytesw.bytebot.service;
+
+import com.amazonaws.services.s3.model.Bucket;
+import com.bytesw.bytebot.http.FileValidationResponse;
+import com.bytesw.bytebot.model.BusinessParameter;
+import com.bytesw.bytebot.repository.BusinessParameterRepository;
+import com.bytesw.xdf.exception.NotFoundException;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.PutObjectRequest;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.util.List;
+import java.util.Optional;
+
+@Service
+@Log4j2
+public class BucketService {
+
+    @Value("${application.bucket.tenant}")
+    private String tenant;
+
+    @Value("${application.bucket.region}")
+    private String region;
+
+    @Value("${application.bucket.fileBucket}")
+    private String fileDir;
+
+    @Autowired
+    private BusinessParameterRepository businessParameterRepository;
+
+    private AmazonS3 getS3Client() {
+        return AmazonS3ClientBuilder.standard()
+                .withRegion(region)
+                .build();
+    }
+
+    public boolean generate(byte[] file, FileValidationResponse response) {
+        Optional<BusinessParameter> businessParameter = businessParameterRepository.findByKey("Bucket");
+        boolean result = false;
+        if (!businessParameter.isPresent()) {
+            throw new NotFoundException("No existe el parametro de negocio del bucket.");
+        }
+        String bucketName = businessParameter.get().getDefaultValue();
+        File mainFile = new File(response.getFileName());
+        try {
+            FileOutputStream stream = new FileOutputStream(mainFile);
+            stream.write(file);
+            String newFile = String.format("%s/%s/1.0/%s/%s",tenant, response.getUuid(), fileDir, mainFile.getName());
+            PutObjectRequest request = new PutObjectRequest(bucketName, newFile, mainFile);
+            AmazonS3 amazonS3 = getS3Client();
+            amazonS3.putObject(request);
+            mainFile.delete();
+            result = true;
+        } catch (Exception e) {
+            log.error("Error al cargar archivo");
+        }
+        return result;
+    }
+
+    public void list() {
+        AmazonS3 s3 = getS3Client();
+        List<Bucket> buckets = s3.listBuckets();
+        System.out.println("Your {S3} buckets are:");
+        for (Bucket b : buckets) {
+            System.out.println("* " + b.getName());
+        }
+    }
+
+}

src/main/java/com/bytesw/bytebot/service/FileManagementService.java

@@ -5,6 +5,7 @@
  */
 package com.bytesw.bytebot.service;
 
+import com.bytesw.bytebot.etl.model.GoalForActions;
 import com.bytesw.bytebot.http.FileValidationResponse;
 import com.bytesw.bytebot.http.FileValidationResult;
 import com.bytesw.bytebot.http.enums.ValidationStatusEnum;
@@ -16,12 +17,9 @@ import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
+
+import com.bytesw.xdf.exception.NotFoundException;
 import lombok.extern.log4j.Log4j2;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.poi.ss.usermodel.Cell;
@@ -282,4 +280,16 @@ public class FileManagementService {
         }
         return isFileHeader;
     }
+
+    @Transactional(propagation = Propagation.REQUIRED)
+    public void deleteGoalForAction(Long id) {
+        Optional<QuestionFile> file = questionFileRepository.findById(id);
+
+        if (!file.isPresent()) {
+            throw new NotFoundException("File no econtrado.");
+        }
+
+        questionFileRepository.delete(file.get());
+    }
+
 }

src/main/java/com/bytesw/bytebot/service/KnowledgeService.java

+package com.bytesw.bytebot.service;
+
+import com.bytesw.bytebot.http.FileValidationResponse;
+import lombok.extern.log4j.Log4j2;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+
+@Service
+@Log4j2
+public class KnowledgeService {
+
+    @Value("${application.bucket.tenant}")
+    private String tenant;
+
+    @Value("${application.knowledge.url}")
+    private String url;
+
+    public void generate(FileValidationResponse response) throws IOException {
+        CloseableHttpClient httpclient = HttpClients.createDefault();
+        HttpPost httppost = new HttpPost(String.format("%s/clients/%s/versions/1.0", url, tenant));
+        String json = String.format("{\"filename_csv\": \"%s\", \"index_name\": \"%s\"}",response.getFileName(), response.getUuid());
+        StringEntity entity = new StringEntity(json);
+        httppost.setEntity(entity);
+        httppost.setHeader("Accept", "application/json");
+        httppost.setHeader("Content-type", "application/json");
+        CloseableHttpResponse res = httpclient.execute(httppost);
+        httpclient.close();
+    }
+
+    public void delete() {
+        HttpPost httpPost = new HttpPost();
+    }
+
+}

src/main/java/com/bytesw/bytebot/service/OrquestadorService.java

 package com.bytesw.bytebot.service;
 
+import com.bytesw.bytebot.http.FileValidationResponse;
+import com.bytesw.bytebot.http.FileValidationResult;
+import com.bytesw.bytebot.http.enums.ValidationStatusEnum;
+import com.bytesw.bytebot.model.BdcControl;
+import com.bytesw.bytebot.model.QuestionFile;
+import com.bytesw.bytebot.model.enums.FrequentQuestionStatusEnum;
+import com.bytesw.bytebot.model.enums.StatusBcdEnum;
+import com.bytesw.bytebot.repository.BdcControlRepository;
+import com.bytesw.bytebot.repository.QuestionFileRepository;
+import com.bytesw.xdf.exception.NotFoundException;
 import lombok.extern.log4j.Log4j2;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
+import org.springframework.web.multipart.MultipartFile;
+
+import java.io.IOException;
+import java.util.Optional;
 
 @Service
 @Transactional
 @Log4j2
 public class OrquestadorService {
+
+    @Autowired
+    private FileManagementService fileManagementService;
+
+    @Autowired
+    private BucketService bucketService;
+
+    @Autowired
+    private BdcControlRepository bdcControlRepository;
+
+    @Autowired
+    private QuestionFileRepository questionFileRepository;
+
+    @Autowired
+    private  KnowledgeService knowledgeService;
+
+
+    public FileValidationResponse executeGenerateBCD(String uuid, MultipartFile file) {
+        BdcControl model = new BdcControl();
+        FileValidationResponse response = new FileValidationResponse();
+        boolean step = false;
+        try{
+            //Valida y guarda en base de datos
+            response = fileManagementService.validateAndSaveFile(uuid,file);
+            if(response.getFileValidationResult().getStatus().getName() != ValidationStatusEnum.OK.getName()) {
+                response.setStatus(FrequentQuestionStatusEnum.ERROR.getName());
+                return response;
+            }
+            model.setUuid(response.getUuid());
+            model.setFileId(response.getId());
+            model.setFileName(response.getFileName());
+            model.setDate(response.getUploadDate());
+            model.setStatus(StatusBcdEnum.CARGADO.getName());
+            model = bdcControlRepository.save(model);
+
+            //Carga archivo a bucket
+            step = bucketService.generate(file.getBytes(), response);
+            if (!step) {
+                throw new NotFoundException("Error a cargar archivo.");
+            }
+            model.setStatus(StatusBcdEnum.INDEXANDO.getName());
+            model = bdcControlRepository.save(model);
+
+            //Generacion del modelo
+            knowledgeService.generate(response);
+            bdcControlRepository.delete(model);
+            response.setStatus(FrequentQuestionStatusEnum.CORRECTO.getName());
+        }catch (Exception e) {
+            response.setStatus(FrequentQuestionStatusEnum.ERROR.getName());
+        }
+        return response;
+    }
+
+    public FileValidationResponse retry(String uuid) {
+        Optional<BdcControl> modelBdc = bdcControlRepository.findByUuid(uuid);
+        if (!modelBdc.isPresent()) {
+            throw new NotFoundException("No se subio archivo a la BD");
+        }
+        boolean step = false;
+        BdcControl model = modelBdc.get();
+        FileValidationResponse response = modelToFileValidation(model);
+        try {
+            if (model.getStatus().equals(StatusBcdEnum.CARGADO.getName())) {
+                Optional<QuestionFile> questionFile = questionFileRepository.findByUuid(uuid);
+                if (!questionFile.isPresent()) {
+                    throw new NotFoundException("No se encontro archivo en la BD");
+                }
+                step = bucketService.generate(questionFile.get().getData(), response);
+                if (!step) {
+                    throw new NotFoundException("Error a cargar archivo.");
+                }
+                model.setStatus(StatusBcdEnum.INDEXANDO.getName());
+                model = bdcControlRepository.save(model);
+            }
+            if (model.getStatus().equals(StatusBcdEnum.INDEXANDO.getName())) {
+                knowledgeService.generate(response);
+            }
+            bdcControlRepository.delete(model);
+            response.setStatus(FrequentQuestionStatusEnum.CORRECTO.getName());
+        } catch (IOException e) {
+            response.setStatus(FrequentQuestionStatusEnum.ERROR.getName());
+        }
+        return response;
+    }
+
+    public void deleteBcd(String uuid) {
+
+    }
+
+    private FileValidationResponse modelToFileValidation(BdcControl model){
+        FileValidationResponse response = new FileValidationResponse();
+        response.setId(model.getId());
+        response.setStatus(model.getStatus());
+        response.setUuid(model.getUuid());
+        response.setFileName(model.getFileName());
+        response.setUploadDate(model.getDate());
+        FileValidationResult result = new FileValidationResult();
+        result.setStatus(ValidationStatusEnum.OK);
+        response.setFileValidationResult(result);
+        return response;
+    }
+
 }