diff --git a/build.gradle b/build.gradle index 6c08a49..d024006 100644 --- a/build.gradle +++ b/build.gradle @@ -55,6 +55,11 @@ dependencies { // pdfbox implementation 'org.apache.pdfbox:pdfbox:2.0.30' + // json 파싱 + implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.2' + + // MockMultipartFile 사용을 위한 의존성 추가 + implementation 'org.springframework.boot:spring-boot-starter-test' // lombok compileOnly 'org.projectlombok:lombok' diff --git a/src/main/java/Sprout_Squad/EyeOn/domain/document/entity/Document.java b/src/main/java/Sprout_Squad/EyeOn/domain/document/entity/Document.java index 49f564f..f1599d0 100644 --- a/src/main/java/Sprout_Squad/EyeOn/domain/document/entity/Document.java +++ b/src/main/java/Sprout_Squad/EyeOn/domain/document/entity/Document.java @@ -47,11 +47,16 @@ public class Document extends BaseEntity { @Column(name = "document_url", nullable = false) private String documentUrl; - public static Document toEntity(DocumentType documentType, String imageUrl, String documentUrl, Form form, User user){ + public void modifyDocument(String documentImageUrl, String documentUrl) { + this.documentImageUrl = documentImageUrl; + this.documentUrl = documentUrl; + } + + public static Document toEntity(DocumentType documentType, String imageUrl, String documentUrl, Form form, String fileName, User user){ return Document.builder() .user(user) .form(form) - .documentName(form.getName()) // 양식 이름과 동일하게 사용 + .documentName(fileName) // 양식 이름과 동일하게 사용 .documentImageUrl(imageUrl) // 작성된 문서 이미지 형태 .documentUrl(documentUrl) // pdf 형태 .documentType(documentType) diff --git a/src/main/java/Sprout_Squad/EyeOn/domain/document/service/DocumentService.java b/src/main/java/Sprout_Squad/EyeOn/domain/document/service/DocumentService.java index b29b920..3fdead6 100644 --- a/src/main/java/Sprout_Squad/EyeOn/domain/document/service/DocumentService.java +++ b/src/main/java/Sprout_Squad/EyeOn/domain/document/service/DocumentService.java @@ -2,6 +2,8 @@ import Sprout_Squad.EyeOn.domain.document.web.dto.*; import Sprout_Squad.EyeOn.global.auth.jwt.UserPrincipal; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.multipart.MultipartFile; import java.io.IOException; import java.util.List; @@ -11,4 +13,8 @@ public interface DocumentService { List getAllDocuments(UserPrincipal userPrincipal); GetSummaryRes getSummary(UserPrincipal userPrincipal, Long id); WriteDocsRes writeDocument(UserPrincipal userPrincipal, Long formId, List writeDocsReqList) throws IOException; + UploadDocumentRes uploadDocument(UserPrincipal userPrincipal, MultipartFile file) throws IOException; + WriteDocsRes rewriteDocument(UserPrincipal userPrincipal, Long documentId, ModifyDocumentReqWrapper modifyDocumentReqWrapper) throws IOException; + List getAdvice(UserPrincipal userPrincipal, Long id) throws IOException; + } diff --git a/src/main/java/Sprout_Squad/EyeOn/domain/document/service/DocumentServiceImpl.java b/src/main/java/Sprout_Squad/EyeOn/domain/document/service/DocumentServiceImpl.java index 467f101..0fdb1b7 100644 --- a/src/main/java/Sprout_Squad/EyeOn/domain/document/service/DocumentServiceImpl.java +++ b/src/main/java/Sprout_Squad/EyeOn/domain/document/service/DocumentServiceImpl.java @@ -10,15 +10,27 @@ import Sprout_Squad.EyeOn.domain.user.repository.UserRepository; import Sprout_Squad.EyeOn.global.auth.exception.CanNotAccessException; import Sprout_Squad.EyeOn.global.auth.jwt.UserPrincipal; +import Sprout_Squad.EyeOn.global.converter.ImgConverter; +import Sprout_Squad.EyeOn.global.external.exception.UnsupportedFileTypeException; import Sprout_Squad.EyeOn.global.external.service.OpenAiService; import Sprout_Squad.EyeOn.global.external.service.PdfService; import Sprout_Squad.EyeOn.global.external.service.S3Service; +import Sprout_Squad.EyeOn.global.flask.dto.GetFieldForModifyRes; +import Sprout_Squad.EyeOn.global.flask.dto.GetModelResForModify; +import Sprout_Squad.EyeOn.global.flask.mapper.FieldLabelMapper; +import Sprout_Squad.EyeOn.global.flask.service.FlaskService; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import lombok.RequiredArgsConstructor; +import org.springframework.mock.web.MockMultipartFile; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.multipart.MultipartFile; import java.io.IOException; +import java.io.InputStream; import java.util.List; +import java.util.stream.Collectors; @Service @RequiredArgsConstructor @@ -29,6 +41,8 @@ public class DocumentServiceImpl implements DocumentService { private final OpenAiService openAiService; private final PdfService pdfService; private final FormRepository formRepository; + private final FlaskService flaskService; + private final FieldLabelMapper fieldLabelMapper; /** * 사용자의 문서 하나 상세 조회 @@ -112,15 +126,145 @@ public WriteDocsRes writeDocument(UserPrincipal userPrincipal, Long formId, List // S3에 pdf 업로드 String fileName = s3Service.generatePdfFileName(); + int dotIndex = fileName.lastIndexOf('.'); + String nameOnly = (dotIndex != -1) ? fileName.substring(0, dotIndex) : fileName; String pdfUrl = s3Service.uploadPdfBytes(fileName, imgToPdf); DocumentType documentType = DocumentType.valueOf(form.getFormType().name()); - Document document = Document.toEntity(documentType, imgUrl, pdfUrl, form, user); + Document document = Document.toEntity(documentType, imgUrl, pdfUrl, form, nameOnly, user); documentRepository.save(document); return WriteDocsRes.from(document); + } + + /** + * 문서 업로드 + */ + @Override + @Transactional + public UploadDocumentRes uploadDocument(UserPrincipal userPrincipal, MultipartFile file) throws IOException { + // 사용자가 존재하지 않을 경우 -> UserNotFoundException + User user = userRepository.getUserById(userPrincipal.getId()); + + String extension = pdfService.getFileExtension(file.getOriginalFilename()).toLowerCase(); + + String fileUrl; + String fileName; + + if (extension.equals("pdf")) { + // PDF -> 이미지 변환 + byte[] pdfBytes = file.getBytes(); + fileUrl = pdfService.convertPdfToImage(pdfBytes); + fileName = s3Service.extractKeyFromUrl(fileUrl); // 변환된 이미지의 S3 key + } else if (extension.equals("jpg") || extension.equals("jpeg") || extension.equals("png")) { + fileName = s3Service.generateFileName(file); + fileUrl = s3Service.uploadFile(fileName, file); + } else { + throw new UnsupportedFileTypeException(); + } + + // 플라스크 서버와 통신하여 파일 유형 받아옴 + DocumentType documentType = DocumentType.from(flaskService.detectType(file, fileName)); + + Document document = Document.toEntity(documentType, fileUrl, fileUrl, null, fileName, user); + documentRepository.save(document); + return UploadDocumentRes.of(document, s3Service.getSize(fileUrl)); + } + + /** + * 문서 수정 + */ + @Override + @Transactional + public WriteDocsRes rewriteDocument(UserPrincipal userPrincipal, Long documentId, + ModifyDocumentReqWrapper modifyDocumentReqWrapper) throws IOException { + // 사용자가 존재하지 않을 경우 -> UserNotFoundException + User user = userRepository.getUserById(userPrincipal.getId()); + + // 문서가 존재하지 않을 경우 -> DocumentNotFoundException + Document document = documentRepository.getDocumentsByDocumentId(documentId); + + // 사용자의 문서가 아닐 경우 -> CanNotAccessException + if(document.getUser() != user) throw new CanNotAccessException(); + + // 실제 문서를 가져옴 + InputStream file = s3Service.downloadFile(document.getDocumentImageUrl()); + + // InputStream을 MultipartFile로 변환 + MultipartFile multipartFile = new MockMultipartFile( + document.getDocumentName(), + document.getDocumentName(), + "application/octet-stream", + file + ); + + + String fileExt = pdfService.getFileExtension(multipartFile.getOriginalFilename()); + + // 모델로부터 작성된 문서의 OCR 및 라벨링 결과를 가져옴 + String res = flaskService.getModifyLabelReq(ImgConverter.toBase64(multipartFile), fileExt); + + // 수정 요청과 비교하여 WritDocsReqList 구성 + WriteDocsReqWrapper writeDocsReqList = flaskService.getModifyRes(res, modifyDocumentReqWrapper); + + // 이미지 url (이미지 덮어쓰기 및 재작성) + String imgUrl = pdfService.rewriteImg(document.getDocumentImageUrl(), writeDocsReqList.data()); + + // pdf + byte[] imgToPdf = pdfService.convertImageToPdf(s3Service.downloadFile(imgUrl)); + + // S3에 pdf 업로드 + String fileName = s3Service.generatePdfFileName(); + String pdfUrl = s3Service.uploadPdfBytes(fileName, imgToPdf); + + document.modifyDocument(imgUrl, pdfUrl); + return WriteDocsRes.from(document); + } + + /** + * 문서 조언 + */ + @Override + public List getAdvice(UserPrincipal userPrincipal, Long id) throws IOException { + // 사용자가 존재하지 않을 경우 -> UserNotFoundException + User user = userRepository.getUserById(userPrincipal.getId()); + + // 문서가 존재하지 않을 경우 -> DocumentNotFoundException + Document document = documentRepository.getDocumentsByDocumentId(id); + + // 사용자의 문서가 아닐 경우 -> CanNotAccessException + if(document.getUser() != user) throw new CanNotAccessException(); + InputStream file = s3Service.downloadFile(document.getDocumentImageUrl()); + + MultipartFile multipartFile = new MockMultipartFile( + document.getDocumentName(), + document.getDocumentName(), + "application/octet-stream", + file + ); + + GetModelResForModify getModelResForModify = flaskService.getResFromModelForModify(multipartFile, document.getDocumentName()); + + // 가공하기 좋은 형태로 변환 + List getFieldForModifyResList = flaskService.getFieldForModify(getModelResForModify); + System.out.println("반환: " + getFieldForModifyResList); + + // GetFieldForModifyRes → GetAdviceReq로 변환 + List adviceReqList = getFieldForModifyResList.stream() + .map(field -> new GetAdviceReq( + field.index(), // i: index + field.displayName(), // d: displayName + field.value() // v: value + )) + .collect(Collectors.toList()); + + System.out.println("reqList" + adviceReqList); + String result = openAiService.getModifyAnalyzeFromOpenAi(adviceReqList, document.getDocumentType()); + ObjectMapper objectMapper = new ObjectMapper(); + // JSON 배열의 각 요소를 GetAdviceRes 객체로 변환하여 리스트로 모음 + return objectMapper.readValue(result, new TypeReference>() {}); } diff --git a/src/main/java/Sprout_Squad/EyeOn/domain/document/web/controller/DocumentController.java b/src/main/java/Sprout_Squad/EyeOn/domain/document/web/controller/DocumentController.java index ada939a..a431d4b 100644 --- a/src/main/java/Sprout_Squad/EyeOn/domain/document/web/controller/DocumentController.java +++ b/src/main/java/Sprout_Squad/EyeOn/domain/document/web/controller/DocumentController.java @@ -1,16 +1,17 @@ package Sprout_Squad.EyeOn.domain.document.web.controller; import Sprout_Squad.EyeOn.domain.document.service.DocumentService; -import Sprout_Squad.EyeOn.domain.document.web.dto.GetDocumentRes; -import Sprout_Squad.EyeOn.domain.document.web.dto.GetSummaryRes; -import Sprout_Squad.EyeOn.domain.document.web.dto.WriteDocsRes; -import Sprout_Squad.EyeOn.domain.document.web.dto.WriteDocsReqWrapper; +import Sprout_Squad.EyeOn.domain.document.web.dto.*; import Sprout_Squad.EyeOn.global.auth.jwt.UserPrincipal; +import Sprout_Squad.EyeOn.global.external.service.PdfService; +import Sprout_Squad.EyeOn.global.external.service.S3Service; +import Sprout_Squad.EyeOn.global.flask.service.FlaskService; import Sprout_Squad.EyeOn.global.response.SuccessResponse; import lombok.RequiredArgsConstructor; import org.springframework.http.ResponseEntity; import org.springframework.security.core.annotation.AuthenticationPrincipal; import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; import java.io.IOException; import java.util.List; @@ -20,7 +21,19 @@ @RequestMapping("/api/document") public class DocumentController { private final DocumentService documentService; + private final FlaskService flaskService; + private final PdfService pdfService; + private final S3Service s3Service; + // 문서 업로드 + @PostMapping + public ResponseEntity> uploadDocument( + @AuthenticationPrincipal UserPrincipal userPrincipal, @RequestPart("file") MultipartFile file) throws IOException { + UploadDocumentRes uploadDocumentRes = documentService.uploadDocument(userPrincipal, file); + return ResponseEntity.ok(SuccessResponse.from(uploadDocumentRes)); + } + + // 문서 상세 조회 @GetMapping("/{documentId}/detail") public ResponseEntity> getDocumentDetail( @AuthenticationPrincipal UserPrincipal userPrincipal, @PathVariable Long documentId) { @@ -28,6 +41,7 @@ public ResponseEntity> getDocumentDetail( return ResponseEntity.ok(SuccessResponse.from(getDocumentRes)); } + // 문서 리스트 조회 @GetMapping("/list") public ResponseEntity>> getDocumentList( @AuthenticationPrincipal UserPrincipal userPrincipal) { @@ -35,6 +49,7 @@ public ResponseEntity>> getDocumentList( return ResponseEntity.ok(SuccessResponse.from(getDocumentResList)); } + // 문서 요약 @GetMapping("/{documentId}/summary") public ResponseEntity> getDocumentSummary( @AuthenticationPrincipal UserPrincipal userPrincipal, @PathVariable Long documentId){ @@ -42,6 +57,7 @@ public ResponseEntity> getDocumentSummary( return ResponseEntity.ok(SuccessResponse.from(getSummaryRes)); } + // 문서 작성 @PostMapping("/{formId}/write") public ResponseEntity> writeDocument( @AuthenticationPrincipal UserPrincipal userPrincipal, @@ -49,4 +65,21 @@ public ResponseEntity> writeDocument( WriteDocsRes writeDocsRes = documentService.writeDocument(userPrincipal, formId, writeDocsReqWrapper.data()); return ResponseEntity.ok(SuccessResponse.from(writeDocsRes)); } + + // 문서 수정 + @PutMapping("/{documentId}") + public ResponseEntity> rewriteDocument( + @AuthenticationPrincipal UserPrincipal userPrincipal, + @PathVariable Long documentId, @RequestBody ModifyDocumentReqWrapper modifyDocumentReqWrapper) throws IOException { + WriteDocsRes writeDocsRes = documentService.rewriteDocument(userPrincipal, documentId, modifyDocumentReqWrapper); + return ResponseEntity.ok(SuccessResponse.from(writeDocsRes)); + } + + // 문서 조언 + @GetMapping("/{documentId}/advice") + public ResponseEntity> getDocumentAdvice( + @AuthenticationPrincipal UserPrincipal userPrincipal, @PathVariable Long documentId) throws IOException { + List getAdviceResList=documentService.getAdvice(userPrincipal, documentId); + return ResponseEntity.ok(SuccessResponse.from(getAdviceResList)); + } } \ No newline at end of file diff --git a/src/main/java/Sprout_Squad/EyeOn/domain/document/web/dto/GetAdviceReq.java b/src/main/java/Sprout_Squad/EyeOn/domain/document/web/dto/GetAdviceReq.java new file mode 100644 index 0000000..a66767b --- /dev/null +++ b/src/main/java/Sprout_Squad/EyeOn/domain/document/web/dto/GetAdviceReq.java @@ -0,0 +1,8 @@ +package Sprout_Squad.EyeOn.domain.document.web.dto; + +public record GetAdviceReq( + int i, // index + String d, // displayName + String v // value +) { +} diff --git a/src/main/java/Sprout_Squad/EyeOn/domain/document/web/dto/GetAdviceRes.java b/src/main/java/Sprout_Squad/EyeOn/domain/document/web/dto/GetAdviceRes.java new file mode 100644 index 0000000..a4dde64 --- /dev/null +++ b/src/main/java/Sprout_Squad/EyeOn/domain/document/web/dto/GetAdviceRes.java @@ -0,0 +1,9 @@ +package Sprout_Squad.EyeOn.domain.document.web.dto; + +public record GetAdviceRes( + int i, // index + String d, // displayName + String v, // value + String a // advice +) { +} diff --git a/src/main/java/Sprout_Squad/EyeOn/domain/document/web/dto/ModifyDocumentReq.java b/src/main/java/Sprout_Squad/EyeOn/domain/document/web/dto/ModifyDocumentReq.java new file mode 100644 index 0000000..1d9179b --- /dev/null +++ b/src/main/java/Sprout_Squad/EyeOn/domain/document/web/dto/ModifyDocumentReq.java @@ -0,0 +1,7 @@ +package Sprout_Squad.EyeOn.domain.document.web.dto; + +public record ModifyDocumentReq( + int i, // index + String v // value +) { +} diff --git a/src/main/java/Sprout_Squad/EyeOn/domain/document/web/dto/ModifyDocumentReqWrapper.java b/src/main/java/Sprout_Squad/EyeOn/domain/document/web/dto/ModifyDocumentReqWrapper.java new file mode 100644 index 0000000..a8cfa3a --- /dev/null +++ b/src/main/java/Sprout_Squad/EyeOn/domain/document/web/dto/ModifyDocumentReqWrapper.java @@ -0,0 +1,8 @@ +package Sprout_Squad.EyeOn.domain.document.web.dto; + +import java.util.List; + +public record ModifyDocumentReqWrapper( + List data +) { +} diff --git a/src/main/java/Sprout_Squad/EyeOn/domain/document/web/dto/UploadDocumentRes.java b/src/main/java/Sprout_Squad/EyeOn/domain/document/web/dto/UploadDocumentRes.java new file mode 100644 index 0000000..658c04d --- /dev/null +++ b/src/main/java/Sprout_Squad/EyeOn/domain/document/web/dto/UploadDocumentRes.java @@ -0,0 +1,14 @@ +package Sprout_Squad.EyeOn.domain.document.web.dto; + +import Sprout_Squad.EyeOn.domain.document.entity.Document; + +public record UploadDocumentRes( + Long documentId, + String name, + long documentSize, + String documentUrl +) { + public static UploadDocumentRes of(Document document, long size) { + return new UploadDocumentRes(document.getId(), document.getDocumentName(), size, document.getDocumentUrl()); + } +} diff --git a/src/main/java/Sprout_Squad/EyeOn/domain/form/service/FormService.java b/src/main/java/Sprout_Squad/EyeOn/domain/form/service/FormService.java index 23e23c7..bda23e7 100644 --- a/src/main/java/Sprout_Squad/EyeOn/domain/form/service/FormService.java +++ b/src/main/java/Sprout_Squad/EyeOn/domain/form/service/FormService.java @@ -3,6 +3,7 @@ import Sprout_Squad.EyeOn.domain.form.entity.enums.FormType; import Sprout_Squad.EyeOn.domain.form.web.dto.*; import Sprout_Squad.EyeOn.global.auth.jwt.UserPrincipal; +import Sprout_Squad.EyeOn.global.flask.dto.GetModelRes; import org.springframework.web.multipart.MultipartFile; import java.io.IOException; @@ -13,6 +14,4 @@ public interface FormService { GetFormRes getOneForm(UserPrincipal userPrincipal, Long formId); List getAllFormsByType(UserPrincipal userPrincipal, FormType formType); FormType getFormType(MultipartFile file, String fileName); - GetModelRes getResFromModel(MultipartFile file, String fileName); - List getField(GetModelRes getModelRes, UserPrincipal userPrincipal); } \ No newline at end of file diff --git a/src/main/java/Sprout_Squad/EyeOn/domain/form/service/FormServiceImpl.java b/src/main/java/Sprout_Squad/EyeOn/domain/form/service/FormServiceImpl.java index 3320aa0..c58094d 100644 --- a/src/main/java/Sprout_Squad/EyeOn/domain/form/service/FormServiceImpl.java +++ b/src/main/java/Sprout_Squad/EyeOn/domain/form/service/FormServiceImpl.java @@ -8,28 +8,17 @@ import Sprout_Squad.EyeOn.domain.user.repository.UserRepository; import Sprout_Squad.EyeOn.global.auth.exception.CanNotAccessException; import Sprout_Squad.EyeOn.global.auth.jwt.UserPrincipal; -import Sprout_Squad.EyeOn.global.converter.ImgConverter; -import Sprout_Squad.EyeOn.global.flask.exception.GetLabelFailedException; -import Sprout_Squad.EyeOn.global.flask.exception.TypeDetectedFiledException; -import Sprout_Squad.EyeOn.global.flask.mapper.FieldLabelMapper; import Sprout_Squad.EyeOn.global.flask.service.FlaskService; import Sprout_Squad.EyeOn.global.external.exception.UnsupportedFileTypeException; import Sprout_Squad.EyeOn.global.external.service.PdfService; import Sprout_Squad.EyeOn.global.external.service.S3Service; -import com.fasterxml.jackson.databind.ObjectMapper; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; -import javax.imageio.ImageIO; -import java.awt.image.BufferedImage; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; -import java.util.Map; @Service @RequiredArgsConstructor @@ -39,128 +28,14 @@ public class FormServiceImpl implements FormService { private final S3Service s3Service; private final PdfService pdfService; private final FlaskService flaskService; - private final FieldLabelMapper fieldLabelMapper; /** * 양식 판별 */ @Override public FormType getFormType(MultipartFile file, String fileName) { - try { - String fileToBase64 = ImgConverter.toBase64(file); - - String fileExtension = pdfService.getFileExtension(fileName); - String type = flaskService.detectType(fileToBase64, fileExtension); - - return FormType.from(type); - - } catch (Exception e){ throw new TypeDetectedFiledException(); } - } - - /** - * 양식 필드 분석 - */ - @Override - public GetModelRes getResFromModel(MultipartFile file, String fileName){ - try { - String fileToBase64 = ImgConverter.toBase64(file); - String fileExtension = pdfService.getFileExtension(fileName); - - // 플라스크 요청 - String jsonRes = flaskService.getLabel(fileToBase64, fileExtension); - - // jsonRes를 파싱 - ObjectMapper mapper = new ObjectMapper(); - Map resultMap = mapper.readValue(jsonRes, Map.class); - - // tokens: 문서에서 추출된 텍스트 조각 - List tokens = (List) resultMap.get("tokens"); -// System.out.println("문서 tokens: "+ tokens); - - // labels: 각 토큰의 labels - List labels = (List) resultMap.get("labels"); -// System.out.println("문서 labels: "+labels); - - // doctype: 문서 유형 - String doctype = (String) resultMap.get("doctype"); - - // rawBboxes: 각 토큰의 bounding box 좌표 정보 - List> rawBboxes = (List>) resultMap.get("bboxes"); - - // bboxes: 각 토큰의 bounding box 좌표 정보 - List> bboxes = new ArrayList<>(); - - // List 형태로 파싱된 좌표 데이터를 List로 변환 - for (List box : rawBboxes) { - List convertedBox = box.stream() - .map(val -> ((Number) val).doubleValue()) // JSON 파싱 결과가 Number 타입 -> 형변환 필요 - .toList(); - bboxes.add(convertedBox); - } - return GetModelRes.of(doctype, tokens, bboxes, labels); - } catch (Exception e){ throw new GetLabelFailedException();} - - } - - /** - * db에 있는 정보를 채워서 반환 - */ - public List getField(GetModelRes getModelRes, UserPrincipal userPrincipal){ -// System.out.println("GetModelRes: " +getModelRes); - - // 사용자가 존재하지 않을 경우 -> UserNotFoundException - User user = userRepository.getUserById(userPrincipal.getId()); - - List tokens = getModelRes.tokens(); - List labels = getModelRes.labels(); - List> bboxes = getModelRes.bboxes(); - - String docType = getModelRes.doctype(); - - List userInputs = new ArrayList<>(); - - // context별 전체 label map 불러오기 - Map> contextLabelMap = fieldLabelMapper.getContextualLabels(); - - // 현재 문서 타입에 해당하는 label 맵 - Map labelMap = contextLabelMap.getOrDefault(docType, Collections.emptyMap()); - Map commonMap = contextLabelMap.getOrDefault("common", Collections.emptyMap()); - - for (int i = 0; i < tokens.size(); i++) { - if ("[BLANK]".equals(tokens.get(i))) { - String label = labels.get(i); - - if (label.endsWith("-FIELD")) { // label이 -FIELD 로 끝나면 - String baseLabel = label.replace("-FIELD", ""); - - // displayName 우선순위: 1) 문서 타입 별 Map → 2) 공통 Map → 3) 기본값 - String displayName = labelMap.getOrDefault(baseLabel, - commonMap.getOrDefault(baseLabel, baseLabel)); - - // 사용자 정보에서 가져올 수 있는 값 매핑 - String value = switch (baseLabel) { - case "B-PERSONAL-NAME", "B-GRANTOR-NAME", "B-DELEGATE-NAME", "B-NAME", "B-SIGN-NAME" -> user.getName(); - case "B-PERSONAL-RRN", "B-GRANTOR-RRN", "B-DELEGATE-RRN" -> user.getResidentNumber(); - case "B-PERSONAL-PHONE", "B-GRANTOR-PHONE", "B-DELEGATE-PHONE" -> user.getPhoneNumber(); - case "B-PERSONAL-ADDR", "B-GRANTOR-ADDR", "B-DELEGATE-ADDR" -> user.getAddress(); - case "B-PERSONAL-EMAIL" -> user.getEmail(); - default -> null; // 나머지는 프론트로부터 값 받기 - }; - - List bbox = bboxes.get(i); - - userInputs.add(new GetFieldRes( - baseLabel, - label, - i, - bbox, - displayName, - value - )); - } - } - } - return userInputs; + String type = flaskService.detectType(file, fileName); + return FormType.from(type); } /** @@ -177,7 +52,6 @@ public UploadFormRes uploadForm(UserPrincipal userPrincipal, MultipartFile file) String fileUrl; String fileName; -// System.out.println("파일 확장자 : " + extension); if (extension.equals("pdf")) { // PDF -> 이미지 변환 byte[] pdfBytes = file.getBytes(); @@ -191,9 +65,6 @@ public UploadFormRes uploadForm(UserPrincipal userPrincipal, MultipartFile file) throw new UnsupportedFileTypeException(); } -// System.out.println("fileName : " + fileName); -// System.out.println("fileUrl : " + fileUrl); - // 플라스크 서버와 통신하여 파일 유형 받아옴 FormType formType = getFormType(file, fileName); diff --git a/src/main/java/Sprout_Squad/EyeOn/domain/form/web/controller/FormController.java b/src/main/java/Sprout_Squad/EyeOn/domain/form/web/controller/FormController.java index 4953463..2dd8e7b 100644 --- a/src/main/java/Sprout_Squad/EyeOn/domain/form/web/controller/FormController.java +++ b/src/main/java/Sprout_Squad/EyeOn/domain/form/web/controller/FormController.java @@ -2,11 +2,12 @@ import Sprout_Squad.EyeOn.domain.form.entity.enums.FormType; import Sprout_Squad.EyeOn.domain.form.service.FormService; -import Sprout_Squad.EyeOn.domain.form.web.dto.GetFieldRes; -import Sprout_Squad.EyeOn.domain.form.web.dto.GetModelRes; +import Sprout_Squad.EyeOn.global.flask.dto.GetFieldForWriteRes; +import Sprout_Squad.EyeOn.global.flask.dto.GetModelRes; import Sprout_Squad.EyeOn.domain.form.web.dto.GetFormRes; import Sprout_Squad.EyeOn.domain.form.web.dto.UploadFormRes; import Sprout_Squad.EyeOn.global.auth.jwt.UserPrincipal; +import Sprout_Squad.EyeOn.global.flask.service.FlaskService; import Sprout_Squad.EyeOn.global.response.SuccessResponse; import lombok.RequiredArgsConstructor; import org.springframework.http.ResponseEntity; @@ -22,6 +23,7 @@ @RequestMapping("/api/form") public class FormController { private final FormService formService; + private final FlaskService flaskService; @PostMapping public ResponseEntity> uploadForm(@AuthenticationPrincipal UserPrincipal userPrincipal, @@ -31,15 +33,14 @@ public ResponseEntity> uploadForm(@Authentication } @PostMapping("/analyze/field") - public ResponseEntity>> getFormField(@AuthenticationPrincipal UserPrincipal userPrincipal, - @RequestPart("file") MultipartFile file) { + public ResponseEntity>> getFormField(@AuthenticationPrincipal UserPrincipal userPrincipal, + @RequestPart("file") MultipartFile file) { String fileName = file.getOriginalFilename(); - GetModelRes getModelRes = formService.getResFromModel(file, fileName); - System.out.println("1차 통과 : " + getModelRes); - List getFieldResList = formService.getField(getModelRes, userPrincipal); + GetModelRes getModelRes = flaskService.getResFromModelForWrite(file, fileName); + List getFieldForWriteResList = flaskService.getFieldForWrite(getModelRes, userPrincipal); - return ResponseEntity.ok(SuccessResponse.from(getFieldResList)); + return ResponseEntity.ok(SuccessResponse.from(getFieldForWriteResList)); } @GetMapping("/{formId}/detail") diff --git a/src/main/java/Sprout_Squad/EyeOn/domain/form/web/dto/GetModelRes.java b/src/main/java/Sprout_Squad/EyeOn/domain/form/web/dto/GetModelRes.java deleted file mode 100644 index b26f6fc..0000000 --- a/src/main/java/Sprout_Squad/EyeOn/domain/form/web/dto/GetModelRes.java +++ /dev/null @@ -1,14 +0,0 @@ -package Sprout_Squad.EyeOn.domain.form.web.dto; - -import java.util.List; - -public record GetModelRes( - String doctype, - List tokens, - List> bboxes, - List labels -) { - public static GetModelRes of(String doctype, List tokens, List> bboxes, List labels) { - return new GetModelRes(doctype, tokens, bboxes, labels); - } -} \ No newline at end of file diff --git a/src/main/java/Sprout_Squad/EyeOn/global/exception/GlobalExceptionHandler.java b/src/main/java/Sprout_Squad/EyeOn/global/exception/GlobalExceptionHandler.java index 7d0b5a4..4ddc7ef 100644 --- a/src/main/java/Sprout_Squad/EyeOn/global/exception/GlobalExceptionHandler.java +++ b/src/main/java/Sprout_Squad/EyeOn/global/exception/GlobalExceptionHandler.java @@ -13,6 +13,7 @@ import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.RestControllerAdvice; import org.springframework.web.method.annotation.MethodArgumentTypeMismatchException; +import org.springframework.web.multipart.support.MissingServletRequestPartException; @RestControllerAdvice @Slf4j @@ -84,6 +85,17 @@ private ResponseEntity handleUserSignupRequiredException(SignupRe return ResponseEntity.status(error.getHttpStatus()).body(error); } + /* RequestPart 누락 시 에러 처리 */ + @ExceptionHandler(MissingServletRequestPartException.class) + private ResponseEntity handleMissingServletRequestPartException(MissingServletRequestPartException e) { + log.error("MissingServletRequestPartException Error: {}", e.getRequestPartName()); + ErrorResponse error = ErrorResponse.of( + GlobalErrorCode.BAD_REQUEST_ERROR, + e.getRequestPartName() + " 파트가 요청에 없습니다." + ); + return ResponseEntity.status(error.getHttpStatus()).body(error); + } + /* 나머지 예외 처리 */ @ExceptionHandler(Exception.class) diff --git a/src/main/java/Sprout_Squad/EyeOn/global/external/service/OpenAiService.java b/src/main/java/Sprout_Squad/EyeOn/global/external/service/OpenAiService.java index 24d5277..a8cc403 100644 --- a/src/main/java/Sprout_Squad/EyeOn/global/external/service/OpenAiService.java +++ b/src/main/java/Sprout_Squad/EyeOn/global/external/service/OpenAiService.java @@ -1,12 +1,19 @@ package Sprout_Squad.EyeOn.global.external.service; +import Sprout_Squad.EyeOn.domain.document.entity.enums.DocumentType; +import Sprout_Squad.EyeOn.domain.document.web.dto.GetAdviceReq; import Sprout_Squad.EyeOn.global.config.OpenAiConfig; import Sprout_Squad.EyeOn.global.external.exception.OpenAiApiException; +import Sprout_Squad.EyeOn.global.flask.dto.GetModelRes; +import Sprout_Squad.EyeOn.global.flask.mapper.FieldLabelMapper; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import lombok.RequiredArgsConstructor; import org.springframework.http.*; import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -15,26 +22,79 @@ public class OpenAiService { private final OpenAiConfig openAiConfig; private final RestTemplate restTemplate; + private final FieldLabelMapper fieldLabelMapper; private static final String CHAT_URL = "https://api.openai.com/v1/chat/completions"; private static final String SUMMARY_PROMPT = "당신은 문서를 요약하는 능력이 아주 뛰어난 사람입니다." + "사진의 문서를 분석하고, 중요한 내용을 빠짐없이 요약해주세요. 반드시 모든 내용을 요약에 포함시켜주어야 합니다." + "텍스트로 제공할 것이기 때문에 강조 표시나 볼드 표시는 필요하지 않습니다."; + private static final String MODIFY_PROMPT = "당신은 문서를 검토하는 능력이 매우 뛰어난 사람입니다. 해당 문서를 검토하세요"+ + "모든 항목을 분석할 필요는 없으며, 오타를 찾거나 잘못된 부분 지적하세요"+ "다음 JSON 배열은 문서를 구조화 시킨 내용입니다. " + + "각 항목은 index(i), displayName(d:표시된 명칭), value(v:입력된 값)를 가지고 있습니다.\n" + + "법적으로 살펴봐야 할 조언이나 문제가 될만할 부분을 간단하게 제시하세요." + + "수정이 필요한 항목에 한해서만 대해 한국어 조언(a)를 추가하여 JSON으로 반환해 주세요. 그러나 ```json이라는 표시는 빼주세요. " + + "조언을 제공할 항목의 d와 v값을 JSON 형태로 제공할 때 표시된 그대로가 아닌 알맞은 띄어쓰기를 제공해주세요. " + + "그러나 이 내용이 a에 들어가서는 안됩니다. \n" + + "형식은 아래와 같아야 합니다:\n" + + "\n" + + "[\n" + + " { \"i\": ..., \"d\": \"...\", \"v\": \"...\", \"a\": \"...\" },\n" + + " ...\n" + + "]\n" + + "\n" + + "설명 없이 배열만 출력하세요.\n" + + "입력은 다음과 같습니다. \n"; + + /** + * 수정할 부분 분석 요청 + */ + public String getModifyAnalyzeFromOpenAi(List fields, DocumentType documentType) { + String json; + try { + json = new ObjectMapper().writeValueAsString(fields); + } catch (JsonProcessingException e) { + throw new RuntimeException("JSON 직렬화 실패", e); + } + + System.out.println("📝 분석 요청 payload: " + json); + + String prompt = "이 문서의 유형은 " + documentType + "입니다.\n" + MODIFY_PROMPT + json; + Map requestBody = createRequestBody(prompt); + ResponseEntity response = sendRequest(requestBody); + return parseResponse(response); + } + + /** * 이미지 요약 요청 */ public String getSummaryFromOpenAi(String imageUrl) { - Map requestBody = createRequestBody(imageUrl, SUMMARY_PROMPT); + Map requestBody = createRequestBodyWithImg(imageUrl, SUMMARY_PROMPT); ResponseEntity response = sendRequest(requestBody); return parseResponse(response); + } + /** + * 프롬프트로만 RequestBody 생성 + */ + public Map createRequestBody(String prompt) { + return Map.of( + "model", openAiConfig.getModel(), + "messages", List.of( + Map.of( + "role", "user", + "content", prompt + ) + ) + ); } + /** - * RequestBody 생성 + * 이미지를 포함한 RequestBody 생성 */ - public Map createRequestBody(String imageUrl, String prompt){ + public Map createRequestBodyWithImg(String imageUrl, String prompt){ return Map.of( "model", openAiConfig.getModel(), "messages", List.of( diff --git a/src/main/java/Sprout_Squad/EyeOn/global/external/service/PdfService.java b/src/main/java/Sprout_Squad/EyeOn/global/external/service/PdfService.java index 39df36b..a8bc00b 100644 --- a/src/main/java/Sprout_Squad/EyeOn/global/external/service/PdfService.java +++ b/src/main/java/Sprout_Squad/EyeOn/global/external/service/PdfService.java @@ -26,6 +26,103 @@ public class PdfService { private final S3Service s3Service; + /** + * S3 url을 받아 기존 양식 위에 적힌 부분들을 하얀색으로 덮어 씌우고 재작성하는 로직 + */ + public String rewriteImg(String s3ImageUrl, List fields) { + try ( + InputStream imageStream = s3Service.downloadFile(s3ImageUrl); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + ) { + // 1. 이미지 불러오기 + BufferedImage image = ImageIO.read(imageStream); + if (image == null) { + throw new IOException("이미지 로딩 실패"); + } + + // 2. 폰트 설정 + InputStream fontStream = getClass().getClassLoader().getResourceAsStream("fonts/NanumGothic.ttf"); + if (fontStream == null) throw new FontNotFoundException(); + Font font = Font.createFont(Font.TRUETYPE_FONT, fontStream).deriveFont(12f); + + Graphics2D g = image.createGraphics(); + g.setFont(font); + g.setColor(Color.BLACK); + + // 3. 텍스트 삽입 + for (WriteDocsReq field : fields) { + + if (field.value() == null) continue; + // bbox : [x0, y0, x1, y1] + List bbox = field.bbox(); + + float imgWidth = image.getWidth(); + float imgHeight = image.getHeight(); + + // bbox 좌표를 이미지 크기에 맞게 변환 + List bboxRaw = field.bbox(); + float x0 = bboxRaw.get(0).floatValue() * imgWidth / 1000; + float y0 = bboxRaw.get(1).floatValue() * imgHeight / 1000; + float x1 = bboxRaw.get(2).floatValue() * imgWidth / 1000; + float y1 = bboxRaw.get(3).floatValue() * imgHeight / 1000; + + // 박스 덮기용 + double ratioXLeft = 0.97; + double ratioXRight = 0.97; + double ratioYTop = 0.80; + double ratioYBottom = 0.97; + + double width = x1 - x0; + double height = y1 - y0; + + double newX0 = x0 + width * (1 - ratioXLeft); + double newX1 = x0 + width * ratioXRight; + double newY0 = y0 + height * (1 - ratioYTop); + double newY1 = y0 + height * ratioYBottom; + + // 박스를 흰색으로 지우기 + g.setColor(Color.WHITE); + g.fillRect((int)newX0, (int)newY0, (int)(newX1 - newX0), (int)(newY1 - newY0)); + + // 텍스트를 박스 안에 적절히 배치 (좌표 보정) + float boxHeight = y1 - y0; + x0 += 15f; + y0 += boxHeight * 0.75f; // 텍스트를 박스의 적절한 위치로 이동 (상단보다는 조금 아래쪽) + + System.out.printf("[좌표] x0=%.2f, y0=%.2f, x1=%.2f, y1=%.2f%n", x0, y0, x1, y1); + + g.setColor(Color.BLACK); + //g.drawRect((int) x0, (int) y0, (int) (x1 - x0), (int) boxHeight); // 디버깅용 사각형 그리기 + // 폰트 크기를 bbox 높이에 맞춰 동적으로 설정 + Font dynamicFont = font.deriveFont( 20f); // 박스 높이에 맞춰 폰트 크기 조정 + g.setFont(dynamicFont); + + System.out.println("[텍스트 출력] value=\"" + field.value() + "\" at (" + x0 + ", " + y0 + ")"); + + // 텍스트 삽입 + g.drawString(field.value(), x0, y0); + } + + + g.dispose(); + + System.out.println("4번"); + // 4. BufferedImage → ByteArrayOutputStream (JPG 저장) + ImageIO.write(image, "png", outputStream); + byte[] imageBytes = outputStream.toByteArray(); + + System.out.println("5번"); + // 5. S3 업로드 + String fileName = generateImageFileName(); + String s3Key = "filled-docs/" + fileName; + return s3Service.uploadImageBytes(s3Key, imageBytes); + + } catch (Exception e) { + e.printStackTrace(); + throw new FileNotCreatedException(); + } + } + /** * S3 url을 받아 양식 위에 문서 작성 로직 */ @@ -51,11 +148,9 @@ public String fillImageFromS3(String s3ImageUrl, List fields) { // 3. 텍스트 삽입 for (WriteDocsReq field : fields) { - if (field.value() == null) continue; // bbox : [x0, y0, x1, y1] List bbox = field.bbox(); - System.out.println("Bounding Box for " + field.displayName() + ": " + bbox); float imgWidth = image.getWidth(); float imgHeight = image.getHeight(); @@ -66,8 +161,9 @@ public String fillImageFromS3(String s3ImageUrl, List fields) { float x1 = bbox.get(2).floatValue() * imgWidth / 1000; // x1 float y1 = bbox.get(3).floatValue() * imgHeight / 1000; // y1 - // 텍스트를 박스 안에 적절히 배치 (y좌표 보정) + // 텍스트를 박스 안에 적절히 배치 (좌표 보정) float boxHeight = y1 - y0; + x0 += 15f; y0 += boxHeight * 0.75f; // 텍스트를 박스의 적절한 위치로 이동 (상단보다는 조금 아래쪽) @@ -156,17 +252,13 @@ public String convertPdfToImage(byte[] pdfBytes) throws IOException { PDDocument document = PDDocument.load(pdfBytes); ByteArrayOutputStream imageOutputStream = new ByteArrayOutputStream(); ) { - System.out.println("잘 들어와요22"); PDFRenderer renderer = new PDFRenderer(document); - System.out.println("잘 들어와요333"); // 첫 페이지를 이미지로 변환 BufferedImage bim = renderer.renderImageWithDPI(0, 300); - System.out.println("잘 들어와요4444"); ImageIO.write(bim, "jpg", imageOutputStream); byte[] imageBytes = imageOutputStream.toByteArray(); - System.out.println("잘 들어와요5"); String fileName = generateImageFileName(); String s3Key = "pdf-preview/" + fileName; diff --git a/src/main/java/Sprout_Squad/EyeOn/global/flask/dto/GetFieldForModifyRes.java b/src/main/java/Sprout_Squad/EyeOn/global/flask/dto/GetFieldForModifyRes.java new file mode 100644 index 0000000..bbfeb97 --- /dev/null +++ b/src/main/java/Sprout_Squad/EyeOn/global/flask/dto/GetFieldForModifyRes.java @@ -0,0 +1,13 @@ +package Sprout_Squad.EyeOn.global.flask.dto; + +import java.util.List; + +public record GetFieldForModifyRes( + String field, + String targetField, + int index, + List bbox, + String displayName, + String value +) { +} diff --git a/src/main/java/Sprout_Squad/EyeOn/domain/form/web/dto/GetFieldRes.java b/src/main/java/Sprout_Squad/EyeOn/global/flask/dto/GetFieldForWriteRes.java similarity index 68% rename from src/main/java/Sprout_Squad/EyeOn/domain/form/web/dto/GetFieldRes.java rename to src/main/java/Sprout_Squad/EyeOn/global/flask/dto/GetFieldForWriteRes.java index 958f880..2e16a6b 100644 --- a/src/main/java/Sprout_Squad/EyeOn/domain/form/web/dto/GetFieldRes.java +++ b/src/main/java/Sprout_Squad/EyeOn/global/flask/dto/GetFieldForWriteRes.java @@ -1,8 +1,8 @@ -package Sprout_Squad.EyeOn.domain.form.web.dto; +package Sprout_Squad.EyeOn.global.flask.dto; import java.util.List; -public record GetFieldRes( +public record GetFieldForWriteRes( String field, String targetField, int index, diff --git a/src/main/java/Sprout_Squad/EyeOn/global/flask/dto/GetModelRes.java b/src/main/java/Sprout_Squad/EyeOn/global/flask/dto/GetModelRes.java new file mode 100644 index 0000000..dbe44a9 --- /dev/null +++ b/src/main/java/Sprout_Squad/EyeOn/global/flask/dto/GetModelRes.java @@ -0,0 +1,21 @@ +package Sprout_Squad.EyeOn.global.flask.dto; + +import java.util.List; + +public record GetModelRes( + String doctype, + List tokens, + List> bboxes, + List labels, + List indices // bboxes와 일대일 대응되는 인덱스 +) { + public static GetModelRes of(String doctype, List tokens, List> bboxes, List labels) { + // index 리스트 생성 + List indices = new java.util.ArrayList<>(); + for (int i = 0; i < bboxes.size(); i++) { + indices.add(i); // bbox index 기준 + } + + return new GetModelRes(doctype, tokens, bboxes, labels, indices); + } +} diff --git a/src/main/java/Sprout_Squad/EyeOn/global/flask/dto/GetModelResForModify.java b/src/main/java/Sprout_Squad/EyeOn/global/flask/dto/GetModelResForModify.java new file mode 100644 index 0000000..7795bf8 --- /dev/null +++ b/src/main/java/Sprout_Squad/EyeOn/global/flask/dto/GetModelResForModify.java @@ -0,0 +1,23 @@ +package Sprout_Squad.EyeOn.global.flask.dto; + +import java.util.List; + +public record GetModelResForModify( + String doctype, + List tokens, + List> bboxes, + List labels, + List indices, // bboxes와 일대일 대응되는 인덱스 + List mergedTokens + +) { + public static GetModelResForModify of(String doctype, List tokens, List> bboxes, List labels, List mergedTokens) { + // index 리스트 생성 + List indices = new java.util.ArrayList<>(); + for (int i = 0; i < bboxes.size(); i++) { + indices.add(i); // bbox index 기준 + } + + return new GetModelResForModify(doctype, tokens, bboxes, labels, indices, mergedTokens); + } +} diff --git a/src/main/java/Sprout_Squad/EyeOn/global/flask/mapper/FieldLabelMapper.java b/src/main/java/Sprout_Squad/EyeOn/global/flask/mapper/FieldLabelMapper.java index 6a3ba73..7a418e4 100644 --- a/src/main/java/Sprout_Squad/EyeOn/global/flask/mapper/FieldLabelMapper.java +++ b/src/main/java/Sprout_Squad/EyeOn/global/flask/mapper/FieldLabelMapper.java @@ -11,7 +11,7 @@ public class FieldLabelMapper { private final Map> labelMap; - // 🟨 클래스 필드 선언 추가 + // 클래스 필드 선언 추가 private final Map resume; private final Map certificate; private final Map consent; @@ -120,15 +120,12 @@ public FieldLabelMapper() { ); this.common = Map.ofEntries( - Map.entry("년", "B-SIGN-YEAR"), - Map.entry("월", "B-SIGN-MONTH"), - Map.entry("일", "B-SIGN-DAY"), - Map.entry("지원자", "B-SIGN-NAME"), - Map.entry("작성일자", "B-DATE-DOCUMENT"), - Map.entry("작성일", "B-DATE-DOCUMENT"), - Map.entry("위임자", "B-SIGN-NAME"), - Map.entry("(인)", "SIGN-SEAL"), - Map.entry("(서명)", "SIGN-SEAL") + Map.entry("B-SIGN-YEAR", "년"), + Map.entry("B-SIGN-MONTH", "월"), + Map.entry("B-SIGN-DAY", "일"), + Map.entry("B-SIGN-NAME", "지원자"), + Map.entry("B-DATE-DOCUMENT", "작성일자"), + Map.entry("B-SIGN-SEAL", "(인)") ); tempMap.put("resume", this.resume); diff --git a/src/main/java/Sprout_Squad/EyeOn/global/flask/mapper/FieldTargetMapper.java b/src/main/java/Sprout_Squad/EyeOn/global/flask/mapper/FieldTargetMapper.java new file mode 100644 index 0000000..fc14d3b --- /dev/null +++ b/src/main/java/Sprout_Squad/EyeOn/global/flask/mapper/FieldTargetMapper.java @@ -0,0 +1,122 @@ +package Sprout_Squad.EyeOn.global.flask.mapper; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class FieldTargetMapper { + private static final Map fieldToTargetField; + + static { + Map map = new HashMap<>(); + + // 이력서 - 자격 + map.put("B-CERTIFICATE-DATE", "B-CERTIFICATE-DATE-FIELD"); + map.put("B-CERTIFICATE-NAME", "B-CERTIFICATE-NAME-FIELD"); + map.put("B-CERTIFICATE-ORG", "B-CERTIFICATE-ORG-FIELD"); + map.put("B-CERTIFICATE-SCORE", "B-CERTIFICATE-SCORE-FIELD"); + + // 이력서 - 학업 + map.put("B-EDU-GPA", "B-EDU-GPA-FIELD"); + map.put("B-EDU-MAJOR", "B-EDU-MAJOR-FIELD"); + map.put("B-EDU-NAME", "B-EDU-NAME-FIELD"); + map.put("B-EDU-PERIOD", "B-EDU-PERIOD-FIELD"); + + // 이력서 - 군복무 + map.put("B-MILITARY-PERIOD", "B-MILITARY-PERIOD-FIELD"); + map.put("B-MILITARY-RANK", "B-MILITARY-RANK-FIELD"); + map.put("B-MILITARY-STATUS", "B-MILITARY-STATUS-FIELD"); + map.put("B-MILITARY-TYPE", "B-MILITARY-TYPE-FIELD"); + + // 이력서 - 경력 + map.put("B-WORK-DUTY", "B-WORK-DUTY-FIELD"); + map.put("B-WORK-PERIOD", "B-WORK-PERIOD-FIELD"); + map.put("B-WORK-PLACE", "B-WORK-PLACE-FIELD"); + map.put("B-WORK-NAME", "B-WORK-NAME-FIELD"); + map.put("B-WORK-POSITION", "B-WORK-POSITION-FIELD"); + + // 업무일지 + map.put("B-DATE", "B-DATE-FIELD"); + map.put("B-DEPT", "B-DEPT-FIELD"); + map.put("B-NAME", "B-NAME-FIELD"); + map.put("B-POSITION", "B-POSITION-FIELD"); + map.put("B-REPORT-ISSUE", "B-REPORT-ISSUE-FIELD"); + map.put("B-TODAY-WORK-DETAIL", "B-TODAY-WORK-DETAIL-FIELD"); + map.put("B-TODAY-WORK-NOTE", "B-TODAY-WORK-NOTE-FIELD"); + map.put("B-TODAY-WORK-TIME", "B-TODAY-WORK-TIME-FIELD"); + map.put("B-TODAY-WORK-TITLE", "B-TODAY-WORK-TITLE-FIELD"); + map.put("B-TOMORROW-WORK-AM-PLAN", "B-TOMORROW-WORK-AM-PLAN-FIELD"); + map.put("B-TOMORROW-WORK-PM-PLAN", "B-TOMORROW-WORK-PM-PLAN-FIELD"); + + + // 자기소개서 + map.put("B-ACADEMIC-LIFE", "B-ACADEMIC-LIFE-FIELD"); + map.put("B-MOTIVATION-FUTURE", "B-MOTIVATION-FUTURE-FIELD"); + map.put("B-PERSONALITY", "B-PERSONALITY-FIELD"); + map.put("B-SELF-GROWTH", "B-SELF-GROWTH-FIELD"); + + // 재직증명서 + map.put("B-CEO-SIGN", "B-CEO-SIGN-FIELD"); + map.put("B-EMPLOY-DEPT", "B-EMPLOY-DEPT-FIELD"); + map.put("B-EMPLOY-PERIOD", "B-EMPLOY-PERIOD-FIELD"); + map.put("B-EMPLOY-POSITION", "B-EMPLOY-POSITION-FIELD"); + map.put("B-PLACE-ADDR", "B-PLACE-ADDR-FIELD"); + map.put("B-PLACE-NAME", "B-PLACE-NAME-FIELD"); + map.put("B-PURPOSE", "B-PURPOSE-FIELD"); + + // 위임장 + map.put("B-DELEGATE-ADDR", "B-DELEGATE-ADDR-FIELD"); + map.put("B-DELEGATE-NAME", "B-DELEGATE-NAME-FIELD"); + map.put("B-DELEGATE-PHONE", "B-DELEGATE-PHONE-FIELD"); + map.put("B-DELEGATE-RELATION", "B-DELEGATE-RELATION-FIELD"); + map.put("B-DELEGATE-RRN", "B-DELEGATE-RRN-FIELD"); + map.put("B-GRANT-CONTENT", "B-GRANT-CONTENT-FIELD"); + map.put("B-GRANTOR-ADDR", "B-GRANTOR-ADDR-FIELD"); + map.put("B-GRANTOR-NAME", "B-GRANTOR-NAME-FIELD"); + map.put("B-GRANTOR-PHONE", "B-GRANTOR-PHONE-FIELD"); + map.put("B-GRANTOR-REASON", "B-GRANTOR-REASON-FIELD"); + map.put("B-GRANTOR-RRN", "B-GRANTOR-RRN-FIELD"); + + // 오타로 보이는 항목 + //map.put("B-GRANT-CONTENT", "B-GRANT-CONTENT-FILED"); + + // ? + map.put("B-DATE-DOCUMENT", "B-DATE-DOCUMENT-FIELD"); + + // common + map.put("B-SIGN-DAY", "B-SIGN-DAY-FIELD"); + map.put("B-SIGN-MONTH", "B-SIGN-MONTH-FIELD"); + map.put("B-SIGN-NAME", "B-SIGN-NAME-FIELD"); + map.put("B-SIGN-YEAR", "B-SIGN-YEAR-FIELD"); + map.put("B-PERSONAL-ADDR", "B-PERSONAL-ADDR-FIELD"); + map.put("B-PERSONAL-EMAIL", "B-PERSONAL-EMAIL-FIELD"); + map.put("B-PERSONAL-NAME", "B-PERSONAL-NAME-FIELD"); + map.put("B-PERSONAL-PHONE", "B-PERSONAL-PHONE-FIELD"); + map.put("B-PERSONAL-RRN", "B-PERSONAL-RRN-FIELD"); + map.put("B-PERSONAL-PHOTO", "B-PERSONAL-PHOTO-FIELD"); + + fieldToTargetField = Collections.unmodifiableMap(map); + } + + public static String getTargetField(String field) { + return fieldToTargetField.get(field); + } + + public static boolean hasTargetField(String field) { + return fieldToTargetField.containsKey(field); + } + + public static Map getAllMappings() { + return fieldToTargetField; + } + + public static String getFieldByTargetField(String targetField) { + for (Map.Entry entry : fieldToTargetField.entrySet()) { + if (entry.getValue().equals(targetField)) { + return entry.getKey(); + } + } + return null; + } + +} diff --git a/src/main/java/Sprout_Squad/EyeOn/global/flask/service/FlaskService.java b/src/main/java/Sprout_Squad/EyeOn/global/flask/service/FlaskService.java index 5e1fcd5..c755ed3 100644 --- a/src/main/java/Sprout_Squad/EyeOn/global/flask/service/FlaskService.java +++ b/src/main/java/Sprout_Squad/EyeOn/global/flask/service/FlaskService.java @@ -1,44 +1,323 @@ package Sprout_Squad.EyeOn.global.flask.service; +import Sprout_Squad.EyeOn.domain.document.web.dto.ModifyDocumentReq; +import Sprout_Squad.EyeOn.domain.document.web.dto.ModifyDocumentReqWrapper; +import Sprout_Squad.EyeOn.domain.document.web.dto.WriteDocsReq; +import Sprout_Squad.EyeOn.domain.document.web.dto.WriteDocsReqWrapper; +import Sprout_Squad.EyeOn.global.flask.dto.GetFieldForModifyRes; +import Sprout_Squad.EyeOn.global.flask.dto.GetFieldForWriteRes; +import Sprout_Squad.EyeOn.global.flask.dto.GetModelRes; +import Sprout_Squad.EyeOn.domain.user.entity.User; +import Sprout_Squad.EyeOn.domain.user.repository.UserRepository; +import Sprout_Squad.EyeOn.global.auth.jwt.UserPrincipal; +import Sprout_Squad.EyeOn.global.converter.ImgConverter; +import Sprout_Squad.EyeOn.global.external.service.PdfService; +import Sprout_Squad.EyeOn.global.flask.dto.GetModelResForModify; +import Sprout_Squad.EyeOn.global.flask.exception.GetLabelFailedException; +import Sprout_Squad.EyeOn.global.flask.exception.TypeDetectedFiledException; +import Sprout_Squad.EyeOn.global.flask.mapper.FieldLabelMapper; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.RequiredArgsConstructor; import org.springframework.http.*; import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; +import org.springframework.web.multipart.MultipartFile; -import java.util.HashMap; -import java.util.Map; +import java.util.*; @Service @RequiredArgsConstructor public class FlaskService { private final RestTemplate restTemplate; + private final PdfService pdfService; + private final FieldLabelMapper fieldLabelMapper; + private final UserRepository userRepository; private final ObjectMapper objectMapper = new ObjectMapper(); private String baseUrl = "http://3.39.215.178:5050"; /** - * 문서 필드 분석 (라벨링) + * 모델에서 문서 분석 결과를 받아 가공 (작성) */ - public String getLabel(String base64Image, String fileExt) throws JsonProcessingException { - Map responseBody = sendFlaskPostRequest("/api/ai/create", base64Image, fileExt); + public GetModelRes getResFromModelForWrite(MultipartFile file, String fileName){ + try { + String fileToBase64 = ImgConverter.toBase64(file); + String fileExtension = pdfService.getFileExtension(fileName); + + // 플라스크 요청 + String jsonRes = getLabelForWrite(fileToBase64, fileExtension); + + // jsonRes를 파싱 + ObjectMapper mapper = new ObjectMapper(); + Map resultMap = mapper.readValue(jsonRes, Map.class); + + // tokens: 문서에서 추출된 텍스트 조각 + List tokens = (List) resultMap.get("tokens"); + + // labels: 각 토큰의 labels + List labels = (List) resultMap.get("labels"); + + // doctype: 문서 유형 + String doctype = (String) resultMap.get("doctype"); + + // rawBboxes: 각 토큰의 bounding box 좌표 정보 + List> rawBboxes = (List>) resultMap.get("bboxes"); + + // bboxes: 각 토큰의 bounding box 좌표 정보 + List> bboxes = new ArrayList<>(); + + // List 형태로 파싱된 좌표 데이터를 List로 변환 + for (List box : rawBboxes) { + List convertedBox = box.stream() + .map(val -> ((Number) val).doubleValue()) // JSON 파싱 결과가 Number 타입 -> 형변환 필요 + .toList(); + bboxes.add(convertedBox); + } + return GetModelRes.of(doctype, tokens, bboxes, labels); + } catch (Exception e){ throw new GetLabelFailedException();} + } + + /** + * 작성을 위한 문서 필드 분석(라벨링) 요청 + */ + public String getLabelForWrite(String base64Image, String fileExt) throws JsonProcessingException { + Map responseBody = sendJsonRequestToFlask("/api/ai/create", base64Image, fileExt); Map result = (Map) responseBody.get("result"); - System.out.println("결과 : " + result); return objectMapper.writeValueAsString(result); // result 전체를 JSON 문자열로 변환 } /** - * 문서 유형 감지 + * 모델에서 문서 분석 결과를 받아 가공 (수정) */ - public String detectType(String base64Image, String fileExt) { - Map responseBody = sendFlaskPostRequest("/api/ai/detect", base64Image, fileExt); - return (String) responseBody.get("doc_type"); + public GetModelResForModify getResFromModelForModify(MultipartFile file, String fileName){ + try { + String fileToBase64 = ImgConverter.toBase64(file); + String fileExtension = pdfService.getFileExtension(fileName); + + // Flask에 요청 + String jsonRes = getLabelForModify(fileToBase64, fileExtension); + + // 응답 파싱 + ObjectMapper mapper = new ObjectMapper(); + Map rootMap = mapper.readValue(jsonRes, Map.class); + Map resultMap = (Map) rootMap.get("layoutlm_result"); + Map mergedMap = (Map) rootMap.get("merged_tokens"); + + String doctype = (String) resultMap.get("doctype"); + List tokens = (List) resultMap.get("tokens"); + List labels = (List) resultMap.get("labels"); + + List> bboxes = new ArrayList<>(); + List> rawBboxes = (List>) resultMap.get("bboxes"); + for (List box : rawBboxes) { + List converted = box.stream() + .map(val -> ((Number) val).doubleValue()) + .toList(); + bboxes.add(converted); + } + + List mergedTokens = (List) mergedMap.get("tokens"); + + // 인덱스 생성 + List indices = new ArrayList<>(); + for (int i = 0; i < bboxes.size(); i++) { + indices.add(i); + } + + return new GetModelResForModify(doctype, tokens, bboxes, labels, indices, mergedTokens); + + } catch (Exception e) { + throw new GetLabelFailedException(); + } + } + + + /** + * 수정을 위한 문서 필드 분석(라벨링) 요청 + */ + public String getLabelForModify(String base64Image, String fileExt) throws JsonProcessingException { + Map responseBody = sendJsonRequestToFlask("/api/ai/modify", base64Image, fileExt); + Map result = (Map) responseBody.get("result"); + return objectMapper.writeValueAsString(result); // result 전체를 JSON 문자열로 변환 + } + + /** + * 문서 타입에 맞는 필드의 라벨명을 반환 + */ + private Map getLabelMap(String docType){ + // context는 중첩된 Map 구조 + Map> context = fieldLabelMapper.getContextualLabels(); + + // doctype에 맞는 Map을 복사 + Map labelMap = new HashMap<>(context.getOrDefault(docType, Collections.emptyMap())); + + // common 항목에 대하여 Map을 복사 + Map commonMap = context.getOrDefault("common", Collections.emptyMap()); + + // labelMap에 공통 항목까지 삽입 + labelMap.putAll(commonMap); + + return labelMap; + } + + /** + * 필드 라벨에 따라 사용자 정보를 채워넣어 반환 + */ + private String resolveValue(String baseLabel, User user) { + return switch (baseLabel) { + case "B-PERSONAL-NAME", "B-GRANTOR-NAME", "B-DELEGATE-NAME", "B-NAME", "B-SIGN-NAME" -> user.getName(); + case "B-PERSONAL-RRN", "B-GRANTOR-RRN", "B-DELEGATE-RRN" -> user.getResidentNumber(); + case "B-PERSONAL-PHONE", "B-GRANTOR-PHONE", "B-DELEGATE-PHONE" -> user.getPhoneNumber(); + case "B-PERSONAL-ADDR", "B-GRANTOR-ADDR", "B-DELEGATE-ADDR" -> user.getAddress(); + case "B-PERSONAL-EMAIL" -> user.getEmail(); + default -> null; // 나머지는 프론트에서 받기 + }; + } + + /** + * 모델로부터 얻은 결과를 가공 및 사용자 정보를 채워넣어 문서 수정 양식에 맞는 형태로 반환 + */ + public List getFieldForModify(GetModelResForModify getModelResForModify) { + List labels = getModelResForModify.labels(); + List mergedTokens = getModelResForModify.mergedTokens(); + List> bboxes = getModelResForModify.bboxes(); + Map labelMap = getLabelMap(getModelResForModify.doctype()); + + List results = new ArrayList<>(); + + for (int i = 0; i < labels.size(); i++) { + String label = labels.get(i); + if (!label.endsWith("-FIELD")) continue; + + String field = label.replace("-FIELD", ""); + String displayName = labelMap.getOrDefault(field, field); + + if (i >= mergedTokens.size()) continue; + + String value = mergedTokens.get(i); + if (value == null || value.trim().equals("[BLANK]")) continue; + + List bbox = bboxes.get(i); + results.add(new GetFieldForModifyRes(field, label, i, bbox, displayName, value)); + } + + return results; + } + + + + /** + * 모델로부터 얻은 결과를 가공 및 사용자 정보를 채워넣어 문서 작성 양식에 맞는 형태로 반환 + */ + public List getFieldForWrite(GetModelRes getModelRes, UserPrincipal userPrincipal) { + User user = userRepository.getUserById(userPrincipal.getId()); + Map labelMap = getLabelMap(getModelRes.doctype()); + List fields = new ArrayList<>(); + + List labels = getModelRes.labels(); + List tokens = getModelRes.tokens(); + List indices = getModelRes.indices(); + List> bboxes = getModelRes.bboxes(); + + for (int i = 0; i < labels.size(); i++) { + String label = labels.get(i); + + if (label.endsWith("-FIELD")) { + String field = label.replace("-FIELD", ""); + String targetField = label; + String displayName = labelMap.getOrDefault(field, field); + String value = resolveValue(field, user); + + int actualIndex = indices.get(i); + List bbox = bboxes.get(i); + + fields.add(new GetFieldForWriteRes( + field, + targetField, + actualIndex, + bbox, + displayName, + value + )); + } + } + + return fields; + } + + + /** + * 수정할 문서 분석 요청 + */ + public String getModifyLabelReq(String base64Image, String fileExt) throws JsonProcessingException { + Map responseBody = sendJsonRequestToFlask("/api/ai/modify", base64Image, fileExt); + Map result = (Map) responseBody.get("result"); + System.out.println(result); + return objectMapper.writeValueAsString(result); + } + + /** + * 수정 문석 분석 결과를 바탕으로 modifyDocumentReq와 비교하여 WriteDocsReqWrapper 를 반환 + */ + public WriteDocsReqWrapper getModifyRes(String aiResult, ModifyDocumentReqWrapper modifyDocumentReqWrapper) { + try { + Map resultMap = objectMapper.readValue(aiResult, Map.class); + + Map layoutlmResult = (Map) resultMap.get("layoutlm_result"); + List labels = (List) layoutlmResult.get("labels"); + List> bboxes = (List>) layoutlmResult.get("bboxes"); + String doctype = (String) layoutlmResult.get("doctype"); + + List resultList = new ArrayList<>(); + + // 클라이언트가 보내준 index는 layoutlm 기준임 + for (ModifyDocumentReq modify : modifyDocumentReqWrapper.data()) { + int index = modify.i(); + String value = modify.v(); + + if (index >= labels.size() || index >= bboxes.size()) continue; + + String targetField = labels.get(index); // B-XXX-FIELD + String field = targetField.replace("-FIELD", ""); // B-XXX + List bbox = bboxes.get(index); + String displayName = fieldLabelMapper.getDisplayName(doctype, field); + + resultList.add(new WriteDocsReq( + field, + targetField, + index, + bbox, + displayName, + value + )); + } + + System.out.println("결과는? : " + resultList); + return new WriteDocsReqWrapper(resultList); + + } catch (Exception e) { + throw new RuntimeException("수정 결과 가공 실패", e); + } + } + + /** + * 문서 유형 감지 바디 구성 + */ + public String detectType(MultipartFile file, String fileName) { + try { + String base64Image = ImgConverter.toBase64(file); + String fileExt = pdfService.getFileExtension(fileName); + Map responseBody = sendJsonRequestToFlask("/api/ai/detect", base64Image, fileExt); + return (String) responseBody.get("doc_type"); + } catch (Exception e) { + throw new TypeDetectedFiledException(); + } } /** - * 공통 요청 처리 메서드 + * 문서 작성, 필드 분석, 문서 수정 필드 분석 요청 처리 메서드 */ - private Map sendFlaskPostRequest(String path, String base64Image, String fileExt) { + private Map sendJsonRequestToFlask(String path, String base64Image, String fileExt) { String endpoint = baseUrl + path; Map requestBody = new HashMap<>();