From 485b4dd4851b1283b2d6d4743d088dc6c7d0f58f Mon Sep 17 00:00:00 2001 From: super <2903208875@qq.com> Date: Tue, 24 Mar 2026 22:22:37 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E6=88=90=E5=93=81=E7=89=8C=E9=83=A8?= =?UTF-8?q?=E5=88=86=E6=94=B9=E9=80=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend-java/pom.xml | 4 + .../com/nanri/aiimage/AiImageApplication.java | 2 + .../config/BrandProgressProperties.java | 13 + .../aiimage/config/PropertiesConfig.java | 2 +- .../aiimage/config/StorageProperties.java | 4 + .../brand/controller/BrandTaskController.java | 207 ++++ .../brand/mapper/BrandCrawlTaskMapper.java | 9 + .../model/dto/BrandCrawlResultFileDto.java | 48 + .../model/dto/BrandCrawlResultRequest.java | 21 + .../brand/model/dto/BrandInvalidBrandDto.java | 18 + .../model/dto/BrandParsedFileCacheDto.java | 16 + .../brand/model/dto/BrandQueryFailedDto.java | 15 + .../brand/model/dto/BrandSourceFileDto.java | 20 + .../model/dto/BrandTaskCreateRequest.java | 27 + .../brand/model/dto/BrandTaskRunRequest.java | 20 + .../model/entity/BrandCrawlTaskEntity.java | 33 + .../brand/model/vo/BrandCrawlPayloadVo.java | 28 + .../modules/brand/model/vo/BrandSimpleVo.java | 13 + .../brand/model/vo/BrandTaskCreateVo.java | 11 + .../brand/model/vo/BrandTaskDetailVo.java | 11 + .../brand/model/vo/BrandTaskItemVo.java | 23 + .../brand/model/vo/BrandTaskListVo.java | 13 + .../vo/LegacyBrandLineProgressInfoVo.java | 27 + .../model/vo/LegacyBrandLineProgressVo.java | 15 + .../model/vo/LegacyBrandTaskCreateVo.java | 9 + .../model/vo/LegacyBrandTaskDetailVo.java | 15 + .../brand/model/vo/LegacyBrandTaskItemVo.java | 44 + .../brand/model/vo/LegacyBrandTaskListVo.java | 11 + .../BrandTaskProgressCacheService.java | 182 +++ .../brand/service/BrandTaskService.java | 1025 +++++++++++++++++ .../controller/DedupeRunController.java | 2 + .../dedupe/mapper/DedupeTotalDataMapper.java | 16 + .../dedupe/service/DedupeRunService.java | 36 +- .../service/DedupeTotalDataService.java | 23 + .../file/service/LocalFileStorageService.java | 2 +- .../file/service/LocalTempCleanupService.java | 116 ++ .../src/main/resources/application.yml | 16 + 37 files changed, 2086 insertions(+), 11 deletions(-) create mode 100644 backend-java/src/main/java/com/nanri/aiimage/config/BrandProgressProperties.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/brand/controller/BrandTaskController.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/brand/mapper/BrandCrawlTaskMapper.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandCrawlResultFileDto.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandCrawlResultRequest.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandInvalidBrandDto.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandParsedFileCacheDto.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandQueryFailedDto.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandSourceFileDto.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandTaskCreateRequest.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandTaskRunRequest.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/entity/BrandCrawlTaskEntity.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandCrawlPayloadVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandSimpleVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandTaskCreateVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandTaskDetailVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandTaskItemVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandTaskListVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandLineProgressInfoVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandLineProgressVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandTaskCreateVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandTaskDetailVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandTaskItemVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandTaskListVo.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskProgressCacheService.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java create mode 100644 backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalTempCleanupService.java diff --git a/backend-java/pom.xml b/backend-java/pom.xml index bf45bcb..187f51e 100644 --- a/backend-java/pom.xml +++ b/backend-java/pom.xml @@ -42,6 +42,10 @@ org.springframework.boot spring-boot-starter-security + + org.springframework.boot + spring-boot-starter-data-redis + com.baomidou mybatis-plus-spring-boot3-starter diff --git a/backend-java/src/main/java/com/nanri/aiimage/AiImageApplication.java b/backend-java/src/main/java/com/nanri/aiimage/AiImageApplication.java index a85d792..f86b27e 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/AiImageApplication.java +++ b/backend-java/src/main/java/com/nanri/aiimage/AiImageApplication.java @@ -2,8 +2,10 @@ package com.nanri.aiimage; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.EnableScheduling; @SpringBootApplication +@EnableScheduling public class AiImageApplication { public static void main(String[] args) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/BrandProgressProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/BrandProgressProperties.java new file mode 100644 index 0000000..c270006 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/config/BrandProgressProperties.java @@ -0,0 +1,13 @@ +package com.nanri.aiimage.config; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@Data +@ConfigurationProperties(prefix = "aiimage.brand-progress") +public class BrandProgressProperties { + private long ttlHours = 24; + private long failedTtlHours = 2; + private long heartbeatTimeoutMinutes = 15; + private String staleCheckCron = "0 */2 * * * *"; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/PropertiesConfig.java b/backend-java/src/main/java/com/nanri/aiimage/config/PropertiesConfig.java index 2b77762..5a099af 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/PropertiesConfig.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/PropertiesConfig.java @@ -4,6 +4,6 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Configuration; @Configuration -@EnableConfigurationProperties({OssProperties.class, StorageProperties.class}) +@EnableConfigurationProperties({OssProperties.class, StorageProperties.class, BrandProgressProperties.class}) public class PropertiesConfig { } diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/StorageProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/StorageProperties.java index da584bf..7eb4830 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/StorageProperties.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/StorageProperties.java @@ -7,4 +7,8 @@ import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties(prefix = "aiimage.storage") public class StorageProperties { private String localTempDir; + private boolean cleanupEnabled = true; + private String cleanupCron = "0 0 */6 * * *"; + private long sourceRetentionHours = 48; + private long resultRetentionHours = 24; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/controller/BrandTaskController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/controller/BrandTaskController.java new file mode 100644 index 0000000..b4ad774 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/controller/BrandTaskController.java @@ -0,0 +1,207 @@ +package com.nanri.aiimage.modules.brand.controller; + +import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultRequest; +import com.nanri.aiimage.modules.brand.model.dto.BrandTaskCreateRequest; +import com.nanri.aiimage.modules.brand.model.vo.BrandCrawlPayloadVo; +import com.nanri.aiimage.modules.brand.model.vo.BrandSimpleVo; +import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandTaskDetailVo; +import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandTaskListVo; +import com.nanri.aiimage.modules.brand.service.BrandTaskService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.validation.Valid; +import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpHeaders; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequiredArgsConstructor +@RequestMapping("/api/brand") +@Tag(name = "品牌任务", description = "品牌任务接口。Java 端只负责下载并解析 OSS 文件、返回待爬取数据、接收爬取结果并生成 xlsx/上传 OSS/落旧表，不负责实际爬虫执行。") +public class BrandTaskController { + + private final BrandTaskService brandTaskService; + + @PostMapping("/tasks") + @Operation( + summary = "创建品牌任务并返回待爬取数据", + description = """ + 创建一条品牌任务记录到旧表 brand_crawl_tasks，随后立即下载并读取 Excel， + 将原始解析结果临时缓存到后端，并把 sheetName、columns、rows、uniqueBrands 等待爬取数据返回给前端。 + + 适用流程： + 1. 前端先拿到 OSS 文件链接； + 2. 再调用本接口创建任务； + 3. 前端拿到返回数据后自行执行品牌爬取； + 4. 爬取结束后调用结果提交接口回传结果。 + """) + @ApiResponses({ + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "创建成功，返回任务 ID 和待爬取数据", content = @Content(schema = @Schema(implementation = BrandCrawlPayloadVo.class))), + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "请求参数不合法或文件列表为空"), + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "500", description = "下载或解析 Excel 或创建任务失败") + }) + public ApiResponse createTask( + @Parameter(description = "用户 ID", required = true) @RequestParam Long userId, + @Valid @RequestBody BrandTaskCreateRequest request) { + return ApiResponse.success(brandTaskService.createTaskAndBuildPayload(userId, request)); + } + + @GetMapping("/tasks") + @Operation( + summary = "获取品牌任务列表", + description = """ + 查询旧表 brand_crawl_tasks 最近的任务列表，返回前端任务面板需要的核心信息。 + + 返回内容包括： + - id：任务 ID； + - file_paths：任务创建时记录的源文件信息； + - desc：任务描述，通常由 strategy + 文件名摘要组成； + - strategy：品牌匹配方式； + - status：任务状态（pending/running/success/failed/cancelled）； + - result_paths：结果文件地址信息，通常包含 urls 和 zip_url； + - error_message：失败原因； + - progress_current / progress_total：当前进度； + - created_at / updated_at：创建和更新时间。 + + 典型用途： + - 页面任务列表展示； + - 判断任务是否已完成； + - 决定是否显示下载按钮； + - 展示失败原因和进度。 + """ + ) + @ApiResponses({ + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功，返回任务列表", content = @Content(schema = @Schema(implementation = LegacyBrandTaskListVo.class))) + }) + public ApiResponse listTasks( + @Parameter(description = "用户 ID", required = true) @RequestParam Long userId) { + return ApiResponse.success(brandTaskService.listTasksLegacy(userId)); + } + + @GetMapping("/tasks/{taskId}") + @Operation( + summary = "获取品牌任务详情", + description = """ + 根据任务 ID 查询单条品牌任务详情，返回任务基础信息 + 文件级摘要进度 + Redis 中基于结果分片累计出的实时行级进度。 + + 返回内容包括： + - task.id：任务 ID； + - task.file_paths：任务绑定的源文件信息； + - task.desc / task.strategy / task.status：任务描述、策略、状态； + - task.result_paths：结果文件地址。单文件场景通常只有 urls，多文件场景通常同时包含 urls 和 zip_url； + - task.error_message：失败原因； + - task.progress_current / task.progress_total：文件级摘要进度，表示已完成文件数 / 总文件数； + - task.created_at / task.updated_at：创建和更新时间； + - line_progress.has_progress：当前是否存在 Redis 实时进度； + - line_progress.info.file_index / file_total：当前处理到第几个文件 / 总文件数； + - line_progress.info.file_name：当前处理文件名； + - line_progress.info.current_line / total_lines：当前文件已处理行数 / 总行数； + - line_progress.info.phase：当前阶段，可能为 crawling / assembling / uploading / failed。 + + 典型用途： + - 前端轮询任务状态； + - 展示当前文件与当前行进度； + - 提交结果后确认任务是否已 success； + - 下载前校验任务结果是否已生成； + - 任务失败时获取具体错误信息。 + """ + ) + @ApiResponses({ + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功，返回单条任务详情", content = @Content(schema = @Schema(implementation = LegacyBrandTaskDetailVo.class))), + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "任务不存在") + }) + public ApiResponse getTask( + @Parameter(description = "品牌任务 ID", required = true) @PathVariable Long taskId) { + return ApiResponse.success(brandTaskService.getTaskDetailLegacy(taskId)); + } + + @PostMapping("/tasks/{taskId}/result") + @Operation( + summary = "提交品牌任务爬取结果", + description = """ + 前端完成品牌爬取后，将按品牌汇总后的结果分片提交给本接口。 + Java 端会累计接收前端每次提交的少量结果（例如 5 个品牌）， + 并直接根据 keptRows / invalidBrands / queryFailedBrands 的数量推进 Redis 中的实时进度。 + 在所有文件都提交完成后再基于创建任务时缓存的原始数据统一： + - 按“品牌”列删除 invalidBrands 对应的原始行，生成主 sheet； + - keptRows 用于统计已处理且保留的品牌进度； + - 生成“不符合品牌”和“查询失败品牌” sheet； + - 输出 xlsx； + - 上传 OSS； + - 回写 brand_crawl_tasks.result_paths/status/error_message； + + 处理期间前端可轮询任务详情接口 `GET /api/brand/tasks/{taskId}`， + 获取 status、progress_current、progress_total、error_message、line_progress 等实时进度信息。 + """) + @ApiResponses({ + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "提交成功，结果文件已生成并落库", content = @Content(schema = @Schema(implementation = BrandSimpleVo.class))), + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "请求参数不合法、fileUrl 不匹配或结果数据为空"), + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "500", description = "生成结果文件或上传 OSS 失败") + }) + public ApiResponse submitResult( + @Parameter(description = "品牌任务 ID", required = true) @PathVariable Long taskId, + @Valid @RequestBody BrandCrawlResultRequest request) { + brandTaskService.submitCrawlResult(taskId, request); + return ApiResponse.success(new BrandSimpleVo(true)); + } + + @PostMapping("/tasks/{taskId}/cancel") + @Operation( + summary = "取消品牌任务", + description = "将任务状态从 pending/running 更新为 cancelled。适用于前端停止本地爬取时同步取消任务记录。" + ) + @ApiResponses({ + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "取消成功", content = @Content(schema = @Schema(implementation = BrandSimpleVo.class))), + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "任务不存在或当前状态不可取消") + }) + public ApiResponse cancelTask( + @Parameter(description = "品牌任务 ID", required = true) @PathVariable Long taskId) { + brandTaskService.cancelTask(taskId); + return ApiResponse.success(new BrandSimpleVo(true)); + } + + @DeleteMapping("/tasks/{taskId}") + @Operation( + summary = "删除品牌任务", + description = "删除旧表 brand_crawl_tasks 中的任务记录。仅允许删除非 running 状态任务。" + ) + @ApiResponses({ + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "删除成功", content = @Content(schema = @Schema(implementation = BrandSimpleVo.class))), + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "任务不存在或正在执行中不可删除") + }) + public ApiResponse deleteTask( + @Parameter(description = "品牌任务 ID", required = true) @PathVariable Long taskId) { + brandTaskService.deleteTask(taskId); + return ApiResponse.success(new BrandSimpleVo(true)); + } + + @GetMapping("/tasks/{taskId}/download") + @Operation( + summary = "下载品牌任务结果", + description = "根据任务 ID 读取 brand_crawl_tasks.result_paths，优先跳转 zip_url；若没有 zip_url，则跳转第一个可用结果 URL。" + ) + @ApiResponses({ + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "302", description = "重定向到 OSS 结果文件地址"), + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "任务不存在或无结果可下载") + }) + public ResponseEntity download( + @Parameter(description = "品牌任务 ID", required = true) @PathVariable Long taskId) { + String url = brandTaskService.resolveDownloadUrl(taskId); + return ResponseEntity.status(302) + .header(HttpHeaders.LOCATION, url) + .build(); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/mapper/BrandCrawlTaskMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/mapper/BrandCrawlTaskMapper.java new file mode 100644 index 0000000..50afef9 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/mapper/BrandCrawlTaskMapper.java @@ -0,0 +1,9 @@ +package com.nanri.aiimage.modules.brand.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface BrandCrawlTaskMapper extends BaseMapper { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandCrawlResultFileDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandCrawlResultFileDto.java new file mode 100644 index 0000000..79804db --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandCrawlResultFileDto.java @@ -0,0 +1,48 @@ +package com.nanri.aiimage.modules.brand.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import lombok.Data; + +import java.util.List; + +@Data +@Schema(description = "单个品牌文件结果分片。前端只需按品牌提交保留品牌、不符合品牌、查询失败品牌；后端会基于创建任务时缓存的原始行数据生成最终结果文件。") +public class BrandCrawlResultFileDto { + + @NotBlank(message = "fileUrl 不能为空") + @Schema(description = "源文件 OSS 链接，必须与创建任务时的文件项一一对应。") + private String fileUrl; + + @Schema(description = "原始文件名。为空时会回退使用任务创建时记录的 originalFilename。") + private String originalFilename; + + @Schema(description = "相对路径。文件夹上传场景下用于生成 zip 时恢复目录层级。") + private String relativePath; + + @Schema(description = "主 sheet 名称。为空时默认输出为 Sheet1。") + private String mainSheetName; + + @NotNull(message = "chunkIndex 不能为空") + @Schema(description = "当前分片序号，从 1 开始。", example = "1") + private Integer chunkIndex; + + @NotNull(message = "chunkTotal 不能为空") + @Schema(description = "当前文件总分片数。", example = "10") + private Integer chunkTotal; + + @Schema(description = "当前文件总行数。后端据此展示精确行级进度。", example = "1000") + private Integer totalLines; + + @Schema(description = "本批处理后应保留的品牌列表。仅用于后端累计进度，不需要回传整行数据。") + private List keptRows; + + @Valid + @Schema(description = "本批处理后判定为不符合品牌的记录列表。会写入“不符合品牌” sheet；删除主 sheet 数据时仅按 brand 字段匹配删除。") + private List invalidBrands; + + @Schema(description = "本批查询失败的品牌列表。后端会写入“查询失败品牌” sheet。") + private List queryFailedBrands; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandCrawlResultRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandCrawlResultRequest.java new file mode 100644 index 0000000..0e2ee37 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandCrawlResultRequest.java @@ -0,0 +1,21 @@ +package com.nanri.aiimage.modules.brand.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; +import lombok.Data; + +import java.util.List; + +@Data +@Schema(description = "品牌爬取结果提交请求。前端按文件分片提交不符合品牌数据和查询失败品牌数据；后端会基于创建任务时缓存的原始行数据生成最终结果文件。") +public class BrandCrawlResultRequest { + + @Schema(description = "品牌匹配方式。应与创建任务时使用的 strategy 保持一致。", example = "Terms") + private String strategy = "Terms"; + + @Valid + @NotEmpty(message = "files 不能为空") + @Schema(description = "本次提交的文件结果分片列表。每个分片只需要提交 fileUrl、chunkIndex、chunkTotal、totalLines、keptRows、invalidBrands、queryFailedBrands。invalidBrands 需包含品牌、国家、状态；主 sheet 删除时仅按品牌字段匹配。") + private List files; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandInvalidBrandDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandInvalidBrandDto.java new file mode 100644 index 0000000..7ba9a75 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandInvalidBrandDto.java @@ -0,0 +1,18 @@ +package com.nanri.aiimage.modules.brand.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "不符合品牌记录。Java 会将该列表输出到结果 xlsx 的“不符合品牌” sheet。") +public class BrandInvalidBrandDto { + + @Schema(description = "命中的品牌名称。") + private String brand; + + @Schema(description = "命中的国家/地区。") + private String country; + + @Schema(description = "命中的品牌状态，例如 已注册、待决。") + private String status; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandParsedFileCacheDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandParsedFileCacheDto.java new file mode 100644 index 0000000..424d81c --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandParsedFileCacheDto.java @@ -0,0 +1,16 @@ +package com.nanri.aiimage.modules.brand.model.dto; + +import lombok.Data; + +import java.util.List; +import java.util.Map; + +@Data +public class BrandParsedFileCacheDto { + private String fileUrl; + private String originalFilename; + private String relativePath; + private String sheetName; + private List columns; + private List> rows; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandQueryFailedDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandQueryFailedDto.java new file mode 100644 index 0000000..1b20fc2 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandQueryFailedDto.java @@ -0,0 +1,15 @@ +package com.nanri.aiimage.modules.brand.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "查询失败品牌记录。Java 会将该列表输出到结果 xlsx 的“查询失败品牌” sheet。") +public class BrandQueryFailedDto { + + @Schema(description = "查询失败的品牌名称。") + private String brand; + + @Schema(description = "失败时间。由前端或爬虫侧生成后原样回传。") + private String time; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandSourceFileDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandSourceFileDto.java new file mode 100644 index 0000000..908b9b0 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandSourceFileDto.java @@ -0,0 +1,20 @@ +package com.nanri.aiimage.modules.brand.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotBlank; +import lombok.Data; + +@Data +@Schema(description = "品牌任务源文件。由 OSS 文件链接和文件元信息组成。") +public class BrandSourceFileDto { + + @NotBlank(message = "fileUrl 不能为空") + @Schema(description = "OSS 文件下载链接，Java 将先下载文件再读取内容。", example = "https://example.oss-cn-hangzhou.aliyuncs.com/source/brand.xlsx") + private String fileUrl; + + @Schema(description = "原始文件名，用于生成任务描述和结果文件名。", example = "品牌样例.xlsx") + private String originalFilename; + + @Schema(description = "相对路径。文件夹上传场景下可传，用于最终 zip 结果保持目录结构。", example = "店铺A/品牌样例.xlsx") + private String relativePath; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandTaskCreateRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandTaskCreateRequest.java new file mode 100644 index 0000000..7d6c6b2 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandTaskCreateRequest.java @@ -0,0 +1,27 @@ +package com.nanri.aiimage.modules.brand.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; +import lombok.Data; + +import java.util.List; + +@Data +@Schema(description = "品牌任务创建请求。前端将 OSS 文件链接和 originalFilename/relativePath 传入本请求。Java 会创建 brand_crawl_tasks 记录并下载文件返回待爬取数据。") +public class BrandTaskCreateRequest { + + @Valid + @NotEmpty(message = "请先上传待处理文件") + @Schema(description = "已上传源文件列表。每个元素对应一个待解析的 Excel 文件。") + private List files; + + @Schema(description = "品牌匹配方式。支持 Terms=精确匹配、Simple=嵌入匹配。默认 Terms。", example = "Terms") + private String strategy = "Terms"; + + @Schema(description = "任务类型。1=立即处理并返回待爬取数据，2=仅登记任务。当前推荐传 1。", example = "1") + private Integer taskType = 1; + + @Schema(description = "文件夹上传场景下可选的归档名称。当前版本仅保留字段，不参与主流程。") + private String archiveName; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandTaskRunRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandTaskRunRequest.java new file mode 100644 index 0000000..63f9068 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandTaskRunRequest.java @@ -0,0 +1,20 @@ +package com.nanri.aiimage.modules.brand.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.List; + +@Data +@Schema(description = "品牌任务兼容立即执行请求") +public class BrandTaskRunRequest { + + @Schema(description = "旧页面传入的本地路径列表，仅兼容旧接口") + private List paths; + + @Schema(description = "品牌匹配方式: Terms/Simple") + private String strategy = "Terms"; + + @Schema(description = "任务类型: 1=立即执行") + private Integer taskType = 1; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/entity/BrandCrawlTaskEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/entity/BrandCrawlTaskEntity.java new file mode 100644 index 0000000..efe7f7b --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/entity/BrandCrawlTaskEntity.java @@ -0,0 +1,33 @@ +package com.nanri.aiimage.modules.brand.model.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +@TableName("brand_crawl_tasks") +public class BrandCrawlTaskEntity { + + @TableId(type = IdType.AUTO) + private Long id; + + private Long userId; + private String filePaths; + private String status; + private Integer taskType; + private String resultPaths; + private String errorMessage; + private Integer progressCurrent; + private Integer progressTotal; + + @TableField("`desc`") + private String desc; + + private String strategy; + private LocalDateTime createdAt; + private LocalDateTime updatedAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandCrawlPayloadVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandCrawlPayloadVo.java new file mode 100644 index 0000000..96c2806 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandCrawlPayloadVo.java @@ -0,0 +1,28 @@ +package com.nanri.aiimage.modules.brand.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.List; +import java.util.Map; + +@Data +@Schema(description = "品牌任务爬取载荷") +public class BrandCrawlPayloadVo { + + private Long taskId; + private String strategy; + private List files; + + @Data + public static class BrandCrawlPayloadFileVo { + private Integer fileIndex; + private String fileUrl; + private String originalFilename; + private String relativePath; + private String sheetName; + private List columns; + private List> rows; + private List uniqueBrands; + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandSimpleVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandSimpleVo.java new file mode 100644 index 0000000..2ab2448 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandSimpleVo.java @@ -0,0 +1,13 @@ +package com.nanri.aiimage.modules.brand.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +@Schema(description = "简单成功响应") +public class BrandSimpleVo { + + private boolean success; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandTaskCreateVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandTaskCreateVo.java new file mode 100644 index 0000000..df7989a --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandTaskCreateVo.java @@ -0,0 +1,11 @@ +package com.nanri.aiimage.modules.brand.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "品牌任务创建结果") +public class BrandTaskCreateVo { + + private Long taskId; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandTaskDetailVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandTaskDetailVo.java new file mode 100644 index 0000000..8f68454 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandTaskDetailVo.java @@ -0,0 +1,11 @@ +package com.nanri.aiimage.modules.brand.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "品牌任务详情") +public class BrandTaskDetailVo { + + private BrandTaskItemVo task; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandTaskItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandTaskItemVo.java new file mode 100644 index 0000000..0458027 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandTaskItemVo.java @@ -0,0 +1,23 @@ +package com.nanri.aiimage.modules.brand.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.List; + +@Data +@Schema(description = "品牌任务项") +public class BrandTaskItemVo { + + private Long id; + private List filePaths; + private String desc; + private String strategy; + private String status; + private Object resultPaths; + private String errorMessage; + private Integer progressCurrent; + private Integer progressTotal; + private String createdAt; + private String updatedAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandTaskListVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandTaskListVo.java new file mode 100644 index 0000000..477771d --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandTaskListVo.java @@ -0,0 +1,13 @@ +package com.nanri.aiimage.modules.brand.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.List; + +@Data +@Schema(description = "品牌任务列表") +public class BrandTaskListVo { + + private List items; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandLineProgressInfoVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandLineProgressInfoVo.java new file mode 100644 index 0000000..f7e0846 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandLineProgressInfoVo.java @@ -0,0 +1,27 @@ +package com.nanri.aiimage.modules.brand.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "品牌任务实时进度详情") +public class LegacyBrandLineProgressInfoVo { + + @Schema(description = "当前处理到第几个文件，从 1 开始") + private Integer file_index; + + @Schema(description = "本次任务总文件数") + private Integer file_total; + + @Schema(description = "当前处理文件名") + private String file_name; + + @Schema(description = "当前文件已处理行数") + private Integer current_line; + + @Schema(description = "当前文件总行数") + private Integer total_lines; + + @Schema(description = "当前阶段：crawling/assembling/uploading/failed") + private String phase; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandLineProgressVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandLineProgressVo.java new file mode 100644 index 0000000..2f3a7d5 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandLineProgressVo.java @@ -0,0 +1,15 @@ +package com.nanri.aiimage.modules.brand.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "品牌任务实时行级进度") +public class LegacyBrandLineProgressVo { + + @Schema(description = "当前是否存在实时进度") + private boolean has_progress; + + @Schema(description = "实时进度详情") + private LegacyBrandLineProgressInfoVo info; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandTaskCreateVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandTaskCreateVo.java new file mode 100644 index 0000000..67b1c25 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandTaskCreateVo.java @@ -0,0 +1,9 @@ +package com.nanri.aiimage.modules.brand.model.vo; + +import lombok.Data; + +@Data +public class LegacyBrandTaskCreateVo { + + private Long task_id; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandTaskDetailVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandTaskDetailVo.java new file mode 100644 index 0000000..3a50cb7 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandTaskDetailVo.java @@ -0,0 +1,15 @@ +package com.nanri.aiimage.modules.brand.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "品牌任务详情响应") +public class LegacyBrandTaskDetailVo { + + @Schema(description = "任务基础信息") + private LegacyBrandTaskItemVo task; + + @Schema(description = "实时行级进度信息，来自 Redis") + private LegacyBrandLineProgressVo line_progress; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandTaskItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandTaskItemVo.java new file mode 100644 index 0000000..566050c --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandTaskItemVo.java @@ -0,0 +1,44 @@ +package com.nanri.aiimage.modules.brand.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.List; + +@Data +@Schema(description = "品牌任务基础信息") +public class LegacyBrandTaskItemVo { + + @Schema(description = "任务 ID") + private Long id; + + @Schema(description = "任务绑定的源文件信息") + private List file_paths; + + @Schema(description = "任务描述") + private String desc; + + @Schema(description = "品牌匹配策略") + private String strategy; + + @Schema(description = "任务状态：pending/running/success/failed/cancelled") + private String status; + + @Schema(description = "结果文件地址信息。单文件场景通常包含 urls，多文件场景通常同时包含 urls 和 zip_url") + private Object result_paths; + + @Schema(description = "失败原因") + private String error_message; + + @Schema(description = "文件级摘要进度：已完成文件数") + private Integer progress_current; + + @Schema(description = "文件级摘要进度：总文件数") + private Integer progress_total; + + @Schema(description = "创建时间") + private String created_at; + + @Schema(description = "更新时间") + private String updated_at; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandTaskListVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandTaskListVo.java new file mode 100644 index 0000000..f878116 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandTaskListVo.java @@ -0,0 +1,11 @@ +package com.nanri.aiimage.modules.brand.model.vo; + +import lombok.Data; + +import java.util.List; + +@Data +public class LegacyBrandTaskListVo { + + private List items; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskProgressCacheService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskProgressCacheService.java new file mode 100644 index 0000000..613bd28 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskProgressCacheService.java @@ -0,0 +1,182 @@ +package com.nanri.aiimage.modules.brand.service; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.config.BrandProgressProperties; +import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultFileDto; +import lombok.RequiredArgsConstructor; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Service; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +@Service +@RequiredArgsConstructor +public class BrandTaskProgressCacheService { + + public static final String PHASE_CRAWLING = "crawling"; + public static final String PHASE_ASSEMBLING = "assembling"; + public static final String PHASE_UPLOADING = "uploading"; + public static final String PHASE_FAILED = "failed"; + + private final StringRedisTemplate stringRedisTemplate; + private final BrandProgressProperties brandProgressProperties; + private final ObjectMapper objectMapper; + + public void saveProgressFromResult(Long taskId, + String fileUrl, + int fileIndex, + int fileTotal, + String fileName, + int currentLine, + int totalLines, + int finishedFiles) { + String key = buildKey(taskId); + String now = String.valueOf(Instant.now().toEpochMilli()); + Map values = new LinkedHashMap<>(); + values.put("phase", PHASE_CRAWLING); + values.put("file_index", String.valueOf(Math.max(fileIndex, 0))); + values.put("file_total", String.valueOf(Math.max(fileTotal, 0))); + values.put("file_name", blankToEmpty(fileName)); + values.put("file_url", blankToEmpty(fileUrl)); + values.put("current_line", String.valueOf(Math.max(Math.min(currentLine, totalLines), 0))); + values.put("total_lines", String.valueOf(Math.max(totalLines, 0))); + values.put("finished_files", String.valueOf(Math.max(finishedFiles, 0))); + values.put("updated_at", now); + values.put("last_heartbeat_at", now); + stringRedisTemplate.opsForHash().putAll(key, values); + stringRedisTemplate.expire(key, Duration.ofHours(brandProgressProperties.getTtlHours())); + } + + public void updatePhase(Long taskId, String phase, int finishedFiles, int fileTotal) { + String key = buildKey(taskId); + String now = String.valueOf(Instant.now().toEpochMilli()); + Map values = new LinkedHashMap<>(); + values.put("phase", normalizePhase(phase)); + values.put("finished_files", String.valueOf(Math.max(finishedFiles, 0))); + values.put("file_total", String.valueOf(Math.max(fileTotal, 0))); + values.put("updated_at", now); + values.put("last_heartbeat_at", now); + stringRedisTemplate.opsForHash().putAll(key, values); + stringRedisTemplate.expire(key, Duration.ofHours(brandProgressProperties.getTtlHours())); + } + + public void markFailed(Long taskId, String message) { + String key = buildKey(taskId); + String now = String.valueOf(Instant.now().toEpochMilli()); + Map values = new LinkedHashMap<>(); + values.put("phase", PHASE_FAILED); + values.put("updated_at", now); + values.put("error_message", blankToEmpty(message)); + stringRedisTemplate.opsForHash().putAll(key, values); + stringRedisTemplate.expire(key, Duration.ofHours(brandProgressProperties.getFailedTtlHours())); + } + + public Map getProgress(Long taskId) { + return stringRedisTemplate.opsForHash().entries(buildKey(taskId)); + } + + public void saveParsedPayload(Long taskId, Object payload) { + try { + stringRedisTemplate.opsForValue().set(buildPayloadKey(taskId), objectMapper.writeValueAsString(payload), Duration.ofHours(brandProgressProperties.getTtlHours())); + } catch (Exception ex) { + throw new BusinessException("暂存品牌原始数据失败"); + } + } + + public T getParsedPayload(Long taskId, TypeReference typeReference) { + String raw = stringRedisTemplate.opsForValue().get(buildPayloadKey(taskId)); + if (raw == null || raw.isBlank()) { + return null; + } + try { + return objectMapper.readValue(raw, typeReference); + } catch (Exception ex) { + throw new BusinessException("读取品牌原始数据失败"); + } + } + + public void delete(Long taskId) { + stringRedisTemplate.delete(buildKey(taskId)); + stringRedisTemplate.delete(buildResultKey(taskId)); + stringRedisTemplate.delete(buildPayloadKey(taskId)); + } + + public void mergeResultChunks(Long taskId, List incomingFiles) { + String resultKey = buildResultKey(taskId); + for (BrandCrawlResultFileDto file : incomingFiles) { + if (file.getChunkIndex() == null || file.getChunkTotal() == null || file.getChunkIndex() <= 0 || file.getChunkTotal() <= 0 || file.getChunkIndex() > file.getChunkTotal()) { + throw new BusinessException("分片参数不合法"); + } + String field = buildChunkField(file.getFileUrl(), file.getChunkIndex()); + try { + stringRedisTemplate.opsForHash().put(resultKey, field, objectMapper.writeValueAsString(file)); + } catch (Exception ex) { + throw new BusinessException("暂存结果分片失败"); + } + } + stringRedisTemplate.expire(resultKey, Duration.ofHours(brandProgressProperties.getTtlHours())); + } + + public Map> groupResultChunksByFile(Long taskId) { + Map stored = stringRedisTemplate.opsForHash().entries(buildResultKey(taskId)); + Map> grouped = new LinkedHashMap<>(); + for (Object value : stored.values()) { + if (!(value instanceof String raw) || raw.isBlank()) { + continue; + } + try { + BrandCrawlResultFileDto file = objectMapper.readValue(raw, BrandCrawlResultFileDto.class); + grouped.computeIfAbsent(file.getFileUrl(), ignored -> new ArrayList<>()).add(file); + } catch (Exception ignored) { + } + } + grouped.values().forEach(list -> list.sort(java.util.Comparator.comparing(BrandCrawlResultFileDto::getChunkIndex))); + return grouped; + } + + public void clearResultChunks(Long taskId) { + stringRedisTemplate.delete(buildResultKey(taskId)); + } + + public String buildKey(Long taskId) { + return "brand:task:progress:" + taskId; + } + + private String buildResultKey(Long taskId) { + return "brand:task:result-chunks:" + taskId; + } + + private String buildChunkField(String fileUrl, Integer chunkIndex) { + return fileUrl + "#" + chunkIndex; + } + + private String buildPayloadKey(Long taskId) { + return "brand:task:parsed-payload:" + taskId; + } + + public long getHeartbeatTimeoutMinutes() { + return brandProgressProperties.getHeartbeatTimeoutMinutes(); + } + + private String normalizePhase(String phase) { + if (phase == null || phase.isBlank()) { + return PHASE_CRAWLING; + } + String value = phase.trim().toLowerCase(); + return switch (value) { + case PHASE_ASSEMBLING, PHASE_UPLOADING, PHASE_FAILED -> value; + default -> PHASE_CRAWLING; + }; + } + + private String blankToEmpty(String value) { + return value == null ? "" : value.trim(); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java new file mode 100644 index 0000000..5b5192c --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java @@ -0,0 +1,1025 @@ +package com.nanri.aiimage.modules.brand.service; + +import cn.hutool.core.io.FileUtil; +import cn.hutool.json.JSONUtil; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.config.BrandProgressProperties; +import com.nanri.aiimage.config.StorageProperties; +import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper; +import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultFileDto; +import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultRequest; +import com.nanri.aiimage.modules.brand.model.dto.BrandInvalidBrandDto; +import com.nanri.aiimage.modules.brand.model.dto.BrandParsedFileCacheDto; +import com.nanri.aiimage.modules.brand.model.dto.BrandQueryFailedDto; +import com.nanri.aiimage.modules.brand.model.dto.BrandSourceFileDto; +import com.nanri.aiimage.modules.brand.model.dto.BrandTaskCreateRequest; +import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity; +import com.nanri.aiimage.modules.brand.model.vo.BrandCrawlPayloadVo; +import com.nanri.aiimage.modules.brand.model.vo.BrandTaskCreateVo; +import com.nanri.aiimage.modules.brand.model.vo.BrandTaskDetailVo; +import com.nanri.aiimage.modules.brand.model.vo.BrandTaskItemVo; +import com.nanri.aiimage.modules.brand.model.vo.BrandTaskListVo; +import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandLineProgressInfoVo; +import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandLineProgressVo; +import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandTaskDetailVo; +import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandTaskItemVo; +import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandTaskListVo; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import lombok.RequiredArgsConstructor; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.DataFormatter; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.ss.usermodel.WorkbookFactory; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.file.Files; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +@Service +@RequiredArgsConstructor +public class BrandTaskService { + + private static final DateTimeFormatter DATETIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm"); + private static final String STATUS_PENDING = "pending"; + private static final String STATUS_RUNNING = "running"; + private static final String STATUS_SUCCESS = "success"; + private static final String STATUS_FAILED = "failed"; + private static final String STATUS_CANCELLED = "cancelled"; + + private final BrandCrawlTaskMapper brandCrawlTaskMapper; + private final OssStorageService ossStorageService; + private final StorageProperties storageProperties; + private final BrandProgressProperties brandProgressProperties; + private final BrandTaskProgressCacheService brandTaskProgressCacheService; + private final ObjectMapper objectMapper; + + public BrandTaskCreateVo createTask(Long userId, BrandTaskCreateRequest request) { + if (userId == null || userId <= 0) { + throw new BusinessException("userId 不合法"); + } + List files = normalizeFiles(request.getFiles()); + if (files.isEmpty()) { + throw new BusinessException("请先上传待处理文件"); + } + String strategy = normalizeStrategy(request.getStrategy()); + int taskType = normalizeTaskType(request.getTaskType(), 2); + + BrandCrawlTaskEntity entity = new BrandCrawlTaskEntity(); + entity.setUserId(userId); + entity.setFilePaths(JSONUtil.toJsonStr(files)); + entity.setStatus(STATUS_PENDING); + entity.setTaskType(taskType); + entity.setResultPaths(null); + entity.setErrorMessage(null); + entity.setProgressCurrent(0); + entity.setProgressTotal(0); + entity.setStrategy(strategy); + entity.setDesc(buildDesc(strategy, files)); + brandCrawlTaskMapper.insert(entity); + + BrandTaskCreateVo vo = new BrandTaskCreateVo(); + vo.setTaskId(entity.getId()); + return vo; + } + + public BrandCrawlPayloadVo createTaskAndBuildPayload(Long userId, BrandTaskCreateRequest request) { + BrandTaskCreateVo created = createTask(userId, request); + return getCrawlPayload(created.getTaskId()); + } + + public BrandTaskListVo listTasks(Long userId) { + List entities = listTaskEntities(userId); + BrandTaskListVo vo = new BrandTaskListVo(); + vo.setItems(entities.stream().map(this::toTaskItem).toList()); + return vo; + } + + public LegacyBrandTaskListVo listTasksLegacy(Long userId) { + List entities = listTaskEntities(userId); + LegacyBrandTaskListVo vo = new LegacyBrandTaskListVo(); + vo.setItems(entities.stream().map(this::toLegacyTaskItem).toList()); + return vo; + } + + public BrandTaskDetailVo getTaskDetail(Long taskId) { + BrandTaskDetailVo vo = new BrandTaskDetailVo(); + vo.setTask(toTaskItem(requireTask(taskId))); + return vo; + } + + public LegacyBrandTaskDetailVo getTaskDetailLegacy(Long taskId) { + BrandCrawlTaskEntity task = requireTask(taskId); + LegacyBrandTaskDetailVo vo = new LegacyBrandTaskDetailVo(); + vo.setTask(toLegacyTaskItem(task)); + vo.setLine_progress(buildLineProgress(taskId)); + return vo; + } + + public BrandCrawlPayloadVo getCrawlPayload(Long taskId) { + BrandCrawlTaskEntity task = requireTask(taskId); + if (STATUS_CANCELLED.equalsIgnoreCase(blankToDefault(task.getStatus(), STATUS_PENDING))) { + throw new BusinessException("任务已取消"); + } + List files = parseSourceFiles(task.getFilePaths()); + if (files.isEmpty()) { + throw new BusinessException("任务没有源文件"); + } + + List payloadFiles = new ArrayList<>(); + List cachedFiles = new ArrayList<>(); + int index = 0; + for (BrandSourceFileDto sourceFile : files) { + index++; + File inputFile = resolveSourceFile(sourceFile); + ParsedBrandFile parsed = parseBrandFile(inputFile); + + BrandCrawlPayloadVo.BrandCrawlPayloadFileVo item = new BrandCrawlPayloadVo.BrandCrawlPayloadFileVo(); + item.setFileIndex(index); + item.setFileUrl(sourceFile.getFileUrl()); + item.setOriginalFilename(resolveOriginalFilename(sourceFile, inputFile)); + item.setRelativePath(sourceFile.getRelativePath()); + item.setSheetName(parsed.sheetName()); + item.setColumns(parsed.columns()); + item.setRows(parsed.rows()); + item.setUniqueBrands(parsed.uniqueBrands()); + payloadFiles.add(item); + + BrandParsedFileCacheDto cacheDto = new BrandParsedFileCacheDto(); + cacheDto.setFileUrl(item.getFileUrl()); + cacheDto.setOriginalFilename(item.getOriginalFilename()); + cacheDto.setRelativePath(item.getRelativePath()); + cacheDto.setSheetName(item.getSheetName()); + cacheDto.setColumns(item.getColumns()); + cacheDto.setRows(item.getRows()); + cachedFiles.add(cacheDto); + } + + brandTaskProgressCacheService.saveParsedPayload(taskId, cachedFiles); + + brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper() + .eq(BrandCrawlTaskEntity::getId, taskId) + .ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED) + .set(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING) + .set(BrandCrawlTaskEntity::getProgressTotal, payloadFiles.size()) + .set(BrandCrawlTaskEntity::getProgressCurrent, 0) + .set(BrandCrawlTaskEntity::getErrorMessage, null)); + brandTaskProgressCacheService.updatePhase(taskId, BrandTaskProgressCacheService.PHASE_CRAWLING, 0, payloadFiles.size()); + + BrandCrawlPayloadVo vo = new BrandCrawlPayloadVo(); + vo.setTaskId(taskId); + vo.setStrategy(normalizeStrategy(task.getStrategy())); + vo.setFiles(payloadFiles); + return vo; + } + + public void submitCrawlResult(Long taskId, BrandCrawlResultRequest request) { + BrandCrawlTaskEntity task = requireTask(taskId); + if (STATUS_CANCELLED.equalsIgnoreCase(blankToDefault(task.getStatus(), STATUS_PENDING))) { + throw new BusinessException("任务已取消"); + } + List sourceFiles = parseSourceFiles(task.getFilePaths()); + if (sourceFiles.isEmpty()) { + throw new BusinessException("任务没有源文件"); + } + + Map sourceByUrl = new LinkedHashMap<>(); + for (BrandSourceFileDto file : sourceFiles) { + sourceByUrl.put(file.getFileUrl(), file); + } + List cachedFiles = brandTaskProgressCacheService.getParsedPayload(taskId, + new TypeReference>() { + }); + if (cachedFiles == null || cachedFiles.isEmpty()) { + throw new BusinessException("任务原始数据已过期，请重新创建任务"); + } + Map cachedByUrl = new LinkedHashMap<>(); + for (BrandParsedFileCacheDto cachedFile : cachedFiles) { + cachedByUrl.put(cachedFile.getFileUrl(), cachedFile); + } + + List resultFiles = request.getFiles() == null ? List.of() : request.getFiles(); + if (resultFiles.isEmpty()) { + throw new BusinessException("files 不能为空"); + } + ensureNoDuplicateResultFiles(resultFiles); + + int totalCount = sourceFiles.size(); + int started = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper() + .eq(BrandCrawlTaskEntity::getId, taskId) + .ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED) + .set(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING) + .set(BrandCrawlTaskEntity::getProgressTotal, totalCount) + .set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now()) + .set(BrandCrawlTaskEntity::getErrorMessage, null)); + if (started == 0) { + throw new BusinessException("任务已取消"); + } + + brandTaskProgressCacheService.mergeResultChunks(taskId, resultFiles); + Map> groupedResults = brandTaskProgressCacheService.groupResultChunksByFile(taskId); + int finishedCount = countCompletedFiles(groupedResults); + updateResultDrivenProgress(taskId, sourceFiles, sourceByUrl, groupedResults, finishedCount, totalCount); + updateTaskProgress(taskId, finishedCount, totalCount); + + if (finishedCount < totalCount) { + return; + } + + brandTaskProgressCacheService.updatePhase(taskId, BrandTaskProgressCacheService.PHASE_ASSEMBLING, finishedCount, totalCount); + + File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "brand-result", String.valueOf(taskId))); + List outputEntries = new ArrayList<>(); + + try { + for (List fileChunks : groupedResults.values()) { + BrandCrawlResultFileDto resultFile = mergeChunks(fileChunks); + BrandSourceFileDto sourceFile = sourceByUrl.get(resultFile.getFileUrl()); + if (sourceFile == null) { + throw new BusinessException("存在未知 fileUrl: " + resultFile.getFileUrl()); + } + BrandParsedFileCacheDto cachedFile = cachedByUrl.get(resultFile.getFileUrl()); + if (cachedFile == null) { + throw new BusinessException("缺少原始缓存数据: " + resultFile.getFileUrl()); + } + String originalFilename = blankToDefault(resultFile.getOriginalFilename(), sourceFile.getOriginalFilename()); + File outputFile = buildNamedOutputFile(outputDir, buildResultFilename(originalFilename)); + writeBrandWorkbook(outputFile, request.getStrategy(), cachedFile, resultFile); + outputEntries.add(new OutputEntry(blankToNull(resultFile.getRelativePath()), outputFile)); + } + + brandTaskProgressCacheService.updatePhase(taskId, BrandTaskProgressCacheService.PHASE_UPLOADING, finishedCount, totalCount); + Map resultPaths = buildAndUploadResult(taskId, outputEntries); + int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper() + .eq(BrandCrawlTaskEntity::getId, taskId) + .ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED) + .set(BrandCrawlTaskEntity::getStatus, STATUS_SUCCESS) + .set(BrandCrawlTaskEntity::getResultPaths, JSONUtil.toJsonStr(resultPaths)) + .set(BrandCrawlTaskEntity::getProgressCurrent, finishedCount) + .set(BrandCrawlTaskEntity::getProgressTotal, totalCount) + .set(BrandCrawlTaskEntity::getErrorMessage, null)); + if (updated == 0) { + throw new BusinessException("任务已取消"); + } + brandTaskProgressCacheService.delete(taskId); + } catch (Exception ex) { + brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper() + .eq(BrandCrawlTaskEntity::getId, taskId) + .ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED) + .set(BrandCrawlTaskEntity::getStatus, STATUS_FAILED) + .set(BrandCrawlTaskEntity::getProgressCurrent, finishedCount) + .set(BrandCrawlTaskEntity::getProgressTotal, totalCount) + .set(BrandCrawlTaskEntity::getErrorMessage, ex.getMessage())); + brandTaskProgressCacheService.markFailed(taskId, ex.getMessage()); + if (ex instanceof BusinessException businessException) { + throw businessException; + } + throw new BusinessException(ex.getMessage()); + } + } + + public void cancelTask(Long taskId) { + int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper() + .eq(BrandCrawlTaskEntity::getId, taskId) + .in(BrandCrawlTaskEntity::getStatus, STATUS_PENDING, STATUS_RUNNING) + .set(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED)); + if (updated == 0) { + throw new BusinessException("任务不存在或无法取消"); + } + brandTaskProgressCacheService.delete(taskId); + } + + public void deleteTask(Long taskId) { + int deleted = brandCrawlTaskMapper.delete(new LambdaQueryWrapper() + .eq(BrandCrawlTaskEntity::getId, taskId) + .ne(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING)); + if (deleted == 0) { + throw new BusinessException("任务不存在或正在执行中无法删除"); + } + } + + public String resolveDownloadUrl(Long taskId) { + BrandCrawlTaskEntity task = requireTask(taskId); + Object raw = parseJsonValue(task.getResultPaths()); + if (!(raw instanceof Map map)) { + throw new BusinessException("无结果可下载"); + } + Object zipUrl = map.get("zip_url"); + if (zipUrl instanceof String zip && !zip.isBlank()) { + return zip; + } + Object urls = map.get("urls"); + if (urls instanceof List list && !list.isEmpty()) { + Object first = list.get(0); + if (first instanceof String url && !url.isBlank()) { + return url; + } + } + throw new BusinessException("无结果可下载"); + } + + private List listTaskEntities(Long userId) { + if (userId == null || userId <= 0) { + throw new BusinessException("userId 不合法"); + } + return brandCrawlTaskMapper.selectList(new LambdaQueryWrapper() + .eq(BrandCrawlTaskEntity::getUserId, userId) + .orderByDesc(BrandCrawlTaskEntity::getId) + .last("limit 100")); + } + + private List normalizeFiles(List files) { + List result = new ArrayList<>(); + if (files == null) { + return result; + } + for (BrandSourceFileDto file : files) { + if (file == null || file.getFileUrl() == null || file.getFileUrl().isBlank()) { + continue; + } + BrandSourceFileDto item = new BrandSourceFileDto(); + item.setFileUrl(file.getFileUrl().trim()); + item.setOriginalFilename(blankToNull(file.getOriginalFilename())); + item.setRelativePath(blankToNull(file.getRelativePath())); + result.add(item); + } + return result; + } + + private List normalizePathFiles(List paths) { + List files = new ArrayList<>(); + if (paths == null) { + return files; + } + for (String path : paths) { + if (path == null || path.isBlank()) { + continue; + } + String trimmed = path.trim(); + File file = new File(trimmed); + if (!file.isFile()) { + continue; + } + BrandSourceFileDto item = new BrandSourceFileDto(); + item.setFileUrl(trimmed); + item.setOriginalFilename(file.getName()); + item.setRelativePath(null); + files.add(item); + } + return files; + } + + private List parseSourceFiles(String rawJson) { + if (rawJson == null || rawJson.isBlank()) { + return List.of(); + } + try { + return objectMapper.readValue(rawJson, new TypeReference>() { + }); + } catch (Exception ex) { + try { + List paths = objectMapper.readValue(rawJson, new TypeReference>() { + }); + return normalizePathFiles(paths); + } catch (Exception innerEx) { + throw new BusinessException("解析任务文件失败"); + } + } + } + + private BrandTaskItemVo toTaskItem(BrandCrawlTaskEntity entity) { + BrandTaskItemVo vo = new BrandTaskItemVo(); + vo.setId(entity.getId()); + vo.setFilePaths(parseJsonList(entity.getFilePaths())); + vo.setDesc(blankToNull(entity.getDesc())); + vo.setStrategy(blankToDefault(entity.getStrategy(), "Terms")); + vo.setStatus(blankToDefault(entity.getStatus(), STATUS_PENDING)); + vo.setResultPaths(parseJsonValue(entity.getResultPaths())); + vo.setErrorMessage(blankToNull(entity.getErrorMessage())); + vo.setProgressCurrent(defaultInteger(entity.getProgressCurrent())); + vo.setProgressTotal(defaultInteger(entity.getProgressTotal())); + vo.setCreatedAt(formatTime(entity.getCreatedAt())); + vo.setUpdatedAt(formatTime(entity.getUpdatedAt())); + return vo; + } + + private LegacyBrandTaskItemVo toLegacyTaskItem(BrandCrawlTaskEntity entity) { + LegacyBrandTaskItemVo vo = new LegacyBrandTaskItemVo(); + vo.setId(entity.getId()); + vo.setFile_paths(parseJsonList(entity.getFilePaths())); + vo.setDesc(blankToNull(entity.getDesc())); + vo.setStrategy(blankToDefault(entity.getStrategy(), "Terms")); + vo.setStatus(blankToDefault(entity.getStatus(), STATUS_PENDING)); + vo.setResult_paths(parseJsonValue(entity.getResultPaths())); + vo.setError_message(blankToNull(entity.getErrorMessage())); + vo.setProgress_current(defaultInteger(entity.getProgressCurrent())); + vo.setProgress_total(defaultInteger(entity.getProgressTotal())); + vo.setCreated_at(formatTime(entity.getCreatedAt())); + vo.setUpdated_at(formatTime(entity.getUpdatedAt())); + return vo; + } + + private List parseJsonList(String rawJson) { + Object value = parseJsonValue(rawJson); + if (value instanceof List list) { + return new ArrayList<>(list); + } + return new ArrayList<>(); + } + + private Object parseJsonValue(String rawJson) { + if (rawJson == null || rawJson.isBlank()) { + return null; + } + try { + return objectMapper.readValue(rawJson, Object.class); + } catch (Exception ex) { + return null; + } + } + + private String formatTime(LocalDateTime time) { + return time == null ? "" : DATETIME_FORMATTER.format(time); + } + + private String normalizeStrategy(String strategy) { + String value = blankToDefault(strategy, "Terms"); + return "Simple".equalsIgnoreCase(value) ? "Simple" : "Terms"; + } + + private int normalizeTaskType(Integer taskType, int defaultValue) { + if (taskType == null) { + return defaultValue; + } + return Objects.equals(taskType, 1) ? 1 : 2; + } + + private void ensureNoDuplicateResultFiles(List resultFiles) { + Set seen = new LinkedHashSet<>(); + for (BrandCrawlResultFileDto resultFile : resultFiles) { + String fileUrl = blankToNull(resultFile.getFileUrl()); + if (fileUrl == null) { + throw new BusinessException("fileUrl 不能为空"); + } + if (!seen.add(fileUrl)) { + throw new BusinessException("存在重复 fileUrl: " + fileUrl); + } + } + } + + private void updateTaskProgress(Long taskId, int finishedCount, int totalCount) { + brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper() + .eq(BrandCrawlTaskEntity::getId, taskId) + .ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED) + .set(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING) + .set(BrandCrawlTaskEntity::getProgressCurrent, finishedCount) + .set(BrandCrawlTaskEntity::getProgressTotal, totalCount) + .set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now()) + .set(BrandCrawlTaskEntity::getErrorMessage, null)); + } + + private void updateResultDrivenProgress(Long taskId, + List sourceFiles, + Map sourceByUrl, + Map> groupedResults, + int finishedCount, + int totalCount) { + if (groupedResults.isEmpty()) { + return; + } + String currentFileUrl = null; + List currentChunks = null; + int currentFileIndex = 0; + for (int i = sourceFiles.size() - 1; i >= 0; i--) { + BrandSourceFileDto sourceFile = sourceFiles.get(i); + List chunks = groupedResults.get(sourceFile.getFileUrl()); + if (chunks == null || chunks.isEmpty()) { + continue; + } + currentFileUrl = sourceFile.getFileUrl(); + currentChunks = chunks; + currentFileIndex = i + 1; + break; + } + if (currentFileUrl == null || currentChunks == null) { + return; + } + BrandSourceFileDto sourceFile = sourceByUrl.get(currentFileUrl); + String fileName = sourceFile == null ? "" : blankToDefault(sourceFile.getOriginalFilename(), sourceFile.getFileUrl()); + int currentLine = countProcessedLines(currentChunks); + int totalLines = Math.max(inferTotalLines(currentChunks), currentLine); + brandTaskProgressCacheService.saveProgressFromResult(taskId, + currentFileUrl, + currentFileIndex, + totalCount, + fileName, + currentLine, + totalLines, + finishedCount); + } + + private int countProcessedLines(List chunks) { + int currentLine = 0; + for (BrandCrawlResultFileDto chunk : chunks) { + if (chunk.getKeptRows() != null) { + currentLine += chunk.getKeptRows().size(); + } + if (chunk.getInvalidBrands() != null) { + currentLine += chunk.getInvalidBrands().size(); + } + if (chunk.getQueryFailedBrands() != null) { + currentLine += chunk.getQueryFailedBrands().size(); + } + } + return currentLine; + } + + private int inferTotalLines(List chunks) { + if (chunks == null || chunks.isEmpty()) { + return 0; + } + BrandCrawlResultFileDto last = chunks.get(chunks.size() - 1); + if (last.getTotalLines() != null && last.getTotalLines() > 0) { + return last.getTotalLines(); + } + if (last.getChunkTotal() == null || last.getChunkTotal() <= 0) { + return countProcessedLines(chunks); + } + int processed = countProcessedLines(chunks); + if (Objects.equals(chunks.size(), last.getChunkTotal())) { + return processed; + } + int averagePerChunk = Math.max(processed / Math.max(chunks.size(), 1), 1); + return averagePerChunk * last.getChunkTotal(); + } + + private String buildDesc(String strategy, List files) { + String joined = files.stream() + .map(file -> blankToDefault(file.getOriginalFilename(), file.getFileUrl())) + .filter(Objects::nonNull) + .map(String::trim) + .filter(name -> !name.isEmpty()) + .limit(20) + .reduce((left, right) -> left + ", " + right) + .orElse(""); + String desc = "[" + strategy + "] " + joined; + return desc.length() > 500 ? desc.substring(0, 500) : desc; + } + + private BrandCrawlTaskEntity requireTask(Long taskId) { + BrandCrawlTaskEntity entity = brandCrawlTaskMapper.selectById(taskId); + if (entity == null) { + throw new BusinessException("任务不存在"); + } + return entity; + } + + private File resolveSourceFile(BrandSourceFileDto sourceFile) { + String fileUrl = sourceFile.getFileUrl(); + if (fileUrl == null || fileUrl.isBlank()) { + throw new BusinessException("fileUrl 不能为空"); + } + File directFile = new File(fileUrl); + if (directFile.isFile()) { + return directFile; + } + return downloadSourceFile(fileUrl); + } + + private File downloadSourceFile(String fileUrl) { + try { + URI uri = URI.create(fileUrl); + String filename = FileUtil.getName(uri.getPath()); + if (filename == null || filename.isBlank()) { + filename = "brand-source.xlsx"; + } + String suffix = FileUtil.extName(filename); + File downloadDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "brand-source-download")); + File tempFile = Files.createTempFile(downloadDir.toPath(), "brand_", suffix.isBlank() ? "" : "." + suffix).toFile(); + try (InputStream inputStream = uri.toURL().openStream()) { + FileUtil.writeFromStream(inputStream, tempFile); + } + return tempFile; + } catch (Exception ex) { + throw new BusinessException("下载源文件失败"); + } + } + + private String resolveOriginalFilename(BrandSourceFileDto sourceFile, File inputFile) { + return blankToDefault(sourceFile.getOriginalFilename(), inputFile.getName()); + } + + private ParsedBrandFile parseBrandFile(File inputFile) { + DataFormatter formatter = new DataFormatter(); + try (FileInputStream fis = new FileInputStream(inputFile); Workbook workbook = WorkbookFactory.create(fis)) { + Sheet sheet = workbook.getSheetAt(0); + Row headerRow = sheet.getRow(0); + if (headerRow == null) { + throw new BusinessException("Excel 表头为空"); + } + List columns = extractHeaders(headerRow, formatter); + if (columns.isEmpty()) { + throw new BusinessException("未读取到有效表头"); + } + Map headerIndexes = buildHeaderIndexes(headerRow, formatter, columns); + List> rows = new ArrayList<>(); + Set uniqueBrands = new LinkedHashSet<>(); + for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) { + Row row = sheet.getRow(rowNum); + if (row == null) { + continue; + } + Map rowData = new LinkedHashMap<>(); + for (String column : columns) { + Integer index = headerIndexes.get(column); + String value = index == null ? "" : normalizeCellText(formatter.formatCellValue(row.getCell(index))); + rowData.put(column, value); + } + rowData.put("__rowIndex", rowNum + 1); + rows.add(rowData); + String brand = normalizeCellText(Objects.toString(rowData.getOrDefault("品牌", ""), "")); + if (!brand.isBlank()) { + uniqueBrands.add(brand); + } + } + return new ParsedBrandFile(sheet.getSheetName(), columns, rows, new ArrayList<>(uniqueBrands)); + } catch (IOException ex) { + throw new BusinessException("读取 Excel 失败"); + } + } + + private List extractHeaders(Row headerRow, DataFormatter formatter) { + List headers = new ArrayList<>(); + Set seen = new LinkedHashSet<>(); + for (int i = 0; i < headerRow.getLastCellNum(); i++) { + Cell cell = headerRow.getCell(i); + String value = normalizeHeaderValue(cell == null ? null : formatter.formatCellValue(cell)); + if (value.isBlank() || seen.contains(value)) { + continue; + } + seen.add(value); + headers.add(value); + if ("缩略图地址8".equals(value)) { + break; + } + } + return headers; + } + + private Map buildHeaderIndexes(Row headerRow, DataFormatter formatter, List columns) { + Map headerIndexes = new LinkedHashMap<>(); + Set allowed = new LinkedHashSet<>(columns); + for (int i = 0; i < headerRow.getLastCellNum(); i++) { + Cell cell = headerRow.getCell(i); + String value = normalizeHeaderValue(cell == null ? null : formatter.formatCellValue(cell)); + if (value.isBlank() || headerIndexes.containsKey(value) || !allowed.contains(value)) { + continue; + } + headerIndexes.put(value, i); + if ("缩略图地址8".equals(value)) { + break; + } + } + return headerIndexes; + } + + private String normalizeHeaderValue(String value) { + String normalized = normalizeCellText(value); + if (normalized.isBlank()) { + return ""; + } + String[] parts = normalized.split("idASIN国家状态价格变体数量", 2); + String trimmed = parts[0].trim(); + return trimmed.isEmpty() ? normalized : trimmed; + } + + private String normalizeCellText(String value) { + if (value == null) { + return ""; + } + return value.replace("\ufeff", "") + .replace("\u3000", " ") + .replace("\r\n", " ") + .replace("\r", " ") + .replace("\n", " ") + .replace("\t", " ") + .trim() + .replaceAll("\\s+", " "); + } + + private void writeBrandWorkbook(File outputFile, + String strategy, + BrandParsedFileCacheDto cachedFile, + BrandCrawlResultFileDto resultFile) throws IOException { + String actualStrategy = normalizeStrategy(strategy); + try (XSSFWorkbook workbook = new XSSFWorkbook()) { + String mainSheetName = blankToDefault(cachedFile.getSheetName(), "Sheet1"); + var mainSheet = workbook.createSheet(mainSheetName); + var headerRow = mainSheet.createRow(0); + List columns = cachedFile.getColumns() == null ? List.of() : cachedFile.getColumns(); + for (int i = 0; i < columns.size(); i++) { + headerRow.createCell(i).setCellValue(columns.get(i)); + } + + Set invalidBrandSet = normalizeBrandSetFromInvalids(resultFile.getInvalidBrands()); + List> sourceRows = cachedFile.getRows() == null ? List.of() : cachedFile.getRows(); + int writeRowIndex = 1; + for (Map rowData : sourceRows) { + String brand = normalizeCellText(Objects.toString(rowData.getOrDefault("品牌", ""), "")); + if (!brand.isBlank() && invalidBrandSet.contains(brand)) { + continue; + } + var row = mainSheet.createRow(writeRowIndex++); + for (int colIndex = 0; colIndex < columns.size(); colIndex++) { + String column = columns.get(colIndex); + row.createCell(colIndex).setCellValue(Objects.toString(rowData.getOrDefault(column, ""), "")); + } + } + + var invalidSheet = workbook.createSheet("不符合品牌"); + var invalidHeader = invalidSheet.createRow(0); + invalidHeader.createCell(0).setCellValue("品牌"); + invalidHeader.createCell(1).setCellValue("国家"); + invalidHeader.createCell(2).setCellValue("状态"); + List invalidBrands = resultFile.getInvalidBrands() == null ? List.of() : resultFile.getInvalidBrands(); + for (int i = 0; i < invalidBrands.size(); i++) { + var row = invalidSheet.createRow(i + 1); + BrandInvalidBrandDto item = invalidBrands.get(i); + row.createCell(0).setCellValue(blankToDefault(item.getBrand(), "")); + row.createCell(1).setCellValue(blankToDefault(item.getCountry(), "")); + row.createCell(2).setCellValue(blankToDefault(item.getStatus(), "")); + } + + var queryFailedSheet = workbook.createSheet("查询失败品牌"); + var queryHeader = queryFailedSheet.createRow(0); + queryHeader.createCell(0).setCellValue("品牌"); + queryHeader.createCell(1).setCellValue("时间"); + List queryFailedBrands = resultFile.getQueryFailedBrands() == null ? List.of() : resultFile.getQueryFailedBrands(); + String nowText = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yy-MM-dd HH:mm:ss")); + for (int i = 0; i < queryFailedBrands.size(); i++) { + var row = queryFailedSheet.createRow(i + 1); + row.createCell(0).setCellValue(blankToDefault(queryFailedBrands.get(i), "")); + row.createCell(1).setCellValue(nowText); + } + + try (FileOutputStream outputStream = new FileOutputStream(outputFile)) { + workbook.write(outputStream); + } + } + } + + private Set normalizeBrandSetFromInvalids(List invalidBrands) { + Set result = new LinkedHashSet<>(); + if (invalidBrands == null) { + return result; + } + for (BrandInvalidBrandDto invalidBrand : invalidBrands) { + if (invalidBrand == null) { + continue; + } + String normalized = normalizeCellText(invalidBrand.getBrand()); + if (!normalized.isBlank()) { + result.add(normalized); + } + } + return result; + } + + private Set normalizeBrandSet(List brands) { + Set result = new LinkedHashSet<>(); + if (brands == null) { + return result; + } + for (String brand : brands) { + String normalized = normalizeCellText(brand); + if (!normalized.isBlank()) { + result.add(normalized); + } + } + return result; + } + + private Map buildAndUploadResult(Long taskId, List entries) throws IOException { + if (entries.isEmpty()) { + throw new BusinessException("没有可上传的结果文件"); + } + List urls = new ArrayList<>(); + for (OutputEntry entry : entries) { + String objectKey = ossStorageService.uploadResultFile(entry.file(), "BRAND"); + urls.add(ossStorageService.generateDownloadUrl(objectKey)); + } + Map result = new LinkedHashMap<>(); + result.put("urls", urls); + if (entries.size() > 1) { + File zipFile = packageAsZip(taskId, entries); + String zipObjectKey = ossStorageService.uploadResultFile(zipFile, "BRAND"); + result.put("zip_url", ossStorageService.generateDownloadUrl(zipObjectKey)); + } + return result; + } + + private File packageAsZip(Long taskId, List entries) throws IOException { + File zipDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "brand-result", String.valueOf(taskId))); + File zipFile = buildNamedOutputFile(zipDir, "brand_task_" + taskId + ".zip"); + try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zipFile))) { + byte[] buffer = new byte[8192]; + for (OutputEntry entry : entries) { + String relativePath = blankToNull(entry.relativePath()); + String entryName; + if (relativePath != null) { + String relativeParent = relativePath.replace('\\', '/'); + int slashIndex = relativeParent.lastIndexOf('/'); + String prefix = slashIndex >= 0 ? relativeParent.substring(0, slashIndex + 1) : ""; + entryName = prefix + entry.file().getName(); + } else { + entryName = entry.file().getName(); + } + zos.putNextEntry(new ZipEntry(entryName)); + try (FileInputStream inputStream = new FileInputStream(entry.file())) { + int len; + while ((len = inputStream.read(buffer)) > 0) { + zos.write(buffer, 0, len); + } + } + zos.closeEntry(); + } + } + return zipFile; + } + + private File buildNamedOutputFile(File outputDir, String filename) { + File candidate = FileUtil.file(outputDir, filename); + if (!candidate.exists()) { + return candidate; + } + String mainName = FileUtil.mainName(filename); + String extName = FileUtil.extName(filename); + int index = 2; + while (true) { + String nextFilename = extName.isEmpty() ? mainName + "_" + index : mainName + "_" + index + "." + extName; + File nextFile = FileUtil.file(outputDir, nextFilename); + if (!nextFile.exists()) { + return nextFile; + } + index++; + } + } + + private String buildResultFilename(String originalFilename) { + String sourceName = blankToDefault(originalFilename, "brand.xlsx"); + return FileUtil.mainName(sourceName) + "_result.xlsx"; + } + + private Integer defaultInteger(Integer value) { + return value == null ? 0 : value; + } + + private String blankToDefault(String value, String defaultValue) { + return value == null || value.isBlank() ? defaultValue : value; + } + + private String blankToNull(String value) { + return value == null || value.isBlank() ? null : value.trim(); + } + + private LegacyBrandLineProgressVo buildLineProgress(Long taskId) { + LegacyBrandLineProgressVo vo = new LegacyBrandLineProgressVo(); + Map progress = brandTaskProgressCacheService.getProgress(taskId); + if (progress.isEmpty()) { + vo.setHas_progress(false); + return vo; + } + LegacyBrandLineProgressInfoVo info = new LegacyBrandLineProgressInfoVo(); + info.setFile_index(parseInteger(progress.get("file_index"))); + info.setFile_total(parseInteger(progress.get("file_total"))); + info.setFile_name(Objects.toString(progress.getOrDefault("file_name", ""), "")); + info.setCurrent_line(parseInteger(progress.get("current_line"))); + info.setTotal_lines(parseInteger(progress.get("total_lines"))); + info.setPhase(Objects.toString(progress.getOrDefault("phase", ""), "")); + vo.setHas_progress(true); + vo.setInfo(info); + return vo; + } + + private int countCompletedFiles(Map> groupedResults) { + int count = 0; + for (List chunks : groupedResults.values()) { + if (!chunks.isEmpty() && Objects.equals(chunks.size(), chunks.get(0).getChunkTotal())) { + count++; + } + } + return count; + } + + private BrandCrawlResultFileDto mergeChunks(List chunks) { + if (chunks == null || chunks.isEmpty()) { + throw new BusinessException("结果分片为空"); + } + BrandCrawlResultFileDto first = chunks.get(0); + if (!Objects.equals(chunks.size(), first.getChunkTotal())) { + throw new BusinessException("文件结果分片未完成"); + } + BrandCrawlResultFileDto merged = new BrandCrawlResultFileDto(); + merged.setFileUrl(first.getFileUrl()); + merged.setOriginalFilename(first.getOriginalFilename()); + merged.setRelativePath(first.getRelativePath()); + merged.setMainSheetName(first.getMainSheetName()); + merged.setChunkIndex(first.getChunkIndex()); + merged.setChunkTotal(first.getChunkTotal()); + merged.setTotalLines(first.getTotalLines()); + List keptRows = new ArrayList<>(); + List invalidBrands = new ArrayList<>(); + List queryFailedBrands = new ArrayList<>(); + for (BrandCrawlResultFileDto chunk : chunks) { + if (chunk.getKeptRows() != null) { + keptRows.addAll(chunk.getKeptRows()); + } + if (chunk.getInvalidBrands() != null) { + invalidBrands.addAll(chunk.getInvalidBrands()); + } + if (chunk.getQueryFailedBrands() != null) { + queryFailedBrands.addAll(chunk.getQueryFailedBrands()); + } + } + merged.setKeptRows(keptRows); + merged.setInvalidBrands(invalidBrands); + merged.setQueryFailedBrands(queryFailedBrands); + return merged; + } + + private Integer parseInteger(Object value) { + if (value == null) { + return 0; + } + try { + return Integer.parseInt(String.valueOf(value)); + } catch (Exception ex) { + return 0; + } + } + + @Transactional + @org.springframework.scheduling.annotation.Scheduled(cron = "${aiimage.brand-progress.stale-check-cron:0 */2 * * * *}") + public void failStaleRunningTasks() { + LocalDateTime threshold = LocalDateTime.now().minusMinutes(brandProgressProperties.getHeartbeatTimeoutMinutes()); + List runningTasks = brandCrawlTaskMapper.selectList(new LambdaQueryWrapper() + .eq(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING) + .lt(BrandCrawlTaskEntity::getUpdatedAt, threshold)); + for (BrandCrawlTaskEntity task : runningTasks) { + Map progress = brandTaskProgressCacheService.getProgress(task.getId()); + if (progress.isEmpty()) { + continue; + } + long lastHeartbeatAt = 0L; + try { + lastHeartbeatAt = Long.parseLong(String.valueOf(progress.getOrDefault("last_heartbeat_at", "0"))); + } catch (Exception ignored) { + } + if (lastHeartbeatAt <= 0) { + continue; + } + LocalDateTime lastHeartbeat = LocalDateTime.ofInstant(Instant.ofEpochMilli(lastHeartbeatAt), ZoneId.systemDefault()); + if (lastHeartbeat.isAfter(threshold)) { + continue; + } + brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper() + .eq(BrandCrawlTaskEntity::getId, task.getId()) + .eq(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING) + .set(BrandCrawlTaskEntity::getStatus, STATUS_FAILED) + .set(BrandCrawlTaskEntity::getErrorMessage, "前端长时间无响应，任务已自动失败")); + brandTaskProgressCacheService.markFailed(task.getId(), "前端长时间无响应，任务已自动失败"); + } + } + + private record ParsedBrandFile(String sheetName, List columns, List> rows, + List uniqueBrands) { + } + + private record OutputEntry(String relativePath, File file) { + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeRunController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeRunController.java index 850266a..26f2571 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeRunController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeRunController.java @@ -46,6 +46,8 @@ public class DedupeRunController { - keepIntegerIds=true 时保留纯数字 ID； - keepUnderscoreIds=true 时保留类似 1_1 的 ID； - keepIntegerMainIdsWhenNoSubIds=true 且文件中不存在 1_1 这类子 ID 时，会自动保留纯数字主 ID； + - 整理后的数据会再按 ASIN 与去重总数据表比对，命中的行不会出现在最终输出文件； + - 上传文件内如果存在重复 ASIN，只保留第一次出现的那一行； - 输出文件命名遵循 原文件名_cleaned.xlsx，若重名自动追加序号。 """) @ApiResponses({ diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/mapper/DedupeTotalDataMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/mapper/DedupeTotalDataMapper.java index 6999757..6b9f8a6 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/mapper/DedupeTotalDataMapper.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/mapper/DedupeTotalDataMapper.java @@ -3,13 +3,29 @@ package com.nanri.aiimage.modules.dedupe.mapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.nanri.aiimage.modules.dedupe.model.entity.DedupeTotalDataEntity; import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.SelectProvider; import java.util.List; +import java.util.StringJoiner; @Mapper public interface DedupeTotalDataMapper extends BaseMapper { @Select("SELECT data_value FROM biz_dedupe_total_data") List selectAllDataValues(); + + @SelectProvider(type = SqlProvider.class, method = "selectExistingDataValues") + List selectExistingDataValues(@Param("values") List values); + + class SqlProvider { + public String selectExistingDataValues(@Param("values") List values) { + StringJoiner placeholders = new StringJoiner(", "); + for (int i = 0; i < values.size(); i++) { + placeholders.add("#{values[" + i + "]}"); + } + return "SELECT data_value FROM biz_dedupe_total_data WHERE data_value IN (" + placeholders + ")"; + } + } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java index dbc004a..2c44bca 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java @@ -32,8 +32,10 @@ import java.nio.file.Files; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -72,7 +74,6 @@ public class DedupeRunService { int successCount = 0; int failedCount = 0; List archiveEntries = new ArrayList<>(); - java.util.Set totalDataValues = dedupeTotalDataService.listComparableValues(); for (DedupeSourceFileDto sourceFile : request.getFiles()) { DedupeResultItemVo item = new DedupeResultItemVo(); @@ -94,8 +95,7 @@ public class DedupeRunService { request.getSelectedColumns(), request.isKeepIntegerIds(), request.isKeepUnderscoreIds(), - request.isKeepIntegerMainIdsWhenNoSubIds(), - totalDataValues + request.isKeepIntegerMainIdsWhenNoSubIds() ); if (folderMode) { @@ -223,8 +223,7 @@ public class DedupeRunService { private void cleanExcelByLegacyRules(File inputFile, File outputFile, List selectedColumns, boolean keepIntegerIds, boolean keepUnderscoreIds, - boolean keepIntegerMainIdsWhenNoSubIds, - java.util.Set totalDataValues) throws Exception { + boolean keepIntegerMainIdsWhenNoSubIds) throws Exception { DataFormatter formatter = new DataFormatter(); try (FileInputStream fis = new FileInputStream(inputFile); Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis); @@ -271,7 +270,8 @@ public class DedupeRunService { // The preliminary global scan for mainIdHasSubIdMap was removed. // We now determine subset existence contextually (per group) during the main loop. } - int outputRowIndex = 1; + List candidateRows = new ArrayList<>(); + Set candidateAsinValues = new HashSet<>(); for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) { Row row = sheet.getRow(rowNum); if (row == null) { @@ -283,10 +283,28 @@ public class DedupeRunService { continue; } } - if (asinColumnIndex != null && !totalDataValues.isEmpty()) { + candidateRows.add(row); + if (asinColumnIndex != null) { String asinValue = dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex))); - if (!asinValue.isBlank() && totalDataValues.contains(asinValue)) { - continue; + if (!asinValue.isBlank()) { + candidateAsinValues.add(asinValue); + } + } + } + + Set matchedAsinValues = dedupeTotalDataService.findExistingComparableValues(candidateAsinValues); + Set writtenAsinValues = new HashSet<>(); + int outputRowIndex = 1; + for (Row row : candidateRows) { + if (asinColumnIndex != null) { + String asinValue = dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex))); + if (!asinValue.isBlank()) { + if (!matchedAsinValues.isEmpty() && matchedAsinValues.contains(asinValue)) { + continue; + } + if (!writtenAsinValues.add(asinValue)) { + continue; + } } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java index 6e54b1e..33b623c 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java @@ -24,6 +24,7 @@ import org.springframework.web.multipart.MultipartFile; import java.io.ByteArrayInputStream; import java.io.InputStream; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -35,6 +36,8 @@ import java.util.concurrent.ConcurrentHashMap; @RequiredArgsConstructor public class DedupeTotalDataService { + private static final int COMPARE_BATCH_SIZE = 1000; + private final DedupeTotalDataMapper dedupeTotalDataMapper; private final Map importProgressMap = new ConcurrentHashMap<>(); private final Map deleteImportProgressMap = new ConcurrentHashMap<>(); @@ -76,6 +79,26 @@ public class DedupeTotalDataService { .collect(java.util.stream.Collectors.toSet()); } + public Set findExistingComparableValues(Collection values) { + if (values == null || values.isEmpty()) { + return Set.of(); + } + List normalizedValues = values.stream() + .map(this::normalizeComparableValueOrBlank) + .filter(value -> !value.isEmpty()) + .distinct() + .toList(); + if (normalizedValues.isEmpty()) { + return Set.of(); + } + Set existingValues = new HashSet<>(); + for (int start = 0; start < normalizedValues.size(); start += COMPARE_BATCH_SIZE) { + int end = Math.min(start + COMPARE_BATCH_SIZE, normalizedValues.size()); + existingValues.addAll(dedupeTotalDataMapper.selectExistingDataValues(normalizedValues.subList(start, end))); + } + return existingValues; + } + public DedupeTotalDataImportStartVo startImport(MultipartFile file) { if (file == null || file.isEmpty()) { throw new BusinessException("请上传 xlsx 文件"); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalFileStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalFileStorageService.java index 865bf52..1263a42 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalFileStorageService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalFileStorageService.java @@ -96,7 +96,7 @@ public class LocalFileStorageService { } } - private File findLocalSourceFile(String fileKey) { + public File findLocalSourceFile(String fileKey) { File baseDir = FileUtil.file(storageProperties.getLocalTempDir()); if (!baseDir.exists()) { return null; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalTempCleanupService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalTempCleanupService.java new file mode 100644 index 0000000..fc70404 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalTempCleanupService.java @@ -0,0 +1,116 @@ +package com.nanri.aiimage.modules.file.service; + +import cn.hutool.core.io.FileUtil; +import com.nanri.aiimage.config.StorageProperties; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.regex.Pattern; + +@Slf4j +@Service +@RequiredArgsConstructor +public class LocalTempCleanupService { + + private static final Pattern ROOT_TEMP_FILE_PATTERN = Pattern.compile("^[a-fA-F0-9]{32}(\\.[^.]+)?$"); + private static final List RESULT_DIR_NAMES = List.of( + "dedupe-result", + "convert-result", + "split-result", + "brand-source-download", + "brand-result" + ); + + private final StorageProperties storageProperties; + + @Scheduled(cron = "${aiimage.storage.cleanup-cron:0 0 */6 * * *}") + public void cleanupLocalTempDir() { + if (!storageProperties.isCleanupEnabled()) { + return; + } + File tempDir = FileUtil.file(storageProperties.getLocalTempDir()); + if (!tempDir.exists() || !tempDir.isDirectory()) { + return; + } + + Instant sourceExpireBefore = Instant.now().minus(storageProperties.getSourceRetentionHours(), ChronoUnit.HOURS); + Instant resultExpireBefore = Instant.now().minus(storageProperties.getResultRetentionHours(), ChronoUnit.HOURS); + + int deletedSourceCount = 0; + int deletedResultCount = 0; + File[] children = tempDir.listFiles(); + if (children == null || children.length == 0) { + return; + } + + for (File child : children) { + try { + if (child.isFile() && isManagedRootTempFile(child) && isExpired(child, sourceExpireBefore)) { + if (FileUtil.del(child)) { + deletedSourceCount++; + } + continue; + } + if (child.isDirectory() && RESULT_DIR_NAMES.contains(child.getName())) { + deletedResultCount += deleteExpiredChildrenRecursively(child, resultExpireBefore); + deleteEmptyDirectories(child, tempDir); + } + } catch (Exception ex) { + log.warn("清理本地临时文件失败: path={}", child.getAbsolutePath(), ex); + } + } + + if (deletedSourceCount > 0 || deletedResultCount > 0) { + log.info("本地临时目录清理完成: sourceDeleted={}, resultDeleted={}, tempDir={}", deletedSourceCount, deletedResultCount, tempDir.getAbsolutePath()); + } + } + + private int deleteExpiredChildrenRecursively(File file, Instant expireBefore) { + int deletedCount = 0; + if (file.isDirectory()) { + File[] children = file.listFiles(); + if (children != null) { + for (File child : children) { + deletedCount += deleteExpiredChildrenRecursively(child, expireBefore); + } + } + if (isExpired(file, expireBefore) && isDirectoryEmpty(file) && FileUtil.del(file)) { + deletedCount++; + } + return deletedCount; + } + if (isExpired(file, expireBefore) && FileUtil.del(file)) { + return 1; + } + return 0; + } + + private void deleteEmptyDirectories(File directory, File stopAt) { + File current = directory; + while (current != null && !current.equals(stopAt) && isDirectoryEmpty(current)) { + if (!FileUtil.del(current)) { + return; + } + current = current.getParentFile(); + } + } + + private boolean isManagedRootTempFile(File file) { + return ROOT_TEMP_FILE_PATTERN.matcher(file.getName()).matches(); + } + + private boolean isExpired(File file, Instant expireBefore) { + return Instant.ofEpochMilli(file.lastModified()).isBefore(expireBefore); + } + + private boolean isDirectoryEmpty(File directory) { + File[] children = directory.listFiles(); + return children == null || children.length == 0; + } +} diff --git a/backend-java/src/main/resources/application.yml b/backend-java/src/main/resources/application.yml index 153f5e6..92262f2 100644 --- a/backend-java/src/main/resources/application.yml +++ b/backend-java/src/main/resources/application.yml @@ -15,6 +15,13 @@ spring: url: ${AIIMAGE_DB_URL:jdbc:mysql://127.0.0.1:3306/aiimage?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false} username: ${AIIMAGE_DB_USERNAME:root} password: ${AIIMAGE_DB_PASSWORD:change-me} + data: + redis: + host: ${AIIMAGE_REDIS_HOST:127.0.0.1} + port: ${AIIMAGE_REDIS_PORT:6379} + password: ${AIIMAGE_REDIS_PASSWORD:} + database: ${AIIMAGE_REDIS_DATABASE:0} + timeout: ${AIIMAGE_REDIS_TIMEOUT:5s} management: health: @@ -56,3 +63,12 @@ aiimage: access-key-secret: ${AIIMAGE_OSS_ACCESS_KEY_SECRET:change-me} storage: local-temp-dir: ${AIIMAGE_STORAGE_LOCAL_TEMP_DIR:./data/tmp} + cleanup-enabled: ${AIIMAGE_STORAGE_CLEANUP_ENABLED:true} + cleanup-cron: ${AIIMAGE_STORAGE_CLEANUP_CRON:0 0 */6 * * *} + source-retention-hours: ${AIIMAGE_STORAGE_SOURCE_RETENTION_HOURS:48} + result-retention-hours: ${AIIMAGE_STORAGE_RESULT_RETENTION_HOURS:24} + brand-progress: + ttl-hours: ${AIIMAGE_BRAND_PROGRESS_TTL_HOURS:24} + failed-ttl-hours: ${AIIMAGE_BRAND_PROGRESS_FAILED_TTL_HOURS:2} + heartbeat-timeout-minutes: ${AIIMAGE_BRAND_PROGRESS_HEARTBEAT_TIMEOUT_MINUTES:15} + stale-check-cron: ${AIIMAGE_BRAND_PROGRESS_STALE_CHECK_CRON:0 */2 * * * *}