super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

完成品牌部分改造

Browse Source
This commit is contained in:
super
2026-03-24 22:22:37 +08:00
parent 4f728107d1
commit 485b4dd485
37 changed files with 2086 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend-java/pom.xml
View File

@@ -42,6 +42,10 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-security</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>
</dependency>
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-spring-boot3-starter</artifactId>

2
backend-java/src/main/java/com/nanri/aiimage/AiImageApplication.java
View File

@@ -2,8 +2,10 @@ package com.nanri.aiimage;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableScheduling;
@SpringBootApplication
@EnableScheduling
public class AiImageApplication {
public static void main(String[] args) {

13
backend-java/src/main/java/com/nanri/aiimage/config/BrandProgressProperties.java Normal file
View File

@@ -0,0 +1,13 @@
package com.nanri.aiimage.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
@Data
@ConfigurationProperties(prefix = "aiimage.brand-progress")
public class BrandProgressProperties {
private long ttlHours = 24;
private long failedTtlHours = 2;
private long heartbeatTimeoutMinutes = 15;
private String staleCheckCron = "0 */2 * * * *";
}

2
backend-java/src/main/java/com/nanri/aiimage/config/PropertiesConfig.java
View File

@@ -4,6 +4,6 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
import org.springframework.context.annotation.Configuration;
@Configuration
@EnableConfigurationProperties({OssProperties.class, StorageProperties.class})
@EnableConfigurationProperties({OssProperties.class, StorageProperties.class, BrandProgressProperties.class})
public class PropertiesConfig {
}

4
backend-java/src/main/java/com/nanri/aiimage/config/StorageProperties.java
View File

@@ -7,4 +7,8 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
@ConfigurationProperties(prefix = "aiimage.storage")
public class StorageProperties {
private String localTempDir;
private boolean cleanupEnabled = true;
private String cleanupCron = "0 0 */6 * * *";
private long sourceRetentionHours = 48;
private long resultRetentionHours = 24;
}

207
backend-java/src/main/java/com/nanri/aiimage/modules/brand/controller/BrandTaskController.java Normal file
View File

@@ -0,0 +1,207 @@
package com.nanri.aiimage.modules.brand.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultRequest;
import com.nanri.aiimage.modules.brand.model.dto.BrandTaskCreateRequest;
import com.nanri.aiimage.modules.brand.model.vo.BrandCrawlPayloadVo;
import com.nanri.aiimage.modules.brand.model.vo.BrandSimpleVo;
import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandTaskDetailVo;
import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandTaskListVo;
import com.nanri.aiimage.modules.brand.service.BrandTaskService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpHeaders;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/brand")
@Tag(name = "品牌任务", description = "品牌任务接口。Java 端只负责下载并解析 OSS 文件、返回待爬取数据、接收爬取结果并生成 xlsx/上传 OSS/落旧表，不负责实际爬虫执行。")
public class BrandTaskController {
private final BrandTaskService brandTaskService;
@PostMapping("/tasks")
@Operation(
summary = "创建品牌任务并返回待爬取数据",
description = """
创建一条品牌任务记录到旧表 brand_crawl_tasks，随后立即下载并读取 Excel，
将原始解析结果临时缓存到后端，并把 sheetName、columns、rows、uniqueBrands 等待爬取数据返回给前端。
适用流程：
1. 前端先拿到 OSS 文件链接；
2. 再调用本接口创建任务；
3. 前端拿到返回数据后自行执行品牌爬取；
4. 爬取结束后调用结果提交接口回传结果。
""")
@ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "创建成功，返回任务 ID 和待爬取数据", content = @Content(schema = @Schema(implementation = BrandCrawlPayloadVo.class))),
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "请求参数不合法或文件列表为空"),
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "500", description = "下载或解析 Excel 或创建任务失败")
})
public ApiResponse<BrandCrawlPayloadVo> createTask(
@Parameter(description = "用户 ID", required = true) @RequestParam Long userId,
@Valid @RequestBody BrandTaskCreateRequest request) {
return ApiResponse.success(brandTaskService.createTaskAndBuildPayload(userId, request));
}
@GetMapping("/tasks")
@Operation(
summary = "获取品牌任务列表",
description = """
查询旧表 brand_crawl_tasks 最近的任务列表，返回前端任务面板需要的核心信息。
返回内容包括：
- id：任务 ID；
- file_paths：任务创建时记录的源文件信息；
- desc：任务描述，通常由 strategy + 文件名摘要组成；
- strategy：品牌匹配方式；
- status：任务状态（pending/running/success/failed/cancelled）；
- result_paths：结果文件地址信息，通常包含 urls 和 zip_url；
- error_message：失败原因；
- progress_current / progress_total：当前进度；
- created_at / updated_at：创建和更新时间。
典型用途：
- 页面任务列表展示；
- 判断任务是否已完成；
- 决定是否显示下载按钮；
- 展示失败原因和进度。
"""
)
@ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功，返回任务列表", content = @Content(schema = @Schema(implementation = LegacyBrandTaskListVo.class)))
})
public ApiResponse<LegacyBrandTaskListVo> listTasks(
@Parameter(description = "用户 ID", required = true) @RequestParam Long userId) {
return ApiResponse.success(brandTaskService.listTasksLegacy(userId));
}
@GetMapping("/tasks/{taskId}")
@Operation(
summary = "获取品牌任务详情",
description = """
根据任务 ID 查询单条品牌任务详情，返回任务基础信息 + 文件级摘要进度 + Redis 中基于结果分片累计出的实时行级进度。
返回内容包括：
- task.id：任务 ID；
- task.file_paths：任务绑定的源文件信息；
- task.desc / task.strategy / task.status：任务描述、策略、状态；
- task.result_paths：结果文件地址。单文件场景通常只有 urls，多文件场景通常同时包含 urls 和 zip_url；
- task.error_message：失败原因；
- task.progress_current / task.progress_total：文件级摘要进度，表示已完成文件数 / 总文件数；
- task.created_at / task.updated_at：创建和更新时间；
- line_progress.has_progress：当前是否存在 Redis 实时进度；
- line_progress.info.file_index / file_total：当前处理到第几个文件 / 总文件数；
- line_progress.info.file_name：当前处理文件名；
- line_progress.info.current_line / total_lines：当前文件已处理行数 / 总行数；
- line_progress.info.phase：当前阶段，可能为 crawling / assembling / uploading / failed。
典型用途：
- 前端轮询任务状态；
- 展示当前文件与当前行进度；
- 提交结果后确认任务是否已 success；
- 下载前校验任务结果是否已生成；
- 任务失败时获取具体错误信息。
"""
)
@ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功，返回单条任务详情", content = @Content(schema = @Schema(implementation = LegacyBrandTaskDetailVo.class))),
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "任务不存在")
})
public ApiResponse<LegacyBrandTaskDetailVo> getTask(
@Parameter(description = "品牌任务 ID", required = true) @PathVariable Long taskId) {
return ApiResponse.success(brandTaskService.getTaskDetailLegacy(taskId));
}
@PostMapping("/tasks/{taskId}/result")
@Operation(
summary = "提交品牌任务爬取结果",
description = """
前端完成品牌爬取后，将按品牌汇总后的结果分片提交给本接口。
Java 端会累计接收前端每次提交的少量结果（例如 5 个品牌），
并直接根据 keptRows / invalidBrands / queryFailedBrands 的数量推进 Redis 中的实时进度。
在所有文件都提交完成后再基于创建任务时缓存的原始数据统一：
- 按“品牌”列删除 invalidBrands 对应的原始行，生成主 sheet；
- keptRows 用于统计已处理且保留的品牌进度；
- 生成“不符合品牌”和“查询失败品牌” sheet；
- 输出 xlsx；
- 上传 OSS；
- 回写 brand_crawl_tasks.result_paths/status/error_message；
处理期间前端可轮询任务详情接口 `GET /api/brand/tasks/{taskId}`，
获取 status、progress_current、progress_total、error_message、line_progress 等实时进度信息。
""")
@ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "提交成功，结果文件已生成并落库", content = @Content(schema = @Schema(implementation = BrandSimpleVo.class))),
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "请求参数不合法、fileUrl 不匹配或结果数据为空"),
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "500", description = "生成结果文件或上传 OSS 失败")
})
public ApiResponse<BrandSimpleVo> submitResult(
@Parameter(description = "品牌任务 ID", required = true) @PathVariable Long taskId,
@Valid @RequestBody BrandCrawlResultRequest request) {
brandTaskService.submitCrawlResult(taskId, request);
return ApiResponse.success(new BrandSimpleVo(true));
}
@PostMapping("/tasks/{taskId}/cancel")
@Operation(
summary = "取消品牌任务",
description = "将任务状态从 pending/running 更新为 cancelled。适用于前端停止本地爬取时同步取消任务记录。"
)
@ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "取消成功", content = @Content(schema = @Schema(implementation = BrandSimpleVo.class))),
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "任务不存在或当前状态不可取消")
})
public ApiResponse<BrandSimpleVo> cancelTask(
@Parameter(description = "品牌任务 ID", required = true) @PathVariable Long taskId) {
brandTaskService.cancelTask(taskId);
return ApiResponse.success(new BrandSimpleVo(true));
}
@DeleteMapping("/tasks/{taskId}")
@Operation(
summary = "删除品牌任务",
description = "删除旧表 brand_crawl_tasks 中的任务记录。仅允许删除非 running 状态任务。"
)
@ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "删除成功", content = @Content(schema = @Schema(implementation = BrandSimpleVo.class))),
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "任务不存在或正在执行中不可删除")
})
public ApiResponse<BrandSimpleVo> deleteTask(
@Parameter(description = "品牌任务 ID", required = true) @PathVariable Long taskId) {
brandTaskService.deleteTask(taskId);
return ApiResponse.success(new BrandSimpleVo(true));
}
@GetMapping("/tasks/{taskId}/download")
@Operation(
summary = "下载品牌任务结果",
description = "根据任务 ID 读取 brand_crawl_tasks.result_paths，优先跳转 zip_url；若没有 zip_url，则跳转第一个可用结果 URL。"
)
@ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "302", description = "重定向到 OSS 结果文件地址"),
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "任务不存在或无结果可下载")
})
public ResponseEntity<Void> download(
@Parameter(description = "品牌任务 ID", required = true) @PathVariable Long taskId) {
String url = brandTaskService.resolveDownloadUrl(taskId);
return ResponseEntity.status(302)
.header(HttpHeaders.LOCATION, url)
.build();
}
}

9
backend-java/src/main/java/com/nanri/aiimage/modules/brand/mapper/BrandCrawlTaskMapper.java Normal file
View File

@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.brand.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface BrandCrawlTaskMapper extends BaseMapper<BrandCrawlTaskEntity> {
}

48
backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandCrawlResultFileDto.java Normal file
View File

@@ -0,0 +1,48 @@
package com.nanri.aiimage.modules.brand.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "单个品牌文件结果分片。前端只需按品牌提交保留品牌、不符合品牌、查询失败品牌；后端会基于创建任务时缓存的原始行数据生成最终结果文件。")
public class BrandCrawlResultFileDto {
@NotBlank(message = "fileUrl 不能为空")
@Schema(description = "源文件 OSS 链接，必须与创建任务时的文件项一一对应。")
private String fileUrl;
@Schema(description = "原始文件名。为空时会回退使用任务创建时记录的 originalFilename。")
private String originalFilename;
@Schema(description = "相对路径。文件夹上传场景下用于生成 zip 时恢复目录层级。")
private String relativePath;
@Schema(description = "主 sheet 名称。为空时默认输出为 Sheet1。")
private String mainSheetName;
@NotNull(message = "chunkIndex 不能为空")
@Schema(description = "当前分片序号，从 1 开始。", example = "1")
private Integer chunkIndex;
@NotNull(message = "chunkTotal 不能为空")
@Schema(description = "当前文件总分片数。", example = "10")
private Integer chunkTotal;
@Schema(description = "当前文件总行数。后端据此展示精确行级进度。", example = "1000")
private Integer totalLines;
@Schema(description = "本批处理后应保留的品牌列表。仅用于后端累计进度，不需要回传整行数据。")
private List<String> keptRows;
@Valid
@Schema(description = "本批处理后判定为不符合品牌的记录列表。会写入“不符合品牌” sheet；删除主 sheet 数据时仅按 brand 字段匹配删除。")
private List<BrandInvalidBrandDto> invalidBrands;
@Schema(description = "本批查询失败的品牌列表。后端会写入“查询失败品牌” sheet。")
private List<String> queryFailedBrands;
}

21
backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandCrawlResultRequest.java Normal file
View File

@@ -0,0 +1,21 @@
package com.nanri.aiimage.modules.brand.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotEmpty;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "品牌爬取结果提交请求。前端按文件分片提交不符合品牌数据和查询失败品牌数据；后端会基于创建任务时缓存的原始行数据生成最终结果文件。")
public class BrandCrawlResultRequest {
@Schema(description = "品牌匹配方式。应与创建任务时使用的 strategy 保持一致。", example = "Terms")
private String strategy = "Terms";
@Valid
@NotEmpty(message = "files 不能为空")
@Schema(description = "本次提交的文件结果分片列表。每个分片只需要提交 fileUrl、chunkIndex、chunkTotal、totalLines、keptRows、invalidBrands、queryFailedBrands。invalidBrands 需包含品牌、国家、状态；主 sheet 删除时仅按品牌字段匹配。")
private List<BrandCrawlResultFileDto> files;
}

18
backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandInvalidBrandDto.java Normal file
View File

@@ -0,0 +1,18 @@
package com.nanri.aiimage.modules.brand.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "不符合品牌记录。Java 会将该列表输出到结果 xlsx 的“不符合品牌” sheet。")
public class BrandInvalidBrandDto {
@Schema(description = "命中的品牌名称。")
private String brand;
@Schema(description = "命中的国家/地区。")
private String country;
@Schema(description = "命中的品牌状态，例如 已注册、待决。")
private String status;
}

16
backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandParsedFileCacheDto.java Normal file
View File

@@ -0,0 +1,16 @@
package com.nanri.aiimage.modules.brand.model.dto;
import lombok.Data;
import java.util.List;
import java.util.Map;
@Data
public class BrandParsedFileCacheDto {
private String fileUrl;
private String originalFilename;
private String relativePath;
private String sheetName;
private List<String> columns;
private List<Map<String, Object>> rows;
}

15
backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandQueryFailedDto.java Normal file
View File

@@ -0,0 +1,15 @@
package com.nanri.aiimage.modules.brand.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "查询失败品牌记录。Java 会将该列表输出到结果 xlsx 的“查询失败品牌” sheet。")
public class BrandQueryFailedDto {
@Schema(description = "查询失败的品牌名称。")
private String brand;
@Schema(description = "失败时间。由前端或爬虫侧生成后原样回传。")
private String time;
}

20
backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandSourceFileDto.java Normal file
View File

@@ -0,0 +1,20 @@
package com.nanri.aiimage.modules.brand.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotBlank;
import lombok.Data;
@Data
@Schema(description = "品牌任务源文件。由 OSS 文件链接和文件元信息组成。")
public class BrandSourceFileDto {
@NotBlank(message = "fileUrl 不能为空")
@Schema(description = "OSS 文件下载链接，Java 将先下载文件再读取内容。", example = "https://example.oss-cn-hangzhou.aliyuncs.com/source/brand.xlsx")
private String fileUrl;
@Schema(description = "原始文件名，用于生成任务描述和结果文件名。", example = "品牌样例.xlsx")
private String originalFilename;
@Schema(description = "相对路径。文件夹上传场景下可传，用于最终 zip 结果保持目录结构。", example = "店铺A/品牌样例.xlsx")
private String relativePath;
}

27
backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandTaskCreateRequest.java Normal file
View File

@@ -0,0 +1,27 @@
package com.nanri.aiimage.modules.brand.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotEmpty;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "品牌任务创建请求。前端将 OSS 文件链接和 originalFilename/relativePath 传入本请求。Java 会创建 brand_crawl_tasks 记录并下载文件返回待爬取数据。")
public class BrandTaskCreateRequest {
@Valid
@NotEmpty(message = "请先上传待处理文件")
@Schema(description = "已上传源文件列表。每个元素对应一个待解析的 Excel 文件。")
private List<BrandSourceFileDto> files;
@Schema(description = "品牌匹配方式。支持 Terms=精确匹配、Simple=嵌入匹配。默认 Terms。", example = "Terms")
private String strategy = "Terms";
@Schema(description = "任务类型。1=立即处理并返回待爬取数据，2=仅登记任务。当前推荐传 1。", example = "1")
private Integer taskType = 1;
@Schema(description = "文件夹上传场景下可选的归档名称。当前版本仅保留字段，不参与主流程。")
private String archiveName;
}

20
backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/dto/BrandTaskRunRequest.java Normal file
View File

@@ -0,0 +1,20 @@
package com.nanri.aiimage.modules.brand.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "品牌任务兼容立即执行请求")
public class BrandTaskRunRequest {
@Schema(description = "旧页面传入的本地路径列表，仅兼容旧接口")
private List<String> paths;
@Schema(description = "品牌匹配方式: Terms/Simple")
private String strategy = "Terms";
@Schema(description = "任务类型: 1=立即执行")
private Integer taskType = 1;
}

33
backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/entity/BrandCrawlTaskEntity.java Normal file
View File

@@ -0,0 +1,33 @@
package com.nanri.aiimage.modules.brand.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("brand_crawl_tasks")
public class BrandCrawlTaskEntity {
@TableId(type = IdType.AUTO)
private Long id;
private Long userId;
private String filePaths;
private String status;
private Integer taskType;
private String resultPaths;
private String errorMessage;
private Integer progressCurrent;
private Integer progressTotal;
@TableField("`desc`")
private String desc;
private String strategy;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}

28
backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandCrawlPayloadVo.java Normal file
View File

@@ -0,0 +1,28 @@
package com.nanri.aiimage.modules.brand.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.List;
import java.util.Map;
@Data
@Schema(description = "品牌任务爬取载荷")
public class BrandCrawlPayloadVo {
private Long taskId;
private String strategy;
private List<BrandCrawlPayloadFileVo> files;
@Data
public static class BrandCrawlPayloadFileVo {
private Integer fileIndex;
private String fileUrl;
private String originalFilename;
private String relativePath;
private String sheetName;
private List<String> columns;
private List<Map<String, Object>> rows;
private List<String> uniqueBrands;
}
}

13
backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandSimpleVo.java Normal file
View File

@@ -0,0 +1,13 @@
package com.nanri.aiimage.modules.brand.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
@Schema(description = "简单成功响应")
public class BrandSimpleVo {
private boolean success;
}

11
backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandTaskCreateVo.java Normal file
View File

@@ -0,0 +1,11 @@
package com.nanri.aiimage.modules.brand.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "品牌任务创建结果")
public class BrandTaskCreateVo {
private Long taskId;
}

11
backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandTaskDetailVo.java Normal file
View File

@@ -0,0 +1,11 @@
package com.nanri.aiimage.modules.brand.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "品牌任务详情")
public class BrandTaskDetailVo {
private BrandTaskItemVo task;
}

23
backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandTaskItemVo.java Normal file
View File

@@ -0,0 +1,23 @@
package com.nanri.aiimage.modules.brand.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "品牌任务项")
public class BrandTaskItemVo {
private Long id;
private List<Object> filePaths;
private String desc;
private String strategy;
private String status;
private Object resultPaths;
private String errorMessage;
private Integer progressCurrent;
private Integer progressTotal;
private String createdAt;
private String updatedAt;
}

13
backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/BrandTaskListVo.java Normal file
View File

@@ -0,0 +1,13 @@
package com.nanri.aiimage.modules.brand.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "品牌任务列表")
public class BrandTaskListVo {
private List<BrandTaskItemVo> items;
}

27
backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandLineProgressInfoVo.java Normal file
View File

@@ -0,0 +1,27 @@
package com.nanri.aiimage.modules.brand.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "品牌任务实时进度详情")
public class LegacyBrandLineProgressInfoVo {
@Schema(description = "当前处理到第几个文件，从 1 开始")
private Integer file_index;
@Schema(description = "本次任务总文件数")
private Integer file_total;
@Schema(description = "当前处理文件名")
private String file_name;
@Schema(description = "当前文件已处理行数")
private Integer current_line;
@Schema(description = "当前文件总行数")
private Integer total_lines;
@Schema(description = "当前阶段：crawling/assembling/uploading/failed")
private String phase;
}

15
backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandLineProgressVo.java Normal file
View File

@@ -0,0 +1,15 @@
package com.nanri.aiimage.modules.brand.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "品牌任务实时行级进度")
public class LegacyBrandLineProgressVo {
@Schema(description = "当前是否存在实时进度")
private boolean has_progress;
@Schema(description = "实时进度详情")
private LegacyBrandLineProgressInfoVo info;
}

9
backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandTaskCreateVo.java Normal file
View File

@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.brand.model.vo;
import lombok.Data;
@Data
public class LegacyBrandTaskCreateVo {
private Long task_id;
}

15
backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandTaskDetailVo.java Normal file
View File

@@ -0,0 +1,15 @@
package com.nanri.aiimage.modules.brand.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "品牌任务详情响应")
public class LegacyBrandTaskDetailVo {
@Schema(description = "任务基础信息")
private LegacyBrandTaskItemVo task;
@Schema(description = "实时行级进度信息，来自 Redis")
private LegacyBrandLineProgressVo line_progress;
}

44
backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandTaskItemVo.java Normal file
View File

@@ -0,0 +1,44 @@
package com.nanri.aiimage.modules.brand.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "品牌任务基础信息")
public class LegacyBrandTaskItemVo {
@Schema(description = "任务 ID")
private Long id;
@Schema(description = "任务绑定的源文件信息")
private List<Object> file_paths;
@Schema(description = "任务描述")
private String desc;
@Schema(description = "品牌匹配策略")
private String strategy;
@Schema(description = "任务状态：pending/running/success/failed/cancelled")
private String status;
@Schema(description = "结果文件地址信息。单文件场景通常包含 urls，多文件场景通常同时包含 urls 和 zip_url")
private Object result_paths;
@Schema(description = "失败原因")
private String error_message;
@Schema(description = "文件级摘要进度：已完成文件数")
private Integer progress_current;
@Schema(description = "文件级摘要进度：总文件数")
private Integer progress_total;
@Schema(description = "创建时间")
private String created_at;
@Schema(description = "更新时间")
private String updated_at;
}

11
backend-java/src/main/java/com/nanri/aiimage/modules/brand/model/vo/LegacyBrandTaskListVo.java Normal file
View File

@@ -0,0 +1,11 @@
package com.nanri.aiimage.modules.brand.model.vo;
import lombok.Data;
import java.util.List;
@Data
public class LegacyBrandTaskListVo {
private List<LegacyBrandTaskItemVo> items;
}

182
backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskProgressCacheService.java Normal file
View File

@@ -0,0 +1,182 @@
package com.nanri.aiimage.modules.brand.service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.BrandProgressProperties;
import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultFileDto;
import lombok.RequiredArgsConstructor;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@Service
@RequiredArgsConstructor
public class BrandTaskProgressCacheService {
public static final String PHASE_CRAWLING = "crawling";
public static final String PHASE_ASSEMBLING = "assembling";
public static final String PHASE_UPLOADING = "uploading";
public static final String PHASE_FAILED = "failed";
private final StringRedisTemplate stringRedisTemplate;
private final BrandProgressProperties brandProgressProperties;
private final ObjectMapper objectMapper;
public void saveProgressFromResult(Long taskId,
String fileUrl,
int fileIndex,
int fileTotal,
String fileName,
int currentLine,
int totalLines,
int finishedFiles) {
String key = buildKey(taskId);
String now = String.valueOf(Instant.now().toEpochMilli());
Map<String, String> values = new LinkedHashMap<>();
values.put("phase", PHASE_CRAWLING);
values.put("file_index", String.valueOf(Math.max(fileIndex, 0)));
values.put("file_total", String.valueOf(Math.max(fileTotal, 0)));
values.put("file_name", blankToEmpty(fileName));
values.put("file_url", blankToEmpty(fileUrl));
values.put("current_line", String.valueOf(Math.max(Math.min(currentLine, totalLines), 0)));
values.put("total_lines", String.valueOf(Math.max(totalLines, 0)));
values.put("finished_files", String.valueOf(Math.max(finishedFiles, 0)));
values.put("updated_at", now);
values.put("last_heartbeat_at", now);
stringRedisTemplate.opsForHash().putAll(key, values);
stringRedisTemplate.expire(key, Duration.ofHours(brandProgressProperties.getTtlHours()));
}
public void updatePhase(Long taskId, String phase, int finishedFiles, int fileTotal) {
String key = buildKey(taskId);
String now = String.valueOf(Instant.now().toEpochMilli());
Map<String, String> values = new LinkedHashMap<>();
values.put("phase", normalizePhase(phase));
values.put("finished_files", String.valueOf(Math.max(finishedFiles, 0)));
values.put("file_total", String.valueOf(Math.max(fileTotal, 0)));
values.put("updated_at", now);
values.put("last_heartbeat_at", now);
stringRedisTemplate.opsForHash().putAll(key, values);
stringRedisTemplate.expire(key, Duration.ofHours(brandProgressProperties.getTtlHours()));
}
public void markFailed(Long taskId, String message) {
String key = buildKey(taskId);
String now = String.valueOf(Instant.now().toEpochMilli());
Map<String, String> values = new LinkedHashMap<>();
values.put("phase", PHASE_FAILED);
values.put("updated_at", now);
values.put("error_message", blankToEmpty(message));
stringRedisTemplate.opsForHash().putAll(key, values);
stringRedisTemplate.expire(key, Duration.ofHours(brandProgressProperties.getFailedTtlHours()));
}
public Map<Object, Object> getProgress(Long taskId) {
return stringRedisTemplate.opsForHash().entries(buildKey(taskId));
}
public void saveParsedPayload(Long taskId, Object payload) {
try {
stringRedisTemplate.opsForValue().set(buildPayloadKey(taskId), objectMapper.writeValueAsString(payload), Duration.ofHours(brandProgressProperties.getTtlHours()));
} catch (Exception ex) {
throw new BusinessException("暂存品牌原始数据失败");
}
}
public <T> T getParsedPayload(Long taskId, TypeReference<T> typeReference) {
String raw = stringRedisTemplate.opsForValue().get(buildPayloadKey(taskId));
if (raw == null || raw.isBlank()) {
return null;
}
try {
return objectMapper.readValue(raw, typeReference);
} catch (Exception ex) {
throw new BusinessException("读取品牌原始数据失败");
}
}
public void delete(Long taskId) {
stringRedisTemplate.delete(buildKey(taskId));
stringRedisTemplate.delete(buildResultKey(taskId));
stringRedisTemplate.delete(buildPayloadKey(taskId));
}
public void mergeResultChunks(Long taskId, List<BrandCrawlResultFileDto> incomingFiles) {
String resultKey = buildResultKey(taskId);
for (BrandCrawlResultFileDto file : incomingFiles) {
if (file.getChunkIndex() == null || file.getChunkTotal() == null || file.getChunkIndex() <= 0 || file.getChunkTotal() <= 0 || file.getChunkIndex() > file.getChunkTotal()) {
throw new BusinessException("分片参数不合法");
}
String field = buildChunkField(file.getFileUrl(), file.getChunkIndex());
try {
stringRedisTemplate.opsForHash().put(resultKey, field, objectMapper.writeValueAsString(file));
} catch (Exception ex) {
throw new BusinessException("暂存结果分片失败");
}
}
stringRedisTemplate.expire(resultKey, Duration.ofHours(brandProgressProperties.getTtlHours()));
}
public Map<String, List<BrandCrawlResultFileDto>> groupResultChunksByFile(Long taskId) {
Map<Object, Object> stored = stringRedisTemplate.opsForHash().entries(buildResultKey(taskId));
Map<String, List<BrandCrawlResultFileDto>> grouped = new LinkedHashMap<>();
for (Object value : stored.values()) {
if (!(value instanceof String raw) || raw.isBlank()) {
continue;
}
try {
BrandCrawlResultFileDto file = objectMapper.readValue(raw, BrandCrawlResultFileDto.class);
grouped.computeIfAbsent(file.getFileUrl(), ignored -> new ArrayList<>()).add(file);
} catch (Exception ignored) {
}
}
grouped.values().forEach(list -> list.sort(java.util.Comparator.comparing(BrandCrawlResultFileDto::getChunkIndex)));
return grouped;
}
public void clearResultChunks(Long taskId) {
stringRedisTemplate.delete(buildResultKey(taskId));
}
public String buildKey(Long taskId) {
return "brand:task:progress:" + taskId;
}
private String buildResultKey(Long taskId) {
return "brand:task:result-chunks:" + taskId;
}
private String buildChunkField(String fileUrl, Integer chunkIndex) {
return fileUrl + "#" + chunkIndex;
}
private String buildPayloadKey(Long taskId) {
return "brand:task:parsed-payload:" + taskId;
}
public long getHeartbeatTimeoutMinutes() {
return brandProgressProperties.getHeartbeatTimeoutMinutes();
}
private String normalizePhase(String phase) {
if (phase == null || phase.isBlank()) {
return PHASE_CRAWLING;
}
String value = phase.trim().toLowerCase();
return switch (value) {
case PHASE_ASSEMBLING, PHASE_UPLOADING, PHASE_FAILED -> value;
default -> PHASE_CRAWLING;
};
}
private String blankToEmpty(String value) {
return value == null ? "" : value.trim();
}
}

1025
backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java Normal file
View File

File diff suppressed because it is too large Load Diff

2
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeRunController.java
View File

@@ -46,6 +46,8 @@ public class DedupeRunController {
- keepIntegerIds=true 时保留纯数字 ID；
- keepUnderscoreIds=true 时保留类似 1_1 的 ID；
- keepIntegerMainIdsWhenNoSubIds=true 且文件中不存在 1_1 这类子 ID 时，会自动保留纯数字主 ID；
- 整理后的数据会再按 ASIN 与去重总数据表比对，命中的行不会出现在最终输出文件；
- 上传文件内如果存在重复 ASIN，只保留第一次出现的那一行；
- 输出文件命名遵循 原文件名_cleaned.xlsx，若重名自动追加序号。
""")
@ApiResponses({

16
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/mapper/DedupeTotalDataMapper.java
View File

@@ -3,13 +3,29 @@ package com.nanri.aiimage.modules.dedupe.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.dedupe.model.entity.DedupeTotalDataEntity;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import org.apache.ibatis.annotations.SelectProvider;
import java.util.List;
import java.util.StringJoiner;
@Mapper
public interface DedupeTotalDataMapper extends BaseMapper<DedupeTotalDataEntity> {
@Select("SELECT data_value FROM biz_dedupe_total_data")
List<String> selectAllDataValues();
@SelectProvider(type = SqlProvider.class, method = "selectExistingDataValues")
List<String> selectExistingDataValues(@Param("values") List<String> values);
class SqlProvider {
public String selectExistingDataValues(@Param("values") List<String> values) {
StringJoiner placeholders = new StringJoiner(", ");
for (int i = 0; i < values.size(); i++) {
placeholders.add("#{values[" + i + "]}");
}
return "SELECT data_value FROM biz_dedupe_total_data WHERE data_value IN (" + placeholders + ")";
}
}
}

36
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java
View File

@@ -32,8 +32,10 @@ import java.nio.file.Files;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -72,7 +74,6 @@ public class DedupeRunService {
int successCount = 0;
int failedCount = 0;
List<DedupeArchiveEntry> archiveEntries = new ArrayList<>();
java.util.Set<String> totalDataValues = dedupeTotalDataService.listComparableValues();
for (DedupeSourceFileDto sourceFile : request.getFiles()) {
DedupeResultItemVo item = new DedupeResultItemVo();
@@ -94,8 +95,7 @@ public class DedupeRunService {
request.getSelectedColumns(),
request.isKeepIntegerIds(),
request.isKeepUnderscoreIds(),
request.isKeepIntegerMainIdsWhenNoSubIds(),
totalDataValues
request.isKeepIntegerMainIdsWhenNoSubIds()
);
if (folderMode) {
@@ -223,8 +223,7 @@ public class DedupeRunService {
private void cleanExcelByLegacyRules(File inputFile, File outputFile, List<String> selectedColumns,
boolean keepIntegerIds, boolean keepUnderscoreIds,
boolean keepIntegerMainIdsWhenNoSubIds,
java.util.Set<String> totalDataValues) throws Exception {
boolean keepIntegerMainIdsWhenNoSubIds) throws Exception {
DataFormatter formatter = new DataFormatter();
try (FileInputStream fis = new FileInputStream(inputFile);
Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis);
@@ -271,7 +270,8 @@ public class DedupeRunService {
// The preliminary global scan for mainIdHasSubIdMap was removed.
// We now determine subset existence contextually (per group) during the main loop.
}
int outputRowIndex = 1;
List<Row> candidateRows = new ArrayList<>();
Set<String> candidateAsinValues = new HashSet<>();
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
@@ -283,10 +283,28 @@ public class DedupeRunService {
continue;
}
}
if (asinColumnIndex != null && !totalDataValues.isEmpty()) {
candidateRows.add(row);
if (asinColumnIndex != null) {
String asinValue = dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex)));
if (!asinValue.isBlank() && totalDataValues.contains(asinValue)) {
continue;
if (!asinValue.isBlank()) {
candidateAsinValues.add(asinValue);
}
}
}
Set<String> matchedAsinValues = dedupeTotalDataService.findExistingComparableValues(candidateAsinValues);
Set<String> writtenAsinValues = new HashSet<>();
int outputRowIndex = 1;
for (Row row : candidateRows) {
if (asinColumnIndex != null) {
String asinValue = dedupeTotalDataService.normalizeComparableValueOrBlank(formatter.formatCellValue(row.getCell(asinColumnIndex)));
if (!asinValue.isBlank()) {
if (!matchedAsinValues.isEmpty() && matchedAsinValues.contains(asinValue)) {
continue;
}
if (!writtenAsinValues.add(asinValue)) {
continue;
}
}
}

23
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java
View File

@@ -24,6 +24,7 @@ import org.springframework.web.multipart.MultipartFile;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -35,6 +36,8 @@ import java.util.concurrent.ConcurrentHashMap;
@RequiredArgsConstructor
public class DedupeTotalDataService {
private static final int COMPARE_BATCH_SIZE = 1000;
private final DedupeTotalDataMapper dedupeTotalDataMapper;
private final Map<String, DedupeTotalDataImportProgressVo> importProgressMap = new ConcurrentHashMap<>();
private final Map<String, DedupeTotalDataImportProgressVo> deleteImportProgressMap = new ConcurrentHashMap<>();
@@ -76,6 +79,26 @@ public class DedupeTotalDataService {
.collect(java.util.stream.Collectors.toSet());
}
public Set<String> findExistingComparableValues(Collection<String> values) {
if (values == null || values.isEmpty()) {
return Set.of();
}
List<String> normalizedValues = values.stream()
.map(this::normalizeComparableValueOrBlank)
.filter(value -> !value.isEmpty())
.distinct()
.toList();
if (normalizedValues.isEmpty()) {
return Set.of();
}
Set<String> existingValues = new HashSet<>();
for (int start = 0; start < normalizedValues.size(); start += COMPARE_BATCH_SIZE) {
int end = Math.min(start + COMPARE_BATCH_SIZE, normalizedValues.size());
existingValues.addAll(dedupeTotalDataMapper.selectExistingDataValues(normalizedValues.subList(start, end)));
}
return existingValues;
}
public DedupeTotalDataImportStartVo startImport(MultipartFile file) {
if (file == null || file.isEmpty()) {
throw new BusinessException("请上传 xlsx 文件");

2
backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalFileStorageService.java
View File

@@ -96,7 +96,7 @@ public class LocalFileStorageService {
}
}
private File findLocalSourceFile(String fileKey) {
public File findLocalSourceFile(String fileKey) {
File baseDir = FileUtil.file(storageProperties.getLocalTempDir());
if (!baseDir.exists()) {
return null;

116
backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalTempCleanupService.java Normal file
View File

@@ -0,0 +1,116 @@
package com.nanri.aiimage.modules.file.service;
import cn.hutool.core.io.FileUtil;
import com.nanri.aiimage.config.StorageProperties;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.io.File;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.regex.Pattern;
@Slf4j
@Service
@RequiredArgsConstructor
public class LocalTempCleanupService {
private static final Pattern ROOT_TEMP_FILE_PATTERN = Pattern.compile("^[a-fA-F0-9]{32}(\\.[^.]+)?$");
private static final List<String> RESULT_DIR_NAMES = List.of(
"dedupe-result",
"convert-result",
"split-result",
"brand-source-download",
"brand-result"
);
private final StorageProperties storageProperties;
@Scheduled(cron = "${aiimage.storage.cleanup-cron:0 0 */6 * * *}")
public void cleanupLocalTempDir() {
if (!storageProperties.isCleanupEnabled()) {
return;
}
File tempDir = FileUtil.file(storageProperties.getLocalTempDir());
if (!tempDir.exists() || !tempDir.isDirectory()) {
return;
}
Instant sourceExpireBefore = Instant.now().minus(storageProperties.getSourceRetentionHours(), ChronoUnit.HOURS);
Instant resultExpireBefore = Instant.now().minus(storageProperties.getResultRetentionHours(), ChronoUnit.HOURS);
int deletedSourceCount = 0;
int deletedResultCount = 0;
File[] children = tempDir.listFiles();
if (children == null || children.length == 0) {
return;
}
for (File child : children) {
try {
if (child.isFile() && isManagedRootTempFile(child) && isExpired(child, sourceExpireBefore)) {
if (FileUtil.del(child)) {
deletedSourceCount++;
}
continue;
}
if (child.isDirectory() && RESULT_DIR_NAMES.contains(child.getName())) {
deletedResultCount += deleteExpiredChildrenRecursively(child, resultExpireBefore);
deleteEmptyDirectories(child, tempDir);
}
} catch (Exception ex) {
log.warn("清理本地临时文件失败: path={}", child.getAbsolutePath(), ex);
}
}
if (deletedSourceCount > 0 || deletedResultCount > 0) {
log.info("本地临时目录清理完成: sourceDeleted={}, resultDeleted={}, tempDir={}", deletedSourceCount, deletedResultCount, tempDir.getAbsolutePath());
}
}
private int deleteExpiredChildrenRecursively(File file, Instant expireBefore) {
int deletedCount = 0;
if (file.isDirectory()) {
File[] children = file.listFiles();
if (children != null) {
for (File child : children) {
deletedCount += deleteExpiredChildrenRecursively(child, expireBefore);
}
}
if (isExpired(file, expireBefore) && isDirectoryEmpty(file) && FileUtil.del(file)) {
deletedCount++;
}
return deletedCount;
}
if (isExpired(file, expireBefore) && FileUtil.del(file)) {
return 1;
}
return 0;
}
private void deleteEmptyDirectories(File directory, File stopAt) {
File current = directory;
while (current != null && !current.equals(stopAt) && isDirectoryEmpty(current)) {
if (!FileUtil.del(current)) {
return;
}
current = current.getParentFile();
}
}
private boolean isManagedRootTempFile(File file) {
return ROOT_TEMP_FILE_PATTERN.matcher(file.getName()).matches();
}
private boolean isExpired(File file, Instant expireBefore) {
return Instant.ofEpochMilli(file.lastModified()).isBefore(expireBefore);
}
private boolean isDirectoryEmpty(File directory) {
File[] children = directory.listFiles();
return children == null || children.length == 0;
}
}

16
backend-java/src/main/resources/application.yml
View File

@@ -15,6 +15,13 @@ spring:
url: ${AIIMAGE_DB_URL:jdbc:mysql://127.0.0.1:3306/aiimage?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false}
username: ${AIIMAGE_DB_USERNAME:root}
password: ${AIIMAGE_DB_PASSWORD:change-me}
data:
redis:
host: ${AIIMAGE_REDIS_HOST:127.0.0.1}
port: ${AIIMAGE_REDIS_PORT:6379}
password: ${AIIMAGE_REDIS_PASSWORD:}
database: ${AIIMAGE_REDIS_DATABASE:0}
timeout: ${AIIMAGE_REDIS_TIMEOUT:5s}
management:
health:
@@ -56,3 +63,12 @@ aiimage:
access-key-secret: ${AIIMAGE_OSS_ACCESS_KEY_SECRET:change-me}
storage:
local-temp-dir: ${AIIMAGE_STORAGE_LOCAL_TEMP_DIR:./data/tmp}
cleanup-enabled: ${AIIMAGE_STORAGE_CLEANUP_ENABLED:true}
cleanup-cron: ${AIIMAGE_STORAGE_CLEANUP_CRON:0 0 */6 * * *}
source-retention-hours: ${AIIMAGE_STORAGE_SOURCE_RETENTION_HOURS:48}
result-retention-hours: ${AIIMAGE_STORAGE_RESULT_RETENTION_HOURS:24}
brand-progress:
ttl-hours: ${AIIMAGE_BRAND_PROGRESS_TTL_HOURS:24}
failed-ttl-hours: ${AIIMAGE_BRAND_PROGRESS_FAILED_TTL_HOURS:2}
heartbeat-timeout-minutes: ${AIIMAGE_BRAND_PROGRESS_HEARTBEAT_TIMEOUT_MINUTES:15}
stale-check-cron: ${AIIMAGE_BRAND_PROGRESS_STALE_CHECK_CRON:0 */2 * * * *}