增加公共下载进度、增加接收SKU、密钥分别存放
This commit is contained in:
super
+128
@@ -0,0 +1,128 @@
|
||||
package com.nanri.aiimage.modules.collectdata.controller;
|
||||
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataCountryPreferenceSaveRequest;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataParseRequest;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataTaskBatchRequest;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataCountryPreferenceVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataDashboardVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataHistoryVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataItemsPageVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataParseVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskBatchVo;
|
||||
import com.nanri.aiimage.modules.collectdata.service.CollectDataService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.Parameter;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import jakarta.validation.Valid;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.PutMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@RequestMapping("/api/collect-data")
|
||||
@Tag(name = "采集数据", description = "采集数据任务接口。前端上传 Excel 后由 Java 解析并落库；Python 端按页拉取明细数据进行采集。")
|
||||
public class CollectDataController {
|
||||
|
||||
private final CollectDataService service;
|
||||
|
||||
@PostMapping("/parse")
|
||||
@Operation(summary = "解析 Excel 并创建任务", description = "解析上传后的 Excel 文件，按行入库到 biz_collect_data_item，并保存任务筛选条件。任务初始状态为 PENDING。")
|
||||
public ApiResponse<CollectDataParseVo> parse(@Valid @RequestBody CollectDataParseRequest request) {
|
||||
return ApiResponse.success(service.parseAndCreateTask(request));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/{taskId}/activate")
|
||||
@Operation(summary = "激活任务", description = "前端推送 Python 队列前调用，将任务状态从 PENDING 修改为 RUNNING，表示已交给 Python 处理。")
|
||||
public ApiResponse<Void> activate(
|
||||
@Parameter(description = "采集任务 ID", required = true, example = "9001")
|
||||
@PathVariable Long taskId,
|
||||
@Parameter(description = "当前用户 ID", required = true, example = "1")
|
||||
@RequestParam("user_id") Long userId) {
|
||||
service.activateTask(taskId, userId);
|
||||
return ApiResponse.success(null);
|
||||
}
|
||||
|
||||
@GetMapping("/tasks/{taskId}/items")
|
||||
@Operation(summary = "分页获取任务明细数据", description = "供 Python 端拉取，默认每页 50 条；返回任务关联的筛选条件，便于 Python 端按筛选条件采集。")
|
||||
public ApiResponse<CollectDataItemsPageVo> items(
|
||||
@Parameter(description = "采集任务 ID", required = true, example = "9001")
|
||||
@PathVariable Long taskId,
|
||||
@Parameter(description = "用户 ID，可选；传入时会校验任务归属")
|
||||
@RequestParam(value = "user_id", required = false) Long userId,
|
||||
@Parameter(description = "页码，从 1 开始", example = "1")
|
||||
@RequestParam(value = "page", required = false, defaultValue = "1") Integer page,
|
||||
@Parameter(description = "每页条数，默认 50，最大 200", example = "50")
|
||||
@RequestParam(value = "page_size", required = false, defaultValue = "50") Integer pageSize) {
|
||||
return ApiResponse.success(service.getItemsPage(taskId, userId, page, pageSize));
|
||||
}
|
||||
|
||||
@GetMapping("/dashboard")
|
||||
@Operation(summary = "查询采集任务总览")
|
||||
public ApiResponse<CollectDataDashboardVo> dashboard(
|
||||
@Parameter(description = "当前用户 ID", required = true, example = "1")
|
||||
@RequestParam("user_id") Long userId) {
|
||||
return ApiResponse.success(service.dashboard(userId));
|
||||
}
|
||||
|
||||
@GetMapping("/history")
|
||||
@Operation(summary = "查询采集任务历史")
|
||||
public ApiResponse<CollectDataHistoryVo> history(
|
||||
@Parameter(description = "当前用户 ID", required = true, example = "1")
|
||||
@RequestParam("user_id") Long userId,
|
||||
@Parameter(description = "返回数量上限，默认 50，最大 100", example = "50")
|
||||
@RequestParam(value = "limit", required = false, defaultValue = "50") Integer limit) {
|
||||
return ApiResponse.success(service.history(userId, limit));
|
||||
}
|
||||
|
||||
@PostMapping("/tasks/progress/batch")
|
||||
@Operation(summary = "批量查询任务进度")
|
||||
public ApiResponse<CollectDataTaskBatchVo> progressBatch(@Valid @RequestBody CollectDataTaskBatchRequest request) {
|
||||
return ApiResponse.success(service.progressBatch(request.getTaskIds()));
|
||||
}
|
||||
|
||||
@DeleteMapping("/tasks/{taskId}")
|
||||
@Operation(summary = "删除任务", description = "删除任务及其明细行、关联结果记录。")
|
||||
public ApiResponse<Void> deleteTask(
|
||||
@Parameter(description = "采集任务 ID", required = true, example = "9001")
|
||||
@PathVariable Long taskId,
|
||||
@Parameter(description = "当前用户 ID", required = true, example = "1")
|
||||
@RequestParam("user_id") Long userId) {
|
||||
service.deleteTask(taskId, userId);
|
||||
return ApiResponse.success(null);
|
||||
}
|
||||
|
||||
@DeleteMapping("/history/{resultId}")
|
||||
@Operation(summary = "删除历史记录")
|
||||
public ApiResponse<Void> deleteHistory(
|
||||
@Parameter(description = "结果记录 ID", required = true, example = "1001")
|
||||
@PathVariable Long resultId,
|
||||
@Parameter(description = "当前用户 ID", required = true, example = "1")
|
||||
@RequestParam("user_id") Long userId) {
|
||||
service.deleteHistory(resultId, userId);
|
||||
return ApiResponse.success(null);
|
||||
}
|
||||
|
||||
@GetMapping("/country-preference")
|
||||
@Operation(summary = "查询国家处理顺序", description = "返回当前用户保存的国家代码列表，顺序即处理顺序。")
|
||||
public ApiResponse<CollectDataCountryPreferenceVo> getCountryPreference(
|
||||
@Parameter(description = "当前用户 ID", required = true, example = "1")
|
||||
@RequestParam("user_id") Long userId) {
|
||||
return ApiResponse.success(service.getCountryPreference(userId));
|
||||
}
|
||||
|
||||
@PutMapping("/country-preference")
|
||||
@Operation(summary = "保存国家处理顺序", description = "保存当前用户的国家勾选与处理顺序。")
|
||||
public ApiResponse<CollectDataCountryPreferenceVo> saveCountryPreference(
|
||||
@Valid @RequestBody CollectDataCountryPreferenceSaveRequest request) {
|
||||
return ApiResponse.success(service.saveCountryPreference(request));
|
||||
}
|
||||
}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
package com.nanri.aiimage.modules.collectdata.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataCountryPrefEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface CollectDataCountryPrefMapper extends BaseMapper<CollectDataCountryPrefEntity> {
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
package com.nanri.aiimage.modules.collectdata.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity;
|
||||
import org.apache.ibatis.annotations.Insert;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Mapper
|
||||
public interface CollectDataItemMapper extends BaseMapper<CollectDataItemEntity> {
|
||||
|
||||
@Insert("""
|
||||
<script>
|
||||
INSERT INTO biz_collect_data_item
|
||||
(task_id, row_index, source_file_key, source_filename, keyword, status_value, extra_json, created_at)
|
||||
VALUES
|
||||
<foreach collection="rows" item="row" separator=",">
|
||||
(#{row.taskId}, #{row.rowIndex}, #{row.sourceFileKey}, #{row.sourceFilename},
|
||||
#{row.keyword}, #{row.statusValue}, #{row.extraJson}, #{row.createdAt})
|
||||
</foreach>
|
||||
</script>
|
||||
""")
|
||||
int insertBatch(@Param("rows") List<CollectDataItemEntity> rows);
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "保存采集数据五国处理顺序：至少 1 个、最多 5 个，代码须为 DE、UK、FR、IT、ES，且不能重复")
|
||||
public class CollectDataCountryPreferenceSaveRequest {
|
||||
|
||||
@NotNull(message = "user_id 不能为空")
|
||||
@JsonProperty("user_id")
|
||||
@Schema(description = "当前用户 ID", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
|
||||
private Long userId;
|
||||
|
||||
@NotEmpty(message = "country_codes 不能为空")
|
||||
@Size(max = 5, message = "country_codes 最多 5 项")
|
||||
@JsonProperty("country_codes")
|
||||
@Schema(description = "国家代码列表（DE、UK、FR、IT、ES），顺序即处理顺序", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private List<String> countryCodes = new ArrayList<>();
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据筛选条件")
|
||||
public class CollectDataFiltersDto {
|
||||
|
||||
@Schema(description = "金额（手动输入），可为空", example = "59.99")
|
||||
private BigDecimal amount;
|
||||
|
||||
@Schema(description = "排名（手动输入），可为空", example = "100000")
|
||||
private Integer rank;
|
||||
|
||||
@Schema(description = "是否包含 FBA 商品")
|
||||
private Boolean fba;
|
||||
|
||||
@Schema(description = "是否包含 FBM 商品")
|
||||
private Boolean fbm;
|
||||
|
||||
@Schema(description = "国家代码列表，按用户拖拽顺序保存", example = "[\"DE\",\"UK\"]")
|
||||
private List<String> countryCodes = new ArrayList<>();
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据解析请求：提交筛选条件和上传后的 Excel 文件，由后端解析并落库。")
|
||||
public class CollectDataParseRequest {
|
||||
|
||||
@JsonProperty("user_id")
|
||||
@NotNull
|
||||
@Schema(description = "当前用户 ID", example = "1", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private Long userId;
|
||||
|
||||
@NotEmpty
|
||||
@Schema(description = "已上传的 Excel 文件列表", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private List<CollectDataSourceFileDto> files;
|
||||
|
||||
@JsonProperty("task_type")
|
||||
@Schema(description = "任务类型，前端可自定义传入；为空时默认 collect-data", example = "collect-data")
|
||||
private String taskType;
|
||||
|
||||
@Valid
|
||||
@Schema(description = "筛选条件")
|
||||
private CollectDataFiltersDto filters;
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据源文件信息")
|
||||
public class CollectDataSourceFileDto {
|
||||
@Schema(description = "上传接口返回的临时文件 key", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private String fileKey;
|
||||
|
||||
@Schema(description = "原始文件名")
|
||||
private String originalFilename;
|
||||
|
||||
@Schema(description = "相对目录路径")
|
||||
private String relativePath;
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "批量查询采集数据任务进度的请求")
|
||||
public class CollectDataTaskBatchRequest {
|
||||
@NotEmpty
|
||||
@Schema(description = "任务 ID 列表", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private List<Long> taskIds;
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.entity;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableField;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Data
|
||||
@TableName("biz_collect_data_country_pref")
|
||||
public class CollectDataCountryPrefEntity {
|
||||
|
||||
@TableId(type = IdType.INPUT)
|
||||
private Long userId;
|
||||
|
||||
@TableField("country_codes_json")
|
||||
private String countryCodesJson;
|
||||
|
||||
private LocalDateTime updatedAt;
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.entity;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Data
|
||||
@TableName("biz_collect_data_item")
|
||||
public class CollectDataItemEntity {
|
||||
|
||||
@TableId(type = IdType.AUTO)
|
||||
private Long id;
|
||||
private Long taskId;
|
||||
private Integer rowIndex;
|
||||
private String sourceFileKey;
|
||||
private String sourceFilename;
|
||||
private String keyword;
|
||||
private String statusValue;
|
||||
private String extraJson;
|
||||
private LocalDateTime createdAt;
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.vo;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据用户五国处理偏好：按顺序排列的国家代码列表，如 DE、UK")
|
||||
public class CollectDataCountryPreferenceVo {
|
||||
|
||||
@JsonProperty("country_codes")
|
||||
@Schema(description = "已选国家代码，顺序即处理顺序；未持久化时返回默认 DE -> UK -> FR -> IT -> ES")
|
||||
private List<String> countryCodes = new ArrayList<>();
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据总览统计")
|
||||
public class CollectDataDashboardVo {
|
||||
@Schema(description = "运行中任务数量")
|
||||
private Long pendingTaskCount;
|
||||
@Schema(description = "已结束任务数量")
|
||||
private Long processedTaskCount;
|
||||
@Schema(description = "成功任务数量")
|
||||
private Long successTaskCount;
|
||||
@Schema(description = "失败任务数量")
|
||||
private Long failedTaskCount;
|
||||
}
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.vo;
|
||||
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataFiltersDto;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据历史/任务记录")
|
||||
public class CollectDataHistoryItemVo {
|
||||
@Schema(description = "结果记录 ID")
|
||||
private Long resultId;
|
||||
|
||||
@Schema(description = "任务 ID")
|
||||
private Long taskId;
|
||||
|
||||
@Schema(description = "任务编号")
|
||||
private String taskNo;
|
||||
|
||||
@Schema(description = "源文件聚合展示名")
|
||||
private String sourceFilename;
|
||||
|
||||
@Schema(description = "结果文件名")
|
||||
private String resultFilename;
|
||||
|
||||
@Schema(description = "下载地址")
|
||||
private String downloadUrl;
|
||||
|
||||
@Schema(description = "任务状态：PENDING / RUNNING / SUCCESS / FAILED")
|
||||
private String taskStatus;
|
||||
|
||||
@Schema(description = "结果是否成功")
|
||||
private Boolean success;
|
||||
|
||||
@Schema(description = "错误信息")
|
||||
private String error;
|
||||
|
||||
@Schema(description = "落库行数")
|
||||
private Integer rowCount;
|
||||
|
||||
@Schema(description = "记录创建时间")
|
||||
private String createdAt;
|
||||
|
||||
@Schema(description = "任务开始时间")
|
||||
private String startedAt;
|
||||
|
||||
@Schema(description = "任务结束时间")
|
||||
private String finishedAt;
|
||||
|
||||
@Schema(description = "任务类型")
|
||||
private String taskType;
|
||||
|
||||
@Schema(description = "解析时保存的筛选条件")
|
||||
private CollectDataFiltersDto filters;
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据历史列表")
|
||||
public class CollectDataHistoryVo {
|
||||
@Schema(description = "历史记录列表")
|
||||
private List<CollectDataHistoryItemVo> items = new ArrayList<>();
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据明细行")
|
||||
public class CollectDataItemVo {
|
||||
@Schema(description = "数据库主键")
|
||||
private Long id;
|
||||
|
||||
@Schema(description = "行号，从 1 开始")
|
||||
private Integer rowIndex;
|
||||
|
||||
@Schema(description = "来源文件 key")
|
||||
private String sourceFileKey;
|
||||
|
||||
@Schema(description = "来源文件名")
|
||||
private String sourceFilename;
|
||||
|
||||
@Schema(description = "关键词列内容")
|
||||
private String keyword;
|
||||
|
||||
@Schema(description = "状态列内容")
|
||||
private String statusValue;
|
||||
|
||||
@Schema(description = "其它列内容（列名 -> 字符串值）")
|
||||
private Map<String, String> extra = new LinkedHashMap<>();
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.vo;
|
||||
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataFiltersDto;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据明细分页结果，默认每页 50 条，供 Python 拉取")
|
||||
public class CollectDataItemsPageVo {
|
||||
@Schema(description = "任务 ID")
|
||||
private Long taskId;
|
||||
|
||||
@Schema(description = "任务编号")
|
||||
private String taskNo;
|
||||
|
||||
@Schema(description = "任务类型")
|
||||
private String taskType;
|
||||
|
||||
@Schema(description = "任务状态")
|
||||
private String taskStatus;
|
||||
|
||||
@Schema(description = "页码，从 1 开始")
|
||||
private Integer page;
|
||||
|
||||
@Schema(description = "每页条数")
|
||||
private Integer pageSize;
|
||||
|
||||
@Schema(description = "本页条数")
|
||||
private Integer count;
|
||||
|
||||
@Schema(description = "总条数")
|
||||
private Long total;
|
||||
|
||||
@Schema(description = "总页数")
|
||||
private Integer totalPages;
|
||||
|
||||
@Schema(description = "本任务关联的筛选条件")
|
||||
private CollectDataFiltersDto filters;
|
||||
|
||||
@Schema(description = "明细列表")
|
||||
private List<CollectDataItemVo> items = new ArrayList<>();
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据解析结果")
|
||||
public class CollectDataParseVo {
|
||||
|
||||
@Schema(description = "新创建的任务 ID", example = "9001")
|
||||
private Long taskId;
|
||||
|
||||
@Schema(description = "任务编号")
|
||||
private String taskNo;
|
||||
|
||||
@Schema(description = "聚合源文件展示名", example = "采集前数据.xlsx 等 1 个文件")
|
||||
private String sourceFilename;
|
||||
|
||||
@Schema(description = "源文件数量")
|
||||
private Integer sourceFileCount;
|
||||
|
||||
@Schema(description = "Excel 中检测到的非空数据总行数")
|
||||
private Integer totalRows;
|
||||
|
||||
@Schema(description = "落库的有效行数")
|
||||
private Integer acceptedRows;
|
||||
|
||||
@Schema(description = "因缺少必要字段被丢弃的行数")
|
||||
private Integer droppedRows;
|
||||
|
||||
@Schema(description = "推荐分页大小，便于 Python 拉取分页数据", example = "50")
|
||||
private Integer pageSize;
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "批量任务进度返回")
|
||||
public class CollectDataTaskBatchVo {
|
||||
@Schema(description = "任务详情列表")
|
||||
private List<CollectDataTaskDetailVo> items = new ArrayList<>();
|
||||
|
||||
@Schema(description = "未找到的任务 ID")
|
||||
private List<Long> missingTaskIds = new ArrayList<>();
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据任务进度详情")
|
||||
public class CollectDataTaskDetailVo {
|
||||
@Schema(description = "任务摘要")
|
||||
private CollectDataTaskSummaryVo task;
|
||||
|
||||
@Schema(description = "结果/历史记录")
|
||||
private List<CollectDataHistoryItemVo> items = new ArrayList<>();
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package com.nanri.aiimage.modules.collectdata.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "采集数据任务摘要")
|
||||
public class CollectDataTaskSummaryVo {
|
||||
@Schema(description = "任务 ID")
|
||||
private Long id;
|
||||
|
||||
@Schema(description = "任务编号")
|
||||
private String taskNo;
|
||||
|
||||
@Schema(description = "任务状态")
|
||||
private String status;
|
||||
|
||||
@Schema(description = "错误信息")
|
||||
private String errorMessage;
|
||||
|
||||
@Schema(description = "创建时间")
|
||||
private String createdAt;
|
||||
|
||||
@Schema(description = "更新时间")
|
||||
private String updatedAt;
|
||||
|
||||
@Schema(description = "结束时间")
|
||||
private String finishedAt;
|
||||
}
|
||||
+732
@@ -0,0 +1,732 @@
|
||||
package com.nanri.aiimage.modules.collectdata.service;
|
||||
|
||||
import cn.hutool.core.util.IdUtil;
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataCountryPreferenceSaveRequest;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataFiltersDto;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataParseRequest;
|
||||
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSourceFileDto;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataCountryPrefEntity;
|
||||
import com.nanri.aiimage.modules.collectdata.model.entity.CollectDataItemEntity;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataCountryPreferenceVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataDashboardVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataHistoryItemVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataHistoryVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataItemVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataItemsPageVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataParseVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskBatchVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskDetailVo;
|
||||
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataTaskSummaryVo;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.ss.usermodel.WorkbookFactory;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class CollectDataService {
|
||||
|
||||
public static final String MODULE_TYPE = "COLLECT_DATA";
|
||||
public static final int DEFAULT_PAGE_SIZE = 50;
|
||||
|
||||
/**
|
||||
* 默认国家偏好顺序：DE -> UK -> FR -> IT -> ES，与前端保持一致。
|
||||
*/
|
||||
public static final List<String> DEFAULT_COUNTRY_PREFERENCE_ORDER = List.of("DE", "UK", "FR", "IT", "ES");
|
||||
|
||||
private static final String STATUS_PENDING = "PENDING";
|
||||
private static final String STATUS_RUNNING = "RUNNING";
|
||||
private static final String STATUS_SUCCESS = "SUCCESS";
|
||||
private static final String STATUS_FAILED = "FAILED";
|
||||
|
||||
private static final String DEFAULT_TASK_TYPE = "collect-data";
|
||||
private static final int ITEM_INSERT_BATCH_SIZE = 500;
|
||||
|
||||
private static final List<String> KEYWORD_HEADER_ALIASES = List.of("关键词", "keyword", "key word");
|
||||
private static final List<String> STATUS_HEADER_ALIASES = List.of("状态", "status");
|
||||
|
||||
private static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
|
||||
private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+");
|
||||
|
||||
private final LocalFileStorageService localFileStorageService;
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final FileResultMapper fileResultMapper;
|
||||
private final CollectDataItemMapper collectDataItemMapper;
|
||||
private final CollectDataCountryPrefMapper collectDataCountryPrefMapper;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final TransactionTemplate transactionTemplate;
|
||||
|
||||
public CollectDataParseVo parseAndCreateTask(CollectDataParseRequest request) {
|
||||
long startedAt = System.currentTimeMillis();
|
||||
if (request == null || request.getUserId() == null || request.getUserId() <= 0) {
|
||||
throw new BusinessException("user_id 不合法");
|
||||
}
|
||||
if (request.getFiles() == null || request.getFiles().isEmpty()) {
|
||||
throw new BusinessException("请先上传 Excel 文件");
|
||||
}
|
||||
List<CollectDataSourceFileDto> sources = request.getFiles().stream()
|
||||
.filter(Objects::nonNull)
|
||||
.filter(f -> f.getFileKey() != null && !f.getFileKey().isBlank())
|
||||
.toList();
|
||||
if (sources.isEmpty()) {
|
||||
throw new BusinessException("请先上传 Excel 文件");
|
||||
}
|
||||
String requestedTaskType = normalize(request.getTaskType());
|
||||
final String taskType = requestedTaskType.isBlank() ? DEFAULT_TASK_TYPE : requestedTaskType;
|
||||
CollectDataFiltersDto filters = request.getFilters() == null ? new CollectDataFiltersDto() : request.getFilters();
|
||||
|
||||
List<ParsedRow> parsedRows = new ArrayList<>();
|
||||
int totalRows = 0;
|
||||
int droppedRows = 0;
|
||||
for (CollectDataSourceFileDto source : sources) {
|
||||
File input = localFileStorageService.findLocalSourceFile(source.getFileKey());
|
||||
if (input == null || !input.exists()) {
|
||||
throw new BusinessException("源文件不存在");
|
||||
}
|
||||
ParsedWorkbook parsed = parseWorkbook(input, source);
|
||||
totalRows += parsed.totalRows();
|
||||
droppedRows += parsed.droppedRows();
|
||||
parsedRows.addAll(parsed.rows());
|
||||
}
|
||||
if (parsedRows.isEmpty()) {
|
||||
throw new BusinessException("未解析到有效数据行");
|
||||
}
|
||||
|
||||
long parsedAt = System.currentTimeMillis();
|
||||
String aggregateFilename = buildAggregateSourceFilenameLabel(sources);
|
||||
PersistedParse persisted = transactionTemplate.execute(status ->
|
||||
persistParsedTask(request, sources, taskType, filters, parsedRows, aggregateFilename));
|
||||
if (persisted == null) {
|
||||
throw new BusinessException("创建采集任务失败");
|
||||
}
|
||||
long persistedAt = System.currentTimeMillis();
|
||||
log.info("[collect-data] parseAndCreateTask userId={} files={} totalRows={} acceptedRows={} droppedRows={} parseMs={} persistMs={} totalMs={}",
|
||||
request.getUserId(), sources.size(), totalRows, parsedRows.size(), droppedRows,
|
||||
parsedAt - startedAt, persistedAt - parsedAt, persistedAt - startedAt);
|
||||
|
||||
CollectDataParseVo vo = new CollectDataParseVo();
|
||||
vo.setTaskId(persisted.task().getId());
|
||||
vo.setTaskNo(persisted.task().getTaskNo());
|
||||
vo.setSourceFilename(aggregateFilename);
|
||||
vo.setSourceFileCount(sources.size());
|
||||
vo.setTotalRows(totalRows);
|
||||
vo.setAcceptedRows(parsedRows.size());
|
||||
vo.setDroppedRows(droppedRows);
|
||||
vo.setPageSize(DEFAULT_PAGE_SIZE);
|
||||
return vo;
|
||||
}
|
||||
|
||||
private PersistedParse persistParsedTask(CollectDataParseRequest request,
|
||||
List<CollectDataSourceFileDto> sources,
|
||||
String taskType,
|
||||
CollectDataFiltersDto filters,
|
||||
List<ParsedRow> parsedRows,
|
||||
String aggregateFilename) {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setTaskNo(MODULE_TYPE + "-" + IdUtil.getSnowflakeNextIdStr());
|
||||
task.setModuleType(MODULE_TYPE);
|
||||
task.setTaskMode("PYTHON_QUEUE");
|
||||
task.setStatus(STATUS_PENDING);
|
||||
task.setSourceFileCount(sources.size());
|
||||
task.setSuccessFileCount(0);
|
||||
task.setFailedFileCount(0);
|
||||
task.setCreatedBy("user:" + request.getUserId());
|
||||
task.setUserId(request.getUserId());
|
||||
task.setCreatedAt(LocalDateTime.now());
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
try {
|
||||
Map<String, Object> requestPayload = new LinkedHashMap<>();
|
||||
requestPayload.put("userId", request.getUserId());
|
||||
requestPayload.put("taskType", taskType);
|
||||
requestPayload.put("filters", filters);
|
||||
requestPayload.put("files", sources);
|
||||
task.setRequestJson(objectMapper.writeValueAsString(requestPayload));
|
||||
task.setResultJson("{}");
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("序列化任务信息失败");
|
||||
}
|
||||
fileTaskMapper.insert(task);
|
||||
|
||||
int rowIndex = 0;
|
||||
LocalDateTime itemCreatedAt = LocalDateTime.now();
|
||||
List<CollectDataItemEntity> itemBatch = new ArrayList<>(Math.min(parsedRows.size(), ITEM_INSERT_BATCH_SIZE));
|
||||
for (ParsedRow parsedRow : parsedRows) {
|
||||
rowIndex++;
|
||||
CollectDataItemEntity entity = new CollectDataItemEntity();
|
||||
entity.setTaskId(task.getId());
|
||||
entity.setRowIndex(rowIndex);
|
||||
entity.setSourceFileKey(parsedRow.sourceFileKey());
|
||||
entity.setSourceFilename(parsedRow.sourceFilename());
|
||||
entity.setKeyword(parsedRow.keyword());
|
||||
entity.setStatusValue(parsedRow.statusValue());
|
||||
try {
|
||||
entity.setExtraJson(objectMapper.writeValueAsString(parsedRow.extra()));
|
||||
} catch (Exception ex) {
|
||||
entity.setExtraJson("{}");
|
||||
}
|
||||
entity.setCreatedAt(itemCreatedAt);
|
||||
itemBatch.add(entity);
|
||||
if (itemBatch.size() >= ITEM_INSERT_BATCH_SIZE) {
|
||||
collectDataItemMapper.insertBatch(itemBatch);
|
||||
itemBatch.clear();
|
||||
}
|
||||
}
|
||||
if (!itemBatch.isEmpty()) {
|
||||
collectDataItemMapper.insertBatch(itemBatch);
|
||||
}
|
||||
|
||||
FileResultEntity result = new FileResultEntity();
|
||||
result.setTaskId(task.getId());
|
||||
result.setModuleType(MODULE_TYPE);
|
||||
result.setSourceFilename(aggregateFilename);
|
||||
result.setSourceFileUrl(buildAggregateSourceFileUrl(sources));
|
||||
result.setRowCount(parsedRows.size());
|
||||
result.setUserId(request.getUserId());
|
||||
result.setCreatedAt(LocalDateTime.now());
|
||||
fileResultMapper.insert(result);
|
||||
|
||||
return new PersistedParse(task, result);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void activateTask(Long taskId, Long userId) {
|
||||
FileTaskEntity task = requireTask(taskId, userId);
|
||||
if (STATUS_SUCCESS.equals(task.getStatus()) || STATUS_FAILED.equals(task.getStatus())) {
|
||||
throw new BusinessException("任务已结束");
|
||||
}
|
||||
task.setStatus(STATUS_RUNNING);
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
fileTaskMapper.updateById(task);
|
||||
}
|
||||
|
||||
public CollectDataDashboardVo dashboard(Long userId) {
|
||||
CollectDataDashboardVo vo = new CollectDataDashboardVo();
|
||||
vo.setPendingTaskCount(countTask(userId, STATUS_RUNNING));
|
||||
vo.setSuccessTaskCount(countTask(userId, STATUS_SUCCESS));
|
||||
vo.setFailedTaskCount(countTask(userId, STATUS_FAILED));
|
||||
long processed = (vo.getSuccessTaskCount() == null ? 0 : vo.getSuccessTaskCount())
|
||||
+ (vo.getFailedTaskCount() == null ? 0 : vo.getFailedTaskCount());
|
||||
vo.setProcessedTaskCount(processed);
|
||||
return vo;
|
||||
}
|
||||
|
||||
public CollectDataHistoryVo history(Long userId, Integer limit) {
|
||||
CollectDataHistoryVo vo = new CollectDataHistoryVo();
|
||||
if (userId == null || userId <= 0) {
|
||||
return vo;
|
||||
}
|
||||
int safeLimit = Math.max(1, Math.min(limit == null ? 50 : limit, 100));
|
||||
List<FileResultEntity> rows = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileResultEntity::getUserId, userId)
|
||||
.orderByDesc(FileResultEntity::getCreatedAt)
|
||||
.last("limit " + safeLimit));
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return vo;
|
||||
}
|
||||
List<Long> taskIds = rows.stream().map(FileResultEntity::getTaskId).filter(Objects::nonNull).distinct().toList();
|
||||
Map<Long, FileTaskEntity> taskMap = loadTaskMap(taskIds);
|
||||
for (FileResultEntity row : rows) {
|
||||
FileTaskEntity task = row.getTaskId() == null ? null : taskMap.get(row.getTaskId());
|
||||
vo.getItems().add(toHistoryItem(row, task));
|
||||
}
|
||||
return vo;
|
||||
}
|
||||
|
||||
public CollectDataTaskBatchVo progressBatch(List<Long> taskIds) {
|
||||
CollectDataTaskBatchVo vo = new CollectDataTaskBatchVo();
|
||||
List<Long> normalizedIds = taskIds == null ? List.of() : taskIds.stream()
|
||||
.filter(id -> id != null && id > 0)
|
||||
.distinct()
|
||||
.toList();
|
||||
if (normalizedIds.isEmpty()) {
|
||||
return vo;
|
||||
}
|
||||
Map<Long, FileTaskEntity> taskMap = loadTaskMap(normalizedIds);
|
||||
List<FileResultEntity> resultRows = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
|
||||
.in(FileResultEntity::getTaskId, normalizedIds)
|
||||
.orderByDesc(FileResultEntity::getCreatedAt));
|
||||
Map<Long, FileResultEntity> resultByTaskId = new LinkedHashMap<>();
|
||||
if (resultRows != null) {
|
||||
for (FileResultEntity row : resultRows) {
|
||||
if (row.getTaskId() != null) {
|
||||
resultByTaskId.putIfAbsent(row.getTaskId(), row);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (Long taskId : normalizedIds) {
|
||||
FileTaskEntity task = taskMap.get(taskId);
|
||||
if (task == null) {
|
||||
vo.getMissingTaskIds().add(taskId);
|
||||
continue;
|
||||
}
|
||||
CollectDataTaskDetailVo detail = new CollectDataTaskDetailVo();
|
||||
detail.setTask(toTaskSummary(task));
|
||||
FileResultEntity row = resultByTaskId.get(taskId);
|
||||
if (row != null) {
|
||||
detail.getItems().add(toHistoryItem(row, task));
|
||||
}
|
||||
vo.getItems().add(detail);
|
||||
}
|
||||
return vo;
|
||||
}
|
||||
|
||||
public CollectDataItemsPageVo getItemsPage(Long taskId, Long userId, Integer page, Integer pageSize) {
|
||||
FileTaskEntity task = requireTask(taskId, userId);
|
||||
int safePage = page == null || page <= 0 ? 1 : page;
|
||||
int safePageSize = pageSize == null || pageSize <= 0 ? DEFAULT_PAGE_SIZE : Math.min(pageSize, 200);
|
||||
|
||||
Long total = collectDataItemMapper.selectCount(new LambdaQueryWrapper<CollectDataItemEntity>()
|
||||
.eq(CollectDataItemEntity::getTaskId, taskId));
|
||||
long totalRows = total == null ? 0L : total;
|
||||
int totalPages = totalRows == 0 ? 0 : (int) ((totalRows + safePageSize - 1) / safePageSize);
|
||||
|
||||
int offset = (safePage - 1) * safePageSize;
|
||||
List<CollectDataItemEntity> rows = collectDataItemMapper.selectList(new LambdaQueryWrapper<CollectDataItemEntity>()
|
||||
.eq(CollectDataItemEntity::getTaskId, taskId)
|
||||
.orderByAsc(CollectDataItemEntity::getRowIndex)
|
||||
.last("limit " + safePageSize + " offset " + Math.max(offset, 0)));
|
||||
|
||||
CollectDataItemsPageVo vo = new CollectDataItemsPageVo();
|
||||
vo.setTaskId(task.getId());
|
||||
vo.setTaskNo(task.getTaskNo());
|
||||
vo.setTaskStatus(task.getStatus());
|
||||
vo.setPage(safePage);
|
||||
vo.setPageSize(safePageSize);
|
||||
vo.setTotal(totalRows);
|
||||
vo.setTotalPages(totalPages);
|
||||
vo.setFilters(extractFiltersFromRequest(task));
|
||||
vo.setTaskType(extractTaskTypeFromRequest(task));
|
||||
|
||||
List<CollectDataItemVo> items = new ArrayList<>();
|
||||
if (rows != null) {
|
||||
for (CollectDataItemEntity row : rows) {
|
||||
items.add(toItemVo(row));
|
||||
}
|
||||
}
|
||||
vo.setItems(items);
|
||||
vo.setCount(items.size());
|
||||
return vo;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void deleteTask(Long taskId, Long userId) {
|
||||
FileTaskEntity task = requireTask(taskId, userId);
|
||||
collectDataItemMapper.delete(new LambdaQueryWrapper<CollectDataItemEntity>()
|
||||
.eq(CollectDataItemEntity::getTaskId, task.getId()));
|
||||
fileResultMapper.delete(new LambdaQueryWrapper<FileResultEntity>()
|
||||
.eq(FileResultEntity::getTaskId, task.getId())
|
||||
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
|
||||
fileTaskMapper.deleteById(task.getId());
|
||||
}
|
||||
|
||||
public void deleteHistory(Long resultId, Long userId) {
|
||||
FileResultEntity row = fileResultMapper.selectById(resultId);
|
||||
if (row == null || !MODULE_TYPE.equals(row.getModuleType()) || !Objects.equals(userId, row.getUserId())) {
|
||||
throw new BusinessException("记录不存在");
|
||||
}
|
||||
fileResultMapper.deleteById(resultId);
|
||||
}
|
||||
|
||||
public CollectDataCountryPreferenceVo getCountryPreference(Long userId) {
|
||||
if (userId == null || userId <= 0) {
|
||||
throw new BusinessException("user_id 不合法");
|
||||
}
|
||||
CollectDataCountryPrefEntity row = collectDataCountryPrefMapper.selectById(userId);
|
||||
CollectDataCountryPreferenceVo vo = new CollectDataCountryPreferenceVo();
|
||||
if (row == null || row.getCountryCodesJson() == null || row.getCountryCodesJson().isBlank()) {
|
||||
vo.getCountryCodes().addAll(DEFAULT_COUNTRY_PREFERENCE_ORDER);
|
||||
return vo;
|
||||
}
|
||||
try {
|
||||
List<String> parsed = objectMapper.readValue(row.getCountryCodesJson(), new TypeReference<List<String>>() {
|
||||
});
|
||||
vo.getCountryCodes().addAll(sanitizeStoredCountryCodes(parsed));
|
||||
} catch (Exception ex) {
|
||||
vo.getCountryCodes().addAll(DEFAULT_COUNTRY_PREFERENCE_ORDER);
|
||||
}
|
||||
return vo;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public CollectDataCountryPreferenceVo saveCountryPreference(CollectDataCountryPreferenceSaveRequest request) {
|
||||
if (request == null || request.getUserId() == null || request.getUserId() <= 0) {
|
||||
throw new BusinessException("user_id 不合法");
|
||||
}
|
||||
List<String> normalized = validateCountryCodesForSave(request.getCountryCodes());
|
||||
String json;
|
||||
try {
|
||||
json = objectMapper.writeValueAsString(normalized);
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("保存国家偏好失败");
|
||||
}
|
||||
CollectDataCountryPrefEntity row = collectDataCountryPrefMapper.selectById(request.getUserId());
|
||||
if (row == null) {
|
||||
row = new CollectDataCountryPrefEntity();
|
||||
row.setUserId(request.getUserId());
|
||||
row.setCountryCodesJson(json);
|
||||
collectDataCountryPrefMapper.insert(row);
|
||||
} else {
|
||||
row.setCountryCodesJson(json);
|
||||
collectDataCountryPrefMapper.updateById(row);
|
||||
}
|
||||
CollectDataCountryPreferenceVo vo = new CollectDataCountryPreferenceVo();
|
||||
vo.getCountryCodes().addAll(normalized);
|
||||
return vo;
|
||||
}
|
||||
|
||||
private static List<String> sanitizeStoredCountryCodes(List<String> raw) {
|
||||
if (raw == null || raw.isEmpty()) {
|
||||
return new ArrayList<>(DEFAULT_COUNTRY_PREFERENCE_ORDER);
|
||||
}
|
||||
LinkedHashSet<String> seen = new LinkedHashSet<>();
|
||||
List<String> out = new ArrayList<>();
|
||||
for (String code : raw) {
|
||||
if (code == null || code.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
String upper = code.trim().toUpperCase(Locale.ROOT);
|
||||
if (!DEFAULT_COUNTRY_PREFERENCE_ORDER.contains(upper)) {
|
||||
continue;
|
||||
}
|
||||
if (seen.add(upper)) {
|
||||
out.add(upper);
|
||||
}
|
||||
}
|
||||
return out.isEmpty() ? new ArrayList<>(DEFAULT_COUNTRY_PREFERENCE_ORDER) : out;
|
||||
}
|
||||
|
||||
private static List<String> validateCountryCodesForSave(List<String> raw) {
|
||||
if (raw == null || raw.isEmpty()) {
|
||||
throw new BusinessException("country_codes 至少选择 1 个国家");
|
||||
}
|
||||
LinkedHashSet<String> seen = new LinkedHashSet<>();
|
||||
List<String> out = new ArrayList<>();
|
||||
for (String code : raw) {
|
||||
if (code == null || code.isBlank()) {
|
||||
throw new BusinessException("country_codes 含空项");
|
||||
}
|
||||
String upper = code.trim().toUpperCase(Locale.ROOT);
|
||||
if (!DEFAULT_COUNTRY_PREFERENCE_ORDER.contains(upper)) {
|
||||
throw new BusinessException("非法国家代码: " + code);
|
||||
}
|
||||
if (!seen.add(upper)) {
|
||||
throw new BusinessException("country_codes 存在重复: " + upper);
|
||||
}
|
||||
out.add(upper);
|
||||
}
|
||||
if (out.size() > DEFAULT_COUNTRY_PREFERENCE_ORDER.size()) {
|
||||
throw new BusinessException("country_codes 最多 5 项");
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private FileTaskEntity requireTask(Long taskId, Long userId) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
throw new BusinessException("任务不存在");
|
||||
}
|
||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
||||
throw new BusinessException("任务不存在");
|
||||
}
|
||||
if (userId != null && userId > 0 && !Objects.equals(userId, task.getUserId())) {
|
||||
throw new BusinessException("任务不存在");
|
||||
}
|
||||
return task;
|
||||
}
|
||||
|
||||
private long countTask(Long userId, String status) {
|
||||
if (userId == null || userId <= 0) {
|
||||
return 0L;
|
||||
}
|
||||
Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||
.eq(FileTaskEntity::getUserId, userId)
|
||||
.eq(FileTaskEntity::getStatus, status));
|
||||
return count == null ? 0L : count;
|
||||
}
|
||||
|
||||
private Map<Long, FileTaskEntity> loadTaskMap(List<Long> taskIds) {
|
||||
Map<Long, FileTaskEntity> taskMap = new LinkedHashMap<>();
|
||||
if (taskIds == null || taskIds.isEmpty()) {
|
||||
return taskMap;
|
||||
}
|
||||
List<FileTaskEntity> tasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.in(FileTaskEntity::getId, taskIds));
|
||||
if (tasks != null) {
|
||||
for (FileTaskEntity task : tasks) {
|
||||
if (task != null && MODULE_TYPE.equals(task.getModuleType())) {
|
||||
taskMap.put(task.getId(), task);
|
||||
}
|
||||
}
|
||||
}
|
||||
return taskMap;
|
||||
}
|
||||
|
||||
private CollectDataHistoryItemVo toHistoryItem(FileResultEntity row, FileTaskEntity task) {
|
||||
CollectDataHistoryItemVo vo = new CollectDataHistoryItemVo();
|
||||
vo.setResultId(row.getId());
|
||||
vo.setTaskId(row.getTaskId());
|
||||
vo.setSourceFilename(row.getSourceFilename());
|
||||
vo.setResultFilename(row.getResultFilename());
|
||||
vo.setRowCount(row.getRowCount());
|
||||
vo.setSuccess(row.getSuccess() != null && row.getSuccess() == 1);
|
||||
vo.setError(row.getErrorMessage());
|
||||
vo.setCreatedAt(formatTime(row.getCreatedAt()));
|
||||
if (task != null) {
|
||||
vo.setTaskNo(task.getTaskNo());
|
||||
vo.setTaskStatus(task.getStatus());
|
||||
vo.setStartedAt(formatTime(task.getCreatedAt()));
|
||||
vo.setFinishedAt(formatTime(task.getFinishedAt()));
|
||||
vo.setTaskType(extractTaskTypeFromRequest(task));
|
||||
vo.setFilters(extractFiltersFromRequest(task));
|
||||
}
|
||||
return vo;
|
||||
}
|
||||
|
||||
private CollectDataTaskSummaryVo toTaskSummary(FileTaskEntity task) {
|
||||
CollectDataTaskSummaryVo vo = new CollectDataTaskSummaryVo();
|
||||
vo.setId(task.getId());
|
||||
vo.setTaskNo(task.getTaskNo());
|
||||
vo.setStatus(task.getStatus());
|
||||
vo.setErrorMessage(task.getErrorMessage());
|
||||
vo.setCreatedAt(formatTime(task.getCreatedAt()));
|
||||
vo.setUpdatedAt(formatTime(task.getUpdatedAt()));
|
||||
vo.setFinishedAt(formatTime(task.getFinishedAt()));
|
||||
return vo;
|
||||
}
|
||||
|
||||
private CollectDataItemVo toItemVo(CollectDataItemEntity row) {
|
||||
CollectDataItemVo vo = new CollectDataItemVo();
|
||||
vo.setId(row.getId());
|
||||
vo.setRowIndex(row.getRowIndex());
|
||||
vo.setSourceFileKey(row.getSourceFileKey());
|
||||
vo.setSourceFilename(row.getSourceFilename());
|
||||
vo.setKeyword(row.getKeyword());
|
||||
vo.setStatusValue(row.getStatusValue());
|
||||
if (row.getExtraJson() != null && !row.getExtraJson().isBlank()) {
|
||||
try {
|
||||
Map<String, String> extra = objectMapper.readValue(row.getExtraJson(), new TypeReference<>() {
|
||||
});
|
||||
if (extra != null) {
|
||||
vo.getExtra().putAll(extra);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.warn("[collect-data] parse extra json failed rowId={} err={}", row.getId(), ex.getMessage());
|
||||
}
|
||||
}
|
||||
return vo;
|
||||
}
|
||||
|
||||
private CollectDataFiltersDto extractFiltersFromRequest(FileTaskEntity task) {
|
||||
if (task == null || task.getRequestJson() == null || task.getRequestJson().isBlank()) {
|
||||
return new CollectDataFiltersDto();
|
||||
}
|
||||
try {
|
||||
JsonNode root = objectMapper.readTree(task.getRequestJson());
|
||||
JsonNode filtersNode = root.get("filters");
|
||||
if (filtersNode == null || filtersNode.isNull()) {
|
||||
return new CollectDataFiltersDto();
|
||||
}
|
||||
return objectMapper.treeToValue(filtersNode, CollectDataFiltersDto.class);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[collect-data] parse request filters failed taskId={} err={}", task.getId(), ex.getMessage());
|
||||
return new CollectDataFiltersDto();
|
||||
}
|
||||
}
|
||||
|
||||
private String extractTaskTypeFromRequest(FileTaskEntity task) {
|
||||
if (task == null || task.getRequestJson() == null || task.getRequestJson().isBlank()) {
|
||||
return DEFAULT_TASK_TYPE;
|
||||
}
|
||||
try {
|
||||
JsonNode root = objectMapper.readTree(task.getRequestJson());
|
||||
JsonNode taskTypeNode = root.get("taskType");
|
||||
String value = taskTypeNode == null || taskTypeNode.isNull() ? null : taskTypeNode.asText(null);
|
||||
return value == null || value.isBlank() ? DEFAULT_TASK_TYPE : value;
|
||||
} catch (Exception ex) {
|
||||
return DEFAULT_TASK_TYPE;
|
||||
}
|
||||
}
|
||||
|
||||
private String buildAggregateSourceFilenameLabel(List<CollectDataSourceFileDto> sources) {
|
||||
if (sources == null || sources.isEmpty()) {
|
||||
return "";
|
||||
}
|
||||
String first = firstNonBlank(sources.get(0).getOriginalFilename(), sources.get(0).getFileKey());
|
||||
if (sources.size() == 1) {
|
||||
return first;
|
||||
}
|
||||
return first + " 等 " + sources.size() + " 个文件";
|
||||
}
|
||||
|
||||
private String buildAggregateSourceFileUrl(List<CollectDataSourceFileDto> sources) {
|
||||
if (sources == null || sources.isEmpty()) {
|
||||
return "";
|
||||
}
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (int i = 0; i < sources.size(); i++) {
|
||||
if (i > 0) {
|
||||
sb.append('|');
|
||||
}
|
||||
sb.append(sources.get(i).getFileKey());
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private ParsedWorkbook parseWorkbook(File input, CollectDataSourceFileDto source) {
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
try (FileInputStream fis = new FileInputStream(input); Workbook workbook = WorkbookFactory.create(fis)) {
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
Row header = sheet.getRow(0);
|
||||
if (header == null) {
|
||||
throw new BusinessException("Excel 表头为空");
|
||||
}
|
||||
|
||||
List<String> headers = new ArrayList<>();
|
||||
for (int i = 0; i < header.getLastCellNum(); i++) {
|
||||
String value = normalize(formatter.formatCellValue(header.getCell(i)));
|
||||
headers.add(value.isBlank() ? "列" + (i + 1) : value);
|
||||
}
|
||||
|
||||
int keywordCol = findHeaderIndex(headers, KEYWORD_HEADER_ALIASES);
|
||||
int statusCol = findHeaderIndex(headers, STATUS_HEADER_ALIASES);
|
||||
|
||||
List<ParsedRow> rows = new ArrayList<>();
|
||||
int totalRows = 0;
|
||||
int droppedRows = 0;
|
||||
String filename = firstNonBlank(source.getOriginalFilename(), input.getName());
|
||||
for (int i = 1; i <= sheet.getLastRowNum(); i++) {
|
||||
Row row = sheet.getRow(i);
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
Map<String, String> extra = new LinkedHashMap<>();
|
||||
String keyword = "";
|
||||
String statusValue = "";
|
||||
boolean nonEmpty = false;
|
||||
for (int c = 0; c < headers.size(); c++) {
|
||||
Cell cell = row.getCell(c);
|
||||
String value = normalize(cell == null ? "" : formatter.formatCellValue(cell));
|
||||
if (!value.isBlank()) {
|
||||
nonEmpty = true;
|
||||
}
|
||||
if (c == keywordCol) {
|
||||
keyword = value;
|
||||
} else if (c == statusCol) {
|
||||
statusValue = value;
|
||||
} else {
|
||||
extra.put(headers.get(c), value);
|
||||
}
|
||||
}
|
||||
if (!nonEmpty) {
|
||||
continue;
|
||||
}
|
||||
totalRows++;
|
||||
if (keywordCol >= 0 && keyword.isBlank()) {
|
||||
droppedRows++;
|
||||
continue;
|
||||
}
|
||||
rows.add(new ParsedRow(source.getFileKey(), filename, keyword, statusValue, extra));
|
||||
}
|
||||
return new ParsedWorkbook(totalRows, droppedRows, rows);
|
||||
} catch (BusinessException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
log.warn("[collect-data] parse workbook failed file={} err={}", input, ex.getMessage());
|
||||
throw new BusinessException("解析 Excel 失败");
|
||||
}
|
||||
}
|
||||
|
||||
private int findHeaderIndex(List<String> headers, List<String> aliases) {
|
||||
if (headers == null || headers.isEmpty() || aliases == null || aliases.isEmpty()) {
|
||||
return -1;
|
||||
}
|
||||
for (int i = 0; i < headers.size(); i++) {
|
||||
String lower = headers.get(i).toLowerCase(Locale.ROOT);
|
||||
for (String alias : aliases) {
|
||||
if (lower.contains(alias.toLowerCase(Locale.ROOT))) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
private String formatTime(LocalDateTime time) {
|
||||
return time == null ? null : time.format(TIME_FORMATTER);
|
||||
}
|
||||
|
||||
private String firstNonBlank(String... values) {
|
||||
if (values == null) {
|
||||
return "";
|
||||
}
|
||||
for (String value : values) {
|
||||
if (value != null && !value.isBlank()) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
private String normalize(String value) {
|
||||
if (value == null) {
|
||||
return "";
|
||||
}
|
||||
String normalized = value.replace(String.valueOf((char) 0xFEFF), "")
|
||||
.replace((char) 0x3000, ' ')
|
||||
.replace("\r\n", " ")
|
||||
.replace("\r", " ")
|
||||
.replace("\n", " ")
|
||||
.replace("\t", " ")
|
||||
.trim();
|
||||
return WHITESPACE_PATTERN.matcher(normalized).replaceAll(" ");
|
||||
}
|
||||
|
||||
private record ParsedRow(String sourceFileKey,
|
||||
String sourceFilename,
|
||||
String keyword,
|
||||
String statusValue,
|
||||
Map<String, String> extra) {
|
||||
}
|
||||
|
||||
private record ParsedWorkbook(int totalRows, int droppedRows, List<ParsedRow> rows) {
|
||||
}
|
||||
|
||||
private record PersistedParse(FileTaskEntity task, FileResultEntity result) {
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user