super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

提交打包

Browse Source
This commit is contained in:
super
2026-05-07 16:21:35 +08:00
parent a7d8f8be6c
commit e5d0b2d9ab
64 changed files with 1368 additions and 875 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/controller/AppearancePatentController.java
+30
View File
@@ -3,6 +3,8 @@ package com.nanri.aiimage.modules.appearancepatent.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.common.util.DownloadHeaderUtil;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParseRequest;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParsedGroupPageDto;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParsedPayloadDto;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSubmitResultRequest;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentTaskBatchRequest;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentDashboardVo;
@@ -44,6 +46,34 @@ public class AppearancePatentController {
return ApiResponse.success(service.parseAndCreateTask(request));
}
@GetMapping("/tasks/{taskId}/parsed-payload")
@Operation(summary = "获取外观专利完整解析载荷", description = "供 Python 队列消费端使用。parse 接口仅返回轻量结果，完整行数据通过本接口按 taskId 拉取。")
public ApiResponse<AppearancePatentParsedPayloadDto> parsedPayload(
@Parameter(description = "外观专利任务 ID", required = true, example = "7004")
@PathVariable Long taskId,
@Parameter(description = "当前用户 ID", required = true, example = "1")
@RequestParam("user_id") Long userId) {
return ApiResponse.success(service.parsedPayload(taskId, userId));
}
@GetMapping("/tasks/{taskId}/queue-payload")
@Operation(summary = "获取外观专利 Python 队列载荷", description = "只返回 Python 当前需要的 groups，避免前端搬运完整解析大数据。")
public ApiResponse<AppearancePatentParsedPayloadDto> queuePayload(
@PathVariable Long taskId,
@RequestParam("user_id") Long userId) {
return ApiResponse.success(service.queuePayload(taskId, userId));
}
@GetMapping("/tasks/{taskId}/parsed-groups")
@Operation(summary = "分页获取外观专利解析分组", description = "供后续优化使用，按页返回解析分组。")
public ApiResponse<AppearancePatentParsedGroupPageDto> parsedGroups(
@PathVariable Long taskId,
@RequestParam("user_id") Long userId,
@RequestParam(value = "page", required = false, defaultValue = "1") Integer page,
@RequestParam(value = "page_size", required = false, defaultValue = "50") Integer pageSize) {
return ApiResponse.success(service.parsedGroupPage(taskId, userId, page, pageSize));
}
@GetMapping("/dashboard")
@Operation(summary = "查询外观专利检测总览", description = "查询当前用户的运行中、成功、失败和已结束任务数量。页面进入时请求一次即可，不需要持续轮询。")
public ApiResponse<AppearancePatentDashboardVo> dashboard(
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentParsedGroupManifestDto.java
+23
View File
@@ -0,0 +1,23 @@
package com.nanri.aiimage.modules.appearancepatent.model.dto;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
public class AppearancePatentParsedGroupManifestDto {
private String aiPrompt;
private String apiKey;
private Integer pageSize;
private Integer totalGroups;
private Integer totalRows;
private List<PageRef> pages = new ArrayList<>();
@Data
public static class PageRef {
private Integer page;
private Integer groupCount;
private String ref;
}
}
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentParsedGroupPageDto.java
+19
View File
@@ -0,0 +1,19 @@
package com.nanri.aiimage.modules.appearancepatent.model.dto;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedGroupVo;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
public class AppearancePatentParsedGroupPageDto {
private String aiPrompt;
private String apiKey;
private Integer page;
private Integer pageSize;
private Integer totalGroups;
private Integer totalRows;
private Boolean hasNext;
private List<AppearancePatentParsedGroupVo> groups = new ArrayList<>();
}
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentParsedRowVo.java
+3 -6
View File
@@ -3,9 +3,6 @@ package com.nanri.aiimage.modules.appearancepatent.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.LinkedHashMap;
import java.util.Map;
@Data
@Schema(description = "外观专利检测解析出的单行数据")
public class AppearancePatentParsedRowVo {
@@ -36,12 +33,12 @@ public class AppearancePatentParsedRowVo {
@Schema(description = "国家或站点。", example = "英国")
private String country;
@Schema(description = "价格。", example = "12.99")
private String price;
@Schema(description = "商品图片 URL 或商品 URL，供 Coze 检测使用。", example = "https://webstatic.aiproxy.vip/output/demo.jpg")
private String url;
@Schema(description = "商品标题。", example = "Women Floral Dress Summer Casual")
private String title;
@Schema(description = "该 Excel 行的原始列值映射。最终生成 xlsx 时可以从这里读取价格等字段。", example = "{\"id\":\"2_1\",\"asin\":\"B0CJ8SNXXV\",\"国家\":\"英国\",\"价格\":\"12.99\"}")
private Map<String, String> values = new LinkedHashMap<>();
}
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java
+74 -56
View File
@@ -13,6 +13,7 @@ import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentCozeClient;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParseRequest;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParsedGroupPageDto;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParsedPayloadDto;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultGroupDto;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
@@ -175,7 +176,7 @@ public class AppearancePatentTaskService {
throw new BusinessException("未解析到有效 ASIN 数据");
}
long parsedAt = System.nanoTime();
List<AppearancePatentParsedGroupVo> groups = buildParsedGroups(allRows);
int groupCount = countParsedGroups(allRows);
long groupedAt = System.nanoTime();
FileTaskEntity task = new FileTaskEntity();
@@ -201,7 +202,7 @@ public class AppearancePatentTaskService {
String aggregateScopeKey = buildAggregateScopeKey(sourceFiles);
String sourceScopeHash = DigestUtil.sha256Hex(aggregateScopeKey);
String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), request.getApiKey(), sourceFiles, mergedHeaders, groups, allRows);
String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), request.getApiKey(), sourceFiles, mergedHeaders, allRows);
long payloadBuiltAt = System.nanoTime();
String parsedPayloadPointer = storeParsedPayload(task.getId(), sourceScopeHash, parsedPayload);
long payloadStoredAt = System.nanoTime();
@@ -242,16 +243,16 @@ public class AppearancePatentTaskService {
vo.setTotalRows(totalRows);
vo.setAcceptedRows(allRows.size());
vo.setDroppedRows(droppedRows);
vo.setGroupCount(groups.size());
vo.setGroupCount(groupCount);
vo.setAiPrompt(normalize(request.getAiPrompt()));
vo.setItems(allRows);
vo.setGroups(groups);
vo.setItems(List.of());
vo.setGroups(List.of());
long finishedAt = System.nanoTime();
log.info("[appearance-patent] parse timing taskId={} files={} rows={} groups={} totalMs={} parseMs={} groupMs={} taskInsertMs={} payloadJsonMs={} payloadStoreMs={} persistMs={} responseMs={}",
task.getId(),
sourceFiles.size(),
allRows.size(),
groups.size(),
groupCount,
elapsedMs(startedAt, finishedAt),
elapsedMs(parseStartedAt, parsedAt),
elapsedMs(parsedAt, groupedAt),
@@ -322,15 +323,7 @@ public class AppearancePatentTaskService {
.map(FileResultEntity::getId)
.filter(Objects::nonNull)
.toList());
List<FileResultEntity> sortedRows = new ArrayList<>();
for (FileResultEntity row : rows) {
FileTaskEntity task = taskMap.get(row.getTaskId());
String taskStatus = task == null ? null : task.getStatus();
if (STATUS_PENDING.equals(taskStatus)) {
continue;
}
sortedRows.add(row);
}
List<FileResultEntity> sortedRows = new ArrayList<>(rows);
sortedRows.sort(Comparator
.comparingInt((FileResultEntity row) -> historyPriority(row, taskMap.get(row.getTaskId()), jobMap.get(row.getId())))
.thenComparing((FileResultEntity row) -> historyActivityTime(row, taskMap.get(row.getTaskId()), jobMap.get(row.getId())),
@@ -968,6 +961,10 @@ public class AppearancePatentTaskService {
return groups;
}
private int countParsedGroups(List<AppearancePatentParsedRowVo> rows) {
return groupRowsByBaseId(rows).size();
}
private Map<String, List<AppearancePatentParsedRowVo>> groupRowsByBaseId(List<AppearancePatentParsedRowVo> rows) {
Map<String, List<AppearancePatentParsedRowVo>> result = new LinkedHashMap<>();
if (rows == null) {
@@ -1997,7 +1994,7 @@ public class AppearancePatentTaskService {
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getDisplayId(), parsedRow.getSourceId()));
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getAsin(), ""));
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getCountry(), ""));
row.createCell(col++).setCellValue(readValueByHeader(parsedRow, "价格", "price"));
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getPrice(), ""));
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getTitle(), "") : firstNonBlank(resultRow.getTitle(), parsedRow.getTitle()));
row.createCell(col++).setCellValue(resultRow == null ? firstNonBlank(parsedRow.getUrl(), "") : firstNonBlank(resultRow.getUrl(), parsedRow.getUrl()));
row.createCell(col++).setCellValue(resultRow == null ? missingReason : userFacingCozeCellValue(resultRow, resultRow.getTitleRisk()));
@@ -2081,10 +2078,10 @@ public class AppearancePatentTaskService {
throw new BusinessException("Excel 表头为空");
}
Map<String, Integer> headerMap = buildHeaderMap(header, formatter);
List<String> headers = readHeaders(header, formatter);
int idCol = findRequiredHeader(headerMap, "id");
int asinCol = findRequiredHeader(headerMap, "asin");
int countryCol = findRequiredHeader(headerMap, "国家", "country");
int priceCol = findOptionalHeaderExact(headerMap, "价格", "price");
int urlCol = findOptionalHeaderExact(headerMap,
"url", "rul", "link", "image", "img", "pic", "picture",
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
@@ -2127,16 +2124,16 @@ public class AppearancePatentTaskService {
vo.setRowToken(buildRowToken(source.getFileKey(), vo.getRowIndex()));
vo.setAsin(asin);
vo.setCountry(country);
vo.setPrice(priceCol >= 0 ? cell(row, priceCol, formatter) : "");
vo.setUrl(urlCol >= 0 ? cell(row, urlCol, formatter) : "");
vo.setTitle(titleCol >= 0 ? cell(row, titleCol, formatter) : "");
vo.setValues(readRowValues(row, headers, formatter));
allRows.add(vo);
}
hydratePromptFields(allRows);
if (allRows.isEmpty()) {
throw new BusinessException("no valid appearance patent rows");
}
return new ParsedWorkbook(total, dropped, headers, allRows);
return new ParsedWorkbook(total, dropped, List.of(), allRows);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
@@ -2202,23 +2199,6 @@ public class AppearancePatentTaskService {
return map;
}
private List<String> readHeaders(Row header, DataFormatter formatter) {
List<String> headers = new ArrayList<>();
for (int i = 0; i < header.getLastCellNum(); i++) {
String val = normalize(formatter.formatCellValue(header.getCell(i)));
headers.add(val.isBlank() ? "列" + (i + 1) : val);
}
return headers;
}
private Map<String, String> readRowValues(Row row, List<String> headers, DataFormatter formatter) {
Map<String, String> values = new LinkedHashMap<>();
for (int i = 0; i < headers.size(); i++) {
values.put(headers.get(i), cell(row, i, formatter));
}
return values;
}
private int findRequiredHeader(Map<String, Integer> map, String... names) {
int idx = findOptionalHeader(map, names);
if (idx < 0) {
@@ -2368,7 +2348,7 @@ public class AppearancePatentTaskService {
private int historyPriority(FileResultEntity row, FileTaskEntity task, TaskFileJobEntity job) {
String taskStatus = task == null ? null : task.getStatus();
if (STATUS_RUNNING.equals(taskStatus) || isHistoryFileBuilding(row, taskStatus, job)) {
if (STATUS_PENDING.equals(taskStatus) || STATUS_RUNNING.equals(taskStatus) || isHistoryFileBuilding(row, taskStatus, job)) {
return 0;
}
return 1;
@@ -2476,15 +2456,15 @@ public class AppearancePatentTaskService {
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
}
private String buildParsedPayloadJson(String aiPrompt, String apiKey, List<AppearancePatentSourceFileDto> sourceFiles, List<String> headers, List<AppearancePatentParsedGroupVo> groups, List<AppearancePatentParsedRowVo> allRows) {
private String buildParsedPayloadJson(String aiPrompt, String apiKey, List<AppearancePatentSourceFileDto> sourceFiles, List<String> headers, List<AppearancePatentParsedRowVo> allRows) {
AppearancePatentParsedPayloadDto payload = new AppearancePatentParsedPayloadDto();
payload.setAiPrompt(normalize(aiPrompt));
payload.setApiKey(normalize(apiKey));
payload.setSourceFiles(sourceFiles == null ? List.of() : sourceFiles);
payload.setHeaders(headers == null ? List.of() : headers);
payload.setItems(List.of());
payload.setGroups(groups == null ? List.of() : groups);
payload.setAllItems(List.of());
payload.setGroups(List.of());
payload.setAllItems(allRows == null ? List.of() : allRows);
return writeJson(payload, "保存解析结果失败");
}
@@ -2552,6 +2532,57 @@ public class AppearancePatentTaskService {
return hydrateParsedPayloadRows(new AppearancePatentParsedPayloadDto());
}
public AppearancePatentParsedPayloadDto parsedPayload(Long taskId, Long userId) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || userId != null && !Objects.equals(userId, task.getUserId())) {
throw new BusinessException("任务不存在");
}
return readParsedPayload(task);
}
public AppearancePatentParsedPayloadDto queuePayload(Long taskId, Long userId) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || userId != null && !Objects.equals(userId, task.getUserId())) {
throw new BusinessException("任务不存在");
}
AppearancePatentParsedPayloadDto payload = readParsedPayload(task);
AppearancePatentParsedPayloadDto queuePayload = new AppearancePatentParsedPayloadDto();
queuePayload.setAiPrompt(payload.getAiPrompt());
queuePayload.setApiKey(payload.getApiKey());
queuePayload.setGroups(payload.getGroups() == null ? List.of() : payload.getGroups());
queuePayload.setItems(List.of());
queuePayload.setAllItems(List.of());
queuePayload.setHeaders(List.of());
queuePayload.setSourceFiles(List.of());
return queuePayload;
}
public AppearancePatentParsedGroupPageDto parsedGroupPage(Long taskId, Long userId, Integer page, Integer pageSize) {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || userId != null && !Objects.equals(userId, task.getUserId())) {
throw new BusinessException("任务不存在");
}
int safePage = Math.max(1, page == null ? 1 : page);
int safePageSize = Math.max(1, Math.min(pageSize == null ? 50 : pageSize, 200));
AppearancePatentParsedPayloadDto payload = readParsedPayload(task);
List<AppearancePatentParsedGroupVo> groups = payload.getGroups() == null ? List.of() : payload.getGroups();
int totalGroups = groups.size();
int totalRows = payload.getAllItems() == null ? 0 : payload.getAllItems().size();
int fromIndex = Math.min((safePage - 1) * safePageSize, totalGroups);
int toIndex = Math.min(fromIndex + safePageSize, totalGroups);
AppearancePatentParsedGroupPageDto vo = new AppearancePatentParsedGroupPageDto();
vo.setAiPrompt(payload.getAiPrompt());
vo.setApiKey(payload.getApiKey());
vo.setPage(safePage);
vo.setPageSize(safePageSize);
vo.setTotalGroups(totalGroups);
vo.setTotalRows(totalRows);
vo.setHasNext(toIndex < totalGroups);
vo.setGroups(fromIndex >= toIndex ? List.of() : groups.subList(fromIndex, toIndex));
return vo;
}
private AppearancePatentParsedPayloadDto hydrateParsedPayloadRows(AppearancePatentParsedPayloadDto payload) {
if (payload == null) {
return new AppearancePatentParsedPayloadDto();
@@ -2575,6 +2606,9 @@ public class AppearancePatentTaskService {
if (payload.getItems() == null || payload.getItems().isEmpty()) {
payload.setItems(rows);
}
if ((payload.getGroups() == null || payload.getGroups().isEmpty()) && !rows.isEmpty()) {
payload.setGroups(buildParsedGroups(rows));
}
return payload;
}
@@ -2656,22 +2690,6 @@ public class AppearancePatentTaskService {
return normalize(id) + "::" + normalize(asin).toUpperCase(Locale.ROOT) + "::" + normalize(country);
}
private String readValueByHeader(AppearancePatentParsedRowVo row, String... candidates) {
if (row == null || row.getValues() == null || row.getValues().isEmpty() || candidates == null) {
return "";
}
for (Map.Entry<String, String> entry : row.getValues().entrySet()) {
String header = normalize(entry.getKey()).toLowerCase(Locale.ROOT);
for (String candidate : candidates) {
String expected = normalize(candidate).toLowerCase(Locale.ROOT);
if (!expected.isBlank() && header.contains(expected)) {
return entry.getValue() == null ? "" : entry.getValue();
}
}
}
return "";
}
private String userFacingCozeCellValue(AppearancePatentResultRowDto row, String value) {
String normalizedValue = normalize(value);
if (!normalizedValue.isBlank() && !isTechnicalCozeFailure(normalizedValue)) {