super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

稳定组成这个文件优化

Browse Source
This commit is contained in:
super
2026-05-02 16:40:13 +08:00
parent 12d84a22e4
commit 934919c699
39 changed files with 126 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/.env
View File

@@ -11,9 +11,9 @@ zn_username=%E8%87%AA%E5%8A%A8%E5%8C%96_Robot
client_name=ShuFuAI client_name=ShuFuAI
# java_api_base=http://47.111.163.154:18080 java_api_base=http://47.111.163.154:18080
# java_api_base=http://127.0.0.1:18080 # java_api_base=http://127.0.0.1:18080
# java_api_base=http://8.136.19.173:18080 # java_api_base=http://8.136.19.173:18080
java_api_base=http://121.196.149.225:18080 # java_api_base=http://121.196.149.225:18080

1
app/assets/appearance-patent-BYYE4_FI.css
View File

File diff suppressed because one or more lines are too long

1
app/assets/appearance-patent-CA1Tj-nN.css
View File

File diff suppressed because one or more lines are too long

1
app/assets/appearance-patent-DaZnvhj3.css
View File

File diff suppressed because one or more lines are too long

1
app/assets/appearance-patent-TI-osB39.css
View File

File diff suppressed because one or more lines are too long

4
app/assets/appearance-patent.js
View File

File diff suppressed because one or more lines are too long

1
app/assets/brand-CABpTGPu.js
View File

@@ -1 +0,0 @@
import{aZ as r}from"./pywebview-BWNiqXug.js";const n="";function a(e){return r(`${n}/api/brand/expand-folder-recursive`,{folder:e})}export{a as e};

1
app/assets/convert-BKSNvX8i.css Normal file
View File

File diff suppressed because one or more lines are too long

1
app/assets/convert-r7MSXq7I.css
View File

File diff suppressed because one or more lines are too long

2
app/assets/convert.js
View File

File diff suppressed because one or more lines are too long

1
app/assets/dedupe-DMeaPUMZ.css
View File

File diff suppressed because one or more lines are too long

1
app/assets/dedupe-DNlVfFj-.css Normal file
View File

File diff suppressed because one or more lines are too long

2
app/assets/dedupe.js
View File

File diff suppressed because one or more lines are too long

1
app/assets/delete-brand-B5ChOXZY.css
View File

File diff suppressed because one or more lines are too long

2
app/assets/delete-brand.js
View File

File diff suppressed because one or more lines are too long

1
app/assets/patrol-delete-CBBdonF4.css Normal file
View File

File diff suppressed because one or more lines are too long

1
app/assets/patrol-delete-CdffrCXY.css
View File

File diff suppressed because one or more lines are too long

1
app/assets/patrol-delete-DyumfGaZ.css
View File

File diff suppressed because one or more lines are too long

2
app/assets/patrol-delete.js
View File

File diff suppressed because one or more lines are too long

1
app/assets/price-track-DMhhVz4u.css
View File

File diff suppressed because one or more lines are too long

2
app/assets/price-track.js
View File

File diff suppressed because one or more lines are too long

1
app/assets/product-risk-CI6qY5U2.css
View File

File diff suppressed because one or more lines are too long

2
app/assets/product-risk.js
View File

File diff suppressed because one or more lines are too long

55
app/assets/pywebview-BWNiqXug.js
View File

File diff suppressed because one or more lines are too long

2
app/assets/pywebview-BFZO8_3q.css → app/assets/pywebview-Dp5dN8OO.css
View File

File diff suppressed because one or more lines are too long

1
app/assets/query-asin-Bw6BQsRN.css
View File

File diff suppressed because one or more lines are too long

2
app/assets/query-asin.js
View File

File diff suppressed because one or more lines are too long

1
app/assets/shop-match-CxCKUsqS.css
View File

File diff suppressed because one or more lines are too long

2
app/assets/shop-match.js
View File

File diff suppressed because one or more lines are too long

1
app/assets/split-BoBVrLdC.css Normal file
View File

File diff suppressed because one or more lines are too long

1
app/assets/split-Cw6sXhAg.css
View File

File diff suppressed because one or more lines are too long

2
app/assets/split.js
View File

File diff suppressed because one or more lines are too long

1
app/assets/zh-cn-BbuLS91u.js
View File

File diff suppressed because one or more lines are too long

30
app/new_web_source/convert.html
View File

@@ -1,16 +1,16 @@
<!doctype html> <!doctype html>
<html lang="zh-CN"> <html lang="zh-CN">
<head> <head>
<meta charset="UTF-8" /> <meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>格式转换 - 数富AI</title> <title>格式转换 - 数富AI</title>
<script type="module" crossorigin src="/assets/convert.js"></script> <script type="module" crossorigin src="/assets/convert.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-BWNiqXug.js"> <link rel="modulepreload" crossorigin href="/assets/pywebview-BtdhG28Q.js">
<link rel="modulepreload" crossorigin href="/assets/brand-CABpTGPu.js"> <link rel="modulepreload" crossorigin href="/assets/brand-Lp7YWffz.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-BFZO8_3q.css"> <link rel="stylesheet" crossorigin href="/assets/pywebview-Dp5dN8OO.css">
<link rel="stylesheet" crossorigin href="/assets/convert-r7MSXq7I.css"> <link rel="stylesheet" crossorigin href="/assets/convert-BKSNvX8i.css">
</head> </head>
<body> <body>
<div id="app"></div> <div id="app"></div>
</body>
</body> </html>

30
app/new_web_source/dedupe.html
View File

@@ -1,16 +1,16 @@
<!doctype html> <!doctype html>
<html lang="zh-CN"> <html lang="zh-CN">
<head> <head>
<meta charset="UTF-8" /> <meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>数据去重 - 数富AI</title> <title>数据去重 - 数富AI</title>
<script type="module" crossorigin src="/assets/dedupe.js"></script> <script type="module" crossorigin src="/assets/dedupe.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-BWNiqXug.js"> <link rel="modulepreload" crossorigin href="/assets/pywebview-BtdhG28Q.js">
<link rel="modulepreload" crossorigin href="/assets/brand-CABpTGPu.js"> <link rel="modulepreload" crossorigin href="/assets/brand-Lp7YWffz.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-BFZO8_3q.css"> <link rel="stylesheet" crossorigin href="/assets/pywebview-Dp5dN8OO.css">
<link rel="stylesheet" crossorigin href="/assets/dedupe-DMeaPUMZ.css"> <link rel="stylesheet" crossorigin href="/assets/dedupe-DNlVfFj-.css">
</head> </head>
<body> <body>
<div id="app"></div> <div id="app"></div>
</body>
</body> </html>

30
app/new_web_source/delete-brand.html
View File

@@ -1,17 +1,17 @@
<!doctype html> <!doctype html>
<html lang="zh-CN"> <html lang="zh-CN">
<head> <head>
<meta charset="UTF-8" /> <meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>删除品牌 - 数富AI</title> <title>删除品牌 - 数富AI</title>
<script type="module" crossorigin src="/assets/delete-brand.js"></script> <script type="module" crossorigin src="/assets/delete-brand.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-BWNiqXug.js"> <link rel="modulepreload" crossorigin href="/assets/pywebview-BtdhG28Q.js">
<link rel="modulepreload" crossorigin href="/assets/brand-CABpTGPu.js"> <link rel="modulepreload" crossorigin href="/assets/brand-Lp7YWffz.js">
<link rel="modulepreload" crossorigin href="/assets/categorized-timers-JPA-olTr.js"> <link rel="modulepreload" crossorigin href="/assets/categorized-timers-JPA-olTr.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-BFZO8_3q.css"> <link rel="stylesheet" crossorigin href="/assets/pywebview-Dp5dN8OO.css">
<link rel="stylesheet" crossorigin href="/assets/delete-brand-B5ChOXZY.css"> <link rel="stylesheet" crossorigin href="/assets/delete-brand-BfMLVSQU.css">
</head> </head>
<body> <body>
<div id="app"></div> <div id="app"></div>
</body>
</body> </html>

30
app/new_web_source/split.html
View File

@@ -1,16 +1,16 @@
<!doctype html> <!doctype html>
<html lang="zh-CN"> <html lang="zh-CN">
<head> <head>
<meta charset="UTF-8" /> <meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>数据拆分 - 数富AI</title> <title>数据拆分 - 数富AI</title>
<script type="module" crossorigin src="/assets/split.js"></script> <script type="module" crossorigin src="/assets/split.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-BWNiqXug.js"> <link rel="modulepreload" crossorigin href="/assets/pywebview-BtdhG28Q.js">
<link rel="modulepreload" crossorigin href="/assets/brand-CABpTGPu.js"> <link rel="modulepreload" crossorigin href="/assets/brand-Lp7YWffz.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-BFZO8_3q.css"> <link rel="stylesheet" crossorigin href="/assets/pywebview-Dp5dN8OO.css">
<link rel="stylesheet" crossorigin href="/assets/split-Cw6sXhAg.css"> <link rel="stylesheet" crossorigin href="/assets/split-BoBVrLdC.css">
</head> </head>
<body> <body>
<div id="app"></div> <div id="app"></div>
</body>
</body> </html>

45
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java
View File

@@ -86,6 +86,8 @@ public class AppearancePatentTaskService {
private static final String STATUS_SUCCESS = "SUCCESS"; private static final String STATUS_SUCCESS = "SUCCESS";
private static final String STATUS_FAILED = "FAILED"; private static final String STATUS_FAILED = "FAILED";
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final int RESULT_ROWS_READ_RETRY_LIMIT = 3;
private static final long RESULT_ROWS_READ_RETRY_DELAY_MS = 500L;
private static final List<String> RESULT_HEADERS = List.of( private static final List<String> RESULT_HEADERS = List.of(
"id", "id",
"asin", "asin",
@@ -759,12 +761,17 @@ public class AppearancePatentTaskService {
} }
String payloadJson = writeJson(rows == null ? List.of() : rows, "结果序列化失败"); String payloadJson = writeJson(rows == null ? List.of() : rows, "结果序列化失败");
payloadJson = writeJson(new ArrayList<>(persistedRows.values()), "appearance patent chunk payload merge failed"); payloadJson = writeJson(new ArrayList<>(persistedRows.values()), "appearance patent chunk payload merge failed");
String storedPayload = transientPayloadStorageService.storeChunkPayload(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson); String oldPayload = chunk.getPayloadJson();
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(chunk.getPayloadJson(), storedPayload); String storedPayload = transientPayloadStorageService.storeChunkPayloadVersioned(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson);
chunk.setPayloadJson(storedPayload); chunk.setPayloadJson(storedPayload);
chunk.setPayloadHash(DigestUtil.sha256Hex(payloadJson)); chunk.setPayloadHash(DigestUtil.sha256Hex(payloadJson));
chunk.setUpdatedAt(LocalDateTime.now()); chunk.setUpdatedAt(LocalDateTime.now());
taskChunkMapper.updateById(chunk); int updated = taskChunkMapper.updateById(chunk);
if (updated <= 0) {
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
throw new IllegalStateException("appearance patent chunk payload update failed");
}
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(oldPayload, storedPayload);
} }
private List<AppearancePatentResultRowDto> expandRows(List<AppearancePatentResultRowDto> representatives, private List<AppearancePatentResultRowDto> expandRows(List<AppearancePatentResultRowDto> representatives,
@@ -1122,7 +1129,7 @@ public class AppearancePatentTaskService {
private void assembleResultWorkbook(FileTaskEntity task, FileResultEntity result) { private void assembleResultWorkbook(FileTaskEntity task, FileResultEntity result) {
AppearancePatentParsedPayloadDto parsed = readParsedPayload(task); AppearancePatentParsedPayloadDto parsed = readParsedPayload(task);
Map<String, AppearancePatentResultRowDto> resultMap = loadPersistedResultRows(task.getId()); Map<String, AppearancePatentResultRowDto> resultMap = loadPersistedResultRowsWithRetry(task.getId(), parsed.getAllItems().size());
long resolvedRows = parsed.getAllItems().stream() long resolvedRows = parsed.getAllItems().stream()
.filter(row -> findResultRow(row, resultMap) != null || findResultRowByAsin(row.getAsin(), resultMap) != null) .filter(row -> findResultRow(row, resultMap) != null || findResultRowByAsin(row.getAsin(), resultMap) != null)
.count(); .count();
@@ -1131,6 +1138,9 @@ public class AppearancePatentTaskService {
.count(); .count();
log.info("[appearance-patent] assemble workbook taskId={} parsedRows={} resultRows={} resolvedRows={} reasonRows={}", log.info("[appearance-patent] assemble workbook taskId={} parsedRows={} resultRows={} resolvedRows={} reasonRows={}",
task.getId(), parsed.getAllItems().size(), resultMap.size(), resolvedRows, reasonRows); task.getId(), parsed.getAllItems().size(), resultMap.size(), resolvedRows, reasonRows);
if (!parsed.getAllItems().isEmpty() && resultMap.isEmpty()) {
throw new BusinessException("外观专利检测结果为空，请稍后重试生成结果文件");
}
File outputDir = new File(storageProperties.getLocalTempDir(), "appearance-patent-result"); File outputDir = new File(storageProperties.getLocalTempDir(), "appearance-patent-result");
if (!outputDir.exists() && !outputDir.mkdirs()) { if (!outputDir.exists() && !outputDir.mkdirs()) {
throw new BusinessException("创建结果目录失败"); throw new BusinessException("创建结果目录失败");
@@ -1164,6 +1174,33 @@ public class AppearancePatentTaskService {
return result; return result;
} }
private Map<String, AppearancePatentResultRowDto> loadPersistedResultRowsWithRetry(Long taskId, int expectedParsedRows) {
Map<String, AppearancePatentResultRowDto> result = loadPersistedResultRows(taskId);
if (expectedParsedRows <= 0 || !result.isEmpty()) {
return result;
}
for (int attempt = 1; attempt <= RESULT_ROWS_READ_RETRY_LIMIT && result.isEmpty(); attempt++) {
sleepBeforeResultRowsRetry(attempt);
result = loadPersistedResultRows(taskId);
if (!result.isEmpty()) {
log.info("[appearance-patent] result rows recovered after retry taskId={} attempt={} rows={}",
taskId, attempt, result.size());
return result;
}
log.warn("[appearance-patent] result rows still empty after retry taskId={} attempt={}/{}",
taskId, attempt, RESULT_ROWS_READ_RETRY_LIMIT);
}
return result;
}
private void sleepBeforeResultRowsRetry(int attempt) {
try {
Thread.sleep(RESULT_ROWS_READ_RETRY_DELAY_MS * attempt);
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
}
}
private void writeResultWorkbook(File xlsx, AppearancePatentParsedPayloadDto parsed, Map<String, AppearancePatentResultRowDto> resultMap) { private void writeResultWorkbook(File xlsx, AppearancePatentParsedPayloadDto parsed, Map<String, AppearancePatentResultRowDto> resultMap) {
try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream fos = new FileOutputStream(xlsx)) { try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream fos = new FileOutputStream(xlsx)) {
Sheet sheet = workbook.createSheet("外观专利检测结果"); Sheet sheet = workbook.createSheet("外观专利检测结果");

7
backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TransientPayloadStorageService.java
View File

@@ -55,6 +55,13 @@ public class TransientPayloadStorageService {
return store("task-chunk", moduleType, taskId, scopeHash, entryKey, content, true); return store("task-chunk", moduleType, taskId, scopeHash, entryKey, content, true);
} }
public String storeChunkPayloadVersioned(String moduleType, Long taskId, String scopeHash, Integer chunkIndex, String content) {
String entryKey = chunkIndex == null
? UUID.randomUUID().toString()
: "chunk-" + chunkIndex + "-" + UUID.randomUUID();
return store("task-chunk", moduleType, taskId, scopeHash, entryKey, content, true);
}
public String resolvePayload(String value, String errorMessage) { public String resolvePayload(String value, String errorMessage) {
String pointer = extractPointer(value); String pointer = extractPointer(value);
if (pointer == null) { if (pointer == null) {