super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

合并分支

Browse Source
This commit is contained in:
super
2026-05-31 09:28:48 +08:00
parent 9835831415
commit ea37d82d73
35 changed files with 200 additions and 7893 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentCozeClient.java
+31 -18
View File
@@ -36,6 +36,10 @@ public class AppearancePatentCozeClient {
private final AtomicLong credentialCursor = new AtomicLong();
public List<AppearancePatentResultRowDto> inspect(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey) {
return inspect(rows, prompt, apiKey, null);
}
public List<AppearancePatentResultRowDto> inspect(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey, String patentToken) {
if (rows == null || rows.isEmpty()) {
return List.of();
}
@@ -44,7 +48,7 @@ public class AppearancePatentCozeClient {
return rows.stream().map(this::copy).toList();
}
try {
return inspectWithFallback(rows, prompt, apiKey);
return inspectWithFallback(rows, prompt, apiKey, patentToken);
} catch (Exception ex) {
String failureMessage = failureMessage(ex);
log.warn("[appearance-patent] coze batch failed size={} err={}", rows.size(), failureMessage);
@@ -53,15 +57,16 @@ public class AppearancePatentCozeClient {
}
public CozeSubmitResponse submitWorkflow(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey) throws Exception {
return submitWorkflow(rows, prompt, apiKey, nextCredential());
return submitWorkflow(rows, prompt, apiKey, null, nextCredential());
}
public CozeSubmitResponse submitWorkflow(List<AppearancePatentResultRowDto> rows,
String prompt,
String apiKey,
String patentToken,
CozeCredentialRef credential) throws Exception {
CozeCredentialRef resolvedCredential = resolveCredential(credential);
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, resolvedCredential));
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, patentToken, resolvedCredential));
ensureSuccess(submitRoot);
return new CozeSubmitResponse(
extractExecuteId(submitRoot),
@@ -103,12 +108,12 @@ public class AppearancePatentCozeClient {
return rows.stream().map(this::copy).map(row -> markFailed(row, failureMessage)).toList();
}
private List<AppearancePatentResultRowDto> inspectWithFallback(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey) {
private List<AppearancePatentResultRowDto> inspectWithFallback(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey, String patentToken) {
try {
if (rows.size() == 1) {
return inspectSingleRowWithRetry(rows, prompt, apiKey);
return inspectSingleRowWithRetry(rows, prompt, apiKey, patentToken);
}
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey);
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, patentToken);
if (attempt.resolvedCount() < rows.size()) {
throw new PartialCozeResultException(attempt.resolvedCount(), rows.size(), attempt.rawResultCount());
}
@@ -119,17 +124,17 @@ public class AppearancePatentCozeClient {
log.warn("[appearance-patent] coze batch fallback split size={} left={} right={} err={}",
rows.size(), middle, rows.size() - middle, failureMessage(ex));
List<AppearancePatentResultRowDto> merged = new ArrayList<>(rows.size());
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(0, middle), prompt, apiKey));
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(middle, rows.size()), prompt, apiKey));
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(0, middle), prompt, apiKey, patentToken));
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(middle, rows.size()), prompt, apiKey, patentToken));
return merged;
}
throw propagate(ex);
}
}
private List<AppearancePatentResultRowDto> inspectPartitionWithFailureFallback(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey) {
private List<AppearancePatentResultRowDto> inspectPartitionWithFailureFallback(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey, String patentToken) {
try {
return inspectWithFallback(rows, prompt, apiKey);
return inspectWithFallback(rows, prompt, apiKey, patentToken);
} catch (Exception ex) {
String failureMessage = failureMessage(ex);
log.warn("[appearance-patent] coze partition failed size={} err={}", rows.size(), failureMessage);
@@ -137,12 +142,12 @@ public class AppearancePatentCozeClient {
}
}
private List<AppearancePatentResultRowDto> inspectSingleRowWithRetry(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey) throws Exception {
private List<AppearancePatentResultRowDto> inspectSingleRowWithRetry(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey, String patentToken) throws Exception {
AppearancePatentResultRowDto row = rows.getFirst();
PartialCozeResultException lastFailure = null;
for (int attemptIndex = 1; attemptIndex <= 3; attemptIndex++) {
try {
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey);
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, patentToken);
if (attempt.resolvedCount() == rows.size()) {
return attempt.mergedRows();
}
@@ -173,8 +178,8 @@ public class AppearancePatentCozeClient {
throw lastFailure == null ? new PartialCozeResultException(0, rows.size(), 0) : lastFailure;
}
private InspectAttempt inspectOnce(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey) throws Exception {
String raw = runWorkflowAsyncAndWait(rows, prompt, apiKey);
private InspectAttempt inspectOnce(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey, String patentToken) throws Exception {
String raw = runWorkflowAsyncAndWait(rows, prompt, apiKey, patentToken);
List<CozeResult> results = parseResults(raw);
if (rows.size() > 1 && !results.isEmpty() && results.stream().noneMatch(this::hasIdentity)) {
throw new PartialCozeResultException(0, rows.size(), results.size());
@@ -183,9 +188,9 @@ public class AppearancePatentCozeClient {
return new InspectAttempt(raw, merged, resolvedCount(merged), results.size());
}
private String runWorkflowAsyncAndWait(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey) throws Exception {
private String runWorkflowAsyncAndWait(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey, String patentToken) throws Exception {
CozeCredentialRef credential = nextCredential();
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, credential));
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, patentToken, credential));
ensureSuccess(submitRoot);
String immediateData = extractResultDataText(submitRoot);
@@ -229,8 +234,9 @@ public class AppearancePatentCozeClient {
private String postWorkflow(List<AppearancePatentResultRowDto> rows,
String prompt,
String apiKey,
String patentToken,
CozeCredentialRef credential) {
Map<String, Object> parameters = buildParameters(rows, prompt, apiKey);
Map<String, Object> parameters = buildParameters(rows, prompt, apiKey, patentToken);
Map<String, Object> body = new LinkedHashMap<>();
body.put("workflow_id", credential.workflowId());
body.put("parameters", parameters);
@@ -350,7 +356,7 @@ public class AppearancePatentCozeClient {
return credentials;
}
private Map<String, Object> buildParameters(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey) {
private Map<String, Object> buildParameters(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey, String patentToken) {
List<String> groupKeys = rows.stream().map(row -> nonBlank(row.getGroupKey(), rowKey(row))).toList();
List<String> rowIds = rows.stream().map(row -> nonBlank(row.getId(), "")).toList();
List<String> asins = rows.stream().map(row -> nonBlank(row.getAsin(), "")).toList();
@@ -370,6 +376,9 @@ public class AppearancePatentCozeClient {
if (apiKey != null && !apiKey.isBlank()) {
parameters.put("api_key", apiKey.trim());
}
if (patentToken != null && !patentToken.isBlank()) {
parameters.put("patent_token", patentToken.trim());
}
return parameters;
}
@@ -383,6 +392,10 @@ public class AppearancePatentCozeClient {
if (apiKey instanceof String apiKeyText && !apiKeyText.isBlank()) {
maskedParameters.put("api_key", maskSecret(apiKeyText));
}
Object patentToken = maskedParameters.get("patent_token");
if (patentToken instanceof String patentTokenText && !patentTokenText.isBlank()) {
maskedParameters.put("patent_token", maskSecret(patentTokenText));
}
masked.put("parameters", maskedParameters);
}
return masked;
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentParseRequest.java
+5
View File
@@ -32,4 +32,9 @@ public class AppearancePatentParseRequest {
@Schema(description = "传递给 Coze workflow parameters.api_key 的任务级密钥。")
@NotBlank(message = "密钥不能为空")
private String apiKey;
@JsonProperty("patent_token")
@JsonAlias({"patentToken"})
@Schema(description = "传递给 Coze workflow parameters.patent_token 的专利汇令牌。非必填。")
private String patentToken;
}
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentParsedGroupManifestDto.java
+1
View File
@@ -9,6 +9,7 @@ import java.util.List;
public class AppearancePatentParsedGroupManifestDto {
private String aiPrompt;
private String apiKey;
private String patentToken;
private Integer pageSize;
private Integer totalGroups;
private Integer totalRows;
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentParsedGroupPageDto.java
+1
View File
@@ -10,6 +10,7 @@ import java.util.List;
public class AppearancePatentParsedGroupPageDto {
private String aiPrompt;
private String apiKey;
private String patentToken;
private Integer page;
private Integer pageSize;
private Integer totalGroups;
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentParsedPayloadDto.java
+3
View File
@@ -17,6 +17,9 @@ public class AppearancePatentParsedPayloadDto {
@Schema(description = "传递给 Coze workflow parameters.api_key 的任务级密钥")
private String apiKey;
@Schema(description = "传递给 Coze workflow parameters.patent_token 的专利汇令牌")
private String patentToken;
@Schema(description = "本次解析的源文件列表")
private List<AppearancePatentSourceFileDto> sourceFiles = new ArrayList<>();
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java
+27 -11
View File
@@ -218,11 +218,11 @@ public class AppearancePatentTaskService {
String aggregateScopeKey = buildAggregateScopeKey(sourceFiles);
String sourceScopeHash = DigestUtil.sha256Hex(aggregateScopeKey);
String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), request.getApiKey(), sourceFiles, mergedHeaders, allRows);
String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), request.getApiKey(), request.getPatentToken(), sourceFiles, mergedHeaders, allRows);
long payloadBuiltAt = System.nanoTime();
String parsedPayloadPointer = storeParsedPayload(task.getId(), sourceScopeHash, parsedPayload);
long payloadStoredAt = System.nanoTime();
task.setResultJson(buildTaskResultJson(request.getAiPrompt(), request.getApiKey(), sourceFiles, parsedPayloadPointer));
task.setResultJson(buildTaskResultJson(request.getAiPrompt(), request.getApiKey(), request.getPatentToken(), sourceFiles, parsedPayloadPointer));
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
@@ -984,10 +984,11 @@ public class AppearancePatentTaskService {
}
String prompt = readAiPrompt(task);
String apiKey = readApiKey(task);
String patentToken = readPatentToken(task);
int batchSize = Math.max(1, properties.getCozeBatchSize());
List<AppearancePatentResultRowDto> result = new ArrayList<>();
for (int i = 0; i < items.size(); i += batchSize) {
result.addAll(cozeClient.inspect(items.subList(i, Math.min(i + batchSize, items.size())), prompt, apiKey));
result.addAll(cozeClient.inspect(items.subList(i, Math.min(i + batchSize, items.size())), prompt, apiKey, patentToken));
if (progressHook != null) {
progressHook.run();
}
@@ -1287,6 +1288,14 @@ public class AppearancePatentTaskService {
}
}
private String readPatentToken(FileTaskEntity task) {
try {
return normalize(readParsedPayload(task).getPatentToken());
} catch (Exception ignored) {
return "";
}
}
private void finalizeTask(FileTaskEntity task, String error, int rowCount, boolean assembleWorkbook) {
String finalError = error;
FileResultEntity result = null;
@@ -1592,6 +1601,7 @@ public class AppearancePatentTaskService {
}
String prompt = readAiPrompt(task);
String apiKey = readApiKey(task);
String patentToken = readPatentToken(task);
int batchSize = Math.max(1, properties.getCozeBatchSize());
List<CozeCandidate> candidates = collectPendingCozeCandidates(task, chunks);
boolean flushRemainder = isResultSubmissionComplete(task.getId());
@@ -1612,7 +1622,7 @@ public class AppearancePatentTaskService {
List<AppearancePatentResultRowDto> batchRows = batchCandidates.stream()
.map(CozeCandidate::row)
.toList();
pending |= submitCozeBatch(task, result, job, batchRows, batchIndex, batchTotal, prompt, apiKey, allRowsByBaseId);
pending |= submitCozeBatch(task, result, job, batchRows, batchIndex, batchTotal, prompt, apiKey, patentToken, allRowsByBaseId);
batchIndex++;
}
return pending || countPendingCozeStates(task.getId()) > 0;
@@ -1626,6 +1636,7 @@ public class AppearancePatentTaskService {
int batchTotal,
String prompt,
String apiKey,
String patentToken,
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId) {
if (batchRows == null || batchRows.isEmpty()) {
return false;
@@ -1645,7 +1656,7 @@ public class AppearancePatentTaskService {
AppearancePatentCozeClient.CozeCredentialRef credential = cozeClient.nextCredential();
try {
AppearancePatentCozeClient.CozeSubmitResponse submit = submitCozeWorkflowThrottled(
batchRows, prompt, apiKey, credential, true);
batchRows, prompt, apiKey, patentToken, credential, true);
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
List<AppearancePatentResultRowDto> cozeRows = mergeUsableCozeRows(batchRows, submit.immediateData());
mergeCozeRowsIntoSubmittedChunks(task, cozeRows, allRowsByBaseId);
@@ -1945,7 +1956,7 @@ public class AppearancePatentTaskService {
}
try {
AppearancePatentCozeClient.CozeSubmitResponse submit =
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task),
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task), readPatentToken(task),
cozeClient.credentialByName(context.credentialName()), false);
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId = loadAllRowsByBaseId(task);
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
@@ -1996,6 +2007,7 @@ public class AppearancePatentTaskService {
List<AppearancePatentResultRowDto> rows,
String prompt,
String apiKey,
String patentToken,
AppearancePatentCozeClient.CozeCredentialRef credential,
boolean allowCredentialFallback) throws Exception {
int attempts = allowCredentialFallback ? Math.max(1, cozeClient.configuredCredentialCount()) : 1;
@@ -2018,7 +2030,7 @@ public class AppearancePatentTaskService {
continue;
}
try (lockHandle; borrowedCredential) {
return cozeClient.submitWorkflow(rows, prompt, apiKey, currentCredential);
return cozeClient.submitWorkflow(rows, prompt, apiKey, patentToken, currentCredential);
} finally {
sleepQuietly(COZE_SUBMIT_MIN_INTERVAL_MILLIS);
}
@@ -2088,7 +2100,7 @@ public class AppearancePatentTaskService {
int partIndex = 1;
for (List<AppearancePatentResultRowDto> partRows : partitions) {
AppearancePatentCozeClient.CozeSubmitResponse submit =
submitCozeWorkflowThrottled(partRows, readAiPrompt(task), readApiKey(task),
submitCozeWorkflowThrottled(partRows, readAiPrompt(task), readApiKey(task), readPatentToken(task),
cozeClient.credentialByName(context.credentialName()), false);
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
List<AppearancePatentResultRowDto> cozeRows =
@@ -2615,7 +2627,7 @@ public class AppearancePatentTaskService {
try {
taskFileJobService.touchRunning(context.jobId());
AppearancePatentCozeClient.CozeSubmitResponse submit =
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task),
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task), readPatentToken(task),
cozeClient.credentialByName(context.credentialName()), false);
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId = loadAllRowsByBaseId(task);
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
@@ -3814,10 +3826,11 @@ public class AppearancePatentTaskService {
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
}
private String buildParsedPayloadJson(String aiPrompt, String apiKey, List<AppearancePatentSourceFileDto> sourceFiles, List<String> headers, List<AppearancePatentParsedRowVo> allRows) {
private String buildParsedPayloadJson(String aiPrompt, String apiKey, String patentToken, List<AppearancePatentSourceFileDto> sourceFiles, List<String> headers, List<AppearancePatentParsedRowVo> allRows) {
AppearancePatentParsedPayloadDto payload = new AppearancePatentParsedPayloadDto();
payload.setAiPrompt(normalize(aiPrompt));
payload.setApiKey(normalize(apiKey));
payload.setPatentToken(normalize(patentToken));
payload.setSourceFiles(sourceFiles == null ? List.of() : sourceFiles);
payload.setHeaders(headers == null ? List.of() : headers);
payload.setItems(List.of());
@@ -3826,10 +3839,11 @@ public class AppearancePatentTaskService {
return writeJson(payload, "保存解析结果失败");
}
private String buildTaskResultJson(String aiPrompt, String apiKey, List<AppearancePatentSourceFileDto> sourceFiles, String parsedPayloadPointer) {
private String buildTaskResultJson(String aiPrompt, String apiKey, String patentToken, List<AppearancePatentSourceFileDto> sourceFiles, String parsedPayloadPointer) {
Map<String, Object> payload = new LinkedHashMap<>();
payload.put("aiPrompt", normalize(aiPrompt));
payload.put("apiKey", normalize(apiKey));
payload.put("patentToken", normalize(patentToken));
payload.put("sourceFileKeys", sourceFiles == null ? List.of() : sourceFiles.stream()
.map(AppearancePatentSourceFileDto::getFileKey)
.filter(Objects::nonNull)
@@ -3907,6 +3921,7 @@ public class AppearancePatentTaskService {
AppearancePatentParsedPayloadDto queuePayload = new AppearancePatentParsedPayloadDto();
queuePayload.setAiPrompt(payload.getAiPrompt());
queuePayload.setApiKey(payload.getApiKey());
queuePayload.setPatentToken(payload.getPatentToken());
queuePayload.setGroups(payload.getGroups() == null ? List.of() : payload.getGroups());
queuePayload.setItems(List.of());
queuePayload.setAllItems(List.of());
@@ -3932,6 +3947,7 @@ public class AppearancePatentTaskService {
AppearancePatentParsedGroupPageDto vo = new AppearancePatentParsedGroupPageDto();
vo.setAiPrompt(payload.getAiPrompt());
vo.setApiKey(payload.getApiKey());
vo.setPatentToken(payload.getPatentToken());
vo.setPage(safePage);
vo.setPageSize(safePageSize);
vo.setTotalGroups(totalGroups);