fix(release): AP 部分缺口出表、平台自费计次与工作台预签名 TTL

- AP 抓取缺行不再一律判死,按覆盖率判定;≥95% 出表成功并保留缺口提示
- 无归属代理提取记 0 号平台自费桶,后台可见并推管理员日告警
- 工作台安装包预签名从 180s 提到可配置 3600s,避免慢网上传在途过期
This commit is contained in:
2026-10-25 14:51:57 +08:00
parent 583dad92ec
commit fcb6b5f545
12 changed files with 305 additions and 35 deletions
@@ -212,7 +212,13 @@ public class AppearancePatentResultJobSupport {
}
}
}
boolean failed = finalError != null && !finalError.isBlank();
// 抓取缺行是“部分数据缺口”,不是致命失败:只要已有可用结果记录、且错误里没有
// 出表/任务级故障,就先进入组装;最终是否成功由 completeLlmFileJob 按覆盖率判定。
// 只有真正凑不出结果(result == null)时才维持 FAILED。
boolean hasError = finalError != null && !finalError.isBlank();
boolean partialGapSalvageable = hasError && isPartialDataGapError(finalError)
&& assembleWorkbook && result != null;
boolean failed = hasError && !partialGapSalvageable;
boolean waitingForAssemble = !failed && assembleWorkbook && !shouldAssembleSynchronously();
task.setStatus(statusResolver.resolve(waitingForAssemble, failed));
task.setSuccessFileCount(failed ? 0 : 1);
@@ -375,13 +381,16 @@ public class AppearancePatentResultJobSupport {
AppearancePatentRowOutcome.Facts deliveryFacts = assembleResultWorkbook(task, result);
progressSupport.saveFileBuildProgress(task, job, totalProgressUnits, totalProgressUnits - 1, "正在上传结果文件");
fileResultMapper.updateById(result);
boolean taskAlreadyFailed = STATUS_FAILED.equals(task.getStatus())
|| (task.getErrorMessage() != null && !task.getErrorMessage().isBlank());
String existingError = task.getErrorMessage();
task.setStatus(statusResolver.resolve(false, taskAlreadyFailed));
task.setSuccessFileCount(taskAlreadyFailed ? 0 : 1);
task.setFailedFileCount(taskAlreadyFailed ? 1 : 0);
task.setErrorMessage(deliverySupport.appendDeliveryGapMessage(taskAlreadyFailed ? existingError : null, deliveryFacts));
boolean taskAlreadyFailed = STATUS_FAILED.equals(task.getStatus());
boolean failed = shouldFailAfterAssembly(taskAlreadyFailed, existingError,
deliveryFacts.expectedRows(), deliveryFacts.coverage(),
properties.getLlmCoverageMinRatio());
task.setStatus(statusResolver.resolve(false, failed));
task.setSuccessFileCount(failed ? 0 : 1);
task.setFailedFileCount(failed ? 1 : 0);
// 成功时也保留缺口文案:用户要看到“N 行抓取失败”,但不能因此把整单判死、逼他重跑。
task.setErrorMessage(deliverySupport.appendDeliveryGapMessage(existingError, deliveryFacts));
task.setUpdatedAt(LocalDateTime.now());
task.setFinishedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
@@ -390,6 +399,46 @@ public class AppearancePatentResultJobSupport {
return true;
}
/**
* 出表后的最终状态判定:已判死 / 覆盖率低于下限 / 非部分缺口的错误 → FAILED。
*
* <p>抓取缺行只允许把缺口写进消息,不允许把已经拿到 95% 以上结论的任务整单判死——
* 那会让用户重跑,二次烧掉整单 LLM 密钥(现网 32288/32252 就是 3/2 行缺口烧掉整单)。
*/
static boolean shouldFailAfterAssembly(boolean taskAlreadyFailed, String existingError,
int expectedRows, double coverage, double minCoverage) {
if (taskAlreadyFailed) {
return true;
}
boolean hasError = existingError != null && !existingError.isBlank();
if (hasError && !isPartialDataGapError(existingError)) {
return true;
}
if (expectedRows <= 0) {
return hasError;
}
return coverage + 1e-9 < minCoverage;
}
/**
* 只认“少量行抓取/采集缺失”这类可带缺口交付的错误;出表失败、任务级故障仍按致命处理。
*/
static boolean isPartialDataGapError(String error) {
if (error == null || error.isBlank()) {
return false;
}
if (error.contains("结果文件生成失败")
|| error.contains("结果文件生成多次失败")
|| error.contains("Generate appearance patent result failed")
|| error.contains("任务不存在")
|| error.contains("任务已自动结束")) {
return false;
}
return error.contains("行抓取失败(未取得商品页数据)")
|| error.contains("行未采集到数据")
|| error.contains("行未完成检测");
}
/**
* 失败兜底:组装作业重试耗尽(WAIT_LLM 停放被 stuck 棘轮判死、或组装异常连吃 5 次重试)后调用,
* 用已回传的分片与已拿到结论的行尽力生成可下载的部分结果文件。
@@ -40,6 +40,9 @@ public class CostGuardScheduler {
static final String LOCK_NAME = "cost-guard-scan";
/** 无归属购买的汇总桶(见 UserSecretUsageService.PLATFORM_USER_ID):不暂停,只告警管理员。 */
static final long PLATFORM_USER_ID = UserSecretUsageService.PLATFORM_USER_ID;
private final CostGuardProperties properties;
private final UserSecretUsageMapper usageMapper;
private final UserTaskGuardService guardService;
@@ -77,7 +80,9 @@ public class CostGuardScheduler {
void runOnce() {
long nowMillis = System.currentTimeMillis();
int windowSeconds = Math.max(60, properties.getWindowSeconds());
Map<Long, UserTotals> current = loadTodayTotals();
Map<Long, UserTotals> all = loadAllTotals();
UserTotals platform = all.remove(PLATFORM_USER_ID);
Map<Long, UserTotals> current = all;
Map<Long, WindowDelta> deltas = new LinkedHashMap<>();
int paused = 0;
for (Map.Entry<Long, UserTotals> entry : current.entrySet()) {
@@ -123,11 +128,57 @@ public class CostGuardScheduler {
}
lastDeltas.clear();
lastDeltas.putAll(deltas);
scanPlatform(platform, nowMillis);
maybeRefreshShared(true);
log.info("[cost-guard] 扫描完成 用户={} 本轮暂停={} 共享密钥快照={}ms",
current.size(), paused, sharedSecretMonitorService.snapshotAtMillis());
}
/**
* 平台自费桶只告警、不暂停:无归属购买没有可暂停的账号,但金额是真实的,
* 超过告警线就推全体管理员,避免像 2026-10 那次一样在后台账上完全不可见
* (cookie 生成器一天买了 3900+ 次 3 分钟档 IP,页面上一个数字都没有)。
*
* <p>基线同样落共享 store,跨实例按真实 elapsed 归一化;同一天只报一次
* (crossed 只在「本轮跨过」时为真),告警内容的数字随当日累计更新。
*/
private void scanPlatform(UserTotals platform, long nowMillis) {
if (platform == null) {
return;
}
int threshold = properties.getPlatformProxyDailyThreshold();
if (threshold <= 0) {
return;
}
CostGuardBaselineStore.Baseline before = baselineStore.get(PLATFORM_USER_ID);
String reason = null;
// 跨天时当日累计会变小:此时上一天的基线(可能已高于告警线)不能拿来判 crossed,
// 否则今天首次越过告警线会被静默吞掉。
boolean newDay = before != null && platform.proxy < before.proxy;
if (before == null || newDay) {
if (platform.proxy >= threshold) {
reason = platformReason(platform.proxy, threshold);
}
} else if (crossed(before.proxy, platform.proxy, threshold)) {
reason = platformReason(platform.proxy, threshold);
}
baselineStore.put(PLATFORM_USER_ID, new CostGuardBaselineStore.Baseline(
platform.proxy, platform.llmCalls, platform.llmTokens, nowMillis));
if (reason == null) {
return;
}
int pushed = notificationDispatchService.pushToAdmins(NotificationService.SCENE_COST_GUARD,
NotificationService.LEVEL_WARNING, "平台自费代理提取超告警线",
reason + "。这笔消耗没有归属账号、不能自动暂停,请检查 cookie 生成器 / 平台兜底池的提取链接档位。",
"cost_guard_platform_proxy:" + LocalDate.now(), null);
log.warn("[cost-guard] 平台自费代理提取告警 次数={} 阈值={} 管理员落库={}",
platform.proxy, threshold, pushed);
}
private String platformReason(long count, int threshold) {
return "今日平台自费(无归属)代理提取累计 " + count + " 次,超过告警线 " + threshold + " 次";
}
/**
* 用户点击继续时调用:把基线对齐到当前值(而不是删掉基线)。删掉会让下一轮走「首轮」
* 分支;首轮现在会按日累计判暂停,用户刚点继续就会被立刻再暂停。
@@ -148,10 +199,17 @@ public class CostGuardScheduler {
}
public Map<Long, UserTotals> loadTodayTotals() {
Map<Long, UserTotals> totals = loadAllTotals();
totals.remove(PLATFORM_USER_ID);
return totals;
}
/** 含平台自费桶(0 号)的当日聚合;按账号暂停的那条路必须先摘掉它。 */
private Map<Long, UserTotals> loadAllTotals() {
List<UserSecretUsageAggregate> rows = usageMapper.sumByUserAndModule(LocalDate.now());
Map<Long, UserTotals> totals = new LinkedHashMap<>();
for (UserSecretUsageAggregate row : rows) {
if (row.getUserId() == null || row.getUserId() <= 0) {
if (row.getUserId() == null || row.getUserId() < 0) {
continue;
}
accumulate(totals.computeIfAbsent(row.getUserId(), key -> new UserTotals()), row);
@@ -42,6 +42,13 @@ public class UserSecretUsageService {
/** 单次上报上限:防止异常客户端把计数写爆。 */
private static final int MAX_REPORT_COUNT = 10_000;
/**
* 平台自费桶:无归属 uid 的代理提取(cookie 生成器、平台兜底池)记到 0 号,
* 后台用量统计显示为「平台自费」,损耗扫描只告警不暂停。只允许 proxy 走这个桶。
*/
public static final long PLATFORM_USER_ID = 0L;
static final String PLATFORM_USER_LABEL = "平台自费";
private final UserSecretUsageMapper usageMapper;
private final AdminUserMapper adminUserMapper;
private final AdminAuthSupport adminAuthSupport;
@@ -66,13 +73,18 @@ public class UserSecretUsageService {
/** 内部通道上报入口:模块非法返回 false(供接口回执),写库失败只告警。 */
public boolean recordReported(Long userId, String moduleKey, int count) {
if (userId == null || userId <= 0 || count <= 0) {
if (userId == null || userId < 0 || count <= 0) {
return false;
}
UserSecretModule module = UserSecretModule.of(moduleKey).orElse(null);
if (module == null) {
return false;
}
// 0 号只收代理:LLM 一定挂在具体账号的密钥上,平台没有"公共 LLM 密钥"这回事。
if (userId == PLATFORM_USER_ID && module != UserSecretModule.PROXY) {
log.warn("[user-secret-usage] 平台自费桶只收 proxy 计次 userId=0 module={}", moduleKey);
return false;
}
return recordModule(userId, module, Math.min(count, MAX_REPORT_COUNT), 0L, 0L);
}
@@ -270,31 +282,32 @@ public class UserSecretUsageService {
private void fillRowMeta(List<AdminUserSecretUsageRowVo> pageItems) {
List<Long> userIds = pageItems.stream()
.map(AdminUserSecretUsageRowVo::getUserId)
.filter(id -> id != null)
.filter(id -> id != null && id > PLATFORM_USER_ID)
.toList();
if (userIds.isEmpty()) {
return;
}
Map<Long, String> usernames = new HashMap<>();
for (AdminUserEntity user : adminUserMapper.selectBatchIds(userIds)) {
if (user.getId() != null) {
usernames.put(user.getId(), normalize(user.getUsername()));
}
}
Map<Long, List<String>> groups = new LinkedHashMap<>();
for (UserGroupRef ref : adminGroupMapper.selectGroupNamesByUserIds(userIds)) {
if (ref.getUserId() == null) {
continue;
if (!userIds.isEmpty()) {
for (AdminUserEntity user : adminUserMapper.selectBatchIds(userIds)) {
if (user.getId() != null) {
usernames.put(user.getId(), normalize(user.getUsername()));
}
}
String name = normalize(ref.getGroupName());
if (name.isEmpty()) {
continue;
for (UserGroupRef ref : adminGroupMapper.selectGroupNamesByUserIds(userIds)) {
if (ref.getUserId() == null) {
continue;
}
String name = normalize(ref.getGroupName());
if (name.isEmpty()) {
continue;
}
groups.computeIfAbsent(ref.getUserId(), key -> new ArrayList<>()).add(name);
}
groups.computeIfAbsent(ref.getUserId(), key -> new ArrayList<>()).add(name);
}
for (AdminUserSecretUsageRowVo item : pageItems) {
item.setUsername(usernames.getOrDefault(item.getUserId(), ""));
item.setGroups(groups.getOrDefault(item.getUserId(), List.of()));
boolean platform = item.getUserId() != null && item.getUserId() == PLATFORM_USER_ID;
item.setUsername(platform ? PLATFORM_USER_LABEL
: usernames.getOrDefault(item.getUserId(), ""));
item.setGroups(platform ? List.of() : groups.getOrDefault(item.getUserId(), List.of()));
}
}
@@ -7,6 +7,7 @@ import com.nanri.aiimage.modules.workbenchclient.mapper.WorkbenchClientPackageMa
import com.nanri.aiimage.modules.workbenchclient.model.entity.WorkbenchClientPackageEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.math.BigInteger;
@@ -44,8 +45,12 @@ public class WorkbenchClientPackageService {
/** 版本号长度上限,与 biz_workbench_client_package.version VARCHAR(64) 对齐。 */
private static final int MAX_VERSION_LENGTH = 64;
/** 直传预签名有效期:3 分钟足够数百 MB 级安装包上传。 */
private static final int PRESIGN_EXPIRY_SECONDS = 180;
/** 直传预签名默认有效期:90MB 级包在慢公网入口上传时,3 分钟窗口会在途过期。 */
private static final int DEFAULT_PRESIGN_EXPIRY_SECONDS = 3600;
/** 允许配置覆盖;上下限防止误配成 0 或过长窗口。 */
@Value("${aiimage.workbench-client.presign-expiry-seconds:3600}")
private int presignExpirySeconds = DEFAULT_PRESIGN_EXPIRY_SECONDS;
/** 公开版本列表上限:够客户端做「指定版本更新」下拉即可。 */
private static final int PUBLIC_LIST_LIMIT = 50;
@@ -118,6 +123,10 @@ public class WorkbenchClientPackageService {
return result;
}
private int effectivePresignExpirySeconds() {
return Math.max(60, Math.min(presignExpirySeconds, 24 * 3600));
}
/**
* 签发直传预签名:对象 key 由「毫秒时间戳 + 随机 UUID + 安全版本号」组成。
* UUID 是 CR-08 的修复点:只靠毫秒时间戳时,多实例时钟相同或同毫秒两名管理员
@@ -130,9 +139,10 @@ public class WorkbenchClientPackageService {
+ OBJECT_NAME_FORMATTER.format(LocalDateTime.now())
+ "-" + UUID.randomUUID().toString().replace("-", "")
+ "-" + safeVersionKey(normalizedVersion) + ".zip";
int expirySeconds = effectivePresignExpirySeconds();
log.info("[workbench-client] 签发客户端包直传预签名 version={} fileName={} objectKey={} expirySeconds={}",
normalizedVersion, normalizedName, objectKey, PRESIGN_EXPIRY_SECONDS);
String uploadUrl = ossStorageService.presignWorkbenchClientUpload(objectKey, PRESIGN_EXPIRY_SECONDS);
normalizedVersion, normalizedName, objectKey, expirySeconds);
String uploadUrl = ossStorageService.presignWorkbenchClientUpload(objectKey, expirySeconds);
Map<String, Object> result = new LinkedHashMap<>();
result.put("object_key", objectKey);
result.put("upload_url", uploadUrl);