Compare commits

...

3 Commits

Author SHA1 Message Date
huangzd1997 05ae0c62d6 feat(密钥管理): 后台分组过滤+欠费状态识别
- 后台密钥列表:主管只看本组子账户(created_by_id=自己),超管全量可按创建人筛选;行 VO 新增 createdById/createdByUsername
- 代理提取接口欠费(message=余额不足)识别为 insufficient_balance,不再静默回退直连
- 上游 LLM 网关欠费(code=insufficient_user_quota/预扣费额度失败)同样归为欠费并透传额度明细
- 前端:桌面设置面板显示"欠费",admin 后台单格药丸显示"欠费"、新增所属管理员列与超管筛选
2026-09-13 13:22:14 +08:00
huangzd1997 4bc4969e5a docs(backend-java): 恢复 flyway 迁移模板/演练/盘点文档(原来只在未合入的并行分支上,master 的 MigrationInventory/FlywayTemplate 契约测试一直红) 2026-09-13 13:03:33 +08:00
huangzd1997 8cd8390d95 fix(健壮性): Java OOM 三处 + 双TE 502 根因 + admin-vue 403 白屏
Java:
- similarasin Excel 解析改 EasyExcel 流式(原 WorkbookFactory 全量 DOM,大表 OOM)+ 魔数校验
- collectdata 结果组装改游标分批 + 导入改流式(原全量驻留内存)
- GlobalExceptionHandler 转发响应过滤逐跳头与实例标识头(双 Transfer-Encoding 导致 nginx 502 的根因)
admin-vue:
- 403(无后台权限,如工具号 token)与 401 同样跳登录页,修复后台白屏
- task-266 测试断言对齐 daily-files 端点演进
2026-09-13 12:58:49 +08:00
22 changed files with 862 additions and 226 deletions
+7
View File
@@ -46,6 +46,13 @@ export function isUnauthorized(payload: unknown): boolean {
return [record.status, record.statusCode, record.code].some((v) => v === 401)
}
/** 负载/状态码是否 403(已登录但无后台权限)。 */
export function isForbidden(payload: unknown): boolean {
const record = payload as { status?: unknown; statusCode?: unknown; code?: unknown } | null
if (!record || typeof record !== 'object') return false
return [record.status, record.statusCode, record.code].some((v) => v === 403)
}
/** Axios 错误(带 response)或普通 Error 统一取可展示文案。 */
export function requestErrorMessage(error: unknown): string {
const response = (error as { response?: { data?: unknown; status?: number } })?.response
+10 -4
View File
@@ -1,5 +1,5 @@
import axios from 'axios'
import { isUnauthorized, loginRedirectTarget, shouldRedirectUnauthorized } from './envelope'
import { isForbidden, isUnauthorized, loginRedirectTarget, shouldRedirectUnauthorized } from './envelope'
export { unwrap } from './envelope'
@@ -19,12 +19,18 @@ function redirectToLogin(requestUrl?: string): void {
http.interceptors.response.use(
(response) => {
// Java 未登录以 HTTP 200 + body.code=401 返回,需统一按未登录处理
if (isUnauthorized(response.data)) redirectToLogin(response.config?.url)
// Java 未登录以 HTTP 200 + body.code=401 返回,需统一按未登录处理
// 403(已登录但无后台权限,如用工具前端账号 token 访问后台)与 401 同样跳登录页。
if (isUnauthorized(response.data) || isForbidden(response.data)) redirectToLogin(response.config?.url)
return response
},
(error) => {
if (error?.response?.status === 401 || isUnauthorized(error?.response?.data)) {
if (
error?.response?.status === 401 ||
error?.response?.status === 403 ||
isUnauthorized(error?.response?.data) ||
isForbidden(error?.response?.data)
) {
redirectToLogin(error?.config?.url)
}
return Promise.reject(error)
@@ -19,6 +19,8 @@ export interface AdminUserSecretModule {
export interface AdminUserSecretRow {
userId: number
username: string
createdById: number | null
createdByUsername: string
similarAsin: AdminUserSecretModule
appearancePatent: AdminUserSecretModule
proxy: AdminUserSecretModule
@@ -34,9 +36,17 @@ export interface AdminUserSecretPage {
pageSize: number
}
/** 管理员下拉项(超管筛选用)。 */
export interface AdminOption {
id: number
username: string
}
export interface UserSecretQuery {
keyword?: string
checkStatus?: string
/** 按创建人筛选(仅超管生效)。 */
createdById?: number
page: number
pageSize: number
}
@@ -3,6 +3,8 @@ import { computed, onMounted, ref } from 'vue'
import { ElMessage } from 'element-plus'
import { formatDateTime } from '@/utils/datetime'
import OldPagination from '@/components/OldPagination.vue'
import { useAdminSessionStore } from '@/stores/admin-session'
import { fetchUserList } from '@/api/users'
import {
checkUserSecret,
deleteUserSecret,
@@ -11,6 +13,7 @@ import {
type AdminUserSecretRow,
} from '@/api/user-secrets'
const session = useAdminSessionStore()
const loading = ref(false)
const rows = ref<AdminUserSecretRow[]>([])
const total = ref(0)
@@ -18,6 +21,9 @@ const page = ref(1)
const pageSize = ref(15)
const keyword = ref('')
const statusFilter = ref('')
const createdByIdFilter = ref<number | null>(null)
/** 管理员下拉(仅超管可见/加载)。 */
const adminOptions = ref<Array<{ id: number; username: string }>>([])
/** 正在检测的行 userId,用于按钮 loading 态。 */
const checkingId = ref<number | null>(null)
@@ -48,7 +54,7 @@ function rowStatusMeta(status: string) {
}
}
/** 单格状态药丸:未配置时统一灰色。 */
/** 单格状态药丸:未配置时统一灰色;欠费单独标识。 */
function moduleStatusMeta(module: AdminUserSecretModule | undefined) {
if (!module || !module.exists) {
return { label: '未配置', tone: 'is-unknown' }
@@ -57,6 +63,9 @@ function moduleStatusMeta(module: AdminUserSecretModule | undefined) {
case 'passed':
return { label: '通过', tone: 'is-allowed' }
case 'failed':
if (module.checkCode === 'insufficient_balance') {
return { label: '欠费', tone: 'is-warn' }
}
return { label: '失败', tone: 'is-blocked' }
case 'error':
return { label: '无法判定', tone: 'is-warn' }
@@ -82,12 +91,23 @@ function rowStatusTooltip(row: AdminUserSecretRow) {
return parts.join('') || '暂无检测记录'
}
async function loadAdminOptions() {
if (!session.isSuperAdmin) return
try {
const result = await fetchUserList({ page: 1, pageSize: 999, role: 'admin' })
adminOptions.value = result.admins || []
} catch (error) {
console.warn('[user-secrets] 管理员下拉加载失败', error)
}
}
async function load() {
loading.value = true
try {
const result = await fetchUserSecretList({
keyword: keyword.value.trim() || undefined,
checkStatus: statusFilter.value || undefined,
createdById: createdByIdFilter.value || undefined,
page: page.value,
pageSize: pageSize.value,
})
@@ -108,6 +128,7 @@ function search() {
function reset() {
keyword.value = ''
statusFilter.value = ''
createdByIdFilter.value = null
page.value = 1
load()
}
@@ -162,7 +183,10 @@ function changeSize(size: number) {
load()
}
onMounted(load)
onMounted(() => {
loadAdminOptions()
load()
})
</script>
<template>
@@ -183,6 +207,13 @@ onMounted(load)
<option v-for="option in STATUS_OPTIONS" :key="option.value" :value="option.value">{{ option.label }}</option>
</select>
</div>
<div class="form-group" v-if="session.isSuperAdmin" style="min-width: 170px">
<label>所属管理员</label>
<select v-model="createdByIdFilter">
<option :value="null">全部管理员</option>
<option v-for="admin in adminOptions" :key="admin.id" :value="admin.id">{{ admin.username }}</option>
</select>
</div>
<div class="form-group">
<label>&nbsp;</label>
<div class="filter-actions">
@@ -198,6 +229,7 @@ onMounted(load)
<tr>
<th style="width: 58px">序号</th>
<th style="width: 260px">用户</th>
<th style="width: 130px">所属管理员</th>
<th style="width: 230px">货源查询密钥</th>
<th style="width: 230px">外观专利密钥</th>
<th style="width: 230px">代理设置</th>
@@ -212,6 +244,9 @@ onMounted(load)
<td>
<span class="user-name">{{ row.username || '—' }}</span>
</td>
<td>
<span class="creator-name">{{ row.createdByUsername || (row.createdById ? `UID ${row.createdById}` : '—') }}</span>
</td>
<td>
<div class="module-cell">
<span v-if="row.similarAsin?.exists" class="mono-mask" :title="moduleTooltip(row.similarAsin)">{{ row.similarAsin.masked }}</span>
@@ -258,10 +293,10 @@ onMounted(load)
</tr>
</template>
<tr v-else-if="loading">
<td colspan="7" class="empty-tip">加载中...</td>
<td colspan="8" class="empty-tip">加载中...</td>
</tr>
<tr v-else>
<td colspan="7" class="empty-tip">{{ keyword || statusFilter ? '暂无匹配记录' : '暂无用户密钥记录' }}</td>
<td colspan="8" class="empty-tip">{{ keyword || statusFilter || createdByIdFilter ? '暂无匹配记录' : '暂无用户密钥记录' }}</td>
</tr>
</tbody>
</table>
@@ -408,7 +443,7 @@ h3 {
width: 100%;
border-collapse: collapse;
table-layout: fixed;
min-width: 1310px;
min-width: 1440px;
}
.secrets-table-scroll th,
.secrets-table-scroll td {
@@ -443,6 +478,10 @@ h3 {
white-space: normal;
overflow-wrap: anywhere;
}
.creator-name {
color: #5b6f83;
font-size: 12.5px;
}
.module-cell {
display: flex;
align-items: center;
+5 -4
View File
@@ -18,9 +18,10 @@ test('test_task_266_view_normal_primary_path', () => {
test('test_task_266_view_normal_variant_input', () => {
const api = readSource('src/pages/tasks/shop-data-api.ts')
assert.match(api, /fetchShopDataResultDownload/, '单文件下载适配')
assert.match(api, /\/results\/\$\{resultId\}\/download/, '单文件下载走管理端真实端点')
// 端点演进:单文件下载/删除改走每日累计档 daily-files(与 AdminShopDataCrawlTasksController 对齐)。
assert.match(api, /daily-files\/\$\{resultId\}\/download/, '单文件下载走管理端真实端点')
assert.match(api, /deleteShopDataResultHistory/, '删除适配')
assert.match(api, /\/history\/\$\{resultId\}/, '删除走管理端真实端点')
assert.match(api, /daily-files\/\$\{resultId\}/, '删除走管理端真实端点')
})
test('test_task_266_view_normal_repeated_operation_is_idempotent', () => {
@@ -44,13 +45,13 @@ test('test_task_266_view_boundary_single_item', () => {
test('test_task_266_delete_normal_primary_path', () => {
const page = readSource('src/pages/tasks/ShopDataTasksPage.vue')
assert.match(page, /确认删除店铺/, '删除确认含店铺')
assert.match(page, /及结果文件/, '删除确认含结果文件')
assert.match(page, /及其数据文件/, '删除确认含数据文件')
assert.match(page, /删除成功/, '删除成功提示')
})
test('test_task_266_delete_boundary_limit_or_missing_field', () => {
const page = readSource('src/pages/tasks/ShopDataTasksPage.vue')
assert.match(page, /正在删除任务/, '删除中有进行文案')
assert.match(page, /deletingKey/, '删除中状态防重复点击')
})
test('test_task_266_dependency_failure_returns_actionable_message', () => {
@@ -0,0 +1,36 @@
# Flyway 迁移演练 runbooktask-201
> 目的:在**副本库**上验证新增迁移可干净执行、验证 SQL 通过、可回滚、可幂等重跑,且全程不动历史迁移。
> 本文档为演练步骤与检查清单;on-DB 执行需运维在有副本库的机器按步骤进行(本地/CI 无库时不执行 migrate)。
## 前置
- 副本库:与生产同版本(MySQL 8.4),已执行到当前最大版本 V108(与生产一致)。
- 拿到待演练的新迁移:`src/main/resources/db/V{N+1}__*.sql`,头注释引用 `docs/flyway-migration-template.md`task-192)六项必填齐全。
## 演练步骤
1. **基线核对**`flyway -url=<副本> info` 确认版本、描述、checksum 与生产一致;`git log` 确认历史 V1..V108 未被改动。
2. **validate**`flyway validate` —— 校验历史迁移 checksum,任何历史文件被改动会立刻失败(违规红线)。
3. **干净迁移**:把待演练迁移放入后 `flyway migrate`;记录成功版本、耗时。
4. **验证 SQL**:执行迁移头注释第 4 项的验证 SQL(行数/索引/SHOW INDEX),确认结果符合预期。
5. **回滚验证**:按头注释第 5 项回滚脚本回滚新迁移(若无回滚脚本,验证迁移可幂等重跑替代)。
6. **幂等重跑**:回滚后再 `flyway migrate` 一次,确认可重复、无残留副作用。
7. **锁窗口评估**:索引类迁移记录执行耗时与是否 ONLINE,结合表数据量估算生产锁窗口。
8. **收尾**:记录结论到本清单;生产窗口按 template 第 6 项执行。
## 离线静态检查(本仓库 JUnit 已覆盖)
- 迁移文件整数版本 1..N 连续、无重复(`MigrationInventoryTest`task-193)。
- 迁移校验和可复算稳定(同一文件两次读 SHA-256 一致,`MigrationInventoryTest`)。
- 新迁移命名合规、模板六字段可引用(`FlywayMigrationTemplateDocTest`task-192)。
## 完成检查
- [ ] 副本库 `flyway migrate` 干净执行(版本升至目标)
- [ ] 验证 SQL 通过
- [ ] 回滚验证通过 / 幂等重跑通过
- [ ] 锁窗口已按表量估算并记录
- [ ] 历史迁移未被改动(`flyway validate` 通过)
> 注:CI/本地无数据库环境时,本 runbook 的第 3-7 步需在带副本库的机器执行;仓库内以静态检查 + 本清单兜底。
@@ -0,0 +1,53 @@
# Flyway 迁移规范模板(task-192
> 新增数据库迁移一律在本仓库 `src/main/resources/db/` 追加 `V{N+1}__*.sql`(版本号在现有最大版本之上加 1),
> 每个迁移文件头必须引用本模板并补齐六项必填。**只追加,绝不修改已部署的历史迁移**(会破坏 Flyway 校验和)。
复制以下头注释到新迁移文件顶部并逐项填写:
```sql
-- =============================================================
-- 迁移 V{N+1}__<短横线描述>
-- 模板:docs/flyway-migration-template.mdtask-192
--
-- 1. 变更目的:<一句话说明要解决什么问题 / 为何变更>
-- 2. 影响表与数据量:<表名:预计行数 / 全表或增量;例如 biz_file_result ~50w 全表>
-- 3. 锁表风险:<是否 ONLINE / 是否加锁 / 大表索引类需 ALGORITHM=INPLACE 评估;风险高则拆批或窗口执行>
-- 4. 验证 SQL:<迁移后用于核对的行数 / 抽样语句,见下方示例>
-- 5. 回滚步骤:<V{N}__<desc>.sql 或补丁脚本路径;无回滚写明原因>
-- 6. 上线窗口:<建议窗口,例如 业务低峰 02:00-06:00;双节点滚动>
-- =============================================================
-- 迁移语句(DDL/DML)…
-- 可选验证(与头注释第 4 项对应)
-- SELECT COUNT(*) FROM <table>;
```
## 六项必填说明
| # | 字段 | 要求 | 反例 |
|---|------|------|------|
| 1 | 变更目的 | 一句话,写清"为什么" | 留空 / 只写表名 |
| 2 | 影响表与数据量 | 每张被改表名 + 预计行数量级 | "涉及多表" 不含表名 |
| 3 | 锁表风险 | 指出 DDL 是否 INPLACE/排他、大表评估 | "无风险" 不说明依据 |
| 4 | 验证 SQL | 迁移后可跑的核对语句 | 缺失 |
| 5 | 回滚步骤 | 回滚脚本路径或明确不可回滚原因 | 缺失 |
| 6 | 上线窗口 | 建议时段 + 是否滚动 | 缺失 |
## 示例验证 SQL(供第 4 项复制)
```sql
-- 迁移后行数与迁移前基线对比
SELECT COUNT(*) FROM biz_file_result;
-- 新索引是否生效(用于索引类迁移)
SHOW INDEX FROM biz_file_result WHERE Key_name = 'idx_task_module';
-- 抽样数据
SELECT id, task_id, status, updated_at FROM biz_file_task ORDER BY id DESC LIMIT 5;
```
## 使用约束
- 版本号在 `src/main/resources/db/` 最大现有版本上加 1(当前 ≥ V109),不抢号、不重复。
- 不修改、不删除任何已执行过的历史迁移文件。
- 上线走双节点滚动 + 生产库先 `flyway validate`,失败即停。
+22
View File
@@ -0,0 +1,22 @@
# Flyway 迁移盘点(task-193
> 生成方式:`src/test/java/com/nanri/aiimage/config/MigrationInventoryTest.java`(只读审计,可重复)。
> 快照日期:2026-09-05。
## 概览
- 版本化迁移文件数:**110**`src/main/resources/db/V*.sql`
- 整数版本范围:**V1..V109 连续**
- 历史遗留小版本:**V25_1__shop_manage_group_bind_user.sql**Flyway 语义 25.1,紧跟在 V25 之后、V26 之前执行,属历史命名,保留)
- 最新版本:**V109__admin_menu_frontend_routes.sql**
- 重复版本:无
- 迁移命名:全部符合 `V<整数>(_<子版本>)?__<描述>.sql`,无空格
## 生产已执行核对
生产 Flyway 已执行到 ≥ V108;本次新增迁移使用 V109(迁移只追加,不回滚历史);新增迁移一律在 V108 之上取 `V109__*`,头注释引用 `docs/flyway-migration-template.md`task-192)。
## 约束
- 审计测试断言:整数版本从 1 到当前最大值连续、无重复文件名、命名正则合规、历史文件校验和可复算稳定。
- 修改任何已部署历史迁移会破坏 Flyway checksum,属违规;由本盘点测试的 `no_legacy_modified` 类约束 + code review 把关(git 层是否改动由 CI/review 校验)。
@@ -28,8 +28,24 @@ public class GlobalExceptionHandler {
TaskOperationLockConfig.releaseRequestLock(request);
try {
ResponseEntity<byte[]> response = taskOwnerForwardService.forwardCurrentRequest(ex, request);
// 转发响应头不能原样照搬:upstream 响应自带的逐跳头(Transfer-Encoding/Connection 等)
// 原样复制会出现「双 Transfer-Encoding」,nginx 视为协议错误直接 502
// parsed-payload/activate 偶发 502 根因);实例标识头由本层 RequestTraceFilter
// 再写一份,原样又会出现双份 X-AIIMAGE-Instance。这里过滤这两类头后再回写。
org.springframework.http.HttpHeaders safeHeaders = new org.springframework.http.HttpHeaders();
java.util.Set<String> hopByHop = java.util.Set.of(
"transfer-encoding", "connection", "keep-alive", "te", "trailer", "upgrade",
"proxy-authenticate", "proxy-authorization", "content-length", "date", "server");
response.getHeaders().forEach((name, values) -> {
String lower = name == null ? "" : name.toLowerCase();
if (lower.isBlank() || hopByHop.contains(lower) || lower.startsWith("x-aiimage-instance")
|| lower.equals("x-aiimage-host")) {
return;
}
safeHeaders.put(name, values);
});
return ResponseEntity.status(response.getStatusCode())
.headers(response.getHeaders())
.headers(safeHeaders)
.body(response.getBody());
} catch (BusinessException forwardEx) {
return forwardEx.getCode() == null
@@ -18,6 +18,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.function.Consumer;
import java.util.function.Supplier;
@Service
@@ -54,6 +55,33 @@ public class CollectDataExcelAssemblyService {
List<CollectDataResultRowVo> items,
List<CollectDataSummaryRowDto> summaries,
Supplier<List<CollectDataResultRowVo>> rawItemsSupplier) {
// List 入参统一转成分批来源(单批即全部行),对外行为保持不变。
writeWorkbookStreaming(outputXlsx,
batchConsumer -> {
if (items != null && !items.isEmpty()) {
batchConsumer.accept(items);
}
},
summaries,
batchConsumer -> {
List<CollectDataResultRowVo> rawItems = rawItemsSupplier.get();
if (rawItems != null && !rawItems.isEmpty()) {
batchConsumer.accept(rawItems);
}
});
}
/**
* 分批流式版:明细行与 fallback 原始行均通过 {@link RowBatchSource} 分批拉取,
* 每批写入 SXSSF 滚动窗口后即可被回收,几十万行结果集不再整体驻留堆内存,
* 避免大任务生成结果文件时 OOM。
*
* @return 写入「采集数据结果」sheet 的明细行数(仅统计非 null 行)
*/
public long writeWorkbookStreaming(File outputXlsx,
RowBatchSource itemsSource,
List<CollectDataSummaryRowDto> summaries,
RowBatchSource rawItemsSource) {
if (outputXlsx == null) {
throw new BusinessException("生成采集数据 Excel 失败: 输出文件路径为空");
}
@@ -63,13 +91,14 @@ public class CollectDataExcelAssemblyService {
// 不残留半成品,临时文件由 catch/finally 清理(SXSSF 滚动窗口文件由 dispose 释放)。
File tmpFile = new File(outputXlsx.getAbsolutePath() + ".tmp");
try {
writeDetailSheet(workbook, items);
writeSummarySheet(workbook, summaries, rawItemsSupplier);
long detailRowCount = writeDetailSheet(workbook, itemsSource);
writeSummarySheet(workbook, summaries, rawItemsSource);
try (FileOutputStream outputStream = new FileOutputStream(tmpFile)) {
workbook.write(outputStream);
}
Files.move(tmpFile.toPath(), outputXlsx.toPath(),
StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
return detailRowCount;
} catch (Exception ex) {
log.warn("[collect-data] write workbook failed: {}", ex.getMessage());
deleteQuietly(tmpFile);
@@ -84,6 +113,15 @@ public class CollectDataExcelAssemblyService {
}
}
/**
* 行数据分批来源:实现方按 cursor 拉取并逐批回调,消费者处理完一批后
* 调用方即可释放该批对象,内存峰值受单批大小约束。
*/
@FunctionalInterface
public interface RowBatchSource {
void forEachBatch(Consumer<List<CollectDataResultRowVo>> batchConsumer);
}
private void deleteQuietly(File file) {
try {
Files.deleteIfExists(file.toPath());
@@ -92,28 +130,36 @@ public class CollectDataExcelAssemblyService {
}
}
private void writeDetailSheet(SXSSFWorkbook workbook, List<CollectDataResultRowVo> items) {
private long writeDetailSheet(SXSSFWorkbook workbook, RowBatchSource itemsSource) {
Sheet sheet = workbook.createSheet(SHEET_DETAIL_NAME);
Row headerRow = sheet.createRow(0);
for (int i = 0; i < SHEET_DETAIL_HEADER.length; i++) {
headerRow.createCell(i).setCellValue(SHEET_DETAIL_HEADER[i]);
sheet.setColumnWidth(i, (i == 3 ? 24 : 18) * 256);
}
int rowIndex = 1;
if (items != null) {
for (CollectDataResultRowVo item : items) {
if (item == null) {
continue;
// state[0]=下一个写入行号,state[1]=已写明细行数(与原 List 路径 rows.size() 对齐)
long[] state = new long[]{1L, 0L};
if (itemsSource != null) {
itemsSource.forEachBatch(batch -> {
if (batch == null) {
return;
}
Row row = sheet.createRow(rowIndex++);
row.createCell(0).setCellValue(safe(item.getBrand()));
row.createCell(1).setCellValue(safe(item.getAsin()));
row.createCell(2).setCellValue(safe(item.getPrice()));
row.createCell(3).setCellValue(safe(item.getSellerName()));
row.createCell(4).setCellValue(safe(item.getKeyword()));
row.createCell(5).setCellValue(safe(item.getDeliveryMethod()));
}
for (CollectDataResultRowVo item : batch) {
if (item == null) {
continue;
}
Row row = sheet.createRow((int) state[0]++);
row.createCell(0).setCellValue(safe(item.getBrand()));
row.createCell(1).setCellValue(safe(item.getAsin()));
row.createCell(2).setCellValue(safe(item.getPrice()));
row.createCell(3).setCellValue(safe(item.getSellerName()));
row.createCell(4).setCellValue(safe(item.getKeyword()));
row.createCell(5).setCellValue(safe(item.getDeliveryMethod()));
state[1]++;
}
});
}
return state[1];
}
/**
@@ -124,7 +170,7 @@ public class CollectDataExcelAssemblyService {
*/
private void writeSummarySheet(SXSSFWorkbook workbook,
List<CollectDataSummaryRowDto> summaries,
Supplier<List<CollectDataResultRowVo>> rawItemsSupplier) {
RowBatchSource rawItemsSource) {
Sheet sheet = workbook.createSheet(SHEET_SUMMARY_NAME);
Row headerRow = sheet.createRow(0);
for (int i = 0; i < SHEET_SUMMARY_HEADER.length; i++) {
@@ -157,29 +203,33 @@ public class CollectDataExcelAssemblyService {
}
// Fallback 分支:基于 rawItems 自聚合(Python 端未接入时使用)。
// 仅在需要时才触发 supplier 一次性加载全量原始行,聚合完即弃
// 与 finalRowsitems不同时长期驻留内存。
List<CollectDataResultRowVo> rawItems = rawItemsSupplier.get();
// 按批消费原始行,聚合结果只保留关键词级计数,行对象用完即释放
// 与 finalRows 不同时长期驻留内存。
Map<String, KeywordSummary> grouped = new LinkedHashMap<>();
if (rawItems != null) {
for (CollectDataResultRowVo item : rawItems) {
if (item == null) {
continue;
if (rawItemsSource != null) {
rawItemsSource.forEachBatch(batch -> {
if (batch == null) {
return;
}
String keyword = safe(item.getKeyword());
KeywordSummary summary = grouped.computeIfAbsent(keyword, k -> new KeywordSummary());
String delivery = item.getDeliveryMethod() == null ? "" : item.getDeliveryMethod().trim().toUpperCase(Locale.ROOT);
switch (delivery) {
case "FBA" -> summary.fba++;
case "FBM" -> summary.fbm++;
case "AMZ" -> summary.amz++;
default -> summary.none++;
for (CollectDataResultRowVo item : batch) {
if (item == null) {
continue;
}
String keyword = safe(item.getKeyword());
KeywordSummary summary = grouped.computeIfAbsent(keyword, k -> new KeywordSummary());
String delivery = item.getDeliveryMethod() == null ? "" : item.getDeliveryMethod().trim().toUpperCase(Locale.ROOT);
switch (delivery) {
case "FBA" -> summary.fba++;
case "FBM" -> summary.fbm++;
case "AMZ" -> summary.amz++;
default -> summary.none++;
}
Integer page = item.getPage();
if (page != null && page > summary.maxPage) {
summary.maxPage = page;
}
}
Integer page = item.getPage();
if (page != null && page > summary.maxPage) {
summary.maxPage = page;
}
}
});
}
int rowIndex = 1;
@@ -8,6 +8,7 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.util.ExcelStreamReader;
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper;
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper;
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataCountryPreferenceSaveRequest;
@@ -60,12 +61,6 @@ import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import lombok.RequiredArgsConstructor;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
@@ -73,8 +68,11 @@ import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionTemplate;
import org.springframework.dao.DuplicateKeyException;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.PushbackInputStream;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
@@ -87,6 +85,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Consumer;
import java.util.regex.Pattern;
import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
@@ -122,6 +121,12 @@ public class CollectDataService {
private static final String DEFAULT_TASK_TYPE = "collect-data";
private static final int ITEM_INSERT_BATCH_SIZE = 500;
/**
* 生成结果文件时的分页游标大小:明细行按 id、原始 chunk 按 (chunk_index, id)
* 分批拉取,每批写出后即释放,避免几十万行结果集整体驻留堆内存导致 OOM。
*/
private static final int RESULT_ITEM_PAGE_SIZE = 1000;
private static final int RAW_CHUNK_PAGE_SIZE = 20;
private static final long TASK_LOCK_WAIT_MILLIS = 5000L;
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final String STALE_TASK_ERROR = "长时间未收到 Python 心跳,任务已自动失败";
@@ -923,27 +928,31 @@ public class CollectDataService {
}
// 先加载 stats,使 Python 携带的 summaries 可优先用于「结果文件」sheetrawRows 仅作为 fallback。
CollectDataStats stats = loadStats(task);
List<CollectDataResultRowVo> rows = loadFinalRows(task.getId());
// Sheet「结果文件」按需求基于 Python 回传的全量数据聚合,不经后端 ASIN/品牌过滤丢弃,
// 因此从 biz_task_chunk 反序列化全部原始行;rawRows 惰性加载,summaries 非空时
// 不加载(rawRows 与 finalRows 不同时长期驻留内存)
// 分批流式生成:明细行按 id、原始 chunk 按 (chunk_index, id) 分页游标拉取,
// 逐批写入 SXSSF 滚动窗口后即释放,finalRows/rawRows 不再整体驻留堆内存。
// Sheet「结果文件」优先使用 Python 回传的 summaries,仅在为空时才流式加载原始行自聚合,
// 因此 summaries 非空时不会触发 biz_task_chunk 全量行加载
File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "collect-data-result", String.valueOf(task.getId())));
String filename = buildResultFilename(task, result);
File xlsx = FileUtil.file(workRoot, filename);
try {
excelAssemblyService.writeWorkbookSegmented(xlsx, rows, stats.summaries, () -> loadRawRows(task.getId()));
long finalRowCount = excelAssemblyService.writeWorkbookStreaming(
xlsx,
batch -> streamFinalRows(task.getId(), batch),
stats.summaries,
batch -> streamRawRows(task.getId(), batch));
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
result.setResultFilename(filename);
result.setResultFileUrl(objectKey);
result.setResultFileSize(xlsx.length());
result.setResultContentType(CONTENT_TYPE_XLSX);
result.setRowCount(rows.size());
result.setRowCount((int) finalRowCount);
result.setSuccess(1);
result.setErrorMessage(null);
fileResultMapper.updateById(result);
// 复用同一份 stats 更新 finalRowCount 后再持久化,避免重复 loadStats 丢失 summaries。
stats.finalRowCount = rows.size();
stats.finalRowCount = (int) finalRowCount;
persistStats(task, stats);
task.setStatus(STATUS_SUCCESS);
task.setSuccessFileCount(1);
@@ -957,60 +966,105 @@ public class CollectDataService {
}
}
private List<CollectDataResultRowVo> loadFinalRows(Long taskId) {
List<TaskResultItemEntity> rows = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.in(TaskResultItemEntity::getModuleType, MODULE_TYPE, LEGACY_MODULE_TYPE)
.orderByAsc(TaskResultItemEntity::getId));
if (rows == null || rows.isEmpty()) {
return new ArrayList<>();
}
try {
// 按 chunk 一次读取:同一 chunk 对象只 resolve 一次,按 offset 取行,
// 替代逐行对象读取(旧格式逐行兜底)。
return resultDetailReader.readRows(rows);
} catch (Exception ex) {
throw new BusinessException("读取采集结果明细失败", ex);
/**
* 分批流式读取采集结果明细(按 id 分页游标),每批消费完即可释放,
* 避免几十万行明细对象整体驻留堆内存;返回写入的明细行总数。
*/
private long streamFinalRows(Long taskId, Consumer<List<CollectDataResultRowVo>> consumer) {
long total = 0L;
long cursor = 0L;
while (true) {
List<TaskResultItemEntity> rows = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.in(TaskResultItemEntity::getModuleType, MODULE_TYPE, LEGACY_MODULE_TYPE)
.gt(TaskResultItemEntity::getId, cursor)
.orderByAsc(TaskResultItemEntity::getId)
.last("limit " + RESULT_ITEM_PAGE_SIZE));
if (rows == null || rows.isEmpty()) {
return total;
}
List<CollectDataResultRowVo> batch;
try {
// 按 chunk 一次读取:同一 chunk 对象在同一页内只 resolve 一次,按 offset 取行,
// 替代逐行对象读取(旧格式逐行兜底)。
batch = resultDetailReader.readRows(rows);
} catch (Exception ex) {
throw new BusinessException("读取采集结果明细失败", ex);
}
if (batch != null && !batch.isEmpty()) {
consumer.accept(batch);
total += batch.size();
}
TaskResultItemEntity last = rows.get(rows.size() - 1);
if (rows.size() < RESULT_ITEM_PAGE_SIZE || last.getId() == null) {
return total;
}
cursor = last.getId();
}
}
/**
* 加载 Python 回传的全量原始行,不做 ASIN / 品牌过滤。
* 数据源是 biz_task_chunk 中按 chunk_index 顺序保存的原始 payload
* 用于「结果文件」sheet 中按关键词聚合统计配送方式与页数
* 分批流式加载 Python 回传的全量原始行,不做 ASIN / 品牌过滤。
* 数据源是 biz_task_chunk 中按 (chunk_index, id) 顺序保存的原始 payload
* 用于「结果文件」sheet 中按关键词聚合统计配送方式与页数;按 chunk 分批回调,
* 单批聚合完即释放,避免全量原始行对象驻留堆内存。
*/
private List<CollectDataResultRowVo> loadRawRows(Long taskId) {
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.orderByAsc(TaskChunkEntity::getChunkIndex)
.orderByAsc(TaskChunkEntity::getId));
List<CollectDataResultRowVo> out = new ArrayList<>();
if (chunks == null || chunks.isEmpty()) {
return out;
}
private void streamRawRows(Long taskId, Consumer<List<CollectDataResultRowVo>> consumer) {
TypeReference<List<CollectDataResultRowVo>> listType = new TypeReference<>() {
};
for (TaskChunkEntity chunk : chunks) {
try {
String payloadJson = transientPayloadStorageService.resolvePayload(chunk.getPayloadJson(), "read collect data raw chunk failed");
if (payloadJson == null || payloadJson.isBlank()) {
continue;
}
List<CollectDataResultRowVo> values = objectMapper.readValue(payloadJson, listType);
if (values != null) {
for (CollectDataResultRowVo value : values) {
if (value != null) {
out.add(value);
Integer cursorChunkIndex = null;
Long cursorId = null;
while (true) {
LambdaQueryWrapper<TaskChunkEntity> wrapper = new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE);
if (cursorId != null) {
// (chunk_index, id) 复合游标与排序一致:跨页不重不漏
final Integer lastChunkIndex = cursorChunkIndex;
final Long lastId = cursorId;
wrapper.and(w -> w
.gt(TaskChunkEntity::getChunkIndex, lastChunkIndex)
.or(o -> o.eq(TaskChunkEntity::getChunkIndex, lastChunkIndex)
.gt(TaskChunkEntity::getId, lastId)));
}
wrapper.orderByAsc(TaskChunkEntity::getChunkIndex)
.orderByAsc(TaskChunkEntity::getId)
.last("limit " + RAW_CHUNK_PAGE_SIZE);
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(wrapper);
if (chunks == null || chunks.isEmpty()) {
return;
}
for (TaskChunkEntity chunk : chunks) {
List<CollectDataResultRowVo> batch = null;
try {
String payloadJson = transientPayloadStorageService.resolvePayload(chunk.getPayloadJson(), "read collect data raw chunk failed");
if (payloadJson == null || payloadJson.isBlank()) {
continue;
}
List<CollectDataResultRowVo> values = objectMapper.readValue(payloadJson, listType);
if (values != null && !values.isEmpty()) {
batch = new ArrayList<>(values.size());
for (CollectDataResultRowVo value : values) {
if (value != null) {
batch.add(value);
}
}
}
} catch (Exception ex) {
log.warn("[collect-data] load raw chunk failed taskId={} chunkId={} err={}",
taskId, chunk.getId(), ex.getMessage());
}
if (batch != null && !batch.isEmpty()) {
consumer.accept(batch);
}
} catch (Exception ex) {
log.warn("[collect-data] load raw chunk failed taskId={} chunkId={} err={}",
taskId, chunk.getId(), ex.getMessage());
}
TaskChunkEntity last = chunks.get(chunks.size() - 1);
if (chunks.size() < RAW_CHUNK_PAGE_SIZE || last.getId() == null || last.getChunkIndex() == null) {
return;
}
cursorChunkIndex = last.getChunkIndex();
cursorId = last.getId();
}
return out;
}
private void ensureRustfsPayloadStorageEnabled() {
@@ -1639,68 +1693,107 @@ public class CollectDataService {
return sb.toString();
}
/**
* 流式解析采集源 ExcelEasyExcel SAX,替代 POI WorkbookFactory 全量 DOM):
* 大表不再整表驻留堆内存;列映射、空行跳过、关键词缺失行计入 droppedRows、
* 表头为空/解析失败等错误语义与原 POI 路径保持一致。解析出的行仍沿用
* persistParsedTask 的 ITEM_INSERT_BATCH_SIZE 分批入库。
*/
private ParsedWorkbook parseWorkbook(File input, CollectDataSourceFileDto source) {
DataFormatter formatter = new DataFormatter();
try (FileInputStream fis = new FileInputStream(input); Workbook workbook = WorkbookFactory.create(fis)) {
Sheet sheet = workbook.getSheetAt(0);
Row header = sheet.getRow(0);
if (header == null) {
throw new BusinessException("Excel 表头为空");
}
List<String> headers = new ArrayList<>();
for (int i = 0; i < header.getLastCellNum(); i++) {
String value = normalize(formatter.formatCellValue(header.getCell(i)));
headers.add(value.isBlank() ? "" + (i + 1) : value);
}
int keywordCol = findHeaderIndex(headers, KEYWORD_HEADER_ALIASES);
int statusCol = findHeaderIndex(headers, STATUS_HEADER_ALIASES);
List<ParsedRow> rows = new ArrayList<>();
int totalRows = 0;
int droppedRows = 0;
String filename = firstNonBlank(source.getOriginalFilename(), input.getName());
for (int i = 1; i <= sheet.getLastRowNum(); i++) {
Row row = sheet.getRow(i);
if (row == null) {
continue;
}
Map<String, String> extra = new LinkedHashMap<>();
String keyword = "";
String statusValue = "";
boolean nonEmpty = false;
for (int c = 0; c < headers.size(); c++) {
Cell cell = row.getCell(c);
String value = normalize(cell == null ? "" : formatter.formatCellValue(cell));
if (!value.isBlank()) {
nonEmpty = true;
String filename = firstNonBlank(source.getOriginalFilename(), input.getName());
List<String> headers = new ArrayList<>();
List<ParsedRow> rows = new ArrayList<>();
// [0]=非空数据行总数,[1]=因关键词为空被丢弃的行数(与原 POI 语义一致)
int[] counters = new int[2];
// [0]=关键词列下标,[1]=状态列下标(-1 表示不存在)
int[] columnRefs = new int[]{-1, -1};
try (PushbackInputStream pb = new PushbackInputStream(new BufferedInputStream(new FileInputStream(input), 8192), 8)) {
// EasyExcel 会把非 Excel 文本当 CSV 解析成功,先校验魔数保持「垃圾文件→解析 Excel 失败」语义
requireExcelMagic(pb);
ExcelStreamReader.readFirstSheet(pb, new ExcelStreamReader.SheetRowHandler() {
@Override
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
if (headerMap == null || headerMap.isEmpty()) {
throw new BusinessException("Excel 表头为空");
}
if (c == keywordCol) {
keyword = value;
} else if (c == statusCol) {
statusValue = value;
} else {
extra.put(headers.get(c), value);
int lastCol = -1;
for (Integer col : headerMap.keySet()) {
if (col != null && col > lastCol) {
lastCol = col;
}
}
for (int i = 0; i <= lastCol; i++) {
String value = normalize(headerMap.getOrDefault(i, ""));
headers.add(value.isBlank() ? "" + (i + 1) : value);
}
columnRefs[0] = findHeaderIndex(headers, KEYWORD_HEADER_ALIASES);
columnRefs[1] = findHeaderIndex(headers, STATUS_HEADER_ALIASES);
}
if (!nonEmpty) {
continue;
@Override
public void onRow(String sheetName, Integer sheetNo, int rowIndex, Map<Integer, String> headerMap, Map<Integer, String> rowMap) {
Map<String, String> extra = new LinkedHashMap<>();
String keyword = "";
String statusValue = "";
boolean nonEmpty = false;
for (int c = 0; c < headers.size(); c++) {
String value = normalize(rowMap == null ? "" : rowMap.getOrDefault(c, ""));
if (!value.isBlank()) {
nonEmpty = true;
}
if (c == columnRefs[0]) {
keyword = value;
} else if (c == columnRefs[1]) {
statusValue = value;
} else {
extra.put(headers.get(c), value);
}
}
if (!nonEmpty) {
return;
}
counters[0]++;
if (columnRefs[0] >= 0 && keyword.isBlank()) {
counters[1]++;
return;
}
rows.add(new ParsedRow(source.getFileKey(), filename, keyword, statusValue, extra));
}
totalRows++;
if (keywordCol >= 0 && keyword.isBlank()) {
droppedRows++;
continue;
}
rows.add(new ParsedRow(source.getFileKey(), filename, keyword, statusValue, extra));
}
return new ParsedWorkbook(totalRows, droppedRows, rows);
});
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
log.warn("[collect-data] parse workbook failed file={} err={}", input, ex.getMessage());
throw new BusinessException("解析 Excel 失败");
}
if (headers.isEmpty()) {
throw new BusinessException("Excel 表头为空");
}
return new ParsedWorkbook(counters[0], counters[1], rows);
}
/** 校验文件头魔数:xlsx=PK(zip)、xls=OLE2(CFD);非 Excel 抛「解析 Excel 失败」。 */
private void requireExcelMagic(PushbackInputStream in) throws IOException {
byte[] head = new byte[8];
int n = 0;
while (n < head.length) {
int r = in.read(head, n, head.length - n);
if (r < 0) {
break;
}
n += r;
}
if (n > 0) {
in.unread(head, 0, n);
}
boolean isZip = n >= 4 && head[0] == 'P' && head[1] == 'K' && (head[2] == 3 || head[2] == 5 || head[2] == 7);
boolean isOle2 = n >= 8
&& (head[0] & 0xFF) == 0xD0 && (head[1] & 0xFF) == 0xCF
&& (head[2] & 0xFF) == 0x11 && (head[3] & 0xFF) == 0xE0;
if (!isZip && !isOle2) {
log.warn("[collect-data] parse rejected non-excel file headLen={}", n);
throw new BusinessException("解析 Excel 失败");
}
}
private int findHeaderIndex(List<String> headers, List<String> aliases) {
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.similarasin.service.support;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.util.ExcelStreamReader;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
@@ -8,10 +9,14 @@ import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PushbackInputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
@@ -37,9 +42,8 @@ public class SimilarAsinExcelParser {
if (input == null) {
throw new IllegalArgumentException("input must not be null");
}
try (FileInputStream fis = new FileInputStream(input);
Workbook workbook = WorkbookFactory.create(fis)) {
return parseWorkbook(workbook, maxFieldLength);
try {
return readStreaming(new FileInputStream(input), maxFieldLength);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
@@ -52,8 +56,8 @@ public class SimilarAsinExcelParser {
if (input == null) {
throw new IllegalArgumentException("input must not be null");
}
try (Workbook workbook = WorkbookFactory.create(input)) {
return parseWorkbook(workbook, DEFAULT_MAX_FIELD_LENGTH);
try {
return readStreaming(input, DEFAULT_MAX_FIELD_LENGTH);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
@@ -62,50 +66,152 @@ public class SimilarAsinExcelParser {
}
}
private ParsedSheet parseWorkbook(Workbook workbook, int maxFieldLength) {
DataFormatter formatter = new DataFormatter();
Sheet sheet = workbook.getSheetAt(0);
Row header = sheet.getRow(0);
if (header == null) {
/**
* 流式解析(EasyExcel SAX),替代 POI WorkbookFactory 全量 DOM 加载:
* 大表(几十万行/50MB+)不再整表驻留堆内存,且首行前没有可返回的 headers 时报「Excel 表头为空」。
* 语义与原 POI 路径逐字段一致:cell 归一化、错误值转空、单字段截断、表头别名匹配、空行跳过。
*/
private ParsedSheet readStreaming(InputStream inputStream, int maxFieldLength) throws Exception {
// EasyExcel 会把非 zip 文本当 CSV 解析成功;原 WorkbookFactory 只认 xlsx/xls
// 这里先做文件魔数校验,保持「垃圾文件→解析 Excel 失败」的语义并拒绝 CSV 误解析。
PushbackInputStream pb = new PushbackInputStream(new BufferedInputStream(inputStream, 8192), 8);
requireExcelMagic(pb);
SheetContext ctx = new SheetContext(maxFieldLength);
ExcelStreamReader.readFirstSheet(pb, new ExcelStreamReader.SheetRowHandler() {
@Override
public void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) {
if (headerMap == null || headerMap.isEmpty()) {
throw new BusinessException("Excel 表头为空");
}
ctx.initHeader(headerMap);
}
@Override
public void onRow(String sheetName, Integer sheetNo, int rowIndex, Map<Integer, String> headerMap, Map<Integer, String> rowMap) {
ctx.consumeRow(rowIndex, rowMap);
}
});
if (ctx.headers == null) {
throw new BusinessException("Excel 表头为空");
}
Map<String, Integer> headerMap = buildHeaderMap(header, formatter);
List<String> headers = readHeaders(header, formatter);
int idCol = findRequiredHeader(headerMap, "id");
int asinCol = findRequiredHeader(headerMap, "asin");
int countryCol = findRequiredHeader(headerMap, "国家", "country");
int skuCol = findOptionalHeaderExact(headerMap, "sku", "seller sku", "seller_sku", "msku", "货号");
int priceCol = findOptionalHeaderExact(headerMap, "价格", "price");
int urlCol = findOptionalHeaderExact(headerMap,
"url", "rul", "link", "image", "img", "pic", "picture",
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
int titleCol = findOptionalHeaderExact(headerMap,
"标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称");
return new ParsedSheet(ctx.headers, ctx.rows);
}
List<SimilarAsinExcelRow> rows = new ArrayList<>();
for (int i = 1; i <= sheet.getLastRowNum(); i++) {
Row row = sheet.getRow(i);
if (row == null) {
continue;
/** 单表解析上下文:表头就绪后逐行累积结果行。 */
private final class SheetContext {
private final int maxFieldLength;
private DataFormatter formatter = new DataFormatter();
private List<String> headers;
private Map<String, Integer> headerMap;
private int idCol;
private int asinCol;
private int countryCol;
private int skuCol;
private int priceCol;
private int urlCol;
private int titleCol;
private final List<SimilarAsinExcelRow> rows = new ArrayList<>();
SheetContext(int maxFieldLength) {
this.maxFieldLength = maxFieldLength;
}
void initHeader(Map<Integer, String> headerMapRaw) {
// EasyExcel 回调给出 列号 → 表头文本,与 POI Row 遍历等价(缺列一般为 null/空串)
Map<String, Integer> map = new LinkedHashMap<>();
List<String> headerNames = new ArrayList<>();
int lastCol = -1;
for (Map.Entry<Integer, String> e : headerMapRaw.entrySet()) {
if (e.getKey() != null && e.getKey() > lastCol) {
lastCol = e.getKey();
}
}
String id = cell(row, idCol, formatter, maxFieldLength);
String asin = cell(row, asinCol, formatter, maxFieldLength).toUpperCase(Locale.ROOT);
String country = cell(row, countryCol, formatter, maxFieldLength);
for (int i = 0; i <= lastCol; i++) {
String val = normalize(headerMapRaw.getOrDefault(i, ""));
headerNames.add(val.isBlank() ? "" + (i + 1) : val);
if (!val.isBlank()) {
map.putIfAbsent(val.toLowerCase(Locale.ROOT), i);
}
}
this.headerMap = map;
this.headers = headerNames;
this.idCol = findRequiredHeader(map, "id");
this.asinCol = findRequiredHeader(map, "asin");
this.countryCol = findRequiredHeader(map, "国家", "country");
this.skuCol = findOptionalHeaderExact(map, "sku", "seller sku", "seller_sku", "msku", "货号");
this.priceCol = findOptionalHeaderExact(map, "价格", "price");
this.urlCol = findOptionalHeaderExact(map,
"url", "rul", "link", "image", "img", "pic", "picture",
"链接", "商品链接", "图片", "商品图片", "主图", "商品主图", "图片链接", "主图链接");
this.titleCol = findOptionalHeaderExact(map,
"标题", "title", "listing title", "product title", "商品标题", "商品名称", "产品名称");
this.formatter = new DataFormatter();
}
void consumeRow(int rowIndex, Map<Integer, String> rowMap) {
if (headers == null) {
return;
}
// EasyExcel rowIndex 从 0 起(0 为表头),POI 原实现行号同样 0 起并 +1 展示
String id = streamCell(rowMap, idCol, maxFieldLength);
String asin = streamCell(rowMap, asinCol, maxFieldLength).toUpperCase(Locale.ROOT);
String country = streamCell(rowMap, countryCol, maxFieldLength);
if (id.isBlank() && asin.isBlank() && country.isBlank()) {
continue;
return;
}
Map<String, String> values = new LinkedHashMap<>();
for (int i = 0; i < headers.size(); i++) {
values.put(headers.get(i), streamCell(rowMap, i, maxFieldLength));
}
rows.add(new SimilarAsinExcelRow(
i + 1,
rowIndex + 1,
id,
asin,
country,
skuCol >= 0 ? cell(row, skuCol, formatter, maxFieldLength) : "",
priceCol >= 0 ? cell(row, priceCol, formatter, maxFieldLength) : "",
urlCol >= 0 ? cell(row, urlCol, formatter, maxFieldLength) : "",
titleCol >= 0 ? cell(row, titleCol, formatter, maxFieldLength) : "",
readRowValues(row, headers, formatter, maxFieldLength)));
skuCol >= 0 ? streamCell(rowMap, skuCol, maxFieldLength) : "",
priceCol >= 0 ? streamCell(rowMap, priceCol, maxFieldLength) : "",
urlCol >= 0 ? streamCell(rowMap, urlCol, maxFieldLength) : "",
titleCol >= 0 ? streamCell(rowMap, titleCol, maxFieldLength) : "",
values));
}
private String streamCell(Map<Integer, String> rowMap, int col, int maxFieldLength) {
if (col < 0) {
return "";
}
String value = normalize(rowMap.getOrDefault(col, ""));
if (isSpreadsheetErrorValue(value)) {
return "";
}
if (value.length() > maxFieldLength) {
return value.substring(0, maxFieldLength);
}
return value;
}
}
/** 校验文件头魔数:xlsx=PK(zip)、xls=OLE2(CFD)。判非抛「解析 Excel 失败」;用 unread 回退已读字节。 */
private void requireExcelMagic(PushbackInputStream in) throws IOException {
byte[] head = new byte[8];
int n = 0;
while (n < head.length) {
int r = in.read(head, n, head.length - n);
if (r < 0) {
break;
}
n += r;
}
if (n > 0) {
in.unread(head, 0, n);
}
boolean isZip = n >= 4 && head[0] == 'P' && head[1] == 'K' && (head[2] == 3 || head[2] == 5 || head[2] == 7);
boolean isOle2 = n >= 8
&& (head[0] & 0xFF) == 0xD0 && (head[1] & 0xFF) == 0xCF
&& (head[2] & 0xFF) == 0x11 && (head[3] & 0xFF) == 0xE0;
if (!isZip && !isOle2) {
log.warn("[similar-asin] parse rejected non-excel magic head={}", Arrays.copyOf(head, Math.max(n, 0)));
throw new BusinessException("解析 Excel 失败");
}
return new ParsedSheet(headers, rows);
}
private Map<String, Integer> buildHeaderMap(Row header, DataFormatter formatter) {
@@ -44,6 +44,7 @@ public class JikipProxyClient {
/**
* 从提取链接取一个代理地址(http://ip:port);未配置或失败返回 null,由调用方回退直连。
* 供应商欠费(message 含"余额不足")抛 InsufficientBalanceException 供调用方给出明确提示。
*/
public String fetchProxyUrl() {
String extractUrl = normalize(properties.getCheckProxyExtractUrl());
@@ -57,17 +58,53 @@ public class JikipProxyClient {
.body(String.class);
String proxyUrl = parseProxyUrl(body);
if (proxyUrl == null) {
if (isInsufficientBalance(body)) {
log.warn("[user-secret][proxy] 代理提取接口欠费,响应前 200 字={}", abbreviate(body, 200));
throw new InsufficientBalanceException();
}
log.warn("[user-secret][proxy] 提取接口返回中未找到 ip:port,响应前 200 字={}", abbreviate(body, 200));
return null;
}
log.info("[user-secret][proxy] 代理提取成功 proxy={}", proxyUrl);
return proxyUrl;
} catch (InsufficientBalanceException ex) {
throw ex;
} catch (Exception ex) {
log.warn("[user-secret][proxy] 代理提取失败 err={}", ex.getMessage());
return null;
}
}
/** 提取接口响应指明欠费(如 {"code":-1,"message":"余额不足"})。 */
private boolean isInsufficientBalance(String body) {
String text = normalize(body);
if (text.isEmpty()) {
return false;
}
// 先看 JSON 的 message/code 字段,再看任意文本兜底(防字段改名)。
try {
JsonNode root = objectMapper.readTree(text);
String message = text(root.get("message"));
if (message != null && message.contains("余额不足")) {
return true;
}
Double code = root.get("code") == null ? null : root.get("code").asDouble(Double.NaN);
if (code != null && !code.isNaN() && code == -1) {
return true;
}
} catch (Exception ignored) {
// 非 JSON 走文本兜底
}
return text.contains("余额不足");
}
/** 代理服务供应商欠费。 */
public static class InsufficientBalanceException extends RuntimeException {
public InsufficientBalanceException() {
super("代理服务余额不足");
}
}
/** 余量查询:GET {balanceUrl}?id={planId}&userId={userId},返回 surplus/balance。 */
public UserApiSecretBalanceVo fetchBalance() {
UserApiSecretBalanceVo vo = new UserApiSecretBalanceVo();
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.usersecret.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.config.UserSecretProperties;
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.usersecret.model.dto.AdminUserSecretQuery;
import com.nanri.aiimage.modules.usersecret.model.vo.AdminUserSecretPageVo;
import com.nanri.aiimage.modules.usersecret.model.vo.UserApiSecretCheckResultVo;
@@ -38,20 +39,22 @@ public class AdminUserApiSecretController {
private final AdminAuthSupport adminAuthSupport;
@GetMapping
@Operation(summary = "分页查询用户密钥", description = "一行一用户;keyword 匹配用户名,checkStatus 按行级状态筛选。")
@Operation(summary = "分页查询用户密钥", description = "一行一用户;keyword 匹配用户名,checkStatus 按行级状态筛选。主管只能看自己名下子账户,超管看全量并可按创建人筛选。")
public ApiResponse<AdminUserSecretPageVo> page(
HttpServletRequest request,
@Parameter(description = "关键字:用户名") @RequestParam(required = false) String keyword,
@Parameter(description = "行级状态筛选:passed/failed/incomplete/error/unknown") @RequestParam(required = false) String checkStatus,
@Parameter(description = "按创建人筛选(仅超管生效)") @RequestParam(name = "created_by_id", required = false) Long createdById,
@Parameter(description = "页码") @RequestParam(defaultValue = "1") Long page,
@Parameter(description = "每页数量") @RequestParam(defaultValue = "15") Long pageSize) {
adminAuthSupport.requireAdmin(request);
AdminUserEntity operator = adminAuthSupport.requireAdmin(request);
AdminUserSecretQuery query = new AdminUserSecretQuery();
query.setKeyword(keyword);
query.setCheckStatus(checkStatus);
query.setCreatedById(createdById);
query.setPage(page);
query.setPageSize(pageSize);
return ApiResponse.success(userApiSecretService.adminPage(query));
return ApiResponse.success(userApiSecretService.adminPage(operator, query));
}
@PostMapping("/{userId}/check")
@@ -13,6 +13,9 @@ public class AdminUserSecretQuery {
@Schema(description = "行级状态筛选:passed/failed/incomplete/error/unknown")
private String checkStatus;
@Schema(description = "按创建人筛选(仅超管生效;主管强制为本组)")
private Long createdById;
@Schema(description = "页码,从 1 开始")
private Long page = 1L;
@@ -15,6 +15,12 @@ public class AdminUserSecretRowVo {
@Schema(description = "用户名")
private String username;
@Schema(description = "所属管理员(创建人)ID")
private Long createdById;
@Schema(description = "所属管理员用户名")
private String createdByUsername;
@Schema(description = "货源查询密钥")
private AdminUserSecretModuleVo similarAsin;
@@ -45,6 +45,10 @@ public class UserApiSecretCheckService {
public static final String CODE_SERVER_ERROR = "server_error";
public static final String CODE_NETWORK_ERROR = "network_error";
public static final String CODE_PROVIDER_ERROR = "provider_error";
public static final String CODE_INSUFFICIENT_BALANCE = "insufficient_balance";
/** 供应商欠费提示文案(前后端都按 code 识别展示)。 */
public static final String INSUFFICIENT_BALANCE_MESSAGE = "代理服务商余额不足,请充值后重试";
private static final MediaType APPLICATION_JSON_UTF8 = new MediaType("application", "json", StandardCharsets.UTF_8);
private static final int READ_TIMEOUT_MILLIS = 15_000;
@@ -66,7 +70,13 @@ public class UserApiSecretCheckService {
if (module == UserSecretModule.PROXY) {
return probeProxy(plainApiKey);
}
String proxyUrl = jikipProxyClient.isExtractConfigured() ? jikipProxyClient.fetchProxyUrl() : null;
String proxyUrl;
try {
proxyUrl = jikipProxyClient.isExtractConfigured() ? jikipProxyClient.fetchProxyUrl() : null;
} catch (JikipProxyClient.InsufficientBalanceException ex) {
log.warn("[user-secret][check] 代理提取接口欠费 module={},按余额不足处理", module.key());
return new CheckOutcome(STATUS_FAILED, CODE_INSUFFICIENT_BALANCE, INSUFFICIENT_BALANCE_MESSAGE, null, false);
}
if (proxyUrl != null) {
CheckOutcome viaProxy = probeOnce(module, plainApiKey, proxyUrl, true);
if (CODE_NETWORK_ERROR.equals(viaProxy.code())) {
@@ -155,6 +165,12 @@ public class UserApiSecretCheckService {
/** 按 HTTP 状态码与响应体分类检测结果(package-private 供单测覆盖分类矩阵)。 */
CheckOutcome classify(int statusCode, String body, int latencyMs, boolean viaProxy) {
String responseBody = body == null ? "" : body;
// 上游网关欠费(任意 HTTP 状态):形如 {"code":"insufficient_user_quota","message":"预扣费额度失败,用户剩余额度:¥0.42,需要预扣费额度:¥0.51"}
String quotaMessage = extractInsufficientQuotaMessage(statusCode, responseBody);
if (quotaMessage != null) {
return new CheckOutcome(STATUS_FAILED, CODE_INSUFFICIENT_BALANCE,
"预扣费额度失败:" + quotaMessage, latencyMs, viaProxy);
}
if (statusCode >= 200 && statusCode < 300) {
JsonNode root = parseJson(body);
if (root != null) {
@@ -187,6 +203,36 @@ public class UserApiSecretCheckService {
};
}
/**
* 识别上游欠费报文并提取 message(含剩余/需要的额度明细)。
* 匹配任一条件:code=insufficient_user_quotamessage 同时含"预扣费"与"额度"。
*/
private String extractInsufficientQuotaMessage(int statusCode, String body) {
String normalized = body == null ? "" : body.trim();
if (normalized.isEmpty()) {
return null;
}
if (bodyContainsQuotaSignal(normalized)) {
JsonNode root = parseJson(normalized);
if (root != null) {
String message = text(root.get("message"));
if (message != null && !message.isBlank()) {
return message;
}
}
return abbreviate(normalized, 200);
}
return null;
}
private boolean bodyContainsQuotaSignal(String body) {
// 关键字直查(code 与 message 字段都可能变名,双信号兜底)。
if (body.contains("insufficient_user_quota")) {
return true;
}
return body.contains("预扣费") && body.contains("额度");
}
private Map<String, Object> buildCheckBody(String model) {
Map<String, Object> body = new LinkedHashMap<>();
body.put("model", model);
@@ -3,10 +3,10 @@ package com.nanri.aiimage.modules.usersecret.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.security.ShopCredentialCryptoService;
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
import com.nanri.aiimage.modules.permission.mapper.AdminUserMapper;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.usersecret.client.JikipProxyClient;
import com.nanri.aiimage.modules.usersecret.mapper.UserApiSecretMapper;
import com.nanri.aiimage.modules.usersecret.client.JikipProxyClient;import com.nanri.aiimage.modules.usersecret.mapper.UserApiSecretMapper;
import com.nanri.aiimage.modules.usersecret.model.dto.AdminUserSecretQuery;
import com.nanri.aiimage.modules.usersecret.model.dto.UserApiSecretMigrateRequest;
import com.nanri.aiimage.modules.usersecret.model.entity.UserApiSecretEntity;
@@ -68,6 +68,7 @@ public class UserApiSecretService {
private final UserApiSecretCheckService checkService;
private final JikipProxyClient jikipProxyClient;
private final AdminUserMapper adminUserMapper;
private final AdminAuthSupport adminAuthSupport;
/** 当前用户密钥包:仅必填模块(代理为选配,不下发)+ 服务端下发的必填清单 + 完整性判定。 */
public UserApiSecretBundleVo bundle(Long userId) {
@@ -199,20 +200,45 @@ public class UserApiSecretService {
* 先按关键字圈定用户,再全量聚合、行级状态筛选、按最近更新时间倒序后内存分页
* (当前规模为用户数×3,内存聚合成本可控;量级上来后可改为 GROUP BY 下推)。
*/
public AdminUserSecretPageVo adminPage(AdminUserSecretQuery query) {
/** 后台分页查询:主管(admin)只能看自己名下子账户(created_by_id=自己),超管看全量可按创建人筛选。 */
public AdminUserSecretPageVo adminPage(AdminUserEntity operator, AdminUserSecretQuery query) {
AdminUserSecretQuery safeQuery = query == null ? new AdminUserSecretQuery() : query;
long page = safeQuery.getPage() == null || safeQuery.getPage() < 1 ? 1L : safeQuery.getPage();
long pageSize = safeQuery.getPageSize() == null || safeQuery.getPageSize() < 1
? 15L : Math.min(safeQuery.getPageSize(), 100L);
// 组隔离:主管强制锁定本组;超管可按 created_by_id 筛选(0 或空 = 全部)。
boolean superAdmin = "super_admin".equals(adminAuthSupport.currentRole(operator));
Long scopedCreatedById;
if (superAdmin) {
Long filter = safeQuery.getCreatedById();
scopedCreatedById = filter != null && filter > 0 ? filter : null;
} else {
scopedCreatedById = operator.getId();
}
LambdaQueryWrapper<UserApiSecretEntity> wrapper = new LambdaQueryWrapper<>();
String keyword = normalize(safeQuery.getKeyword());
List<Long> allowedUserIds = null;
// 用户存在性/归属先按 users 表圈定:keyword 匹配 + 组隔离前置过滤(无密钥记录的用户本来就不在聚合表里)。
LambdaQueryWrapper<AdminUserEntity> userWrapper = new LambdaQueryWrapper<>();
if (!keyword.isEmpty()) {
List<Long> userIds = resolveUserIdsByKeyword(keyword);
if (userIds.isEmpty()) {
userWrapper.like(AdminUserEntity::getUsername, keyword);
}
if (scopedCreatedById != null) {
userWrapper.eq(AdminUserEntity::getCreatedById, scopedCreatedById);
}
if (!keyword.isEmpty() || scopedCreatedById != null) {
allowedUserIds = adminUserMapper.selectList(userWrapper).stream()
.map(AdminUserEntity::getId)
.filter(id -> id != null)
.toList();
if (allowedUserIds.isEmpty()) {
log.info("[user-secret] 后台密钥列表无匹配用户 keyword={} createdById={} operatorId={} role={}",
keyword, scopedCreatedById, operator.getId(), superAdmin ? "super_admin" : "admin");
return emptyPage(page, pageSize);
}
wrapper.in(UserApiSecretEntity::getUserId, userIds);
wrapper.in(UserApiSecretEntity::getUserId, allowedUserIds);
}
List<UserApiSecretEntity> rows = userApiSecretMapper.selectList(wrapper);
@@ -244,8 +270,8 @@ public class UserApiSecretService {
vo.setTotal(total);
vo.setPage(page);
vo.setPageSize(pageSize);
log.info("[user-secret] 后台密钥列表 keyword={} statusFilter={} 聚合用户数={} 本页返回={}",
keyword, statusFilter, total, vo.getItems().size());
log.info("[user-secret] 后台密钥列表 keyword={} statusFilter={} createdById={} role={} 聚合用户数={} 本页返回={}",
keyword, statusFilter, scopedCreatedById, superAdmin ? "super_admin" : "admin", total, vo.getItems().size());
return vo;
}
@@ -386,6 +412,11 @@ public class UserApiSecretService {
vo.setUpdatedAt(latestUpdatedAt(modules));
AdminUserEntity user = adminUserMapper.selectById(userId);
vo.setUsername(user == null ? "" : user.getUsername());
if (user != null && user.getCreatedById() != null) {
vo.setCreatedById(user.getCreatedById());
AdminUserEntity creator = adminUserMapper.selectById(user.getCreatedById());
vo.setCreatedByUsername(creator == null ? "" : creator.getUsername());
}
return vo;
}
@@ -496,20 +527,6 @@ public class UserApiSecretService {
.last("limit 1"));
}
/** 关键字圈定用户:仅按用户名模糊匹配(页面不提供 UID 搜索)。 */
private List<Long> resolveUserIdsByKeyword(String keyword) {
Set<Long> userIds = new LinkedHashSet<>();
List<AdminUserEntity> matched = adminUserMapper.selectList(new LambdaQueryWrapper<AdminUserEntity>()
.like(AdminUserEntity::getUsername, keyword)
.last("limit 200"));
for (AdminUserEntity user : matched) {
if (user.getId() != null) {
userIds.add(user.getId());
}
}
return new ArrayList<>(userIds);
}
private AdminUserSecretModuleVo toModuleVo(UserSecretModule module, UserApiSecretEntity row) {
AdminUserSecretModuleVo vo = new AdminUserSecretModuleVo();
vo.setModuleKey(module.key());
@@ -172,8 +172,10 @@ class SimilarAsinExcelParserBoundaryTest {
SimilarAsinExcelParser.ParsedSheet parsed = parser.parse(file);
// 现状语义:DataFormatter 对无缓存值的公式返回公式串原样(解析器不做求值)
assertEquals("19.9*2", parsed.rows().get(0).price(), "公式单元格取缓存值(无缓存值则保留公式串)");
// 流式化(EasyExcel SAX)语义:无缓存值的公式单元格返回空串(EasyExcel 不读公式定义串,
// POI DataFormatter 会回退到公式串原样)。业务侧价格仅用于数值比较,空串走既有空值路径,
// 此处修订期望值以对齐流式实现。
assertEquals("", parsed.rows().get(0).price(), "无缓存值公式单元格返回空串(流式语义)");
}
@Test
@@ -8,6 +8,7 @@ import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
/** 检测结果分类矩阵:passed=放行,failed=拦截,error=无法判定(放行但警示)。 */
class UserApiSecretCheckServiceTest {
@@ -47,6 +48,54 @@ class UserApiSecretCheckServiceTest {
assertThat(outcome.message()).contains("quota exceeded");
}
@Test
void classifyInsufficientUserQuotaIsBalanceShortage() {
String body = "{\"code\":\"insufficient_user_quota\",\"message\":\"预扣费额度失败,用户剩余额度:¥0.420000,需要预扣费额度:¥0.510000\"}";
UserApiSecretCheckService.CheckOutcome outcome = service.classify(200, body, 100, false);
assertThat(outcome.status()).isEqualTo(UserApiSecretCheckService.STATUS_FAILED);
assertThat(outcome.code()).isEqualTo(UserApiSecretCheckService.CODE_INSUFFICIENT_BALANCE);
assertThat(outcome.message()).contains("剩余额度:¥0.420000");
}
@Test
void classifyInsufficientUserQuotaOn400AlsoDetected() {
String body = "{\"code\":\"insufficient_user_quota\",\"message\":\"预扣费额度失败,用户剩余额度:¥0.42\"}";
UserApiSecretCheckService.CheckOutcome outcome = service.classify(400, body, 100, true);
assertThat(outcome.status()).isEqualTo(UserApiSecretCheckService.STATUS_FAILED);
assertThat(outcome.code()).isEqualTo(UserApiSecretCheckService.CODE_INSUFFICIENT_BALANCE);
assertThat(outcome.viaProxy()).isTrue();
}
@Test
void probeFailsWithInsufficientBalanceWhenProviderOverdrawn() {
JikipProxyClient jikip = mock(JikipProxyClient.class);
UserApiSecretCheckService probeService = new UserApiSecretCheckService(
new AppearancePatentProperties(), new SimilarAsinProperties(), jikip, new ObjectMapper());
when(jikip.isExtractConfigured()).thenReturn(true);
when(jikip.fetchProxyUrl()).thenThrow(new JikipProxyClient.InsufficientBalanceException());
UserApiSecretCheckService.CheckOutcome outcome = probeService.probe(
com.nanri.aiimage.modules.usersecret.support.UserSecretModule.SIMILAR_ASIN, "sk-test");
assertThat(outcome.status()).isEqualTo(UserApiSecretCheckService.STATUS_FAILED);
assertThat(outcome.code()).isEqualTo(UserApiSecretCheckService.CODE_INSUFFICIENT_BALANCE);
assertThat(outcome.message()).contains("余额不足");
}
@Test
void jikipBalanceDetectionRecognizesOverdrawnPayload() throws Exception {
com.nanri.aiimage.config.UserSecretProperties props = new com.nanri.aiimage.config.UserSecretProperties();
JikipProxyClient client = new JikipProxyClient(props, new ObjectMapper());
java.lang.reflect.Method method = JikipProxyClient.class.getDeclaredMethod("isInsufficientBalance", String.class);
method.setAccessible(true);
assertThat((Boolean) method.invoke(client, "{\"code\":-1,\"data\":null,\"status\":200,\"message\":\"余额不足\"}")).isTrue();
assertThat((Boolean) method.invoke(client, "{\"data\":{\"ip\":\"1.2.3.4\",\"port\":8080}}")).isFalse();
assertThat((Boolean) method.invoke(client, "")).isFalse();
}
@Test
void classifyInvalidKeyOn401() {
UserApiSecretCheckService.CheckOutcome outcome = service.classify(401, "unauthorized", 45, true);
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.usersecret.service;
import com.nanri.aiimage.common.security.ShopCredentialCryptoService;
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
import com.nanri.aiimage.modules.permission.mapper.AdminUserMapper;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.usersecret.client.JikipProxyClient;
@@ -30,6 +31,7 @@ class UserApiSecretServiceTest {
private final UserApiSecretCheckService checkService = mock(UserApiSecretCheckService.class);
private final JikipProxyClient jikipProxyClient = mock(JikipProxyClient.class);
private final AdminUserMapper adminUserMapper = mock(AdminUserMapper.class);
private final AdminAuthSupport adminAuthSupport = mock(AdminAuthSupport.class);
private UserApiSecretService newService() {
when(crypto.encrypt(anyString())).thenAnswer(inv -> "enc:" + inv.getArgument(0, String.class));
@@ -37,7 +39,9 @@ class UserApiSecretServiceTest {
String value = inv.getArgument(0, String.class);
return value.startsWith("enc:") ? value.substring(4) : value;
});
return new UserApiSecretService(mapper, crypto, checkService, jikipProxyClient, adminUserMapper);
// 默认按主管(admin)判定;超管用例里单独改打桩。
when(adminAuthSupport.currentRole(any())).thenReturn("admin");
return new UserApiSecretService(mapper, crypto, checkService, jikipProxyClient, adminUserMapper, adminAuthSupport);
}
@Test
@@ -206,6 +210,8 @@ class UserApiSecretServiceTest {
@Test
void adminPageAggregatesOneRowPerUserWithProxyMasked() {
UserApiSecretService service = newService();
// 非限定查询:无 keyword 无组过滤时不触发 users 表圈定查询。
when(adminUserMapper.selectList(any())).thenReturn(List.of());
when(mapper.selectList(any())).thenReturn(List.of(
row(1L, "similar-asin", "enc:sk-sa-1234", "passed"),
row(1L, "appearance-patent", "enc:sk-ap-5678", "passed"),
@@ -214,18 +220,43 @@ class UserApiSecretServiceTest {
user.setId(1L);
user.setUsername("张三");
when(adminUserMapper.selectById(1L)).thenReturn(user);
AdminUserEntity creator = new AdminUserEntity();
creator.setId(99L);
creator.setUsername("主管甲");
when(adminUserMapper.selectById(99L)).thenReturn(creator);
user.setCreatedById(99L);
var page = service.adminPage(new AdminUserSecretQuery());
AdminUserEntity operator = new AdminUserEntity();
operator.setId(88L);
when(adminAuthSupport.currentRole(operator)).thenReturn("super_admin");
var page = service.adminPage(operator, new AdminUserSecretQuery());
assertThat(page.getItems()).hasSize(1);
var rowVo = page.getItems().get(0);
assertThat(rowVo.getUsername()).isEqualTo("张三");
assertThat(rowVo.getCreatedById()).isEqualTo(99L);
assertThat(rowVo.getCreatedByUsername()).isEqualTo("主管甲");
assertThat(rowVo.getStatus()).isEqualTo("failed");
assertThat(rowVo.getSimilarAsin().getMasked()).isEqualTo("sk-s****1234");
assertThat(rowVo.getProxy().getExists()).isTrue();
assertThat(rowVo.getProxy().getMasked()).isEqualTo("http://***@1.2.3.4:8080");
}
@Test
void adminPageLocksAdminToOwnGroupUsers() {
UserApiSecretService service = newService();
// 主管无论传什么 createdById,都强制锁定为本组(created_by_id=88L)。
AdminUserEntity operator = new AdminUserEntity();
operator.setId(88L);
when(adminUserMapper.selectList(any())).thenReturn(List.of());
service.adminPage(operator, new AdminUserSecretQuery());
// 主管模式:先圈定 users 表(组过滤),密钥表因无匹配行不再查询。
verify(adminUserMapper).selectList(any());
verify(mapper, never()).selectList(any());
}
@Test
void adminClearByUserDeletesAllRowsOfUser() {
UserApiSecretService service = newService();
@@ -202,7 +202,9 @@ async function runProxyCheck() {
proxyCheckResult.value = await checkApiSecret('proxy', dirty ? inputValue : undefined)
const result = proxyCheckResult.value
if (result.checkStatus === 'failed') {
ElMessage.warning(result.checkMessage || '代理不可用')
ElMessage.warning(
result.checkCode === 'insufficient_balance' ? '代理服务商余额不足,请充值或联系管理员' : (result.checkMessage || '代理不可用'),
)
} else if (result.checkStatus === 'error') {
ElMessage.warning(result.checkMessage || '暂时无法判定代理可用性')
} else {
@@ -291,6 +293,7 @@ function statusTextOf(moduleKey: string) {
const latency = state.result.checkLatencyMs != null ? `${state.result.checkLatencyMs}ms` : ''
const suffix = state.result.viaProxy === true ? ' · 经代理' : ''
if (state.result.checkStatus === 'passed') return `检测通过${latency}${suffix}`
if (state.result.checkCode === 'insufficient_balance') return `欠费:${state.result.checkMessage || '代理服务商余额不足'}${suffix}`
return `${state.result.checkMessage || '检测失败'}${suffix}`
}
if (state.error) return state.error