Compare commits
36 Commits
d189d94c3b
..
master
| Author | SHA1 | Date | |
|---|---|---|---|
| 5b8105ec2b | |||
| 5fcb449946 | |||
| ff68426c69 | |||
| 46be044121 | |||
| e0303f9cba | |||
| 9d39705c77 | |||
| 986df86e89 | |||
| bd359411a9 | |||
| 3634ea1d62 | |||
| 3ce0569c59 | |||
| d2d95f0b71 | |||
| 2a51006888 | |||
| a89de129ea | |||
| ddefcbed56 | |||
| 3137299bfe | |||
| 1fe3368c5a | |||
| 7aea3a0a50 | |||
| 188aedec84 | |||
| d5952945dd | |||
| 6e1689dfe5 | |||
| 4f16a02658 | |||
| 5e9a59b327 | |||
| ab09cff427 | |||
| 47a9520a82 | |||
| 7a6c3c3fa3 | |||
| 803b5d583d | |||
| 1c52cd529b | |||
| ab28b168ec | |||
| 367b4b7553 | |||
| 25c7323c47 | |||
| aea0e16279 | |||
| f566573fce | |||
| 8fcceb3226 | |||
| ed0d6575c8 | |||
| f2ada02383 | |||
| 5e5816cd74 |
@@ -0,0 +1,44 @@
|
|||||||
|
import { expect, test, type Page } from '@playwright/test'
|
||||||
|
|
||||||
|
// 后台管理列表「创建时间 + 更新时间」两列验收(2026-09-19 需求):
|
||||||
|
// 8 个实体管理列表统一展示创建/更新时间,数据取自各列表 VO。
|
||||||
|
// mock 夹具统一给 createdAt=2026-03-01 09:15:00、updatedAt=2026-09-18 16:42:30,
|
||||||
|
// 页面统一经 formatDateTime 归一为「YYYY-MM-DD HH:mm:ss」。
|
||||||
|
|
||||||
|
const CREATED = '2026-03-01 09:15:00'
|
||||||
|
const UPDATED = '2026-09-18 16:42:30'
|
||||||
|
|
||||||
|
const PAGES: Array<{ title: string; path: string; heading: string }> = [
|
||||||
|
{ title: '用户管理', path: '/admin-vue/account/users', heading: '用户列表' },
|
||||||
|
{ title: '菜单管理', path: '/admin-vue/account/menus', heading: '菜单' },
|
||||||
|
{ title: '密钥管理', path: '/admin-vue/account/user-secrets', heading: '用户密钥列表' },
|
||||||
|
{ title: '不符合ASIN数据', path: '/admin-vue/asin-center/invalid', heading: '品牌数据列表' },
|
||||||
|
{ title: '数据去重总数据', path: '/admin-vue/asin-center/registry', heading: '数据去重总数据' },
|
||||||
|
{ title: '查询ASIN', path: '/admin-vue/asin-center/query', heading: '查询' },
|
||||||
|
{ title: '最低价ASIN', path: '/admin-vue/asin-center/skip-price', heading: '最低价' },
|
||||||
|
{ title: '商品类目', path: '/admin-vue/asin-center/categories', heading: '商品类目' },
|
||||||
|
]
|
||||||
|
|
||||||
|
async function openPage(page: Page, path: string) {
|
||||||
|
await page.goto(path)
|
||||||
|
await expect(page.locator('.panel-box table tbody tr').first()).toBeVisible({ timeout: 15000 })
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const spec of PAGES) {
|
||||||
|
test(`${spec.title}:表格展示创建时间与更新时间`, async ({ page }) => {
|
||||||
|
const errors: string[] = []
|
||||||
|
page.on('pageerror', (error) => errors.push(error.message))
|
||||||
|
|
||||||
|
await openPage(page, spec.path)
|
||||||
|
|
||||||
|
const table = page.locator('.panel-box table').first()
|
||||||
|
// 表头出现两列
|
||||||
|
await expect(table.locator('thead')).toContainText('创建时间')
|
||||||
|
await expect(table.locator('thead')).toContainText('更新时间')
|
||||||
|
// 数据行渲染出真实时间值(非空、非占位符)
|
||||||
|
await expect(table.locator('tbody')).toContainText(CREATED)
|
||||||
|
await expect(table.locator('tbody')).toContainText(UPDATED)
|
||||||
|
|
||||||
|
expect(errors).toEqual([])
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -220,13 +220,183 @@ const server = createServer((req, res) => {
|
|||||||
return json(res, {
|
return json(res, {
|
||||||
success: true,
|
success: true,
|
||||||
data: {
|
data: {
|
||||||
items: [{ id: 1, username: 'admin', role: 'super_admin', creatorUsername: 'system', createdAt: '2026-01-01 00:00:00' }],
|
items: [{ id: 1, username: 'admin', role: 'super_admin', creatorUsername: 'system', created_at: '2026-03-01 09:15:00', updated_at: '2026-09-18 16:42:30' }],
|
||||||
total: 1,
|
total: 1,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if (url === '/api/admin/permission-menus') {
|
if (url === '/api/admin/permission-menus') {
|
||||||
return json(res, { success: true, data: { items: [] } })
|
// 菜单管理列表:一条根节点,带创建/更新时间(验收时间列)。
|
||||||
|
return json(res, {
|
||||||
|
success: true,
|
||||||
|
data: [
|
||||||
|
{
|
||||||
|
id: 1,
|
||||||
|
name: '用户管理',
|
||||||
|
column_key: 'admin_users',
|
||||||
|
parent_id: null,
|
||||||
|
menu_type: 'admin',
|
||||||
|
route_path: 'account/users',
|
||||||
|
sort_order: 10,
|
||||||
|
created_at: '2026-03-01T09:15:00',
|
||||||
|
updated_at: '2026-09-18T16:42:30',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if (url === '/api/admin/invalid-asin-data') {
|
||||||
|
return json(res, {
|
||||||
|
success: true,
|
||||||
|
data: {
|
||||||
|
items: [
|
||||||
|
{
|
||||||
|
id: 1,
|
||||||
|
dataValue: 'B0INVALID1',
|
||||||
|
brand: '测试品牌',
|
||||||
|
groupId: 1,
|
||||||
|
groupName: '测试分组',
|
||||||
|
recordSource: 'MANUAL',
|
||||||
|
createdAt: '2026-03-01 09:15:00',
|
||||||
|
updatedAt: '2026-09-18 16:42:30',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
total: 1,
|
||||||
|
page: 1,
|
||||||
|
pageSize: 10,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if (url === '/api/admin/dedupe-total-data') {
|
||||||
|
return json(res, {
|
||||||
|
success: true,
|
||||||
|
data: {
|
||||||
|
items: [
|
||||||
|
{
|
||||||
|
id: 1,
|
||||||
|
dataValue: 'B0DEDUPE01',
|
||||||
|
country: 'DE',
|
||||||
|
groupId: 1,
|
||||||
|
groupName: '测试分组',
|
||||||
|
uploaderUserId: 1,
|
||||||
|
username: 'admin',
|
||||||
|
createdAt: '2026-03-01 09:15:00',
|
||||||
|
updatedAt: '2026-09-18 16:42:30',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
total: 1,
|
||||||
|
page: 1,
|
||||||
|
pageSize: 10,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if (url === '/api/admin/query-asins') {
|
||||||
|
return json(res, {
|
||||||
|
success: true,
|
||||||
|
data: {
|
||||||
|
items: [
|
||||||
|
{
|
||||||
|
id: 1,
|
||||||
|
groupId: 1,
|
||||||
|
groupName: '测试分组',
|
||||||
|
shopName: '测试店铺',
|
||||||
|
asinDe: 'B0QUERY001',
|
||||||
|
asinUk: '',
|
||||||
|
asinFr: '',
|
||||||
|
asinIt: '',
|
||||||
|
asinEs: '',
|
||||||
|
createdAt: '2026-03-01 09:15:00',
|
||||||
|
updatedAt: '2026-09-18 16:42:30',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
total: 1,
|
||||||
|
page: 1,
|
||||||
|
pageSize: 10,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if (url === '/api/admin/skip-price-asins') {
|
||||||
|
return json(res, {
|
||||||
|
success: true,
|
||||||
|
data: {
|
||||||
|
items: [
|
||||||
|
{
|
||||||
|
id: 1,
|
||||||
|
groupId: 1,
|
||||||
|
groupName: '测试分组',
|
||||||
|
shopName: '测试店铺',
|
||||||
|
asinDe: 'B0SKIP0001',
|
||||||
|
minimumPriceDe: 9.99,
|
||||||
|
asinUk: '',
|
||||||
|
minimumPriceUk: null,
|
||||||
|
asinFr: '',
|
||||||
|
minimumPriceFr: null,
|
||||||
|
asinIt: '',
|
||||||
|
minimumPriceIt: null,
|
||||||
|
asinEs: '',
|
||||||
|
minimumPriceEs: null,
|
||||||
|
createdAt: '2026-03-01 09:15:00',
|
||||||
|
updatedAt: '2026-09-18 16:42:30',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
total: 1,
|
||||||
|
page: 1,
|
||||||
|
pageSize: 10,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if (url === '/api/admin/product-categories/children') {
|
||||||
|
return json(res, {
|
||||||
|
success: true,
|
||||||
|
data: {
|
||||||
|
items: [
|
||||||
|
{
|
||||||
|
id: 1,
|
||||||
|
parentId: null,
|
||||||
|
name: '护肤品',
|
||||||
|
categoryKey: 'skincare',
|
||||||
|
sortOrder: 10,
|
||||||
|
description: '测试备注',
|
||||||
|
isBuiltin: true,
|
||||||
|
childCount: 0,
|
||||||
|
level: 0,
|
||||||
|
path: '护肤品',
|
||||||
|
createdAt: '2026-03-01 09:15:00',
|
||||||
|
updatedAt: '2026-09-18 16:42:30',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
total: 1,
|
||||||
|
page: 1,
|
||||||
|
pageSize: 10,
|
||||||
|
hasMore: false,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if (url === '/api/admin/user-secrets') {
|
||||||
|
const emptyModule = { moduleKey: '', moduleLabel: '', masked: '', full: '', exists: false, checkStatus: 'unknown', checkCode: '', checkMessage: '', checkLatencyMs: null, checkedAt: null, updatedAt: null }
|
||||||
|
return json(res, {
|
||||||
|
success: true,
|
||||||
|
data: {
|
||||||
|
items: [
|
||||||
|
{
|
||||||
|
userId: 1,
|
||||||
|
username: 'admin',
|
||||||
|
groups: [],
|
||||||
|
leaderUsername: '',
|
||||||
|
similarAsin: { ...emptyModule, moduleKey: 'similar-asin', moduleLabel: '货源查询密钥' },
|
||||||
|
appearancePatent: { ...emptyModule, moduleKey: 'appearance-patent', moduleLabel: '外观专利密钥' },
|
||||||
|
proxy: { ...emptyModule, moduleKey: 'proxy', moduleLabel: '代理设置' },
|
||||||
|
status: 'unknown',
|
||||||
|
statusMessage: '',
|
||||||
|
createdAt: '2026-03-01 09:15:00',
|
||||||
|
updatedAt: '2026-09-18 16:42:30',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
total: 1,
|
||||||
|
page: 1,
|
||||||
|
pageSize: 15,
|
||||||
|
groupOptions: [],
|
||||||
|
},
|
||||||
|
})
|
||||||
}
|
}
|
||||||
if (url === '/api/admin/shop-manage-groups') {
|
if (url === '/api/admin/shop-manage-groups') {
|
||||||
return json(res, { success: true, data: { items: [] } })
|
return json(res, { success: true, data: { items: [] } })
|
||||||
|
|||||||
@@ -30,6 +30,8 @@ export interface AdminUserSecretRow {
|
|||||||
proxy: AdminUserSecretModule
|
proxy: AdminUserSecretModule
|
||||||
status: string
|
status: string
|
||||||
statusMessage: string
|
statusMessage: string
|
||||||
|
/** 首次配置时间(三模块中最早);从未配置为 null。 */
|
||||||
|
createdAt: string | null
|
||||||
updatedAt: string | null
|
updatedAt: string | null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -23,6 +23,8 @@ export function toAdminUserItem(raw: unknown): AdminUser | null {
|
|||||||
if (createdById !== null) item.createdById = createdById
|
if (createdById !== null) item.createdById = createdById
|
||||||
const createdAt = text(r.created_at)
|
const createdAt = text(r.created_at)
|
||||||
if (createdAt) item.createdAt = createdAt
|
if (createdAt) item.createdAt = createdAt
|
||||||
|
const updatedAt = text(r.updated_at)
|
||||||
|
if (updatedAt) item.updatedAt = updatedAt
|
||||||
const creator = text(r.creator_username)
|
const creator = text(r.creator_username)
|
||||||
if (creator) item.creatorUsername = creator
|
if (creator) item.creatorUsername = creator
|
||||||
const abbr = text(r.pinyin_abbr)
|
const abbr = text(r.pinyin_abbr)
|
||||||
|
|||||||
@@ -241,6 +241,7 @@ onMounted(loadMenus)
|
|||||||
<th style="width: 100px">菜单类型</th>
|
<th style="width: 100px">菜单类型</th>
|
||||||
<th style="width: 120px">上级菜单</th>
|
<th style="width: 120px">上级菜单</th>
|
||||||
<th style="width: 170px">创建时间</th>
|
<th style="width: 170px">创建时间</th>
|
||||||
|
<th style="width: 170px">更新时间</th>
|
||||||
<th style="width: 170px">操作</th>
|
<th style="width: 170px">操作</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
@@ -271,6 +272,7 @@ onMounted(loadMenus)
|
|||||||
<td><span class="menu-type-text">{{ menuTypeLabel(row.menuType) }}</span></td>
|
<td><span class="menu-type-text">{{ menuTypeLabel(row.menuType) }}</span></td>
|
||||||
<td>{{ parentNameOf(row) }}</td>
|
<td>{{ parentNameOf(row) }}</td>
|
||||||
<td>{{ formatDateTime(row.createdAt) }}</td>
|
<td>{{ formatDateTime(row.createdAt) }}</td>
|
||||||
|
<td>{{ formatDateTime(row.updatedAt) }}</td>
|
||||||
<td class="ops-cell">
|
<td class="ops-cell">
|
||||||
<button class="btn btn-sm" type="button" @click="openEdit(row)">编辑</button>
|
<button class="btn btn-sm" type="button" @click="openEdit(row)">编辑</button>
|
||||||
<button
|
<button
|
||||||
@@ -286,10 +288,10 @@ onMounted(loadMenus)
|
|||||||
</tr>
|
</tr>
|
||||||
</template>
|
</template>
|
||||||
<tr v-else-if="loading">
|
<tr v-else-if="loading">
|
||||||
<td colspan="7" class="empty-tip">加载中...</td>
|
<td colspan="8" class="empty-tip">加载中...</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr v-else>
|
<tr v-else>
|
||||||
<td colspan="7" class="empty-tip">{{ filterName || filterType ? '暂无匹配菜单' : '暂无菜单,请先在上方新增' }}</td>
|
<td colspan="8" class="empty-tip">{{ filterName || filterType ? '暂无匹配菜单' : '暂无菜单,请先在上方新增' }}</td>
|
||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
|
|||||||
@@ -77,7 +77,7 @@ watch(
|
|||||||
node-key="id"
|
node-key="id"
|
||||||
show-checkbox
|
show-checkbox
|
||||||
default-expand-all
|
default-expand-all
|
||||||
:props="{ label: 'name', children: 'children' }"
|
:props="{ label: 'name', children: 'children', disabled: 'disabled' }"
|
||||||
@check="onCheck"
|
@check="onCheck"
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
@@ -89,7 +89,7 @@ watch(
|
|||||||
node-key="id"
|
node-key="id"
|
||||||
show-checkbox
|
show-checkbox
|
||||||
default-expand-all
|
default-expand-all
|
||||||
:props="{ label: 'name', children: 'children' }"
|
:props="{ label: 'name', children: 'children', disabled: 'disabled' }"
|
||||||
@check="onCheck"
|
@check="onCheck"
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -233,6 +233,8 @@ onMounted(load)
|
|||||||
<th style="width: 230px">外观专利密钥</th>
|
<th style="width: 230px">外观专利密钥</th>
|
||||||
<th style="width: 380px">代理设置</th>
|
<th style="width: 380px">代理设置</th>
|
||||||
<th style="width: 130px">状态</th>
|
<th style="width: 130px">状态</th>
|
||||||
|
<th style="width: 170px">创建时间</th>
|
||||||
|
<th style="width: 170px">更新时间</th>
|
||||||
<th style="width: 170px">操作</th>
|
<th style="width: 170px">操作</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
@@ -278,6 +280,8 @@ onMounted(load)
|
|||||||
{{ rowStatusMeta(row.status).label }}
|
{{ rowStatusMeta(row.status).label }}
|
||||||
</span>
|
</span>
|
||||||
</td>
|
</td>
|
||||||
|
<td>{{ formatDateTime(row.createdAt) }}</td>
|
||||||
|
<td>{{ formatDateTime(row.updatedAt) }}</td>
|
||||||
<td class="ops-cell">
|
<td class="ops-cell">
|
||||||
<button
|
<button
|
||||||
class="btn btn-sm"
|
class="btn btn-sm"
|
||||||
@@ -292,10 +296,10 @@ onMounted(load)
|
|||||||
</tr>
|
</tr>
|
||||||
</template>
|
</template>
|
||||||
<tr v-else-if="loading">
|
<tr v-else-if="loading">
|
||||||
<td :colspan="isSuperAdmin ? 8 : 7" class="empty-tip">加载中...</td>
|
<td :colspan="isSuperAdmin ? 10 : 9" class="empty-tip">加载中...</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr v-else>
|
<tr v-else>
|
||||||
<td :colspan="isSuperAdmin ? 8 : 7" class="empty-tip">{{ keyword || statusFilter || groupFilter ? '暂无匹配记录' : '暂无用户密钥记录' }}</td>
|
<td :colspan="isSuperAdmin ? 10 : 9" class="empty-tip">{{ keyword || statusFilter || groupFilter ? '暂无匹配记录' : '暂无用户密钥记录' }}</td>
|
||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
|
|||||||
@@ -169,6 +169,7 @@ onMounted(loadUsers)
|
|||||||
<th style="width: 140px">角色</th>
|
<th style="width: 140px">角色</th>
|
||||||
<th style="width: 160px">所属管理员</th>
|
<th style="width: 160px">所属管理员</th>
|
||||||
<th style="width: 180px">创建时间</th>
|
<th style="width: 180px">创建时间</th>
|
||||||
|
<th style="width: 180px">更新时间</th>
|
||||||
<th style="width: 160px">操作</th>
|
<th style="width: 160px">操作</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
@@ -180,6 +181,7 @@ onMounted(loadUsers)
|
|||||||
<td>{{ roleLabel(row.role) }}</td>
|
<td>{{ roleLabel(row.role) }}</td>
|
||||||
<td>{{ row.creatorUsername || '-' }}</td>
|
<td>{{ row.creatorUsername || '-' }}</td>
|
||||||
<td>{{ formatDateTime(row.createdAt) }}</td>
|
<td>{{ formatDateTime(row.createdAt) }}</td>
|
||||||
|
<td>{{ formatDateTime(row.updatedAt) }}</td>
|
||||||
<td class="ops-cell">
|
<td class="ops-cell">
|
||||||
<button
|
<button
|
||||||
class="btn btn-sm"
|
class="btn btn-sm"
|
||||||
@@ -203,10 +205,10 @@ onMounted(loadUsers)
|
|||||||
</tr>
|
</tr>
|
||||||
</template>
|
</template>
|
||||||
<tr v-else-if="loading">
|
<tr v-else-if="loading">
|
||||||
<td colspan="6" class="empty-tip">加载中...</td>
|
<td colspan="7" class="empty-tip">加载中...</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr v-else>
|
<tr v-else>
|
||||||
<td colspan="6" class="empty-tip">{{ userListEmptyHint() }}</td>
|
<td colspan="7" class="empty-tip">{{ userListEmptyHint() }}</td>
|
||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ export interface MenuManageNode {
|
|||||||
routePath: string
|
routePath: string
|
||||||
sortOrder: number
|
sortOrder: number
|
||||||
createdAt?: string
|
createdAt?: string
|
||||||
|
updatedAt?: string
|
||||||
children?: MenuManageNode[]
|
children?: MenuManageNode[]
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -90,6 +91,8 @@ export function parseMenuManageItem(raw: unknown): MenuManageNode | null {
|
|||||||
if (rootColumnKey) node.rootColumnKey = rootColumnKey
|
if (rootColumnKey) node.rootColumnKey = rootColumnKey
|
||||||
const createdAt = text(record.created_at)
|
const createdAt = text(record.created_at)
|
||||||
if (createdAt) node.createdAt = createdAt
|
if (createdAt) node.createdAt = createdAt
|
||||||
|
const updatedAt = text(record.updated_at)
|
||||||
|
if (updatedAt) node.updatedAt = updatedAt
|
||||||
return node
|
return node
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -16,6 +16,11 @@ export interface MenuOptionNode {
|
|||||||
parentId: number | null
|
parentId: number | null
|
||||||
/** 所属菜单类型(admin 后台 / app 前端客户端),提交时按类型分区落库。 */
|
/** 所属菜单类型(admin 后台 / app 前端客户端),提交时按类型分区落库。 */
|
||||||
type: string
|
type: string
|
||||||
|
/**
|
||||||
|
* 当前操作者无权授予(后端 grantable=false)时置灰:仍展示并回显已勾选,
|
||||||
|
* 但不允许改勾选。非超管只能授自己已有的菜单,勾到越权项会让整笔保存回滚。
|
||||||
|
*/
|
||||||
|
disabled?: boolean
|
||||||
children?: MenuOptionNode[]
|
children?: MenuOptionNode[]
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -39,6 +44,8 @@ export function parsePermissionMenuItem(raw: unknown, type = ''): MenuOptionNode
|
|||||||
sort: sortRaw === null ? 0 : sortRaw,
|
sort: sortRaw === null ? 0 : sortRaw,
|
||||||
parentId: parentId === null ? null : parentId,
|
parentId: parentId === null ? null : parentId,
|
||||||
type,
|
type,
|
||||||
|
// 缺省(菜单管理页等未标记的接口)按可授予处理,保持旧行为
|
||||||
|
disabled: record.grantable === false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -202,6 +202,7 @@ onMounted(() => {
|
|||||||
<th v-if="isSuperAdmin" style="width: 140px">分组</th>
|
<th v-if="isSuperAdmin" style="width: 140px">分组</th>
|
||||||
<th style="width: 110px">来源</th>
|
<th style="width: 110px">来源</th>
|
||||||
<th style="width: 170px">创建时间</th>
|
<th style="width: 170px">创建时间</th>
|
||||||
|
<th style="width: 170px">更新时间</th>
|
||||||
<th style="width: 150px">操作</th>
|
<th style="width: 150px">操作</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
@@ -218,6 +219,7 @@ onMounted(() => {
|
|||||||
</span>
|
</span>
|
||||||
</td>
|
</td>
|
||||||
<td>{{ formatDateTime(row.createdAt) }}</td>
|
<td>{{ formatDateTime(row.createdAt) }}</td>
|
||||||
|
<td>{{ formatDateTime(row.updatedAt) }}</td>
|
||||||
<td class="ops-cell">
|
<td class="ops-cell">
|
||||||
<button class="btn btn-sm" type="button" @click="openEdit(row)">编辑</button>
|
<button class="btn btn-sm" type="button" @click="openEdit(row)">编辑</button>
|
||||||
<button class="btn btn-sm btn-danger" type="button" @click="remove(row)">删除</button>
|
<button class="btn btn-sm btn-danger" type="button" @click="remove(row)">删除</button>
|
||||||
@@ -225,10 +227,10 @@ onMounted(() => {
|
|||||||
</tr>
|
</tr>
|
||||||
</template>
|
</template>
|
||||||
<tr v-else-if="loading">
|
<tr v-else-if="loading">
|
||||||
<td :colspan="isSuperAdmin ? 7 : 6" class="empty-tip">加载中...</td>
|
<td :colspan="isSuperAdmin ? 8 : 7" class="empty-tip">加载中...</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr v-else>
|
<tr v-else>
|
||||||
<td :colspan="isSuperAdmin ? 7 : 6" class="empty-tip">暂无数据</td>
|
<td :colspan="isSuperAdmin ? 8 : 7" class="empty-tip">暂无数据</td>
|
||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
|
|||||||
@@ -375,6 +375,7 @@ onMounted(() => {
|
|||||||
<th>用户名</th>
|
<th>用户名</th>
|
||||||
<th v-if="isSuperAdmin">分组</th>
|
<th v-if="isSuperAdmin">分组</th>
|
||||||
<th>创建时间</th>
|
<th>创建时间</th>
|
||||||
|
<th>更新时间</th>
|
||||||
<th>操作</th>
|
<th>操作</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
@@ -387,6 +388,7 @@ onMounted(() => {
|
|||||||
<td>{{ row.username || '-' }}</td>
|
<td>{{ row.username || '-' }}</td>
|
||||||
<td v-if="isSuperAdmin">{{ row.groupName || '未分组' }}</td>
|
<td v-if="isSuperAdmin">{{ row.groupName || '未分组' }}</td>
|
||||||
<td>{{ formatDateTime(row.createdAt) }}</td>
|
<td>{{ formatDateTime(row.createdAt) }}</td>
|
||||||
|
<td>{{ formatDateTime(row.updatedAt) }}</td>
|
||||||
<td class="ops-cell">
|
<td class="ops-cell">
|
||||||
<button class="btn btn-sm" type="button" @click="openEdit(row)">编辑</button>
|
<button class="btn btn-sm" type="button" @click="openEdit(row)">编辑</button>
|
||||||
<button class="btn btn-sm btn-danger" type="button" @click="removeRow(row)">删除</button>
|
<button class="btn btn-sm btn-danger" type="button" @click="removeRow(row)">删除</button>
|
||||||
@@ -394,10 +396,10 @@ onMounted(() => {
|
|||||||
</tr>
|
</tr>
|
||||||
</template>
|
</template>
|
||||||
<tr v-else-if="loading">
|
<tr v-else-if="loading">
|
||||||
<td :colspan="isSuperAdmin ? 7 : 6" class="empty-tip">加载中...</td>
|
<td :colspan="isSuperAdmin ? 8 : 7" class="empty-tip">加载中...</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr v-else>
|
<tr v-else>
|
||||||
<td :colspan="isSuperAdmin ? 7 : 6" class="empty-tip">暂无总数据</td>
|
<td :colspan="isSuperAdmin ? 8 : 7" class="empty-tip">暂无总数据</td>
|
||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import {
|
|||||||
} from './product-category-model.ts'
|
} from './product-category-model.ts'
|
||||||
import { categoryDraftError, createProductCategoryDraft, toCategorySaveRequest, type ProductCategoryDraft } from './product-category-editor-model.ts'
|
import { categoryDraftError, createProductCategoryDraft, toCategorySaveRequest, type ProductCategoryDraft } from './product-category-editor-model.ts'
|
||||||
import { buildProductCategoryExportUrl } from './product-category-export.ts'
|
import { buildProductCategoryExportUrl } from './product-category-export.ts'
|
||||||
|
import { formatDateTime } from '@/utils/datetime'
|
||||||
import OldPagination from '@/components/OldPagination.vue'
|
import OldPagination from '@/components/OldPagination.vue'
|
||||||
|
|
||||||
const loading = ref(false)
|
const loading = ref(false)
|
||||||
@@ -330,6 +331,8 @@ onMounted(loadTree)
|
|||||||
<th style="width: 90px">排序</th>
|
<th style="width: 90px">排序</th>
|
||||||
<th style="width: 120px">来源</th>
|
<th style="width: 120px">来源</th>
|
||||||
<th>备注</th>
|
<th>备注</th>
|
||||||
|
<th style="width: 170px">创建时间</th>
|
||||||
|
<th style="width: 170px">更新时间</th>
|
||||||
<th style="width: 150px">操作</th>
|
<th style="width: 150px">操作</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
@@ -337,7 +340,7 @@ onMounted(loadTree)
|
|||||||
<template v-if="treeRows.length">
|
<template v-if="treeRows.length">
|
||||||
<template v-for="row in treeRows" :key="String(row.node.id)">
|
<template v-for="row in treeRows" :key="String(row.node.id)">
|
||||||
<tr v-if="isCategoryLoadMoreNode(row.node)" class="load-more-row">
|
<tr v-if="isCategoryLoadMoreNode(row.node)" class="load-more-row">
|
||||||
<td colspan="6">
|
<td colspan="8">
|
||||||
<div class="load-more-cell">
|
<div class="load-more-cell">
|
||||||
<span class="tree-indent" :style="{ width: `${(row.depth + 1) * 24}px` }"></span>
|
<span class="tree-indent" :style="{ width: `${(row.depth + 1) * 24}px` }"></span>
|
||||||
<button class="btn btn-sm btn-secondary" type="button" :disabled="moreLoading(row.node)" @click="loadMoreChildren(row.node)">
|
<button class="btn btn-sm btn-secondary" type="button" :disabled="moreLoading(row.node)" @click="loadMoreChildren(row.node)">
|
||||||
@@ -376,6 +379,8 @@ onMounted(loadTree)
|
|||||||
<span v-if="row.node.description" class="node-desc" :title="row.node.description">{{ row.node.description }}</span>
|
<span v-if="row.node.description" class="node-desc" :title="row.node.description">{{ row.node.description }}</span>
|
||||||
<span v-else class="dim">—</span>
|
<span v-else class="dim">—</span>
|
||||||
</td>
|
</td>
|
||||||
|
<td>{{ formatDateTime(row.node.createdAt) }}</td>
|
||||||
|
<td>{{ formatDateTime(row.node.updatedAt) }}</td>
|
||||||
<td class="ops-cell">
|
<td class="ops-cell">
|
||||||
<button class="btn btn-sm" type="button" @click="openEdit(row.node)">编辑</button>
|
<button class="btn btn-sm" type="button" @click="openEdit(row.node)">编辑</button>
|
||||||
<button
|
<button
|
||||||
@@ -392,10 +397,10 @@ onMounted(loadTree)
|
|||||||
</template>
|
</template>
|
||||||
</template>
|
</template>
|
||||||
<tr v-else-if="loading">
|
<tr v-else-if="loading">
|
||||||
<td colspan="6" class="empty-tip">加载中...</td>
|
<td colspan="8" class="empty-tip">加载中...</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr v-else>
|
<tr v-else>
|
||||||
<td colspan="6" class="empty-tip">暂无商品类目</td>
|
<td colspan="8" class="empty-tip">暂无商品类目</td>
|
||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
<tbody v-else>
|
<tbody v-else>
|
||||||
@@ -411,6 +416,8 @@ onMounted(loadTree)
|
|||||||
<span v-if="row.description" class="node-desc" :title="row.description">{{ row.description }}</span>
|
<span v-if="row.description" class="node-desc" :title="row.description">{{ row.description }}</span>
|
||||||
<span v-else class="dim">—</span>
|
<span v-else class="dim">—</span>
|
||||||
</td>
|
</td>
|
||||||
|
<td>{{ formatDateTime(row.createdAt) }}</td>
|
||||||
|
<td>{{ formatDateTime(row.updatedAt) }}</td>
|
||||||
<td class="ops-cell">
|
<td class="ops-cell">
|
||||||
<button class="btn btn-sm" type="button" @click="openEdit(row)">编辑</button>
|
<button class="btn btn-sm" type="button" @click="openEdit(row)">编辑</button>
|
||||||
<button
|
<button
|
||||||
@@ -426,10 +433,10 @@ onMounted(loadTree)
|
|||||||
</tr>
|
</tr>
|
||||||
</template>
|
</template>
|
||||||
<tr v-else-if="searchLoading">
|
<tr v-else-if="searchLoading">
|
||||||
<td colspan="6" class="empty-tip">搜索中...</td>
|
<td colspan="8" class="empty-tip">搜索中...</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr v-else>
|
<tr v-else>
|
||||||
<td colspan="6" class="empty-tip">暂无搜索结果</td>
|
<td colspan="8" class="empty-tip">暂无搜索结果</td>
|
||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
|
|||||||
@@ -4,6 +4,7 @@
|
|||||||
import { computed, onMounted, reactive, ref } from 'vue'
|
import { computed, onMounted, reactive, ref } from 'vue'
|
||||||
import { ElMessage, ElMessageBox } from 'element-plus'
|
import { ElMessage, ElMessageBox } from 'element-plus'
|
||||||
import CopyText from '@/components/CopyText.vue'
|
import CopyText from '@/components/CopyText.vue'
|
||||||
|
import { formatDateTime } from '@/utils/datetime'
|
||||||
import { createQueryAsin, fetchQueryAsinList, fetchShopNamesByGroup } from './query-asin-api.ts'
|
import { createQueryAsin, fetchQueryAsinList, fetchShopNamesByGroup } from './query-asin-api.ts'
|
||||||
import { QUERY_ASIN_COUNTRIES, queryAsinDisplayRows, type QueryAsinItem } from './query-asin-model.ts'
|
import { QUERY_ASIN_COUNTRIES, queryAsinDisplayRows, type QueryAsinItem } from './query-asin-model.ts'
|
||||||
import { asinCountryLabel } from './asin-country.ts'
|
import { asinCountryLabel } from './asin-country.ts'
|
||||||
@@ -394,6 +395,8 @@ onMounted(() => {
|
|||||||
<th style="width: 15%">店铺名</th>
|
<th style="width: 15%">店铺名</th>
|
||||||
<th style="width: 24%">ASIN</th>
|
<th style="width: 24%">ASIN</th>
|
||||||
<th style="width: 12%">国家</th>
|
<th style="width: 12%">国家</th>
|
||||||
|
<th style="width: 170px">创建时间</th>
|
||||||
|
<th style="width: 170px">更新时间</th>
|
||||||
<th style="width: 9%">操作</th>
|
<th style="width: 9%">操作</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
@@ -413,6 +416,8 @@ onMounted(() => {
|
|||||||
</td>
|
</td>
|
||||||
<td>{{ asinCountryLabel(row.country || '') }}</td>
|
<td>{{ asinCountryLabel(row.country || '') }}</td>
|
||||||
<template v-if="row.isFirst">
|
<template v-if="row.isFirst">
|
||||||
|
<td :rowspan="row.rowspan">{{ formatDateTime(row.item.createdAt) }}</td>
|
||||||
|
<td :rowspan="row.rowspan">{{ formatDateTime(row.item.updatedAt) }}</td>
|
||||||
<td :rowspan="row.rowspan" class="asin-col-actions">
|
<td :rowspan="row.rowspan" class="asin-col-actions">
|
||||||
<button class="btn btn-sm" type="button" @click="openConfig(row.item as QueryAsinItem)">配置</button>
|
<button class="btn btn-sm" type="button" @click="openConfig(row.item as QueryAsinItem)">配置</button>
|
||||||
<button class="btn btn-sm btn-danger" type="button" @click="removeShopAsin(row.item as QueryAsinItem)">删除</button>
|
<button class="btn btn-sm btn-danger" type="button" @click="removeShopAsin(row.item as QueryAsinItem)">删除</button>
|
||||||
@@ -421,10 +426,10 @@ onMounted(() => {
|
|||||||
</tr>
|
</tr>
|
||||||
</template>
|
</template>
|
||||||
<tr v-else-if="loading">
|
<tr v-else-if="loading">
|
||||||
<td :colspan="isSuperAdmin ? 6 : 5" class="empty-tip">加载中...</td>
|
<td :colspan="isSuperAdmin ? 8 : 7" class="empty-tip">加载中...</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr v-else>
|
<tr v-else>
|
||||||
<td :colspan="isSuperAdmin ? 6 : 5" class="empty-tip">暂无数据</td>
|
<td :colspan="isSuperAdmin ? 8 : 7" class="empty-tip">暂无数据</td>
|
||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
|
|||||||
@@ -4,6 +4,7 @@
|
|||||||
import { computed, onMounted, reactive, ref } from 'vue'
|
import { computed, onMounted, reactive, ref } from 'vue'
|
||||||
import { ElMessage, ElMessageBox } from 'element-plus'
|
import { ElMessage, ElMessageBox } from 'element-plus'
|
||||||
import CopyText from '@/components/CopyText.vue'
|
import CopyText from '@/components/CopyText.vue'
|
||||||
|
import { formatDateTime } from '@/utils/datetime'
|
||||||
import { createSkipPriceAsin, fetchSkipPriceList } from './skip-price-api.ts'
|
import { createSkipPriceAsin, fetchSkipPriceList } from './skip-price-api.ts'
|
||||||
import { skipPriceDisplayRows, type SkipPriceItem } from './skip-price-model.ts'
|
import { skipPriceDisplayRows, type SkipPriceItem } from './skip-price-model.ts'
|
||||||
import { asinCountryLabel } from './asin-country.ts'
|
import { asinCountryLabel } from './asin-country.ts'
|
||||||
@@ -447,6 +448,8 @@ onMounted(() => {
|
|||||||
<th style="width: 24%">ASIN</th>
|
<th style="width: 24%">ASIN</th>
|
||||||
<th style="width: 12%">国家</th>
|
<th style="width: 12%">国家</th>
|
||||||
<th style="width: 10%">最低价</th>
|
<th style="width: 10%">最低价</th>
|
||||||
|
<th style="width: 170px">创建时间</th>
|
||||||
|
<th style="width: 170px">更新时间</th>
|
||||||
<th style="width: 9%">操作</th>
|
<th style="width: 9%">操作</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
@@ -467,6 +470,8 @@ onMounted(() => {
|
|||||||
<td>{{ asinCountryLabel(row.country || '') }}</td>
|
<td>{{ asinCountryLabel(row.country || '') }}</td>
|
||||||
<td class="price-cell">{{ row.minimumPrice !== '' ? row.minimumPrice : '-' }}</td>
|
<td class="price-cell">{{ row.minimumPrice !== '' ? row.minimumPrice : '-' }}</td>
|
||||||
<template v-if="row.isFirst">
|
<template v-if="row.isFirst">
|
||||||
|
<td :rowspan="row.rowspan">{{ formatDateTime(row.item.createdAt) }}</td>
|
||||||
|
<td :rowspan="row.rowspan">{{ formatDateTime(row.item.updatedAt) }}</td>
|
||||||
<td :rowspan="row.rowspan" class="asin-col-actions">
|
<td :rowspan="row.rowspan" class="asin-col-actions">
|
||||||
<button class="btn btn-sm" type="button" @click="openConfig(row.item as SkipPriceItem)">配置</button>
|
<button class="btn btn-sm" type="button" @click="openConfig(row.item as SkipPriceItem)">配置</button>
|
||||||
<button class="btn btn-sm btn-danger" type="button" @click="removeRow(row.item as SkipPriceItem)">删除</button>
|
<button class="btn btn-sm btn-danger" type="button" @click="removeRow(row.item as SkipPriceItem)">删除</button>
|
||||||
@@ -475,10 +480,10 @@ onMounted(() => {
|
|||||||
</tr>
|
</tr>
|
||||||
</template>
|
</template>
|
||||||
<tr v-else-if="loading">
|
<tr v-else-if="loading">
|
||||||
<td :colspan="isSuperAdmin ? 7 : 6" class="empty-tip">加载中...</td>
|
<td :colspan="isSuperAdmin ? 9 : 8" class="empty-tip">加载中...</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr v-else>
|
<tr v-else>
|
||||||
<td :colspan="isSuperAdmin ? 7 : 6" class="empty-tip">暂无数据</td>
|
<td :colspan="isSuperAdmin ? 9 : 8" class="empty-tip">暂无数据</td>
|
||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ export interface DedupeTotalItem {
|
|||||||
uploaderUserId: number | null
|
uploaderUserId: number | null
|
||||||
username: string
|
username: string
|
||||||
createdAt?: string
|
createdAt?: string
|
||||||
|
updatedAt?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface DedupeTotalPageResult {
|
export interface DedupeTotalPageResult {
|
||||||
@@ -51,6 +52,8 @@ export function toDedupeTotalItem(raw: unknown): DedupeTotalItem | null {
|
|||||||
}
|
}
|
||||||
const createdAt = text(record.createdAt ?? record.created_at)
|
const createdAt = text(record.createdAt ?? record.created_at)
|
||||||
if (createdAt) item.createdAt = createdAt
|
if (createdAt) item.createdAt = createdAt
|
||||||
|
const updatedAt = text(record.updatedAt ?? record.updated_at)
|
||||||
|
if (updatedAt) item.updatedAt = updatedAt
|
||||||
return item
|
return item
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -14,6 +14,8 @@ export interface ProductCategoryNode {
|
|||||||
childCount: number
|
childCount: number
|
||||||
level: number | null
|
level: number | null
|
||||||
path: string
|
path: string
|
||||||
|
createdAt?: string
|
||||||
|
updatedAt?: string
|
||||||
children: ProductCategoryNode[]
|
children: ProductCategoryNode[]
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -72,6 +74,10 @@ export function toProductCategoryNode(raw: unknown): ProductCategoryNode | null
|
|||||||
path: text(record.path),
|
path: text(record.path),
|
||||||
children: [],
|
children: [],
|
||||||
}
|
}
|
||||||
|
const createdAt = text(record.createdAt ?? record.created_at)
|
||||||
|
if (createdAt) node.createdAt = createdAt
|
||||||
|
const updatedAt = text(record.updatedAt ?? record.updated_at)
|
||||||
|
if (updatedAt) node.updatedAt = updatedAt
|
||||||
if (Array.isArray(record.children)) {
|
if (Array.isArray(record.children)) {
|
||||||
node.children = record.children
|
node.children = record.children
|
||||||
.map((child) => toProductCategoryNode(child))
|
.map((child) => toProductCategoryNode(child))
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ export interface InvalidAsinItem {
|
|||||||
groupName: string
|
groupName: string
|
||||||
recordSource: string
|
recordSource: string
|
||||||
createdAt?: string
|
createdAt?: string
|
||||||
|
updatedAt?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface InvalidAsinPageResult {
|
export interface InvalidAsinPageResult {
|
||||||
@@ -74,6 +75,8 @@ export function toInvalidAsinItem(raw: unknown): InvalidAsinItem | null {
|
|||||||
}
|
}
|
||||||
const createdAt = text(record.createdAt ?? record.created_at)
|
const createdAt = text(record.createdAt ?? record.created_at)
|
||||||
if (createdAt) item.createdAt = createdAt
|
if (createdAt) item.createdAt = createdAt
|
||||||
|
const updatedAt = text(record.updatedAt ?? record.updated_at)
|
||||||
|
if (updatedAt) item.updatedAt = updatedAt
|
||||||
return item
|
return item
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ export interface AdminUser {
|
|||||||
createdById?: number | null
|
createdById?: number | null
|
||||||
creatorUsername?: string
|
creatorUsername?: string
|
||||||
createdAt?: string
|
createdAt?: string
|
||||||
|
updatedAt?: string
|
||||||
pinyinAbbr?: string
|
pinyinAbbr?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -82,7 +82,9 @@ test('align_query_asin_page_layout_wiring', () => {
|
|||||||
assert.match(page, /新增 ASIN/, '顶栏补新增 ASIN 按钮')
|
assert.match(page, /新增 ASIN/, '顶栏补新增 ASIN 按钮')
|
||||||
assert.match(page, /:rowspan="row\.rowspan"/, '表格用原生 rowspan 合并(像素复刻旧版)')
|
assert.match(page, /:rowspan="row\.rowspan"/, '表格用原生 rowspan 合并(像素复刻旧版)')
|
||||||
assert.match(page, /CopyText/, 'ASIN 单元格点击复制')
|
assert.match(page, /CopyText/, 'ASIN 单元格点击复制')
|
||||||
assert.doesNotMatch(page, /更新时间/, '去掉更新时间列(参考无此列)')
|
// 需求变更:后台管理列表统一展示「创建时间 + 更新时间」,覆盖早期「不加更新时间」的像素对齐。
|
||||||
|
assert.match(page, /创建时间/, '表格含创建时间列')
|
||||||
|
assert.match(page, /更新时间/, '表格含更新时间列')
|
||||||
assert.match(page, /queryAsinDisplayRows/, '行展开走纯模型')
|
assert.match(page, /queryAsinDisplayRows/, '行展开走纯模型')
|
||||||
assert.match(page, /请完整填写分组、店铺名、国家和 ASIN/, '新增弹窗校验文案对齐')
|
assert.match(page, /请完整填写分组、店铺名、国家和 ASIN/, '新增弹窗校验文案对齐')
|
||||||
assert.match(page, /请先选择分组/, '店铺下拉未选分组时占位对齐')
|
assert.match(page, /请先选择分组/, '店铺下拉未选分组时占位对齐')
|
||||||
|
|||||||
@@ -82,7 +82,9 @@ test('align_skip_price_page_layout_wiring', () => {
|
|||||||
assert.match(page, /新增 ASIN/, '顶栏补新增 ASIN 按钮')
|
assert.match(page, /新增 ASIN/, '顶栏补新增 ASIN 按钮')
|
||||||
assert.match(page, /:rowspan="row\.rowspan"/, '表格用原生 rowspan 合并(像素复刻旧版)')
|
assert.match(page, /:rowspan="row\.rowspan"/, '表格用原生 rowspan 合并(像素复刻旧版)')
|
||||||
assert.match(page, /CopyText/, 'ASIN 单元格点击复制')
|
assert.match(page, /CopyText/, 'ASIN 单元格点击复制')
|
||||||
assert.doesNotMatch(page, /更新时间/, '去掉更新时间列(参考无此列)')
|
// 需求变更:后台管理列表统一展示「创建时间 + 更新时间」,覆盖早期「不加更新时间」的像素对齐。
|
||||||
|
assert.match(page, /创建时间/, '表格含创建时间列')
|
||||||
|
assert.match(page, /更新时间/, '表格含更新时间列')
|
||||||
assert.match(page, /skipPriceDisplayRows/, '行展开走纯模型')
|
assert.match(page, /skipPriceDisplayRows/, '行展开走纯模型')
|
||||||
assert.match(page, /请完整填写分组、店铺名、国家和 ASIN/, '新增弹窗校验文案对齐')
|
assert.match(page, /请完整填写分组、店铺名、国家和 ASIN/, '新增弹窗校验文案对齐')
|
||||||
assert.match(page, /最低价格式不正确/, '最低价格式校验对齐')
|
assert.match(page, /最低价格式不正确/, '最低价格式校验对齐')
|
||||||
|
|||||||
@@ -0,0 +1,86 @@
|
|||||||
|
import test from 'node:test'
|
||||||
|
import assert from 'node:assert/strict'
|
||||||
|
import { readSource } from './helpers.ts'
|
||||||
|
import {
|
||||||
|
buildMenuOptionTree,
|
||||||
|
compactDirectGrantIds,
|
||||||
|
parseMenuOptionList,
|
||||||
|
parsePermissionMenuItem,
|
||||||
|
} from '../src/pages/account/user-menu-auth.ts'
|
||||||
|
|
||||||
|
// 2026-09-16 线上事故:非超管(普通管理员)在授权树里勾到自己无权授予的菜单后,
|
||||||
|
// 后端 ensureGrantable 抛 403 并回滚整笔事务——创建用户与保存权限双双失败,
|
||||||
|
// 前端只显示「普通管理员只能分配自己已有的菜单权限」。
|
||||||
|
// 修复:菜单列表按操作者标记 grantable,前端把不可授予的节点置灰不可勾。
|
||||||
|
//
|
||||||
|
// 注意:这里是「置灰」而非「隐藏」。授权保存是整树替换,隐藏会让超管早先授予、
|
||||||
|
// 而操作者自己没有的菜单在提交时被当作取消勾选删掉(与 09-13「权限自己没掉」同类)。
|
||||||
|
|
||||||
|
test('align_user_menu_grantable_false_maps_to_disabled_node', () => {
|
||||||
|
const locked = parsePermissionMenuItem(
|
||||||
|
{ id: 5, name: '查询ASIN', parent_id: null, sort_order: 1, grantable: false },
|
||||||
|
'admin',
|
||||||
|
)
|
||||||
|
assert.equal(locked?.disabled, true, 'grantable=false → 节点置灰')
|
||||||
|
|
||||||
|
const allowed = parsePermissionMenuItem(
|
||||||
|
{ id: 6, name: '店铺管理', parent_id: null, sort_order: 2, grantable: true },
|
||||||
|
'admin',
|
||||||
|
)
|
||||||
|
assert.equal(allowed?.disabled, false, 'grantable=true → 可勾选')
|
||||||
|
|
||||||
|
// 菜单管理页等未标记 grantable 的接口必须保持旧行为(全部可勾选)
|
||||||
|
const unmarked = parsePermissionMenuItem({ id: 7, name: '菜单权限配置', parent_id: null, sort_order: 3 }, 'admin')
|
||||||
|
assert.equal(unmarked?.disabled, false, '缺省 grantable 视为可授予')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('align_user_menu_grantable_survives_tree_build', () => {
|
||||||
|
const nodes = parseMenuOptionList(
|
||||||
|
[
|
||||||
|
{ id: 98, name: '账号与权限', parent_id: null, sort_order: 1, grantable: false },
|
||||||
|
{ id: 7, name: '用户管理', parent_id: 98, sort_order: 1, grantable: false },
|
||||||
|
{ id: 100, name: '店铺管理', parent_id: null, sort_order: 2, grantable: true },
|
||||||
|
],
|
||||||
|
'admin',
|
||||||
|
)
|
||||||
|
const tree = buildMenuOptionTree(nodes)
|
||||||
|
const account = tree.find((node) => node.id === 98)
|
||||||
|
assert.equal(account?.disabled, true, '分组节点置灰')
|
||||||
|
assert.equal(account?.children?.[0]?.disabled, true, '子节点置灰随树保留')
|
||||||
|
assert.equal(tree.find((node) => node.id === 100)?.disabled, false, '可授予节点不受影响')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('align_user_menu_grantable_disabled_node_still_compactable', () => {
|
||||||
|
// 已持有但无权授予的节点会保持勾选并原样提交,压缩逻辑不能因 disabled 漏掉它
|
||||||
|
const tree = buildMenuOptionTree(
|
||||||
|
parseMenuOptionList(
|
||||||
|
[
|
||||||
|
{ id: 98, name: '账号与权限', parent_id: null, sort_order: 1, grantable: false },
|
||||||
|
{ id: 7, name: '用户管理', parent_id: 98, sort_order: 1, grantable: false },
|
||||||
|
],
|
||||||
|
'admin',
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assert.deepEqual(compactDirectGrantIds([98, 7], tree), [98], '父级已勾选时仍压缩掉后代')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('align_user_menu_grantable_wired_end_to_end', () => {
|
||||||
|
const tree = readSource('src/pages/account/UserMenuAuthTree.vue')
|
||||||
|
assert.match(tree, /disabled: 'disabled'/, 'el-tree 按 disabled 键置灰节点')
|
||||||
|
|
||||||
|
const vo = readSource(
|
||||||
|
'../backend-java/src/main/java/com/nanri/aiimage/modules/permission/model/vo/PermissionMenuItemVo.java',
|
||||||
|
)
|
||||||
|
assert.match(vo, /private Boolean grantable;/, 'VO 暴露 grantable')
|
||||||
|
|
||||||
|
const controller = readSource(
|
||||||
|
'../backend-java/src/main/java/com/nanri/aiimage/modules/permission/controller/PermissionMenuController.java',
|
||||||
|
)
|
||||||
|
assert.match(controller, /permissionMenuService\.list\(requireAdmin\(request\), menuType\)/, '列表接口传入操作者')
|
||||||
|
|
||||||
|
const service = readSource(
|
||||||
|
'../backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuService.java',
|
||||||
|
)
|
||||||
|
assert.match(service, /resolveGrantableMenuIds/, '按操作者计算可授予集')
|
||||||
|
assert.match(service, /ensureGrantable\(operator, grantIds, userId\)/, '保存校验传入目标用户以放行既有授权')
|
||||||
|
})
|
||||||
@@ -25,4 +25,12 @@ public final class BusinessCodes {
|
|||||||
|
|
||||||
/** 任务归属其它实例,需转发。 */
|
/** 任务归属其它实例,需转发。 */
|
||||||
public static final int TASK_OWNER_FORWARD = 40903;
|
public static final int TASK_OWNER_FORWARD = 40903;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 提交结果的目标任务已不存在(通常是被删除)。
|
||||||
|
* 语义:本次提交无意义,响应 success=false 且带该码,调用方应放弃而不是反复重试。
|
||||||
|
* 与 {@link #TASK_ALREADY_FINISHED} 的区别:那个还留了任务记录(可幂等忽略),
|
||||||
|
* 这个任务已经没了——如实报错,否则任务被误删时结果会被静默吞掉。
|
||||||
|
*/
|
||||||
|
public static final int TASK_NOT_FOUND = 40401;
|
||||||
}
|
}
|
||||||
|
|||||||
+40
-5
@@ -1,18 +1,21 @@
|
|||||||
package com.nanri.aiimage.common.exception;
|
package com.nanri.aiimage.common.exception;
|
||||||
|
|
||||||
import com.nanri.aiimage.common.api.ApiResponse;
|
import com.nanri.aiimage.common.api.ApiResponse;
|
||||||
|
import com.nanri.aiimage.common.security.DeviceSessionPolicy;
|
||||||
import com.nanri.aiimage.common.service.TaskOwnerForwardService;
|
import com.nanri.aiimage.common.service.TaskOwnerForwardService;
|
||||||
import com.nanri.aiimage.config.TaskOperationLockConfig;
|
import com.nanri.aiimage.config.TaskOperationLockConfig;
|
||||||
import jakarta.servlet.http.HttpServletRequest;
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
import jakarta.validation.ConstraintViolationException;
|
import jakarta.validation.ConstraintViolationException;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.http.HttpStatus;
|
||||||
import org.springframework.http.ResponseEntity;
|
import org.springframework.http.ResponseEntity;
|
||||||
import org.springframework.web.bind.MethodArgumentNotValidException;
|
import org.springframework.web.bind.MethodArgumentNotValidException;
|
||||||
import org.springframework.web.bind.annotation.ExceptionHandler;
|
import org.springframework.web.bind.annotation.ExceptionHandler;
|
||||||
import org.springframework.web.bind.annotation.RestControllerAdvice;
|
import org.springframework.web.bind.annotation.RestControllerAdvice;
|
||||||
import org.springframework.web.context.request.async.AsyncRequestNotUsableException;
|
import org.springframework.web.context.request.async.AsyncRequestNotUsableException;
|
||||||
import org.springframework.web.server.ResponseStatusException;
|
import org.springframework.web.server.ResponseStatusException;
|
||||||
|
import org.springframework.web.servlet.resource.NoResourceFoundException;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
|
||||||
@@ -52,22 +55,40 @@ public class GlobalExceptionHandler {
|
|||||||
? ApiResponse.fail(forwardEx.getMessage())
|
? ApiResponse.fail(forwardEx.getMessage())
|
||||||
: ApiResponse.fail(forwardEx.getCode(), forwardEx.getMessage());
|
: ApiResponse.fail(forwardEx.getCode(), forwardEx.getMessage());
|
||||||
} catch (Exception forwardEx) {
|
} catch (Exception forwardEx) {
|
||||||
|
// 转发失败是**瞬时基础设施故障**(归属实例正在滚动重启),不是业务结论,
|
||||||
|
// 更不能表达成「任务不存活」。原先返回 ApiResponse.fail(40903) —— HTTP 200
|
||||||
|
// 加 data:null,而客户端那句 bool((resp.json().get("data") or {}).get("alive"))
|
||||||
|
// 会把「拿不到数据」折叠成 alive=false,于是客户端把**健康的长任务主动停掉**:
|
||||||
|
// 2026-09-18 任务 28616 就是这么死的(归属节点 server-110 重启窗口内,心跳经
|
||||||
|
// nginx 落到 server-121,转发 3 次 Connection refused 后返回空 data)。
|
||||||
|
// 改为 503 + 空 body:新客户端按状态码判为「未知」继续跑;老客户端因 body 不是
|
||||||
|
// JSON、resp.json() 抛异常,同样落到「未知」。顺带让这类故障在 HTTP 指标里可见
|
||||||
|
// (原先记成 200,监控完全看不到滚动重启期间丢了多少心跳)。
|
||||||
log.warn("[instance-routing] forward failed taskId={} operation={} owner={} current={} msg={}",
|
log.warn("[instance-routing] forward failed taskId={} operation={} owner={} current={} msg={}",
|
||||||
ex.getTaskId(), ex.getOperation(), ex.getOwnerInstanceId(), ex.getCurrentInstanceId(),
|
ex.getTaskId(), ex.getOperation(), ex.getOwnerInstanceId(), ex.getCurrentInstanceId(),
|
||||||
forwardEx.getMessage(), forwardEx);
|
forwardEx.getMessage(), forwardEx);
|
||||||
return ApiResponse.fail(40903, "任务归属实例转发失败: " + forwardEx.getMessage());
|
return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE).build();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ExceptionHandler(BusinessException.class)
|
@ExceptionHandler(BusinessException.class)
|
||||||
public ApiResponse<Void> handleBusinessException(BusinessException ex) {
|
public ApiResponse<Void> handleBusinessException(BusinessException ex, HttpServletRequest request) {
|
||||||
|
// 业务异常此前完全不记日志:2026-09-16 线上「保存权限/创建用户」双双失败时,
|
||||||
|
// 服务端只留 RequestTraceFilter 的 200 一行,根因只能靠反推响应体字节数才定位到。
|
||||||
|
// 401/4011(未登录、被顶下线)属于轮询类接口的常态噪声,降为 debug 以免淹没真实业务错。
|
||||||
|
if (isRoutineAuthNoise(ex.getCode())) {
|
||||||
|
log.debug("[business] {} {} code={} message={}", request.getMethod(), request.getRequestURI(),
|
||||||
|
ex.getCode(), ex.getMessage());
|
||||||
|
} else {
|
||||||
|
log.warn("[business] {} {} code={} message={}", request.getMethod(), request.getRequestURI(),
|
||||||
|
ex.getCode(), ex.getMessage());
|
||||||
|
}
|
||||||
if (Integer.valueOf(BusinessCodes.TASK_ALREADY_FINISHED).equals(ex.getCode())) {
|
if (Integer.valueOf(BusinessCodes.TASK_ALREADY_FINISHED).equals(ex.getCode())) {
|
||||||
// 幂等忽略:任务已结束时的重复提交无副作用,按成功返回,避免客户端反复重试
|
// 幂等忽略:任务已结束时的重复提交无副作用,按成功返回,避免客户端反复重试
|
||||||
return ApiResponse.success("任务已结束,忽略重复提交", null);
|
return ApiResponse.success("任务已结束,忽略重复提交", null);
|
||||||
}
|
}
|
||||||
if (Integer.valueOf(BusinessCodes.TASK_BUSY).equals(ex.getCode())) {
|
if (Integer.valueOf(BusinessCodes.TASK_BUSY).equals(ex.getCode())) {
|
||||||
// 锁竞争:必须如实返回失败 + 可重试码,否则 worker 会把「未落库」当成功而停止重试
|
// 锁竞争:必须如实返回失败 + 可重试码,否则 worker 会把「未落库」当成功而停止重试
|
||||||
log.warn("[business] 任务忙,调用方应稍后重试: {}", ex.getMessage());
|
|
||||||
return ApiResponse.fail(BusinessCodes.TASK_BUSY, ex.getMessage());
|
return ApiResponse.fail(BusinessCodes.TASK_BUSY, ex.getMessage());
|
||||||
}
|
}
|
||||||
return ex.getCode() == null
|
return ex.getCode() == null
|
||||||
@@ -75,9 +96,23 @@ public class GlobalExceptionHandler {
|
|||||||
: ApiResponse.fail(ex.getCode(), ex.getMessage());
|
: ApiResponse.fail(ex.getCode(), ex.getMessage());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ExceptionHandler(NoResourceFoundException.class)
|
||||||
|
public ResponseEntity<ApiResponse<Void>> handleNoResourceFoundException(NoResourceFoundException ex) {
|
||||||
|
// 静态资源 404。绝大部分是外部扫描器在探测 /.env、/credentials、aliyun.json、oss.json
|
||||||
|
// 这类云凭据文件(线上单节点一天 580 条)。此前落到 handleException 里,既刷 ERROR 堆栈,
|
||||||
|
// 又把探测响应伪装成 HTTP 200;这里降为 debug 并如实返回 404。
|
||||||
|
log.debug("static resource not found: {}", ex.getMessage());
|
||||||
|
return ResponseEntity.status(HttpStatus.NOT_FOUND).body(ApiResponse.fail("资源不存在"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 未登录 / 登录态失效 / 被其他设备顶下线:按 401 语义的常态噪声,不占 WARN。 */
|
||||||
|
private boolean isRoutineAuthNoise(Integer code) {
|
||||||
|
return Integer.valueOf(401).equals(code)
|
||||||
|
|| Integer.valueOf(DeviceSessionPolicy.CODE_KICKED).equals(code);
|
||||||
|
}
|
||||||
|
|
||||||
@ExceptionHandler(MethodArgumentNotValidException.class)
|
@ExceptionHandler(MethodArgumentNotValidException.class)
|
||||||
public ApiResponse<Void> handleMethodArgumentNotValidException(MethodArgumentNotValidException ex) {
|
public ApiResponse<Void> handleMethodArgumentNotValidException(MethodArgumentNotValidException ex) { String message = ex.getBindingResult().getFieldError() != null
|
||||||
String message = ex.getBindingResult().getFieldError() != null
|
|
||||||
? ex.getBindingResult().getFieldError().getDefaultMessage()
|
? ex.getBindingResult().getFieldError().getDefaultMessage()
|
||||||
: "参数校验失败";
|
: "参数校验失败";
|
||||||
return ApiResponse.fail(message);
|
return ApiResponse.fail(message);
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
package com.nanri.aiimage.common.model.entity;
|
package com.nanri.aiimage.common.model.entity;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.annotation.FieldStrategy;
|
||||||
import com.baomidou.mybatisplus.annotation.IdType;
|
import com.baomidou.mybatisplus.annotation.IdType;
|
||||||
import com.baomidou.mybatisplus.annotation.TableField;
|
import com.baomidou.mybatisplus.annotation.TableField;
|
||||||
import com.baomidou.mybatisplus.annotation.TableId;
|
import com.baomidou.mybatisplus.annotation.TableId;
|
||||||
@@ -24,5 +25,13 @@ public class AdminUserEntity {
|
|||||||
private Long createdById;
|
private Long createdById;
|
||||||
@TableField("created_at")
|
@TableField("created_at")
|
||||||
private LocalDateTime createdAt;
|
private LocalDateTime createdAt;
|
||||||
|
/**
|
||||||
|
* 更新时间(V131 新增):由数据库维护(DEFAULT CURRENT_TIMESTAMP + ON UPDATE CURRENT_TIMESTAMP)。
|
||||||
|
*
|
||||||
|
* <p>禁止应用显式写:MySQL 在 UPDATE 语句显式给该列赋值时不会触发自动更新,
|
||||||
|
* 而本表写回多为 selectById → 改字段 → updateById(实体带着旧值),一旦写回就会冻结更新时间。
|
||||||
|
*/
|
||||||
|
@TableField(value = "updated_at", insertStrategy = FieldStrategy.NEVER, updateStrategy = FieldStrategy.NEVER)
|
||||||
|
private LocalDateTime updatedAt;
|
||||||
private String machine;
|
private String machine;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import io.jsonwebtoken.Claims;
|
|||||||
import jakarta.servlet.http.Cookie;
|
import jakarta.servlet.http.Cookie;
|
||||||
import jakarta.servlet.http.HttpServletRequest;
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.http.HttpHeaders;
|
import org.springframework.http.HttpHeaders;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
@@ -19,6 +20,7 @@ import java.nio.charset.StandardCharsets;
|
|||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
@Component
|
@Component
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public class AdminAuthSupport {
|
public class AdminAuthSupport {
|
||||||
@@ -62,8 +64,30 @@ public class AdminAuthSupport {
|
|||||||
return user;
|
return user;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 当前用户必须是管理员或超级管理员,否则抛 403。 */
|
/**
|
||||||
public AdminUserEntity requireAdmin(HttpServletRequest request) {
|
* 解析当前请求 JWT 中**签名的**设备标识(deviceId claim);识别不出时返回空串。
|
||||||
|
*
|
||||||
|
* <p>无 token、token 过期/非法、内部令牌通道调用一律返回空串——调用方必须把空串
|
||||||
|
* 当作"来源不明"做保守判定,绝不据此放宽任何限制。只认签名 claim,不接受
|
||||||
|
* X-Device-Id 请求头(头由客户端可控,见 {@link DeviceSessionPolicy} 类注释)。</p>
|
||||||
|
*
|
||||||
|
* <p>本方法只做识别、不做鉴权,因此解析失败不抛异常,仅记日志后返回空串,
|
||||||
|
* 避免把匿名/内部调用直接升级成 401。</p>
|
||||||
|
*/
|
||||||
|
public String currentDeviceId(HttpServletRequest request) {
|
||||||
|
String token = resolveToken(request);
|
||||||
|
if (token == null || token.isBlank()) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return DeviceSessionPolicy.claimDeviceId(jwtService.parse(token));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[auth] 解析 token 取设备标识失败,按来源不明处理: {}", ex.getMessage());
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 当前用户必须是管理员或超级管理员,否则抛 403。 */ public AdminUserEntity requireAdmin(HttpServletRequest request) {
|
||||||
AdminUserEntity user = requireUser(request);
|
AdminUserEntity user = requireUser(request);
|
||||||
String role = currentRole(user);
|
String role = currentRole(user);
|
||||||
if (role == null) {
|
if (role == null) {
|
||||||
|
|||||||
+61
-6
@@ -13,10 +13,12 @@ import org.springframework.http.ResponseEntity;
|
|||||||
import org.springframework.http.client.SimpleClientHttpRequestFactory;
|
import org.springframework.http.client.SimpleClientHttpRequestFactory;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.util.StreamUtils;
|
import org.springframework.util.StreamUtils;
|
||||||
|
import org.springframework.web.client.ResourceAccessException;
|
||||||
import org.springframework.web.client.RestClient;
|
import org.springframework.web.client.RestClient;
|
||||||
import org.springframework.web.util.ContentCachingRequestWrapper;
|
import org.springframework.web.util.ContentCachingRequestWrapper;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.net.ConnectException;
|
||||||
import java.util.Enumeration;
|
import java.util.Enumeration;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
@@ -28,6 +30,11 @@ public class TaskOwnerForwardService {
|
|||||||
|
|
||||||
public static final String FORWARDED_HEADER = "X-AIIMAGE-Owner-Forwarded";
|
public static final String FORWARDED_HEADER = "X-AIIMAGE-Owner-Forwarded";
|
||||||
|
|
||||||
|
/** 连接类失败的重试次数(含首次)。对端滚动重启时通常几秒内即可恢复。 */
|
||||||
|
private static final int CONNECT_RETRY_TIMES = 3;
|
||||||
|
/** 第 n 次重试前的退避:1s、2s(总等待不超过 3s,不长时间占用请求线程)。 */
|
||||||
|
private static final long CONNECT_RETRY_BACKOFF_MILLIS = 1000L;
|
||||||
|
|
||||||
private static final Set<String> HOP_BY_HOP_HEADERS = Set.of(
|
private static final Set<String> HOP_BY_HOP_HEADERS = Set.of(
|
||||||
"connection",
|
"connection",
|
||||||
"keep-alive",
|
"keep-alive",
|
||||||
@@ -58,12 +65,60 @@ public class TaskOwnerForwardService {
|
|||||||
HttpHeaders headers = copyForwardHeaders(request, ex.getCurrentInstanceId());
|
HttpHeaders headers = copyForwardHeaders(request, ex.getCurrentInstanceId());
|
||||||
log.info("[instance-routing] forward {} taskId={} operation={} owner={} current={} url={}",
|
log.info("[instance-routing] forward {} taskId={} operation={} owner={} current={} url={}",
|
||||||
method, ex.getTaskId(), ex.getOperation(), ex.getOwnerInstanceId(), ex.getCurrentInstanceId(), url);
|
method, ex.getTaskId(), ex.getOperation(), ex.getOwnerInstanceId(), ex.getCurrentInstanceId(), url);
|
||||||
return restClient().method(method)
|
return forwardWithConnectRetry(method, url, headers, body, ex);
|
||||||
.uri(url)
|
}
|
||||||
.headers(target -> target.addAll(headers))
|
|
||||||
.body(body)
|
/**
|
||||||
.retrieve()
|
* 转发带连接级重试。
|
||||||
.toEntity(byte[].class);
|
*
|
||||||
|
* <p>对端实例在部署窗口(原地换 JAR + 两节点滚动重启)内会有几秒的 Connection refused。
|
||||||
|
* 连接都没建立起来说明请求没到达对端,此时重放是安全的;而读超时不重试——对端可能
|
||||||
|
* 已经在处理,盲目重放会造成重复提交。线上由此丢过用户提交的结果。
|
||||||
|
*/
|
||||||
|
private ResponseEntity<byte[]> forwardWithConnectRetry(HttpMethod method, String url,
|
||||||
|
HttpHeaders headers, byte[] body,
|
||||||
|
TaskOwnerMismatchException ex) {
|
||||||
|
RuntimeException lastError = null;
|
||||||
|
for (int attempt = 1; attempt <= CONNECT_RETRY_TIMES; attempt++) {
|
||||||
|
try {
|
||||||
|
return restClient().method(method)
|
||||||
|
.uri(url)
|
||||||
|
.headers(target -> target.addAll(headers))
|
||||||
|
.body(body)
|
||||||
|
.retrieve()
|
||||||
|
.toEntity(byte[].class);
|
||||||
|
} catch (ResourceAccessException accessError) {
|
||||||
|
if (!isConnectFailure(accessError)) {
|
||||||
|
throw accessError;
|
||||||
|
}
|
||||||
|
lastError = accessError;
|
||||||
|
log.warn("[instance-routing] 转发连接失败,第 {}/{} 次 url={} taskId={} 原因={}",
|
||||||
|
attempt, CONNECT_RETRY_TIMES, url, ex.getTaskId(), accessError.getMessage());
|
||||||
|
if (attempt < CONNECT_RETRY_TIMES) {
|
||||||
|
sleepQuietly(CONNECT_RETRY_BACKOFF_MILLIS * attempt);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw lastError;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static boolean isConnectFailure(Throwable error) {
|
||||||
|
Throwable cursor = error;
|
||||||
|
while (cursor != null) {
|
||||||
|
if (cursor instanceof ConnectException) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
cursor = cursor.getCause();
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void sleepQuietly(long millis) {
|
||||||
|
try {
|
||||||
|
Thread.sleep(millis);
|
||||||
|
} catch (InterruptedException interrupted) {
|
||||||
|
Thread.currentThread().interrupt();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private String resolveUrl(TaskOwnerMismatchException ex, String path) {
|
private String resolveUrl(TaskOwnerMismatchException ex, String path) {
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
|||||||
import com.nanri.aiimage.common.api.ApiResponse;
|
import com.nanri.aiimage.common.api.ApiResponse;
|
||||||
import com.nanri.aiimage.common.exception.BusinessException;
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
import com.nanri.aiimage.common.security.AdminAuthSupport;
|
import com.nanri.aiimage.common.security.AdminAuthSupport;
|
||||||
|
import com.nanri.aiimage.common.security.DeviceSessionPolicy;
|
||||||
import jakarta.servlet.FilterChain;
|
import jakarta.servlet.FilterChain;
|
||||||
import jakarta.servlet.ServletException;
|
import jakarta.servlet.ServletException;
|
||||||
import jakarta.servlet.http.HttpServletRequest;
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
@@ -153,7 +154,14 @@ public class AdminApiGuardFilter extends OncePerRequestFilter {
|
|||||||
ApiResponse<Void> body = ex.getCode() == null
|
ApiResponse<Void> body = ex.getCode() == null
|
||||||
? ApiResponse.fail(ex.getMessage())
|
? ApiResponse.fail(ex.getMessage())
|
||||||
: ApiResponse.fail(ex.getCode(), ex.getMessage());
|
: ApiResponse.fail(ex.getCode(), ex.getMessage());
|
||||||
log.warn("[admin-guard] {} {} rejected: {}", request.getMethod(), request.getRequestURI(), ex.getMessage());
|
// 401(登录已过期)与被顶下线是前端定时轮询(/api/notifications/summary、/api/user-secrets 等)
|
||||||
|
// 的常态:线上单节点一天近 3000 条,会把真实业务错误淹没。与 GlobalExceptionHandler
|
||||||
|
// 的 isRoutineAuthNoise 同一口径降为 debug。
|
||||||
|
if (isRoutineAuthNoise(ex.getCode())) {
|
||||||
|
log.debug("[admin-guard] {} {} rejected: {}", request.getMethod(), request.getRequestURI(), ex.getMessage());
|
||||||
|
} else {
|
||||||
|
log.warn("[admin-guard] {} {} rejected: {}", request.getMethod(), request.getRequestURI(), ex.getMessage());
|
||||||
|
}
|
||||||
response.setStatus(HttpServletResponse.SC_OK);
|
response.setStatus(HttpServletResponse.SC_OK);
|
||||||
response.setContentType("application/json;charset=UTF-8");
|
response.setContentType("application/json;charset=UTF-8");
|
||||||
response.getWriter().write(objectMapper.writeValueAsString(body));
|
response.getWriter().write(objectMapper.writeValueAsString(body));
|
||||||
@@ -168,6 +176,12 @@ public class AdminApiGuardFilter extends OncePerRequestFilter {
|
|||||||
chain.doFilter(request, response);
|
chain.doFilter(request, response);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** 登录态过期 / 被其他设备顶下线:前端轮询的常态噪声,不占 WARN。 */
|
||||||
|
private static boolean isRoutineAuthNoise(Integer code) {
|
||||||
|
return Integer.valueOf(401).equals(code)
|
||||||
|
|| Integer.valueOf(DeviceSessionPolicy.CODE_KICKED).equals(code);
|
||||||
|
}
|
||||||
|
|
||||||
/** 命中受保护前缀(/api/admin、/debug、自助接口、用户态工具前缀及其子路径)才进入鉴权,其余请求直接放行。 */
|
/** 命中受保护前缀(/api/admin、/debug、自助接口、用户态工具前缀及其子路径)才进入鉴权,其余请求直接放行。 */
|
||||||
private boolean isGuarded(String uri) {
|
private boolean isGuarded(String uri) {
|
||||||
if (matchesPrefix(uri, ADMIN_API_PREFIX) || matchesPrefix(uri, DEBUG_PREFIX)) {
|
if (matchesPrefix(uri, ADMIN_API_PREFIX) || matchesPrefix(uri, DEBUG_PREFIX)) {
|
||||||
|
|||||||
@@ -23,6 +23,14 @@ public class BrandCheckProperties {
|
|||||||
* 10 次尝试合计约 45s,既能覆盖限流窗口又不会让单个品牌长时间占住查询线程。
|
* 10 次尝试合计约 45s,既能覆盖限流窗口又不会让单个品牌长时间占住查询线程。
|
||||||
*/
|
*/
|
||||||
private int retryMaxIntervalMillis = 10000;
|
private int retryMaxIntervalMillis = 10000;
|
||||||
|
/**
|
||||||
|
* 一次批量检查的总耗时上限(毫秒):整批用尽后不再重试,剩余品牌按「查询失败」收尾。
|
||||||
|
* 上限的意义不是省时间,而是给「分片回传」这类同步调用方一个时延上界——上游卡死时
|
||||||
|
* 单品牌 10 次重试曾把一次回传拖到 103.5 秒(taskId 28599),客户端重试预算耗尽后
|
||||||
|
* 中止了整个采集。首轮请求始终执行,故上游只是略慢时不会误降级。
|
||||||
|
* 设为 0 或负数表示不限制。
|
||||||
|
*/
|
||||||
|
private int totalTimeoutMillis = 90000;
|
||||||
private int connectTimeoutMillis = 10000;
|
private int connectTimeoutMillis = 10000;
|
||||||
private int readTimeoutMillis = 60000;
|
private int readTimeoutMillis = 60000;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -67,4 +67,12 @@ public class NotificationProperties {
|
|||||||
|
|
||||||
/** 已读通知保留天数(超期自动清理),默认 90 天。 */
|
/** 已读通知保留天数(超期自动清理),默认 90 天。 */
|
||||||
private int readRetentionDays = 90;
|
private int readRetentionDays = 90;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 未读通知保留天数,默认 180 天(比已读长一倍)。
|
||||||
|
*
|
||||||
|
* <p>未读通知此前永不清理:不看铃铛的用户会无限累积。保留期给得更宽,
|
||||||
|
* 是因为未读意味着"用户可能还没看到",但也不能永远留着。
|
||||||
|
*/
|
||||||
|
private int unreadRetentionDays = 180;
|
||||||
}
|
}
|
||||||
|
|||||||
+20
-1
@@ -29,7 +29,14 @@ public class TransientStorageProperties {
|
|||||||
*/
|
*/
|
||||||
private long maxTotalConcurrentOperations = 0;
|
private long maxTotalConcurrentOperations = 0;
|
||||||
private long acquirePermitTimeoutMillis = 2000;
|
private long acquirePermitTimeoutMillis = 2000;
|
||||||
private long baseRetryDelayMillis = 500;
|
/**
|
||||||
|
* 首次重试前的基础退避。
|
||||||
|
*
|
||||||
|
* <p>线上高频的重试诱因是 `unexpected end of stream`——那是**立即失败**(连接被 RustFS
|
||||||
|
* 重置后 OkHttp 读响应即报错),不是等超时,所以 500ms 基本是白等:每天上千次累计十几分钟。
|
||||||
|
* 降到 200ms 保留退避语义(真遇到服务端过载仍会退让),又不至于让用户等太久。
|
||||||
|
*/
|
||||||
|
private long baseRetryDelayMillis = 200;
|
||||||
private long maxRetryDelayMillis = 5000;
|
private long maxRetryDelayMillis = 5000;
|
||||||
private long retryJitterMillis = 250;
|
private long retryJitterMillis = 250;
|
||||||
private long failureWindowSeconds = 60;
|
private long failureWindowSeconds = 60;
|
||||||
@@ -37,7 +44,19 @@ public class TransientStorageProperties {
|
|||||||
private long failureCooldownMillis = 10000;
|
private long failureCooldownMillis = 10000;
|
||||||
private int dispatcherMaxRequests = 56;
|
private int dispatcherMaxRequests = 56;
|
||||||
private int dispatcherMaxRequestsPerHost = 56;
|
private int dispatcherMaxRequestsPerHost = 56;
|
||||||
|
/**
|
||||||
|
* 空闲连接保留数。
|
||||||
|
*
|
||||||
|
* <p>2026-09-17 曾试过设 0(彻底不复用)来验证"unexpected end of stream 是复用死连接导致的"
|
||||||
|
* 这一假设——**实测照旧失败**(新容器起来后第一次请求就中招)。至此已排除公网链路、
|
||||||
|
* keepAlive 过长、连接复用三项;用 mc 并发压 200 个小对象也全部成功,说明服务端没问题。
|
||||||
|
* 剩余方向指向 MinIO Java SDK / OkHttp 与 RustFS 的协议细节,故恢复默认的连接复用。
|
||||||
|
*/
|
||||||
private int connectionPoolMaxIdle = 5;
|
private int connectionPoolMaxIdle = 5;
|
||||||
|
/**
|
||||||
|
* 空闲连接在池里的保留时长。曾由 300000 调到 30000 试图减少 unexpected end of stream,
|
||||||
|
* 实测无改善(该现象与连接复用无关,见 {@link #connectionPoolMaxIdle} 的排查记录),故恢复原值。
|
||||||
|
*/
|
||||||
private long connectionPoolKeepAliveMillis = 300000;
|
private long connectionPoolKeepAliveMillis = 300000;
|
||||||
private long warnPayloadBytes = 5L * 1024 * 1024;
|
private long warnPayloadBytes = 5L * 1024 * 1024;
|
||||||
private long maxPayloadBytes = 50L * 1024 * 1024;
|
private long maxPayloadBytes = 50L * 1024 * 1024;
|
||||||
|
|||||||
+3
-1
@@ -76,8 +76,10 @@ public class AdminConsoleController {
|
|||||||
@Operation(summary = "当前登录管理员的可见后台菜单树")
|
@Operation(summary = "当前登录管理员的可见后台菜单树")
|
||||||
public ApiResponse<Map<String, Object>> currentUserMenus(HttpServletRequest request) {
|
public ApiResponse<Map<String, Object>> currentUserMenus(HttpServletRequest request) {
|
||||||
AdminUserEntity operator = adminAuthSupport.requireAdminOrInternal(request);
|
AdminUserEntity operator = adminAuthSupport.requireAdminOrInternal(request);
|
||||||
|
// 补全祖先分组:部分授权用户(只授权了子页面)也要看到「一级分组 + 子页面」层级,
|
||||||
|
// 与超管的菜单组织顺序一致;分组节点无页面路由,不构成权限扩展。
|
||||||
List<PermissionMenuItemVo> menus = permissionMenuService.getUserColumnPermissions(
|
List<PermissionMenuItemVo> menus = permissionMenuService.getUserColumnPermissions(
|
||||||
operator, operator.getId(), PermissionMenuService.MENU_TYPE_ADMIN);
|
operator, operator.getId(), PermissionMenuService.MENU_TYPE_ADMIN, true);
|
||||||
menus = AdminMenuTreeBuilder.filterByMenuType(menus, PermissionMenuService.MENU_TYPE_ADMIN);
|
menus = AdminMenuTreeBuilder.filterByMenuType(menus, PermissionMenuService.MENU_TYPE_ADMIN);
|
||||||
List<Map<String, Object>> items = AdminMenuTreeBuilder.toMapList(AdminMenuTreeBuilder.build(menus));
|
List<Map<String, Object>> items = AdminMenuTreeBuilder.toMapList(AdminMenuTreeBuilder.build(menus));
|
||||||
return ApiResponse.success(Map.of("items", items));
|
return ApiResponse.success(Map.of("items", items));
|
||||||
|
|||||||
+2
@@ -16,6 +16,8 @@ public class AdminUserItemVo {
|
|||||||
private String creatorUsername;
|
private String creatorUsername;
|
||||||
@JsonProperty("created_at")
|
@JsonProperty("created_at")
|
||||||
private String createdAt;
|
private String createdAt;
|
||||||
|
@JsonProperty("updated_at")
|
||||||
|
private String updatedAt;
|
||||||
@JsonProperty("pinyin_abbr")
|
@JsonProperty("pinyin_abbr")
|
||||||
private String pinyinAbbr;
|
private String pinyinAbbr;
|
||||||
}
|
}
|
||||||
|
|||||||
+2
@@ -338,6 +338,8 @@ public class AdminUserService {
|
|||||||
vo.setCreatorUsername(creatorId == null ? "" : creatorMap.getOrDefault(creatorId, ""));
|
vo.setCreatorUsername(creatorId == null ? "" : creatorMap.getOrDefault(creatorId, ""));
|
||||||
LocalDateTime createdAt = entity.getCreatedAt();
|
LocalDateTime createdAt = entity.getCreatedAt();
|
||||||
vo.setCreatedAt(createdAt == null ? "" : createdAt.format(CREATED_AT_FORMATTER));
|
vo.setCreatedAt(createdAt == null ? "" : createdAt.format(CREATED_AT_FORMATTER));
|
||||||
|
LocalDateTime updatedAt = entity.getUpdatedAt();
|
||||||
|
vo.setUpdatedAt(updatedAt == null ? "" : updatedAt.format(CREATED_AT_FORMATTER));
|
||||||
vo.setPinyinAbbr(PinyinAbbrUtil.abbr(entity.getUsername()));
|
vo.setPinyinAbbr(PinyinAbbrUtil.abbr(entity.getUsername()));
|
||||||
return vo;
|
return vo;
|
||||||
}
|
}
|
||||||
|
|||||||
+105
-4
@@ -424,6 +424,53 @@ public class AppearancePatentTaskService {
|
|||||||
|
|
||||||
public void submitResult(Long taskId, AppearancePatentSubmitResultRequest request) {
|
public void submitResult(Long taskId, AppearancePatentSubmitResultRequest request) {
|
||||||
submitResultLocked(taskId, request);
|
submitResultLocked(taskId, request);
|
||||||
|
// 分片补传后自动恢复终态失败的组装 job —— 走与删除品牌同一套口径,
|
||||||
|
// 此前外观专利没有接该入口,分片补齐后只能人工重置 job(线上任务 28459 即如此)。
|
||||||
|
maybeRecoverTerminalFailedAssemble(taskId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 补传恢复:此前因分片缺失导致组装 job 重试耗尽(终态失败),客户端补传缺口后
|
||||||
|
* 把失败的组装 job 重置为 PENDING 重新派发({@code resetTerminalFailedForRecovery} 自行补发 dispatch 事件)。
|
||||||
|
*
|
||||||
|
* <p>best-effort:恢复失败不得影响补传本身——分片已经落库,恢复只是让后续组装继续推进。
|
||||||
|
* 常态(无终态失败 job)下只查两次即返回,不触发分片扫描。
|
||||||
|
*/
|
||||||
|
private void maybeRecoverTerminalFailedAssemble(Long taskId) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
FileResultEntity result = findResultRecord(taskId);
|
||||||
|
if (result == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (taskFileJobService.hasSuccessfulAssembleJob(taskId, MODULE_TYPE, result.getId())) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!taskFileJobService.isTerminalFailedAssembleJob(taskId, MODULE_TYPE, result.getId())) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!isResultSubmissionComplete(taskId)) {
|
||||||
|
log.info("[appearance-patent] 组装 job 终态失败但分片仍未补传完整,暂不恢复 taskId={} resultId={}",
|
||||||
|
taskId, result.getId());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
log.info("[appearance-patent] 分片已补传完整,恢复终态失败的组装 job taskId={} resultId={}",
|
||||||
|
taskId, result.getId());
|
||||||
|
taskFileJobService.resetTerminalFailedForRecovery(taskId, MODULE_TYPE, result.getId());
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[appearance-patent] 补传恢复检查失败(不影响本次补传)taskId={} err={}", taskId, ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 按 task 取结果行(不创建);不存在返回 null。 */
|
||||||
|
private FileResultEntity findResultRecord(Long taskId) {
|
||||||
|
List<FileResultEntity> rows = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
||||||
|
.eq(FileResultEntity::getTaskId, taskId)
|
||||||
|
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.last("limit 1"));
|
||||||
|
return rows == null || rows.isEmpty() ? null : rows.getFirst();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void submitResultLocked(Long taskId, AppearancePatentSubmitResultRequest request) {
|
private void submitResultLocked(Long taskId, AppearancePatentSubmitResultRequest request) {
|
||||||
@@ -1065,6 +1112,14 @@ public class AppearancePatentTaskService {
|
|||||||
long activeAssembleJobs = taskFileJobService.countActiveAssembleJobs(taskId, MODULE_TYPE);
|
long activeAssembleJobs = taskFileJobService.countActiveAssembleJobs(taskId, MODULE_TYPE);
|
||||||
log.info("[appearance-patent] stale recovery probe taskId={} uploadComplete={} activeAssembleJobs={} persistedRows={}",
|
log.info("[appearance-patent] stale recovery probe taskId={} uploadComplete={} activeAssembleJobs={} persistedRows={}",
|
||||||
taskId, uploadComplete, activeAssembleJobs, hasPersistedResultRows(taskId));
|
taskId, uploadComplete, activeAssembleJobs, hasPersistedResultRows(taskId));
|
||||||
|
// 已有「重试耗尽且已终态收尾」的 assemble job:说明恢复已经试过、缺失是永久的。
|
||||||
|
// 再重建只会每 30 秒空转一轮,而且恢复过程刷新任务心跳会让任务永远 RUNNING
|
||||||
|
// (线上任务 28459 实测:48 分钟里每隔 30 秒重建一次 job)。返回 false 交给
|
||||||
|
// finalizeStaleTask 按失败收尾,用户看到明确失败而不是无限等待。
|
||||||
|
if (taskFileJobService.hasExhaustedAssembleJob(taskId, MODULE_TYPE)) {
|
||||||
|
log.warn("[appearance-patent] stale recovery 放弃:已有重试耗尽的 assemble job,按失败收尾 taskId={}", taskId);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
if (!hasPersistedResultRows(taskId)) {
|
if (!hasPersistedResultRows(taskId)) {
|
||||||
log.info("[appearance-patent] stale recovery aborted because no persisted rows taskId={}", taskId);
|
log.info("[appearance-patent] stale recovery aborted because no persisted rows taskId={}", taskId);
|
||||||
return false;
|
return false;
|
||||||
@@ -1215,6 +1270,7 @@ public class AppearancePatentTaskService {
|
|||||||
if (rows == null || rows.isEmpty()) {
|
if (rows == null || rows.isEmpty()) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
String conflictDetail = "未发生冲突";
|
||||||
for (int attempt = 1; attempt <= CHUNK_PAYLOAD_MERGE_RETRY_LIMIT; attempt++) {
|
for (int attempt = 1; attempt <= CHUNK_PAYLOAD_MERGE_RETRY_LIMIT; attempt++) {
|
||||||
TaskChunkEntity chunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
TaskChunkEntity chunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
@@ -1256,13 +1312,35 @@ public class AppearancePatentTaskService {
|
|||||||
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(oldPayload, storedPayload);
|
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(oldPayload, storedPayload);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
|
// CAS 冲突:读回行上的当前哈希,否则只有一句 conflict 无从判断是谁改的(线上 28459 至今未能定位)
|
||||||
|
String currentHash = currentPayloadHash(chunk.getId());
|
||||||
|
conflictDetail = "期望hash=" + oldPayloadHash + " 当前hash=" + currentHash;
|
||||||
if (attempt < CHUNK_PAYLOAD_MERGE_RETRY_LIMIT) {
|
if (attempt < CHUNK_PAYLOAD_MERGE_RETRY_LIMIT) {
|
||||||
log.warn("[appearance-patent] chunk payload CAS conflict, retry merge taskId={} scopeHash={} chunk={} attempt={}/{}",
|
log.warn("[appearance-patent] chunk payload CAS conflict, retry merge taskId={} scopeHash={} chunk={} attempt={}/{} 期望hash={} 当前hash={}",
|
||||||
taskId, scopeHash, chunkIndex, attempt, CHUNK_PAYLOAD_MERGE_RETRY_LIMIT);
|
taskId, scopeHash, chunkIndex, attempt, CHUNK_PAYLOAD_MERGE_RETRY_LIMIT, oldPayloadHash, currentHash);
|
||||||
|
// 还要重试:这次写的对象会被下次重写,先删掉避免堆积
|
||||||
|
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
|
||||||
|
} else {
|
||||||
|
// 终局失败:保留这次写入的版本化对象,作为读路径「同槽位兄弟对象」兜底的恢复源。
|
||||||
|
// 行没指过去不该让该分片永久判死——删掉它才是线上 28459 丢数据的形态。
|
||||||
|
log.error("[appearance-patent] chunk payload 合并冲突重试耗尽,保留兜底对象 taskId={} scopeHash={} chunk={} 期望hash={} 当前hash={} 保留对象={}",
|
||||||
|
taskId, scopeHash, chunkIndex, oldPayloadHash, currentHash, storedPayload);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
throw new IllegalStateException("appearance patent chunk payload update conflict");
|
throw new IllegalStateException("appearance patent chunk payload update conflict " + conflictDetail);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 读回 chunk 行当前的 payload_hash,用于 CAS 冲突定位(行已不存在/读取失败时返回可读标记)。 */
|
||||||
|
private String currentPayloadHash(Long chunkId) {
|
||||||
|
if (chunkId == null) {
|
||||||
|
return "chunkId 为空";
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
TaskChunkEntity latest = taskChunkMapper.selectById(chunkId);
|
||||||
|
return latest == null ? "行已不存在" : latest.getPayloadHash();
|
||||||
|
} catch (Exception ex) {
|
||||||
|
return "读取失败:" + ex.getMessage();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<AppearancePatentResultRowDto> expandRows(List<AppearancePatentResultRowDto> representatives,
|
private List<AppearancePatentResultRowDto> expandRows(List<AppearancePatentResultRowDto> representatives,
|
||||||
@@ -2915,12 +2993,35 @@ public class AppearancePatentTaskService {
|
|||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
log.warn("[appearance-patent] read chunk payload failed taskId={} chunk={} err={}",
|
log.warn("[appearance-patent] read chunk payload failed taskId={} chunk={} err={}",
|
||||||
chunk.getTaskId(), chunk.getChunkIndex(), ex.getMessage());
|
chunk.getTaskId(), chunk.getChunkIndex(), ex.getMessage());
|
||||||
|
if (isPayloadMissing(ex)) {
|
||||||
|
// payload 对象已不在(被清理或从未写入):重试多少次都读不回来。继续抛会让
|
||||||
|
// ASSEMBLE_RESULT job 的终态回调每轮重跑兜底组装 → 再读同一个缺失对象 → 无限循环
|
||||||
|
// (线上任务 28459 每 10~30 秒重试一次)。跳过该分片,让任务按已有分片出部分结果,
|
||||||
|
// 与品牌/相似ASIN「失败也产出可下载的部分结果」同一口径。
|
||||||
|
log.warn("[appearance-patent] chunk payload 已不存在,跳过该分片(任务按已有分片出结果)"
|
||||||
|
+ " taskId={} chunk={}", chunk.getTaskId(), chunk.getChunkIndex());
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
throw new BusinessException("appearance patent chunk payload read failed chunk="
|
throw new BusinessException("appearance patent chunk payload read failed chunk="
|
||||||
+ chunk.getChunkIndex() + ": " + ex.getMessage(), ex);
|
+ chunk.getChunkIndex() + ": " + ex.getMessage(), ex);
|
||||||
}
|
}
|
||||||
return rows;
|
return rows;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** payload 对象已不存在(RustFS 返回 NoSuchKey:message 为 "The specified key does not exist.")。
|
||||||
|
* 只有这种"重试也没用"的缺失才允许跳过;网络类失败仍照旧抛出以便重试。 */
|
||||||
|
private static boolean isPayloadMissing(Throwable error) {
|
||||||
|
Throwable cursor = error;
|
||||||
|
while (cursor != null) {
|
||||||
|
String message = cursor.getMessage();
|
||||||
|
if (message != null && message.contains("does not exist")) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
cursor = cursor.getCause();
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
private String rowKey(AppearancePatentParsedRowVo row) {
|
private String rowKey(AppearancePatentParsedRowVo row) {
|
||||||
if (row == null) {
|
if (row == null) {
|
||||||
return "";
|
return "";
|
||||||
|
|||||||
+14
-2
@@ -95,10 +95,14 @@ public class BrandCheckClient {
|
|||||||
|
|
||||||
public BrandCheckBatchResult checkAll(List<String> brands, String strategy) {
|
public BrandCheckBatchResult checkAll(List<String> brands, String strategy) {
|
||||||
List<String> distinctBrands = distinctNonBlank(brands);
|
List<String> distinctBrands = distinctNonBlank(brands);
|
||||||
|
// 整批共用一个耗时预算:上游 16890 卡死时,单品牌 10 次重试曾把一次分片回传拖到
|
||||||
|
// 103.5 秒(taskId 28599),客户端重试预算耗尽后中止了整个采集。预算用尽即停止重试。
|
||||||
|
long budgetMillis = properties.getTotalTimeoutMillis();
|
||||||
|
long deadlineNanos = budgetMillis > 0L ? System.nanoTime() + budgetMillis * 1_000_000L : Long.MAX_VALUE;
|
||||||
List<CompletableFuture<BrandCheckOutcome>> futures = new ArrayList<>(distinctBrands.size());
|
List<CompletableFuture<BrandCheckOutcome>> futures = new ArrayList<>(distinctBrands.size());
|
||||||
for (String brand : distinctBrands) {
|
for (String brand : distinctBrands) {
|
||||||
futures.add(CompletableFuture.supplyAsync(
|
futures.add(CompletableFuture.supplyAsync(
|
||||||
() -> checkOneBrand(brand, strategy), checkExecutor));
|
() -> checkOneBrand(brand, strategy, deadlineNanos), checkExecutor));
|
||||||
}
|
}
|
||||||
List<Object> failedData = new ArrayList<>();
|
List<Object> failedData = new ArrayList<>();
|
||||||
List<Object> queryFailedData = new ArrayList<>();
|
List<Object> queryFailedData = new ArrayList<>();
|
||||||
@@ -110,11 +114,19 @@ public class BrandCheckClient {
|
|||||||
return new BrandCheckBatchResult(distinctBrands, failedData, queryFailedData);
|
return new BrandCheckBatchResult(distinctBrands, failedData, queryFailedData);
|
||||||
}
|
}
|
||||||
|
|
||||||
private BrandCheckOutcome checkOneBrand(String brand, String strategy) {
|
private BrandCheckOutcome checkOneBrand(String brand, String strategy, long deadlineNanos) {
|
||||||
int attempts = Math.max(1, properties.getRetryTimes());
|
int attempts = Math.max(1, properties.getRetryTimes());
|
||||||
BrandCheckResponse response = null;
|
BrandCheckResponse response = null;
|
||||||
Exception lastFailure = null;
|
Exception lastFailure = null;
|
||||||
for (int attempt = 1; attempt <= attempts; attempt++) {
|
for (int attempt = 1; attempt <= attempts; attempt++) {
|
||||||
|
// 预算用尽就不再重试,按查询失败收尾。只掐「重试」不打断已发出的请求,
|
||||||
|
// 故最坏耗时 ≈ 预算 + 一次请求的读超时;首轮始终执行,避免上游只是慢一点时被误降级。
|
||||||
|
if (attempt > 1 && System.nanoTime() >= deadlineNanos) {
|
||||||
|
log.warn("[brand-check] 整批耗时预算用尽,停止重试 brand={} attempt={}/{} lastErr={}",
|
||||||
|
brand, attempt, attempts,
|
||||||
|
lastFailure == null ? "query_faild_data 持续非空" : lastFailure.getMessage());
|
||||||
|
break;
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
response = check(brand, strategy);
|
response = check(brand, strategy);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
|
|||||||
+24
@@ -3,7 +3,31 @@ package com.nanri.aiimage.modules.brand.mapper;
|
|||||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||||
import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity;
|
import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity;
|
||||||
import org.apache.ibatis.annotations.Mapper;
|
import org.apache.ibatis.annotations.Mapper;
|
||||||
|
import org.apache.ibatis.annotations.Param;
|
||||||
|
import org.apache.ibatis.annotations.Select;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
@Mapper
|
@Mapper
|
||||||
public interface BrandCrawlTaskMapper extends BaseMapper<BrandCrawlTaskEntity> {
|
public interface BrandCrawlTaskMapper extends BaseMapper<BrandCrawlTaskEntity> {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 查询超过保留期的终态品牌检测任务 id(保留期清理用,只取 id 不拉整行——历史行的
|
||||||
|
* file_paths/result_paths JSON 字段可能很大)。
|
||||||
|
*
|
||||||
|
* <p>终态集合与 {@code BrandTaskService} 的状态机一致(success/failed/cancelled),
|
||||||
|
* pending/running 绝不返回(删了正在跑的任务,结果回传会找不到任务行)。
|
||||||
|
* 时间线用 updated_at,与 BrandTaskStaleRepairSpiImpl 的陈旧判定同款口径,
|
||||||
|
* 可命中 V120 的 idx_brand_crawl_task_status_updated(status, updated_at) 索引。
|
||||||
|
*/
|
||||||
|
@Select("""
|
||||||
|
SELECT id FROM brand_crawl_tasks
|
||||||
|
WHERE status IN ('success', 'failed', 'cancelled')
|
||||||
|
AND updated_at < #{cutoff}
|
||||||
|
ORDER BY id ASC
|
||||||
|
LIMIT #{batchSize}
|
||||||
|
""")
|
||||||
|
List<Long> selectExpiredTerminalTaskIds(@Param("cutoff") LocalDateTime cutoff,
|
||||||
|
@Param("batchSize") int batchSize);
|
||||||
}
|
}
|
||||||
|
|||||||
+93
@@ -0,0 +1,93 @@
|
|||||||
|
package com.nanri.aiimage.modules.brand.service;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||||
|
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.scheduling.annotation.Scheduled;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 品牌检测任务(brand_crawl_tasks)的保留期清理(2026-09 审核:该表只增不删,永久累积)。
|
||||||
|
*
|
||||||
|
* <p>该表自建、不写 biz_file_task,故不在 ModuleHistoryCleanupService 的清理名单里,
|
||||||
|
* 此前没有任何删除路径。这里只查过期终态任务的 id,逐个走
|
||||||
|
* {@link BrandTaskService#deleteTask(Long)} 既有删除入口 —— 它已处理任务行删除 +
|
||||||
|
* 存储数据清理(brandTaskStorageService.deleteTaskData)+ 进度缓存清理,本类不重新实现删除逻辑。
|
||||||
|
*
|
||||||
|
* <p>双节点用 job 锁保证单实例执行;每批小批量(默认 50)逐个删,避免单次跑太久占住锁。
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
public class BrandTaskRetentionService {
|
||||||
|
|
||||||
|
/** 单轮最多处理的批次数(每批 batchSize 个任务),剩余留给下一轮。 */
|
||||||
|
static final int MAX_BATCHES_PER_RUN = 20;
|
||||||
|
|
||||||
|
private final BrandCrawlTaskMapper brandCrawlTaskMapper;
|
||||||
|
private final BrandTaskService brandTaskService;
|
||||||
|
private final DistributedJobLockService distributedJobLockService;
|
||||||
|
|
||||||
|
@Value("${aiimage.brand.task-retention-days:90}")
|
||||||
|
private int retentionDays = 90;
|
||||||
|
|
||||||
|
@Value("${aiimage.brand.task-retention-batch-size:50}")
|
||||||
|
private int retentionBatchSize = 50;
|
||||||
|
|
||||||
|
@Scheduled(cron = "${aiimage.brand.task-retention-cron:0 45 4 * * *}")
|
||||||
|
public void purgeExpiredTasks() {
|
||||||
|
int days = Math.max(1, retentionDays);
|
||||||
|
int batchSize = Math.max(1, retentionBatchSize);
|
||||||
|
LocalDateTime cutoff = LocalDateTime.now().minusDays(days);
|
||||||
|
|
||||||
|
// 单轮 20 批 × 50 个任务的删除(含存储数据清理)可能跑较久,锁 TTL 给足 30 分钟
|
||||||
|
DistributedJobLockService.LockHandle lockHandle =
|
||||||
|
distributedJobLockService.tryLock("brand:task-retention", Duration.ofMinutes(30));
|
||||||
|
if (lockHandle == null) {
|
||||||
|
log.info("[brand-retention] 任务保留清理跳过:另一实例持锁");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try (lockHandle) {
|
||||||
|
int totalDeleted = 0;
|
||||||
|
int totalFailed = 0;
|
||||||
|
int batches = 0;
|
||||||
|
while (batches < MAX_BATCHES_PER_RUN) {
|
||||||
|
// 只取 id:历史行的 file_paths/result_paths JSON 字段可能很大
|
||||||
|
List<Long> taskIds = brandCrawlTaskMapper.selectExpiredTerminalTaskIds(cutoff, batchSize);
|
||||||
|
if (taskIds.isEmpty()) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
batches++;
|
||||||
|
int deletedInBatch = 0;
|
||||||
|
for (Long taskId : taskIds) {
|
||||||
|
try {
|
||||||
|
brandTaskService.deleteTask(taskId);
|
||||||
|
deletedInBatch++;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
// 单个任务删除失败只记日志继续:一个坏任务不能卡住整轮
|
||||||
|
log.warn("[brand-retention] 任务删除失败 taskId={} msg={}", taskId, ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
totalDeleted += deletedInBatch;
|
||||||
|
totalFailed += taskIds.size() - deletedInBatch;
|
||||||
|
if (deletedInBatch == 0) {
|
||||||
|
log.warn("[brand-retention] 整批任务均删除失败,提前结束本轮以免反复重试同一批");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (taskIds.size() < batchSize) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.info("[brand-retention] 任务保留清理完成 cutoff={} retentionDays={} deleted={} failed={} batches={}",
|
||||||
|
cutoff, days, totalDeleted, totalFailed, batches);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[brand-retention] 任务保留清理失败 cutoff={} msg={}", cutoff, ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+79
-36
@@ -782,6 +782,39 @@ public class CollectDataService {
|
|||||||
throw new BusinessException("request is empty");
|
throw new BusinessException("request is empty");
|
||||||
}
|
}
|
||||||
ensureRustfsPayloadStorageEnabled();
|
ensureRustfsPayloadStorageEnabled();
|
||||||
|
|
||||||
|
// 锁外预检:任务不存在/已结束时立即失败,不为终态任务白跑去重查询与品牌检测。
|
||||||
|
// 只做快速失败,并发正确性仍由锁内的重读复核保证。
|
||||||
|
FileTaskEntity probe = fileTaskMapper.selectById(taskId);
|
||||||
|
if (probe == null || !MODULE_TYPE.equals(probe.getModuleType())) {
|
||||||
|
throw new BusinessException("任务不存在");
|
||||||
|
}
|
||||||
|
if (STATUS_SUCCESS.equals(probe.getStatus()) || STATUS_FAILED.equals(probe.getStatus())) {
|
||||||
|
log.warn("[collect-data] 任务已结束,拒绝重复提交 taskId={} status={}", taskId, probe.getStatus());
|
||||||
|
throw new BusinessException(40901, "任务已结束,拒绝重复提交");
|
||||||
|
}
|
||||||
|
|
||||||
|
// 归一化 / 去重过滤 / 品牌检测放在锁外:品牌检测是同步远程调用,上游 16890 抖动时
|
||||||
|
// 单品牌 10 次重试合计上百秒(taskId 28599 实测:chunk 回传在锁内等品牌检测 103.5 秒,
|
||||||
|
// 期间心跳与客户端重试全部撞 40902 拿不到锁,客户端 5 次重试预算耗尽后中止整个采集)。
|
||||||
|
// 这几步只依赖本批入参、不写任务状态,放锁外不改变 chunk 落库的串行语义。
|
||||||
|
List<CollectDataResultRowVo> rows = normalizeSubmitRows(request.getItems());
|
||||||
|
buildParseLimits().validateChunkRowCount(rows.size());
|
||||||
|
// 「结果文件」sheet 需要 Python 回传的全量数据(不经任何后端过滤),
|
||||||
|
// 因此 chunk payload 写入未筛除空 ASIN 的全量 rows;
|
||||||
|
// 仅当 ASIN 非空时才进入去重 / 无效品牌 / 品牌检测链路并落 biz_task_result_item。
|
||||||
|
List<CollectDataResultRowVo> rowsForFiltering = new ArrayList<>(rows.size());
|
||||||
|
for (CollectDataResultRowVo row : rows) {
|
||||||
|
if (row.getAsin() != null && !row.getAsin().isBlank()) {
|
||||||
|
rowsForFiltering.add(row);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
long prepareStartAt = System.currentTimeMillis();
|
||||||
|
CollectDataBatchQuery.FilterResult filtered = collectDataBatchQuery.filter(rowsForFiltering);
|
||||||
|
CollectDataBrandBatchFilter.BrandBatchOutcome brandOutcome = brandBatchFilter.filter(filtered.kept());
|
||||||
|
log.info("[collect-data] 锁外预处理完成 taskId={} rows={} 去重后={} 品牌检测耗时={}ms",
|
||||||
|
taskId, rows.size(), filtered.kept().size(), System.currentTimeMillis() - prepareStartAt);
|
||||||
|
|
||||||
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
|
try (TaskDistributedLockService.LockHandle ignored = acquireTaskLockOrThrow(taskId)) {
|
||||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
||||||
@@ -819,25 +852,23 @@ public class CollectDataService {
|
|||||||
return buildSubmitVo(task, result, chunkIndex, chunkTotal, request, 0);
|
return buildSubmitVo(task, result, chunkIndex, chunkTotal, request, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
List<CollectDataResultRowVo> rows = normalizeSubmitRows(request.getItems());
|
|
||||||
buildParseLimits().validateChunkRowCount(rows.size());
|
|
||||||
CollectDataStats stats = loadStats(task);
|
CollectDataStats stats = loadStats(task);
|
||||||
stats.receivedRows += rows.size();
|
stats.receivedRows += rows.size();
|
||||||
stats.currentChunkRows = rows.size();
|
stats.currentChunkRows = rows.size();
|
||||||
|
|
||||||
// 「结果文件」sheet 需要 Python 回传的全量数据(不经任何后端过滤),
|
|
||||||
// 因此 chunk payload 写入未筛除空 ASIN 的全量 rows;
|
|
||||||
// 仅当 ASIN 非空时才进入去重 / 无效品牌 / 品牌检测链路并落 biz_task_result_item。
|
|
||||||
List<CollectDataResultRowVo> rowsForFiltering = new ArrayList<>(rows.size());
|
|
||||||
for (CollectDataResultRowVo row : rows) {
|
|
||||||
if (row.getAsin() != null && !row.getAsin().isBlank()) {
|
|
||||||
rowsForFiltering.add(row);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
CollectDataBatchQuery.FilterResult filtered = collectDataBatchQuery.filter(rowsForFiltering);
|
|
||||||
stats.dedupeFilteredCount += filtered.dedupeFilteredCount();
|
stats.dedupeFilteredCount += filtered.dedupeFilteredCount();
|
||||||
stats.invalidFilteredCount += filtered.invalidFilteredCount();
|
stats.invalidFilteredCount += filtered.invalidFilteredCount();
|
||||||
List<CollectDataResultRowVo> accepted = filterByBrandCheck(filtered.kept(), stats);
|
stats.brandRejectedCount += brandOutcome.rejected().size();
|
||||||
|
stats.brandQueryFailedCount += brandOutcome.queryFailed().size();
|
||||||
|
// 只有被服务端判定为无效品牌(rejected)的行才写入无效品牌表;
|
||||||
|
// queryFailed 是品牌检测服务端到端失败(如缺 X-Token 422/超时/限流),
|
||||||
|
// 一并写入会把故障期间被误伤的品牌永久拉黑,后续同品牌商品全部被
|
||||||
|
// invalidFiltered 过滤(task-27265 实测:通过 Python 过滤的 21 行中
|
||||||
|
// 8 行查询失败被写表后误杀,最终结果 Excel 0 行)。rejected 为空时
|
||||||
|
// 不触发任何写入,避免空批次无意义调用。
|
||||||
|
if (!brandOutcome.rejected().isEmpty()) {
|
||||||
|
invalidAsinBatchWriter.writeBatch(brandOutcome.rejected());
|
||||||
|
}
|
||||||
|
List<CollectDataResultRowVo> accepted = brandOutcome.accepted();
|
||||||
// 结果明细改为 chunk 级存储:整个 chunk 的 accepted 行共享一个
|
// 结果明细改为 chunk 级存储:整个 chunk 的 accepted 行共享一个
|
||||||
// RustFS 对象(deterministic key,同 chunk 重提覆盖同一对象),
|
// RustFS 对象(deterministic key,同 chunk 重提覆盖同一对象),
|
||||||
// biz_task_result_item.payload_json 只存 {chunk, offset, payload} 引用。
|
// biz_task_result_item.payload_json 只存 {chunk, offset, payload} 引用。
|
||||||
@@ -872,6 +903,14 @@ public class CollectDataService {
|
|||||||
|
|
||||||
if (request.getError() != null && !request.getError().isBlank()) {
|
if (request.getError() != null && !request.getError().isBlank()) {
|
||||||
markTaskFailed(task, result, request.getError(), stats);
|
markTaskFailed(task, result, request.getError(), stats);
|
||||||
|
// 失败但已收到分片:照常组装结果文件,让用户能下载已采集的数据。
|
||||||
|
// 此前失败分支只标失败不组装,已落库的数据也没有任何结果文件可下载
|
||||||
|
// (taskId 28599:55 个分片全部收到、187 行明细已落库,用户却拿不到文件)。
|
||||||
|
if (hasReceivedChunks(taskId)) {
|
||||||
|
enqueueFinalWorkbook(task, result, stats);
|
||||||
|
log.warn("[collect-data] 任务失败仍组装部分结果 taskId={} error={} finalRows={}",
|
||||||
|
taskId, request.getError(), stats.finalRowCount);
|
||||||
|
}
|
||||||
} else if (Boolean.TRUE.equals(request.getDone())) {
|
} else if (Boolean.TRUE.equals(request.getDone())) {
|
||||||
enqueueFinalWorkbook(task, result, stats);
|
enqueueFinalWorkbook(task, result, stats);
|
||||||
} else {
|
} else {
|
||||||
@@ -933,22 +972,6 @@ public class CollectDataService {
|
|||||||
return rows;
|
return rows;
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<CollectDataResultRowVo> filterByBrandCheck(List<CollectDataResultRowVo> rows, CollectDataStats stats) {
|
|
||||||
CollectDataBrandBatchFilter.BrandBatchOutcome outcome = brandBatchFilter.filter(rows);
|
|
||||||
stats.brandRejectedCount += outcome.rejected().size();
|
|
||||||
stats.brandQueryFailedCount += outcome.queryFailed().size();
|
|
||||||
// 只有被服务端判定为无效品牌(rejected)的行才写入无效品牌表;
|
|
||||||
// queryFailed 是品牌检测服务端到端失败(如缺 X-Token 422/超时/限流),
|
|
||||||
// 一并写入会把故障期间被误伤的品牌永久拉黑,后续同品牌商品全部被
|
|
||||||
// invalidFiltered 过滤(task-27265 实测:通过 Python 过滤的 21 行中
|
|
||||||
// 8 行查询失败被写表后误杀,最终结果 Excel 0 行)。rejected 为空时
|
|
||||||
// 不触发任何写入,避免空批次无意义调用。
|
|
||||||
if (!outcome.rejected().isEmpty()) {
|
|
||||||
invalidAsinBatchWriter.writeBatch(outcome.rejected());
|
|
||||||
}
|
|
||||||
return outcome.accepted();
|
|
||||||
}
|
|
||||||
|
|
||||||
private void persistChunk(Long taskId,
|
private void persistChunk(Long taskId,
|
||||||
String scopeKey,
|
String scopeKey,
|
||||||
String scopeHash,
|
String scopeHash,
|
||||||
@@ -1011,7 +1034,8 @@ public class CollectDataService {
|
|||||||
result.setResultFileSize(0L);
|
result.setResultFileSize(0L);
|
||||||
result.setResultContentType(CONTENT_TYPE_XLSX);
|
result.setResultContentType(CONTENT_TYPE_XLSX);
|
||||||
result.setRowCount(stats.finalRowCount);
|
result.setRowCount(stats.finalRowCount);
|
||||||
result.setErrorMessage(null);
|
// 不清 errorMessage:失败任务的部分结果组装也走这里,清掉会让用户看不到真实失败原因
|
||||||
|
// (成功路径的 errorMessage 本来就为 null,无需清理)。
|
||||||
fileResultMapper.updateById(result);
|
fileResultMapper.updateById(result);
|
||||||
task.setUpdatedAt(LocalDateTime.now());
|
task.setUpdatedAt(LocalDateTime.now());
|
||||||
fileTaskMapper.updateById(task);
|
fileTaskMapper.updateById(task);
|
||||||
@@ -1060,6 +1084,22 @@ public class CollectDataService {
|
|||||||
stats.summaries,
|
stats.summaries,
|
||||||
batch -> streamRawRows(task.getId(), batch));
|
batch -> streamRawRows(task.getId(), batch));
|
||||||
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
|
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
|
||||||
|
|
||||||
|
// 复用同一份 stats 更新 finalRowCount 后再持久化,避免重复 loadStats 丢失 summaries。
|
||||||
|
stats.finalRowCount = (int) finalRowCount;
|
||||||
|
persistStats(task, stats);
|
||||||
|
|
||||||
|
// 失败原因先留存:下面的乐观写入会清空 result.errorMessage,任务已被判失败时要用它恢复。
|
||||||
|
String failureReason = result.getErrorMessage();
|
||||||
|
if (failureReason == null || failureReason.isBlank()) {
|
||||||
|
failureReason = task.getErrorMessage();
|
||||||
|
}
|
||||||
|
if (failureReason == null || failureReason.isBlank()) {
|
||||||
|
failureReason = "任务失败,结果文件为已采集的部分数据";
|
||||||
|
}
|
||||||
|
|
||||||
|
// 结果行先按成功乐观写入:保持「结果文件先于任务成功落库」的时序,
|
||||||
|
// 万一进程在这两步之间退出,任务仍是 RUNNING,会被陈旧巡检重新组装(可自愈)。
|
||||||
result.setResultFilename(filename);
|
result.setResultFilename(filename);
|
||||||
result.setResultFileUrl(objectKey);
|
result.setResultFileUrl(objectKey);
|
||||||
result.setResultFileSize(xlsx.length());
|
result.setResultFileSize(xlsx.length());
|
||||||
@@ -1069,10 +1109,7 @@ public class CollectDataService {
|
|||||||
result.setErrorMessage(null);
|
result.setErrorMessage(null);
|
||||||
fileResultMapper.updateById(result);
|
fileResultMapper.updateById(result);
|
||||||
|
|
||||||
// 复用同一份 stats 更新 finalRowCount 后再持久化,避免重复 loadStats 丢失 summaries。
|
// 条件更新:任务可能已被判失败(客户端上报失败 / 陈旧判死与结果文件组装并发)——
|
||||||
stats.finalRowCount = (int) finalRowCount;
|
|
||||||
persistStats(task, stats);
|
|
||||||
// 条件更新:任务可能已被 /fail 标为 FAILED(客户端报错与结果文件组装并发)——
|
|
||||||
// 无条件 updateById 会把 FAILED 覆盖成 SUCCESS(错误信息被清、用户看到「假成功」)。
|
// 无条件 updateById 会把 FAILED 覆盖成 SUCCESS(错误信息被清、用户看到「假成功」)。
|
||||||
// 结果文件已生成,故失败态下仍保留文件,只是不覆盖状态。
|
// 结果文件已生成,故失败态下仍保留文件,只是不覆盖状态。
|
||||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||||
@@ -1085,7 +1122,13 @@ public class CollectDataService {
|
|||||||
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
|
.set(FileTaskEntity::getUpdatedAt, LocalDateTime.now())
|
||||||
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
|
.set(FileTaskEntity::getFinishedAt, LocalDateTime.now()));
|
||||||
if (updated == 0) {
|
if (updated == 0) {
|
||||||
log.warn("[collect-data] 结果文件已生成但任务已是 FAILED,保留失败态不覆盖 taskId={}", task.getId());
|
// 任务已是 FAILED:结果记录改回失败语义并保留真实原因,但文件 URL 照常保留,
|
||||||
|
// 用户看到「失败 + 原因」的同时仍能下载已采集的部分结果。
|
||||||
|
result.setSuccess(0);
|
||||||
|
result.setErrorMessage(failureReason);
|
||||||
|
fileResultMapper.updateById(result);
|
||||||
|
log.warn("[collect-data] 结果文件已生成但任务已是 FAILED,保留失败态与原因 taskId={} rows={} reason={}",
|
||||||
|
task.getId(), finalRowCount, failureReason);
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
FileUtil.del(xlsx);
|
FileUtil.del(xlsx);
|
||||||
|
|||||||
+9
@@ -1,6 +1,8 @@
|
|||||||
package com.nanri.aiimage.modules.dedupe.model.entity;
|
package com.nanri.aiimage.modules.dedupe.model.entity;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.annotation.FieldStrategy;
|
||||||
import com.baomidou.mybatisplus.annotation.IdType;
|
import com.baomidou.mybatisplus.annotation.IdType;
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableField;
|
||||||
import com.baomidou.mybatisplus.annotation.TableId;
|
import com.baomidou.mybatisplus.annotation.TableId;
|
||||||
import com.baomidou.mybatisplus.annotation.TableName;
|
import com.baomidou.mybatisplus.annotation.TableName;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -19,5 +21,12 @@ public class DedupeTotalDataEntity {
|
|||||||
private Long uploaderUserId;
|
private Long uploaderUserId;
|
||||||
private String uploaderUsername;
|
private String uploaderUsername;
|
||||||
private LocalDateTime createdAt;
|
private LocalDateTime createdAt;
|
||||||
|
/**
|
||||||
|
* 更新时间:由数据库维护(DEFAULT CURRENT_TIMESTAMP + ON UPDATE CURRENT_TIMESTAMP)。
|
||||||
|
*
|
||||||
|
* <p>禁止应用显式写:MySQL 在 UPDATE 语句显式给该列赋值时不会触发自动更新,
|
||||||
|
* 而本表写回为 selectById → 改字段 → updateById(实体带着旧值),一旦写回就会冻结更新时间。
|
||||||
|
*/
|
||||||
|
@TableField(value = "updated_at", insertStrategy = FieldStrategy.NEVER, updateStrategy = FieldStrategy.NEVER)
|
||||||
private LocalDateTime updatedAt;
|
private LocalDateTime updatedAt;
|
||||||
}
|
}
|
||||||
|
|||||||
+3
@@ -32,4 +32,7 @@ public class DedupeTotalDataItemVo {
|
|||||||
|
|
||||||
@Schema(description = "创建时间")
|
@Schema(description = "创建时间")
|
||||||
private LocalDateTime createdAt;
|
private LocalDateTime createdAt;
|
||||||
|
|
||||||
|
@Schema(description = "更新时间")
|
||||||
|
private LocalDateTime updatedAt;
|
||||||
}
|
}
|
||||||
|
|||||||
+82
-73
@@ -42,6 +42,7 @@ import java.util.concurrent.Executors;
|
|||||||
import java.util.concurrent.Future;
|
import java.util.concurrent.Future;
|
||||||
import java.util.concurrent.Semaphore;
|
import java.util.concurrent.Semaphore;
|
||||||
import java.util.concurrent.atomic.AtomicInteger;
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
import java.util.function.Supplier;
|
||||||
import java.util.zip.ZipEntry;
|
import java.util.zip.ZipEntry;
|
||||||
import java.util.zip.ZipOutputStream;
|
import java.util.zip.ZipOutputStream;
|
||||||
|
|
||||||
@@ -487,7 +488,7 @@ public class DedupeRunService {
|
|||||||
readResult.scannedRows = new AtomicInteger(0);
|
readResult.scannedRows = new AtomicInteger(0);
|
||||||
readResult.filteredFbaRows = new AtomicInteger(0);
|
readResult.filteredFbaRows = new AtomicInteger(0);
|
||||||
|
|
||||||
PendingMainIdGroup pendingMainIdGroup = new PendingMainIdGroup();
|
IdRuleRowPicker rowPicker = new IdRuleRowPicker(keepIntegerIds, keepUnderscoreIds, keepIntegerMainIdsWhenNoSubIds);
|
||||||
// 表头列索引缓存:由 onHeader 填充,数据行为空时使用
|
// 表头列索引缓存:由 onHeader 填充,数据行为空时使用
|
||||||
final Map<String, Integer>[] headerIndexCache = new Map[]{Map.of()};
|
final Map<String, Integer>[] headerIndexCache = new Map[]{Map.of()};
|
||||||
|
|
||||||
@@ -526,23 +527,16 @@ public class DedupeRunService {
|
|||||||
}
|
}
|
||||||
Integer idColumnIndex = headerIndex.get("id");
|
Integer idColumnIndex = headerIndex.get("id");
|
||||||
if (idColumnIndex == null) {
|
if (idColumnIndex == null) {
|
||||||
readResult.rows.add(buildCandidateRow(rowMap, headerIndex, orderedSelectedColumns, null));
|
rowPicker.addAlways(buildCandidateRow(rowMap, headerIndex, orderedSelectedColumns, null));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
appendRowByIdRule(
|
rowPicker.select(
|
||||||
normalizeCellText(cellText(rowMap, idColumnIndex)),
|
normalizeCellText(cellText(rowMap, idColumnIndex)),
|
||||||
rowMap,
|
() -> buildCandidateRow(rowMap, headerIndex, orderedSelectedColumns, null)
|
||||||
headerIndex,
|
|
||||||
orderedSelectedColumns,
|
|
||||||
keepIntegerIds,
|
|
||||||
keepUnderscoreIds,
|
|
||||||
keepIntegerMainIdsWhenNoSubIds,
|
|
||||||
pendingMainIdGroup,
|
|
||||||
readResult.rows
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
pendingMainIdGroup.flush(readResult.rows);
|
readResult.rows = rowPicker.resolve();
|
||||||
long readNs = elapsedNs(readStartNs);
|
long readNs = elapsedNs(readStartNs);
|
||||||
|
|
||||||
Set<String> candidateAsinValues = new HashSet<>();
|
Set<String> candidateAsinValues = new HashSet<>();
|
||||||
@@ -560,8 +554,9 @@ public class DedupeRunService {
|
|||||||
try (SXSSFWorkbook outputWorkbook = new SXSSFWorkbook(200)) {
|
try (SXSSFWorkbook outputWorkbook = new SXSSFWorkbook(200)) {
|
||||||
org.apache.poi.ss.usermodel.Sheet outputSheet = outputWorkbook.createSheet(WorkbookUtil.createSafeSheetName(readResult.sheetName.isBlank() ? "Sheet1" : readResult.sheetName));
|
org.apache.poi.ss.usermodel.Sheet outputSheet = outputWorkbook.createSheet(WorkbookUtil.createSafeSheetName(readResult.sheetName.isBlank() ? "Sheet1" : readResult.sheetName));
|
||||||
Row outputHeaderRow = outputSheet.createRow(0);
|
Row outputHeaderRow = outputSheet.createRow(0);
|
||||||
for (int i = 0; i < selectedColumns.size(); i++) {
|
// 表头必须与数据行同序:数据按 orderedSelectedColumns 取值,表头写 selectedColumns 会整体错列
|
||||||
outputHeaderRow.createCell(i).setCellValue(selectedColumns.get(i));
|
for (int i = 0; i < orderedSelectedColumns.size(); i++) {
|
||||||
|
outputHeaderRow.createCell(i).setCellValue(orderedSelectedColumns.get(i));
|
||||||
}
|
}
|
||||||
|
|
||||||
Set<String> writtenAsinValues = new HashSet<>();
|
Set<String> writtenAsinValues = new HashSet<>();
|
||||||
@@ -659,34 +654,80 @@ public class DedupeRunService {
|
|||||||
return new DedupeCandidateRow(selectedValues, asinValue);
|
return new DedupeCandidateRow(selectedValues, asinValue);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void appendRowByIdRule(String idValue, Map<Integer, String> rowMap,
|
/**
|
||||||
Map<String, Integer> headerIndex, List<String> selectedColumns,
|
* 按 ID 保留规则挑选输出行。
|
||||||
boolean keepIntegerIds, boolean keepUnderscoreIds,
|
*
|
||||||
boolean keepIntegerMainIdsWhenNoSubIds,
|
* <p>主链接行是否保留,只取决于「该主 ID 是否出现过子链接行」,与两类行在源文件中的
|
||||||
PendingMainIdGroup pendingMainIdGroup,
|
* 先后顺序无关。旧实现用「暂存主链接行 + 后到的同主 ID 子链接行把它丢弃」的方式,
|
||||||
List<DedupeCandidateRow> rows) {
|
* 一旦顺序是「子链接在前、主链接在后」(子行到来时暂存区尚空,无人记录该主 ID 已有
|
||||||
if (idValue == null || idValue.isBlank()) {
|
* 子链接),主链接就会一路存活到收尾,导致同一主 ID 的主/子链接同时出现在结果里。</p>
|
||||||
return;
|
*/
|
||||||
|
static final class IdRuleRowPicker {
|
||||||
|
|
||||||
|
private final boolean keepIntegerIds;
|
||||||
|
private final boolean keepUnderscoreIds;
|
||||||
|
private final boolean keepIntegerMainIdsWhenNoSubIds;
|
||||||
|
/** 出现过子链接行的主 ID 集合,与源文件顺序无关 */
|
||||||
|
private final Set<String> mainIdsWithSubRows = new HashSet<>();
|
||||||
|
private final List<PickedRow> pickedRows = new ArrayList<>();
|
||||||
|
private int subRowCount = 0;
|
||||||
|
private int mainRowCount = 0;
|
||||||
|
|
||||||
|
IdRuleRowPicker(boolean keepIntegerIds, boolean keepUnderscoreIds, boolean keepIntegerMainIdsWhenNoSubIds) {
|
||||||
|
this.keepIntegerIds = keepIntegerIds;
|
||||||
|
this.keepUnderscoreIds = keepUnderscoreIds;
|
||||||
|
this.keepIntegerMainIdsWhenNoSubIds = keepIntegerMainIdsWhenNoSubIds;
|
||||||
}
|
}
|
||||||
String mainId = extractMainId(idValue);
|
|
||||||
if (pendingMainIdGroup.hasDifferentMainId(mainId)) {
|
/** ID 不参与保留规则判定的行(如无 id 列或 id 为空)直接保留。 */
|
||||||
pendingMainIdGroup.flush(rows);
|
void addAlways(DedupeCandidateRow row) {
|
||||||
|
pickedRows.add(new PickedRow(row, null));
|
||||||
}
|
}
|
||||||
if (isUnderscoreId(idValue)) {
|
|
||||||
pendingMainIdGroup.discardIfSameMainId(mainId);
|
/** 按 ID 形态与保留规则挑选;rowSupplier 仅在确定保留时才求值。 */
|
||||||
if (keepUnderscoreIds) {
|
void select(String idValue, Supplier<DedupeCandidateRow> rowSupplier) {
|
||||||
rows.add(buildCandidateRow(rowMap, headerIndex, selectedColumns, null));
|
if (idValue == null || idValue.isBlank()) {
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (isIntegerId(idValue)) {
|
|
||||||
if (keepIntegerIds) {
|
|
||||||
rows.add(buildCandidateRow(rowMap, headerIndex, selectedColumns, null));
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (keepIntegerMainIdsWhenNoSubIds) {
|
String mainId = extractMainId(idValue);
|
||||||
pendingMainIdGroup.add(mainId, buildCandidateRow(rowMap, headerIndex, selectedColumns, null));
|
if (isUnderscoreId(idValue)) {
|
||||||
|
subRowCount++;
|
||||||
|
mainIdsWithSubRows.add(mainId);
|
||||||
|
if (keepUnderscoreIds) {
|
||||||
|
pickedRows.add(new PickedRow(rowSupplier.get(), null));
|
||||||
|
}
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
if (isIntegerId(idValue)) {
|
||||||
|
if (keepIntegerIds) {
|
||||||
|
pickedRows.add(new PickedRow(rowSupplier.get(), null));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (keepIntegerMainIdsWhenNoSubIds) {
|
||||||
|
mainRowCount++;
|
||||||
|
pickedRows.add(new PickedRow(rowSupplier.get(), mainId));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 收尾统一判定:带条件的主链接行仅在该主 ID 没有子链接行时保留。 */
|
||||||
|
List<DedupeCandidateRow> resolve() {
|
||||||
|
List<DedupeCandidateRow> resolved = new ArrayList<>(pickedRows.size());
|
||||||
|
int droppedMainRowCount = 0;
|
||||||
|
for (PickedRow picked : pickedRows) {
|
||||||
|
if (picked.conditionalMainId() != null && mainIdsWithSubRows.contains(picked.conditionalMainId())) {
|
||||||
|
droppedMainRowCount++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
resolved.add(picked.row());
|
||||||
|
}
|
||||||
|
log.info("dedupe id rules subRows={} mainRows={} mainIdsWithSubRows={} droppedMainRows={} keptRows={}",
|
||||||
|
subRowCount, mainRowCount, mainIdsWithSubRows.size(), droppedMainRowCount, resolved.size());
|
||||||
|
return resolved;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** conditionalMainId 非空表示该行是主链接行,需等收尾时看同主 ID 有无子链接行 */
|
||||||
|
private record PickedRow(DedupeCandidateRow row, String conditionalMainId) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -730,7 +771,7 @@ public class DedupeRunService {
|
|||||||
return String.join("/", parts);
|
return String.join("/", parts);
|
||||||
}
|
}
|
||||||
|
|
||||||
private String extractMainId(String text) {
|
static String extractMainId(String text) {
|
||||||
if (text == null || text.isBlank()) {
|
if (text == null || text.isBlank()) {
|
||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
@@ -744,7 +785,7 @@ public class DedupeRunService {
|
|||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean isIntegerId(String text) {
|
static boolean isIntegerId(String text) {
|
||||||
if (text == null || text.isEmpty()) {
|
if (text == null || text.isEmpty()) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -756,7 +797,7 @@ public class DedupeRunService {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean isUnderscoreId(String text) {
|
static boolean isUnderscoreId(String text) {
|
||||||
if (text == null || text.length() < 3) {
|
if (text == null || text.length() < 3) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -952,38 +993,6 @@ public class DedupeRunService {
|
|||||||
private AtomicInteger filteredFbaRows;
|
private AtomicInteger filteredFbaRows;
|
||||||
}
|
}
|
||||||
|
|
||||||
private record DedupeCandidateRow(List<String> selectedValues, String asinValue) {
|
record DedupeCandidateRow(List<String> selectedValues, String asinValue) {
|
||||||
}
|
|
||||||
|
|
||||||
private static final class PendingMainIdGroup {
|
|
||||||
private String mainId;
|
|
||||||
private final List<DedupeCandidateRow> rows = new ArrayList<>();
|
|
||||||
|
|
||||||
private void add(String nextMainId, DedupeCandidateRow row) {
|
|
||||||
if (hasDifferentMainId(nextMainId)) {
|
|
||||||
rows.clear();
|
|
||||||
}
|
|
||||||
mainId = nextMainId;
|
|
||||||
rows.add(row);
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean hasDifferentMainId(String nextMainId) {
|
|
||||||
return mainId != null && (nextMainId == null || nextMainId.isBlank() || !mainId.equals(nextMainId));
|
|
||||||
}
|
|
||||||
|
|
||||||
private void discardIfSameMainId(String nextMainId) {
|
|
||||||
if (mainId != null && mainId.equals(nextMainId)) {
|
|
||||||
rows.clear();
|
|
||||||
mainId = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void flush(List<DedupeCandidateRow> outputRows) {
|
|
||||||
if (!rows.isEmpty()) {
|
|
||||||
outputRows.addAll(rows);
|
|
||||||
rows.clear();
|
|
||||||
}
|
|
||||||
mainId = null;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+1
@@ -1398,6 +1398,7 @@ public class DedupeTotalDataService {
|
|||||||
vo.setUploaderUserId(entity.getUploaderUserId());
|
vo.setUploaderUserId(entity.getUploaderUserId());
|
||||||
vo.setUsername(entity.getUploaderUsername());
|
vo.setUsername(entity.getUploaderUsername());
|
||||||
vo.setCreatedAt(entity.getCreatedAt());
|
vo.setCreatedAt(entity.getCreatedAt());
|
||||||
|
vo.setUpdatedAt(entity.getUpdatedAt());
|
||||||
return vo;
|
return vo;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+23
-1
@@ -27,6 +27,7 @@ import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
|||||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskResumeService;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
@@ -92,6 +93,8 @@ public class DeleteBrandStaleTaskService {
|
|||||||
private final ShopDataCrawlTaskService shopDataCrawlTaskService;
|
private final ShopDataCrawlTaskService shopDataCrawlTaskService;
|
||||||
private final TaskScopeStateMapper taskScopeStateMapper;
|
private final TaskScopeStateMapper taskScopeStateMapper;
|
||||||
private final TaskHeartbeatPositionService taskHeartbeatPositionService;
|
private final TaskHeartbeatPositionService taskHeartbeatPositionService;
|
||||||
|
/** 客户端中断任务的自动续跑(保留失败记录 + 重排队续跑任务,V129)。 */
|
||||||
|
private final TaskResumeService taskResumeService;
|
||||||
|
|
||||||
@Value("${aiimage.temp-dir.retention-hours:24}")
|
@Value("${aiimage.temp-dir.retention-hours:24}")
|
||||||
private long tempDirRetentionHours;
|
private long tempDirRetentionHours;
|
||||||
@@ -114,13 +117,16 @@ public class DeleteBrandStaleTaskService {
|
|||||||
ShopMatchStaleCheckStats queryAsinStats = failStaleQueryAsinTasks();
|
ShopMatchStaleCheckStats queryAsinStats = failStaleQueryAsinTasks();
|
||||||
ShopMatchStaleCheckStats withdrawStats = failStaleWithdrawTasks();
|
ShopMatchStaleCheckStats withdrawStats = failStaleWithdrawTasks();
|
||||||
NoResultUploadStaleCheckStats noUploadStats = failNoResultUploadTasks();
|
NoResultUploadStaleCheckStats noUploadStats = failNoResultUploadTasks();
|
||||||
|
// 客户端重启中断的任务:除了标失败(客户端上报,用户能看到原因),还要重新排队
|
||||||
|
// 一条 PENDING 续跑任务交给客户端兜底拉取执行,否则长任务一遇客户端更新就整个白跑
|
||||||
|
TaskResumeService.ResumeStats resumeStats = resumeInterruptedSafely();
|
||||||
for (Map.Entry<String, Runnable> delegated : delegatedStaleChecks().entrySet()) {
|
for (Map.Entry<String, Runnable> delegated : delegatedStaleChecks().entrySet()) {
|
||||||
runModuleStaleCheck(delegated.getKey(), delegated.getValue());
|
runModuleStaleCheck(delegated.getKey(), delegated.getValue());
|
||||||
}
|
}
|
||||||
// 周期每 2 分钟一轮,各模块 summary 合并为单行,避免定期刷屏
|
// 周期每 2 分钟一轮,各模块 summary 合并为单行,避免定期刷屏
|
||||||
// 注意:每段占位符数量必须与实参一致——此前每段 5 个占位符只传 4 个参数,
|
// 注意:每段占位符数量必须与实参一致——此前每段 5 个占位符只传 4 个参数,
|
||||||
// 导致 withdraw 之后的取值整体错位、末尾 elapsedMs/thread 打成字面量
|
// 导致 withdraw 之后的取值整体错位、末尾 elapsedMs/thread 打成字面量
|
||||||
log.info("[stale-check] summary product-risk(s={} f={} x={} p={}) price-track(s={} f={} x={} p={}) shop-match(s={} f={} x={} p={}) patrol-delete(s={} f={} x={} p={}) query-asin(s={} f={} x={} p={}) withdraw(s={} f={} x={} p={}) no-upload(c={} f={} x={}) elapsedMs={} thread={}",
|
log.info("[stale-check] summary product-risk(s={} f={} x={} p={}) price-track(s={} f={} x={} p={}) shop-match(s={} f={} x={} p={}) patrol-delete(s={} f={} x={} p={}) query-asin(s={} f={} x={} p={}) withdraw(s={} f={} x={} p={}) no-upload(c={} f={} x={}) resume(s={} r={} k={} u={}) elapsedMs={} thread={}",
|
||||||
stats.scannedTaskCount, stats.finalizedTaskCount, stats.failedTaskCount, stats.skippedTaskCount,
|
stats.scannedTaskCount, stats.finalizedTaskCount, stats.failedTaskCount, stats.skippedTaskCount,
|
||||||
priceTrackStats.scannedTaskCount, priceTrackStats.finalizedTaskCount, priceTrackStats.failedTaskCount, priceTrackStats.skippedTaskCount,
|
priceTrackStats.scannedTaskCount, priceTrackStats.finalizedTaskCount, priceTrackStats.failedTaskCount, priceTrackStats.skippedTaskCount,
|
||||||
shopMatchStats.scannedTaskCount, shopMatchStats.finalizedTaskCount, shopMatchStats.failedTaskCount, shopMatchStats.skippedTaskCount,
|
shopMatchStats.scannedTaskCount, shopMatchStats.finalizedTaskCount, shopMatchStats.failedTaskCount, shopMatchStats.skippedTaskCount,
|
||||||
@@ -128,11 +134,27 @@ public class DeleteBrandStaleTaskService {
|
|||||||
queryAsinStats.scannedTaskCount, queryAsinStats.finalizedTaskCount, queryAsinStats.failedTaskCount, queryAsinStats.skippedTaskCount,
|
queryAsinStats.scannedTaskCount, queryAsinStats.finalizedTaskCount, queryAsinStats.failedTaskCount, queryAsinStats.skippedTaskCount,
|
||||||
withdrawStats.scannedTaskCount, withdrawStats.finalizedTaskCount, withdrawStats.failedTaskCount, withdrawStats.skippedTaskCount,
|
withdrawStats.scannedTaskCount, withdrawStats.finalizedTaskCount, withdrawStats.failedTaskCount, withdrawStats.skippedTaskCount,
|
||||||
noUploadStats.scannedTaskCount, noUploadStats.failedTaskCount, noUploadStats.skippedTaskCount,
|
noUploadStats.scannedTaskCount, noUploadStats.failedTaskCount, noUploadStats.skippedTaskCount,
|
||||||
|
resumeStats.scannedTaskCount, resumeStats.resumedTaskCount, resumeStats.skippedTaskCount,
|
||||||
|
resumeStats.unsupportedTaskCount,
|
||||||
System.currentTimeMillis() - startedAt,
|
System.currentTimeMillis() - startedAt,
|
||||||
Thread.currentThread().getName());
|
Thread.currentThread().getName());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 自动续跑巡检的隔离壳:续跑失败绝不能拖垮判死主流程。
|
||||||
|
* 判死是任务状态的兜底(不做会留下永远 RUNNING 的孤儿),优先级高于续跑;
|
||||||
|
* 这里失败只记日志并返回空统计,2 分钟后的下一轮自然重试。
|
||||||
|
*/
|
||||||
|
private TaskResumeService.ResumeStats resumeInterruptedSafely() {
|
||||||
|
try {
|
||||||
|
return taskResumeService.resumeInterruptedTasks();
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[task-resume] 自动续跑巡检失败(不影响本轮判死): {}", ex.getMessage(), ex);
|
||||||
|
return new TaskResumeService.ResumeStats();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 委派式陈旧判死:moduleType → 处理动作。
|
* 委派式陈旧判死:moduleType → 处理动作。
|
||||||
* {@code TaskModuleRegistry} 中 delegatedStaleCheck=true 的模块都必须在这里登记,
|
* {@code TaskModuleRegistry} 中 delegatedStaleCheck=true 的模块都必须在这里登记,
|
||||||
|
|||||||
+17
@@ -2,8 +2,25 @@ package com.nanri.aiimage.modules.devicelog.mapper;
|
|||||||
|
|
||||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||||
import com.nanri.aiimage.modules.devicelog.model.entity.DeviceLogFileEntity;
|
import com.nanri.aiimage.modules.devicelog.model.entity.DeviceLogFileEntity;
|
||||||
|
import org.apache.ibatis.annotations.Delete;
|
||||||
import org.apache.ibatis.annotations.Mapper;
|
import org.apache.ibatis.annotations.Mapper;
|
||||||
|
import org.apache.ibatis.annotations.Param;
|
||||||
|
|
||||||
|
import java.time.LocalDate;
|
||||||
|
|
||||||
@Mapper
|
@Mapper
|
||||||
public interface DeviceLogFileMapper extends BaseMapper<DeviceLogFileEntity> {
|
public interface DeviceLogFileMapper extends BaseMapper<DeviceLogFileEntity> {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 按日志日期分批删除保留期外的元数据行。
|
||||||
|
*
|
||||||
|
* <p>只删 {@code log_date} 早于 cutoff 的行:这条线正好是查询侧可见窗口的边界,
|
||||||
|
* 即"早已看不见、只剩占位"的行,删掉不影响任何读取路径。
|
||||||
|
*/
|
||||||
|
@Delete("""
|
||||||
|
DELETE FROM device_log_file
|
||||||
|
WHERE log_date < #{cutoff}
|
||||||
|
LIMIT #{batchSize}
|
||||||
|
""")
|
||||||
|
int deleteOlderThanBatch(@Param("cutoff") LocalDate cutoff, @Param("batchSize") int batchSize);
|
||||||
}
|
}
|
||||||
|
|||||||
+77
@@ -0,0 +1,77 @@
|
|||||||
|
package com.nanri.aiimage.modules.devicelog.service;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||||
|
import com.nanri.aiimage.config.DeviceLogOssProperties;
|
||||||
|
import com.nanri.aiimage.modules.devicelog.mapper.DeviceLogFileMapper;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.scheduling.annotation.Scheduled;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.LocalDate;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 设备日志元数据的保留期清理。
|
||||||
|
*
|
||||||
|
* <p>对象侧一直有生命周期(主机B 独立 MinIO 桶,由运维 mc 定时任务按 7 天回收),
|
||||||
|
* 但 {@code device_log_file} 的元数据行此前**只增不删**:客户端每 60 秒上报一轮,
|
||||||
|
* 每台设备每天登记若干「来源+设备+文件名+日期」行,长期运行会无限累积。
|
||||||
|
*
|
||||||
|
* <p>查询侧本就只展示保留期内的行({@link DeviceLogService#page} 用同一个
|
||||||
|
* {@code retentionDays} 过滤),这里按完全相同的边界删掉早已不可见的行。
|
||||||
|
* 保留天数直接复用 {@code aiimage.device-log-oss.retention-days},
|
||||||
|
* 与对象侧共用同一个配置项,避免两边各写一份天数后悄悄漂移。
|
||||||
|
*
|
||||||
|
* <p>对象本身仍由桶生命周期负责回收,本任务不碰对象:两条时间线按「对象修改时间」
|
||||||
|
* 与「日志日期」衡量,可能有几天错位,但对象最终仍会被桶规则删除,不会永久残留。
|
||||||
|
*
|
||||||
|
* <p>双节点用 job 锁保证单实例执行;分批删除并限制单轮批次数,避免一次跑太久占住锁。
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
public class DeviceLogRetentionService {
|
||||||
|
|
||||||
|
/** 单轮最多删除的批次数(每批 batchSize 行),剩余留给下一轮。 */
|
||||||
|
static final int MAX_BATCHES_PER_RUN = 20;
|
||||||
|
|
||||||
|
private final DeviceLogFileMapper deviceLogFileMapper;
|
||||||
|
private final DeviceLogOssProperties properties;
|
||||||
|
private final DistributedJobLockService distributedJobLockService;
|
||||||
|
|
||||||
|
@Value("${aiimage.device-log.retention-batch-size:2000}")
|
||||||
|
private int retentionBatchSize = 2000;
|
||||||
|
|
||||||
|
@Scheduled(cron = "${aiimage.device-log.retention-cron:0 40 4 * * *}")
|
||||||
|
public void purgeExpiredMetadata() {
|
||||||
|
int days = properties.retentionDaysOrDefault();
|
||||||
|
int batchSize = Math.max(100, retentionBatchSize);
|
||||||
|
// 与查询侧可见窗口同一条边界:page() 只展示 log_date >= 今天-(days-1) 的行
|
||||||
|
LocalDate cutoff = LocalDate.now().minusDays(days - 1L);
|
||||||
|
|
||||||
|
DistributedJobLockService.LockHandle lockHandle =
|
||||||
|
distributedJobLockService.tryLock("device-log:metadata-retention", Duration.ofMinutes(15));
|
||||||
|
if (lockHandle == null) {
|
||||||
|
log.info("[device-log] 元数据保留清理跳过:另一实例持锁");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try (lockHandle) {
|
||||||
|
int totalDeleted = 0;
|
||||||
|
int batches = 0;
|
||||||
|
while (batches < MAX_BATCHES_PER_RUN) {
|
||||||
|
int deleted = deviceLogFileMapper.deleteOlderThanBatch(cutoff, batchSize);
|
||||||
|
batches++;
|
||||||
|
totalDeleted += deleted;
|
||||||
|
if (deleted < batchSize) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.info("[device-log] 元数据保留清理完成 cutoff={} retentionDays={} deleted={} batches={}",
|
||||||
|
cutoff, days, totalDeleted, batches);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[device-log] 元数据保留清理失败 cutoff={} msg={}", cutoff, ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+159
-1
@@ -5,10 +5,14 @@ import io.micrometer.core.instrument.DistributionSummary;
|
|||||||
import io.micrometer.core.instrument.MeterRegistry;
|
import io.micrometer.core.instrument.MeterRegistry;
|
||||||
import io.micrometer.core.instrument.Timer;
|
import io.micrometer.core.instrument.Timer;
|
||||||
import io.minio.GetObjectArgs;
|
import io.minio.GetObjectArgs;
|
||||||
|
import io.minio.ListObjectsArgs;
|
||||||
import io.minio.MinioClient;
|
import io.minio.MinioClient;
|
||||||
import io.minio.PutObjectArgs;
|
import io.minio.PutObjectArgs;
|
||||||
import io.minio.RemoveObjectArgs;
|
import io.minio.RemoveObjectArgs;
|
||||||
|
import io.minio.Result;
|
||||||
import io.minio.StatObjectArgs;
|
import io.minio.StatObjectArgs;
|
||||||
|
import io.minio.errors.ErrorResponseException;
|
||||||
|
import io.minio.messages.Item;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import okhttp3.ConnectionPool;
|
import okhttp3.ConnectionPool;
|
||||||
import okhttp3.Dispatcher;
|
import okhttp3.Dispatcher;
|
||||||
@@ -20,7 +24,11 @@ import org.springframework.stereotype.Service;
|
|||||||
import java.io.ByteArrayInputStream;
|
import java.io.ByteArrayInputStream;
|
||||||
import java.io.ByteArrayOutputStream;
|
import java.io.ByteArrayOutputStream;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.UUID;
|
||||||
import java.util.concurrent.Semaphore;
|
import java.util.concurrent.Semaphore;
|
||||||
import java.util.concurrent.ThreadLocalRandom;
|
import java.util.concurrent.ThreadLocalRandom;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
@@ -38,6 +46,17 @@ public class RustfsObjectStorageService {
|
|||||||
private static final String OP_STAT = "stat";
|
private static final String OP_STAT = "stat";
|
||||||
private static final String OP_TOTAL = "total";
|
private static final String OP_TOTAL = "total";
|
||||||
|
|
||||||
|
/** S3 确定性错误码:重试不会改变结果,应立即失败而不是白打两次请求。 */
|
||||||
|
private static final Set<String> NON_RETRYABLE_S3_CODES = Set.of(
|
||||||
|
"NoSuchKey", "NoSuchBucket", "NoSuchVersion",
|
||||||
|
"AccessDenied", "InvalidAccessKeyId", "SignatureDoesNotMatch", "InvalidBucketName");
|
||||||
|
|
||||||
|
/** 标准 UUID 字符串长度(8-4-4-4-12),用于识别版本化对象 key。 */
|
||||||
|
private static final int UUID_STRING_LENGTH = 36;
|
||||||
|
|
||||||
|
/** 兄弟对象兜底列出的上限:只为找回同槽位的版本化对象,不需要列全。 */
|
||||||
|
private static final int MAX_SIBLING_LIST_KEYS = 50;
|
||||||
|
|
||||||
private final TransientStorageProperties properties;
|
private final TransientStorageProperties properties;
|
||||||
private final ObjectProvider<MeterRegistry> meterRegistryProvider;
|
private final ObjectProvider<MeterRegistry> meterRegistryProvider;
|
||||||
private final ObjectProvider<RustfsDeleteRetryService> deleteRetryServiceProvider;
|
private final ObjectProvider<RustfsDeleteRetryService> deleteRetryServiceProvider;
|
||||||
@@ -99,7 +118,22 @@ public class RustfsObjectStorageService {
|
|||||||
return uploadBytes(objectKey, bytes, verifyAfterUpload);
|
return uploadBytes(objectKey, bytes, verifyAfterUpload);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 三参重载:是否做「上传失败补偿删除」按对象 key 形态自动判定。
|
||||||
|
*
|
||||||
|
* <p>只有版本化 key(末段以 UUID 结尾)是本次写入独占的;确定性 key 会被重传重写复用,
|
||||||
|
* 删它就可能删掉别的 DB 行仍在引用的对象(2026-09-17 线上任务 28459 的载荷对象就是这么丢的)。
|
||||||
|
*/
|
||||||
public String uploadBytes(String objectKey, byte[] content, boolean verifyAfterUpload) {
|
public String uploadBytes(String objectKey, byte[] content, boolean verifyAfterUpload) {
|
||||||
|
return uploadBytes(objectKey, content, verifyAfterUpload, isVersionedObjectKey(objectKey));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param compensateDeleteOnFailure put 已完成、但后续可见性校验失败时,是否把该对象排进删除补偿队列。
|
||||||
|
* 仅当调用方能确认「该对象不会被其它写入复用时」才可传 true。
|
||||||
|
*/
|
||||||
|
public String uploadBytes(String objectKey, byte[] content, boolean verifyAfterUpload,
|
||||||
|
boolean compensateDeleteOnFailure) {
|
||||||
long deadlineNanos = operationDeadlineNanos();
|
long deadlineNanos = operationDeadlineNanos();
|
||||||
if (!isConfigured()) {
|
if (!isConfigured()) {
|
||||||
throw new IllegalStateException("transient storage is not configured");
|
throw new IllegalStateException("transient storage is not configured");
|
||||||
@@ -129,13 +163,45 @@ public class RustfsObjectStorageService {
|
|||||||
}
|
}
|
||||||
return uploadedObjectKey;
|
return uploadedObjectKey;
|
||||||
} catch (RuntimeException ex) {
|
} catch (RuntimeException ex) {
|
||||||
if (putCompleted.get()) {
|
if (putCompleted.get() && compensateDeleteOnFailure) {
|
||||||
enqueueDeleteRetry(objectKey, ex);
|
enqueueDeleteRetry(objectKey, ex);
|
||||||
|
} else if (putCompleted.get()) {
|
||||||
|
// 共享 key 会被重传重写:此处删除可能删掉别的行正在引用的对象,交给保留期清理兜底。
|
||||||
|
// 线上任务 28459 的 chunk-462/473/484 就是被这条无条件删除队列删掉的。
|
||||||
|
log.warn("[rustfs] 跳过上传失败补偿删除(对象非本次独占,可能被复用)objectKey={} err={}",
|
||||||
|
objectKey, ex.getMessage());
|
||||||
}
|
}
|
||||||
throw ex;
|
throw ex;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 对象 key 是否为「本次写入独占」的版本化 key:末段(去掉 {@code .json} 后缀)以 UUID 结尾。
|
||||||
|
*
|
||||||
|
* <p>只看末段——UUID 出现在中间段(如 scopeHash)不代表该对象被独占;解析失败一律按共享处理
|
||||||
|
* (保守:宁可留下孤儿对象,也不删掉可能仍被引用的对象)。
|
||||||
|
*/
|
||||||
|
static boolean isVersionedObjectKey(String objectKey) {
|
||||||
|
if (objectKey == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
String key = objectKey.trim();
|
||||||
|
if (key.endsWith(".json")) {
|
||||||
|
key = key.substring(0, key.length() - ".json".length());
|
||||||
|
}
|
||||||
|
int slash = key.lastIndexOf('/');
|
||||||
|
String lastSegment = slash < 0 ? key : key.substring(slash + 1);
|
||||||
|
if (lastSegment.length() < UUID_STRING_LENGTH) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
UUID.fromString(lastSegment.substring(lastSegment.length() - UUID_STRING_LENGTH));
|
||||||
|
return true;
|
||||||
|
} catch (IllegalArgumentException ex) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public String readObjectAsString(String objectKey) {
|
public String readObjectAsString(String objectKey) {
|
||||||
byte[] bytes = readObjectBytes(objectKey);
|
byte[] bytes = readObjectBytes(objectKey);
|
||||||
return new String(bytes, StandardCharsets.UTF_8);
|
return new String(bytes, StandardCharsets.UTF_8);
|
||||||
@@ -175,6 +241,43 @@ public class RustfsObjectStorageService {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 列出前缀下的对象 key,按最后修改时间倒序(最新在前)。
|
||||||
|
*
|
||||||
|
* <p>只服务于「指针指向的对象已不存在、需要找回同槽位的版本化兄弟对象」这一兜底路径,
|
||||||
|
* 因此刻意不做重试、不参与失败窗口记账:列出失败直接抛错,由调用方按原错误语义处理。
|
||||||
|
*/
|
||||||
|
public List<String> listObjectKeysNewestFirst(String prefix, int limit) {
|
||||||
|
if (!isConfigured()) {
|
||||||
|
throw new IllegalStateException("transient storage is not configured");
|
||||||
|
}
|
||||||
|
int safeLimit = Math.max(1, Math.min(limit, MAX_SIBLING_LIST_KEYS));
|
||||||
|
long deadlineNanos = operationDeadlineNanos();
|
||||||
|
List<String[]> entries = new ArrayList<>();
|
||||||
|
try {
|
||||||
|
Iterable<Result<Item>> results = buildClient(deadlineNanos).listObjects(ListObjectsArgs.builder()
|
||||||
|
.bucket(properties.getBucket())
|
||||||
|
.prefix(prefix == null ? "" : prefix)
|
||||||
|
.recursive(true)
|
||||||
|
.maxKeys(safeLimit)
|
||||||
|
.build());
|
||||||
|
for (Result<Item> result : results) {
|
||||||
|
Item item = result.get();
|
||||||
|
entries.add(new String[]{item.objectName(),
|
||||||
|
item.lastModified() == null ? "" : item.lastModified().toString()});
|
||||||
|
}
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new IllegalStateException("transient storage list failed prefix=" + prefix
|
||||||
|
+ " err=" + ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
entries.sort((left, right) -> right[1].compareTo(left[1]));
|
||||||
|
List<String> keys = new ArrayList<>(entries.size());
|
||||||
|
for (String[] entry : entries) {
|
||||||
|
keys.add(entry[0]);
|
||||||
|
}
|
||||||
|
return keys;
|
||||||
|
}
|
||||||
|
|
||||||
public void deleteObject(String objectKey) {
|
public void deleteObject(String objectKey) {
|
||||||
deleteObject(objectKey, true, operationDeadlineNanos());
|
deleteObject(objectKey, true, operationDeadlineNanos());
|
||||||
}
|
}
|
||||||
@@ -238,6 +341,12 @@ public class RustfsObjectStorageService {
|
|||||||
resetFailureWindow(operation);
|
resetFailureWindow(operation);
|
||||||
}
|
}
|
||||||
recordOperation(operation, "success", elapsedNanos(startedAt));
|
recordOperation(operation, "success", elapsedNanos(startedAt));
|
||||||
|
if (attempt > 1) {
|
||||||
|
// 重试后成功必须留 INFO 结论:线上一天上千条 "operation failed, retrying"
|
||||||
|
// 却没有任何结论日志,无法判断这些上传最后到底落盘了没有。
|
||||||
|
log.info("[rustfs] 重试后成功 operation={} objectKey={} attempt={}/{} durationMs={} bytes={}",
|
||||||
|
operation, objectKey, attempt, maxRetries, elapsedMillis(startedAt), bytes);
|
||||||
|
}
|
||||||
log.debug("[rustfs] operation success operation={} objectKey={} attempt={}/{} durationMs={} bytes={}",
|
log.debug("[rustfs] operation success operation={} objectKey={} attempt={}/{} durationMs={} bytes={}",
|
||||||
operation, objectKey, attempt, maxRetries, elapsedMillis(startedAt), bytes);
|
operation, objectKey, attempt, maxRetries, elapsedMillis(startedAt), bytes);
|
||||||
return result;
|
return result;
|
||||||
@@ -245,6 +354,15 @@ public class RustfsObjectStorageService {
|
|||||||
last = ex;
|
last = ex;
|
||||||
recordOperation(operation, attempt < maxRetries ? "retry" : "failure", elapsedNanos(startedAt));
|
recordOperation(operation, attempt < maxRetries ? "retry" : "failure", elapsedNanos(startedAt));
|
||||||
recordFailure(operation, objectKey, ex);
|
recordFailure(operation, objectKey, ex);
|
||||||
|
if (isNonRetryable(ex)) {
|
||||||
|
// 确定性错误(NoSuchKey / AccessDenied…):重试多少次结果都一样。
|
||||||
|
// 线上 read 一个已被清理的 chunk 就会连打 3 次请求、还被记成 ERROR。
|
||||||
|
log.warn("[rustfs] 确定性错误,不重试 operation={} objectKey={} err={}",
|
||||||
|
operation, objectKey, ex.getMessage());
|
||||||
|
throw ex instanceof RuntimeException runtimeException
|
||||||
|
? runtimeException
|
||||||
|
: new IllegalStateException(ex);
|
||||||
|
}
|
||||||
if (attempt < maxRetries) {
|
if (attempt < maxRetries) {
|
||||||
delayMillis = retryDelayMillis(attempt);
|
delayMillis = retryDelayMillis(attempt);
|
||||||
log.warn("[rustfs] operation failed, retrying operation={} objectKey={} attempt={}/{} delayMs={} err={}",
|
log.warn("[rustfs] operation failed, retrying operation={} objectKey={} attempt={}/{} delayMs={} err={}",
|
||||||
@@ -262,6 +380,9 @@ public class RustfsObjectStorageService {
|
|||||||
operation, objectKey, deadlineNanos);
|
operation, objectKey, deadlineNanos);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// 重试耗尽:留 ERROR 结论 + 最后一次错误,否则只看到一串 retrying,无从判断是否真丢数据
|
||||||
|
log.error("[rustfs] 重试耗尽,最终失败 operation={} objectKey={} maxRetries={} 最后一次错误={}",
|
||||||
|
operation, objectKey, maxRetries, last == null ? "无" : last.getMessage(), last);
|
||||||
throw new IllegalStateException("failed to " + operation + " payload in transient storage", last);
|
throw new IllegalStateException("failed to " + operation + " payload in transient storage", last);
|
||||||
} finally {
|
} finally {
|
||||||
if (totalAcquired) {
|
if (totalAcquired) {
|
||||||
@@ -270,6 +391,27 @@ public class RustfsObjectStorageService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 是否为「重试也没用」的确定性错误(NoSuchKey / AccessDenied / 签名错误…)。
|
||||||
|
*
|
||||||
|
* <p>只有网络类与 5xx 类失败才值得重试;确定性错误重试多少次结果都一样。
|
||||||
|
*/
|
||||||
|
private static boolean isNonRetryable(Throwable error) {
|
||||||
|
Throwable cursor = error;
|
||||||
|
while (cursor != null) {
|
||||||
|
if (cursor instanceof ErrorResponseException responseException) {
|
||||||
|
String code = responseException.errorResponse() == null
|
||||||
|
? null
|
||||||
|
: responseException.errorResponse().code();
|
||||||
|
if (code != null && NON_RETRYABLE_S3_CODES.contains(code)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cursor = cursor.getCause();
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
private void acquirePermit(String operation, String objectKey, Semaphore semaphore, long deadlineNanos) {
|
private void acquirePermit(String operation, String objectKey, Semaphore semaphore, long deadlineNanos) {
|
||||||
try {
|
try {
|
||||||
checkDeadline(operation, objectKey, deadlineNanos);
|
checkDeadline(operation, objectKey, deadlineNanos);
|
||||||
@@ -336,6 +478,22 @@ public class RustfsObjectStorageService {
|
|||||||
Math.max(1L, properties.getConnectionPoolKeepAliveMillis()),
|
Math.max(1L, properties.getConnectionPoolKeepAliveMillis()),
|
||||||
TimeUnit.MILLISECONDS))
|
TimeUnit.MILLISECONDS))
|
||||||
.retryOnConnectionFailure(true)
|
.retryOnConnectionFailure(true)
|
||||||
|
.addNetworkInterceptor(chain -> {
|
||||||
|
okhttp3.Request request = chain.request();
|
||||||
|
okhttp3.RequestBody body = request.body();
|
||||||
|
// OkHttp 对「长度为 0 的请求体」不会写 Content-Length,于是请求既无
|
||||||
|
// Content-Length 也无 Transfer-Encoding(HTTP/1.1 不允许这样)。
|
||||||
|
// RustFS 对此直接回 411 Length Required,客户端读到不完整响应就报
|
||||||
|
// unexpected end of stream,进而重试——线上抓包实测 90 秒内 515 次 411,
|
||||||
|
// 而上传空内容(content 为 null/空)在业务里是常态。
|
||||||
|
// 用 network interceptor 在协议层补上该头(普通 interceptor 会被
|
||||||
|
// BridgeInterceptor 按 body 长度覆盖掉,加了也不生效)。
|
||||||
|
if (body != null && body.contentLength() == 0L
|
||||||
|
&& request.header("Content-Length") == null) {
|
||||||
|
request = request.newBuilder().header("Content-Length", "0").build();
|
||||||
|
}
|
||||||
|
return chain.proceed(request);
|
||||||
|
})
|
||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
return httpClient;
|
return httpClient;
|
||||||
|
|||||||
+9
@@ -1,6 +1,8 @@
|
|||||||
package com.nanri.aiimage.modules.invalidasin.model.entity;
|
package com.nanri.aiimage.modules.invalidasin.model.entity;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.annotation.FieldStrategy;
|
||||||
import com.baomidou.mybatisplus.annotation.IdType;
|
import com.baomidou.mybatisplus.annotation.IdType;
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableField;
|
||||||
import com.baomidou.mybatisplus.annotation.TableId;
|
import com.baomidou.mybatisplus.annotation.TableId;
|
||||||
import com.baomidou.mybatisplus.annotation.TableName;
|
import com.baomidou.mybatisplus.annotation.TableName;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
@@ -18,5 +20,12 @@ public class InvalidAsinDataEntity {
|
|||||||
private Long groupId;
|
private Long groupId;
|
||||||
private String recordSource;
|
private String recordSource;
|
||||||
private LocalDateTime createdAt;
|
private LocalDateTime createdAt;
|
||||||
|
/**
|
||||||
|
* 更新时间:由数据库维护(DEFAULT CURRENT_TIMESTAMP + ON UPDATE CURRENT_TIMESTAMP)。
|
||||||
|
*
|
||||||
|
* <p>禁止应用显式写:MySQL 在 UPDATE 语句显式给该列赋值时不会触发自动更新,
|
||||||
|
* 而本表写回为 selectById → 改字段 → updateById(实体带着旧值),一旦写回就会冻结更新时间。
|
||||||
|
*/
|
||||||
|
@TableField(value = "updated_at", insertStrategy = FieldStrategy.NEVER, updateStrategy = FieldStrategy.NEVER)
|
||||||
private LocalDateTime updatedAt;
|
private LocalDateTime updatedAt;
|
||||||
}
|
}
|
||||||
|
|||||||
+3
@@ -29,4 +29,7 @@ public class InvalidAsinDataItemVo {
|
|||||||
|
|
||||||
@Schema(description = "创建时间")
|
@Schema(description = "创建时间")
|
||||||
private LocalDateTime createdAt;
|
private LocalDateTime createdAt;
|
||||||
|
|
||||||
|
@Schema(description = "更新时间")
|
||||||
|
private LocalDateTime updatedAt;
|
||||||
}
|
}
|
||||||
|
|||||||
+1
@@ -245,6 +245,7 @@ public class InvalidAsinDataService {
|
|||||||
vo.setGroupName(groupName == null ? "" : groupName);
|
vo.setGroupName(groupName == null ? "" : groupName);
|
||||||
vo.setRecordSource(isManualRecord(entity) ? RECORD_SOURCE_MANUAL : RECORD_SOURCE_AUTO);
|
vo.setRecordSource(isManualRecord(entity) ? RECORD_SOURCE_MANUAL : RECORD_SOURCE_AUTO);
|
||||||
vo.setCreatedAt(entity.getCreatedAt());
|
vo.setCreatedAt(entity.getCreatedAt());
|
||||||
|
vo.setUpdatedAt(entity.getUpdatedAt());
|
||||||
return vo;
|
return vo;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+10
-6
@@ -44,23 +44,27 @@ public class MaixiangConsoleClient {
|
|||||||
return hasText(properties.getPriceTrackApiUrl()) && hasText(properties.getMaixiangConsoleToken());
|
return hasText(properties.getPriceTrackApiUrl()) && hasText(properties.getMaixiangConsoleToken());
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 批量任务列表(all_task)。createdBefore 非空时只取创建时间不晚于该时刻的任务(SQL 字符串比较)。 */
|
/**
|
||||||
public BatchTaskPage batchTasks(int status, LocalDateTime createdBefore, int pageSize) {
|
* 批量任务列表(all_task)。updateTimeBefore 非空时只取「最后更新」不晚于该时刻的任务
|
||||||
|
* (SQL 字符串比较)。用 update_time 过滤而不是 create_time:desc(id) 分页下按创建时间过滤
|
||||||
|
* 只能看到最新创建的一批,积压深处的老任务(真正停滞的)永远看不到 —— 2026-09-15 滞留漏报的根因。
|
||||||
|
*/
|
||||||
|
public BatchTaskPage batchTasks(int status, LocalDateTime updateTimeBefore, int pageSize) {
|
||||||
String url = buildUrl("/api/console/batch/tasks",
|
String url = buildUrl("/api/console/batch/tasks",
|
||||||
"status=" + status,
|
"status=" + status,
|
||||||
"page=1",
|
"page=1",
|
||||||
"page_size=" + pageSize,
|
"page_size=" + pageSize,
|
||||||
createdBefore == null ? null : "end_time=" + TIME_FORMAT.format(createdBefore));
|
updateTimeBefore == null ? null : "update_time_end=" + TIME_FORMAT.format(updateTimeBefore));
|
||||||
return parseBatchTaskPage(fetch(url, "批量任务列表 status=" + status));
|
return parseBatchTaskPage(fetch(url, "批量任务列表 status=" + status));
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 单任务列表(task_record)。 */
|
/** 单任务列表(task_record)。updateTimeBefore 语义同 {@link #batchTasks(int, LocalDateTime, int)}。 */
|
||||||
public SingleTaskPage singleTasks(int status, LocalDateTime createdBefore, int pageSize) {
|
public SingleTaskPage singleTasks(int status, LocalDateTime updateTimeBefore, int pageSize) {
|
||||||
String url = buildUrl("/api/console/tasks",
|
String url = buildUrl("/api/console/tasks",
|
||||||
"status=" + status,
|
"status=" + status,
|
||||||
"page=1",
|
"page=1",
|
||||||
"page_size=" + pageSize,
|
"page_size=" + pageSize,
|
||||||
createdBefore == null ? null : "end_time=" + TIME_FORMAT.format(createdBefore));
|
updateTimeBefore == null ? null : "update_time_end=" + TIME_FORMAT.format(updateTimeBefore));
|
||||||
return parseSingleTaskPage(fetch(url, "单任务列表 status=" + status));
|
return parseSingleTaskPage(fetch(url, "单任务列表 status=" + status));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+6
-2
@@ -44,20 +44,24 @@ public class AdminNotificationController {
|
|||||||
|
|
||||||
@GetMapping
|
@GetMapping
|
||||||
@Operation(summary = "通知分页列表",
|
@Operation(summary = "通知分页列表",
|
||||||
description = "onlyUnread=true 时只返回未读;keyword 模糊匹配标题/内容;startDate/endDate 为年月日闭区间。")
|
description = "onlyUnread=true 时只返回未读;keyword 模糊匹配标题/内容;"
|
||||||
|
+ "category 按类型大类筛选(system_error/task_error/config_error/system_notice);"
|
||||||
|
+ "startDate/endDate 为年月日闭区间。")
|
||||||
public ApiResponse<NotificationPageVo> page(
|
public ApiResponse<NotificationPageVo> page(
|
||||||
HttpServletRequest request,
|
HttpServletRequest request,
|
||||||
@Parameter(description = "页码") @RequestParam(defaultValue = "1") Long page,
|
@Parameter(description = "页码") @RequestParam(defaultValue = "1") Long page,
|
||||||
@Parameter(description = "每页数量") @RequestParam(defaultValue = "20") Long pageSize,
|
@Parameter(description = "每页数量") @RequestParam(defaultValue = "20") Long pageSize,
|
||||||
@Parameter(description = "只看未读") @RequestParam(required = false, defaultValue = "false") Boolean onlyUnread,
|
@Parameter(description = "只看未读") @RequestParam(required = false, defaultValue = "false") Boolean onlyUnread,
|
||||||
@Parameter(description = "关键字(标题/内容)") @RequestParam(required = false) String keyword,
|
@Parameter(description = "关键字(标题/内容)") @RequestParam(required = false) String keyword,
|
||||||
|
@Parameter(description = "类型大类:system_error/task_error/config_error/system_notice")
|
||||||
|
@RequestParam(required = false) String category,
|
||||||
@Parameter(description = "起始日期(yyyy-MM-dd,含)")
|
@Parameter(description = "起始日期(yyyy-MM-dd,含)")
|
||||||
@RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate startDate,
|
@RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate startDate,
|
||||||
@Parameter(description = "结束日期(yyyy-MM-dd,含)")
|
@Parameter(description = "结束日期(yyyy-MM-dd,含)")
|
||||||
@RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate endDate) {
|
@RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate endDate) {
|
||||||
Long userId = currentAdminId(request);
|
Long userId = currentAdminId(request);
|
||||||
return ApiResponse.success(notificationService.page(userId, NotificationService.AUDIENCE_ADMIN,
|
return ApiResponse.success(notificationService.page(userId, NotificationService.AUDIENCE_ADMIN,
|
||||||
NotificationPageQuery.of(page, pageSize, onlyUnread, keyword, startDate, endDate)));
|
NotificationPageQuery.of(page, pageSize, onlyUnread, keyword, category, startDate, endDate)));
|
||||||
}
|
}
|
||||||
|
|
||||||
@PostMapping("/{id}/read")
|
@PostMapping("/{id}/read")
|
||||||
|
|||||||
+6
-2
@@ -44,20 +44,24 @@ public class NotificationController {
|
|||||||
|
|
||||||
@GetMapping
|
@GetMapping
|
||||||
@Operation(summary = "通知分页列表",
|
@Operation(summary = "通知分页列表",
|
||||||
description = "onlyUnread=true 时只返回未读;keyword 模糊匹配标题/内容;startDate/endDate 为年月日闭区间。")
|
description = "onlyUnread=true 时只返回未读;keyword 模糊匹配标题/内容;"
|
||||||
|
+ "category 按类型大类筛选(system_error/task_error/config_error/system_notice);"
|
||||||
|
+ "startDate/endDate 为年月日闭区间。")
|
||||||
public ApiResponse<NotificationPageVo> page(
|
public ApiResponse<NotificationPageVo> page(
|
||||||
HttpServletRequest request,
|
HttpServletRequest request,
|
||||||
@Parameter(description = "页码") @RequestParam(defaultValue = "1") Long page,
|
@Parameter(description = "页码") @RequestParam(defaultValue = "1") Long page,
|
||||||
@Parameter(description = "每页数量") @RequestParam(defaultValue = "20") Long pageSize,
|
@Parameter(description = "每页数量") @RequestParam(defaultValue = "20") Long pageSize,
|
||||||
@Parameter(description = "只看未读") @RequestParam(required = false, defaultValue = "false") Boolean onlyUnread,
|
@Parameter(description = "只看未读") @RequestParam(required = false, defaultValue = "false") Boolean onlyUnread,
|
||||||
@Parameter(description = "关键字(标题/内容)") @RequestParam(required = false) String keyword,
|
@Parameter(description = "关键字(标题/内容)") @RequestParam(required = false) String keyword,
|
||||||
|
@Parameter(description = "类型大类:system_error/task_error/config_error/system_notice")
|
||||||
|
@RequestParam(required = false) String category,
|
||||||
@Parameter(description = "起始日期(yyyy-MM-dd,含)")
|
@Parameter(description = "起始日期(yyyy-MM-dd,含)")
|
||||||
@RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate startDate,
|
@RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate startDate,
|
||||||
@Parameter(description = "结束日期(yyyy-MM-dd,含)")
|
@Parameter(description = "结束日期(yyyy-MM-dd,含)")
|
||||||
@RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate endDate) {
|
@RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate endDate) {
|
||||||
Long userId = currentUserId(request);
|
Long userId = currentUserId(request);
|
||||||
return ApiResponse.success(notificationService.page(userId, NotificationService.AUDIENCE_USER,
|
return ApiResponse.success(notificationService.page(userId, NotificationService.AUDIENCE_USER,
|
||||||
NotificationPageQuery.of(page, pageSize, onlyUnread, keyword, startDate, endDate)));
|
NotificationPageQuery.of(page, pageSize, onlyUnread, keyword, category, startDate, endDate)));
|
||||||
}
|
}
|
||||||
|
|
||||||
@PostMapping("/{id}/read")
|
@PostMapping("/{id}/read")
|
||||||
|
|||||||
+7
-3
@@ -5,8 +5,9 @@ import lombok.Data;
|
|||||||
import java.time.LocalDate;
|
import java.time.LocalDate;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 通知列表查询条件:分页 + 未读过滤 + 关键字(标题/内容模糊)+ 创建日期区间(年月日闭区间)。
|
* 通知列表查询条件:分页 + 未读过滤 + 关键字(标题/内容模糊)+ 创建日期区间(年月日闭区间)
|
||||||
* 铃铛面板的「按天搜索」与「内容搜索」都走这里,空值表示不限制。
|
* + 类型大类(system_error/task_error/config_error/system_notice)。
|
||||||
|
* 铃铛面板的「按天搜索」「内容搜索」「类型筛选」都走这里,空值表示不限制。
|
||||||
*/
|
*/
|
||||||
@Data
|
@Data
|
||||||
public class NotificationPageQuery {
|
public class NotificationPageQuery {
|
||||||
@@ -15,16 +16,19 @@ public class NotificationPageQuery {
|
|||||||
private Long pageSize;
|
private Long pageSize;
|
||||||
private Boolean onlyUnread;
|
private Boolean onlyUnread;
|
||||||
private String keyword;
|
private String keyword;
|
||||||
|
private String category;
|
||||||
private LocalDate startDate;
|
private LocalDate startDate;
|
||||||
private LocalDate endDate;
|
private LocalDate endDate;
|
||||||
|
|
||||||
public static NotificationPageQuery of(Long page, Long pageSize, Boolean onlyUnread,
|
public static NotificationPageQuery of(Long page, Long pageSize, Boolean onlyUnread,
|
||||||
String keyword, LocalDate startDate, LocalDate endDate) {
|
String keyword, String category,
|
||||||
|
LocalDate startDate, LocalDate endDate) {
|
||||||
NotificationPageQuery query = new NotificationPageQuery();
|
NotificationPageQuery query = new NotificationPageQuery();
|
||||||
query.setPage(page);
|
query.setPage(page);
|
||||||
query.setPageSize(pageSize);
|
query.setPageSize(pageSize);
|
||||||
query.setOnlyUnread(onlyUnread);
|
query.setOnlyUnread(onlyUnread);
|
||||||
query.setKeyword(keyword);
|
query.setKeyword(keyword);
|
||||||
|
query.setCategory(category);
|
||||||
query.setStartDate(startDate);
|
query.setStartDate(startDate);
|
||||||
query.setEndDate(endDate);
|
query.setEndDate(endDate);
|
||||||
return query;
|
return query;
|
||||||
|
|||||||
+2
@@ -11,6 +11,8 @@ public class NotificationItemVo {
|
|||||||
private Long id;
|
private Long id;
|
||||||
/** 场景:secret_balance/secret_invalid/task_failed/service_down/system */
|
/** 场景:secret_balance/secret_invalid/task_failed/service_down/system */
|
||||||
private String scene;
|
private String scene;
|
||||||
|
/** 类型大类(铃铛筛选用):system_error/task_error/config_error/system_notice */
|
||||||
|
private String category;
|
||||||
/** 级别:info/warning/error */
|
/** 级别:info/warning/error */
|
||||||
private String level;
|
private String level;
|
||||||
private String title;
|
private String title;
|
||||||
|
|||||||
+3
-3
@@ -104,7 +104,7 @@ public class MaixiangAnomalyScanner {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 批量任务停滞:创建早于阈值、仍处 status=0/1 且 update_time 不再推进。 */
|
/** 批量任务停滞:最后更新早于阈值、仍处 status=0/1(服务端按 update_time 过滤后取回)。 */
|
||||||
void checkStuckBatchTasks(NotificationDispatchService.AdminAudience audience, LocalDateTime now) {
|
void checkStuckBatchTasks(NotificationDispatchService.AdminAudience audience, LocalDateTime now) {
|
||||||
int thresholdMinutes = Math.max(1, properties.getMaixiangStuckMinutes());
|
int thresholdMinutes = Math.max(1, properties.getMaixiangStuckMinutes());
|
||||||
LocalDateTime cutoff = now.minusMinutes(thresholdMinutes);
|
LocalDateTime cutoff = now.minusMinutes(thresholdMinutes);
|
||||||
@@ -112,7 +112,7 @@ public class MaixiangAnomalyScanner {
|
|||||||
for (int status : new int[]{0, 1}) {
|
for (int status : new int[]{0, 1}) {
|
||||||
candidates.addAll(consoleClient.batchTasks(status, cutoff, PAGE_SIZE).items());
|
candidates.addAll(consoleClient.batchTasks(status, cutoff, PAGE_SIZE).items());
|
||||||
}
|
}
|
||||||
// 创建时间早于 cutoff 但仍在正常推进的大任务(update_time 晚于 cutoff)不算停滞
|
// 服务端已按 update_time ≤ cutoff 过滤;这里再兜一层,防止接口参数被忽略时误报正常推进的大任务
|
||||||
List<BatchTaskItem> stuck = new ArrayList<>();
|
List<BatchTaskItem> stuck = new ArrayList<>();
|
||||||
for (BatchTaskItem task : candidates) {
|
for (BatchTaskItem task : candidates) {
|
||||||
if (task.updateTime() != null && !task.updateTime().isAfter(cutoff)) {
|
if (task.updateTime() != null && !task.updateTime().isAfter(cutoff)) {
|
||||||
@@ -135,7 +135,7 @@ public class MaixiangAnomalyScanner {
|
|||||||
push(audience, "麦象批量任务停滞", content, "maixiang_stuck_batch:" + now.format(DAY_FORMAT));
|
push(audience, "麦象批量任务停滞", content, "maixiang_stuck_batch:" + now.format(DAY_FORMAT));
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 单任务滞留:创建早于阈值、仍处 status=0/1(跟价等单任务应秒级完成)。 */
|
/** 单任务滞留:最后更新早于阈值、仍处 status=0/1(跟价等单任务应秒级完成)。 */
|
||||||
void checkStuckSingleTasks(NotificationDispatchService.AdminAudience audience, LocalDateTime now) {
|
void checkStuckSingleTasks(NotificationDispatchService.AdminAudience audience, LocalDateTime now) {
|
||||||
int thresholdMinutes = Math.max(1, properties.getMaixiangSingleStuckMinutes());
|
int thresholdMinutes = Math.max(1, properties.getMaixiangSingleStuckMinutes());
|
||||||
LocalDateTime cutoff = now.minusMinutes(thresholdMinutes);
|
LocalDateTime cutoff = now.minusMinutes(thresholdMinutes);
|
||||||
|
|||||||
+5
-3
@@ -273,15 +273,17 @@ public class NotificationScanScheduler {
|
|||||||
"service_down:" + serviceKey + ":" + hour, null);
|
"service_down:" + serviceKey + ":" + hour, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 已读通知保留期清理:每天最多一次。 */
|
/** 通知保留期清理:每天最多一次;已读与未读各按自己的保留天数。 */
|
||||||
private void cleanupExpiredIfNeeded() {
|
private void cleanupExpiredIfNeeded() {
|
||||||
LocalDate today = LocalDate.now();
|
LocalDate today = LocalDate.now();
|
||||||
if (today.equals(lastCleanupDate)) {
|
if (today.equals(lastCleanupDate)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
lastCleanupDate = today;
|
lastCleanupDate = today;
|
||||||
LocalDateTime cutoff = LocalDateTime.now().minusDays(Math.max(1, properties.getReadRetentionDays()));
|
LocalDateTime readCutoff = LocalDateTime.now().minusDays(Math.max(1, properties.getReadRetentionDays()));
|
||||||
notificationService.cleanupReadBefore(cutoff);
|
notificationService.cleanupReadBefore(readCutoff);
|
||||||
|
LocalDateTime unreadCutoff = LocalDateTime.now().minusDays(Math.max(1, properties.getUnreadRetentionDays()));
|
||||||
|
notificationService.cleanupUnreadBefore(unreadCutoff);
|
||||||
}
|
}
|
||||||
|
|
||||||
private String dedupeKeyOf(BucketKey key) {
|
private String dedupeKeyOf(BucketKey key) {
|
||||||
|
|||||||
+86
-7
@@ -15,6 +15,7 @@ import org.springframework.stereotype.Service;
|
|||||||
import java.time.LocalDateTime;
|
import java.time.LocalDateTime;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 站内通知存取:桌面端用户(audience=user)与后台管理员(audience=admin)共用一张表,
|
* 站内通知存取:桌面端用户(audience=user)与后台管理员(audience=admin)共用一张表,
|
||||||
@@ -40,6 +41,27 @@ public class NotificationService {
|
|||||||
public static final String SCENE_MAIXIANG_ANOMALY = "maixiang_anomaly";
|
public static final String SCENE_MAIXIANG_ANOMALY = "maixiang_anomaly";
|
||||||
public static final String SCENE_SYSTEM = "system";
|
public static final String SCENE_SYSTEM = "system";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 通知大类:scene 太细(技术语义),铃铛按用户看得懂的四类归并筛选。
|
||||||
|
* null/空分类表示不筛选,见 {@link #scenesOfCategory}。
|
||||||
|
*/
|
||||||
|
public static final String CATEGORY_SYSTEM_ERROR = "system_error";
|
||||||
|
public static final String CATEGORY_TASK_ERROR = "task_error";
|
||||||
|
public static final String CATEGORY_CONFIG_ERROR = "config_error";
|
||||||
|
public static final String CATEGORY_SYSTEM_NOTICE = "system_notice";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* scene → 大类。未知 scene(历史遗留或新增未登记)归入「系统通知」,
|
||||||
|
* 保证任一通知只属于一类、筛选项互斥且完备。
|
||||||
|
*/
|
||||||
|
private static final Map<String, String> SCENE_TO_CATEGORY = Map.of(
|
||||||
|
SCENE_SERVICE_DOWN, CATEGORY_SYSTEM_ERROR,
|
||||||
|
SCENE_MAIXIANG_ANOMALY, CATEGORY_SYSTEM_ERROR,
|
||||||
|
SCENE_TASK_FAILED, CATEGORY_TASK_ERROR,
|
||||||
|
SCENE_SECRET_BALANCE, CATEGORY_CONFIG_ERROR,
|
||||||
|
SCENE_SECRET_INVALID, CATEGORY_CONFIG_ERROR,
|
||||||
|
SCENE_SYSTEM, CATEGORY_SYSTEM_NOTICE);
|
||||||
|
|
||||||
private static final long MAX_PAGE_SIZE = 100L;
|
private static final long MAX_PAGE_SIZE = 100L;
|
||||||
private static final int TITLE_MAX_LENGTH = 128;
|
private static final int TITLE_MAX_LENGTH = 128;
|
||||||
private static final int CONTENT_MAX_LENGTH = 512;
|
private static final int CONTENT_MAX_LENGTH = 512;
|
||||||
@@ -143,12 +165,35 @@ public class NotificationService {
|
|||||||
vo.setPage(safePage);
|
vo.setPage(safePage);
|
||||||
vo.setPageSize(safeSize);
|
vo.setPageSize(safeSize);
|
||||||
vo.setUnreadCount(unreadCount(userId, audience));
|
vo.setUnreadCount(unreadCount(userId, audience));
|
||||||
log.info("[notification] 列表查询 userId={} audience={} page={} size={} onlyUnread={} keyword={} 起={} 止={} 命中={}",
|
log.info("[notification] 列表查询 userId={} audience={} page={} size={} onlyUnread={} category={} keyword={} 起={} 止={} 命中={}",
|
||||||
userId, audience, safePage, safeSize, onlyUnread, normalize(safe.getKeyword()),
|
userId, audience, safePage, safeSize, onlyUnread, normalize(safe.getCategory()),
|
||||||
safe.getStartDate(), safe.getEndDate(), total);
|
normalize(safe.getKeyword()), safe.getStartDate(), safe.getEndDate(), total);
|
||||||
return vo;
|
return vo;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** scene → 用户可见大类;未知 scene(历史遗留/新增未登记)一律归「系统通知」。 */
|
||||||
|
public static String categoryOf(String scene) {
|
||||||
|
return SCENE_TO_CATEGORY.getOrDefault(normalize(scene), CATEGORY_SYSTEM_NOTICE);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 大类对应的 scene 集合,供查询侧做 scene IN (...) 过滤。
|
||||||
|
* 空值或未知大类返回空列表,调用方据此跳过该筛选(不报错,避免前端传错值就查不到数据)。
|
||||||
|
*/
|
||||||
|
public static List<String> scenesOfCategory(String category) {
|
||||||
|
String wanted = normalize(category);
|
||||||
|
if (wanted.isEmpty()) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
List<String> scenes = new ArrayList<>();
|
||||||
|
for (Map.Entry<String, String> entry : SCENE_TO_CATEGORY.entrySet()) {
|
||||||
|
if (entry.getValue().equals(wanted)) {
|
||||||
|
scenes.add(entry.getKey());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return scenes;
|
||||||
|
}
|
||||||
|
|
||||||
/** 摘要:未读数 + 最新通知 id(前端轮询判断是否有新通知)。 */
|
/** 摘要:未读数 + 最新通知 id(前端轮询判断是否有新通知)。 */
|
||||||
public NotificationSummaryVo summary(Long userId, String audience) {
|
public NotificationSummaryVo summary(Long userId, String audience) {
|
||||||
NotificationSummaryVo vo = new NotificationSummaryVo();
|
NotificationSummaryVo vo = new NotificationSummaryVo();
|
||||||
@@ -217,6 +262,17 @@ public class NotificationService {
|
|||||||
return deleted;
|
return deleted;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** 清理指定时间之前仍未读的通知(保留期由调用方决定,比已读给得更宽)。 */
|
||||||
|
public int cleanupUnreadBefore(LocalDateTime cutoff) {
|
||||||
|
int deleted = userNotificationMapper.delete(new LambdaQueryWrapper<UserNotificationEntity>()
|
||||||
|
.isNull(UserNotificationEntity::getReadAt)
|
||||||
|
.lt(UserNotificationEntity::getCreatedAt, cutoff));
|
||||||
|
if (deleted > 0) {
|
||||||
|
log.info("[notification] 清理历史未读通知 cutoff={} 删除={} 条", cutoff, deleted);
|
||||||
|
}
|
||||||
|
return deleted;
|
||||||
|
}
|
||||||
|
|
||||||
private LambdaQueryWrapper<UserNotificationEntity> baseWrapper(Long userId, String audience, boolean onlyUnread,
|
private LambdaQueryWrapper<UserNotificationEntity> baseWrapper(Long userId, String audience, boolean onlyUnread,
|
||||||
NotificationPageQuery query) {
|
NotificationPageQuery query) {
|
||||||
LambdaQueryWrapper<UserNotificationEntity> wrapper = new LambdaQueryWrapper<UserNotificationEntity>()
|
LambdaQueryWrapper<UserNotificationEntity> wrapper = new LambdaQueryWrapper<UserNotificationEntity>()
|
||||||
@@ -231,9 +287,11 @@ public class NotificationService {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 列表筛选:关键字模糊匹配标题/内容;日期按「年月日」闭区间
|
* 列表筛选:关键字模糊匹配标题/内容;日期按「年月日」闭区间
|
||||||
* (起日 00:00 起含、止日次日 00:00 前不含,避免当天 23:59 漏行)。
|
* (起日 00:00 起含、止日次日 00:00 前不含,避免当天 23:59 漏行);
|
||||||
|
* 类型按大类映射成 scene 集合过滤。
|
||||||
*/
|
*/
|
||||||
private void applyFilters(LambdaQueryWrapper<UserNotificationEntity> wrapper, NotificationPageQuery query) {
|
private void applyFilters(LambdaQueryWrapper<UserNotificationEntity> wrapper, NotificationPageQuery query) {
|
||||||
|
applyCategoryFilter(wrapper, query.getCategory());
|
||||||
String keyword = normalize(query.getKeyword());
|
String keyword = normalize(query.getKeyword());
|
||||||
if (!keyword.isEmpty()) {
|
if (!keyword.isEmpty()) {
|
||||||
wrapper.and(nested -> nested.like(UserNotificationEntity::getTitle, keyword)
|
wrapper.and(nested -> nested.like(UserNotificationEntity::getTitle, keyword)
|
||||||
@@ -247,8 +305,28 @@ public class NotificationService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean existsByDedupeKey(String dedupeKey) {
|
/**
|
||||||
return selectByDedupeKey(dedupeKey) != null;
|
* 类型筛选:「系统通知」是兜底类,除 scene=system 外还要包含所有未登记 scene
|
||||||
|
* (否则直接改库塞入的 scene 在筛选时会凭空消失),故表达为「= system 或 不在其它三类里」;
|
||||||
|
* 未知分类一律不追加条件,避免前端传错值就查不到任何数据。
|
||||||
|
*/
|
||||||
|
private void applyCategoryFilter(LambdaQueryWrapper<UserNotificationEntity> wrapper, String category) {
|
||||||
|
String wanted = normalize(category);
|
||||||
|
List<String> scenes = wanted.isEmpty() ? List.of() : scenesOfCategory(wanted);
|
||||||
|
if (scenes.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (CATEGORY_SYSTEM_NOTICE.equals(wanted)) {
|
||||||
|
List<String> classified = new ArrayList<>(SCENE_TO_CATEGORY.keySet());
|
||||||
|
classified.remove(SCENE_SYSTEM);
|
||||||
|
wrapper.and(nested -> nested.eq(UserNotificationEntity::getScene, SCENE_SYSTEM)
|
||||||
|
.or().notIn(UserNotificationEntity::getScene, classified));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
wrapper.in(UserNotificationEntity::getScene, scenes);
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean existsByDedupeKey(String dedupeKey) { return selectByDedupeKey(dedupeKey) != null;
|
||||||
}
|
}
|
||||||
|
|
||||||
private UserNotificationEntity selectByDedupeKey(String dedupeKey) {
|
private UserNotificationEntity selectByDedupeKey(String dedupeKey) {
|
||||||
@@ -262,6 +340,7 @@ public class NotificationService {
|
|||||||
NotificationItemVo vo = new NotificationItemVo();
|
NotificationItemVo vo = new NotificationItemVo();
|
||||||
vo.setId(row.getId());
|
vo.setId(row.getId());
|
||||||
vo.setScene(row.getScene());
|
vo.setScene(row.getScene());
|
||||||
|
vo.setCategory(categoryOf(row.getScene()));
|
||||||
vo.setLevel(row.getLevel());
|
vo.setLevel(row.getLevel());
|
||||||
vo.setTitle(row.getTitle());
|
vo.setTitle(row.getTitle());
|
||||||
vo.setContent(row.getContent());
|
vo.setContent(row.getContent());
|
||||||
@@ -271,7 +350,7 @@ public class NotificationService {
|
|||||||
return vo;
|
return vo;
|
||||||
}
|
}
|
||||||
|
|
||||||
private String normalize(String value) {
|
private static String normalize(String value) {
|
||||||
return value == null ? "" : value.trim();
|
return value == null ? "" : value.trim();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+2
-2
@@ -56,8 +56,8 @@ public class PermissionMenuController {
|
|||||||
@Operation(summary = "查询菜单权限列表")
|
@Operation(summary = "查询菜单权限列表")
|
||||||
public ApiResponse<List<PermissionMenuItemVo>> listMenus(HttpServletRequest request,
|
public ApiResponse<List<PermissionMenuItemVo>> listMenus(HttpServletRequest request,
|
||||||
@Parameter(description = "菜单类型: app/admin") @RequestParam(required = false) String menuType) {
|
@Parameter(description = "菜单类型: app/admin") @RequestParam(required = false) String menuType) {
|
||||||
requireAdmin(request);
|
// 传入操作者:授权树据此把「非超管无权授予」的菜单标记为不可勾选
|
||||||
return ApiResponse.success(permissionMenuService.list(menuType));
|
return ApiResponse.success(permissionMenuService.list(requireAdmin(request), menuType));
|
||||||
}
|
}
|
||||||
|
|
||||||
@PostMapping("/permission-menus")
|
@PostMapping("/permission-menus")
|
||||||
|
|||||||
+9
@@ -1,5 +1,6 @@
|
|||||||
package com.nanri.aiimage.modules.permission.model.entity;
|
package com.nanri.aiimage.modules.permission.model.entity;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.annotation.FieldStrategy;
|
||||||
import com.baomidou.mybatisplus.annotation.IdType;
|
import com.baomidou.mybatisplus.annotation.IdType;
|
||||||
import com.baomidou.mybatisplus.annotation.TableField;
|
import com.baomidou.mybatisplus.annotation.TableField;
|
||||||
import com.baomidou.mybatisplus.annotation.TableId;
|
import com.baomidou.mybatisplus.annotation.TableId;
|
||||||
@@ -22,4 +23,12 @@ public class PermissionMenuEntity {
|
|||||||
private String routePath;
|
private String routePath;
|
||||||
private Integer sortOrder;
|
private Integer sortOrder;
|
||||||
private LocalDateTime createdAt;
|
private LocalDateTime createdAt;
|
||||||
|
/**
|
||||||
|
* 更新时间(V131 新增):由数据库维护(DEFAULT CURRENT_TIMESTAMP + ON UPDATE CURRENT_TIMESTAMP)。
|
||||||
|
*
|
||||||
|
* <p>禁止应用显式写:MySQL 在 UPDATE 语句显式给该列赋值时不会触发自动更新,
|
||||||
|
* 而本表写回多为 selectById → 改字段 → updateById(实体带着旧值),一旦写回就会冻结更新时间。
|
||||||
|
*/
|
||||||
|
@TableField(value = "updated_at", insertStrategy = FieldStrategy.NEVER, updateStrategy = FieldStrategy.NEVER)
|
||||||
|
private LocalDateTime updatedAt;
|
||||||
}
|
}
|
||||||
|
|||||||
+8
@@ -24,4 +24,12 @@ public class PermissionMenuItemVo {
|
|||||||
private Integer sortOrder;
|
private Integer sortOrder;
|
||||||
@JsonProperty("created_at")
|
@JsonProperty("created_at")
|
||||||
private LocalDateTime createdAt;
|
private LocalDateTime createdAt;
|
||||||
|
@JsonProperty("updated_at")
|
||||||
|
private LocalDateTime updatedAt;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 当前操作者能否把该菜单授予他人。仅授权树接口(list)填充;菜单 CRUD 回显等
|
||||||
|
* 其他接口留空,语义为「不限制」。
|
||||||
|
*/
|
||||||
|
private Boolean grantable;
|
||||||
}
|
}
|
||||||
|
|||||||
+99
-7
@@ -57,8 +57,35 @@ public class PermissionMenuService {
|
|||||||
|
|
||||||
/** Returns the flat menu catalog, including each item's direct parent ID. */
|
/** Returns the flat menu catalog, including each item's direct parent ID. */
|
||||||
public List<PermissionMenuItemVo> list(String menuType) {
|
public List<PermissionMenuItemVo> list(String menuType) {
|
||||||
|
return list(null, menuType);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 授权树用:非超管只能授予自己已持有的菜单(与 {@link #ensureGrantable} 同一判据),
|
||||||
|
* 这里把结论标进 {@code grantable},让前端把不可授予的节点置灰,避免「勾了才在保存时
|
||||||
|
* 整单被 403 回滚」——创建用户与保存权限共用这一棵树,勾到一个越权项会连建号一起失败。
|
||||||
|
* operator 为空或超管时不限制。
|
||||||
|
*/
|
||||||
|
public List<PermissionMenuItemVo> list(AdminUserEntity operator, String menuType) {
|
||||||
List<PermissionMenuEntity> menus = loadMenus(menuType);
|
List<PermissionMenuEntity> menus = loadMenus(menuType);
|
||||||
return toItemVos(menus, menus);
|
List<PermissionMenuItemVo> items = toItemVos(menus, menus);
|
||||||
|
Set<Long> grantableIds = resolveGrantableMenuIds(operator);
|
||||||
|
for (PermissionMenuItemVo item : items) {
|
||||||
|
item.setGrantable(grantableIds == null || grantableIds.contains(item.getId()));
|
||||||
|
}
|
||||||
|
return items;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 操作者可授予的菜单 id 全集(自身直接授权 + 其后代)。
|
||||||
|
*
|
||||||
|
* @return null 表示不限制(超管或无操作者)
|
||||||
|
*/
|
||||||
|
private Set<Long> resolveGrantableMenuIds(AdminUserEntity operator) {
|
||||||
|
if (operator == null || operator.getId() == null || isSuperAdmin(operator)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return expandDescendantIds(new LinkedHashSet<>(loadDirectColumnIds(operator.getId())), loadMenus(null));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -262,6 +289,20 @@ public class PermissionMenuService {
|
|||||||
public List<PermissionMenuItemVo> getUserColumnPermissions(AdminUserEntity operator,
|
public List<PermissionMenuItemVo> getUserColumnPermissions(AdminUserEntity operator,
|
||||||
Long userId,
|
Long userId,
|
||||||
String menuType) {
|
String menuType) {
|
||||||
|
return getUserColumnPermissions(operator, userId, menuType, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Controller-facing variant with target-user scope checks.
|
||||||
|
*
|
||||||
|
* <p>includeAncestorGroups=true 时额外把「有可见后代的祖先节点」并入返回集合,仅用于
|
||||||
|
* 后台侧边栏还原二级分组层级(分组自身无页面路由,不代表授权扩展);权限判定与客户端
|
||||||
|
* 「组键命中即整组放行」的权限键集合语义都不受影响,默认 false 保持原行为。</p>
|
||||||
|
*/
|
||||||
|
public List<PermissionMenuItemVo> getUserColumnPermissions(AdminUserEntity operator,
|
||||||
|
Long userId,
|
||||||
|
String menuType,
|
||||||
|
boolean includeAncestorGroups) {
|
||||||
AdminUserEntity user = getUserById(userId);
|
AdminUserEntity user = getUserById(userId);
|
||||||
ensureTargetAccessible(operator, user);
|
ensureTargetAccessible(operator, user);
|
||||||
List<PermissionMenuEntity> menus = loadMenus(menuType);
|
List<PermissionMenuEntity> menus = loadMenus(menuType);
|
||||||
@@ -273,8 +314,9 @@ public class PermissionMenuService {
|
|||||||
return List.of();
|
return List.of();
|
||||||
}
|
}
|
||||||
Set<Long> effectiveIds = expandDescendantIds(directIds, menus);
|
Set<Long> effectiveIds = expandDescendantIds(directIds, menus);
|
||||||
|
Set<Long> visibleIds = includeAncestorGroups ? includeAncestorIds(effectiveIds, menus) : effectiveIds;
|
||||||
List<PermissionMenuEntity> effectiveMenus = menus.stream()
|
List<PermissionMenuEntity> effectiveMenus = menus.stream()
|
||||||
.filter(menu -> menu.getId() != null && effectiveIds.contains(menu.getId()))
|
.filter(menu -> menu.getId() != null && visibleIds.contains(menu.getId()))
|
||||||
.toList();
|
.toList();
|
||||||
return toItemVos(effectiveMenus, menus);
|
return toItemVos(effectiveMenus, menus);
|
||||||
}
|
}
|
||||||
@@ -528,7 +570,7 @@ public class PermissionMenuService {
|
|||||||
.filter(id -> !protectedIds.contains(id))
|
.filter(id -> !protectedIds.contains(id))
|
||||||
.toList();
|
.toList();
|
||||||
}
|
}
|
||||||
ensureGrantable(operator, grantIds);
|
ensureGrantable(operator, grantIds, userId);
|
||||||
LinkedHashSet<Long> finalGrantIds = new LinkedHashSet<>(grantIds);
|
LinkedHashSet<Long> finalGrantIds = new LinkedHashSet<>(grantIds);
|
||||||
|
|
||||||
if (!protectedIds.isEmpty()) {
|
if (!protectedIds.isEmpty()) {
|
||||||
@@ -724,6 +766,44 @@ public class PermissionMenuService {
|
|||||||
return effective;
|
return effective;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 把可见节点沿 parentId 链上的祖先并入集合,仅供后台侧边栏还原「一级分组 + 子页面」
|
||||||
|
* 的展示层级:分组节点无页面路由、不代表授权扩展,权限判定另有独立方法。父链缺失
|
||||||
|
* (孤儿数据)时截断;出现自环/成环时由 visited 兜底终止。
|
||||||
|
*/
|
||||||
|
private Set<Long> includeAncestorIds(Set<Long> effectiveIds, List<PermissionMenuEntity> menus) {
|
||||||
|
Map<Long, PermissionMenuEntity> menuMap = new HashMap<>();
|
||||||
|
for (PermissionMenuEntity menu : menus) {
|
||||||
|
if (menu.getId() != null) {
|
||||||
|
menuMap.put(menu.getId(), menu);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Set<Long> withAncestors = new LinkedHashSet<>(effectiveIds);
|
||||||
|
int addedCount = 0;
|
||||||
|
for (Long id : effectiveIds) {
|
||||||
|
PermissionMenuEntity current = menuMap.get(id);
|
||||||
|
Set<Long> visited = new HashSet<>();
|
||||||
|
while (current != null && current.getId() != null && visited.add(current.getId())) {
|
||||||
|
Long parentId = current.getParentId();
|
||||||
|
if (parentId == null) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
PermissionMenuEntity parent = menuMap.get(parentId);
|
||||||
|
if (parent == null) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (withAncestors.add(parent.getId())) {
|
||||||
|
addedCount++;
|
||||||
|
}
|
||||||
|
current = parent;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (addedCount > 0) {
|
||||||
|
log.debug("[menu-tree-ancestor] 侧边栏树补充祖先分组节点 {} 个(仅展示层级,不改变授权)", addedCount);
|
||||||
|
}
|
||||||
|
return withAncestors;
|
||||||
|
}
|
||||||
|
|
||||||
private PermissionMenuEntity getMenuById(Long id) {
|
private PermissionMenuEntity getMenuById(Long id) {
|
||||||
PermissionMenuEntity entity = permissionMenuMapper.selectById(id);
|
PermissionMenuEntity entity = permissionMenuMapper.selectById(id);
|
||||||
if (entity == null) {
|
if (entity == null) {
|
||||||
@@ -780,14 +860,25 @@ public class PermissionMenuService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Throws 403 when a non-super admin requests grants outside their own effective set. */
|
/**
|
||||||
private void ensureGrantable(AdminUserEntity operator, List<Long> requestedIds) {
|
* Throws 403 when a non-super admin **新增** grants outside their own effective set.
|
||||||
|
*
|
||||||
|
* <p>目标用户已持有的直接授权不参与校验:那是既有事实(通常由超管分配),普通管理员
|
||||||
|
* 编辑该用户时整树提交会把它原样带回,若一并判为越权,整笔事务会回滚——连建号、改密
|
||||||
|
* 都做不成(生产 2026-09-16 现象:勾到一个越权项,创建用户与保存权限双双失败)。
|
||||||
|
* 只放行「保留已有」不放行「新增」,因此不构成提权。
|
||||||
|
*/
|
||||||
|
private void ensureGrantable(AdminUserEntity operator, List<Long> requestedIds, Long targetUserId) {
|
||||||
if (operator == null || isSuperAdmin(operator)) {
|
if (operator == null || isSuperAdmin(operator)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
List<PermissionMenuEntity> menus = loadMenus(null);
|
// 操作者自身 id 缺失(异常数据)时按「无任何可授予项」从严处理,不放行
|
||||||
Set<Long> effective = expandDescendantIds(new LinkedHashSet<>(loadDirectColumnIds(operator.getId())), menus);
|
Set<Long> effective = Objects.requireNonNullElse(resolveGrantableMenuIds(operator), Set.of());
|
||||||
|
Set<Long> keptIds = targetUserId == null
|
||||||
|
? Set.of()
|
||||||
|
: new LinkedHashSet<>(loadDirectColumnIds(targetUserId));
|
||||||
Set<Long> denied = requestedIds.stream()
|
Set<Long> denied = requestedIds.stream()
|
||||||
|
.filter(id -> !keptIds.contains(id))
|
||||||
.filter(id -> !effective.contains(id))
|
.filter(id -> !effective.contains(id))
|
||||||
.collect(Collectors.toCollection(LinkedHashSet::new));
|
.collect(Collectors.toCollection(LinkedHashSet::new));
|
||||||
if (!denied.isEmpty()) {
|
if (!denied.isEmpty()) {
|
||||||
@@ -1069,6 +1160,7 @@ public class PermissionMenuService {
|
|||||||
vo.setRoutePath(entity.getRoutePath());
|
vo.setRoutePath(entity.getRoutePath());
|
||||||
vo.setSortOrder(entity.getSortOrder());
|
vo.setSortOrder(entity.getSortOrder());
|
||||||
vo.setCreatedAt(entity.getCreatedAt());
|
vo.setCreatedAt(entity.getCreatedAt());
|
||||||
|
vo.setUpdatedAt(entity.getUpdatedAt());
|
||||||
return vo;
|
return vo;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+18
@@ -2,8 +2,26 @@ package com.nanri.aiimage.modules.pricetrack.mapper;
|
|||||||
|
|
||||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||||
import com.nanri.aiimage.modules.pricetrack.model.entity.PriceTrackLoopRunEntity;
|
import com.nanri.aiimage.modules.pricetrack.model.entity.PriceTrackLoopRunEntity;
|
||||||
|
import org.apache.ibatis.annotations.Delete;
|
||||||
import org.apache.ibatis.annotations.Mapper;
|
import org.apache.ibatis.annotations.Mapper;
|
||||||
|
import org.apache.ibatis.annotations.Param;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
@Mapper
|
@Mapper
|
||||||
public interface PriceTrackLoopRunMapper extends BaseMapper<PriceTrackLoopRunEntity> {
|
public interface PriceTrackLoopRunMapper extends BaseMapper<PriceTrackLoopRunEntity> {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 分批删除已结束且超过保留期的循环批次行。
|
||||||
|
*
|
||||||
|
* <p>只删终态行:RUNNING 的批次删了会让后续 dispatch / childFinished 找不到记录。
|
||||||
|
* 时间线用 COALESCE 兜底——终态行本应写 finished_at,历史行可能只有 updated_at。
|
||||||
|
*/
|
||||||
|
@Delete("""
|
||||||
|
DELETE FROM biz_price_track_loop_run
|
||||||
|
WHERE status IN ('SUCCESS', 'FAILED', 'STOPPED')
|
||||||
|
AND COALESCE(finished_at, updated_at, created_at) < #{cutoff}
|
||||||
|
LIMIT #{batchSize}
|
||||||
|
""")
|
||||||
|
int deleteFinishedBefore(@Param("cutoff") LocalDateTime cutoff, @Param("batchSize") int batchSize);
|
||||||
}
|
}
|
||||||
+2
@@ -29,6 +29,8 @@ public class PriceTrackLoopRunEntity {
|
|||||||
@TableField(value = "active_task_id", updateStrategy = FieldStrategy.ALWAYS)
|
@TableField(value = "active_task_id", updateStrategy = FieldStrategy.ALWAYS)
|
||||||
private Long activeTaskId;
|
private Long activeTaskId;
|
||||||
private Boolean stopRequested;
|
private Boolean stopRequested;
|
||||||
|
/** 因客户端中断自动重派当前轮的次数:封顶用,避免会话持续不可用时无限重派(V129)。 */
|
||||||
|
private Integer resumeAttempt;
|
||||||
private String errorMessage;
|
private String errorMessage;
|
||||||
private LocalDateTime createdAt;
|
private LocalDateTime createdAt;
|
||||||
private LocalDateTime updatedAt;
|
private LocalDateTime updatedAt;
|
||||||
|
|||||||
+70
@@ -0,0 +1,70 @@
|
|||||||
|
package com.nanri.aiimage.modules.pricetrack.service;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||||
|
import com.nanri.aiimage.modules.pricetrack.mapper.PriceTrackLoopRunMapper;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.scheduling.annotation.Scheduled;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 跟价循环批次表(biz_price_track_loop_run)的保留期清理。
|
||||||
|
*
|
||||||
|
* <p>每次「循环跟价」插一行,且只在请求停止时更新——全模块此前没有任何删除路径(纯遗漏)。
|
||||||
|
* 读取侧只有两个入口:按 id 取单条(用户当下正在看的那一轮)与「当前是否有活跃循环」,
|
||||||
|
* 没有任何历史列表查询,所以终态批次过了保留期即可安全删除。
|
||||||
|
*
|
||||||
|
* <p>只删终态(SUCCESS/FAILED/STOPPED)行,RUNNING 一律不动——正在跑的循环被删会让
|
||||||
|
* 后续 dispatch / childFinished 找不到记录而中断。
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
public class PriceTrackLoopRunRetentionService {
|
||||||
|
|
||||||
|
/** 单轮最多删除的批次数,剩余留给下一轮。 */
|
||||||
|
static final int MAX_BATCHES_PER_RUN = 20;
|
||||||
|
|
||||||
|
private final PriceTrackLoopRunMapper loopRunMapper;
|
||||||
|
private final DistributedJobLockService distributedJobLockService;
|
||||||
|
|
||||||
|
@Value("${aiimage.price-track.loop-run-retention-days:30}")
|
||||||
|
private int retentionDays = 30;
|
||||||
|
|
||||||
|
@Value("${aiimage.price-track.loop-run-retention-batch-size:500}")
|
||||||
|
private int retentionBatchSize = 500;
|
||||||
|
|
||||||
|
@Scheduled(cron = "${aiimage.price-track.loop-run-retention-cron:0 20 4 * * *}")
|
||||||
|
public void purgeExpiredLoopRuns() {
|
||||||
|
int days = Math.max(1, retentionDays);
|
||||||
|
int batchSize = Math.max(50, retentionBatchSize);
|
||||||
|
LocalDateTime cutoff = LocalDateTime.now().minusDays(days);
|
||||||
|
|
||||||
|
DistributedJobLockService.LockHandle lockHandle =
|
||||||
|
distributedJobLockService.tryLock("price-track:loop-run-retention", Duration.ofMinutes(15));
|
||||||
|
if (lockHandle == null) {
|
||||||
|
log.info("[price-track-loop] 批次保留清理跳过:另一实例持锁");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try (lockHandle) {
|
||||||
|
int totalDeleted = 0;
|
||||||
|
int batches = 0;
|
||||||
|
while (batches < MAX_BATCHES_PER_RUN) {
|
||||||
|
int deleted = loopRunMapper.deleteFinishedBefore(cutoff, batchSize);
|
||||||
|
batches++;
|
||||||
|
totalDeleted += deleted;
|
||||||
|
if (deleted < batchSize) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.info("[price-track-loop] 批次保留清理完成 cutoff={} retentionDays={} deleted={} batches={}",
|
||||||
|
cutoff, days, totalDeleted, batches);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[price-track-loop] 批次保留清理失败 cutoff={} msg={}", cutoff, ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+47
@@ -39,6 +39,14 @@ public class PriceTrackLoopRunService {
|
|||||||
private static final String STATUS_STOPPED = "STOPPED";
|
private static final String STATUS_STOPPED = "STOPPED";
|
||||||
private static final String EXECUTION_MODE_FINITE = "FINITE";
|
private static final String EXECUTION_MODE_FINITE = "FINITE";
|
||||||
private static final String EXECUTION_MODE_INFINITE = "INFINITE";
|
private static final String EXECUTION_MODE_INFINITE = "INFINITE";
|
||||||
|
/** 客户端异常中断的任务 errorMessage 前缀(TaskHeartbeatService.markInterrupted 写入)。 */
|
||||||
|
private static final String CLIENT_INTERRUPT_ERROR_PREFIX = "客户端异常中断";
|
||||||
|
/**
|
||||||
|
* 中断后自动重派当前轮的次数上限。
|
||||||
|
* 会话持续不可用(账号被风控、紫鸟未就绪)时,不封顶会无限重派、无限重开浏览器——
|
||||||
|
* 2026-09-18 任务 28587 就是这样白烧了 5 小时。
|
||||||
|
*/
|
||||||
|
private static final int MAX_AUTO_RESUME_ATTEMPT = 3;
|
||||||
|
|
||||||
private final PriceTrackLoopRunMapper loopRunMapper;
|
private final PriceTrackLoopRunMapper loopRunMapper;
|
||||||
private final FileTaskMapper fileTaskMapper;
|
private final FileTaskMapper fileTaskMapper;
|
||||||
@@ -259,6 +267,33 @@ public class PriceTrackLoopRunService {
|
|||||||
entity.setActiveTaskId(null);
|
entity.setActiveTaskId(null);
|
||||||
entity.setUpdatedAt(LocalDateTime.now());
|
entity.setUpdatedAt(LocalDateTime.now());
|
||||||
if (STATUS_FAILED.equals(task.getStatus())) {
|
if (STATUS_FAILED.equals(task.getStatus())) {
|
||||||
|
// 用户已请求停止时绝不续派:停止意图优先于自动恢复(否则点完停止循环还会自己转起来)
|
||||||
|
if (Boolean.TRUE.equals(entity.getStopRequested())) {
|
||||||
|
markStopped(entity, null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// 客户端重启中断(markInterrupted 写入的前缀)不终止循环:清空 active_task_id 后保持
|
||||||
|
// RUNNING,客户端下次 dispatchNext 会拿到**同一店铺、同一轮次**的 childTaskRequest,
|
||||||
|
// 等于原地续跑——页内已处理的 ASIN 由服务端 skip_asins 去重,不会重复改价。
|
||||||
|
if (isClientInterrupt(task) && currentResumeAttempt(entity) < MAX_AUTO_RESUME_ATTEMPT) {
|
||||||
|
int attempt = currentResumeAttempt(entity) + 1;
|
||||||
|
entity.setStatus(STATUS_RUNNING);
|
||||||
|
entity.setErrorMessage(null);
|
||||||
|
entity.setFinishedAt(null);
|
||||||
|
entity.setResumeAttempt(attempt);
|
||||||
|
loopRunMapper.updateById(entity);
|
||||||
|
log.warn("[price-track-loop] 子任务因客户端中断失败,自动重派当前轮 loopRunId={} childTaskId={} "
|
||||||
|
+ "round={} shopIndex={} resumeAttempt={}/{} error={}",
|
||||||
|
entity.getId(), childTaskId, entity.getCurrentRound(), entity.getCurrentShopIndex(),
|
||||||
|
attempt, MAX_AUTO_RESUME_ATTEMPT, task.getErrorMessage());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (isClientInterrupt(task)) {
|
||||||
|
log.warn("[price-track-loop] 客户端中断续跑已达上限 {} 次,终止循环 loopRunId={} childTaskId={} "
|
||||||
|
+ "round={} shopIndex={}",
|
||||||
|
MAX_AUTO_RESUME_ATTEMPT, entity.getId(), childTaskId,
|
||||||
|
entity.getCurrentRound(), entity.getCurrentShopIndex());
|
||||||
|
}
|
||||||
entity.setStatus(STATUS_FAILED);
|
entity.setStatus(STATUS_FAILED);
|
||||||
entity.setErrorMessage(task.getErrorMessage() == null || task.getErrorMessage().isBlank()
|
entity.setErrorMessage(task.getErrorMessage() == null || task.getErrorMessage().isBlank()
|
||||||
? "子任务执行失败"
|
? "子任务执行失败"
|
||||||
@@ -269,6 +304,8 @@ public class PriceTrackLoopRunService {
|
|||||||
entity.getId(), childTaskId, entity.getCurrentRound(), entity.getCurrentShopIndex(), entity.getErrorMessage());
|
entity.getId(), childTaskId, entity.getCurrentRound(), entity.getCurrentShopIndex(), entity.getErrorMessage());
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
// 子任务成功 → 中断续跑计数归零(否则历史上的中断会一直占用封顶额度)
|
||||||
|
entity.setResumeAttempt(0);
|
||||||
List<PriceTrackMatchShopsVo.PriceTrackShopQueueItem> items = parseShops(entity);
|
List<PriceTrackMatchShopsVo.PriceTrackShopQueueItem> items = parseShops(entity);
|
||||||
if (items.isEmpty()) {
|
if (items.isEmpty()) {
|
||||||
entity.setStatus(STATUS_FAILED);
|
entity.setStatus(STATUS_FAILED);
|
||||||
@@ -300,6 +337,16 @@ public class PriceTrackLoopRunService {
|
|||||||
entity.getId(), childTaskId, entity.getStatus(), entity.getCurrentRound(), entity.getCurrentShopIndex(), entity.getActiveTaskId());
|
entity.getId(), childTaskId, entity.getStatus(), entity.getCurrentRound(), entity.getCurrentShopIndex(), entity.getActiveTaskId());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** 子任务失败原因是否为「客户端异常中断」(客户端重启上报,可自动重派续跑)。 */
|
||||||
|
private static boolean isClientInterrupt(FileTaskEntity task) {
|
||||||
|
String message = task == null ? null : task.getErrorMessage();
|
||||||
|
return message != null && message.startsWith(CLIENT_INTERRUPT_ERROR_PREFIX);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int currentResumeAttempt(PriceTrackLoopRunEntity entity) {
|
||||||
|
return entity.getResumeAttempt() == null ? 0 : entity.getResumeAttempt();
|
||||||
|
}
|
||||||
|
|
||||||
private void reconcileWithTerminalChild(PriceTrackLoopRunEntity entity) {
|
private void reconcileWithTerminalChild(PriceTrackLoopRunEntity entity) {
|
||||||
if (entity == null || entity.getActiveTaskId() == null || isTerminal(entity.getStatus())) {
|
if (entity == null || entity.getActiveTaskId() == null || isTerminal(entity.getStatus())) {
|
||||||
return;
|
return;
|
||||||
|
|||||||
+74
-10
@@ -653,13 +653,11 @@ public class PriceTrackTaskService {
|
|||||||
}
|
}
|
||||||
changed = true;
|
changed = true;
|
||||||
if (payload.getError() != null && !payload.getError().isBlank()) {
|
if (payload.getError() != null && !payload.getError().isBlank()) {
|
||||||
markResultFailed(fr, payload.getError());
|
finalizeFailedShop(fr, shopKey, payload, payload.getError());
|
||||||
priceTrackTaskCacheService.removeShopMergedPayload(taskId, shopKey);
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (Boolean.FALSE.equals(payload.getSuccess())) {
|
if (Boolean.FALSE.equals(payload.getSuccess())) {
|
||||||
markResultFailed(fr, "shop processing failed");
|
finalizeFailedShop(fr, shopKey, payload, "shop processing failed");
|
||||||
priceTrackTaskCacheService.removeShopMergedPayload(taskId, shopKey);
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
handleSkipAsinDeletionSignals(shopKey, payload);
|
handleSkipAsinDeletionSignals(shopKey, payload);
|
||||||
@@ -815,6 +813,39 @@ public class PriceTrackTaskService {
|
|||||||
updateTaskStatusFromLatestRows(task, latest);
|
updateTaskStatusFromLatestRows(task, latest);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 失败店铺的收尾:任务照旧标 FAILED(原因回显给用户),但已经把跑出来的行组装成
|
||||||
|
* 部分结果文件随任务交付——否则用户只看到「失败」,却拿不到「哪些 ASIN 实际已被
|
||||||
|
* 跟价/改价」的记录(会话掉线这类"跑了前几页才断"的场景尤其需要)。
|
||||||
|
*
|
||||||
|
* <p>组装出来的行仍是失败态(success=0 + errorMessage),任务状态因此不变,
|
||||||
|
* 只是多了个可下载的文件;只有一行可用数据都没有时才退化成纯失败。
|
||||||
|
*/
|
||||||
|
private void finalizeFailedShop(FileResultEntity fr, String shopKey,
|
||||||
|
PriceTrackSubmitResultRequest.ShopResult payload, String errorMessage) {
|
||||||
|
String finalMessage = errorMessage;
|
||||||
|
try {
|
||||||
|
PriceTrackSubmitResultRequest.ShopResult merged = mergeShopPayload(fr.getTaskId(), shopKey, payload);
|
||||||
|
int rows = countPayloadRows(merged);
|
||||||
|
if (rows > 0) {
|
||||||
|
markResultFailed(fr, errorMessage);
|
||||||
|
enqueueResultFileAssembly(fr, shopKey, merged, true);
|
||||||
|
priceTrackTaskCacheService.removeShopMergedPayload(fr.getTaskId(), shopKey);
|
||||||
|
log.info("[price-track] 失败店铺仍产出部分结果 taskId={} shop={} rows={} error={}",
|
||||||
|
fr.getTaskId(), shopKey, rows, errorMessage);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
log.info("[price-track] 失败店铺无可用行,不产出结果文件 taskId={} shop={} error={}",
|
||||||
|
fr.getTaskId(), shopKey, errorMessage);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[price-track] 失败店铺部分结果排队失败,仅标记失败 taskId={} shop={} msg={}",
|
||||||
|
fr.getTaskId(), shopKey, ex.getMessage(), ex);
|
||||||
|
finalMessage = errorMessage + "(部分结果组装排队失败:" + ex.getMessage() + ")";
|
||||||
|
}
|
||||||
|
markResultFailed(fr, finalMessage);
|
||||||
|
priceTrackTaskCacheService.removeShopMergedPayload(fr.getTaskId(), shopKey);
|
||||||
|
}
|
||||||
|
|
||||||
private void markResultFailed(FileResultEntity fr, String message) {
|
private void markResultFailed(FileResultEntity fr, String message) {
|
||||||
fr.setSuccess(0);
|
fr.setSuccess(0);
|
||||||
fr.setErrorMessage(message);
|
fr.setErrorMessage(message);
|
||||||
@@ -841,11 +872,18 @@ public class PriceTrackTaskService {
|
|||||||
java.io.File workRoot = cn.hutool.core.io.FileUtil.mkdir(
|
java.io.File workRoot = cn.hutool.core.io.FileUtil.mkdir(
|
||||||
cn.hutool.core.io.FileUtil.file(System.getProperty("java.io.tmpdir"), "price-track-result", String.valueOf(result.getTaskId())));
|
cn.hutool.core.io.FileUtil.file(System.getProperty("java.io.tmpdir"), "price-track-result", String.valueOf(result.getTaskId())));
|
||||||
java.io.File xlsx = cn.hutool.core.io.FileUtil.file(workRoot, stem + ".xlsx");
|
java.io.File xlsx = cn.hutool.core.io.FileUtil.file(workRoot, stem + ".xlsx");
|
||||||
|
// 失败店铺的部分结果(errorMessage 已写明原因):组装完成时保留失败态,只挂文件。
|
||||||
|
// 否则组装一落地就把行"洗成成功",用户再也看不到「这个店铺其实没跑完」。
|
||||||
|
boolean partialFailure = result.getErrorMessage() != null && !result.getErrorMessage().isBlank();
|
||||||
try {
|
try {
|
||||||
excelAssemblyService.writeWorkbook(xlsx, countries);
|
excelAssemblyService.writeWorkbook(xlsx, countries);
|
||||||
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
|
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
|
||||||
result.setSuccess(1);
|
if (partialFailure) {
|
||||||
result.setErrorMessage(null);
|
result.setSuccess(0);
|
||||||
|
} else {
|
||||||
|
result.setSuccess(1);
|
||||||
|
result.setErrorMessage(null);
|
||||||
|
}
|
||||||
result.setResultFilename(stem + ".xlsx");
|
result.setResultFilename(stem + ".xlsx");
|
||||||
result.setResultFileUrl(objectKey);
|
result.setResultFileUrl(objectKey);
|
||||||
result.setResultFileSize(xlsx.length());
|
result.setResultFileSize(xlsx.length());
|
||||||
@@ -885,9 +923,20 @@ public class PriceTrackTaskService {
|
|||||||
private void enqueueResultFileAssembly(FileResultEntity result,
|
private void enqueueResultFileAssembly(FileResultEntity result,
|
||||||
String shopKey,
|
String shopKey,
|
||||||
PriceTrackSubmitResultRequest.ShopResult payload) {
|
PriceTrackSubmitResultRequest.ShopResult payload) {
|
||||||
|
enqueueResultFileAssembly(result, shopKey, payload, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param preserveFailure 失败店铺的部分结果:保留 success=0 + 失败原因,只把文件挂上去,
|
||||||
|
* 任务状态不变(仍 FAILED),用户仍能下载已跑出来的行
|
||||||
|
*/
|
||||||
|
private void enqueueResultFileAssembly(FileResultEntity result,
|
||||||
|
String shopKey,
|
||||||
|
PriceTrackSubmitResultRequest.ShopResult payload,
|
||||||
|
boolean preserveFailure) {
|
||||||
applyServerModifyCounts(result.getTaskId(), payload);
|
applyServerModifyCounts(result.getTaskId(), payload);
|
||||||
taskResultPayloadService.saveLatest(result.getTaskId(), MODULE_TYPE, shopKey, payload);
|
taskResultPayloadService.saveLatest(result.getTaskId(), MODULE_TYPE, shopKey, payload);
|
||||||
markResultFilePending(result, shopKey, payload);
|
markResultFilePending(result, shopKey, payload, preserveFailure);
|
||||||
taskFileJobService.enqueueAssembleResult(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey);
|
taskFileJobService.enqueueAssembleResult(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -993,15 +1042,24 @@ public class PriceTrackTaskService {
|
|||||||
|
|
||||||
private void markResultFilePending(FileResultEntity result,
|
private void markResultFilePending(FileResultEntity result,
|
||||||
String shopKey,
|
String shopKey,
|
||||||
PriceTrackSubmitResultRequest.ShopResult payload) {
|
PriceTrackSubmitResultRequest.ShopResult payload,
|
||||||
|
boolean preserveFailure) {
|
||||||
Map<String, List<PriceTrackSubmitResultRequest.AsinResult>> countries =
|
Map<String, List<PriceTrackSubmitResultRequest.AsinResult>> countries =
|
||||||
excelAssemblyService.normalizeCountriesMap(payload.getCountries());
|
excelAssemblyService.normalizeCountriesMap(payload.getCountries());
|
||||||
String displayName = payload.getShopName() != null && !payload.getShopName().isBlank()
|
String displayName = payload.getShopName() != null && !payload.getShopName().isBlank()
|
||||||
? payload.getShopName().trim()
|
? payload.getShopName().trim()
|
||||||
: shopKey;
|
: shopKey;
|
||||||
String stem = safeFileStem(displayName);
|
String stem = safeFileStem(displayName);
|
||||||
result.setSuccess(1);
|
if (preserveFailure) {
|
||||||
result.setErrorMessage(null);
|
// 失败店铺的部分结果:成功态与失败原因都不能动,只标「文件名已定、文件待组装」
|
||||||
|
result.setSuccess(0);
|
||||||
|
if (result.getErrorMessage() == null || result.getErrorMessage().isBlank()) {
|
||||||
|
result.setErrorMessage("店铺未跑完,仅产出部分结果");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
result.setSuccess(1);
|
||||||
|
result.setErrorMessage(null);
|
||||||
|
}
|
||||||
result.setResultFilename(stem + ".xlsx");
|
result.setResultFilename(stem + ".xlsx");
|
||||||
result.setResultFileUrl(null);
|
result.setResultFileUrl(null);
|
||||||
result.setResultFileSize(0L);
|
result.setResultFileSize(0L);
|
||||||
@@ -2164,6 +2222,12 @@ public class PriceTrackTaskService {
|
|||||||
}
|
}
|
||||||
ok++;
|
ok++;
|
||||||
} else if (failed) {
|
} else if (failed) {
|
||||||
|
// 失败行也可能正在组装"部分结果"文件:文件没落地前不能让任务提前终态,
|
||||||
|
// 否则前端一停轮询,下载按钮永远不出现(失败任务的结果文件同样要能下载)
|
||||||
|
if (isResultAwaitingFileAssembly(fr, jobMap.get(fr.getId()))) {
|
||||||
|
allDone = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
fail++;
|
fail++;
|
||||||
allErrors.add(fr.getSourceFilename() + ": " + fr.getErrorMessage());
|
allErrors.add(fr.getSourceFilename() + ": " + fr.getErrorMessage());
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
+68
-12
@@ -652,9 +652,8 @@ public class ProductRiskTaskService {
|
|||||||
}
|
}
|
||||||
matchedShopCount++;
|
matchedShopCount++;
|
||||||
if (payload.getError() != null && !payload.getError().isBlank()) {
|
if (payload.getError() != null && !payload.getError().isBlank()) {
|
||||||
markResultFailed(fr, payload.getError());
|
finalizeFailedShop(fr, shopKey, mergeShopPayload(taskId, shopKey, payload),
|
||||||
batchErrors.add(shopKey + ": " + payload.getError());
|
payload.getError(), batchErrors);
|
||||||
productRiskTaskCacheService.removeShopMergedPayload(taskId, shopKey);
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -759,9 +758,8 @@ public class ProductRiskTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (cachedPayload.getError() != null && !cachedPayload.getError().isBlank()) {
|
if (cachedPayload.getError() != null && !cachedPayload.getError().isBlank()) {
|
||||||
markResultFailed(fr, cachedPayload.getError());
|
// 陈旧收尾同样保留已跑出来的行:失败原因照常回显,文件顺手组装
|
||||||
batchErrors.add(shopKey + ": " + cachedPayload.getError());
|
finalizeFailedShop(fr, shopKey, cachedPayload, cachedPayload.getError(), batchErrors);
|
||||||
productRiskTaskCacheService.removeShopMergedPayload(taskId, shopKey);
|
|
||||||
changed = true;
|
changed = true;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -852,6 +850,9 @@ public class ProductRiskTaskService {
|
|||||||
String stem = safeFileStem(displayName);
|
String stem = safeFileStem(displayName);
|
||||||
File xlsx = FileUtil.file(workRoot, stem + ".xlsx");
|
File xlsx = FileUtil.file(workRoot, stem + ".xlsx");
|
||||||
File zip = FileUtil.file(workRoot, stem + ".zip");
|
File zip = FileUtil.file(workRoot, stem + ".zip");
|
||||||
|
// 失败店铺的部分结果(errorMessage 已写明原因):组装完成时保留失败态,只挂文件。
|
||||||
|
// 否则组装一落地就把行"洗成成功",用户再也看不到「这个店铺其实没跑完」。
|
||||||
|
boolean partialFailure = fr.getErrorMessage() != null && !fr.getErrorMessage().isBlank();
|
||||||
try {
|
try {
|
||||||
excelAssemblyService.writeWorkbook(xlsx, displayName, countries);
|
excelAssemblyService.writeWorkbook(xlsx, displayName, countries);
|
||||||
ZipUtil.zip(zip, false, xlsx);
|
ZipUtil.zip(zip, false, xlsx);
|
||||||
@@ -861,8 +862,12 @@ public class ProductRiskTaskService {
|
|||||||
fr.setResultFileSize(zip.length());
|
fr.setResultFileSize(zip.length());
|
||||||
fr.setResultContentType(CONTENT_TYPE_ZIP);
|
fr.setResultContentType(CONTENT_TYPE_ZIP);
|
||||||
fr.setRowCount(excelAssemblyService.countRows(countries));
|
fr.setRowCount(excelAssemblyService.countRows(countries));
|
||||||
fr.setSuccess(1);
|
if (partialFailure) {
|
||||||
fr.setErrorMessage(null);
|
fr.setSuccess(0);
|
||||||
|
} else {
|
||||||
|
fr.setSuccess(1);
|
||||||
|
fr.setErrorMessage(null);
|
||||||
|
}
|
||||||
fileResultMapper.updateById(fr);
|
fileResultMapper.updateById(fr);
|
||||||
} finally {
|
} finally {
|
||||||
FileUtil.del(xlsx);
|
FileUtil.del(xlsx);
|
||||||
@@ -899,13 +904,56 @@ public class ProductRiskTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void enqueueResultFileAssembly(FileResultEntity result, String shopKey, ProductRiskShopPayloadDto payload) {
|
private void enqueueResultFileAssembly(FileResultEntity result, String shopKey, ProductRiskShopPayloadDto payload) {
|
||||||
|
enqueueResultFileAssembly(result, shopKey, payload, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param preserveFailure 失败店铺的部分结果:保留 success=0 + 失败原因,只把文件挂上去,
|
||||||
|
* 任务状态不变(仍 FAILED),用户仍能下载已跑出来的行
|
||||||
|
*/
|
||||||
|
private void enqueueResultFileAssembly(FileResultEntity result, String shopKey,
|
||||||
|
ProductRiskShopPayloadDto payload, boolean preserveFailure) {
|
||||||
taskResultPayloadService.saveLatest(result.getTaskId(), MODULE_TYPE, shopKey, payload);
|
taskResultPayloadService.saveLatest(result.getTaskId(), MODULE_TYPE, shopKey, payload);
|
||||||
taskResultItemService.replaceResultSnapshot(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey, payload);
|
taskResultItemService.replaceResultSnapshot(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey, payload);
|
||||||
markResultFilePending(result, shopKey, payload);
|
markResultFilePending(result, shopKey, payload, preserveFailure);
|
||||||
taskFileJobService.enqueueAssembleResult(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey);
|
taskFileJobService.enqueueAssembleResult(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void markResultFilePending(FileResultEntity result, String shopKey, ProductRiskShopPayloadDto payload) {
|
/**
|
||||||
|
* 失败店铺的收尾:任务照旧标 FAILED(原因回显给用户),但已经把跑出来的行组装成
|
||||||
|
* 部分结果文件随任务交付——否则用户只看到「失败」,却拿不到「哪些 ASIN 实际已被处理」
|
||||||
|
* 的记录(会话掉线这类"跑了几个国家才断"的场景尤其需要)。一行可用数据都没有时才退化成纯失败。
|
||||||
|
*/
|
||||||
|
private void finalizeFailedShop(FileResultEntity fr, String shopKey,
|
||||||
|
ProductRiskShopPayloadDto mergedPayload, String errorMessage,
|
||||||
|
List<String> batchErrors) {
|
||||||
|
int rows = mergedPayload == null ? 0 : countPayloadRows(mergedPayload);
|
||||||
|
String message = errorMessage;
|
||||||
|
if (rows > 0) {
|
||||||
|
markResultFailed(fr, errorMessage);
|
||||||
|
try {
|
||||||
|
enqueueResultFileAssembly(fr, shopKey, mergedPayload, true);
|
||||||
|
batchErrors.add(shopKey + ": " + errorMessage);
|
||||||
|
productRiskTaskCacheService.removeShopMergedPayload(fr.getTaskId(), shopKey);
|
||||||
|
log.info("[product-risk] 失败店铺仍产出部分结果 taskId={} shop={} rows={} error={}",
|
||||||
|
fr.getTaskId(), shopKey, rows, errorMessage);
|
||||||
|
return;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
message = errorMessage + "(部分结果组装排队失败:" + (ex.getMessage() == null ? "未知错误" : ex.getMessage()) + ")";
|
||||||
|
log.warn("[product-risk] 失败店铺部分结果排队失败,仅标记失败 taskId={} shop={} msg={}",
|
||||||
|
fr.getTaskId(), shopKey, ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
log.info("[product-risk] 失败店铺无可用行,不产出结果文件 taskId={} shop={} error={}",
|
||||||
|
fr.getTaskId(), shopKey, errorMessage);
|
||||||
|
}
|
||||||
|
markResultFailed(fr, message);
|
||||||
|
batchErrors.add(shopKey + ": " + message);
|
||||||
|
productRiskTaskCacheService.removeShopMergedPayload(fr.getTaskId(), shopKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void markResultFilePending(FileResultEntity result, String shopKey,
|
||||||
|
ProductRiskShopPayloadDto payload, boolean preserveFailure) {
|
||||||
Map<String, List<ProductRiskRowDto>> countries = excelAssemblyService.normalizeCountriesMap(payload.getCountries());
|
Map<String, List<ProductRiskRowDto>> countries = excelAssemblyService.normalizeCountriesMap(payload.getCountries());
|
||||||
String displayName = payload.getShopName() != null && !payload.getShopName().isBlank()
|
String displayName = payload.getShopName() != null && !payload.getShopName().isBlank()
|
||||||
? payload.getShopName().trim()
|
? payload.getShopName().trim()
|
||||||
@@ -916,8 +964,16 @@ public class ProductRiskTaskService {
|
|||||||
result.setResultFileSize(0L);
|
result.setResultFileSize(0L);
|
||||||
result.setResultContentType(CONTENT_TYPE_ZIP);
|
result.setResultContentType(CONTENT_TYPE_ZIP);
|
||||||
result.setRowCount(excelAssemblyService.countRows(countries));
|
result.setRowCount(excelAssemblyService.countRows(countries));
|
||||||
result.setSuccess(1);
|
if (preserveFailure) {
|
||||||
result.setErrorMessage(null);
|
// 失败店铺的部分结果:成功态与失败原因都不能动,只标「文件名已定、文件待组装」
|
||||||
|
result.setSuccess(0);
|
||||||
|
if (result.getErrorMessage() == null || result.getErrorMessage().isBlank()) {
|
||||||
|
result.setErrorMessage("店铺未跑完,仅产出部分结果");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
result.setSuccess(1);
|
||||||
|
result.setErrorMessage(null);
|
||||||
|
}
|
||||||
fileResultMapper.updateById(result);
|
fileResultMapper.updateById(result);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+9
-3
@@ -1,6 +1,7 @@
|
|||||||
package com.nanri.aiimage.modules.publish.controller;
|
package com.nanri.aiimage.modules.publish.controller;
|
||||||
|
|
||||||
import com.nanri.aiimage.common.api.ApiResponse;
|
import com.nanri.aiimage.common.api.ApiResponse;
|
||||||
|
import com.nanri.aiimage.common.security.AdminAuthSupport;
|
||||||
import com.nanri.aiimage.modules.publish.model.dto.PublishParseRequest;
|
import com.nanri.aiimage.modules.publish.model.dto.PublishParseRequest;
|
||||||
import com.nanri.aiimage.modules.publish.model.dto.PublishSubmitResultRequest;
|
import com.nanri.aiimage.modules.publish.model.dto.PublishSubmitResultRequest;
|
||||||
import com.nanri.aiimage.modules.publish.model.dto.PublishTaskBatchRequest;
|
import com.nanri.aiimage.modules.publish.model.dto.PublishTaskBatchRequest;
|
||||||
@@ -14,6 +15,7 @@ import com.nanri.aiimage.modules.publish.service.PublishTaskService;
|
|||||||
import io.swagger.v3.oas.annotations.Operation;
|
import io.swagger.v3.oas.annotations.Operation;
|
||||||
import io.swagger.v3.oas.annotations.Parameter;
|
import io.swagger.v3.oas.annotations.Parameter;
|
||||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||||
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
import jakarta.validation.Valid;
|
import jakarta.validation.Valid;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||||
@@ -34,6 +36,7 @@ import com.nanri.aiimage.modules.task.model.vo.TaskProgressLightBatchVo;
|
|||||||
public class PublishController {
|
public class PublishController {
|
||||||
|
|
||||||
private final PublishTaskService publishTaskService;
|
private final PublishTaskService publishTaskService;
|
||||||
|
private final AdminAuthSupport adminAuthSupport;
|
||||||
|
|
||||||
@PostMapping("/parse")
|
@PostMapping("/parse")
|
||||||
@Operation(
|
@Operation(
|
||||||
@@ -57,15 +60,18 @@ public class PublishController {
|
|||||||
@PostMapping("/tasks/{taskId}/files/{fileId}/activate")
|
@PostMapping("/tasks/{taskId}/files/{fileId}/activate")
|
||||||
@Operation(
|
@Operation(
|
||||||
summary = "激活任务中的单个文件",
|
summary = "激活任务中的单个文件",
|
||||||
description = "前端派发 Python 队列前调用。taskId 和 fileId 用于定位文件,user_id 可省略;如传入则校验任务归属。同一任务同一时间只允许一个 RUNNING 文件;重复激活当前 RUNNING 文件为幂等操作。")
|
description = "前端派发 Python 队列前调用。taskId 和 fileId 用于定位文件,user_id 可省略;如传入则校验任务归属。同一任务同一时间只允许一个 RUNNING 文件;重复激活当前 RUNNING 文件为幂等操作。"
|
||||||
|
+ "店铺互斥按 (发起方设备, 店铺) 判定:同一台机器上同一店铺只允许一个任务在跑(该机器上该店铺只有一个紫鸟浏览器会话,并发会互相切换国家);不同设备各自持有独立会话,允许同一店铺并行跑不同国家。"
|
||||||
|
+ "设备号取自 JWT 签名的 deviceId claim,缺失时退回按店铺全局互斥。")
|
||||||
public ApiResponse<Void> activateFile(
|
public ApiResponse<Void> activateFile(
|
||||||
@Parameter(description = "上架任务 ID", required = true, example = "9001")
|
@Parameter(description = "上架任务 ID", required = true, example = "9001")
|
||||||
@PathVariable Long taskId,
|
@PathVariable Long taskId,
|
||||||
@Parameter(description = "任务内文件 ID", required = true, example = "9101")
|
@Parameter(description = "任务内文件 ID", required = true, example = "9101")
|
||||||
@PathVariable Long fileId,
|
@PathVariable Long fileId,
|
||||||
@Parameter(description = "可选的任务所属用户 ID;省略时由 taskId 反查", required = false, example = "1")
|
@Parameter(description = "可选的任务所属用户 ID;省略时由 taskId 反查", required = false, example = "1")
|
||||||
@RequestParam(value = "user_id", required = false) Long userId) {
|
@RequestParam(value = "user_id", required = false) Long userId,
|
||||||
publishTaskService.activateFile(taskId, fileId, userId);
|
HttpServletRequest request) {
|
||||||
|
publishTaskService.activateFile(taskId, fileId, userId, adminAuthSupport.currentDeviceId(request));
|
||||||
return ApiResponse.success(null);
|
return ApiResponse.success(null);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+2
@@ -20,6 +20,8 @@ public class PublishFileEntity {
|
|||||||
private Integer matched;
|
private Integer matched;
|
||||||
private String shopId;
|
private String shopId;
|
||||||
private Long matchedUserId;
|
private Long matchedUserId;
|
||||||
|
/** 激活该文件时客户端所在设备(JWT 签名的 deviceId);空表示来源不明,按全局店铺互斥保守处理。 */
|
||||||
|
private String deviceId;
|
||||||
private String platform;
|
private String platform;
|
||||||
private String companyName;
|
private String companyName;
|
||||||
private String matchStatus;
|
private String matchStatus;
|
||||||
|
|||||||
+97
@@ -0,0 +1,97 @@
|
|||||||
|
package com.nanri.aiimage.modules.publish.service;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.scheduling.annotation.Scheduled;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 上架任务的保留期清理(2026-09 审核:PUBLISH 的任务历史此前完全没有清理,永久累积)。
|
||||||
|
*
|
||||||
|
* <p>涉及 biz_file_task / biz_file_result / biz_publish_item(每个上传 Excel 的每一行落一行,
|
||||||
|
* 单任务可达数万行)/ biz_publish_file,此前只有用户手动删任务才回收。这里刻意不复用
|
||||||
|
* ModuleHistoryCleanupService:上架单任务行数太大,且删除必须经过业务删除入口回收结果对象。
|
||||||
|
*
|
||||||
|
* <p>删除动作复用 {@link PublishTaskService#deleteTaskForRetention(Long)}(与用户删任务
|
||||||
|
* 同一套删除实现:明细/结果/分片载荷 + 事务提交后回收结果对象),本类只做「查一批过期 id →
|
||||||
|
* 逐个调用 → 计数」。该入口保留状态校验,只删终态任务,PENDING/RUNNING 不会被碰。
|
||||||
|
*
|
||||||
|
* <p>双节点用 job 锁保证单实例执行;每批小批量(默认 50)逐个删,避免单次事务过长。
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
public class PublishTaskRetentionService {
|
||||||
|
|
||||||
|
/** 单轮最多处理的批次数(每批 batchSize 个任务),剩余留给下一轮。 */
|
||||||
|
static final int MAX_BATCHES_PER_RUN = 20;
|
||||||
|
|
||||||
|
private final FileTaskMapper fileTaskMapper;
|
||||||
|
private final PublishTaskService publishTaskService;
|
||||||
|
private final DistributedJobLockService distributedJobLockService;
|
||||||
|
|
||||||
|
@Value("${aiimage.publish.task-retention-days:90}")
|
||||||
|
private int retentionDays = 90;
|
||||||
|
|
||||||
|
@Value("${aiimage.publish.task-retention-batch-size:50}")
|
||||||
|
private int retentionBatchSize = 50;
|
||||||
|
|
||||||
|
@Scheduled(cron = "${aiimage.publish.task-retention-cron:0 30 4 * * *}")
|
||||||
|
public void purgeExpiredTasks() {
|
||||||
|
int days = Math.max(1, retentionDays);
|
||||||
|
int batchSize = Math.max(1, retentionBatchSize);
|
||||||
|
LocalDateTime cutoff = LocalDateTime.now().minusDays(days);
|
||||||
|
|
||||||
|
// 单轮 20 批 × 50 个任务的域内删除(含结果对象回收)可能跑较久,锁 TTL 给足 30 分钟
|
||||||
|
DistributedJobLockService.LockHandle lockHandle =
|
||||||
|
distributedJobLockService.tryLock("publish:task-retention", Duration.ofMinutes(30));
|
||||||
|
if (lockHandle == null) {
|
||||||
|
log.info("[publish-retention] 任务保留清理跳过:另一实例持锁");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try (lockHandle) {
|
||||||
|
int totalDeleted = 0;
|
||||||
|
int totalFailed = 0;
|
||||||
|
int batches = 0;
|
||||||
|
while (batches < MAX_BATCHES_PER_RUN) {
|
||||||
|
// 只取 id:上架任务行可携带很大的 request_json/result_json,不拉整行
|
||||||
|
List<Long> taskIds = fileTaskMapper.selectExpiredTerminalTaskIds(
|
||||||
|
PublishTaskService.MODULE_TYPE, cutoff, batchSize);
|
||||||
|
if (taskIds.isEmpty()) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
batches++;
|
||||||
|
int deletedInBatch = 0;
|
||||||
|
for (Long taskId : taskIds) {
|
||||||
|
try {
|
||||||
|
publishTaskService.deleteTaskForRetention(taskId);
|
||||||
|
deletedInBatch++;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
// 单个任务删除失败只记日志继续:一个坏任务不能卡住整轮
|
||||||
|
log.warn("[publish-retention] 任务删除失败 taskId={} msg={}", taskId, ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
totalDeleted += deletedInBatch;
|
||||||
|
totalFailed += taskIds.size() - deletedInBatch;
|
||||||
|
if (deletedInBatch == 0) {
|
||||||
|
log.warn("[publish-retention] 整批任务均删除失败,提前结束本轮以免反复重试同一批");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (taskIds.size() < batchSize) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.info("[publish-retention] 任务保留清理完成 cutoff={} retentionDays={} deleted={} failed={} batches={}",
|
||||||
|
cutoff, days, totalDeleted, totalFailed, batches);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[publish-retention] 任务保留清理失败 cutoff={} msg={}", cutoff, ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+95
-1
@@ -96,6 +96,12 @@ public class PublishTaskService {
|
|||||||
private static final String STATUS_RUNNING = "RUNNING";
|
private static final String STATUS_RUNNING = "RUNNING";
|
||||||
private static final String STATUS_SUCCESS = "SUCCESS";
|
private static final String STATUS_SUCCESS = "SUCCESS";
|
||||||
private static final String STATUS_FAILED = "FAILED";
|
private static final String STATUS_FAILED = "FAILED";
|
||||||
|
/**
|
||||||
|
* 保留期清理允许删除的终态集合(与 ModuleHistoryCleanupService 的 TERMINAL_STATUSES 一致)。
|
||||||
|
* PENDING/RUNNING 绝不在此列;PUBLISH 实际只会写 SUCCESS/FAILED,取消态是无害的超集。
|
||||||
|
*/
|
||||||
|
private static final Set<String> RETENTION_TERMINAL_STATUSES =
|
||||||
|
Set.of(STATUS_SUCCESS, STATUS_FAILED, "CANCELLED", "CANCELED");
|
||||||
private static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
|
private static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
|
||||||
|
|
||||||
private final LocalFileStorageService localFileStorageService;
|
private final LocalFileStorageService localFileStorageService;
|
||||||
@@ -168,7 +174,14 @@ public class PublishTaskService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void activateFile(Long taskId, Long fileId, Long userId) {
|
/**
|
||||||
|
* 激活任务中的单个文件。
|
||||||
|
*
|
||||||
|
* @param deviceId 发起方设备标识(JWT 签名的 deviceId claim);空串/空白表示来源不明
|
||||||
|
* (旧客户端 token 无该 claim、内部令牌调用),此时退回全局店铺互斥
|
||||||
|
*/
|
||||||
|
public void activateFile(Long taskId, Long fileId, Long userId, String deviceId) {
|
||||||
|
String device = deviceId == null ? "" : deviceId.trim();
|
||||||
try (TaskDistributedLockService.LockHandle lock =
|
try (TaskDistributedLockService.LockHandle lock =
|
||||||
taskDistributedLockService.acquire(MODULE_TYPE, taskId)) {
|
taskDistributedLockService.acquire(MODULE_TYPE, taskId)) {
|
||||||
if (lock == null) {
|
if (lock == null) {
|
||||||
@@ -192,16 +205,61 @@ public class PublishTaskService {
|
|||||||
if (runningFile != null) {
|
if (runningFile != null) {
|
||||||
throw new BusinessException("同一任务已有文件正在执行: " + runningFile.getSourceFilename());
|
throw new BusinessException("同一任务已有文件正在执行: " + runningFile.getSourceFilename());
|
||||||
}
|
}
|
||||||
|
// 店铺级互斥:同一台设备上同一店铺同一时刻只允许一个上架任务在跑。
|
||||||
|
// 2026-09-17 事故(28519/28520/28521 相继提交同一店铺「林洪武」):同一台机器上同一
|
||||||
|
// 店铺被多个任务并发打开,客户端 startBrowser 全部返回 -10000,三个任务一起失败。
|
||||||
|
// 激活是任务真正开跑的唯一入口,在这里挡掉并带出占用中的任务号,用户才知道要等谁。
|
||||||
|
//
|
||||||
|
// 互斥键是 (设备, 店铺) 而不是店铺:紫鸟浏览器会话是**每台机器一份**,不同客户端
|
||||||
|
// 各自持有独立会话,同一家店可以在两台机器上并行跑不同国家(2026-09-18 任务 28624
|
||||||
|
// 在另一台机器上被 28616 误挡)。真正必须串行的是同一台设备——那里只有一个会话,
|
||||||
|
// 两个任务会互相切换国家(add_product.py 的 SwitchingCountries 是会话级状态,任务
|
||||||
|
// 中途断线重连还会再切一次),轻则失败,重则把 A 国家的商品提交进 B 国家的店铺。
|
||||||
|
// 设备号取自 JWT 签名的 deviceId claim;为空(旧客户端 token 无该 claim / 内部令牌
|
||||||
|
// 调用)时退回改动前的全局店铺互斥,保守不放宽。
|
||||||
|
//
|
||||||
|
// 注意:本校验与随后的状态更新之间仍有极小竞态窗口(两个请求恰好同时通过校验);
|
||||||
|
// 真正的串行由客户端店铺锁保证,这一层的目的是尽早给出明确提示,避免白传文件与重复执行。
|
||||||
|
String shopName = file.getShopName();
|
||||||
|
if (shopName != null && !shopName.isBlank()) {
|
||||||
|
LambdaQueryWrapper<PublishFileEntity> shopRunningQuery = new LambdaQueryWrapper<PublishFileEntity>()
|
||||||
|
.eq(PublishFileEntity::getShopName, shopName)
|
||||||
|
.eq(PublishFileEntity::getStatus, STATUS_RUNNING)
|
||||||
|
.ne(PublishFileEntity::getTaskId, taskId)
|
||||||
|
.orderByAsc(PublishFileEntity::getId)
|
||||||
|
.last("limit 1");
|
||||||
|
if (device.isEmpty()) {
|
||||||
|
log.info("[publish] 激活无设备标识,按全局店铺互斥判定 taskId={} fileId={} shop={}",
|
||||||
|
taskId, fileId, shopName);
|
||||||
|
} else {
|
||||||
|
// 本设备的 RUNNING 行,以及设备未知的存量行(旧客户端/内部调用,NULL 或空串)
|
||||||
|
// ——后者无法判断落在哪台机器上,一律保守视为可能同机。
|
||||||
|
shopRunningQuery.and(wrapper -> wrapper
|
||||||
|
.eq(PublishFileEntity::getDeviceId, device)
|
||||||
|
.or().isNull(PublishFileEntity::getDeviceId)
|
||||||
|
.or().eq(PublishFileEntity::getDeviceId, ""));
|
||||||
|
}
|
||||||
|
PublishFileEntity shopRunning = publishFileMapper.selectOne(shopRunningQuery);
|
||||||
|
if (shopRunning != null) {
|
||||||
|
log.warn("[publish] 店铺互斥拦截 taskId={} fileId={} shop={} device={} 占用任务={} 占用设备={}",
|
||||||
|
taskId, fileId, shopName, device,
|
||||||
|
shopRunning.getTaskId(), shopRunning.getDeviceId());
|
||||||
|
throw new BusinessException("店铺「" + shopName + "」已有上架任务正在执行(任务 "
|
||||||
|
+ shopRunning.getTaskId() + "),请等它完成后再提交");
|
||||||
|
}
|
||||||
|
}
|
||||||
int updated = publishFileMapper.update(null, new LambdaUpdateWrapper<PublishFileEntity>()
|
int updated = publishFileMapper.update(null, new LambdaUpdateWrapper<PublishFileEntity>()
|
||||||
.eq(PublishFileEntity::getId, fileId)
|
.eq(PublishFileEntity::getId, fileId)
|
||||||
.eq(PublishFileEntity::getTaskId, taskId)
|
.eq(PublishFileEntity::getTaskId, taskId)
|
||||||
.eq(PublishFileEntity::getStatus, STATUS_PENDING)
|
.eq(PublishFileEntity::getStatus, STATUS_PENDING)
|
||||||
.set(PublishFileEntity::getStatus, STATUS_RUNNING)
|
.set(PublishFileEntity::getStatus, STATUS_RUNNING)
|
||||||
|
.set(PublishFileEntity::getDeviceId, device.isEmpty() ? null : device)
|
||||||
.set(PublishFileEntity::getUpdatedAt, LocalDateTime.now())
|
.set(PublishFileEntity::getUpdatedAt, LocalDateTime.now())
|
||||||
.set(PublishFileEntity::getErrorMessage, null));
|
.set(PublishFileEntity::getErrorMessage, null));
|
||||||
if (updated <= 0) {
|
if (updated <= 0) {
|
||||||
throw new BusinessException("文件激活失败,请刷新后重试");
|
throw new BusinessException("文件激活失败,请刷新后重试");
|
||||||
}
|
}
|
||||||
|
log.info("[publish] 文件激活成功 taskId={} fileId={} shop={} device={}", taskId, fileId, shopName, device);
|
||||||
if (STATUS_PENDING.equals(task.getStatus())) {
|
if (STATUS_PENDING.equals(task.getStatus())) {
|
||||||
fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||||
.eq(FileTaskEntity::getId, taskId)
|
.eq(FileTaskEntity::getId, taskId)
|
||||||
@@ -566,6 +624,37 @@ public class PublishTaskService {
|
|||||||
@Transactional
|
@Transactional
|
||||||
public void deleteTask(Long taskId, Long userId) {
|
public void deleteTask(Long taskId, Long userId) {
|
||||||
FileTaskEntity task = requireTaskForDeletion(taskId, userId);
|
FileTaskEntity task = requireTaskForDeletion(taskId, userId);
|
||||||
|
deleteTaskWithRelatedRows(task);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 保留期清理专用删除入口(内部调用,定时任务没有用户身份)。
|
||||||
|
*
|
||||||
|
* <p>与 {@link #deleteTask(Long, Long)} 复用同一套删除实现(明细行/结果记录/分片载荷 +
|
||||||
|
* 事务提交后回收结果对象),仅跳过用户归属校验;状态校验保留 —— 只允许删终态任务,
|
||||||
|
* PENDING/RUNNING 一律不删(正在跑的任务被删会让分片回传、结果组装找不到任务行)。
|
||||||
|
* 行已被删除或非上架任务时幂等跳过,不抛异常。
|
||||||
|
*/
|
||||||
|
@Transactional
|
||||||
|
public void deleteTaskForRetention(Long taskId) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
throw new BusinessException("taskId 不合法");
|
||||||
|
}
|
||||||
|
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||||
|
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
||||||
|
log.info("[publish] 保留期删除跳过:任务已不存在或非上架任务 taskId={}", taskId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!isRetentionTerminal(task.getStatus())) {
|
||||||
|
log.warn("[publish] 保留期删除跳过非终态任务 taskId={} status={}", taskId, task.getStatus());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
deleteTaskWithRelatedRows(task);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 删除任务行与全部关联数据,并在事务提交后回收远端对象(deleteTask 与保留期清理共用)。 */
|
||||||
|
private void deleteTaskWithRelatedRows(FileTaskEntity task) {
|
||||||
|
Long taskId = task.getId();
|
||||||
List<FileResultEntity> results = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
List<FileResultEntity> results = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
||||||
.eq(FileResultEntity::getTaskId, taskId)
|
.eq(FileResultEntity::getTaskId, taskId)
|
||||||
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
|
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
|
||||||
@@ -1953,6 +2042,11 @@ public class PublishTaskService {
|
|||||||
return STATUS_SUCCESS.equals(status) || STATUS_FAILED.equals(status);
|
return STATUS_SUCCESS.equals(status) || STATUS_FAILED.equals(status);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** 保留期清理的终态判定:大小写不敏感,PENDING/RUNNING 一律返回 false。 */
|
||||||
|
private static boolean isRetentionTerminal(String status) {
|
||||||
|
return status != null && RETENTION_TERMINAL_STATUSES.contains(status.trim().toUpperCase(Locale.ROOT));
|
||||||
|
}
|
||||||
|
|
||||||
private List<Long> normalizeTaskIds(List<Long> taskIds) {
|
private List<Long> normalizeTaskIds(List<Long> taskIds) {
|
||||||
if (taskIds == null) {
|
if (taskIds == null) {
|
||||||
return List.of();
|
return List.of();
|
||||||
|
|||||||
+9
@@ -27,6 +27,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlTaskBatchVo
|
|||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlCreateTaskVo;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlCreateTaskVo;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlHistoryVo;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlHistoryVo;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||||
import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo;
|
import com.nanri.aiimage.common.model.vo.ProductRiskDashboardVo;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
@@ -121,6 +122,7 @@ public class ShopDataCrawlTaskService {
|
|||||||
private final InstanceMetadata instanceMetadata;
|
private final InstanceMetadata instanceMetadata;
|
||||||
private final ShopDataCrawlDailyFileService dailyFileService;
|
private final ShopDataCrawlDailyFileService dailyFileService;
|
||||||
private final ShopDataCrawlItemStoreService shopDataCrawlItemStoreService;
|
private final ShopDataCrawlItemStoreService shopDataCrawlItemStoreService;
|
||||||
|
private final DuplicateCheckRefreshPort duplicateCheckRefreshPort;
|
||||||
private final PlatformTransactionManager transactionManager;
|
private final PlatformTransactionManager transactionManager;
|
||||||
private final TaskProgressLightAssembler taskProgressLightAssembler;
|
private final TaskProgressLightAssembler taskProgressLightAssembler;
|
||||||
|
|
||||||
@@ -2116,6 +2118,13 @@ public class ShopDataCrawlTaskService {
|
|||||||
shopDataCrawlItemStoreService.saveShopBatchFromSnapshot(
|
shopDataCrawlItemStoreService.saveShopBatchFromSnapshot(
|
||||||
snapshot.getShopName(), itemBatchDate, accumulatedItems,
|
snapshot.getShopName(), itemBatchDate, accumulatedItems,
|
||||||
snapshot.getResultId(), task.getId(), baseDailyFile == null ? null : baseDailyFile.getId());
|
snapshot.getResultId(), task.getId(), baseDailyFile == null ? null : baseDailyFile.getId());
|
||||||
|
// 明细已落库:请求撞款重扫(异步合并执行,不阻塞归档;端口契约保证不抛错)
|
||||||
|
try {
|
||||||
|
duplicateCheckRefreshPort.requestRefresh("shop-data-crawl:" + snapshot.getShopName());
|
||||||
|
} catch (RuntimeException ex) {
|
||||||
|
log.warn("[shop-data-crawl] 请求撞款重扫失败(忽略,不影响归档) shop={} msg={}",
|
||||||
|
snapshot.getShopName(), ex.getMessage());
|
||||||
|
}
|
||||||
int rowCount = excelAssemblyService.writeWorkbook(outputXlsx, accumulatedItems);
|
int rowCount = excelAssemblyService.writeWorkbook(outputXlsx, accumulatedItems);
|
||||||
String objectKey = ossStorageService.uploadResultFile(outputXlsx, MODULE_TYPE);
|
String objectKey = ossStorageService.uploadResultFile(outputXlsx, MODULE_TYPE);
|
||||||
if (blank(objectKey)) {
|
if (blank(objectKey)) {
|
||||||
|
|||||||
+17
@@ -0,0 +1,17 @@
|
|||||||
|
package com.nanri.aiimage.modules.shopdatacrawl.spi;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 采集明细就绪后的撞款重扫触发端口(2026-09:店铺数据采集落库后即时刷新重复检查)。
|
||||||
|
*
|
||||||
|
* <p>实现方在 shopduplicatecheck 模块({@code ShopDataDuplicateCheckScanService})。
|
||||||
|
* 契约:实现必须异步执行、去抖合并,不得阻塞调用方、不得向外抛出异常。
|
||||||
|
*/
|
||||||
|
public interface DuplicateCheckRefreshPort {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 请求一次撞款重扫(异步;合并窗口内的多次触发聚合为一次扫描)。
|
||||||
|
*
|
||||||
|
* @param reason 触发来源,仅用于日志排查
|
||||||
|
*/
|
||||||
|
void requestRefresh(String reason);
|
||||||
|
}
|
||||||
+19
@@ -4,9 +4,13 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
|||||||
import com.nanri.aiimage.modules.shopduplicatecheck.model.dto.ScanLightRowDto;
|
import com.nanri.aiimage.modules.shopduplicatecheck.model.dto.ScanLightRowDto;
|
||||||
import com.nanri.aiimage.modules.shopduplicatecheck.model.dto.ScanFullRowDto;
|
import com.nanri.aiimage.modules.shopduplicatecheck.model.dto.ScanFullRowDto;
|
||||||
import com.nanri.aiimage.modules.shopduplicatecheck.model.entity.ShopDataDuplicateScanEntity;
|
import com.nanri.aiimage.modules.shopduplicatecheck.model.entity.ShopDataDuplicateScanEntity;
|
||||||
|
import org.apache.ibatis.annotations.Delete;
|
||||||
import org.apache.ibatis.annotations.Mapper;
|
import org.apache.ibatis.annotations.Mapper;
|
||||||
|
import org.apache.ibatis.annotations.Param;
|
||||||
import org.apache.ibatis.annotations.Select;
|
import org.apache.ibatis.annotations.Select;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
@Mapper
|
@Mapper
|
||||||
public interface ShopDataDuplicateScanMapper extends BaseMapper<ShopDataDuplicateScanEntity> {
|
public interface ShopDataDuplicateScanMapper extends BaseMapper<ShopDataDuplicateScanEntity> {
|
||||||
|
|
||||||
@@ -20,4 +24,19 @@ public interface ShopDataDuplicateScanMapper extends BaseMapper<ShopDataDuplicat
|
|||||||
+ "created_at AS createdAt FROM shop_data_duplicate_scan "
|
+ "created_at AS createdAt FROM shop_data_duplicate_scan "
|
||||||
+ "WHERE status = 'SUCCESS' ORDER BY id DESC LIMIT 1")
|
+ "WHERE status = 'SUCCESS' ORDER BY id DESC LIMIT 1")
|
||||||
ScanFullRowDto selectLatestFullRow();
|
ScanFullRowDto selectLatestFullRow();
|
||||||
|
|
||||||
|
/** 第 N 新的行 id(offset 从 0 计);行数不足时返回 null,供保留清理计算保护线。 */
|
||||||
|
@Select("SELECT id FROM shop_data_duplicate_scan ORDER BY id DESC LIMIT 1 OFFSET #{offset}")
|
||||||
|
Long selectNthNewestId(@Param("offset") int offset);
|
||||||
|
|
||||||
|
/** 分批删除保留期外、且早于保护线的历史扫描行。 */
|
||||||
|
@Delete("""
|
||||||
|
DELETE FROM shop_data_duplicate_scan
|
||||||
|
WHERE created_at < #{cutoff}
|
||||||
|
AND id < #{protectFromId}
|
||||||
|
LIMIT #{batchSize}
|
||||||
|
""")
|
||||||
|
int deleteOlderThanBatch(@Param("cutoff") LocalDateTime cutoff,
|
||||||
|
@Param("protectFromId") long protectFromId,
|
||||||
|
@Param("batchSize") int batchSize);
|
||||||
}
|
}
|
||||||
|
|||||||
+32
-2
@@ -9,6 +9,7 @@ import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryRes
|
|||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlItemStoreService;
|
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlItemStoreService;
|
||||||
|
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||||
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDataDuplicateScanMapper;
|
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDataDuplicateScanMapper;
|
||||||
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDuplicateCheckItemMapper;
|
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDuplicateCheckItemMapper;
|
||||||
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDuplicateCheckSourceMapper;
|
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDuplicateCheckSourceMapper;
|
||||||
@@ -21,6 +22,7 @@ import com.nanri.aiimage.modules.shopduplicatecheck.model.entity.ShopDataDuplica
|
|||||||
import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateScanPayload;
|
import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateScanPayload;
|
||||||
import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateScanSummary;
|
import com.nanri.aiimage.modules.shopduplicatecheck.model.payload.DuplicateScanSummary;
|
||||||
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.DuplicateCheckAggregator;
|
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.DuplicateCheckAggregator;
|
||||||
|
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.DuplicateCheckRefreshScheduler;
|
||||||
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.DuplicateCheckWorkbookParser;
|
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.DuplicateCheckWorkbookParser;
|
||||||
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.RawRow;
|
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.RawRow;
|
||||||
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.ShopParsed;
|
import com.nanri.aiimage.modules.shopduplicatecheck.service.support.ShopParsed;
|
||||||
@@ -43,14 +45,14 @@ import java.util.Set;
|
|||||||
import java.util.concurrent.atomic.AtomicLong;
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 撞款扫描:每日 00:00 定时 + force 同步全量重扫。
|
* 撞款扫描:每日 00:00 定时 + force 同步全量重扫 + 采集落库触发的异步合并重扫。
|
||||||
* 数据源 = 采集明细表(biz_shop_data_crawl_item,采集先落库再更新文件),
|
* 数据源 = 采集明细表(biz_shop_data_crawl_item,采集先落库再更新文件),
|
||||||
* 直接查库聚合 shops/items/summary 落库 shop_data_duplicate_scan(输出契约不变)。
|
* 直接查库聚合 shops/items/summary 落库 shop_data_duplicate_scan(输出契约不变)。
|
||||||
* 双实例通过 Redis 分布式锁防重;扫描失败落 FAILED 行并上抛,force 场景由端点转 409/500。
|
* 双实例通过 Redis 分布式锁防重;扫描失败落 FAILED 行并上抛,force 场景由端点转 409/500。
|
||||||
*/
|
*/
|
||||||
@Service
|
@Service
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public class ShopDataDuplicateCheckScanService {
|
public class ShopDataDuplicateCheckScanService implements DuplicateCheckRefreshPort {
|
||||||
|
|
||||||
public static final String SCAN_LOCK = "shop-data-duplicate-check:scan";
|
public static final String SCAN_LOCK = "shop-data-duplicate-check:scan";
|
||||||
static final DateTimeFormatter SCANNED_AT_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
|
static final DateTimeFormatter SCANNED_AT_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
|
||||||
@@ -71,6 +73,9 @@ public class ShopDataDuplicateCheckScanService {
|
|||||||
private final AtomicLong cachedRowId = new AtomicLong(-1L);
|
private final AtomicLong cachedRowId = new AtomicLong(-1L);
|
||||||
private volatile CachedScan cachedScan;
|
private volatile CachedScan cachedScan;
|
||||||
|
|
||||||
|
/** 采集落库触发的异步合并重扫调度器(单飞 + 去抖 + 锁忙重试)。 */
|
||||||
|
private final DuplicateCheckRefreshScheduler refreshScheduler;
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
public ShopDataDuplicateCheckScanService(ShopDataDuplicateScanMapper scanMapper,
|
public ShopDataDuplicateCheckScanService(ShopDataDuplicateScanMapper scanMapper,
|
||||||
ShopDuplicateCheckSourceMapper sourceMapper,
|
ShopDuplicateCheckSourceMapper sourceMapper,
|
||||||
@@ -86,6 +91,7 @@ public class ShopDataDuplicateCheckScanService {
|
|||||||
this.objectMapper = objectMapper;
|
this.objectMapper = objectMapper;
|
||||||
this.itemMapper = itemMapper;
|
this.itemMapper = itemMapper;
|
||||||
this.itemStoreService = itemStoreService;
|
this.itemStoreService = itemStoreService;
|
||||||
|
this.refreshScheduler = new DuplicateCheckRefreshScheduler(this::runRefreshOnce);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 读侧视图:scanned_at 为最新 SUCCESS 行 created_at(yyyy-MM-dd HH:mm:ss)。 */
|
/** 读侧视图:scanned_at 为最新 SUCCESS 行 created_at(yyyy-MM-dd HH:mm:ss)。 */
|
||||||
@@ -122,6 +128,30 @@ public class ShopDataDuplicateCheckScanService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** 采集明细落库后的重扫请求(端口实现):异步合并执行,不阻塞、不抛错。 */
|
||||||
|
@Override
|
||||||
|
public void requestRefresh(String reason) {
|
||||||
|
refreshScheduler.request(reason);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 调度器单次扫描动作:锁被占返回 LOCK_BUSY 供其重试;失败只记日志(FAILED 行已落库)。 */
|
||||||
|
private DuplicateCheckRefreshScheduler.Outcome runRefreshOnce() {
|
||||||
|
try {
|
||||||
|
scanNow();
|
||||||
|
return DuplicateCheckRefreshScheduler.Outcome.DONE;
|
||||||
|
} catch (BusinessException ex) {
|
||||||
|
if (ex.getCode() != null && ex.getCode() == 409) {
|
||||||
|
log.info("[shop-duplicate-check] 自动重扫未执行:其它扫描进行中 msg={}", ex.getMessage());
|
||||||
|
return DuplicateCheckRefreshScheduler.Outcome.LOCK_BUSY;
|
||||||
|
}
|
||||||
|
log.warn("[shop-duplicate-check] 自动重扫失败 code={} msg={}", ex.getCode(), ex.getMessage());
|
||||||
|
return DuplicateCheckRefreshScheduler.Outcome.FAILED;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.error("[shop-duplicate-check] 自动重扫异常", ex);
|
||||||
|
return DuplicateCheckRefreshScheduler.Outcome.FAILED;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/** 最新 SUCCESS 扫描视图;无扫描结果返回 null。 */
|
/** 最新 SUCCESS 扫描视图;无扫描结果返回 null。 */
|
||||||
public DuplicateScanView loadLatest() {
|
public DuplicateScanView loadLatest() {
|
||||||
ScanLightRowDto light = scanMapper.selectLatestLightRow();
|
ScanLightRowDto light = scanMapper.selectLatestLightRow();
|
||||||
|
|||||||
+89
@@ -0,0 +1,89 @@
|
|||||||
|
package com.nanri.aiimage.modules.shopduplicatecheck.service;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||||
|
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDataDuplicateScanMapper;
|
||||||
|
import com.nanri.aiimage.modules.shopduplicatecheck.model.dto.ScanLightRowDto;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.scheduling.annotation.Scheduled;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 撞款扫描结果表(shop_data_duplicate_scan)的保留期清理。
|
||||||
|
*
|
||||||
|
* <p>扫描每晚自动跑一轮、管理员还能手动触发,每次插入一行**含整份聚合 payload 的结果**;
|
||||||
|
* 但读取侧只认最新一行({@code selectLatestLightRow} / {@code selectLatestFullRow} 都是
|
||||||
|
* {@code ORDER BY id DESC LIMIT 1}),历史行纯属占用磁盘,此前无任何删除路径。
|
||||||
|
*
|
||||||
|
* <p>保护策略:无论多旧,始终保留按 id 最新的若干行——
|
||||||
|
* 万一扫描停摆很久,界面上仍能看到最后一份结果,而不是被清理任务顺手抹掉。
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
public class ShopDataDuplicateScanRetentionService {
|
||||||
|
|
||||||
|
/** 单轮最多删除的批次数,剩余留给下一轮。 */
|
||||||
|
static final int MAX_BATCHES_PER_RUN = 20;
|
||||||
|
|
||||||
|
/** 无论时间多久都保留的最新行数(最新 SUCCESS 行必然在其中)。 */
|
||||||
|
static final int PROTECTED_ROWS = 10;
|
||||||
|
|
||||||
|
private final ShopDataDuplicateScanMapper scanMapper;
|
||||||
|
private final DistributedJobLockService distributedJobLockService;
|
||||||
|
|
||||||
|
@Value("${aiimage.shop-duplicate-check.scan-retention-days:7}")
|
||||||
|
private int retentionDays = 7;
|
||||||
|
|
||||||
|
@Value("${aiimage.shop-duplicate-check.scan-retention-batch-size:200}")
|
||||||
|
private int retentionBatchSize = 200;
|
||||||
|
|
||||||
|
@Scheduled(cron = "${aiimage.shop-duplicate-check.scan-retention-cron:0 50 3 * * *}")
|
||||||
|
public void purgeExpiredScans() {
|
||||||
|
int days = Math.max(1, retentionDays);
|
||||||
|
int batchSize = Math.max(20, retentionBatchSize);
|
||||||
|
LocalDateTime cutoff = LocalDateTime.now().minusDays(days);
|
||||||
|
|
||||||
|
DistributedJobLockService.LockHandle lockHandle =
|
||||||
|
distributedJobLockService.tryLock("shop-duplicate-check:scan-retention", Duration.ofMinutes(15));
|
||||||
|
if (lockHandle == null) {
|
||||||
|
log.info("[shop-duplicate-check] 扫描结果保留清理跳过:另一实例持锁");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try (lockHandle) {
|
||||||
|
// 两条保护线取更靠前(更小)的那个 id——保护线之内的行一律不删:
|
||||||
|
// ① 最新的 PROTECTED_ROWS 行:万一扫描停摆很久,界面上仍能看到最后一份结果;
|
||||||
|
// ② 最新 SUCCESS 行:读侧只认它。连续失败多日时它可能已滑出保护窗口,
|
||||||
|
// 按时间线会被删掉、界面直接空白,所以单独兜一条。
|
||||||
|
Long nthNewestId = scanMapper.selectNthNewestId(PROTECTED_ROWS - 1);
|
||||||
|
ScanLightRowDto latestSuccess = scanMapper.selectLatestLightRow();
|
||||||
|
long protectFromId = minId(nthNewestId, latestSuccess == null ? null : latestSuccess.getId());
|
||||||
|
int totalDeleted = 0;
|
||||||
|
int batches = 0;
|
||||||
|
while (batches < MAX_BATCHES_PER_RUN) {
|
||||||
|
int deleted = scanMapper.deleteOlderThanBatch(cutoff, protectFromId, batchSize);
|
||||||
|
batches++;
|
||||||
|
totalDeleted += deleted;
|
||||||
|
if (deleted < batchSize) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.info("[shop-duplicate-check] 扫描结果保留清理完成 cutoff={} retentionDays={} protectFromId={} deleted={} batches={}",
|
||||||
|
cutoff, days, protectFromId, totalDeleted, batches);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[shop-duplicate-check] 扫描结果保留清理失败 cutoff={} msg={}", cutoff, ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 取两条保护线里更靠前(更小)的 id;都为 null 表示无需保护(等价于不设限)。 */
|
||||||
|
private static long minId(Long first, Long second) {
|
||||||
|
if (first == null) {
|
||||||
|
return second == null ? Long.MAX_VALUE : second;
|
||||||
|
}
|
||||||
|
return second == null ? first : Math.min(first, second);
|
||||||
|
}
|
||||||
|
}
|
||||||
+138
@@ -0,0 +1,138 @@
|
|||||||
|
package com.nanri.aiimage.modules.shopduplicatecheck.service.support;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.util.ThreadPools;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
import java.util.concurrent.ExecutorService;
|
||||||
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
|
import java.util.function.Supplier;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 采集落库触发的撞款重扫调度器:合并窗口去抖 + 单飞 + 锁忙重试。
|
||||||
|
*
|
||||||
|
* <p>语义:{@link #request} 只置位并异步执行,永不阻塞调用方、永不向外抛错;
|
||||||
|
* 合并窗口内的多次触发聚合为一次扫描;扫描动作执行期间到达的触发在下一轮执行;
|
||||||
|
* 扫描因分布式锁被占用未执行({@link Outcome#LOCK_BUSY})时按固定间隔重试有限次。
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
public class DuplicateCheckRefreshScheduler {
|
||||||
|
|
||||||
|
/** 单次扫描动作的终态:完成 / 锁被占(可重试)/ 失败(不重试,等下次触发或定时扫描)。 */
|
||||||
|
public enum Outcome {
|
||||||
|
DONE, LOCK_BUSY, FAILED
|
||||||
|
}
|
||||||
|
|
||||||
|
private static final long DEFAULT_DEBOUNCE_MILLIS = 10_000L;
|
||||||
|
private static final long DEFAULT_LOCK_RETRY_MILLIS = 20_000L;
|
||||||
|
private static final int DEFAULT_MAX_LOCK_RETRIES = 6;
|
||||||
|
|
||||||
|
private final Supplier<Outcome> scanAction;
|
||||||
|
private final long debounceMillis;
|
||||||
|
private final long lockRetryMillis;
|
||||||
|
private final int maxLockRetries;
|
||||||
|
private final ExecutorService executor;
|
||||||
|
private final AtomicBoolean pending = new AtomicBoolean(false);
|
||||||
|
private final AtomicBoolean running = new AtomicBoolean(false);
|
||||||
|
|
||||||
|
public DuplicateCheckRefreshScheduler(Supplier<Outcome> scanAction) {
|
||||||
|
this(scanAction, DEFAULT_DEBOUNCE_MILLIS, DEFAULT_LOCK_RETRY_MILLIS, DEFAULT_MAX_LOCK_RETRIES);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 测试用:注入更短的窗口与重试参数。 */
|
||||||
|
DuplicateCheckRefreshScheduler(Supplier<Outcome> scanAction, long debounceMillis,
|
||||||
|
long lockRetryMillis, int maxLockRetries) {
|
||||||
|
this.scanAction = scanAction;
|
||||||
|
this.debounceMillis = Math.max(0L, debounceMillis);
|
||||||
|
this.lockRetryMillis = Math.max(0L, lockRetryMillis);
|
||||||
|
this.maxLockRetries = Math.max(0, maxLockRetries);
|
||||||
|
this.executor = ThreadPools.boundedFixed("shop-dup-refresh", 1, 8);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 请求一次重扫(异步、去抖合并)。调用方不被阻塞,也不会收到异常。 */
|
||||||
|
public void request(String reason) {
|
||||||
|
pending.set(true);
|
||||||
|
if (running.compareAndSet(false, true)) {
|
||||||
|
submit(reason);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void submit(String reason) {
|
||||||
|
try {
|
||||||
|
log.info("[shop-duplicate-check] 触发撞款重扫(异步合并执行,窗口={}ms) reason={}", debounceMillis, reason);
|
||||||
|
executor.execute(this::drain);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
// 提交失败(如线程池拒绝)时复位单飞标记,避免后续触发被永久吞掉
|
||||||
|
running.set(false);
|
||||||
|
log.warn("[shop-duplicate-check] 撞款重扫任务提交失败 reason={} msg={}", reason, ex.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void drain() {
|
||||||
|
try {
|
||||||
|
while (true) {
|
||||||
|
// 合并窗口:窗口内到达的多次触发聚合为同一轮扫描
|
||||||
|
if (!sleepQuietly(debounceMillis)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!pending.compareAndSet(true, false)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
int lockRetries = 0;
|
||||||
|
while (true) {
|
||||||
|
Outcome outcome = runOnceSafely();
|
||||||
|
if (outcome != Outcome.LOCK_BUSY) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (lockRetries >= maxLockRetries) {
|
||||||
|
log.warn("[shop-duplicate-check] 撞款重扫连续 {} 次未取得扫描锁,放弃本轮(等待下次触发或定时扫描)",
|
||||||
|
lockRetries + 1);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
lockRetries++;
|
||||||
|
log.info("[shop-duplicate-check] 撞款重扫未取得扫描锁,{}ms 后重试(第 {}/{} 次)",
|
||||||
|
lockRetryMillis, lockRetries, maxLockRetries);
|
||||||
|
if (!sleepQuietly(lockRetryMillis)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
running.set(false);
|
||||||
|
// 竞态兜底:running 复位前到达的触发可能没能提交,补一次
|
||||||
|
if (pending.get() && running.compareAndSet(false, true)) {
|
||||||
|
submit("race-guard");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 执行一次扫描动作;动作自身异常也被吸收(调度器对外零抛出)。 */
|
||||||
|
private Outcome runOnceSafely() {
|
||||||
|
long startedAt = System.currentTimeMillis();
|
||||||
|
try {
|
||||||
|
Outcome outcome = scanAction.get();
|
||||||
|
long elapsed = System.currentTimeMillis() - startedAt;
|
||||||
|
if (outcome == Outcome.DONE) {
|
||||||
|
log.info("[shop-duplicate-check] 采集后自动重扫完成 耗时={}ms", elapsed);
|
||||||
|
} else if (outcome == Outcome.FAILED) {
|
||||||
|
log.warn("[shop-duplicate-check] 采集后自动重扫失败 耗时={}ms", elapsed);
|
||||||
|
}
|
||||||
|
return outcome == null ? Outcome.FAILED : outcome;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.error("[shop-duplicate-check] 采集后自动重扫异常", ex);
|
||||||
|
return Outcome.FAILED;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static boolean sleepQuietly(long millis) {
|
||||||
|
if (millis <= 0) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
Thread.sleep(millis);
|
||||||
|
return true;
|
||||||
|
} catch (InterruptedException ex) {
|
||||||
|
Thread.currentThread().interrupt();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+8
-1
@@ -1,5 +1,6 @@
|
|||||||
package com.nanri.aiimage.modules.shopkey.model.entity;
|
package com.nanri.aiimage.modules.shopkey.model.entity;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.annotation.FieldStrategy;
|
||||||
import com.baomidou.mybatisplus.annotation.IdType;
|
import com.baomidou.mybatisplus.annotation.IdType;
|
||||||
import com.baomidou.mybatisplus.annotation.TableField;
|
import com.baomidou.mybatisplus.annotation.TableField;
|
||||||
import com.baomidou.mybatisplus.annotation.TableId;
|
import com.baomidou.mybatisplus.annotation.TableId;
|
||||||
@@ -39,6 +40,12 @@ public class QueryAsinEntity {
|
|||||||
@TableField("created_at")
|
@TableField("created_at")
|
||||||
private LocalDateTime createdAt;
|
private LocalDateTime createdAt;
|
||||||
|
|
||||||
@TableField("updated_at")
|
/**
|
||||||
|
* 更新时间:由数据库维护(DEFAULT CURRENT_TIMESTAMP + ON UPDATE CURRENT_TIMESTAMP)。
|
||||||
|
*
|
||||||
|
* <p>禁止应用显式写:本表更新走 selectById → 改字段 → updateById(实体带着旧值),
|
||||||
|
* 显式写回旧值会触发 MySQL 的「显式赋值不自动更新」规则,把更新时间冻结在首次写入时刻。
|
||||||
|
*/
|
||||||
|
@TableField(value = "updated_at", insertStrategy = FieldStrategy.NEVER, updateStrategy = FieldStrategy.NEVER)
|
||||||
private LocalDateTime updatedAt;
|
private LocalDateTime updatedAt;
|
||||||
}
|
}
|
||||||
|
|||||||
+7
-1
@@ -56,6 +56,12 @@ public class SkipPriceAsinEntity {
|
|||||||
@TableField("created_at")
|
@TableField("created_at")
|
||||||
private LocalDateTime createdAt;
|
private LocalDateTime createdAt;
|
||||||
|
|
||||||
@TableField("updated_at")
|
/**
|
||||||
|
* 更新时间:由数据库维护(DEFAULT CURRENT_TIMESTAMP + ON UPDATE CURRENT_TIMESTAMP)。
|
||||||
|
*
|
||||||
|
* <p>禁止应用显式写:本表更新走 selectById → 改字段 → updateById(实体带着旧值),
|
||||||
|
* 显式写回旧值会触发 MySQL 的「显式赋值不自动更新」规则,把更新时间冻结在首次写入时刻。
|
||||||
|
*/
|
||||||
|
@TableField(value = "updated_at", insertStrategy = FieldStrategy.NEVER, updateStrategy = FieldStrategy.NEVER)
|
||||||
private LocalDateTime updatedAt;
|
private LocalDateTime updatedAt;
|
||||||
}
|
}
|
||||||
|
|||||||
+67
-11
@@ -676,8 +676,7 @@ public class ShopMatchTaskService {
|
|||||||
changed = true;
|
changed = true;
|
||||||
ShopMatchShopPayloadDto merged = mergeShopPayload(taskId, shopKey, incoming);
|
ShopMatchShopPayloadDto merged = mergeShopPayload(taskId, shopKey, incoming);
|
||||||
if (incoming.getError() != null && !incoming.getError().isBlank()) {
|
if (incoming.getError() != null && !incoming.getError().isBlank()) {
|
||||||
markResultFailed(result, incoming.getError().trim());
|
finalizeFailedShop(result, shopKey, merged, incoming.getError().trim());
|
||||||
shopMatchTaskCacheService.removeShopMergedPayload(taskId, shopKey);
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (!isShopPayloadCompleted(merged)) {
|
if (!isShopPayloadCompleted(merged)) {
|
||||||
@@ -756,9 +755,8 @@ public class ShopMatchTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (cachedPayload.getError() != null && !cachedPayload.getError().isBlank()) {
|
if (cachedPayload.getError() != null && !cachedPayload.getError().isBlank()) {
|
||||||
markResultFailed(result, cachedPayload.getError());
|
// 陈旧收尾同样保留已跑出来的行:失败原因照常回显,文件顺手组装
|
||||||
batchErrors.add(shopKey + ": " + cachedPayload.getError());
|
batchErrors.add(shopKey + ": " + finalizeFailedShop(result, shopKey, cachedPayload, cachedPayload.getError()));
|
||||||
shopMatchTaskCacheService.removeShopMergedPayload(taskId, shopKey);
|
|
||||||
changed = true;
|
changed = true;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -819,6 +817,9 @@ public class ShopMatchTaskService {
|
|||||||
String displayName = payload.getShopName() != null && !payload.getShopName().isBlank() ? payload.getShopName().trim() : shopKey;
|
String displayName = payload.getShopName() != null && !payload.getShopName().isBlank() ? payload.getShopName().trim() : shopKey;
|
||||||
String stem = safeFileStem(displayName);
|
String stem = safeFileStem(displayName);
|
||||||
File xlsx = FileUtil.file(workRoot, stem + ".xlsx");
|
File xlsx = FileUtil.file(workRoot, stem + ".xlsx");
|
||||||
|
// 失败店铺的部分结果(errorMessage 已写明原因):组装完成时保留失败态,只挂文件。
|
||||||
|
// 否则组装一落地就把行"洗成成功",用户再也看不到「这个店铺其实没跑完」。
|
||||||
|
boolean partialFailure = result.getErrorMessage() != null && !result.getErrorMessage().isBlank();
|
||||||
try {
|
try {
|
||||||
excelAssemblyService.writeWorkbook(xlsx, countries);
|
excelAssemblyService.writeWorkbook(xlsx, countries);
|
||||||
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
|
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
|
||||||
@@ -827,8 +828,12 @@ public class ShopMatchTaskService {
|
|||||||
result.setResultFileSize(xlsx.length());
|
result.setResultFileSize(xlsx.length());
|
||||||
result.setResultContentType(CONTENT_TYPE_XLSX);
|
result.setResultContentType(CONTENT_TYPE_XLSX);
|
||||||
result.setRowCount(excelAssemblyService.countRows(countries));
|
result.setRowCount(excelAssemblyService.countRows(countries));
|
||||||
result.setSuccess(1);
|
if (partialFailure) {
|
||||||
result.setErrorMessage(null);
|
result.setSuccess(0);
|
||||||
|
} else {
|
||||||
|
result.setSuccess(1);
|
||||||
|
result.setErrorMessage(null);
|
||||||
|
}
|
||||||
fileResultMapper.updateById(result);
|
fileResultMapper.updateById(result);
|
||||||
} finally {
|
} finally {
|
||||||
FileUtil.del(xlsx);
|
FileUtil.del(xlsx);
|
||||||
@@ -861,12 +866,55 @@ public class ShopMatchTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void enqueueResultFileAssembly(FileResultEntity result, String shopKey, ShopMatchShopPayloadDto payload) {
|
private void enqueueResultFileAssembly(FileResultEntity result, String shopKey, ShopMatchShopPayloadDto payload) {
|
||||||
|
enqueueResultFileAssembly(result, shopKey, payload, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param preserveFailure 失败店铺的部分结果:保留 success=0 + 失败原因,只把文件挂上去,
|
||||||
|
* 任务状态不变(仍 FAILED),用户仍能下载已跑出来的行
|
||||||
|
*/
|
||||||
|
private void enqueueResultFileAssembly(FileResultEntity result, String shopKey,
|
||||||
|
ShopMatchShopPayloadDto payload, boolean preserveFailure) {
|
||||||
taskResultPayloadService.saveLatest(result.getTaskId(), MODULE_TYPE, shopKey, payload);
|
taskResultPayloadService.saveLatest(result.getTaskId(), MODULE_TYPE, shopKey, payload);
|
||||||
markResultFilePending(result, shopKey, payload);
|
markResultFilePending(result, shopKey, payload, preserveFailure);
|
||||||
taskFileJobService.enqueueAssembleResult(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey);
|
taskFileJobService.enqueueAssembleResult(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void markResultFilePending(FileResultEntity result, String shopKey, ShopMatchShopPayloadDto payload) {
|
/**
|
||||||
|
* 失败店铺的收尾:任务照旧标 FAILED(原因回显给用户),但已经把跑出来的行组装成
|
||||||
|
* 部分结果文件随任务交付——否则用户只看到「失败」,却拿不到「哪些 ASIN 实际已被匹配」
|
||||||
|
* 的记录(会话掉线这类"跑了几个国家才断"的场景尤其需要)。一行可用数据都没有时才退化成纯失败。
|
||||||
|
*
|
||||||
|
* @return 最终写入结果记录的失败原因(组装排队失败时会附上原因)
|
||||||
|
*/
|
||||||
|
private String finalizeFailedShop(FileResultEntity result, String shopKey,
|
||||||
|
ShopMatchShopPayloadDto mergedPayload, String errorMessage) {
|
||||||
|
int rows = mergedPayload == null ? 0 : countPayloadRows(mergedPayload);
|
||||||
|
String message = errorMessage;
|
||||||
|
if (rows > 0) {
|
||||||
|
markResultFailed(result, errorMessage);
|
||||||
|
try {
|
||||||
|
enqueueResultFileAssembly(result, shopKey, mergedPayload, true);
|
||||||
|
shopMatchTaskCacheService.removeShopMergedPayload(result.getTaskId(), shopKey);
|
||||||
|
log.info("[shop-match] 失败店铺仍产出部分结果 taskId={} shop={} rows={} error={}",
|
||||||
|
result.getTaskId(), shopKey, rows, errorMessage);
|
||||||
|
return errorMessage;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
message = errorMessage + "(部分结果组装排队失败:" + (ex.getMessage() == null ? "未知错误" : ex.getMessage()) + ")";
|
||||||
|
log.warn("[shop-match] 失败店铺部分结果排队失败,仅标记失败 taskId={} shop={} msg={}",
|
||||||
|
result.getTaskId(), shopKey, ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
log.info("[shop-match] 失败店铺无可用行,不产出结果文件 taskId={} shop={} error={}",
|
||||||
|
result.getTaskId(), shopKey, errorMessage);
|
||||||
|
}
|
||||||
|
markResultFailed(result, message);
|
||||||
|
shopMatchTaskCacheService.removeShopMergedPayload(result.getTaskId(), shopKey);
|
||||||
|
return message;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void markResultFilePending(FileResultEntity result, String shopKey,
|
||||||
|
ShopMatchShopPayloadDto payload, boolean preserveFailure) {
|
||||||
Map<String, List<ShopMatchRowDto>> countries = excelAssemblyService.normalizeCountriesMap(payload.getCountries());
|
Map<String, List<ShopMatchRowDto>> countries = excelAssemblyService.normalizeCountriesMap(payload.getCountries());
|
||||||
String displayName = payload.getShopName() != null && !payload.getShopName().isBlank() ? payload.getShopName().trim() : shopKey;
|
String displayName = payload.getShopName() != null && !payload.getShopName().isBlank() ? payload.getShopName().trim() : shopKey;
|
||||||
String stem = safeFileStem(displayName);
|
String stem = safeFileStem(displayName);
|
||||||
@@ -875,8 +923,16 @@ public class ShopMatchTaskService {
|
|||||||
result.setResultFileSize(0L);
|
result.setResultFileSize(0L);
|
||||||
result.setResultContentType(CONTENT_TYPE_XLSX);
|
result.setResultContentType(CONTENT_TYPE_XLSX);
|
||||||
result.setRowCount(excelAssemblyService.countRows(countries));
|
result.setRowCount(excelAssemblyService.countRows(countries));
|
||||||
result.setSuccess(1);
|
if (preserveFailure) {
|
||||||
result.setErrorMessage(null);
|
// 失败店铺的部分结果:成功态与失败原因都不能动,只标「文件名已定、文件待组装」
|
||||||
|
result.setSuccess(0);
|
||||||
|
if (result.getErrorMessage() == null || result.getErrorMessage().isBlank()) {
|
||||||
|
result.setErrorMessage("店铺未跑完,仅产出部分结果");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
result.setSuccess(1);
|
||||||
|
result.setErrorMessage(null);
|
||||||
|
}
|
||||||
fileResultMapper.updateById(result);
|
fileResultMapper.updateById(result);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+2
-1
@@ -277,7 +277,8 @@ public class SimilarAsinResultRowDto {
|
|||||||
@Schema(description = "阿里巴巴商品图片或商品 URL")
|
@Schema(description = "阿里巴巴商品图片或商品 URL")
|
||||||
private String url;
|
private String url;
|
||||||
|
|
||||||
@JsonAlias({"price", "浠锋牸"})
|
// "浠锋牸" 是 "价格" 被按 GBK 解读后的乱码,历史载荷里出现过,保留兼容
|
||||||
|
@JsonAlias({"price", "价格", "浠锋牸"})
|
||||||
@Schema(description = "阿里巴巴候选商品价格")
|
@Schema(description = "阿里巴巴候选商品价格")
|
||||||
private Object price;
|
private Object price;
|
||||||
|
|
||||||
|
|||||||
+9
@@ -87,6 +87,15 @@ public class SimilarAsinChunkPayloadSupport {
|
|||||||
recordChunkReadFailure(chunk, false, msg);
|
recordChunkReadFailure(chunk, false, msg);
|
||||||
return rows;
|
return rows;
|
||||||
}
|
}
|
||||||
|
// payload 对象已不存在(RustFS 返回 NoSuchKey):重试多少次都读不回来,跳过而不是
|
||||||
|
// 把组装/收尾永久拖死——与上面的 typeMismatch 分支、以及 collect-data 的降级口径一致。
|
||||||
|
// 线上 appearance-patent 28459 就因同类场景每 10~30 秒重试一次(见同批修复)。
|
||||||
|
if (msg.contains("does not exist")) {
|
||||||
|
log.warn("[similar-asin] chunk payload 已不存在,跳过该分片 taskId={} chunk={} err={}",
|
||||||
|
chunk.getTaskId(), chunk.getChunkIndex(), msg);
|
||||||
|
recordChunkReadFailure(chunk, false, msg);
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
log.warn("[similar-asin] read chunk payload failed taskId={} chunk={} crossInstance={} err={}",
|
log.warn("[similar-asin] read chunk payload failed taskId={} chunk={} crossInstance={} err={}",
|
||||||
chunk.getTaskId(), chunk.getChunkIndex(), crossInstance, msg);
|
chunk.getTaskId(), chunk.getChunkIndex(), crossInstance, msg);
|
||||||
recordChunkReadFailure(chunk, crossInstance, msg);
|
recordChunkReadFailure(chunk, crossInstance, msg);
|
||||||
|
|||||||
+31
-7
@@ -15,6 +15,7 @@ import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
|||||||
import com.fasterxml.jackson.databind.JsonNode;
|
import com.fasterxml.jackson.databind.JsonNode;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.nanri.aiimage.common.exception.BusinessException;
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import com.nanri.aiimage.common.exception.BusinessCodes;
|
||||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
||||||
@@ -213,6 +214,7 @@ public class SimilarAsinPipelineSupport {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
int maxAttempts = 3;
|
int maxAttempts = 3;
|
||||||
|
String conflictDetail = "未发生冲突";
|
||||||
for (int attempt = 1; attempt <= maxAttempts; attempt++) {
|
for (int attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||||
TaskChunkEntity chunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
TaskChunkEntity chunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
@@ -274,13 +276,35 @@ public class SimilarAsinPipelineSupport {
|
|||||||
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(oldPayload, storedPayload);
|
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(oldPayload, storedPayload);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
|
// CAS 冲突:读回行上的当前哈希,否则只有一句 conflict 无从判断是谁改的
|
||||||
|
String currentHash = currentPayloadHash(chunk.getId());
|
||||||
|
conflictDetail = "期望hash=" + oldPayloadHash + " 当前hash=" + currentHash;
|
||||||
if (attempt < maxAttempts) {
|
if (attempt < maxAttempts) {
|
||||||
log.warn("[similar-asin] chunk payload CAS conflict, retry merge taskId={} scopeHash={} chunk={} attempt={}/{}",
|
log.warn("[similar-asin] chunk payload CAS conflict, retry merge taskId={} scopeHash={} chunk={} attempt={}/{} 期望hash={} 当前hash={}",
|
||||||
taskId, scopeHash, chunkIndex, attempt, maxAttempts);
|
taskId, scopeHash, chunkIndex, attempt, maxAttempts, oldPayloadHash, currentHash);
|
||||||
|
// 还要重试:这次写的对象会被下次重写,先删掉避免堆积
|
||||||
|
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
|
||||||
|
} else {
|
||||||
|
// 终局失败:保留这次写入的版本化对象,作为读路径「同槽位兄弟对象」兜底的恢复源。
|
||||||
|
// 与外观专利同一口径——行没指过去不该让该分片永久判死。
|
||||||
|
log.error("[similar-asin] chunk payload 合并冲突重试耗尽,保留兜底对象 taskId={} scopeHash={} chunk={} 期望hash={} 当前hash={} 保留对象={}",
|
||||||
|
taskId, scopeHash, chunkIndex, oldPayloadHash, currentHash, storedPayload);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
throw new IllegalStateException("相似ASIN分片载荷更新失败");
|
throw new IllegalStateException("相似ASIN分片载荷更新失败 " + conflictDetail);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 读回 chunk 行当前的 payload_hash,用于 CAS 冲突定位(行已不存在/读取失败时返回可读标记)。 */
|
||||||
|
private String currentPayloadHash(Long chunkId) {
|
||||||
|
if (chunkId == null) {
|
||||||
|
return "chunkId 为空";
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
TaskChunkEntity latest = taskChunkMapper.selectById(chunkId);
|
||||||
|
return latest == null ? "行已不存在" : latest.getPayloadHash();
|
||||||
|
} catch (Exception ex) {
|
||||||
|
return "读取失败:" + ex.getMessage();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -468,7 +492,7 @@ public class SimilarAsinPipelineSupport {
|
|||||||
public PreparedSubmittedChunk prepareSubmittedChunk(Long taskId, SimilarAsinSubmitResultRequest request) {
|
public PreparedSubmittedChunk prepareSubmittedChunk(Long taskId, SimilarAsinSubmitResultRequest request) {
|
||||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
||||||
throw new BusinessException("任务不存在");
|
throw new BusinessException(BusinessCodes.TASK_NOT_FOUND, "任务不存在");
|
||||||
}
|
}
|
||||||
if (!STATUS_RUNNING.equals(task.getStatus())) {
|
if (!STATUS_RUNNING.equals(task.getStatus())) {
|
||||||
throw new BusinessException("任务不是运行中状态");
|
throw new BusinessException("任务不是运行中状态");
|
||||||
@@ -509,7 +533,7 @@ public class SimilarAsinPipelineSupport {
|
|||||||
Long taskId = prepared.taskId();
|
Long taskId = prepared.taskId();
|
||||||
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
||||||
throw new BusinessException("任务不存在");
|
throw new BusinessException(BusinessCodes.TASK_NOT_FOUND, "任务不存在");
|
||||||
}
|
}
|
||||||
if (!STATUS_RUNNING.equals(task.getStatus())) {
|
if (!STATUS_RUNNING.equals(task.getStatus())) {
|
||||||
throw new BusinessException("任务不是运行中状态");
|
throw new BusinessException("任务不是运行中状态");
|
||||||
@@ -561,7 +585,7 @@ public class SimilarAsinPipelineSupport {
|
|||||||
public FinalizeTaskResult completeSubmittedChunk(SubmitContext context) {
|
public FinalizeTaskResult completeSubmittedChunk(SubmitContext context) {
|
||||||
FileTaskEntity task = fileTaskMapper.selectById(context.task().getId());
|
FileTaskEntity task = fileTaskMapper.selectById(context.task().getId());
|
||||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
||||||
throw new BusinessException("任务不存在");
|
throw new BusinessException(BusinessCodes.TASK_NOT_FOUND, "任务不存在");
|
||||||
}
|
}
|
||||||
if (!STATUS_RUNNING.equals(task.getStatus())) {
|
if (!STATUS_RUNNING.equals(task.getStatus())) {
|
||||||
log.info("[similar-asin] skip completion because task already finalized taskId={} status={}",
|
log.info("[similar-asin] skip completion because task already finalized taskId={} status={}",
|
||||||
|
|||||||
@@ -3,7 +3,35 @@ package com.nanri.aiimage.modules.task.mapper;
|
|||||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
import org.apache.ibatis.annotations.Mapper;
|
import org.apache.ibatis.annotations.Mapper;
|
||||||
|
import org.apache.ibatis.annotations.Param;
|
||||||
|
import org.apache.ibatis.annotations.Select;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
@Mapper
|
@Mapper
|
||||||
public interface FileTaskMapper extends BaseMapper<FileTaskEntity> {
|
public interface FileTaskMapper extends BaseMapper<FileTaskEntity> {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 查询某模块下超过保留期的终态任务 id(保留期清理用,只取 id 不拉整行)。
|
||||||
|
*
|
||||||
|
* <p>终态集合与 ModuleHistoryCleanupService 的 TERMINAL_STATUSES 一致:PENDING/RUNNING 绝不返回——
|
||||||
|
* 正在跑的任务被删会让分片回传、结果组装找不到任务行。
|
||||||
|
*
|
||||||
|
* <p>时间线只用 updated_at:该列由 DB 的 ON UPDATE CURRENT_TIMESTAMP 维护,任何一次行更新
|
||||||
|
* (含终态写入)都会刷新,不会早于 finished_at(finished_at 在任务重置时会被置空,不可单独依赖);
|
||||||
|
* 且能命中 V120 的 idx_biz_file_task_status_updated(status, updated_at) 索引,避免每轮全表扫描。
|
||||||
|
* 注意不要在这里包 COALESCE(finished_at, updated_at):表达式会让该索引失效。
|
||||||
|
*/
|
||||||
|
@Select("""
|
||||||
|
SELECT id FROM biz_file_task
|
||||||
|
WHERE module_type = #{moduleType}
|
||||||
|
AND status IN ('SUCCESS', 'FAILED', 'CANCELLED', 'CANCELED')
|
||||||
|
AND updated_at < #{cutoff}
|
||||||
|
ORDER BY id ASC
|
||||||
|
LIMIT #{batchSize}
|
||||||
|
""")
|
||||||
|
List<Long> selectExpiredTerminalTaskIds(@Param("moduleType") String moduleType,
|
||||||
|
@Param("cutoff") LocalDateTime cutoff,
|
||||||
|
@Param("batchSize") int batchSize);
|
||||||
}
|
}
|
||||||
|
|||||||
+4
@@ -26,6 +26,10 @@ public class FileTaskEntity {
|
|||||||
private String createdBy;
|
private String createdBy;
|
||||||
private Long userId;
|
private Long userId;
|
||||||
private String ownerInstanceId;
|
private String ownerInstanceId;
|
||||||
|
/** 续跑来源任务 ID:本行是客户端中断后由服务端自动重排队的续跑任务时非空(V129)。 */
|
||||||
|
private Long resumeOfTaskId;
|
||||||
|
/** 续跑代数:0=原始任务,N=第 N 次自动续跑(封顶用,V129)。 */
|
||||||
|
private Integer resumeAttempt;
|
||||||
private LocalDateTime createdAt;
|
private LocalDateTime createdAt;
|
||||||
private LocalDateTime updatedAt;
|
private LocalDateTime updatedAt;
|
||||||
private LocalDateTime finishedAt;
|
private LocalDateTime finishedAt;
|
||||||
|
|||||||
+133
-21
@@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
|||||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||||
import com.nanri.aiimage.config.ModuleCleanupProperties;
|
import com.nanri.aiimage.config.ModuleCleanupProperties;
|
||||||
|
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||||
import com.nanri.aiimage.modules.task.spi.CollectDataItemCleanupSpi;
|
import com.nanri.aiimage.modules.task.spi.CollectDataItemCleanupSpi;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
@@ -44,6 +45,8 @@ public class ModuleHistoryCleanupService {
|
|||||||
private static final String COLLECT_DATA_MODULE_TYPE = "COLLECT_DATA";
|
private static final String COLLECT_DATA_MODULE_TYPE = "COLLECT_DATA";
|
||||||
private static final int DEFAULT_BATCH_SIZE = 500;
|
private static final int DEFAULT_BATCH_SIZE = 500;
|
||||||
private static final int LOG_SAMPLE_IDS = 5;
|
private static final int LOG_SAMPLE_IDS = 5;
|
||||||
|
/** 指针收集触顶时任务组的最大二分拆分深度(500 任务拆到单个任务约需 9 层)。 */
|
||||||
|
private static final int MAX_POINTER_SPLIT_DEPTH = 20;
|
||||||
|
|
||||||
private final ModuleCleanupProperties moduleCleanupProperties;
|
private final ModuleCleanupProperties moduleCleanupProperties;
|
||||||
private final FileTaskMapper fileTaskMapper;
|
private final FileTaskMapper fileTaskMapper;
|
||||||
@@ -57,14 +60,15 @@ public class ModuleHistoryCleanupService {
|
|||||||
private final CollectDataItemCleanupSpi collectDataItemCleanupSpi;
|
private final CollectDataItemCleanupSpi collectDataItemCleanupSpi;
|
||||||
private final DistributedJobLockService distributedJobLockService;
|
private final DistributedJobLockService distributedJobLockService;
|
||||||
private final TransientPayloadDeleteOrchestrator transientPayloadDeleteOrchestrator;
|
private final TransientPayloadDeleteOrchestrator transientPayloadDeleteOrchestrator;
|
||||||
|
private final OssStorageService ossStorageService;
|
||||||
private final TransactionTemplate transactionTemplate;
|
private final TransactionTemplate transactionTemplate;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 单批最多收集的 payload 指针数:超过即截断(保底可重试),
|
* 单组最多收集的 payload 指针数:超过即把任务组二分拆分(见 {@link #cleanupTaskGroup}),
|
||||||
* 防止单任务行数异常巨大时无界收集造成内存增长。
|
* 防止单任务行数异常巨大时无界收集造成内存增长。
|
||||||
*/
|
*/
|
||||||
@Value("${aiimage.module-cleanup.max-collect-payloads:10000}")
|
@Value("${aiimage.module-cleanup.max-collect-payloads:50000}")
|
||||||
private int maxCollectPayloadsPerRun = 10000;
|
private int maxCollectPayloadsPerRun = 50000;
|
||||||
|
|
||||||
public ModuleHistoryCleanupService(ModuleCleanupProperties moduleCleanupProperties,
|
public ModuleHistoryCleanupService(ModuleCleanupProperties moduleCleanupProperties,
|
||||||
FileTaskMapper fileTaskMapper,
|
FileTaskMapper fileTaskMapper,
|
||||||
@@ -78,6 +82,7 @@ public class ModuleHistoryCleanupService {
|
|||||||
CollectDataItemCleanupSpi collectDataItemCleanupSpi,
|
CollectDataItemCleanupSpi collectDataItemCleanupSpi,
|
||||||
DistributedJobLockService distributedJobLockService,
|
DistributedJobLockService distributedJobLockService,
|
||||||
TransientPayloadDeleteOrchestrator transientPayloadDeleteOrchestrator,
|
TransientPayloadDeleteOrchestrator transientPayloadDeleteOrchestrator,
|
||||||
|
OssStorageService ossStorageService,
|
||||||
PlatformTransactionManager platformTransactionManager) {
|
PlatformTransactionManager platformTransactionManager) {
|
||||||
this.moduleCleanupProperties = moduleCleanupProperties;
|
this.moduleCleanupProperties = moduleCleanupProperties;
|
||||||
this.fileTaskMapper = fileTaskMapper;
|
this.fileTaskMapper = fileTaskMapper;
|
||||||
@@ -91,6 +96,7 @@ public class ModuleHistoryCleanupService {
|
|||||||
this.collectDataItemCleanupSpi = collectDataItemCleanupSpi;
|
this.collectDataItemCleanupSpi = collectDataItemCleanupSpi;
|
||||||
this.distributedJobLockService = distributedJobLockService;
|
this.distributedJobLockService = distributedJobLockService;
|
||||||
this.transientPayloadDeleteOrchestrator = transientPayloadDeleteOrchestrator;
|
this.transientPayloadDeleteOrchestrator = transientPayloadDeleteOrchestrator;
|
||||||
|
this.ossStorageService = ossStorageService;
|
||||||
this.transactionTemplate = new TransactionTemplate(platformTransactionManager);
|
this.transactionTemplate = new TransactionTemplate(platformTransactionManager);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -161,20 +167,11 @@ public class ModuleHistoryCleanupService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (!batchTaskIds.isEmpty()) {
|
if (!batchTaskIds.isEmpty()) {
|
||||||
final List<Long> taskIds = batchTaskIds;
|
BatchTally tally = new BatchTally();
|
||||||
final List<Long> collectDataTaskIds = batchCollectDataTaskIds;
|
cleanupTaskGroup(moduleTypes, batchTaskIds, batchCollectDataTaskIds, 0, tally);
|
||||||
final List<String> types = moduleTypes;
|
totalBatches += tally.batches;
|
||||||
final List<String> collected = new ArrayList<>();
|
totalDeletedTasks += tally.deletedTasks;
|
||||||
transactionTemplate.executeWithoutResult(status -> {
|
totalCollectedPointers += tally.collectedPointers;
|
||||||
collected.addAll(collectPayloadPointers(types, taskIds, maxCollectPayloadsPerRun));
|
|
||||||
int deletedRows = deleteRows(types, taskIds, collectDataTaskIds);
|
|
||||||
submitAndFlush(collected);
|
|
||||||
log.info("[module-cleanup] batch: taskIds={}, deletedRows={}, collectedPointers={}",
|
|
||||||
formatIdSample(taskIds, LOG_SAMPLE_IDS), deletedRows, collected.size());
|
|
||||||
});
|
|
||||||
totalBatches++;
|
|
||||||
totalDeletedTasks += taskIds.size();
|
|
||||||
totalCollectedPointers += collected.size();
|
|
||||||
}
|
}
|
||||||
if (pageMaxId <= cursor) {
|
if (pageMaxId <= cursor) {
|
||||||
log.warn("[module-cleanup] keyset cursor did not advance, abort loop cursor={}", cursor);
|
log.warn("[module-cleanup] keyset cursor did not advance, abort loop cursor={}", cursor);
|
||||||
@@ -190,7 +187,107 @@ public class ModuleHistoryCleanupService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 一个事务内完成:收集指针 → 删除本批行 → 行删完后提交清理队列并 flush。 */
|
/**
|
||||||
|
* 清理一组过期任务:收集 payload 指针 → 删除行 → 提交对象回收。
|
||||||
|
*
|
||||||
|
* <p>指针数触顶时把任务组二分拆分重试,保证「凡被删除的行,其 transient 指针一定已进回收队列」——
|
||||||
|
* 截断会让指针随行一起消失,对象再无引用可查、永久留在桶里。
|
||||||
|
* 拆到单个任务仍触顶则整组不删并记 error(宁可留脏行待下轮重试,也不制造孤儿对象);
|
||||||
|
* 运维可据此调大 {@code aiimage.module-cleanup.max-collect-payloads} 后自动收敛。
|
||||||
|
*/
|
||||||
|
private void cleanupTaskGroup(List<String> moduleTypes, List<Long> taskIds, List<Long> collectDataTaskIds,
|
||||||
|
int depth, BatchTally tally) {
|
||||||
|
if (taskIds.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
List<String> collected = collectPayloadPointers(moduleTypes, taskIds, maxCollectPayloadsPerRun);
|
||||||
|
boolean capped = collected.size() >= maxCollectPayloadsPerRun;
|
||||||
|
if (capped && taskIds.size() > 1 && depth < MAX_POINTER_SPLIT_DEPTH) {
|
||||||
|
int half = taskIds.size() / 2;
|
||||||
|
log.warn("[module-cleanup] payload 指针数触顶 {},任务组二分拆分重试: depth={}, size={}",
|
||||||
|
maxCollectPayloadsPerRun, depth, taskIds.size());
|
||||||
|
cleanupTaskGroup(moduleTypes, new ArrayList<>(taskIds.subList(0, half)),
|
||||||
|
collectDataTaskIds, depth + 1, tally);
|
||||||
|
cleanupTaskGroup(moduleTypes, new ArrayList<>(taskIds.subList(half, taskIds.size())),
|
||||||
|
collectDataTaskIds, depth + 1, tally);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (capped) {
|
||||||
|
log.error("[module-cleanup] payload 指针数触顶且无法再拆分,本轮保留任务行待下轮重试: taskIds={}",
|
||||||
|
formatIdSample(taskIds, LOG_SAMPLE_IDS));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
final List<Long> ids = List.copyOf(taskIds);
|
||||||
|
final Set<Long> collectDataIdSet = Set.copyOf(collectDataTaskIds);
|
||||||
|
final List<Long> groupCollectDataIds = ids.stream().filter(collectDataIdSet::contains).toList();
|
||||||
|
List<String> resultObjectKeys = collectResultObjectKeys(moduleTypes, ids);
|
||||||
|
transactionTemplate.executeWithoutResult(status -> {
|
||||||
|
int deletedRows = deleteRows(moduleTypes, ids, groupCollectDataIds);
|
||||||
|
log.info("[module-cleanup] batch: taskIds={}, deletedRows={}, collectedPointers={}, resultObjects={}",
|
||||||
|
formatIdSample(ids, LOG_SAMPLE_IDS), deletedRows, collected.size(), resultObjectKeys.size());
|
||||||
|
});
|
||||||
|
// 远端删除一律放在事务提交之后:事务里做,一旦回滚就会出现「行还在、对象已被删」的悬空引用
|
||||||
|
submitAndFlush(collected);
|
||||||
|
deleteResultObjects(resultObjectKeys);
|
||||||
|
tally.batches++;
|
||||||
|
tally.deletedTasks += ids.size();
|
||||||
|
tally.collectedPointers += collected.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 收集随行删除的结果文件对象 key(file_result.result_file_url、task_file_job.result_file_url)。
|
||||||
|
*
|
||||||
|
* <p>这些对象此前从不回收——行删掉后再没有任何地方记录过它们,桶里只能靠生命周期规则兜底;
|
||||||
|
* 一旦某天桶规则被调整(历史上就误配过全桶 30 天过期),就会变成永久垃圾。
|
||||||
|
*/
|
||||||
|
private List<String> collectResultObjectKeys(List<String> moduleTypes, List<Long> cleanupTaskIds) {
|
||||||
|
java.util.Set<String> keys = new java.util.LinkedHashSet<>();
|
||||||
|
List<FileResultEntity> results = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
||||||
|
.in(FileResultEntity::getModuleType, moduleTypes)
|
||||||
|
.in(FileResultEntity::getTaskId, cleanupTaskIds));
|
||||||
|
for (FileResultEntity result : results) {
|
||||||
|
addObjectKey(keys, result.getResultFileUrl());
|
||||||
|
}
|
||||||
|
List<TaskFileJobEntity> jobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||||
|
.in(TaskFileJobEntity::getModuleType, moduleTypes)
|
||||||
|
.in(TaskFileJobEntity::getTaskId, cleanupTaskIds));
|
||||||
|
for (TaskFileJobEntity job : jobs) {
|
||||||
|
addObjectKey(keys, job.getResultFileUrl());
|
||||||
|
}
|
||||||
|
return new ArrayList<>(keys);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addObjectKey(java.util.Set<String> keys, String value) {
|
||||||
|
if (value != null && !value.isBlank()) {
|
||||||
|
keys.add(value.trim());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 逐个回收结果对象;单个失败只记日志,不影响其余对象与后续批次。 */
|
||||||
|
private void deleteResultObjects(List<String> objectKeys) {
|
||||||
|
if (objectKeys.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
int failed = 0;
|
||||||
|
for (String objectKey : objectKeys) {
|
||||||
|
try {
|
||||||
|
ossStorageService.deleteObject(objectKey);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
failed++;
|
||||||
|
log.warn("[module-cleanup] 结果对象删除失败 key={} msg={}", objectKey, ex.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.info("[module-cleanup] 结果对象回收完成 count={} failed={}", objectKeys.size(), failed);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 分组拆分后回传计数:批次数、删除任务数、收集指针数。 */
|
||||||
|
private static final class BatchTally {
|
||||||
|
private int batches;
|
||||||
|
private int deletedTasks;
|
||||||
|
private int collectedPointers;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 一个事务内完成:删除本批行 → 行删完后提交清理队列并 flush(指针已由调用方收集)。 */
|
||||||
private int deleteRows(List<String> moduleTypes, List<Long> cleanupTaskIds, List<Long> collectDataTaskIds) {
|
private int deleteRows(List<String> moduleTypes, List<Long> cleanupTaskIds, List<Long> collectDataTaskIds) {
|
||||||
taskFileJobMapper.delete(new LambdaQueryWrapper<TaskFileJobEntity>()
|
taskFileJobMapper.delete(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||||
.in(TaskFileJobEntity::getModuleType, moduleTypes)
|
.in(TaskFileJobEntity::getModuleType, moduleTypes)
|
||||||
@@ -257,8 +354,11 @@ public class ModuleHistoryCleanupService {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 删除前批量收集将随行删除的 payload 指针(chunk.payloadJson、
|
* 删除前批量收集将随行删除的 payload 指针(chunk.payloadJson、
|
||||||
* scope_state.parsedPayloadJson / stateJson),去重并保持稳定顺序;
|
* scope_state.parsedPayloadJson / stateJson、result_item.payloadJson、
|
||||||
* 达到 {@code max} 上限即截断,防止异常巨大的任务行数引发无界收集。
|
* result_payload.payloadJson),去重并保持稳定顺序。
|
||||||
|
*
|
||||||
|
* <p>必须覆盖**所有**存放 transient 指针的列:行一旦删除,漏收的指针就再也无法
|
||||||
|
* 定位对象,桶里会永久残留孤儿对象。新增写入 payload 指针的列时必须同步加进来。
|
||||||
*/
|
*/
|
||||||
private List<String> collectPayloadPointers(List<String> moduleTypes, List<Long> cleanupTaskIds, int max) {
|
private List<String> collectPayloadPointers(List<String> moduleTypes, List<Long> cleanupTaskIds, int max) {
|
||||||
java.util.Set<String> pointers = new java.util.LinkedHashSet<>();
|
java.util.Set<String> pointers = new java.util.LinkedHashSet<>();
|
||||||
@@ -275,8 +375,20 @@ public class ModuleHistoryCleanupService {
|
|||||||
collectPointer(pointers, scopeState.getParsedPayloadJson(), max);
|
collectPointer(pointers, scopeState.getParsedPayloadJson(), max);
|
||||||
collectPointer(pointers, scopeState.getStateJson(), max);
|
collectPointer(pointers, scopeState.getStateJson(), max);
|
||||||
}
|
}
|
||||||
|
List<TaskResultItemEntity> resultItems = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||||
|
.in(TaskResultItemEntity::getModuleType, moduleTypes)
|
||||||
|
.in(TaskResultItemEntity::getTaskId, cleanupTaskIds));
|
||||||
|
for (TaskResultItemEntity resultItem : resultItems) {
|
||||||
|
collectPointer(pointers, resultItem.getPayloadJson(), max);
|
||||||
|
}
|
||||||
|
List<TaskResultPayloadEntity> resultPayloads = taskResultPayloadMapper.selectList(new LambdaQueryWrapper<TaskResultPayloadEntity>()
|
||||||
|
.in(TaskResultPayloadEntity::getModuleType, moduleTypes)
|
||||||
|
.in(TaskResultPayloadEntity::getTaskId, cleanupTaskIds));
|
||||||
|
for (TaskResultPayloadEntity resultPayload : resultPayloads) {
|
||||||
|
collectPointer(pointers, resultPayload.getPayloadJson(), max);
|
||||||
|
}
|
||||||
if (pointers.size() >= max) {
|
if (pointers.size() >= max) {
|
||||||
log.warn("[module-cleanup] payload pointer collection truncated at max={}", max);
|
log.warn("[module-cleanup] payload pointer collection reached max={}, 交由调用方拆分重试", max);
|
||||||
}
|
}
|
||||||
return new ArrayList<>(pointers);
|
return new ArrayList<>(pointers);
|
||||||
}
|
}
|
||||||
|
|||||||
+28
-2
@@ -298,12 +298,19 @@ public class TaskFileJobService {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Transactional
|
|
||||||
public int resetStuckRunningJobs(int stuckMinutes, int limit) {
|
public int resetStuckRunningJobs(int stuckMinutes, int limit) {
|
||||||
return resetStuckRunningJobsDetailed(stuckMinutes, limit).resetCount();
|
return resetStuckRunningJobsDetailed(stuckMinutes, limit).resetCount();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Transactional
|
/**
|
||||||
|
* 卡住任务自愈扫描。
|
||||||
|
*
|
||||||
|
* <p>刻意**不加 @Transactional**:原实现在一个事务里 SELECT 最多 200 行再逐行 UPDATE,
|
||||||
|
* 而两台实例各跑一份 @Scheduled,双方拿到同一批 RUNNING 行后互相等行锁,直接造成线上
|
||||||
|
* 每天上百次 "Lock wait timeout exceeded"(biz_task_file_job)。
|
||||||
|
* 这里的每行更新本就带 status + updatedAt 的 CAS 条件(幂等),逐行独立提交更安全:
|
||||||
|
* 锁即时释放,CAS 不匹配的一方返回 0 行即可,也不会因中途异常回滚掉已修好的行。
|
||||||
|
*/
|
||||||
public StuckJobResetResult resetStuckRunningJobsDetailed(int stuckMinutes, int limit) {
|
public StuckJobResetResult resetStuckRunningJobsDetailed(int stuckMinutes, int limit) {
|
||||||
LocalDateTime threshold = LocalDateTime.now().minusMinutes(Math.max(5, stuckMinutes));
|
LocalDateTime threshold = LocalDateTime.now().minusMinutes(Math.max(5, stuckMinutes));
|
||||||
List<TaskFileJobEntity> jobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
|
List<TaskFileJobEntity> jobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||||
@@ -636,6 +643,25 @@ public class TaskFileJobService {
|
|||||||
.set(TaskFileJobEntity::getTerminalCallbackAt, LocalDateTime.now()));
|
.set(TaskFileJobEntity::getTerminalCallbackAt, LocalDateTime.now()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 该任务是否已有「重试耗尽且已走完终态回调」的组装 job。
|
||||||
|
*
|
||||||
|
* <p>用于卡死恢复(stale recovery)判断"再重建一次还有没有意义":数据永久缺失时,
|
||||||
|
* 每轮重建只会再失败一次,而恢复过程又会刷新任务心跳,导致任务永远 RUNNING、
|
||||||
|
* 恢复每 30 秒空转一轮(线上任务 28459 实测)。
|
||||||
|
*/
|
||||||
|
public boolean hasExhaustedAssembleJob(Long taskId, String moduleType) {
|
||||||
|
if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return taskFileJobMapper.selectCount(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||||
|
.eq(TaskFileJobEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskFileJobEntity::getModuleType, moduleType)
|
||||||
|
.eq(TaskFileJobEntity::getJobType, JOB_TYPE_ASSEMBLE_RESULT)
|
||||||
|
.ge(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
|
||||||
|
.isNotNull(TaskFileJobEntity::getTerminalCallbackAt)) > 0;
|
||||||
|
}
|
||||||
|
|
||||||
public TaskFileJobEntity findAssembleJob(Long taskId, String moduleType, Long resultId) {
|
public TaskFileJobEntity findAssembleJob(Long taskId, String moduleType, Long resultId) {
|
||||||
return findJob(taskId, moduleType, resultId, JOB_TYPE_ASSEMBLE_RESULT);
|
return findJob(taskId, moduleType, resultId, JOB_TYPE_ASSEMBLE_RESULT);
|
||||||
}
|
}
|
||||||
|
|||||||
+190
@@ -0,0 +1,190 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.service;
|
||||||
|
|
||||||
|
import cn.hutool.core.util.IdUtil;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.spi.ClientTaskPullSpi;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Locale;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 客户端中断任务的自动续跑:**保留失败记录 + 重新排队一条 PENDING 续跑任务**。
|
||||||
|
*
|
||||||
|
* <p>场景:客户端被更新脚本/任务管理器杀掉时,{@code TaskHeartbeatService.markInterrupted} 会把在跑的任务
|
||||||
|
* 标 FAILED(原因「客户端异常中断: …」),用户能看到发生过什么;但长任务(采集/相似ASIN/外观专利/单次跟价)
|
||||||
|
* 一遇客户端重启就整个白跑,需要有人把它重新放回队列。本服务负责这件事。
|
||||||
|
*
|
||||||
|
* <p>续跑方式刻意复用已有链路:新建的任务落 PENDING,由 {@link TaskClientPullService} 的兜底拉取
|
||||||
|
* (客户端每分钟 {@code GET /api/tasks/pull-pending})领走执行——不再新造第二套派发机制。
|
||||||
|
* 只有注册了 {@link ClientTaskPullSpi} 的模块才会被续跑(这些模块的载荷能由服务端自行组装);
|
||||||
|
* 上架/改价等写操作模块不在其中,避免盲目重跑。
|
||||||
|
*
|
||||||
|
* <p>封顶 {@code max-attempt}:会话持续不可用(账号被风控、紫鸟未就绪)时,不封顶会无限重排队。
|
||||||
|
* 计数写在 {@code biz_file_task.resume_attempt}(V129),随续跑任务代代递增。
|
||||||
|
*
|
||||||
|
* <p>幂等:同一原任务已存在续跑任务({@code resume_of_task_id} 反查)则跳过,重复扫描安全。
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@Service
|
||||||
|
public class TaskResumeService {
|
||||||
|
|
||||||
|
private static final String STATUS_FAILED = "FAILED";
|
||||||
|
private static final String STATUS_PENDING = "PENDING";
|
||||||
|
/** 客户端重启中断的失败原因前缀,由 TaskHeartbeatService.markInterrupted 写入。 */
|
||||||
|
private static final String CLIENT_INTERRUPT_PREFIX = "客户端异常中断";
|
||||||
|
|
||||||
|
private final FileTaskMapper fileTaskMapper;
|
||||||
|
/** moduleType → 模块兜底载荷实现;只有注册了 SPI 的模块才能被自动续跑。 */
|
||||||
|
private final Map<String, ClientTaskPullSpi> resumeHandlers;
|
||||||
|
|
||||||
|
@Value("${aiimage.task-resume.enabled:true}")
|
||||||
|
private boolean enabled;
|
||||||
|
|
||||||
|
/** 续跑代数上限:达到后不再重排队,任务保持 FAILED 等人工介入。 */
|
||||||
|
@Value("${aiimage.task-resume.max-attempt:3}")
|
||||||
|
private int maxAttempt;
|
||||||
|
|
||||||
|
/** 只处理最近这段时间内中断的任务,避免开机扫描历史积压。 */
|
||||||
|
@Value("${aiimage.task-resume.window-minutes:30}")
|
||||||
|
private long windowMinutes;
|
||||||
|
|
||||||
|
@Value("${aiimage.task-resume.limit:20}")
|
||||||
|
private int limit;
|
||||||
|
|
||||||
|
public TaskResumeService(FileTaskMapper fileTaskMapper, List<ClientTaskPullSpi> pullSpiHandlers) {
|
||||||
|
this.fileTaskMapper = fileTaskMapper;
|
||||||
|
Map<String, ClientTaskPullSpi> index = new LinkedHashMap<>();
|
||||||
|
if (pullSpiHandlers != null) {
|
||||||
|
for (ClientTaskPullSpi handler : pullSpiHandlers) {
|
||||||
|
String moduleType = handler.moduleType();
|
||||||
|
if (moduleType == null || moduleType.isBlank()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
index.put(moduleType.trim().toUpperCase(Locale.ROOT), handler);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.resumeHandlers = Map.copyOf(index);
|
||||||
|
log.info("[task-resume] 可自动续跑模块注册完成 count={} modules={}", index.size(), index.keySet());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 扫描一轮:把最近因客户端中断而失败、且未续跑过的任务重新排队。 */
|
||||||
|
public ResumeStats resumeInterruptedTasks() {
|
||||||
|
ResumeStats stats = new ResumeStats();
|
||||||
|
if (!enabled) {
|
||||||
|
return stats;
|
||||||
|
}
|
||||||
|
if (resumeHandlers.isEmpty()) {
|
||||||
|
log.warn("[task-resume] 没有注册任何 ClientTaskPullSpi,跳过本轮续跑");
|
||||||
|
return stats;
|
||||||
|
}
|
||||||
|
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(Math.max(1L, windowMinutes));
|
||||||
|
int safeLimit = Math.max(1, limit);
|
||||||
|
List<FileTaskEntity> candidates = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||||
|
.eq(FileTaskEntity::getStatus, STATUS_FAILED)
|
||||||
|
.likeRight(FileTaskEntity::getErrorMessage, CLIENT_INTERRUPT_PREFIX)
|
||||||
|
.in(FileTaskEntity::getModuleType, resumeHandlers.keySet())
|
||||||
|
.lt(FileTaskEntity::getResumeAttempt, maxAttempt)
|
||||||
|
.ge(FileTaskEntity::getFinishedAt, cutoff)
|
||||||
|
.orderByAsc(FileTaskEntity::getId)
|
||||||
|
.last("limit " + safeLimit));
|
||||||
|
if (candidates.isEmpty()) {
|
||||||
|
stats.unsupportedTaskCount = countUnsupportedInterrupts(cutoff);
|
||||||
|
return stats;
|
||||||
|
}
|
||||||
|
stats.unsupportedTaskCount = countUnsupportedInterrupts(cutoff);
|
||||||
|
stats.scannedTaskCount = candidates.size();
|
||||||
|
for (FileTaskEntity original : candidates) {
|
||||||
|
if (original.getUserId() == null || original.getUserId() <= 0) {
|
||||||
|
log.warn("[task-resume] 原任务没有归属用户,跳过 taskId={}", original.getId());
|
||||||
|
stats.skippedTaskCount++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// 幂等:同一原任务已经排过一次续跑就跳过(重复扫描 / 双实例竞态下不会重复建单)
|
||||||
|
if (hasResumeChild(original.getId())) {
|
||||||
|
stats.skippedTaskCount++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
FileTaskEntity resume = buildResumeTask(original);
|
||||||
|
try {
|
||||||
|
fileTaskMapper.insert(resume);
|
||||||
|
stats.resumedTaskCount++;
|
||||||
|
log.warn("[task-resume] 已自动重新排队 taskId={} moduleType={} userId={} 续跑任务={} 代数={}/{} 中断原因={}",
|
||||||
|
original.getId(), original.getModuleType(), original.getUserId(),
|
||||||
|
resume.getId(), resume.getResumeAttempt(), maxAttempt, original.getErrorMessage());
|
||||||
|
} catch (Exception ex) {
|
||||||
|
stats.skippedTaskCount++;
|
||||||
|
log.error("[task-resume] 重新排队失败 taskId={} moduleType={} err={}",
|
||||||
|
original.getId(), original.getModuleType(), ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return stats;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 统计「因客户端中断而失败、但模块不支持自动续跑」的任务数。
|
||||||
|
*
|
||||||
|
* <p>这些模块(上架/改价/审批/商品管理采集/跟价的非循环任务等)的续跑载荷需要**用户在页面上选的
|
||||||
|
* 执行参数**(如 ziniao_version),而这份选择只存在于派发那一刻的浏览器里、没落到服务端
|
||||||
|
* request_json —— 自动重排队会用错参数。因此它们只做**可见**:数量进巡检 summary,
|
||||||
|
* 运维据此人工重跑;将来把这类参数回写落库后即可纳入续跑白名单。
|
||||||
|
*/
|
||||||
|
private int countUnsupportedInterrupts(LocalDateTime cutoff) {
|
||||||
|
try {
|
||||||
|
Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper<FileTaskEntity>()
|
||||||
|
.eq(FileTaskEntity::getStatus, STATUS_FAILED)
|
||||||
|
.likeRight(FileTaskEntity::getErrorMessage, CLIENT_INTERRUPT_PREFIX)
|
||||||
|
.notIn(FileTaskEntity::getModuleType, resumeHandlers.keySet())
|
||||||
|
.ge(FileTaskEntity::getFinishedAt, cutoff));
|
||||||
|
return count == null ? 0 : count.intValue();
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[task-resume] 统计不支持续跑的中断任务失败(忽略): {}", ex.getMessage());
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 该原任务是否已经有续跑任务(反查 resume_of_task_id)。 */
|
||||||
|
private boolean hasResumeChild(Long originalTaskId) {
|
||||||
|
Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper<FileTaskEntity>()
|
||||||
|
.eq(FileTaskEntity::getResumeOfTaskId, originalTaskId));
|
||||||
|
return count != null && count > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
private FileTaskEntity buildResumeTask(FileTaskEntity original) {
|
||||||
|
LocalDateTime now = LocalDateTime.now();
|
||||||
|
FileTaskEntity resume = new FileTaskEntity();
|
||||||
|
resume.setTaskNo(original.getModuleType() + "-" + IdUtil.getSnowflakeNextIdStr());
|
||||||
|
resume.setModuleType(original.getModuleType());
|
||||||
|
resume.setTaskMode(original.getTaskMode());
|
||||||
|
resume.setStatus(STATUS_PENDING);
|
||||||
|
resume.setSourceFileCount(original.getSourceFileCount());
|
||||||
|
resume.setSuccessFileCount(0);
|
||||||
|
resume.setFailedFileCount(0);
|
||||||
|
// 原任务的请求参数整体带走:各模块的 ClientTaskPullSpi 从 request_json / 关联表还原执行参数
|
||||||
|
resume.setRequestJson(original.getRequestJson());
|
||||||
|
resume.setCreatedBy(original.getCreatedBy());
|
||||||
|
resume.setUserId(original.getUserId());
|
||||||
|
resume.setResumeOfTaskId(original.getId());
|
||||||
|
int attempt = original.getResumeAttempt() == null ? 0 : original.getResumeAttempt();
|
||||||
|
resume.setResumeAttempt(attempt + 1);
|
||||||
|
resume.setCreatedAt(now);
|
||||||
|
resume.setUpdatedAt(now);
|
||||||
|
return resume;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 单轮统计(合并进 stale-check 的 summary 日志,避免定期刷屏)。 */
|
||||||
|
public static final class ResumeStats {
|
||||||
|
public int scannedTaskCount;
|
||||||
|
public int resumedTaskCount;
|
||||||
|
public int skippedTaskCount;
|
||||||
|
/** 模块不支持自动续跑的中断任务数(仅计数,供运维人工重跑)。 */
|
||||||
|
public int unsupportedTaskCount;
|
||||||
|
}
|
||||||
|
}
|
||||||
+67
-5
@@ -30,6 +30,7 @@ import java.util.Locale;
|
|||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
import java.util.zip.GZIPInputStream;
|
import java.util.zip.GZIPInputStream;
|
||||||
import java.util.zip.GZIPOutputStream;
|
import java.util.zip.GZIPOutputStream;
|
||||||
|
|
||||||
@@ -150,8 +151,20 @@ public class TransientPayloadStorageService {
|
|||||||
return decodeStoredPayloadBytes(readLocalPayloadBytes(stripLocalInstanceId(localKey)));
|
return decodeStoredPayloadBytes(readLocalPayloadBytes(stripLocalInstanceId(localKey)));
|
||||||
}
|
}
|
||||||
if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) {
|
if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) {
|
||||||
return decodeStoredPayloadBytes(
|
String objectKey = pointer.substring(RUSTFS_POINTER_PREFIX.length());
|
||||||
rustfsObjectStorageService.readObjectBytes(pointer.substring(RUSTFS_POINTER_PREFIX.length())));
|
try {
|
||||||
|
return decodeStoredPayloadBytes(rustfsObjectStorageService.readObjectBytes(objectKey));
|
||||||
|
} catch (RuntimeException readException) {
|
||||||
|
String sibling = findVersionedChunkSibling(objectKey, readException);
|
||||||
|
if (sibling == null) {
|
||||||
|
throw readException;
|
||||||
|
}
|
||||||
|
// 2026-09-17 线上任务 28459:行指向的普通 key 被误删,但同一分片槽位的版本化对象还在。
|
||||||
|
// 读出它即可让任务按已有数据出结果,不必整单失败。
|
||||||
|
log.warn("[transient-payload] chunk 载荷对象不存在,回退同槽位版本化对象 pointer={} sibling={}",
|
||||||
|
pointer, sibling);
|
||||||
|
return decodeStoredPayloadBytes(rustfsObjectStorageService.readObjectBytes(sibling));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (pointer.startsWith(OSS_POINTER_PREFIX)) {
|
if (pointer.startsWith(OSS_POINTER_PREFIX)) {
|
||||||
return decodeStoredPayloadBytes(
|
return decodeStoredPayloadBytes(
|
||||||
@@ -164,6 +177,53 @@ public class TransientPayloadStorageService {
|
|||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** chunk 载荷槽位的 entryKey 形态:{@code chunk-<index>}(版本化写入则形如 {@code chunk-<index>-<uuid>})。 */
|
||||||
|
private static final Pattern CHUNK_ENTRY_KEY_PATTERN = Pattern.compile("chunk-\\d+");
|
||||||
|
|
||||||
|
/** 兄弟对象查找上限:只为找回同槽位对象,不需要列全。 */
|
||||||
|
private static final int MAX_SIBLING_LOOKUP_KEYS = 50;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 指针对象已不存在时,尝试找回同一分片槽位的版本化兄弟对象。
|
||||||
|
*
|
||||||
|
* <p>两个条件同时满足才兜底,避免读到无关对象或掩盖真实故障:
|
||||||
|
* <ol>
|
||||||
|
* <li>末段 entryKey 是 {@code chunk-<index>} 形态——只有这种槽位才有「版本化兄弟」语义;</li>
|
||||||
|
* <li>失败原因是对象确实不存在(NoSuchKey)——权限/网络类失败照旧上抛以便重试。</li>
|
||||||
|
* </ol>
|
||||||
|
*/
|
||||||
|
private String findVersionedChunkSibling(String objectKey, Throwable cause) {
|
||||||
|
if (!isObjectMissing(cause)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
int slash = objectKey.lastIndexOf('/');
|
||||||
|
String directory = slash < 0 ? "" : objectKey.substring(0, slash + 1);
|
||||||
|
String fileName = slash < 0 ? objectKey : objectKey.substring(slash + 1);
|
||||||
|
if (!fileName.endsWith(".json")) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String entryKey = fileName.substring(0, fileName.length() - ".json".length());
|
||||||
|
if (!CHUNK_ENTRY_KEY_PATTERN.matcher(entryKey).matches()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
List<String> candidates = rustfsObjectStorageService.listObjectKeysNewestFirst(
|
||||||
|
directory + entryKey + "-", MAX_SIBLING_LOOKUP_KEYS);
|
||||||
|
return candidates.isEmpty() ? null : candidates.getFirst();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 对象确已不存在:RustFS 返回 NoSuchKey,message 为 "The specified key does not exist."。 */
|
||||||
|
private static boolean isObjectMissing(Throwable error) {
|
||||||
|
Throwable cursor = error;
|
||||||
|
while (cursor != null) {
|
||||||
|
String message = cursor.getMessage();
|
||||||
|
if (message != null && message.contains("does not exist")) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
cursor = cursor.getCause();
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
public void deletePayloadIfPresent(String value) {
|
public void deletePayloadIfPresent(String value) {
|
||||||
String pointer = extractPointer(value);
|
String pointer = extractPointer(value);
|
||||||
if (pointer == null) {
|
if (pointer == null) {
|
||||||
@@ -209,8 +269,10 @@ public class TransientPayloadStorageService {
|
|||||||
*
|
*
|
||||||
* <p>判断口径:
|
* <p>判断口径:
|
||||||
* <ul>
|
* <ul>
|
||||||
* <li>{@code biz_task_chunk.payload_json} 命中 > 1 行(> 1 表示除了 caller 视角下
|
* <li>{@code biz_task_chunk.payload_json} 命中任意行(≥ 1)→ 视为仍被引用。
|
||||||
* 自己即将释放的那一行之外,至少还有别的 chunk 行也指向同一对象)→ 视为仍被引用。</li>
|
* 曾用 {@code > 1} 作判据,等于放行「恰好还有 1 行引用」的情况,会把对方仍在用的对象
|
||||||
|
* 删掉(2026-09-17 线上任务 28459:合并成功后指针未落库 + 旧对象被删 → 该分片永久读不到)。
|
||||||
|
* 物理删除本就约定在 DB 行删除之后执行,故调用方正常路径下引用数必然为 0。</li>
|
||||||
* <li>{@code biz_task_scope_state.parsed_payload_json}/{@code state_json} 命中 > 0 行 →
|
* <li>{@code biz_task_scope_state.parsed_payload_json}/{@code state_json} 命中 > 0 行 →
|
||||||
* 视为仍被引用(这两个字段不是 caller 自身行的常见持有者,命中即非自我引用)。</li>
|
* 视为仍被引用(这两个字段不是 caller 自身行的常见持有者,命中即非自我引用)。</li>
|
||||||
* </ul>
|
* </ul>
|
||||||
@@ -248,7 +310,7 @@ public class TransientPayloadStorageService {
|
|||||||
// 解析不出 taskId 时按原口径全局查,行为与改造前一致。
|
// 解析不出 taskId 时按原口径全局查,行为与改造前一致。
|
||||||
Long pointerTaskId = extractTaskId(pointer);
|
Long pointerTaskId = extractTaskId(pointer);
|
||||||
Long chunkCount = referencedChunkCount(pointerTaskId, values);
|
Long chunkCount = referencedChunkCount(pointerTaskId, values);
|
||||||
if (chunkCount != null && chunkCount > 1L) {
|
if (chunkCount != null && chunkCount > 0L) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
Long scopeStateCount = referencedScopeStateCount(pointerTaskId, values);
|
Long scopeStateCount = referencedScopeStateCount(pointerTaskId, values);
|
||||||
|
|||||||
+9
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.usersecret.mapper;
|
|||||||
|
|
||||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||||
import com.nanri.aiimage.modules.usersecret.model.entity.UserSecretUsageEntity;
|
import com.nanri.aiimage.modules.usersecret.model.entity.UserSecretUsageEntity;
|
||||||
|
import org.apache.ibatis.annotations.Delete;
|
||||||
import org.apache.ibatis.annotations.Insert;
|
import org.apache.ibatis.annotations.Insert;
|
||||||
import org.apache.ibatis.annotations.Mapper;
|
import org.apache.ibatis.annotations.Mapper;
|
||||||
import org.apache.ibatis.annotations.Param;
|
import org.apache.ibatis.annotations.Param;
|
||||||
@@ -24,4 +25,12 @@ public interface UserSecretUsageMapper extends BaseMapper<UserSecretUsageEntity>
|
|||||||
@Param("moduleKey") String moduleKey,
|
@Param("moduleKey") String moduleKey,
|
||||||
@Param("businessDate") LocalDate businessDate,
|
@Param("businessDate") LocalDate businessDate,
|
||||||
@Param("count") int count);
|
@Param("count") int count);
|
||||||
|
|
||||||
|
/** 分批删除保留期外的用量日统计行(该表此前只增不删)。 */
|
||||||
|
@Delete("""
|
||||||
|
DELETE FROM biz_user_secret_usage_daily
|
||||||
|
WHERE business_date < #{cutoff}
|
||||||
|
LIMIT #{batchSize}
|
||||||
|
""")
|
||||||
|
int deleteBefore(@Param("cutoff") LocalDate cutoff, @Param("batchSize") int batchSize);
|
||||||
}
|
}
|
||||||
|
|||||||
+3
@@ -37,6 +37,9 @@ public class AdminUserSecretRowVo {
|
|||||||
@Schema(description = "行级状态说明")
|
@Schema(description = "行级状态说明")
|
||||||
private String statusMessage;
|
private String statusMessage;
|
||||||
|
|
||||||
|
@Schema(description = "首次配置时间(三模块中最早)")
|
||||||
|
private LocalDateTime createdAt;
|
||||||
|
|
||||||
@Schema(description = "最近更新时间(三模块中最晚)")
|
@Schema(description = "最近更新时间(三模块中最晚)")
|
||||||
private LocalDateTime updatedAt;
|
private LocalDateTime updatedAt;
|
||||||
}
|
}
|
||||||
|
|||||||
+14
@@ -41,6 +41,7 @@ import java.time.LocalDate;
|
|||||||
import java.time.LocalDateTime;
|
import java.time.LocalDateTime;
|
||||||
import java.time.format.DateTimeFormatter;
|
import java.time.format.DateTimeFormatter;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collection;
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
@@ -607,6 +608,7 @@ public class UserApiSecretService implements UserSecretCleanupPort {
|
|||||||
vo.setStatus(summarizeRowStatus(modules));
|
vo.setStatus(summarizeRowStatus(modules));
|
||||||
vo.setStatusMessage(summarizeRowMessage(modules, vo.getStatus()));
|
vo.setStatusMessage(summarizeRowMessage(modules, vo.getStatus()));
|
||||||
vo.setUpdatedAt(latestUpdatedAt(modules));
|
vo.setUpdatedAt(latestUpdatedAt(modules));
|
||||||
|
vo.setCreatedAt(earliestCreatedAt(moduleRows.values()));
|
||||||
AdminUserEntity user = adminUserMapper.selectById(userId);
|
AdminUserEntity user = adminUserMapper.selectById(userId);
|
||||||
vo.setUsername(user == null ? "" : user.getUsername());
|
vo.setUsername(user == null ? "" : user.getUsername());
|
||||||
vo.setGroups(List.of());
|
vo.setGroups(List.of());
|
||||||
@@ -673,6 +675,18 @@ public class UserApiSecretService implements UserSecretCleanupPort {
|
|||||||
return latest;
|
return latest;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** 行级创建时间:该用户已配置模块中最早的一条 created_at;一条都没有则为 null。 */
|
||||||
|
private LocalDateTime earliestCreatedAt(Collection<UserApiSecretEntity> rows) {
|
||||||
|
LocalDateTime earliest = null;
|
||||||
|
for (UserApiSecretEntity row : rows) {
|
||||||
|
LocalDateTime value = row.getCreatedAt();
|
||||||
|
if (value != null && (earliest == null || value.isBefore(earliest))) {
|
||||||
|
earliest = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return earliest;
|
||||||
|
}
|
||||||
|
|
||||||
private void upsert(Long userId, String moduleKey, String plainValue, String source) {
|
private void upsert(Long userId, String moduleKey, String plainValue, String source) {
|
||||||
LocalDateTime now = LocalDateTime.now();
|
LocalDateTime now = LocalDateTime.now();
|
||||||
UserApiSecretEntity existing = selectOne(userId, moduleKey);
|
UserApiSecretEntity existing = selectOne(userId, moduleKey);
|
||||||
|
|||||||
+69
@@ -0,0 +1,69 @@
|
|||||||
|
package com.nanri.aiimage.modules.usersecret.service;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||||
|
import com.nanri.aiimage.modules.usersecret.mapper.UserSecretUsageMapper;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.scheduling.annotation.Scheduled;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.LocalDate;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 密钥用量日统计表(biz_user_secret_usage_daily)的保留期清理。
|
||||||
|
*
|
||||||
|
* <p>每次调用按「用户 × 模块 × 业务日」upsert 累加,只增不删;量级是
|
||||||
|
* 用户数 × 模块数 × 天数,单行很小但长期累积无上限。
|
||||||
|
*
|
||||||
|
* <p>保留期取 400 天(而不是常见的一年):用量页要支持同比/跨年对比,
|
||||||
|
* 恰好一年前的数据仍有价值,留一点余量避免跨年时把刚过期的上年数据删掉。
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
public class UserSecretUsageRetentionService {
|
||||||
|
|
||||||
|
/** 单轮最多删除的批次数,剩余留给下一轮。 */
|
||||||
|
static final int MAX_BATCHES_PER_RUN = 20;
|
||||||
|
|
||||||
|
private final UserSecretUsageMapper usageMapper;
|
||||||
|
private final DistributedJobLockService distributedJobLockService;
|
||||||
|
|
||||||
|
@Value("${aiimage.user-secret.usage-retention-days:400}")
|
||||||
|
private int retentionDays = 400;
|
||||||
|
|
||||||
|
@Value("${aiimage.user-secret.usage-retention-batch-size:2000}")
|
||||||
|
private int retentionBatchSize = 2000;
|
||||||
|
|
||||||
|
@Scheduled(cron = "${aiimage.user-secret.usage-retention-cron:0 10 4 * * *}")
|
||||||
|
public void purgeExpiredUsage() {
|
||||||
|
int days = Math.max(1, retentionDays);
|
||||||
|
int batchSize = Math.max(100, retentionBatchSize);
|
||||||
|
LocalDate cutoff = LocalDate.now().minusDays(days);
|
||||||
|
|
||||||
|
DistributedJobLockService.LockHandle lockHandle =
|
||||||
|
distributedJobLockService.tryLock("user-secret:usage-retention", Duration.ofMinutes(15));
|
||||||
|
if (lockHandle == null) {
|
||||||
|
log.info("[user-secret] 用量统计保留清理跳过:另一实例持锁");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try (lockHandle) {
|
||||||
|
int totalDeleted = 0;
|
||||||
|
int batches = 0;
|
||||||
|
while (batches < MAX_BATCHES_PER_RUN) {
|
||||||
|
int deleted = usageMapper.deleteBefore(cutoff, batchSize);
|
||||||
|
batches++;
|
||||||
|
totalDeleted += deleted;
|
||||||
|
if (deleted < batchSize) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.info("[user-secret] 用量统计保留清理完成 cutoff={} retentionDays={} deleted={} batches={}",
|
||||||
|
cutoff, days, totalDeleted, batches);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[user-secret] 用量统计保留清理失败 cutoff={} msg={}", cutoff, ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+62
@@ -0,0 +1,62 @@
|
|||||||
|
package com.nanri.aiimage.modules.ziniao.memory.service;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.scheduling.annotation.Scheduled;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 紫鸟记忆存储(biz_ziniao_memory_store)的过期行清理。
|
||||||
|
*
|
||||||
|
* <p>{@link ZiniaoMemoryStoreService#deleteExpired} 早就写好了,但**全库没有任何调用方**——
|
||||||
|
* 属于典型的"实现了却没接线":过期行只有读取命中时才会被顺手删掉一行,
|
||||||
|
* 店铺下线、改名后遗留的 key 行永远不会有人再读到,于是永久残留。
|
||||||
|
*
|
||||||
|
* <p>这里把它挂上定时任务。单独成类而不是直接给 store 加 {@code @Scheduled}:
|
||||||
|
* store 是被广泛注入的存储组件,不该为了清理任务再依赖分布式锁。
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
public class ZiniaoMemoryExpiredCleanupService {
|
||||||
|
|
||||||
|
/** 单轮最多删除的批次数,剩余留给下一轮。 */
|
||||||
|
static final int MAX_BATCHES_PER_RUN = 20;
|
||||||
|
|
||||||
|
private final ZiniaoMemoryStoreService memoryStoreService;
|
||||||
|
private final DistributedJobLockService distributedJobLockService;
|
||||||
|
|
||||||
|
@Value("${aiimage.ziniao.memory-expired-cleanup-batch-size:500}")
|
||||||
|
private int batchSize = 500;
|
||||||
|
|
||||||
|
@Scheduled(cron = "${aiimage.ziniao.memory-expired-cleanup-cron:0 50 4 * * *}")
|
||||||
|
public void purgeExpired() {
|
||||||
|
int size = Math.max(50, batchSize);
|
||||||
|
|
||||||
|
DistributedJobLockService.LockHandle lockHandle =
|
||||||
|
distributedJobLockService.tryLock("ziniao:memory-expired-cleanup", Duration.ofMinutes(15));
|
||||||
|
if (lockHandle == null) {
|
||||||
|
log.info("[ziniao-memory] 过期行清理跳过:另一实例持锁");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try (lockHandle) {
|
||||||
|
int totalDeleted = 0;
|
||||||
|
int batches = 0;
|
||||||
|
while (batches < MAX_BATCHES_PER_RUN) {
|
||||||
|
int deleted = memoryStoreService.deleteExpired(size);
|
||||||
|
batches++;
|
||||||
|
totalDeleted += deleted;
|
||||||
|
if (deleted < size) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.info("[ziniao-memory] 过期行清理完成 deleted={} batches={}", totalDeleted, batches);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[ziniao-memory] 过期行清理失败 msg={}", ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -213,6 +213,31 @@ aiimage:
|
|||||||
cron: ${AIIMAGE_MODULE_CLEANUP_CRON:0 0 0 * * *}
|
cron: ${AIIMAGE_MODULE_CLEANUP_CRON:0 0 0 * * *}
|
||||||
retention-days: ${AIIMAGE_MODULE_CLEANUP_RETENTION_DAYS:7}
|
retention-days: ${AIIMAGE_MODULE_CLEANUP_RETENTION_DAYS:7}
|
||||||
module-types: ${AIIMAGE_MODULE_CLEANUP_MODULE_TYPES:DEDUPE,SPLIT,CONVERT,DELETE_BRAND,PRODUCT_RISK_RESOLVE,PRICE_TRACK,SHOP_MATCH,PATROL_DELETE,QUERY_ASIN,WITHDRAW,APPEARANCE_PATENT,SIMILAR_ASIN,COLLECT_DATA}
|
module-types: ${AIIMAGE_MODULE_CLEANUP_MODULE_TYPES:DEDUPE,SPLIT,CONVERT,DELETE_BRAND,PRODUCT_RISK_RESOLVE,PRICE_TRACK,SHOP_MATCH,PATROL_DELETE,QUERY_ASIN,WITHDRAW,APPEARANCE_PATENT,SIMILAR_ASIN,COLLECT_DATA}
|
||||||
|
# 以下为各业务表的保留期清理(2026-09-16 统一补齐):这些表此前只增不删,
|
||||||
|
# 或只有"读的时候顺手删一行"这类碰运气的清理。全部按天分批删除、带分布式锁单实例执行。
|
||||||
|
device-log:
|
||||||
|
retention-batch-size: ${AIIMAGE_DEVICE_LOG_RETENTION_BATCH_SIZE:2000}
|
||||||
|
retention-cron: ${AIIMAGE_DEVICE_LOG_RETENTION_CRON:0 40 4 * * *}
|
||||||
|
# 上架/品牌检测的任务历史:此前完全没有清理,保留期给足业务余量(默认 90 天)。
|
||||||
|
# 每批只取 50 个任务——上架单任务可达数万行,批量太大会让单次删除事务过长。
|
||||||
|
publish:
|
||||||
|
task-retention-days: ${AIIMAGE_PUBLISH_TASK_RETENTION_DAYS:90}
|
||||||
|
task-retention-batch-size: ${AIIMAGE_PUBLISH_TASK_RETENTION_BATCH_SIZE:50}
|
||||||
|
task-retention-cron: ${AIIMAGE_PUBLISH_TASK_RETENTION_CRON:0 30 4 * * *}
|
||||||
|
brand:
|
||||||
|
task-retention-days: ${AIIMAGE_BRAND_TASK_RETENTION_DAYS:90}
|
||||||
|
task-retention-batch-size: ${AIIMAGE_BRAND_TASK_RETENTION_BATCH_SIZE:50}
|
||||||
|
task-retention-cron: ${AIIMAGE_BRAND_TASK_RETENTION_CRON:0 45 4 * * *}
|
||||||
|
price-track:
|
||||||
|
loop-run-retention-days: ${AIIMAGE_PRICE_TRACK_LOOP_RUN_RETENTION_DAYS:30}
|
||||||
|
loop-run-retention-batch-size: ${AIIMAGE_PRICE_TRACK_LOOP_RUN_RETENTION_BATCH_SIZE:500}
|
||||||
|
loop-run-retention-cron: ${AIIMAGE_PRICE_TRACK_LOOP_RUN_RETENTION_CRON:0 20 4 * * *}
|
||||||
|
shop-duplicate-check:
|
||||||
|
# 每行含整份聚合 payload(实测约 2.3MB/行),而读取侧只认最新一行,
|
||||||
|
# 保留期给 7 天足够排查;行数下限由服务里的 PROTECTED_ROWS 兜底
|
||||||
|
scan-retention-days: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_DAYS:7}
|
||||||
|
scan-retention-batch-size: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_BATCH_SIZE:200}
|
||||||
|
scan-retention-cron: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_CRON:0 50 3 * * *}
|
||||||
permission-schema-init:
|
permission-schema-init:
|
||||||
enabled: ${AIIMAGE_PERMISSION_SCHEMA_INIT_ENABLED:false}
|
enabled: ${AIIMAGE_PERMISSION_SCHEMA_INIT_ENABLED:false}
|
||||||
task-pressure:
|
task-pressure:
|
||||||
@@ -254,6 +279,17 @@ aiimage:
|
|||||||
module-types: ${AIIMAGE_CLIENT_TASK_PULL_MODULE_TYPES:SIMILAR_ASIN,COLLECT_DATA,APPEARANCE_PATENT}
|
module-types: ${AIIMAGE_CLIENT_TASK_PULL_MODULE_TYPES:SIMILAR_ASIN,COLLECT_DATA,APPEARANCE_PATENT}
|
||||||
min-pending-minutes: ${AIIMAGE_CLIENT_TASK_PULL_MIN_PENDING_MINUTES:5}
|
min-pending-minutes: ${AIIMAGE_CLIENT_TASK_PULL_MIN_PENDING_MINUTES:5}
|
||||||
limit: ${AIIMAGE_CLIENT_TASK_PULL_LIMIT:5}
|
limit: ${AIIMAGE_CLIENT_TASK_PULL_LIMIT:5}
|
||||||
|
# 客户端中断任务的自动续跑(V129):客户端被更新脚本/任务管理器杀掉时,在跑的任务会被
|
||||||
|
# 客户端启动时上报中断并标 FAILED(用户能看到真实原因),这里把这些任务重新排队成 PENDING,
|
||||||
|
# 由上面的兜底拉取通道交给在线客户端继续跑——长任务不再因为一次客户端更新就整个白跑。
|
||||||
|
# 只有注册了 ClientTaskPullSpi 的模块会被续跑(相似ASIN/采集/外观专利),
|
||||||
|
# 上架/改价等写操作模块刻意不在内,避免盲目重跑。
|
||||||
|
# enabled 默认跟随兜底拉取开关:拉了没人领的话,续跑任务只会积压并被 stale 判死。
|
||||||
|
task-resume:
|
||||||
|
enabled: ${AIIMAGE_TASK_RESUME_ENABLED:${AIIMAGE_CLIENT_TASK_PULL_ENABLED:false}}
|
||||||
|
max-attempt: ${AIIMAGE_TASK_RESUME_MAX_ATTEMPT:3}
|
||||||
|
window-minutes: ${AIIMAGE_TASK_RESUME_WINDOW_MINUTES:30}
|
||||||
|
limit: ${AIIMAGE_TASK_RESUME_LIMIT:20}
|
||||||
coze-task:
|
coze-task:
|
||||||
max-concurrent: ${AIIMAGE_COZE_TASK_MAX_CONCURRENT:12}
|
max-concurrent: ${AIIMAGE_COZE_TASK_MAX_CONCURRENT:12}
|
||||||
brand-check:
|
brand-check:
|
||||||
@@ -264,6 +300,7 @@ aiimage:
|
|||||||
retry-times: ${AIIMAGE_BRAND_CHECK_RETRY_TIMES:10}
|
retry-times: ${AIIMAGE_BRAND_CHECK_RETRY_TIMES:10}
|
||||||
retry-interval-millis: ${AIIMAGE_BRAND_CHECK_RETRY_INTERVAL_MILLIS:1000}
|
retry-interval-millis: ${AIIMAGE_BRAND_CHECK_RETRY_INTERVAL_MILLIS:1000}
|
||||||
retry-max-interval-millis: ${AIIMAGE_BRAND_CHECK_RETRY_MAX_INTERVAL_MILLIS:10000}
|
retry-max-interval-millis: ${AIIMAGE_BRAND_CHECK_RETRY_MAX_INTERVAL_MILLIS:10000}
|
||||||
|
total-timeout-millis: ${AIIMAGE_BRAND_CHECK_TOTAL_TIMEOUT_MILLIS:90000}
|
||||||
connect-timeout-millis: ${AIIMAGE_BRAND_CHECK_CONNECT_TIMEOUT_MILLIS:10000}
|
connect-timeout-millis: ${AIIMAGE_BRAND_CHECK_CONNECT_TIMEOUT_MILLIS:10000}
|
||||||
read-timeout-millis: ${AIIMAGE_BRAND_CHECK_READ_TIMEOUT_MILLIS:60000}
|
read-timeout-millis: ${AIIMAGE_BRAND_CHECK_READ_TIMEOUT_MILLIS:60000}
|
||||||
appearance-patent:
|
appearance-patent:
|
||||||
@@ -352,6 +389,10 @@ aiimage:
|
|||||||
jikip-user-id: ${AIIMAGE_USER_SECRET_JIKIP_USER_ID:}
|
jikip-user-id: ${AIIMAGE_USER_SECRET_JIKIP_USER_ID:}
|
||||||
# 巡检发现欠费/密钥失效时是否推送站内通知(铃铛)
|
# 巡检发现欠费/密钥失效时是否推送站内通知(铃铛)
|
||||||
notify-enabled: ${AIIMAGE_USER_SECRET_NOTIFY_ENABLED:true}
|
notify-enabled: ${AIIMAGE_USER_SECRET_NOTIFY_ENABLED:true}
|
||||||
|
# 用量日统计保留期:取 400 天而非整年,避免跨年时把刚过期的上年数据删掉(含同比对比场景)
|
||||||
|
usage-retention-days: ${AIIMAGE_USER_SECRET_USAGE_RETENTION_DAYS:400}
|
||||||
|
usage-retention-batch-size: ${AIIMAGE_USER_SECRET_USAGE_RETENTION_BATCH_SIZE:2000}
|
||||||
|
usage-retention-cron: ${AIIMAGE_USER_SECRET_USAGE_RETENTION_CRON:0 10 4 * * *}
|
||||||
# 站内通知(铃铛):任务失败扫描 + 下游服务健康探测
|
# 站内通知(铃铛):任务失败扫描 + 下游服务健康探测
|
||||||
notification:
|
notification:
|
||||||
scan-enabled: ${AIIMAGE_NOTIFICATION_SCAN_ENABLED:true}
|
scan-enabled: ${AIIMAGE_NOTIFICATION_SCAN_ENABLED:true}
|
||||||
@@ -375,6 +416,8 @@ aiimage:
|
|||||||
maixiang-queue-pending-threshold: ${AIIMAGE_NOTIFICATION_MAIXIANG_QUEUE_PENDING_THRESHOLD:300}
|
maixiang-queue-pending-threshold: ${AIIMAGE_NOTIFICATION_MAIXIANG_QUEUE_PENDING_THRESHOLD:300}
|
||||||
maixiang-queue-processing-threshold: ${AIIMAGE_NOTIFICATION_MAIXIANG_QUEUE_PROCESSING_THRESHOLD:100}
|
maixiang-queue-processing-threshold: ${AIIMAGE_NOTIFICATION_MAIXIANG_QUEUE_PROCESSING_THRESHOLD:100}
|
||||||
read-retention-days: ${AIIMAGE_NOTIFICATION_READ_RETENTION_DAYS:90}
|
read-retention-days: ${AIIMAGE_NOTIFICATION_READ_RETENTION_DAYS:90}
|
||||||
|
# 未读通知也设保留期(此前永不清理,不看铃铛的用户会无限累积);给得比已读宽一倍
|
||||||
|
unread-retention-days: ${AIIMAGE_NOTIFICATION_UNREAD_RETENTION_DAYS:180}
|
||||||
security:
|
security:
|
||||||
shop-credential-key: ${AIIMAGE_SHOP_CREDENTIAL_KEY:change-me-shop-credential-key}
|
shop-credential-key: ${AIIMAGE_SHOP_CREDENTIAL_KEY:change-me-shop-credential-key}
|
||||||
internal-token: ${AIIMAGE_INTERNAL_TOKEN:}
|
internal-token: ${AIIMAGE_INTERNAL_TOKEN:}
|
||||||
@@ -406,6 +449,9 @@ aiimage:
|
|||||||
open-store-force-download-path: ${AIIMAGE_ZINIAO_OPEN_STORE_FORCE_DOWNLOAD_PATH:}
|
open-store-force-download-path: ${AIIMAGE_ZINIAO_OPEN_STORE_FORCE_DOWNLOAD_PATH:}
|
||||||
open-store-extra-args: ${AIIMAGE_ZINIAO_OPEN_STORE_EXTRA_ARGS:--disable-gpu start-maximized}
|
open-store-extra-args: ${AIIMAGE_ZINIAO_OPEN_STORE_EXTRA_ARGS:--disable-gpu start-maximized}
|
||||||
session-ttl-hours: ${AIIMAGE_ZINIAO_SESSION_TTL_HOURS:2}
|
session-ttl-hours: ${AIIMAGE_ZINIAO_SESSION_TTL_HOURS:2}
|
||||||
|
# 记忆存储过期行清理:deleteExpired 早已实现但一直无调用方,过期行只有被读到才顺手删一行
|
||||||
|
memory-expired-cleanup-batch-size: ${AIIMAGE_ZINIAO_MEMORY_EXPIRED_CLEANUP_BATCH_SIZE:500}
|
||||||
|
memory-expired-cleanup-cron: ${AIIMAGE_ZINIAO_MEMORY_EXPIRED_CLEANUP_CRON:0 50 4 * * *}
|
||||||
shops-cache-minutes: ${AIIMAGE_ZINIAO_SHOPS_CACHE_MINUTES:30}
|
shops-cache-minutes: ${AIIMAGE_ZINIAO_SHOPS_CACHE_MINUTES:30}
|
||||||
connect-timeout-seconds: ${AIIMAGE_ZINIAO_CONNECT_TIMEOUT_SECONDS:5}
|
connect-timeout-seconds: ${AIIMAGE_ZINIAO_CONNECT_TIMEOUT_SECONDS:5}
|
||||||
read-timeout-seconds: ${AIIMAGE_ZINIAO_READ_TIMEOUT_SECONDS:15}
|
read-timeout-seconds: ${AIIMAGE_ZINIAO_READ_TIMEOUT_SECONDS:15}
|
||||||
|
|||||||
@@ -0,0 +1,55 @@
|
|||||||
|
-- V129: 客户端中断后的自动续跑(保留失败记录 + 服务端重新排队)
|
||||||
|
--
|
||||||
|
-- 背景:2026-09-18 任务 28587(跟价,uid 977 店铺「张美莺」)在客户端 09:51/09:57 被重启后中断,
|
||||||
|
-- 旧行为是任务一直挂 RUNNING 到 stale 兜底判死(最长 2 小时),长任务一旦撞上客户端更新就白跑。
|
||||||
|
-- 新契约:中断任务**保留失败记录**(用户能看到「因客户端重启中断」),同时由服务端自动重新排队续跑。
|
||||||
|
--
|
||||||
|
-- 两条续跑路径各需要一个计数:
|
||||||
|
-- 1) biz_file_task.resume_of_task_id + resume_attempt —— 通用模块(相似ASIN/采集/外观专利/跟价单次任务)
|
||||||
|
-- 由 TaskResumeService 复制出一条 PENDING 续跑任务,交给客户端兜底拉取;代数用于封顶。
|
||||||
|
-- 2) biz_price_track_loop_run.resume_attempt —— 跟价循环由 loop_run 驱动,中断时不让循环终止,
|
||||||
|
-- 而是清掉 active_task_id 重新派发当前轮(客户端下次 dispatch 即拿到同一店铺/轮次),同样封顶。
|
||||||
|
--
|
||||||
|
-- 封顶的意义:会话持续不可用(如账号被风控)时,不封顶会无限重排队、无限重开浏览器。
|
||||||
|
--
|
||||||
|
-- 风险:ADD COLUMN 走 INSTANT,两张表均为小表,秒级完成;建议低峰执行。
|
||||||
|
-- 回滚:ALTER TABLE biz_file_task DROP COLUMN resume_of_task_id, DROP COLUMN resume_attempt;
|
||||||
|
-- ALTER TABLE biz_price_track_loop_run DROP COLUMN resume_attempt;
|
||||||
|
|
||||||
|
SET @db_name = DATABASE();
|
||||||
|
|
||||||
|
SET @col_exists := (
|
||||||
|
SELECT COUNT(*) FROM information_schema.COLUMNS
|
||||||
|
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_file_task' AND COLUMN_NAME = 'resume_of_task_id'
|
||||||
|
);
|
||||||
|
SET @sql := IF(@col_exists = 0,
|
||||||
|
'ALTER TABLE biz_file_task ADD COLUMN resume_of_task_id BIGINT NULL COMMENT ''续跑来源任务ID(客户端中断后自动重排队)'' AFTER owner_instance_id',
|
||||||
|
'SELECT 1'
|
||||||
|
);
|
||||||
|
PREPARE stmt FROM @sql;
|
||||||
|
EXECUTE stmt;
|
||||||
|
DEALLOCATE PREPARE stmt;
|
||||||
|
|
||||||
|
SET @col_exists := (
|
||||||
|
SELECT COUNT(*) FROM information_schema.COLUMNS
|
||||||
|
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_file_task' AND COLUMN_NAME = 'resume_attempt'
|
||||||
|
);
|
||||||
|
SET @sql := IF(@col_exists = 0,
|
||||||
|
'ALTER TABLE biz_file_task ADD COLUMN resume_attempt INT NOT NULL DEFAULT 0 COMMENT ''续跑代数:0=原始任务,N=第N次自动续跑'' AFTER resume_of_task_id',
|
||||||
|
'SELECT 1'
|
||||||
|
);
|
||||||
|
PREPARE stmt FROM @sql;
|
||||||
|
EXECUTE stmt;
|
||||||
|
DEALLOCATE PREPARE stmt;
|
||||||
|
|
||||||
|
SET @col_exists := (
|
||||||
|
SELECT COUNT(*) FROM information_schema.COLUMNS
|
||||||
|
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_price_track_loop_run' AND COLUMN_NAME = 'resume_attempt'
|
||||||
|
);
|
||||||
|
SET @sql := IF(@col_exists = 0,
|
||||||
|
'ALTER TABLE biz_price_track_loop_run ADD COLUMN resume_attempt INT NOT NULL DEFAULT 0 COMMENT ''中断后自动重派当前轮的次数(用于封顶)'' AFTER stop_requested',
|
||||||
|
'SELECT 1'
|
||||||
|
);
|
||||||
|
PREPARE stmt FROM @sql;
|
||||||
|
EXECUTE stmt;
|
||||||
|
DEALLOCATE PREPARE stmt;
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
-- V130: biz_publish_file 增加 device_id 列(激活该文件时客户端所在设备)
|
||||||
|
--
|
||||||
|
-- 背景:同店铺互斥原本只按 shop_name 全局判定(PublishTaskService.activateFile)。
|
||||||
|
-- 但紫鸟浏览器会话是**每台机器一份**:不同客户端各自持有独立的店铺会话,同一家店
|
||||||
|
-- 完全可以在两台机器上并行跑不同国家。原来的全局判定把这种合法的跨机器并行也挡了
|
||||||
|
-- (2026-09-18 任务 28624 被 28616 误挡:两条在不同机器上)。
|
||||||
|
-- 真正必须串行的是「同一台机器上的同一家店」——那里只有一个浏览器会话,两个任务会
|
||||||
|
-- 互相切换国家(add_product.py 的 SwitchingCountries 是会话级状态,任务中途断线重连
|
||||||
|
-- 还会再切一次),轻则失败,重则把 A 国家的商品提交进 B 国家的店铺。
|
||||||
|
--
|
||||||
|
-- 因此互斥键由 shop_name 改为 (device_id, shop_name)。device 取自 JWT 里**签名的**
|
||||||
|
-- deviceId claim(绝不使用客户端可控的 X-Device-Id 请求头,见 DeviceSessionPolicy 注释)。
|
||||||
|
--
|
||||||
|
-- 空值语义:NULL/空串表示"来源不明"——旧客户端(未升级、token 无 claim)或内部令牌调用。
|
||||||
|
-- 该情形退回改动前的全局店铺互斥,保持保守,不因迁移把风险放开。存量 RUNNING 行均为 NULL,
|
||||||
|
-- 因此会继续全挡直到跑完,随后新激活的行都带设备号,跨机器并行自然生效。
|
||||||
|
--
|
||||||
|
-- 风险:ADD COLUMN 走 INSTANT/INPLACE,生产该表仅数百行,秒级完成;建议低峰执行。
|
||||||
|
-- 回滚:ALTER TABLE biz_publish_file DROP COLUMN device_id;
|
||||||
|
|
||||||
|
SET @db_name = DATABASE();
|
||||||
|
|
||||||
|
SET @col_exists := (
|
||||||
|
SELECT COUNT(*) FROM information_schema.COLUMNS
|
||||||
|
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_publish_file' AND COLUMN_NAME = 'device_id'
|
||||||
|
);
|
||||||
|
SET @sql := IF(@col_exists = 0,
|
||||||
|
'ALTER TABLE biz_publish_file ADD COLUMN device_id VARCHAR(128) NULL COMMENT ''device that activated this file, from signed JWT deviceId claim'' AFTER matched_user_id',
|
||||||
|
'SELECT 1'
|
||||||
|
);
|
||||||
|
PREPARE stmt FROM @sql;
|
||||||
|
EXECUTE stmt;
|
||||||
|
DEALLOCATE PREPARE stmt;
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
-- V131: users(用户)、columns(后台菜单) 两表增加 updated_at 列
|
||||||
|
--
|
||||||
|
-- 背景:后台「用户管理」「菜单管理」列表要展示「创建时间 + 更新时间」,但这两张历史表
|
||||||
|
-- 建表时只落了一个 created_at,更新时间无从取值。补一列由数据库维护的 updated_at
|
||||||
|
-- (DEFAULT CURRENT_TIMESTAMP + ON UPDATE CURRENT_TIMESTAMP),应用侧不写它。
|
||||||
|
--
|
||||||
|
-- 与 V125 同样的取舍:列交给数据库维护,Java 实体上标注
|
||||||
|
-- insertStrategy=NEVER / updateStrategy=NEVER,保证 MyBatis-Plus 永不显式写。
|
||||||
|
-- 这一步是必要的——本项目的写回模式多为 selectById → 改字段 → updateById(实体带着旧值),
|
||||||
|
-- 而 MySQL 的规则是「UPDATE 语句显式给某列赋值时不触发该列的自动更新」,
|
||||||
|
-- 不禁写就会把读出来的旧值写回去,更新时间会一直停在第一次写入的值。
|
||||||
|
--
|
||||||
|
-- 存量行:ADD COLUMN 的 DEFAULT CURRENT_TIMESTAMP 会把已有行填成迁移执行时刻,
|
||||||
|
-- 不是真实的历史变更时间(历史上也没有记录,无法还原),属已知取舍。
|
||||||
|
--
|
||||||
|
-- 风险:两表均小(users 百余行、columns 数十行),ADD COLUMN 秒级完成。
|
||||||
|
-- 回滚:ALTER TABLE `users` DROP COLUMN updated_at; ALTER TABLE `columns` DROP COLUMN updated_at;
|
||||||
|
|
||||||
|
SET @db_name = DATABASE();
|
||||||
|
|
||||||
|
-- users(用户表)
|
||||||
|
SET @col_exists := (
|
||||||
|
SELECT COUNT(*) FROM information_schema.COLUMNS
|
||||||
|
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'users' AND COLUMN_NAME = 'updated_at'
|
||||||
|
);
|
||||||
|
SET @sql := IF(@col_exists = 0,
|
||||||
|
'ALTER TABLE `users` ADD COLUMN `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT ''更新时间'' AFTER `created_at`',
|
||||||
|
'SELECT 1'
|
||||||
|
);
|
||||||
|
PREPARE stmt FROM @sql;
|
||||||
|
EXECUTE stmt;
|
||||||
|
DEALLOCATE PREPARE stmt;
|
||||||
|
|
||||||
|
-- columns(后台菜单表)
|
||||||
|
SET @col_exists := (
|
||||||
|
SELECT COUNT(*) FROM information_schema.COLUMNS
|
||||||
|
WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'columns' AND COLUMN_NAME = 'updated_at'
|
||||||
|
);
|
||||||
|
SET @sql := IF(@col_exists = 0,
|
||||||
|
'ALTER TABLE `columns` ADD COLUMN `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT ''更新时间'' AFTER `created_at`',
|
||||||
|
'SELECT 1'
|
||||||
|
);
|
||||||
|
PREPARE stmt FROM @sql;
|
||||||
|
EXECUTE stmt;
|
||||||
|
DEALLOCATE PREPARE stmt;
|
||||||
+85
@@ -0,0 +1,85 @@
|
|||||||
|
package com.nanri.aiimage.common.exception;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.api.ApiResponse;
|
||||||
|
import com.nanri.aiimage.common.service.TaskOwnerForwardService;
|
||||||
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.mockito.InjectMocks;
|
||||||
|
import org.mockito.Mock;
|
||||||
|
import org.mockito.junit.jupiter.MockitoExtension;
|
||||||
|
import org.springframework.http.HttpStatus;
|
||||||
|
import org.springframework.http.ResponseEntity;
|
||||||
|
import org.springframework.web.client.ResourceAccessException;
|
||||||
|
|
||||||
|
import java.net.ConnectException;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 跨实例转发的失败语义。
|
||||||
|
*
|
||||||
|
* <p>2026-09-18 任务 28616:归属节点 server-110 滚动重启期间,客户端心跳经 nginx 落到
|
||||||
|
* server-121,转发 3 次 Connection refused。当时这里返回 {@code ApiResponse.fail(40903)}
|
||||||
|
* ——HTTP 200 + {@code data:null},而客户端用
|
||||||
|
* {@code bool((resp.json().get("data") or {}).get("alive"))} 解析,把「拿不到数据」折叠成
|
||||||
|
* {@code alive=false},于是客户端把一个跑到 66/253 的健康上架任务主动停掉、关闭店铺。
|
||||||
|
* 修复后转发失败返回 503 + 空 body:新客户端按状态码判为「未知」,老客户端因 body 不是
|
||||||
|
* JSON、解析抛异常同样落到「未知」,两边都不会再自杀。</p>
|
||||||
|
*/
|
||||||
|
@ExtendWith(MockitoExtension.class)
|
||||||
|
class GlobalExceptionHandlerTest {
|
||||||
|
|
||||||
|
@Mock private TaskOwnerForwardService taskOwnerForwardService;
|
||||||
|
|
||||||
|
@InjectMocks private GlobalExceptionHandler handler;
|
||||||
|
|
||||||
|
private static TaskOwnerMismatchException ownerMismatch() {
|
||||||
|
return new TaskOwnerMismatchException(28616L, "PUBLISH task heartbeat", "server-110", "server-121");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void forwardConnectFailureReturns503WithEmptyBody() {
|
||||||
|
when(taskOwnerForwardService.forwardCurrentRequest(any(), any()))
|
||||||
|
.thenThrow(new ResourceAccessException("Connection refused",
|
||||||
|
new ConnectException("Connection refused")));
|
||||||
|
|
||||||
|
Object result = handler.handleTaskOwnerMismatchException(ownerMismatch(), mock(HttpServletRequest.class));
|
||||||
|
|
||||||
|
ResponseEntity<?> response = assertInstanceOf(ResponseEntity.class, result);
|
||||||
|
assertEquals(HttpStatus.SERVICE_UNAVAILABLE, response.getStatusCode());
|
||||||
|
// 空 body 是关键:一旦带上 JSON,老客户端的 bool((data or {}).get("alive")) 又会判成「死」
|
||||||
|
assertNull(response.getBody(), "转发失败必须无响应体,否则老客户端会把未知当成任务已死");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void successfulForwardPassesThroughUpstreamStatusAndBody() {
|
||||||
|
byte[] upstream = "{\"success\":true,\"data\":{\"alive\":true}}".getBytes(java.nio.charset.StandardCharsets.UTF_8);
|
||||||
|
when(taskOwnerForwardService.forwardCurrentRequest(any(), any()))
|
||||||
|
.thenReturn(ResponseEntity.ok(upstream));
|
||||||
|
|
||||||
|
Object result = handler.handleTaskOwnerMismatchException(ownerMismatch(), mock(HttpServletRequest.class));
|
||||||
|
|
||||||
|
ResponseEntity<?> response = assertInstanceOf(ResponseEntity.class, result);
|
||||||
|
assertEquals(HttpStatus.OK, response.getStatusCode());
|
||||||
|
assertEquals(upstream, response.getBody(), "转发成功时上游响应体必须原样透传");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void configErrorKeepsBusinessEnvelopeInsteadOfServiceUnavailable() {
|
||||||
|
// 路由未配置 / 检测到转发循环属于配置错误,不是瞬时故障:保留业务信封,不回 503
|
||||||
|
when(taskOwnerForwardService.forwardCurrentRequest(any(), any()))
|
||||||
|
.thenThrow(new BusinessException(40903, "任务归属实例未配置服务路由:server-110"));
|
||||||
|
|
||||||
|
Object result = handler.handleTaskOwnerMismatchException(ownerMismatch(), mock(HttpServletRequest.class));
|
||||||
|
|
||||||
|
ApiResponse<?> response = assertInstanceOf(ApiResponse.class, result);
|
||||||
|
assertFalse(response.isSuccess(), "配置错误仍按业务失败返回");
|
||||||
|
}
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user