feat(保留期清理): 补齐 7 处只增不删的数据;修结果对象孤儿
审核发现一批表/元数据只增不删,且对象存储存在活跃泄漏: - module-cleanup 漏收 task_result_item / task_result_payload 的 payload 指针, 每晚删完行就在 json-server 桶留下孤儿对象;指针收集触顶从"截断后照样删整组" 改为任务组二分拆分,拆到单任务仍触顶则整组不删并记 error(截断会让指针随行消失) - 结果文件对象此前从不回收(只删 DB 行、对象留在桶里),现改为事务提交后 按 file_result / task_file_job 的 result_file_url 逐个回收 - 新增 7 个保留期清理:device_log_file 元数据、price_track_loop_run、撞款扫描行、 密钥用量日统计、紫鸟记忆过期行、PUBLISH 任务、BRAND 任务。后两者刻意不复用 module-cleanup(BRAND 不写 biz_file_task;PUBLISH 单任务可达数万行), 改为调各自既有的业务删除入口,保证结果对象与子表一起回收 - 未读通知补保留期(180 天,已读仍 90 天);ZiniaoMemoryStoreService.deleteExpired 此前全库无调用方(实现了没接线),现已挂定时任务 全部沿用既有范式:@Scheduled + 分布式锁单实例 + 分批 + 单轮批次数上限 + 失败只记日志。
This commit is contained in:
@@ -213,6 +213,29 @@ aiimage:
|
||||
cron: ${AIIMAGE_MODULE_CLEANUP_CRON:0 0 0 * * *}
|
||||
retention-days: ${AIIMAGE_MODULE_CLEANUP_RETENTION_DAYS:7}
|
||||
module-types: ${AIIMAGE_MODULE_CLEANUP_MODULE_TYPES:DEDUPE,SPLIT,CONVERT,DELETE_BRAND,PRODUCT_RISK_RESOLVE,PRICE_TRACK,SHOP_MATCH,PATROL_DELETE,QUERY_ASIN,WITHDRAW,APPEARANCE_PATENT,SIMILAR_ASIN,COLLECT_DATA}
|
||||
# 以下为各业务表的保留期清理(2026-09-16 统一补齐):这些表此前只增不删,
|
||||
# 或只有"读的时候顺手删一行"这类碰运气的清理。全部按天分批删除、带分布式锁单实例执行。
|
||||
device-log:
|
||||
retention-batch-size: ${AIIMAGE_DEVICE_LOG_RETENTION_BATCH_SIZE:2000}
|
||||
retention-cron: ${AIIMAGE_DEVICE_LOG_RETENTION_CRON:0 40 4 * * *}
|
||||
# 上架/品牌检测的任务历史:此前完全没有清理,保留期给足业务余量(默认 90 天)。
|
||||
# 每批只取 50 个任务——上架单任务可达数万行,批量太大会让单次删除事务过长。
|
||||
publish:
|
||||
task-retention-days: ${AIIMAGE_PUBLISH_TASK_RETENTION_DAYS:90}
|
||||
task-retention-batch-size: ${AIIMAGE_PUBLISH_TASK_RETENTION_BATCH_SIZE:50}
|
||||
task-retention-cron: ${AIIMAGE_PUBLISH_TASK_RETENTION_CRON:0 30 4 * * *}
|
||||
brand:
|
||||
task-retention-days: ${AIIMAGE_BRAND_TASK_RETENTION_DAYS:90}
|
||||
task-retention-batch-size: ${AIIMAGE_BRAND_TASK_RETENTION_BATCH_SIZE:50}
|
||||
task-retention-cron: ${AIIMAGE_BRAND_TASK_RETENTION_CRON:0 45 4 * * *}
|
||||
price-track:
|
||||
loop-run-retention-days: ${AIIMAGE_PRICE_TRACK_LOOP_RUN_RETENTION_DAYS:30}
|
||||
loop-run-retention-batch-size: ${AIIMAGE_PRICE_TRACK_LOOP_RUN_RETENTION_BATCH_SIZE:500}
|
||||
loop-run-retention-cron: ${AIIMAGE_PRICE_TRACK_LOOP_RUN_RETENTION_CRON:0 20 4 * * *}
|
||||
shop-duplicate-check:
|
||||
scan-retention-days: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_DAYS:90}
|
||||
scan-retention-batch-size: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_BATCH_SIZE:200}
|
||||
scan-retention-cron: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_CRON:0 50 3 * * *}
|
||||
permission-schema-init:
|
||||
enabled: ${AIIMAGE_PERMISSION_SCHEMA_INIT_ENABLED:false}
|
||||
task-pressure:
|
||||
@@ -352,6 +375,10 @@ aiimage:
|
||||
jikip-user-id: ${AIIMAGE_USER_SECRET_JIKIP_USER_ID:}
|
||||
# 巡检发现欠费/密钥失效时是否推送站内通知(铃铛)
|
||||
notify-enabled: ${AIIMAGE_USER_SECRET_NOTIFY_ENABLED:true}
|
||||
# 用量日统计保留期:取 400 天而非整年,避免跨年时把刚过期的上年数据删掉(含同比对比场景)
|
||||
usage-retention-days: ${AIIMAGE_USER_SECRET_USAGE_RETENTION_DAYS:400}
|
||||
usage-retention-batch-size: ${AIIMAGE_USER_SECRET_USAGE_RETENTION_BATCH_SIZE:2000}
|
||||
usage-retention-cron: ${AIIMAGE_USER_SECRET_USAGE_RETENTION_CRON:0 10 4 * * *}
|
||||
# 站内通知(铃铛):任务失败扫描 + 下游服务健康探测
|
||||
notification:
|
||||
scan-enabled: ${AIIMAGE_NOTIFICATION_SCAN_ENABLED:true}
|
||||
@@ -375,6 +402,8 @@ aiimage:
|
||||
maixiang-queue-pending-threshold: ${AIIMAGE_NOTIFICATION_MAIXIANG_QUEUE_PENDING_THRESHOLD:300}
|
||||
maixiang-queue-processing-threshold: ${AIIMAGE_NOTIFICATION_MAIXIANG_QUEUE_PROCESSING_THRESHOLD:100}
|
||||
read-retention-days: ${AIIMAGE_NOTIFICATION_READ_RETENTION_DAYS:90}
|
||||
# 未读通知也设保留期(此前永不清理,不看铃铛的用户会无限累积);给得比已读宽一倍
|
||||
unread-retention-days: ${AIIMAGE_NOTIFICATION_UNREAD_RETENTION_DAYS:180}
|
||||
security:
|
||||
shop-credential-key: ${AIIMAGE_SHOP_CREDENTIAL_KEY:change-me-shop-credential-key}
|
||||
internal-token: ${AIIMAGE_INTERNAL_TOKEN:}
|
||||
@@ -406,6 +435,9 @@ aiimage:
|
||||
open-store-force-download-path: ${AIIMAGE_ZINIAO_OPEN_STORE_FORCE_DOWNLOAD_PATH:}
|
||||
open-store-extra-args: ${AIIMAGE_ZINIAO_OPEN_STORE_EXTRA_ARGS:--disable-gpu start-maximized}
|
||||
session-ttl-hours: ${AIIMAGE_ZINIAO_SESSION_TTL_HOURS:2}
|
||||
# 记忆存储过期行清理:deleteExpired 早已实现但一直无调用方,过期行只有被读到才顺手删一行
|
||||
memory-expired-cleanup-batch-size: ${AIIMAGE_ZINIAO_MEMORY_EXPIRED_CLEANUP_BATCH_SIZE:500}
|
||||
memory-expired-cleanup-cron: ${AIIMAGE_ZINIAO_MEMORY_EXPIRED_CLEANUP_CRON:0 50 4 * * *}
|
||||
shops-cache-minutes: ${AIIMAGE_ZINIAO_SHOPS_CACHE_MINUTES:30}
|
||||
connect-timeout-seconds: ${AIIMAGE_ZINIAO_CONNECT_TIMEOUT_SECONDS:5}
|
||||
read-timeout-seconds: ${AIIMAGE_ZINIAO_READ_TIMEOUT_SECONDS:15}
|
||||
|
||||
Reference in New Issue
Block a user