super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

task-76: JSON owner 查询迁移到显式列并补充任务/状态复合索引

Browse Source
This commit is contained in:
黄自达
2026-08-30 20:51:07 +08:00
parent e9ba3689ba
commit 04f45bebec
4 changed files with 324 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskFileJobEntity.java
+2
View File
@@ -17,6 +17,8 @@ public class TaskFileJobEntity {
private String moduleType;
private Long resultId;
private String scopeKey;
/** 从 scope_key 提取的显式 owner（task:1:owner:instance-a → instance-a），支持等值索引查询。 */
private String owner;
private String jobType;
private String status;
private Integer retryCount;
backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java
+21 -10
View File
@@ -40,6 +40,7 @@ public class TaskFileJobService {
entity.setModuleType(moduleType);
entity.setResultId(resultId);
entity.setScopeKey(scopeKey);
entity.setOwner(ownerFromScopeKey(scopeKey));
entity.setJobType(JOB_TYPE_ASSEMBLE_RESULT);
entity.setStatus("PENDING");
entity.setRetryCount(0);
@@ -68,6 +69,7 @@ public class TaskFileJobService {
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, existing.getId())
.set(TaskFileJobEntity::getScopeKey, scopeKey)
.set(TaskFileJobEntity::getOwner, ownerFromScopeKey(scopeKey))
.set(TaskFileJobEntity::getStatus, "PENDING")
.set(TaskFileJobEntity::getErrorMessage, null)
.set(TaskFileJobEntity::getUpdatedAt, now)
@@ -103,14 +105,13 @@ public class TaskFileJobService {
if (normalizedOwner.isBlank()) {
return claimRunnableJobs(limit);
}
String ownerMarker = ":owner:" + normalizedOwner;
int safeLimit = Math.max(1, Math.min(limit, 100));
List<TaskFileJobEntity> ownerJobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
.in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED"))
.lt(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
.in(TaskFileJobEntity::getModuleType, List.of("APPEARANCE_PATENT", "SIMILAR_ASIN", "PUBLISH", "SHOP_DATA_CRAWL"))
.like(TaskFileJobEntity::getScopeKey, ownerMarker)
.eq(TaskFileJobEntity::getOwner, normalizedOwner)
.orderByAsc(TaskFileJobEntity::getUpdatedAt)
.last("limit " + safeLimit));
@@ -122,9 +123,7 @@ public class TaskFileJobService {
.and(wrapper -> wrapper
.notIn(TaskFileJobEntity::getModuleType, List.of("APPEARANCE_PATENT", "SIMILAR_ASIN", "PUBLISH", "SHOP_DATA_CRAWL"))
.or()
.isNull(TaskFileJobEntity::getScopeKey)
.or()
.notLike(TaskFileJobEntity::getScopeKey, ":owner:"))
.isNull(TaskFileJobEntity::getOwner))
.orderByAsc(TaskFileJobEntity::getUpdatedAt)
.last("limit " + (safeLimit - ownerJobs.size()));
candidates.addAll(taskFileJobMapper.selectList(genericWrapper));
@@ -157,14 +156,13 @@ public class TaskFileJobService {
if (normalizedOwner.isBlank()) {
return listRunnableJobs(limit);
}
String ownerMarker = ":owner:" + normalizedOwner;
int safeLimit = Math.max(1, Math.min(limit, 100));
List<TaskFileJobEntity> ownerJobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
.in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED"))
.lt(TaskFileJobEntity::getRetryCount, MAX_RETRY_COUNT)
.in(TaskFileJobEntity::getModuleType, List.of("APPEARANCE_PATENT", "SIMILAR_ASIN", "PUBLISH", "SHOP_DATA_CRAWL"))
.like(TaskFileJobEntity::getScopeKey, ownerMarker)
.eq(TaskFileJobEntity::getOwner, normalizedOwner)
.orderByAsc(TaskFileJobEntity::getUpdatedAt)
.last("limit " + safeLimit));
if (ownerJobs.size() >= safeLimit) {
@@ -178,9 +176,7 @@ public class TaskFileJobService {
.and(wrapper -> wrapper
.notIn(TaskFileJobEntity::getModuleType, List.of("APPEARANCE_PATENT", "SIMILAR_ASIN", "PUBLISH", "SHOP_DATA_CRAWL"))
.or()
.isNull(TaskFileJobEntity::getScopeKey)
.or()
.notLike(TaskFileJobEntity::getScopeKey, ":owner:"))
.isNull(TaskFileJobEntity::getOwner))
.orderByAsc(TaskFileJobEntity::getUpdatedAt)
.last("limit " + (safeLimit - jobs.size()));
jobs.addAll(taskFileJobMapper.selectList(genericWrapper));
@@ -586,6 +582,21 @@ public class TaskFileJobService {
.eq(TaskFileJobEntity::getResultId, resultId));
}
private static final String OWNER_MARKER = ":owner:";
/** 从 scope_key（task:1:owner:instance-a）提取 owner；无标记或空白返回 null。 */
static String ownerFromScopeKey(String scopeKey) {
if (scopeKey == null || scopeKey.isBlank()) {
return null;
}
int index = scopeKey.lastIndexOf(OWNER_MARKER);
if (index < 0) {
return null;
}
String owner = scopeKey.substring(index + OWNER_MARKER.length()).trim();
return owner.isBlank() ? null : owner;
}
private TaskFileJobEntity findJob(Long taskId, String moduleType, Long resultId, String jobType) {
if (taskId == null || moduleType == null || moduleType.isBlank() || resultId == null || jobType == null || jobType.isBlank()) {
return null;
backend-java/src/main/resources/db/V96__task_file_job_owner_column_index.sql
+41
View File
@@ -0,0 +1,41 @@
-- Task 76：biz_task_file_job 增加显式 owner 列并补充 (module_type, owner, status, retry_count) 复合索引。
-- owner 从 scope_key 的 ":owner:" 标记提取（task:1:owner:instance-a → instance-a），
-- 存量行回填，owner 等值查询不再依赖 LIKE '%:owner:%' 扫描。
SET @db_name = DATABASE();
SET @col_exists := (
SELECT COUNT(*)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = @db_name
AND TABLE_NAME = 'biz_task_file_job'
AND COLUMN_NAME = 'owner'
);
SET @sql := IF(@col_exists = 0,
'ALTER TABLE biz_task_file_job ADD COLUMN owner VARCHAR(128) NULL COMMENT ''explicit owner extracted from scope_key'' AFTER scope_key',
'SELECT 1'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
UPDATE biz_task_file_job
SET owner = SUBSTRING(scope_key, LOCATE(':owner:', scope_key) + 7)
WHERE owner IS NULL
AND scope_key IS NOT NULL
AND LOCATE(':owner:', scope_key) > 0;
SET @idx_exists := (
SELECT COUNT(*)
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = @db_name
AND TABLE_NAME = 'biz_task_file_job'
AND INDEX_NAME = 'idx_file_job_module_owner_status_retry'
);
SET @sql := IF(@idx_exists = 0,
'ALTER TABLE biz_task_file_job ADD INDEX idx_file_job_module_owner_status_retry (module_type, owner, status, retry_count)',
'SELECT 1'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;