完成后端架构重构等
This commit is contained in:
@@ -0,0 +1,9 @@
|
||||
package com.nanri.aiimage.modules.task.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface TaskChunkMapper extends BaseMapper<TaskChunkEntity> {
|
||||
}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
package com.nanri.aiimage.modules.task.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
|
||||
@Mapper
|
||||
public interface TaskScopeStateMapper extends BaseMapper<TaskScopeStateEntity> {
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
package com.nanri.aiimage.modules.task.model.entity;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Data
|
||||
@TableName("biz_task_chunk")
|
||||
public class TaskChunkEntity {
|
||||
|
||||
@TableId(type = IdType.AUTO)
|
||||
private Long id;
|
||||
private Long taskId;
|
||||
private String moduleType;
|
||||
private String scopeKey;
|
||||
private String scopeHash;
|
||||
private Integer chunkIndex;
|
||||
private Integer chunkTotal;
|
||||
private String payloadJson;
|
||||
private String payloadHash;
|
||||
private LocalDateTime createdAt;
|
||||
private LocalDateTime updatedAt;
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package com.nanri.aiimage.modules.task.model.entity;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Data
|
||||
@TableName("biz_task_scope_state")
|
||||
public class TaskScopeStateEntity {
|
||||
|
||||
@TableId(type = IdType.AUTO)
|
||||
private Long id;
|
||||
private Long taskId;
|
||||
private String moduleType;
|
||||
private String scopeKey;
|
||||
private String scopeHash;
|
||||
private String parsedPayloadJson;
|
||||
private String stateJson;
|
||||
private Integer chunkTotal;
|
||||
private Integer receivedChunkCount;
|
||||
private Integer completed;
|
||||
private LocalDateTime lastChunkAt;
|
||||
private String lastError;
|
||||
private LocalDateTime createdAt;
|
||||
private LocalDateTime updatedAt;
|
||||
}
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.attribute.FileTime;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class TaskScopePayloadBufferMaintenanceService {
|
||||
|
||||
private static final Duration RECOVERY_LOCK_TTL = Duration.ofMinutes(10);
|
||||
private static final Duration CLEANUP_LOCK_TTL = Duration.ofMinutes(10);
|
||||
|
||||
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
|
||||
private final TaskPressureProperties taskPressureProperties;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
public void recoverBufferedPayloadsOnStartup() {
|
||||
DistributedJobLockService.LockHandle lockHandle =
|
||||
distributedJobLockService.tryLock("task-scope-buffer:recover", RECOVERY_LOCK_TTL);
|
||||
if (lockHandle == null) {
|
||||
log.info("[task-scope-buffer] skip startup recovery because another instance holds the lock");
|
||||
return;
|
||||
}
|
||||
try (lockHandle) {
|
||||
Path root = taskScopePayloadStorageService.getBufferedPayloadRoot();
|
||||
if (!Files.isDirectory(root)) {
|
||||
return;
|
||||
}
|
||||
int limit = Math.max(0, taskPressureProperties.getScopePayloadRecoveryMaxFiles());
|
||||
if (limit == 0) {
|
||||
return;
|
||||
}
|
||||
int recovered = 0;
|
||||
int scanned = 0;
|
||||
for (Path file : listBufferedPayloadFiles(root, limit)) {
|
||||
scanned++;
|
||||
try {
|
||||
BufferFileRef ref = parseBufferFile(root, file);
|
||||
if (ref == null) {
|
||||
Files.deleteIfExists(file);
|
||||
continue;
|
||||
}
|
||||
if (taskScopePayloadStorageService.recoverBufferedScopePayload(ref.taskId(), ref.moduleType(), ref.scopeHash())) {
|
||||
recovered++;
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.warn("[task-scope-buffer] startup recovery failed path={} msg={}", file, ex.getMessage());
|
||||
}
|
||||
}
|
||||
if (scanned > 0) {
|
||||
log.info("[task-scope-buffer] startup recovery finished scanned={} recovered={} limit={}", scanned, recovered, limit);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Scheduled(cron = "${aiimage.task-pressure.scope-payload-cleanup-cron:15 */30 * * * *}")
|
||||
public void cleanupExpiredBufferedPayloads() {
|
||||
DistributedJobLockService.LockHandle lockHandle =
|
||||
distributedJobLockService.tryLock("task-scope-buffer:cleanup", CLEANUP_LOCK_TTL);
|
||||
if (lockHandle == null) {
|
||||
log.info("[task-scope-buffer] skip cleanup because another instance holds the lock");
|
||||
return;
|
||||
}
|
||||
try (lockHandle) {
|
||||
Path root = taskScopePayloadStorageService.getBufferedPayloadRoot();
|
||||
if (!Files.isDirectory(root)) {
|
||||
return;
|
||||
}
|
||||
Instant cutoff = Instant.now().minus(Duration.ofHours(Math.max(1L, taskPressureProperties.getScopePayloadBufferRetentionHours())));
|
||||
int deleted = 0;
|
||||
for (Path file : listBufferedPayloadFiles(root, Integer.MAX_VALUE)) {
|
||||
try {
|
||||
FileTime lastModified = Files.getLastModifiedTime(file);
|
||||
if (lastModified.toInstant().isAfter(cutoff)) {
|
||||
continue;
|
||||
}
|
||||
Files.deleteIfExists(file);
|
||||
deleted++;
|
||||
deleteEmptyParents(file.getParent(), root);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[task-scope-buffer] cleanup failed path={} msg={}", file, ex.getMessage());
|
||||
}
|
||||
}
|
||||
if (deleted > 0) {
|
||||
log.info("[task-scope-buffer] cleanup finished deleted={} root={}", deleted, root);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private List<Path> listBufferedPayloadFiles(Path root, int limit) {
|
||||
try (var walk = Files.walk(root, 4)) {
|
||||
return walk
|
||||
.filter(Files::isRegularFile)
|
||||
.filter(path -> path.getFileName().toString().endsWith(".json"))
|
||||
.sorted(Comparator.comparing(this::safeLastModified))
|
||||
.limit(limit)
|
||||
.toList();
|
||||
} catch (IOException ex) {
|
||||
log.warn("[task-scope-buffer] scan failed root={} msg={}", root, ex.getMessage());
|
||||
return List.of();
|
||||
}
|
||||
}
|
||||
|
||||
private Instant safeLastModified(Path path) {
|
||||
try {
|
||||
return Files.getLastModifiedTime(path).toInstant();
|
||||
} catch (IOException ex) {
|
||||
return Instant.EPOCH;
|
||||
}
|
||||
}
|
||||
|
||||
private BufferFileRef parseBufferFile(Path root, Path file) {
|
||||
Path relative = root.relativize(file);
|
||||
if (relative.getNameCount() != 3) {
|
||||
return null;
|
||||
}
|
||||
String moduleType = relative.getName(0).toString().toUpperCase();
|
||||
String taskIdRaw = relative.getName(1).toString();
|
||||
String fileName = relative.getName(2).toString();
|
||||
if (!fileName.endsWith(".json")) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
long taskId = Long.parseLong(taskIdRaw);
|
||||
String scopeHash = fileName.substring(0, fileName.length() - 5);
|
||||
if (scopeHash.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return new BufferFileRef(moduleType, taskId, scopeHash);
|
||||
} catch (NumberFormatException ex) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private void deleteEmptyParents(Path start, Path root) {
|
||||
Path current = start;
|
||||
while (current != null && !current.equals(root)) {
|
||||
try (var list = Files.list(current)) {
|
||||
if (list.findAny().isPresent()) {
|
||||
return;
|
||||
}
|
||||
} catch (IOException ex) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
Files.deleteIfExists(current);
|
||||
} catch (IOException ex) {
|
||||
return;
|
||||
}
|
||||
current = current.getParent();
|
||||
}
|
||||
}
|
||||
|
||||
private record BufferFileRef(String moduleType, Long taskId, String scopeHash) {}
|
||||
}
|
||||
+542
@@ -0,0 +1,542 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class TaskScopePayloadStorageService {
|
||||
|
||||
private static final Set<String> OSS_BACKED_STATE_MODULES = new HashSet<>(Set.of(
|
||||
"SHOP_MATCH",
|
||||
"PRODUCT_RISK_RESOLVE",
|
||||
"PRICE_TRACK",
|
||||
"PATROL_DELETE"
|
||||
));
|
||||
private static final String OSS_POINTER_PREFIX = "oss:";
|
||||
private static final String LOCAL_BUFFER_DIR = "task-scope-buffer";
|
||||
|
||||
private final TaskScopeStateMapper taskScopeStateMapper;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final OssStorageService ossStorageService;
|
||||
private final TaskPressureProperties taskPressureProperties;
|
||||
|
||||
@Transactional
|
||||
public <T> void saveScopePayload(Long taskId, String moduleType, String scopeKey, T payload) {
|
||||
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey) || payload == null) {
|
||||
return;
|
||||
}
|
||||
String normalizedScopeKey = normalize(scopeKey);
|
||||
String scopeHash = hash(normalizedScopeKey);
|
||||
String payloadJson = writeJson(payload, "写入任务范围载荷失败");
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash);
|
||||
if (shouldStoreStatePayloadInOss(moduleType)) {
|
||||
saveOssBackedScopePayload(taskId, moduleType, normalizedScopeKey, scopeHash, payloadJson, now, state);
|
||||
return;
|
||||
}
|
||||
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
|
||||
if (state == null) {
|
||||
TaskScopeStateEntity entity = new TaskScopeStateEntity();
|
||||
entity.setTaskId(taskId);
|
||||
entity.setModuleType(moduleType);
|
||||
entity.setScopeKey(normalizedScopeKey);
|
||||
entity.setScopeHash(scopeHash);
|
||||
entity.setStateJson(stateJson);
|
||||
entity.setChunkTotal(0);
|
||||
entity.setReceivedChunkCount(0);
|
||||
entity.setCompleted(0);
|
||||
entity.setLastChunkAt(now);
|
||||
entity.setLastError(null);
|
||||
entity.setCreatedAt(now);
|
||||
entity.setUpdatedAt(now);
|
||||
try {
|
||||
taskScopeStateMapper.insert(entity);
|
||||
return;
|
||||
} catch (DuplicateKeyException ignored) {
|
||||
state = getScopeState(taskId, moduleType, scopeHash);
|
||||
}
|
||||
}
|
||||
if (state == null) {
|
||||
throw new BusinessException("写入任务范围载荷失败");
|
||||
}
|
||||
deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson);
|
||||
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getId, state.getId())
|
||||
.set(TaskScopeStateEntity::getScopeKey, normalizedScopeKey)
|
||||
.set(TaskScopeStateEntity::getStateJson, stateJson)
|
||||
.set(TaskScopeStateEntity::getLastChunkAt, now)
|
||||
.set(TaskScopeStateEntity::getUpdatedAt, now));
|
||||
}
|
||||
|
||||
public <T> T getScopePayload(Long taskId, String moduleType, String scopeKey, Class<T> clazz) {
|
||||
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey)) {
|
||||
return null;
|
||||
}
|
||||
TaskScopeStateEntity state = getScopeState(taskId, moduleType, hash(normalize(scopeKey)));
|
||||
String payloadJson = resolveStatePayload(state);
|
||||
if (payloadJson == null) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return objectMapper.readValue(payloadJson, clazz);
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("读取任务范围载荷失败");
|
||||
}
|
||||
}
|
||||
|
||||
public <T> Map<String, T> getAllScopePayload(Long taskId, String moduleType, Class<T> clazz) {
|
||||
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
|
||||
return Map.of();
|
||||
}
|
||||
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.select(TaskScopeStateEntity::getScopeKey, TaskScopeStateEntity::getScopeHash, TaskScopeStateEntity::getStateJson)
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, moduleType)
|
||||
.isNotNull(TaskScopeStateEntity::getStateJson));
|
||||
if (states == null || states.isEmpty()) {
|
||||
return Map.of();
|
||||
}
|
||||
Map<String, T> result = new LinkedHashMap<>();
|
||||
for (TaskScopeStateEntity state : states) {
|
||||
String payloadJson = resolveStatePayload(state);
|
||||
if (payloadJson == null || isBlank(state.getScopeKey())) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
result.put(state.getScopeKey(), objectMapper.readValue(payloadJson, clazz));
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("读取任务范围载荷失败");
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void removeScopePayload(Long taskId, String moduleType, String scopeKey) {
|
||||
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey)) {
|
||||
return;
|
||||
}
|
||||
TaskScopeStateEntity state = getScopeState(taskId, moduleType, hash(normalize(scopeKey)));
|
||||
if (state == null) {
|
||||
return;
|
||||
}
|
||||
deleteBufferedPayload(taskId, moduleType, state.getScopeHash());
|
||||
deleteStatePayloadIfNeeded(state.getStateJson());
|
||||
boolean hasParsedPayload = !isBlank(state.getParsedPayloadJson());
|
||||
if (hasParsedPayload) {
|
||||
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getId, state.getId())
|
||||
.set(TaskScopeStateEntity::getStateJson, null)
|
||||
.set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
return;
|
||||
}
|
||||
taskScopeStateMapper.deleteById(state.getId());
|
||||
}
|
||||
|
||||
public boolean hasAnyScopePayload(Long taskId, String moduleType) {
|
||||
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
|
||||
return false;
|
||||
}
|
||||
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, moduleType)
|
||||
.isNotNull(TaskScopeStateEntity::getStateJson));
|
||||
return count != null && count > 0;
|
||||
}
|
||||
|
||||
public long countScopePayload(Long taskId, String moduleType) {
|
||||
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
|
||||
return 0L;
|
||||
}
|
||||
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, moduleType)
|
||||
.isNotNull(TaskScopeStateEntity::getStateJson));
|
||||
return count == null ? 0L : count;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void deleteTaskScopePayloads(Long taskId, String moduleType) {
|
||||
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
|
||||
return;
|
||||
}
|
||||
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, moduleType));
|
||||
if (states == null || states.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
for (TaskScopeStateEntity state : states) {
|
||||
if (state == null || state.getId() == null) {
|
||||
continue;
|
||||
}
|
||||
deleteBufferedPayload(taskId, moduleType, state.getScopeHash());
|
||||
deleteStatePayloadIfNeeded(state.getStateJson());
|
||||
if (!isBlank(state.getParsedPayloadJson())) {
|
||||
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getId, state.getId())
|
||||
.set(TaskScopeStateEntity::getStateJson, null)
|
||||
.set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
} else {
|
||||
taskScopeStateMapper.deleteById(state.getId());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public <T> void saveParsedPayloadMap(Long taskId, String moduleType, Map<String, T> payloadByScope) {
|
||||
if (taskId == null || taskId <= 0 || isBlank(moduleType) || payloadByScope == null || payloadByScope.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
for (Map.Entry<String, T> entry : payloadByScope.entrySet()) {
|
||||
if (isBlank(entry.getKey()) || entry.getValue() == null) {
|
||||
continue;
|
||||
}
|
||||
String scopeKey = normalize(entry.getKey());
|
||||
String scopeHash = hash(scopeKey);
|
||||
String parsedPayloadJson = writeJson(entry.getValue(), "写入任务解析载荷失败");
|
||||
TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash);
|
||||
if (state == null) {
|
||||
TaskScopeStateEntity entity = new TaskScopeStateEntity();
|
||||
entity.setTaskId(taskId);
|
||||
entity.setModuleType(moduleType);
|
||||
entity.setScopeKey(scopeKey);
|
||||
entity.setScopeHash(scopeHash);
|
||||
entity.setParsedPayloadJson(parsedPayloadJson);
|
||||
entity.setChunkTotal(0);
|
||||
entity.setReceivedChunkCount(0);
|
||||
entity.setCompleted(0);
|
||||
entity.setCreatedAt(now);
|
||||
entity.setUpdatedAt(now);
|
||||
try {
|
||||
taskScopeStateMapper.insert(entity);
|
||||
continue;
|
||||
} catch (DuplicateKeyException ignored) {
|
||||
state = getScopeState(taskId, moduleType, scopeHash);
|
||||
}
|
||||
}
|
||||
if (state == null) {
|
||||
throw new BusinessException("写入任务解析载荷失败");
|
||||
}
|
||||
deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), parsedPayloadJson);
|
||||
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getId, state.getId())
|
||||
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
|
||||
.set(TaskScopeStateEntity::getParsedPayloadJson, parsedPayloadJson)
|
||||
.set(TaskScopeStateEntity::getUpdatedAt, now));
|
||||
}
|
||||
}
|
||||
|
||||
public <T> Map<String, T> getParsedPayloadMap(Long taskId, String moduleType, Class<T> clazz) {
|
||||
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
|
||||
return Map.of();
|
||||
}
|
||||
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.select(TaskScopeStateEntity::getScopeKey, TaskScopeStateEntity::getParsedPayloadJson)
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, moduleType)
|
||||
.isNotNull(TaskScopeStateEntity::getParsedPayloadJson));
|
||||
if (states == null || states.isEmpty()) {
|
||||
return Map.of();
|
||||
}
|
||||
Map<String, T> result = new LinkedHashMap<>();
|
||||
for (TaskScopeStateEntity state : states) {
|
||||
if (isBlank(state.getParsedPayloadJson()) || isBlank(state.getScopeKey())) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
result.put(state.getScopeKey(), objectMapper.readValue(resolveParsedPayload(state.getParsedPayloadJson()), clazz));
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("读取任务原始载荷失败");
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public boolean recoverBufferedScopePayload(Long taskId, String moduleType, String scopeHash) {
|
||||
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeHash)) {
|
||||
return false;
|
||||
}
|
||||
String payloadJson = readBufferedPayload(taskId, moduleType, scopeHash);
|
||||
if (payloadJson == null) {
|
||||
return false;
|
||||
}
|
||||
TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash);
|
||||
if (state == null) {
|
||||
deleteBufferedPayload(taskId, moduleType, scopeHash);
|
||||
return false;
|
||||
}
|
||||
if (!shouldStoreStatePayloadInOss(moduleType)) {
|
||||
deleteBufferedPayload(taskId, moduleType, scopeHash);
|
||||
return false;
|
||||
}
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
|
||||
deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson);
|
||||
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getId, state.getId())
|
||||
.set(TaskScopeStateEntity::getStateJson, stateJson)
|
||||
.set(TaskScopeStateEntity::getLastChunkAt, now)
|
||||
.set(TaskScopeStateEntity::getUpdatedAt, now));
|
||||
deleteBufferedPayload(taskId, moduleType, scopeHash);
|
||||
return true;
|
||||
}
|
||||
|
||||
public Path getBufferedPayloadRoot() {
|
||||
return Path.of(System.getProperty("java.io.tmpdir"), LOCAL_BUFFER_DIR);
|
||||
}
|
||||
|
||||
private TaskScopeStateEntity getScopeState(Long taskId, String moduleType, String scopeHash) {
|
||||
return taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||
.eq(TaskScopeStateEntity::getModuleType, moduleType)
|
||||
.eq(TaskScopeStateEntity::getScopeHash, scopeHash)
|
||||
.last("limit 1"));
|
||||
}
|
||||
|
||||
private String storeStatePayload(String moduleType, Long taskId, String scopeHash, String payloadJson) {
|
||||
if (!shouldStoreStatePayloadInOss(moduleType)) {
|
||||
return payloadJson;
|
||||
}
|
||||
try {
|
||||
String pointer = OSS_POINTER_PREFIX + ossStorageService.uploadTaskScopePayload(moduleType, taskId, scopeHash, payloadJson);
|
||||
return objectMapper.writeValueAsString(pointer);
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("写入任务范围载荷失败");
|
||||
}
|
||||
}
|
||||
|
||||
private String resolveStatePayload(TaskScopeStateEntity state) {
|
||||
if (state == null || isBlank(state.getStateJson())) {
|
||||
return null;
|
||||
}
|
||||
String bufferedPayload = readBufferedPayload(state.getTaskId(), state.getModuleType(), state.getScopeHash());
|
||||
if (bufferedPayload != null) {
|
||||
return bufferedPayload;
|
||||
}
|
||||
String ossPointer = extractOssPointer(state.getStateJson());
|
||||
if (ossPointer == null) {
|
||||
return state.getStateJson();
|
||||
}
|
||||
try {
|
||||
return ossStorageService.readObjectAsString(stripOssPointerPrefix(ossPointer));
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("读取任务范围载荷失败");
|
||||
}
|
||||
}
|
||||
|
||||
private void deleteStatePayloadIfNeeded(String value) {
|
||||
String ossPointer = extractOssPointer(value);
|
||||
if (ossPointer == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
ossStorageService.deleteObject(stripOssPointerPrefix(ossPointer));
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
}
|
||||
|
||||
private String resolveParsedPayload(String value) {
|
||||
if (isBlank(value) || !isOssPointer(value)) {
|
||||
return value;
|
||||
}
|
||||
try {
|
||||
return ossStorageService.readObjectAsString(stripOssPointerPrefix(value));
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("读取任务解析载荷失败");
|
||||
}
|
||||
}
|
||||
|
||||
private void deleteParsedPayloadIfNeeded(String oldValue, String newValue) {
|
||||
if (isBlank(oldValue) || oldValue.equals(newValue) || !isOssPointer(oldValue)) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
ossStorageService.deleteObject(stripOssPointerPrefix(oldValue));
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
}
|
||||
|
||||
private void deleteReplacedStatePayloadIfNeeded(String oldValue, String newValue) {
|
||||
if (isBlank(oldValue) || oldValue.equals(newValue)) {
|
||||
return;
|
||||
}
|
||||
deleteStatePayloadIfNeeded(oldValue);
|
||||
}
|
||||
|
||||
private void saveOssBackedScopePayload(Long taskId,
|
||||
String moduleType,
|
||||
String normalizedScopeKey,
|
||||
String scopeHash,
|
||||
String payloadJson,
|
||||
LocalDateTime now,
|
||||
TaskScopeStateEntity state) {
|
||||
if (state == null) {
|
||||
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
|
||||
TaskScopeStateEntity entity = new TaskScopeStateEntity();
|
||||
entity.setTaskId(taskId);
|
||||
entity.setModuleType(moduleType);
|
||||
entity.setScopeKey(normalizedScopeKey);
|
||||
entity.setScopeHash(scopeHash);
|
||||
entity.setStateJson(stateJson);
|
||||
entity.setChunkTotal(0);
|
||||
entity.setReceivedChunkCount(0);
|
||||
entity.setCompleted(0);
|
||||
entity.setLastChunkAt(now);
|
||||
entity.setLastError(null);
|
||||
entity.setCreatedAt(now);
|
||||
entity.setUpdatedAt(now);
|
||||
try {
|
||||
taskScopeStateMapper.insert(entity);
|
||||
deleteBufferedPayload(taskId, moduleType, scopeHash);
|
||||
return;
|
||||
} catch (DuplicateKeyException ignored) {
|
||||
state = getScopeState(taskId, moduleType, scopeHash);
|
||||
}
|
||||
}
|
||||
if (state == null) {
|
||||
throw new BusinessException("刷新任务范围缓冲载荷失败");
|
||||
}
|
||||
|
||||
writeBufferedPayload(taskId, moduleType, scopeHash, payloadJson);
|
||||
if (!shouldFlushBufferedPayload(state, now)) {
|
||||
return;
|
||||
}
|
||||
|
||||
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
|
||||
deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson);
|
||||
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||
.eq(TaskScopeStateEntity::getId, state.getId())
|
||||
.set(TaskScopeStateEntity::getScopeKey, normalizedScopeKey)
|
||||
.set(TaskScopeStateEntity::getStateJson, stateJson)
|
||||
.set(TaskScopeStateEntity::getLastChunkAt, now)
|
||||
.set(TaskScopeStateEntity::getUpdatedAt, now));
|
||||
deleteBufferedPayload(taskId, moduleType, scopeHash);
|
||||
}
|
||||
|
||||
private String writeJson(Object value, String message) {
|
||||
try {
|
||||
return objectMapper.writeValueAsString(value);
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException(message);
|
||||
}
|
||||
}
|
||||
|
||||
private String normalize(String value) {
|
||||
return value == null ? "" : value.trim();
|
||||
}
|
||||
|
||||
private boolean isBlank(String value) {
|
||||
return value == null || value.isBlank();
|
||||
}
|
||||
|
||||
private boolean shouldStoreStatePayloadInOss(String moduleType) {
|
||||
return OSS_BACKED_STATE_MODULES.contains(normalize(moduleType).toUpperCase());
|
||||
}
|
||||
|
||||
private boolean isOssPointer(String value) {
|
||||
return !isBlank(value) && value.startsWith(OSS_POINTER_PREFIX);
|
||||
}
|
||||
|
||||
private String extractOssPointer(String value) {
|
||||
if (isBlank(value)) {
|
||||
return null;
|
||||
}
|
||||
if (isOssPointer(value)) {
|
||||
return value;
|
||||
}
|
||||
try {
|
||||
String decoded = objectMapper.readValue(value, String.class);
|
||||
return isOssPointer(decoded) ? decoded : null;
|
||||
} catch (Exception ignored) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String stripOssPointerPrefix(String value) {
|
||||
return value.substring(OSS_POINTER_PREFIX.length());
|
||||
}
|
||||
|
||||
private boolean shouldFlushBufferedPayload(TaskScopeStateEntity state, LocalDateTime now) {
|
||||
long flushIntervalMillis = Math.max(0L, taskPressureProperties.getScopePayloadFlushIntervalMillis());
|
||||
if (flushIntervalMillis <= 0L || state.getUpdatedAt() == null) {
|
||||
return true;
|
||||
}
|
||||
return java.time.Duration.between(state.getUpdatedAt(), now).toMillis() >= flushIntervalMillis;
|
||||
}
|
||||
|
||||
private void writeBufferedPayload(Long taskId, String moduleType, String scopeHash, String payloadJson) {
|
||||
try {
|
||||
Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash);
|
||||
Files.createDirectories(path.getParent());
|
||||
Files.writeString(path, payloadJson, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE);
|
||||
} catch (IOException ex) {
|
||||
throw new BusinessException("写入任务范围缓冲载荷失败");
|
||||
}
|
||||
}
|
||||
|
||||
private String readBufferedPayload(Long taskId, String moduleType, String scopeHash) {
|
||||
try {
|
||||
Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash);
|
||||
if (!Files.isRegularFile(path)) {
|
||||
return null;
|
||||
}
|
||||
return Files.readString(path);
|
||||
} catch (IOException ex) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private void deleteBufferedPayload(Long taskId, String moduleType, String scopeHash) {
|
||||
try {
|
||||
Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash);
|
||||
Files.deleteIfExists(path);
|
||||
} catch (IOException ignored) {
|
||||
}
|
||||
}
|
||||
|
||||
private Path buildBufferedPayloadPath(Long taskId, String moduleType, String scopeHash) {
|
||||
String normalizedModuleType = normalize(moduleType).toLowerCase();
|
||||
return Path.of(System.getProperty("java.io.tmpdir"), LOCAL_BUFFER_DIR, normalizedModuleType, String.valueOf(taskId), scopeHash + ".json");
|
||||
}
|
||||
|
||||
private String hash(String value) {
|
||||
try {
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
byte[] bytes = digest.digest(normalize(value).getBytes(StandardCharsets.UTF_8));
|
||||
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
||||
for (byte b : bytes) {
|
||||
sb.append(String.format("%02x", b));
|
||||
}
|
||||
return sb.toString();
|
||||
} catch (Exception ex) {
|
||||
throw new IllegalStateException("failed to hash scope key", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user