新增内容,新增拉起软件层

This commit is contained in:
super
2026-06-08 09:19:15 +08:00
parent 1dabf8f55f
commit 687b193645
67 changed files with 1552 additions and 189 deletions
@@ -30,6 +30,15 @@ public interface TaskImageCacheMapper extends BaseMapper<TaskImageCacheEntity> {
@Select("SELECT image_bytes FROM biz_task_image_cache WHERE url_hash = #{urlHash} LIMIT 1")
byte[] selectBytesByUrlHash(@Param("urlHash") String urlHash);
@Select("SELECT COALESCE(SUM(byte_size), 0) FROM biz_task_image_cache")
Long sumByteSize();
@Select("SELECT COALESCE(SUM(byte_size), 0) FROM (SELECT byte_size FROM biz_task_image_cache ORDER BY last_used_at LIMIT #{limit}) oldest")
Long sumOldestBatchByteSize(@Param("limit") int limit);
@Delete("DELETE FROM biz_task_image_cache WHERE last_used_at < #{cutoff} ORDER BY last_used_at LIMIT #{limit}")
int deleteExpiredBatch(@Param("cutoff") LocalDateTime cutoff, @Param("limit") int limit);
@Delete("DELETE FROM biz_task_image_cache ORDER BY last_used_at LIMIT #{limit}")
int deleteOldestBatch(@Param("limit") int limit);
}
@@ -39,18 +39,18 @@ public class TaskImageCacheCleanupService {
int maxBatches = Math.max(1, cleanupProperties.getMaxBatchesPerRun());
LocalDateTime cutoff = LocalDateTime.now().minusDays(retentionDays);
int deletedTotal = 0;
int batches = 0;
while (batches < maxBatches) {
int expiredDeletedTotal = 0;
int expiredBatches = 0;
while (expiredBatches < maxBatches) {
int deleted = taskImageCacheMapper.deleteExpiredBatch(cutoff, batchSize);
if (deleted <= 0) {
break;
}
deletedTotal += deleted;
batches++;
expiredDeletedTotal += deleted;
expiredBatches++;
if (!lockHandle.renew(CLEANUP_LOCK_TTL)) {
log.warn("[task-image-cache-cleanup] stop because lock renew failed deletedTotal={} batches={}",
deletedTotal, batches);
expiredDeletedTotal, expiredBatches);
break;
}
if (deleted < batchSize) {
@@ -58,12 +58,49 @@ public class TaskImageCacheCleanupService {
}
}
if (deletedTotal > 0 || batches >= maxBatches) {
log.info("[task-image-cache-cleanup] finished cutoff={} retentionDays={} deletedTotal={} batches={} batchSize={} maxBatches={}",
cutoff, retentionDays, deletedTotal, batches, batchSize, maxBatches);
long maxBytes = cleanupProperties.getMaxBytes();
long targetBytes = cleanupProperties.getTargetBytes();
int capacityDeletedTotal = 0;
int capacityBatches = 0;
if (maxBytes > 0L) {
long actualTargetBytes = targetBytes > 0L && targetBytes < maxBytes ? targetBytes : maxBytes / 2L;
long currentBytes = nullToZero(taskImageCacheMapper.sumByteSize());
boolean overCapacity = currentBytes > maxBytes;
while (overCapacity && currentBytes > actualTargetBytes && capacityBatches < maxBatches) {
long batchBytes = nullToZero(taskImageCacheMapper.sumOldestBatchByteSize(batchSize));
if (batchBytes <= 0L) {
break;
}
int deleted = taskImageCacheMapper.deleteOldestBatch(batchSize);
if (deleted <= 0) {
break;
}
capacityDeletedTotal += deleted;
capacityBatches++;
currentBytes = Math.max(0L, currentBytes - batchBytes);
if (!lockHandle.renew(CLEANUP_LOCK_TTL)) {
log.warn("[task-image-cache-cleanup] stop capacity cleanup because lock renew failed deletedTotal={} batches={}",
capacityDeletedTotal, capacityBatches);
break;
}
if (deleted < batchSize) {
break;
}
}
}
if (expiredDeletedTotal > 0 || expiredBatches >= maxBatches
|| capacityDeletedTotal > 0 || capacityBatches >= maxBatches) {
log.info("[task-image-cache-cleanup] finished cutoff={} retentionDays={} expiredDeleted={} expiredBatches={} capacityDeleted={} capacityBatches={} batchSize={} maxBatches={} maxBytes={} targetBytes={}",
cutoff, retentionDays, expiredDeletedTotal, expiredBatches, capacityDeletedTotal,
capacityBatches, batchSize, maxBatches, maxBytes, targetBytes);
}
} catch (Exception ex) {
log.warn("[task-image-cache-cleanup] failed msg={}", ex.getMessage(), ex);
}
}
private static long nullToZero(Long value) {
return value == null ? 0L : value;
}
}
@@ -312,20 +312,39 @@ public class TransientPayloadStorageService {
return content;
}
String objectKey = buildObjectKey(category, moduleType, taskId, scopeHash, entryKey);
long rawBytes = payloadBytes(content);
if (isPositiveLimit(properties.getWarnPayloadBytes()) && rawBytes > properties.getWarnPayloadBytes()) {
log.warn("[transient-payload] payload size exceeds warn threshold category={} moduleType={} taskId={} objectKey={} rawBytes={} warnBytes={}",
category, moduleType, taskId, objectKey, rawBytes, properties.getWarnPayloadBytes());
}
boolean rawOversize = isPositiveLimit(properties.getMaxPayloadBytes()) && rawBytes > properties.getMaxPayloadBytes();
String storedContent = encodeStoredPayload(content);
long storedBytes = payloadBytes(storedContent);
boolean storedOversize = isPositiveLimit(properties.getMaxStoredPayloadBytes()) && storedBytes > properties.getMaxStoredPayloadBytes();
if (rawOversize || storedOversize) {
log.warn("[transient-payload] payload size exceeds rustfs limit category={} moduleType={} taskId={} objectKey={} rawBytes={} storedBytes={} maxRawBytes={} maxStoredBytes={} fallbackToLocal={}",
category, moduleType, taskId, objectKey, rawBytes, storedBytes,
properties.getMaxPayloadBytes(), properties.getMaxStoredPayloadBytes(), properties.isFallbackToLocalOnOversize());
if (!properties.isFallbackToLocalOnOversize()) {
throw new IllegalStateException("transient payload exceeds configured size limit: " + objectKey);
}
}
String pointer = null;
boolean rustfsFallbackToLocal = false;
if (rustfsObjectStorageService.isConfigured()) {
boolean rustfsFallbackToLocal = rawOversize || storedOversize;
if (!rustfsFallbackToLocal && rustfsObjectStorageService.isConfigured()) {
try {
pointer = RUSTFS_POINTER_PREFIX + rustfsObjectStorageService.uploadText(objectKey, storedContent, verifyAfterUpload);
} catch (Exception ex) {
rustfsFallbackToLocal = true;
// 升级为 ERROR:rustfs 失败后只能落到本地,多实例下其他节点读不到,必须能告警。
log.error("[transient-payload] rustfs upload failed, fallback to local store instanceId={} category={} taskId={} objectKey={} err={}",
instanceMetadata.getInstanceId(), category, taskId, objectKey, ex.getMessage());
log.error("[transient-payload] rustfs upload failed, fallback to local store instanceId={} category={} moduleType={} taskId={} objectKey={} rawBytes={} storedBytes={} err={}",
instanceMetadata.getInstanceId(), category, moduleType, taskId, objectKey, rawBytes, storedBytes, ex.getMessage());
}
}
if (pointer == null) {
if (rustfsFallbackToLocal) {
rustfsObjectStorageService.recordLocalFallback();
}
pointer = storeLocal(objectKey, storedContent);
}
// P0-3:记录本次 store 是否走 local 兜底,供调用方在拿到 pointer 后立即查询。
@@ -429,6 +448,14 @@ public class TransientPayloadStorageService {
}
}
private long payloadBytes(String content) {
return Objects.requireNonNullElse(content, "").getBytes(StandardCharsets.UTF_8).length;
}
private boolean isPositiveLimit(long value) {
return value > 0L;
}
private String decodeStoredPayload(String storedContent) {
if (storedContent == null || storedContent.isBlank()) {
return storedContent;