task-80: 补充 JVM 堆、直接内存、临时磁盘和连接池容量配置说明
This commit is contained in:
@@ -0,0 +1,39 @@
|
|||||||
|
package com.nanri.aiimage.config;
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@ConfigurationProperties(prefix = "aiimage.capacity-plan")
|
||||||
|
public class CapacityPlanProperties {
|
||||||
|
|
||||||
|
/** JVM 堆上限(-Xmx),0 表示交由启动参数决定。 */
|
||||||
|
private long heapMaxBytes = 6L * 1024 * 1024 * 1024;
|
||||||
|
|
||||||
|
/** JVM 直接内存上限(-XX:MaxDirectMemorySize),0 表示交由启动参数决定。 */
|
||||||
|
private long directMaxBytes = 1L * 1024 * 1024 * 1024;
|
||||||
|
|
||||||
|
/** 临时磁盘(./data/tmp)为一次大任务预留的字节上限,0 会触发警告。 */
|
||||||
|
private long tempDiskReserveBytes = 4L * 1024 * 1024 * 1024;
|
||||||
|
|
||||||
|
/** 数据库连接池(Hikari maximum-pool-size)。 */
|
||||||
|
private int dbPoolMaxSize = 30;
|
||||||
|
|
||||||
|
/** 外部 HTTP 客户端(Coze/品牌/紫鸟)连接池容量。 */
|
||||||
|
private int httpClientPoolMaxSize = 32;
|
||||||
|
|
||||||
|
/** RustFS/MinIO OkHttp 连接池容量。 */
|
||||||
|
private int rustfsPoolMaxSize = 56;
|
||||||
|
|
||||||
|
/** 三个连接池之和的上限:超过视为配置异常,启动时警告。 */
|
||||||
|
private int poolTotalMax = 150;
|
||||||
|
|
||||||
|
/** 堆上限占机器物理内存的最大比例(超过触发警告)。 */
|
||||||
|
private double heapRatioWarn = 0.70;
|
||||||
|
|
||||||
|
/** 直接内存上限占机器物理内存的最大比例(超过触发警告)。 */
|
||||||
|
private double directRatioWarn = 0.15;
|
||||||
|
|
||||||
|
/** 临时磁盘预留占机器物理内存的最大比例(超过触发警告)。 */
|
||||||
|
private double tempDiskRatioWarn = 0.50;
|
||||||
|
}
|
||||||
@@ -0,0 +1,102 @@
|
|||||||
|
package com.nanri.aiimage.config;
|
||||||
|
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.boot.CommandLineRunner;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Task 80:JVM 堆、直接内存、临时磁盘和连接池容量配置说明。把所有容量边界
|
||||||
|
* 收敛为一个可校验、可记录的 CapacityPlan:启动时打印摘要,配置越限时输出
|
||||||
|
* 明确警告(不吞配置、不改默认行为),方便按机器内存调整 -Xmx/-XX:
|
||||||
|
* MaxDirectMemorySize 与三个连接池。
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@Component
|
||||||
|
public class CapacityPlanService implements CommandLineRunner {
|
||||||
|
|
||||||
|
private final CapacityPlanProperties properties;
|
||||||
|
private final long machineMaxMemoryBytes;
|
||||||
|
|
||||||
|
/** 供测试注入机器内存字节数;生产构造时取 Runtime.maxMemory()。 */
|
||||||
|
public CapacityPlanService(CapacityPlanProperties properties) {
|
||||||
|
this(properties, maxMemoryBytes());
|
||||||
|
}
|
||||||
|
|
||||||
|
CapacityPlanService(CapacityPlanProperties properties, long machineMaxMemoryBytes) {
|
||||||
|
this.properties = properties;
|
||||||
|
this.machineMaxMemoryBytes = machineMaxMemoryBytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static long maxMemoryBytes() {
|
||||||
|
try {
|
||||||
|
return Runtime.getRuntime().maxMemory();
|
||||||
|
} catch (Exception ex) {
|
||||||
|
return 0L;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public CapacityPlan buildPlan() {
|
||||||
|
if (properties.getDbPoolMaxSize() < 0 || properties.getHttpClientPoolMaxSize() < 0
|
||||||
|
|| properties.getRustfsPoolMaxSize() < 0 || properties.getPoolTotalMax() < 0) {
|
||||||
|
throw new IllegalArgumentException("连接池容量不能为负值: db=" + properties.getDbPoolMaxSize()
|
||||||
|
+ " http=" + properties.getHttpClientPoolMaxSize()
|
||||||
|
+ " rustfs=" + properties.getRustfsPoolMaxSize());
|
||||||
|
}
|
||||||
|
List<String> warnings = new ArrayList<>();
|
||||||
|
if (machineMaxMemoryBytes > 0) {
|
||||||
|
warnRatio(warnings, "堆", properties.getHeapMaxBytes(), properties.getHeapRatioWarn());
|
||||||
|
warnRatio(warnings, "直接内存", properties.getDirectMaxBytes(), properties.getDirectRatioWarn());
|
||||||
|
warnRatio(warnings, "临时磁盘预留", properties.getTempDiskReserveBytes(), properties.getTempDiskRatioWarn());
|
||||||
|
} else {
|
||||||
|
warnings.add("无法取得机器内存,容量比例校验跳过");
|
||||||
|
}
|
||||||
|
if (properties.getTempDiskReserveBytes() <= 0) {
|
||||||
|
warnings.add("临时磁盘预留为 0,大任务可能无界写盘");
|
||||||
|
}
|
||||||
|
int poolTotal = properties.getDbPoolMaxSize()
|
||||||
|
+ properties.getHttpClientPoolMaxSize()
|
||||||
|
+ properties.getRustfsPoolMaxSize();
|
||||||
|
if (poolTotal > properties.getPoolTotalMax()) {
|
||||||
|
warnings.add("连接池总和 " + poolTotal + " 超过上限 " + properties.getPoolTotalMax());
|
||||||
|
}
|
||||||
|
return new CapacityPlan(properties.getHeapMaxBytes(), properties.getDirectMaxBytes(),
|
||||||
|
properties.getTempDiskReserveBytes(), properties.getDbPoolMaxSize(),
|
||||||
|
properties.getHttpClientPoolMaxSize(), properties.getRustfsPoolMaxSize(),
|
||||||
|
poolTotal, machineMaxMemoryBytes, List.copyOf(warnings));
|
||||||
|
}
|
||||||
|
|
||||||
|
private void warnRatio(List<String> warnings, String label, long bytes, double ratioWarn) {
|
||||||
|
if (bytes > 0 && (double) bytes > machineMaxMemoryBytes * ratioWarn) {
|
||||||
|
warnings.add(label + "上限 " + bytes + " 超过机器内存 " + machineMaxMemoryBytes
|
||||||
|
+ " 的 " + (int) (ratioWarn * 100) + "%");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void run(String... args) {
|
||||||
|
CapacityPlan plan = buildPlan();
|
||||||
|
log.info("[capacity] JVM 堆上限={} 直接内存={} 临时磁盘预留={} 连接池 db={} http={} rustfs={} 总和={} 机器内存={}",
|
||||||
|
plan.heapMaxBytes(), plan.directMaxBytes(), plan.tempDiskReserveBytes(),
|
||||||
|
plan.dbPoolMaxSize(), plan.httpClientPoolMaxSize(), plan.rustfsPoolMaxSize(),
|
||||||
|
plan.poolTotal(), plan.machineMaxMemoryBytes());
|
||||||
|
for (String warning : plan.warnings()) {
|
||||||
|
log.warn("[capacity] {}", warning);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public record CapacityPlan(
|
||||||
|
long heapMaxBytes,
|
||||||
|
long directMaxBytes,
|
||||||
|
long tempDiskReserveBytes,
|
||||||
|
int dbPoolMaxSize,
|
||||||
|
int httpClientPoolMaxSize,
|
||||||
|
int rustfsPoolMaxSize,
|
||||||
|
int poolTotal,
|
||||||
|
long machineMaxMemoryBytes,
|
||||||
|
List<String> warnings
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,6 +4,6 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
|
|||||||
import org.springframework.context.annotation.Configuration;
|
import org.springframework.context.annotation.Configuration;
|
||||||
|
|
||||||
@Configuration
|
@Configuration
|
||||||
@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, BrandCheckProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, TaskImageCacheCleanupProperties.class, AppearancePatentProperties.class, SimilarAsinProperties.class, ImageVideoProperties.class, InstanceRoutingProperties.class})
|
@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, BrandCheckProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, TaskImageCacheCleanupProperties.class, AppearancePatentProperties.class, SimilarAsinProperties.class, ImageVideoProperties.class, InstanceRoutingProperties.class, CapacityPlanProperties.class})
|
||||||
public class PropertiesConfig {
|
public class PropertiesConfig {
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -151,6 +151,17 @@ aiimage:
|
|||||||
transient-payload-retention-hours: ${AIIMAGE_STORAGE_TRANSIENT_PAYLOAD_RETENTION_HOURS:72}
|
transient-payload-retention-hours: ${AIIMAGE_STORAGE_TRANSIENT_PAYLOAD_RETENTION_HOURS:72}
|
||||||
temp-dir:
|
temp-dir:
|
||||||
retention-hours: ${AIIMAGE_TEMP_DIR_RETENTION_HOURS:24}
|
retention-hours: ${AIIMAGE_TEMP_DIR_RETENTION_HOURS:24}
|
||||||
|
capacity-plan:
|
||||||
|
heap-max-bytes: ${AIIMAGE_CAPACITY_PLAN_HEAP_MAX_BYTES:6442450944}
|
||||||
|
direct-max-bytes: ${AIIMAGE_CAPACITY_PLAN_DIRECT_MAX_BYTES:1073741824}
|
||||||
|
temp-disk-reserve-bytes: ${AIIMAGE_CAPACITY_PLAN_TEMP_DISK_RESERVE_BYTES:4294967296}
|
||||||
|
db-pool-max-size: ${AIIMAGE_DB_POOL_MAX_SIZE:30}
|
||||||
|
http-client-pool-max-size: ${AIIMAGE_CAPACITY_PLAN_HTTP_CLIENT_POOL_MAX_SIZE:32}
|
||||||
|
rustfs-pool-max-size: ${AIIMAGE_CAPACITY_PLAN_RUSTFS_POOL_MAX_SIZE:56}
|
||||||
|
pool-total-max: ${AIIMAGE_CAPACITY_PLAN_POOL_TOTAL_MAX:150}
|
||||||
|
heap-ratio-warn: ${AIIMAGE_CAPACITY_PLAN_HEAP_RATIO_WARN:0.70}
|
||||||
|
direct-ratio-warn: ${AIIMAGE_CAPACITY_PLAN_DIRECT_RATIO_WARN:0.15}
|
||||||
|
temp-disk-ratio-warn: ${AIIMAGE_CAPACITY_PLAN_TEMP_DISK_RATIO_WARN:0.50}
|
||||||
brand-progress:
|
brand-progress:
|
||||||
ttl-hours: ${AIIMAGE_BRAND_PROGRESS_TTL_HOURS:24}
|
ttl-hours: ${AIIMAGE_BRAND_PROGRESS_TTL_HOURS:24}
|
||||||
failed-ttl-hours: ${AIIMAGE_BRAND_PROGRESS_FAILED_TTL_HOURS:2}
|
failed-ttl-hours: ${AIIMAGE_BRAND_PROGRESS_FAILED_TTL_HOURS:2}
|
||||||
|
|||||||
@@ -0,0 +1,149 @@
|
|||||||
|
package com.nanri.aiimage.config;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.config.CapacityPlanService.CapacityPlan;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Task 80:补充 JVM 堆、直接内存、临时磁盘和连接池容量配置说明。容量计划
|
||||||
|
* (CapacityPlan)集中描述四类容量边界,构造时注入机器内存以便确定性校验:
|
||||||
|
* 堆/直接内存不得超机器内存比例、临时磁盘预留必须非零、连接池总和有上限。
|
||||||
|
*/
|
||||||
|
class CapacityPlanServiceTest {
|
||||||
|
|
||||||
|
// ---- 1. 默认路径:默认配置下生成完整容量计划,四类容量齐备且无越限警告 ----
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_task_080_task_normal_default_path() {
|
||||||
|
CapacityPlanProperties properties = new CapacityPlanProperties();
|
||||||
|
CapacityPlanService service = new CapacityPlanService(properties, machineBytes(32L * 1024 * 1024 * 1024));
|
||||||
|
|
||||||
|
CapacityPlan plan = service.buildPlan();
|
||||||
|
|
||||||
|
assertNotNull(plan);
|
||||||
|
assertEquals(properties.getHeapMaxBytes(), plan.heapMaxBytes(), "堆上限来自配置");
|
||||||
|
assertEquals(properties.getDirectMaxBytes(), plan.directMaxBytes(), "直接内存上限来自配置");
|
||||||
|
assertEquals(properties.getTempDiskReserveBytes(), plan.tempDiskReserveBytes(), "临时磁盘预留来自配置");
|
||||||
|
assertEquals(properties.getDbPoolMaxSize(), plan.dbPoolMaxSize(), "DB 连接池来自配置");
|
||||||
|
assertEquals(properties.getHttpClientPoolMaxSize(), plan.httpClientPoolMaxSize(), "外部 HTTP 连接池来自配置");
|
||||||
|
assertEquals(properties.getRustfsPoolMaxSize(), plan.rustfsPoolMaxSize(), "RustFS 连接池来自配置");
|
||||||
|
assertTrue(plan.warnings().isEmpty(), "默认配置(32G 机器)不应有越限警告");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 2. 批量:多连接池全部纳入汇总校验,总和上限生效 ----
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_task_080_task_normal_multiple_items() {
|
||||||
|
CapacityPlanProperties properties = new CapacityPlanProperties();
|
||||||
|
properties.setDbPoolMaxSize(200);
|
||||||
|
properties.setHttpClientPoolMaxSize(200);
|
||||||
|
properties.setRustfsPoolMaxSize(200);
|
||||||
|
CapacityPlanService service = new CapacityPlanService(properties, machineBytes(32L * 1024 * 1024 * 1024));
|
||||||
|
|
||||||
|
CapacityPlan plan = service.buildPlan();
|
||||||
|
|
||||||
|
assertEquals(200 + 200 + 200, plan.poolTotal(), "三个连接池总和必须纳入汇总");
|
||||||
|
assertTrue(plan.warnings().stream().anyMatch(w -> w.contains("连接池")),
|
||||||
|
"连接池总和超上限必须产生警告");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 3. 幂等:重复生成容量计划结果稳定 ----
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_task_080_task_normal_repeated_operation_is_idempotent() {
|
||||||
|
CapacityPlanProperties properties = new CapacityPlanProperties();
|
||||||
|
CapacityPlanService service = new CapacityPlanService(properties, machineBytes(32L * 1024 * 1024 * 1024));
|
||||||
|
|
||||||
|
CapacityPlan first = service.buildPlan();
|
||||||
|
CapacityPlan second = service.buildPlan();
|
||||||
|
CapacityPlan third = service.buildPlan();
|
||||||
|
|
||||||
|
assertEquals(first, second, "重复生成结果必须稳定");
|
||||||
|
assertEquals(second, third, "重复生成结果必须稳定");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 4. 空输入:未配置项走默认值兜底,不创建无效资源 ----
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_task_080_task_boundary_empty_input() {
|
||||||
|
CapacityPlanProperties properties = new CapacityPlanProperties();
|
||||||
|
properties.setHeapMaxBytes(0);
|
||||||
|
properties.setDirectMaxBytes(0);
|
||||||
|
properties.setTempDiskReserveBytes(0);
|
||||||
|
CapacityPlanService service = new CapacityPlanService(properties, machineBytes(32L * 1024 * 1024 * 1024));
|
||||||
|
|
||||||
|
CapacityPlan plan = service.buildPlan();
|
||||||
|
|
||||||
|
assertEquals(0, plan.heapMaxBytes(), "显式配置 0 即 0,不猜测");
|
||||||
|
assertTrue(plan.warnings().stream().anyMatch(w -> w.contains("临时磁盘")),
|
||||||
|
"临时磁盘预留为 0 必须警告,避免无界写盘");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 5. 单元素:只配置一项时其余走默认值,整体计划仍完整 ----
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_task_080_task_boundary_single_item() {
|
||||||
|
CapacityPlanProperties properties = new CapacityPlanProperties();
|
||||||
|
properties.setDbPoolMaxSize(5);
|
||||||
|
CapacityPlanService service = new CapacityPlanService(properties, machineBytes(32L * 1024 * 1024 * 1024));
|
||||||
|
|
||||||
|
CapacityPlan plan = service.buildPlan();
|
||||||
|
|
||||||
|
assertEquals(5, plan.dbPoolMaxSize(), "单独配置的项生效");
|
||||||
|
assertEquals(properties.getHeapMaxBytes(), plan.heapMaxBytes(), "未配置项走默认值");
|
||||||
|
assertTrue(plan.poolTotal() > 0, "单配置项不破坏其余默认项");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 6. 上限/超限:堆超机器内存比例被拒绝并警告,不发生无界分配 ----
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_task_080_task_boundary_limit_and_overflow() {
|
||||||
|
CapacityPlanProperties properties = new CapacityPlanProperties();
|
||||||
|
properties.setHeapMaxBytes(24L * 1024 * 1024 * 1024); // 32G 机器上的 75% > 70% 阈值
|
||||||
|
CapacityPlanService service = new CapacityPlanService(properties, machineBytes(32L * 1024 * 1024 * 1024));
|
||||||
|
|
||||||
|
CapacityPlan plan = service.buildPlan();
|
||||||
|
|
||||||
|
assertTrue(plan.warnings().stream().anyMatch(w -> w.contains("堆")),
|
||||||
|
"堆超过机器内存比例必须警告");
|
||||||
|
assertEquals(24L * 1024 * 1024 * 1024, plan.heapMaxBytes(), "警告不吞配置,但明确标注");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 7. 非法参数:负值配置抛出可识别错误 ----
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_task_080_task_invalid_input_rejected() {
|
||||||
|
CapacityPlanProperties properties = new CapacityPlanProperties();
|
||||||
|
properties.setDbPoolMaxSize(-1);
|
||||||
|
CapacityPlanService service = new CapacityPlanService(properties, machineBytes(32L * 1024 * 1024 * 1024));
|
||||||
|
|
||||||
|
IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, service::buildPlan);
|
||||||
|
|
||||||
|
assertTrue(ex.getMessage().contains("连接池"), "负连接池必须抛出可识别错误消息");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 8. 依赖失败:机器内存信息不可用时降级为保守计划,不泄漏 ----
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void test_task_080_task_dependency_failure_releases_resources() {
|
||||||
|
CapacityPlanProperties properties = new CapacityPlanProperties();
|
||||||
|
CapacityPlanService service = new CapacityPlanService(properties, 0);
|
||||||
|
|
||||||
|
CapacityPlan plan = service.buildPlan();
|
||||||
|
|
||||||
|
assertNotNull(plan);
|
||||||
|
assertEquals(0, plan.machineMaxMemoryBytes(), "机器内存不可用时为 0,不猜测");
|
||||||
|
assertTrue(plan.warnings().stream().anyMatch(w -> w.contains("机器内存")),
|
||||||
|
"无法取得机器内存必须警告,避免过度承诺");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static long machineBytes(long bytes) {
|
||||||
|
return bytes;
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user