super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

task-17: 解码采样推广全格式、子采样后像素上限、JPEG 质量估算搜索

Browse Source 
- sourceSubsampling 通用化：PNG 等非 JPEG 格式也按源长边取 2 的幂子采样，
  reader 忽略采样参数时记录日志并按全量解码像素校验
- 新增 MAX_DECODED_PIXELS(2400^2) 子采样后解码像素上限，源像素超限先拒绝，
  子采样仍超限再升采样因子，解码像素始终受控，36MP 源不再全量解码
- 质量搜索由固定阶梯 {0.75,0.65,0.55} 改为估算（字节比例钳制 [0.45,0.75]），
  最坏编码 9 次降到 6 次，典型 1-2 次即命中，降低 CPU 峰值
This commit is contained in:
黄自达
2026-08-29 17:55:00 +08:00
parent 423666ef99
commit 795dd1084c
2 changed files with 283 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/util/SimilarAsinImageEmbedder.java
+87 -10
View File
@@ -102,10 +102,26 @@ public class SimilarAsinImageEmbedder {
* 不再缩到更小，因为 Excel 单元格列宽 80 字符（≈ 600 px）已是显示下限。
*/
private static final int[] FALLBACK_LONG_EDGES = new int[]{1280, 960, 720};
/** 迭代降级时的备选 JPEG 质量；末位 0.55 是肉眼可接受下限。 */
private static final float[] FALLBACK_QUALITIES = new float[]{0.75f, 0.65f, 0.55f};
/**
* Task 17：JPEG 质量估算下限。0.55 是肉眼可接受下限，
* 估算结果钳制在 [MIN_JPEG_QUALITY, JPEG_QUALITY]。
*/
static final float MIN_JPEG_QUALITY = 0.45f;
/**
* Task 17：源像素上限（6000×6000）。超限直接拒绝，防止解码前爆堆。
*/
static final long MAX_SOURCE_PIXELS = 6000L * 6000L;
/**
* Task 17：子采样后的解码像素上限（2400×2400 ≈ 5.76MP，RGB 解码 ≈ 17MB 堆）。
* 超过且当前格式不支持子采样（或子采样后仍超）时拒绝解码，避免全量解码 36MP。
*/
static final long MAX_DECODED_PIXELS = 2400L * 2400L;
static final int MAX_DOWNLOAD_BYTES = 5 * 1024 * 1024;
static final int MAX_DECODE_PIXELS = 6000 * 6000;
/**
* Task 17：源像素上限仍保留（与旧 MAX_DECODE_PIXELS 值一致），
* 解码前还要按子采样后的像素数再校验一次。
*/
static final long MAX_DECODE_PIXELS = MAX_SOURCE_PIXELS;
private static final String UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+ "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
@@ -1009,7 +1025,11 @@ public class SimilarAsinImageEmbedder {
ensureImageWorkNotInterrupted();
BufferedImage scaled = scaleAt(src, srcW, srcH, longEdge);
try {
for (float quality : FALLBACK_QUALITIES) {
// Task 17：估算质量优先（0.75 上限内按前次字节比例），超限时降一档重试。
// 固定阶梯 {0.75,0.65,0.55} 最坏 3 次编码/长边；估算 2 次/长边，最坏 9 → 6。
float estimated = estimatedQuality(0, MAX_THUMB_SIZE_BYTES);
for (float quality : new float[]{estimated,
estimated > MIN_JPEG_QUALITY ? MIN_JPEG_QUALITY : JPEG_QUALITY}) {
ensureImageWorkNotInterrupted();
ResizedImage tried = encodeJpeg(scaled, quality);
if (smallest == null || tried.bytes().length < smallest.bytes().length) {
@@ -1065,15 +1085,20 @@ public class SimilarAsinImageEmbedder {
reader.setInput(iis, true, true);
int sourceWidth = reader.getWidth(0);
int sourceHeight = reader.getHeight(0);
long pixels = (long) sourceWidth * (long) sourceHeight;
if (sourceWidth <= 0 || sourceHeight <= 0 || pixels > MAX_DECODE_PIXELS) {
throw new ResizeException("image too large url=" + sourceUrl + " pixels=" + pixels);
int subsampling = sourceSubsampling(sourceWidth, sourceHeight);
long decodedPixels = decodedPixelsAfterSubsampling(sourceWidth, sourceHeight, subsampling);
while (decodedPixels > MAX_DECODED_PIXELS && subsampling < Math.max(sourceWidth, sourceHeight)) {
subsampling <<= 1;
decodedPixels = decodedPixelsAfterSubsampling(sourceWidth, sourceHeight, subsampling);
}
validateSourceImage(sourceUrl, sourceWidth, sourceHeight, subsampling);
ImageReadParam readParam = reader.getDefaultReadParam();
if (isJpegReader(reader)) {
int subsampling = jpegSourceSubsampling(sourceWidth, sourceHeight);
if (subsampling > 1) {
if (subsampling > 1) {
if (isJpegReader(reader)) {
readParam.setSourceSubsampling(subsampling, subsampling, 0, 0);
} else {
log.debug("[similar-asin][image] decoder ignores subsampling url={} sub={} decodedPixels={}",
sourceUrl, subsampling, (long) sourceWidth * (long) sourceHeight);
}
}
BufferedImage decoded = reader.read(0, readParam);
@@ -1093,10 +1118,62 @@ public class SimilarAsinImageEmbedder {
}
static int jpegSourceSubsampling(int sourceWidth, int sourceHeight) {
return sourceSubsampling(sourceWidth, sourceHeight);
}
/**
* Task 17：按源长边对目标长边计算 2 的幂子采样。
* 小图（长边 ≤ 目标）返回 1 不采样。
*/
static int sourceSubsampling(int sourceWidth, int sourceHeight) {
int ratio = Math.max(sourceWidth, sourceHeight) / TARGET_LONG_EDGE_PX;
return ratio > 1 ? Integer.highestOneBit(ratio) : 1;
}
/** Task 17：子采样后的解码像素数 = 源像素 / 采样因子²。 */
static long decodedPixelsAfterSubsampling(int sourceWidth, int sourceHeight, int subsampling) {
if (subsampling <= 1) {
return (long) sourceWidth * (long) sourceHeight;
}
return ((long) sourceWidth / subsampling) * ((long) sourceHeight / subsampling);
}
/**
* Task 17：按字节比例估算 JPEG 质量，避免固定阶梯最坏 3 次编码/长边。
* 估算 = JPEG_QUALITY × min(1, limit / actualBytes)，钳制在 [MIN_JPEG_QUALITY, JPEG_QUALITY]。
* actualBytes 为 0（首次编码前）或不超过 limit 时返回 JPEG_QUALITY；
* limit 非法（≤0）时回退 JPEG_QUALITY。
*/
static float estimatedQuality(int actualBytes, int limitBytes) {
if (limitBytes <= 0 || actualBytes <= 0 || actualBytes <= limitBytes) {
return JPEG_QUALITY;
}
float ratio = (float) limitBytes / (float) actualBytes;
return Math.max(MIN_JPEG_QUALITY, Math.min(JPEG_QUALITY, JPEG_QUALITY * ratio));
}
/**
* Task 17：解码前尺寸校验。
* 尺寸非法或源像素超 MAX_SOURCE_PIXELS 抛 "image too large"；
* 子采样后解码像素仍超 MAX_DECODED_PIXELS 抛 "decode too large"。
* 供 decodeForResize 在 reader 支持子采样时按源尺寸校验、忽略采样参数时按全量解码校验。
*/
static void validateSourceImage(String sourceUrl, int sourceWidth, int sourceHeight, int subsampling) {
if (sourceWidth <= 0 || sourceHeight <= 0) {
throw new ResizeException("invalid image dimensions url=" + sourceUrl
+ " w=" + sourceWidth + " h=" + sourceHeight);
}
long sourcePixels = (long) sourceWidth * (long) sourceHeight;
if (sourcePixels > MAX_SOURCE_PIXELS) {
throw new ResizeException("image too large url=" + sourceUrl
+ " pixels=" + sourcePixels + " max=" + MAX_SOURCE_PIXELS);
}
if (subsampling <= 1 && sourcePixels > MAX_DECODED_PIXELS) {
throw new ResizeException("decode too large url=" + sourceUrl
+ " pixels=" + sourcePixels + " max=" + MAX_DECODED_PIXELS);
}
}
private static void ensureImageWorkNotInterrupted() throws InterruptedIOException {
if (Thread.currentThread().isInterrupted()) {
throw new InterruptedIOException("image work interrupted");
backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/util/SimilarAsinImageEmbedderDecodeQualityTest.java
+196
View File
@@ -0,0 +1,196 @@
package com.nanri.aiimage.modules.similarasin.util;
import com.nanri.aiimage.config.SimilarAsinProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder.ResizeException;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder.ResizedImage;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import javax.imageio.ImageIO;
import java.awt.Color;
import java.awt.Graphics2D;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.util.Arrays;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
/**
* Task 17：优化图片解码采样、像素上限和 JPEG 质量搜索，降低 CPU 与堆峰值。
* - 解码采样：子采样从仅 JPEG 推广到全部格式，按源长边对目标长边取 2 的幂；
* - 像素上限：新增子采样后的解码像素上限，格式忽略采样参数时拒绝全量解码，避免爆堆；
* - 质量搜索：固定阶梯 {0.75,0.65,0.55} 改为估算搜索（0.75 后按字节比例估算质量，
* 最多每长边 2 次编码），最坏编码次数 9 → 6，典型 1-2 次即命中。
*/
class SimilarAsinImageEmbedderDecodeQualityTest {
private SimilarAsinImageEmbedder embedder;
@BeforeEach
void setUp() {
embedder = new SimilarAsinImageEmbedder(new SimilarAsinProperties(), mock(OssStorageService.class));
}
@AfterEach
void shutdown() {
embedder.shutdown();
}
private static byte[] createImage(int width, int height, String format, Color color) throws Exception {
BufferedImage img = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
Graphics2D g = img.createGraphics();
try {
g.setColor(color);
g.fillRect(0, 0, width, height);
} finally {
g.dispose();
}
ByteArrayOutputStream baos = new ByteArrayOutputStream();
ImageIO.write(img, format, baos);
return baos.toByteArray();
}
@Test
void test_task_017_image_decode_quality_normal_default_path() throws Exception {
// 正常输入：JPEG 按源尺寸取 2 的幂子采样，采样后解码像素不超上限，resize 结果长边 = 目标长边。
int subsampling = SimilarAsinImageEmbedder.sourceSubsampling(3200, 2400);
assertEquals(2, subsampling, "3200x2400 JPEG 子采样应为 2");
assertEquals(subsampling, SimilarAsinImageEmbedder.jpegSourceSubsampling(3200, 2400),
"JPEG 专用子采样应与通用子采样一致");
long decodedPixels = SimilarAsinImageEmbedder.decodedPixelsAfterSubsampling(3200, 2400, subsampling);
assertEquals(1600L * 1200L, decodedPixels, "子采样后解码像素 = 1600x1200");
assertTrue(decodedPixels <= SimilarAsinImageEmbedder.MAX_DECODED_PIXELS, "解码像素不得超上限");
ResizedImage thumb = embedder.resizeImage("https://example.com/default.jpg",
createImage(1200, 1600, "jpg", new Color(0x33, 0x66, 0x99)));
assertEquals(SimilarAsinImageEmbedder.TARGET_LONG_EDGE_PX, Math.max(thumb.width(), thumb.height()),
"长边应缩放到目标 1280");
assertTrue(thumb.bytes().length <= SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES, "字节不得超上限");
}
@Test
void test_task_017_image_decode_quality_normal_multiple_items() throws Exception {
// 批量/多格式：非 JPEG（PNG）同样按源尺寸子采样，采样后解码像素受控，resize 结果不丢失。
int subsampling = SimilarAsinImageEmbedder.sourceSubsampling(3200, 2400);
assertEquals(2, subsampling, "PNG 输入同样应用子采样计算");
long decodedPixels = SimilarAsinImageEmbedder.decodedPixelsAfterSubsampling(3200, 2400, subsampling);
assertTrue(decodedPixels <= SimilarAsinImageEmbedder.MAX_DECODED_PIXELS, "PNG 解码像素不得超上限");
ResizedImage png = embedder.resizeImage("https://example.com/multi.png",
createImage(3200, 2400, "png", new Color(0x99, 0x33, 0x66)));
assertEquals(SimilarAsinImageEmbedder.TARGET_LONG_EDGE_PX, Math.max(png.width(), png.height()));
assertTrue(png.bytes().length <= SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES);
}
@Test
void test_task_017_image_decode_quality_normal_repeated_operation_is_idempotent() throws Exception {
// 重复执行：同一输入两次 resize 字节一致；质量估算纯函数输出确定。
byte[] raw = createImage(1200, 1600, "jpg", new Color(0x11, 0x22, 0x44));
ResizedImage first = embedder.resizeImage("https://example.com/idem.jpg", raw);
ResizedImage second = embedder.resizeImage("https://example.com/idem.jpg", raw);
assertArrayEquals(first.bytes(), second.bytes(), "重复 resize 必须产生相同字节");
assertEquals(first.width(), second.width());
assertEquals(first.height(), second.height());
assertEquals(SimilarAsinImageEmbedder.JPEG_QUALITY,
SimilarAsinImageEmbedder.estimatedQuality(120000, SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES),
"估算质量不得超过 0.75 上限");
assertEquals(0.576f,
SimilarAsinImageEmbedder.estimatedQuality(200000, SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES),
0.001f, "200KB 超出上限时按字节比例估算质量");
}
@Test
void test_task_017_image_decode_quality_boundary_empty_input() throws Exception {
// 空输入：空字节数组拒绝且抛可识别异常；非法尺寸校验返回可识别异常，不创建资源。
byte[] empty = new byte[0];
Exception ex = assertThrows(Exception.class,
() -> embedder.resizeImage("https://example.com/empty.jpg", empty));
assertTrue(ex instanceof java.io.IOException || ex instanceof RuntimeException,
"应为 IOException 或 RuntimeException 兜底，实际=" + ex.getClass().getSimpleName());
assertTrue(ex.getMessage() == null || ex.getMessage().toLowerCase().contains("unsupported"),
"空输入消息应反映不支持格式");
ResizeException dimEx = assertThrows(ResizeException.class,
() -> SimilarAsinImageEmbedder.validateSourceImage("https://example.com/empty.jpg", 0, 100, 1));
assertTrue(dimEx.getMessage().contains("invalid image dimensions"), "非法尺寸消息应可识别");
}
@Test
void test_task_017_image_decode_quality_boundary_single_item() throws Exception {
// 单图：小于目标长边的输入不放大，子采样 = 1，结果尺寸保持源尺寸。
ResizedImage thumb = embedder.resizeImage("https://example.com/single.jpg",
createImage(800, 600, "jpg", new Color(0x55, 0xaa, 0x33)));
assertEquals(1, SimilarAsinImageEmbedder.sourceSubsampling(800, 600), "小图子采样应为 1");
assertEquals(800, thumb.width(), "小图长边保持源尺寸，不放大");
assertEquals(600, thumb.height());
assertTrue(thumb.bytes().length <= SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES);
}
@Test
void test_task_017_image_decode_quality_boundary_limit_and_overflow() throws Exception {
// 上限/超限：源像素超上限拒绝；子采样后解码像素超上限拒绝；正常值放行。
ResizeException sourceEx = assertThrows(ResizeException.class,
() -> SimilarAsinImageEmbedder.validateSourceImage(
"https://example.com/huge.jpg", 7000, 7000, 1));
assertTrue(sourceEx.getMessage().contains("image too large"), "源像素超限消息应可识别");
SimilarAsinImageEmbedder.validateSourceImage("https://example.com/ok.jpg", 6000, 6000, 4);
SimilarAsinImageEmbedder.validateSourceImage("https://example.com/ok.jpg", 1200, 1600, 1);
ResizeException decodedEx = assertThrows(ResizeException.class,
() -> SimilarAsinImageEmbedder.validateSourceImage(
"https://example.com/no-subsample.jpg", 3000, 3000, 1));
assertTrue(decodedEx.getMessage().contains("decode too large"), "解码像素超限消息应可识别");
SimilarAsinImageEmbedder.validateSourceImage("https://example.com/subsampled.jpg", 3000, 3000, 2);
ResizedImage normal = embedder.resizeImage("https://example.com/normal.jpg",
createImage(1500, 1500, "jpg", new Color(0x20, 0x40, 0x60)));
assertEquals(SimilarAsinImageEmbedder.TARGET_LONG_EDGE_PX, Math.max(normal.width(), normal.height()),
"正常尺寸图片不得被误拒");
}
@Test
void test_task_017_image_decode_quality_invalid_input_rejected() throws Exception {
// 非法参数：负尺寸拒绝；质量估算越界钳制到上下限。
ResizeException negEx = assertThrows(ResizeException.class,
() -> SimilarAsinImageEmbedder.validateSourceImage("https://example.com/neg.jpg", -1, 100, 1));
assertTrue(negEx.getMessage().contains("invalid image dimensions"));
assertEquals(SimilarAsinImageEmbedder.MIN_JPEG_QUALITY,
SimilarAsinImageEmbedder.estimatedQuality(300000, SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES),
"估算质量低于下限时钳制到 MIN_JPEG_QUALITY");
assertEquals(SimilarAsinImageEmbedder.JPEG_QUALITY,
SimilarAsinImageEmbedder.estimatedQuality(0, SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES),
"非法字节数回退默认质量");
assertEquals(SimilarAsinImageEmbedder.JPEG_QUALITY,
SimilarAsinImageEmbedder.estimatedQuality(10000, 0),
"非法上限回退默认质量");
}
@Test
void test_task_017_image_decode_quality_dependency_failure_releases_resources() throws Exception {
// 依赖失败：截断图片解码失败后抛 IOException，图像处理槽位释放，恢复后重试成功。
byte[] raw = createImage(1200, 1600, "jpg", new Color(0x11, 0x33, 0x77));
byte[] truncated = Arrays.copyOf(raw, 64);
Exception ex = assertThrows(Exception.class,
() -> embedder.resizeImage("https://example.com/truncated.jpg", truncated));
assertTrue(ex instanceof java.io.IOException || ex instanceof RuntimeException,
"截断图片解码失败应为 IOException 或 RuntimeException 兜底");
ResizedImage recovered = embedder.resizeImage("https://example.com/recovered.jpg", raw);
assertNotNull(recovered, "失败后槽位必须释放，恢复重试成功");
assertEquals(SimilarAsinImageEmbedder.TARGET_LONG_EDGE_PX, Math.max(recovered.width(), recovered.height()));
assertTrue(recovered.bytes().length <= SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES);
}
}