super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

task-284(收尾/在途批次): 页头按钮归位卡内toolbar+店铺数据记录改名+弹窗报错清除; Java page_size配对/按角色数据范围过滤/撞款导出XLSX/Excel图改WPS DISPIMG/V110查询索引

Browse Source
This commit is contained in:
黄自达
2026-09-06 20:17:14 +08:00
parent 898fae4cb3
commit 26e062d32c
24 changed files with 269 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/productcategory/controller/ProductCategoryController.java
+3 -3
View File
@@ -52,9 +52,9 @@ public class ProductCategoryController {
@Operation(summary = "分页查询指定父级下的商品类目")
public ApiResponse<ProductCategoryListVo> children(
HttpServletRequest request,
@RequestParam(value = "parentId", required = false) Long parentId,
@RequestParam(value = "parent_id", required = false) Long parentId,
@RequestParam(value = "page", defaultValue = "1") Long page,
@RequestParam(value = "pageSize", defaultValue = "20") Long pageSize) {
@RequestParam(value = "page_size", defaultValue = "20") Long pageSize) {
adminAuthSupport.requireAdminOrInternal(request);
return ApiResponse.success(productCategoryService.children(parentId, page, pageSize));
}
@@ -65,7 +65,7 @@ public class ProductCategoryController {
HttpServletRequest request,
@RequestParam(value = "keyword", required = false) String keyword,
@RequestParam(value = "page", defaultValue = "1") Long page,
@RequestParam(value = "pageSize", defaultValue = "20") Long pageSize) {
@RequestParam(value = "page_size", defaultValue = "20") Long pageSize) {
adminAuthSupport.requireAdminOrInternal(request);
return ApiResponse.success(productCategoryService.search(keyword, page, pageSize));
}
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/controller/AdminShopDataCrawlTasksController.java
+22 -3
View File
@@ -7,6 +7,7 @@ import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
import com.nanri.aiimage.modules.permission.model.entity.AdminUserEntity;
import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlDownloadRowDto;
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDuplicateCheckSourceMapper;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlAdminTasksService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
@@ -38,6 +39,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
@@ -75,6 +77,7 @@ public class AdminShopDataCrawlTasksController {
private final AdminAuthSupport adminAuthSupport;
private final PermissionMenuService permissionMenuService;
private final ShopDataCrawlAdminTasksService adminTasksService;
private final ShopDuplicateCheckSourceMapper duplicateCheckSourceMapper;
@GetMapping
@Operation(summary = "店铺数据抓取记录：按店铺分组分页列表（每店最新结果文件）")
@@ -87,12 +90,14 @@ public class AdminShopDataCrawlTasksController {
@RequestParam(name = "country", required = false) String country,
@RequestParam(name = "created_from", required = false) String createdFrom,
@RequestParam(name = "created_to", required = false) String createdTo) {
requireShopDataCrawlTaskAccess(request);
AdminUserEntity operator = requireShopDataCrawlTaskAccess(request);
int safePage = Math.max(1, parseInt(page, 1, "page 参数不合法"));
int safePageSize = Math.min(PAGE_SIZE_MAX,
Math.max(10, parseInt(pageSize, PAGE_SIZE_DEFAULT, "page_size 参数不合法")));
boolean superAdmin = "super_admin".equals(adminAuthSupport.currentRole(operator));
List<String> visibleShopKeys = superAdmin ? null : resolveVisibleShopNameKeys(operator.getId());
Map<String, Object> payload = adminTasksService.listShopGroups(
safePage, safePageSize, shopName, groupName, country, createdFrom, createdTo);
safePage, safePageSize, shopName, groupName, country, createdFrom, createdTo, visibleShopKeys);
return ApiResponse.success(payload);
}
@@ -235,7 +240,7 @@ public class AdminShopDataCrawlTasksController {
return new ArrayList<>(ids);
}
private void requireShopDataCrawlTaskAccess(HttpServletRequest request) {
private AdminUserEntity requireShopDataCrawlTaskAccess(HttpServletRequest request) {
AdminUserEntity operator;
try {
operator = adminAuthSupport.requireAdmin(request);
@@ -246,6 +251,20 @@ public class AdminShopDataCrawlTasksController {
}
}
permissionMenuService.requireShopDataCrawlTaskAccess(operator);
return operator;
}
/** 非超管返回其可管店铺名（trim+小写去重，与撞款可见口径一致）；超管调用方传 null 表示全量。 */
private List<String> resolveVisibleShopNameKeys(Long operatorId) {
List<String> names = duplicateCheckSourceMapper.selectManagedShopNames(operatorId);
if (names == null || names.isEmpty()) {
return List.of();
}
return names.stream()
.filter(name -> name != null && !name.isBlank())
.map(name -> name.trim().toLowerCase(Locale.ROOT))
.distinct()
.toList();
}
private AdminUserEntity resolveInternalOperator(HttpServletRequest request) {
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/mapper/ShopDataCrawlAdminTasksMapper.java
+9
View File
@@ -163,6 +163,15 @@ public interface ShopDataCrawlAdminTasksMapper {
if (p.get("createdTo") != null) {
sql.append(" AND t.created_at <= #{createdTo}");
}
Object visibleShopKeys = p.get("visibleShopKeys");
if (visibleShopKeys instanceof List<?> keys) {
if (keys.isEmpty()) {
sql.append(" AND 1 = 0");
} else {
sql.append(" AND LOWER(TRIM(COALESCE(r.source_filename, ''))) IN (")
.append(inPlaceholders(p, "visibleShopKeys", "lv")).append(')');
}
}
return sql.toString();
}
backend-java/src/main/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlAdminTasksService.java
+6 -1
View File
@@ -44,7 +44,8 @@ public class ShopDataCrawlAdminTasksService {
* 分页列表组装（组总数 → 当前页店铺组 → 每店最新结果行 + 文件任务/分组名补充）。
*/
public Map<String, Object> listShopGroups(int page, int pageSize, String shopName, String groupName,
String country, String createdFromRaw, String createdToRaw) {
String country, String createdFromRaw, String createdToRaw,
List<String> visibleShopKeys) {
LocalDateTime createdFrom = parseAdminDateTime("created_from", createdFromRaw);
LocalDateTime createdTo = parseAdminDateTime("created_to", createdToRaw);
String countryFilter = blankToNull(country);
@@ -60,6 +61,10 @@ public class ShopDataCrawlAdminTasksService {
p.put("country", countryFilter);
p.put("createdFrom", createdFrom);
p.put("createdTo", createdTo);
if (visibleShopKeys != null) {
p.put("visibleShopKeys", visibleShopKeys);
flattenList(p, visibleShopKeys, "visibleShopKeys", "lv");
}
long total = adminTasksMapper.countShopGroups(p);
List<Map<String, Object>> items = new ArrayList<>();
backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/controller/AdminShopDataDuplicateCheckController.java
+4 -4
View File
@@ -202,12 +202,12 @@ public class AdminShopDataDuplicateCheckController {
ShopDataDuplicateCheckQueryService.Filters.clean(asin, shopName, country, site, dateFrom, dateTo);
Set<String> visibleKeys = queryService.resolveVisibleShopKeys(operator.id(), operator.superAdmin());
try {
response.setContentType("text/csv; charset=utf-8");
DownloadHeaderUtil.setAttachment(response, "shop-data-duplicate-check.csv");
response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
DownloadHeaderUtil.setAttachment(response, "shop-data-duplicate-check.xlsx");
response.setHeader("Cache-Control", "no-store");
queryService.writeExport(scan, visibleKeys, viewMode, filters, response.getOutputStream());
queryService.writeExportXlsx(scan, visibleKeys, viewMode, filters, response.getOutputStream());
} catch (Exception ex) {
log.warn("[shop-duplicate-check] 导出 CSV 失败: {}", ex.getMessage());
log.warn("[shop-duplicate-check] 导出 Excel 失败: {}", ex.getMessage());
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "导出失败");
}
}
backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/DuplicateCheckXlsxWriter.java
+38
View File
@@ -0,0 +1,38 @@
package com.nanri.aiimage.modules.shopduplicatecheck.service;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import java.io.IOException;
import java.io.OutputStream;
import java.util.List;
/** 撞款导出 .xlsx 写出（逐行 = 一条上架记录，首行为表头）。 */
public final class DuplicateCheckXlsxWriter {
private DuplicateCheckXlsxWriter() {
}
public static void write(OutputStream out, List<List<String>> rows) throws IOException {
try (SXSSFWorkbook workbook = new SXSSFWorkbook(100)) {
Sheet sheet = workbook.createSheet("店铺数据");
int rowCount = rows == null ? 0 : rows.size();
for (int i = 0; i < rowCount; i++) {
List<String> row = rows.get(i);
Row xlsRow = sheet.createRow(i);
int colCount = row == null ? 0 : row.size();
for (int c = 0; c < colCount; c++) {
String value = row.get(c);
xlsRow.createCell(c).setCellValue(value == null ? "" : value);
}
}
int cols = rows == null || rows.isEmpty() ? 0 : rows.get(0).size();
for (int c = 0; c < cols; c++) {
sheet.setColumnWidth(c, 14 * 256);
}
workbook.write(out);
workbook.dispose();
}
}
}
backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateCheckQueryService.java
+13 -1
View File
@@ -395,6 +395,18 @@ public class ShopDataDuplicateCheckQueryService {
public void writeExport(DuplicateScanView view, Set<String> visibleKeys,
String viewMode, Filters filters, OutputStream out) throws IOException {
DuplicateCheckCsvWriter.write(out, buildExportRows(view, visibleKeys, viewMode, filters));
}
/** 撞款/台账导出为 .xlsx（后台「导出」按钮下载）。 */
public void writeExportXlsx(DuplicateScanView view, Set<String> visibleKeys,
String viewMode, Filters filters, OutputStream out) throws IOException {
DuplicateCheckXlsxWriter.write(out, buildExportRows(view, visibleKeys, viewMode, filters));
}
/** 组装导出行：首行表头 + 逐条上架记录（撞款 view=monitor 过滤同店≥2，台账 view=all 全量）。 */
private List<List<String>> buildExportRows(DuplicateScanView view, Set<String> visibleKeys,
String viewMode, Filters filters) {
boolean monitor = "monitor".equals(viewMode);
List<DuplicateItem> cropped = cropItems(view.payload().items(), visibleKeys);
List<DuplicateItem> matched = filterItems(cropped, monitor, filters);
@@ -412,7 +424,7 @@ public class ShopDataDuplicateCheckQueryService {
for (Object[] row : flat) {
rows.add(toStringRow(row));
}
DuplicateCheckCsvWriter.write(out, rows);
return rows;
}
public List<DuplicateItem> filterItems(List<DuplicateItem> items, boolean monitor, Filters f) {
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java
+9 -10
View File
@@ -41,7 +41,7 @@ import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinExcelPar
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinGroupingConverter;
import com.nanri.aiimage.modules.similarasin.service.support.SimilarAsinHistoryAssembler;
import com.nanri.aiimage.modules.similarasin.util.BoundedImageCache;
import com.nanri.aiimage.modules.similarasin.util.ExcelCellImageWriter;
import com.nanri.aiimage.modules.similarasin.util.WpsCellImageWriter;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinLogSupport;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
@@ -4539,9 +4539,8 @@ public class SimilarAsinTaskService {
Map<String, SimilarAsinResultRowDto> resultMap,
Map<String, SimilarAsinImageEmbedder.ResizedImage> taskImageCache) {
ensureResultAssemblyNotInterrupted();
// Excel 365 "Place in Cell" 图片：写完 workbook 后 patch richData 单元格图片结构。
// 这不是浮动 Drawing，因此点击图片区域会选中单元格，图片不能被拖到任意位置，也不需要工作表保护。
ExcelCellImageWriter.Session excelCellImageSession = ExcelCellImageWriter.createSession();
// 单元格内嵌图片：写完 workbook 后 patch WPS DISPIMG 结构（WPS/钉钉/飞书/Excel365 均可显示）。
WpsCellImageWriter.Session cellImageSession = WpsCellImageWriter.createSession();
SimilarAsinImageEmbedder.ImageSpool imageSpool;
try {
Path spoolDir = Files.createTempDirectory(xlsx.getParentFile().toPath(), "similar-asin-images-");
@@ -4642,11 +4641,11 @@ public class SimilarAsinTaskService {
// 行高固定到 IMAGE_ROW_HEIGHT_POINTS（Excel 上限 409pt）：
row.setHeightInPoints(SimilarAsinImageEmbedder.IMAGE_ROW_HEIGHT_POINTS);
imageEmbedder.embedAsExcelCellImage(rowIndex, IMG_COL_MAIN,
resultRow.getMainUrl(), row, taskImageCache, excelCellImageSession, imageSpool);
resultRow.getMainUrl(), row, taskImageCache, cellImageSession, imageSpool);
imageEmbedder.embedAsExcelCellImage(rowIndex, IMG_COL_PUZZLE1,
resultRow.getPuzzleImg1(), row, taskImageCache, excelCellImageSession, imageSpool);
resultRow.getPuzzleImg1(), row, taskImageCache, cellImageSession, imageSpool);
imageEmbedder.embedAsExcelCellImage(rowIndex, IMG_COL_PUZZLE2,
resultRow.getPuzzleImg2(), row, taskImageCache, excelCellImageSession, imageSpool);
resultRow.getPuzzleImg2(), row, taskImageCache, cellImageSession, imageSpool);
}
rowIndex++;
}
@@ -4654,14 +4653,14 @@ public class SimilarAsinTaskService {
workbook.write(fos);
workbook.dispose();
}
if (!excelCellImageSession.isEmpty()) {
if (!cellImageSession.isEmpty()) {
try {
ensureResultAssemblyNotInterrupted();
ExcelCellImageWriter.patchXlsxFile(xlsx, excelCellImageSession);
WpsCellImageWriter.patchXlsxFile(xlsx, cellImageSession);
} catch (IOException ex) {
String cause = ex.toString();
log.error("[similar-asin] excel cell image patch failed xlsx={} images={} err={}",
xlsx.getAbsolutePath(), excelCellImageSession.size(), cause, ex);
xlsx.getAbsolutePath(), cellImageSession.size(), cause, ex);
throw new BusinessException("写入相似ASIN结果 Excel 单元格图片失败: " + cause, ex);
}
}
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/util/SimilarAsinImageEmbedder.java
+23 -33
View File
@@ -569,8 +569,6 @@ public class SimilarAsinImageEmbedder {
* 失败时返回 null；成功返回缩略图。给 Excel richData 单元格图片写入路径使用。
*/
private ResizedImage downloadAndResize(String trimmedUrl,
int colIdx,
Row row,
Map<String, ResizedImage> taskImageCache) {
long startedNanos = System.nanoTime();
try {
@@ -605,7 +603,6 @@ public class SimilarAsinImageEmbedder {
log.warn("[similar-asin][image] embed-fail url={} elapsedMs={} err={}",
trimmedUrl, elapsedMs(startedNanos), errorSummary(ex));
}
row.createCell(colIdx).setCellValue(trimmedUrl);
return null;
}
@@ -614,7 +611,7 @@ public class SimilarAsinImageEmbedder {
String url,
Row row,
Map<String, ResizedImage> taskImageCache,
ExcelCellImageWriter.Session excelCellImageSession) {
WpsCellImageWriter.Session excelCellImageSession) {
return embedAsExcelCellImage(rowIdx, colIdx, url, row, taskImageCache, excelCellImageSession, null);
}
@@ -623,46 +620,39 @@ public class SimilarAsinImageEmbedder {
String url,
Row row,
Map<String, ResizedImage> taskImageCache,
ExcelCellImageWriter.Session excelCellImageSession,
WpsCellImageWriter.Session excelCellImageSession,
ImageSpool imageSpool) {
if (url == null || url.isBlank() || excelCellImageSession == null) {
return null;
}
String trimmedUrl = url.trim();
SpoolImage spooled = imageSpool == null ? null : imageSpool.get(trimmedUrl);
String imageId = null;
ImageDim dim = null;
if (spooled != null) {
Cell cell = row.createCell(colIdx);
cell.setCellValue("#VALUE!");
excelCellImageSession.registerImage(rowIdx, colIdx, spooled.path());
return new ImageDim(spooled.width(), spooled.height());
}
if (imageSpool != null) {
row.createCell(colIdx).setCellValue(trimmedUrl);
// 预取已落盘：按路径注册，patch 阶段再读文件，避免整份图片字节驻留堆内存。
imageId = excelCellImageSession.registerImageByPath(spooled.path());
dim = new ImageDim(spooled.width(), spooled.height());
} else if (imageSpool != null) {
// 预取未命中说明该 URL 下载失败：不重试、不把 URL 文本写进单元格，留空并打日志。
log.warn("[similar-asin][image] cell-image-miss url={} colIdx={} (prefetch did not spool), leave blank",
trimmedUrl, colIdx);
return null;
}
ResizedImage thumb = downloadAndResize(trimmedUrl, colIdx, row, taskImageCache);
if (thumb == null) {
return null;
}
try {
Cell cell = row.createCell(colIdx);
cell.setCellValue("#VALUE!");
if (imageSpool == null) {
excelCellImageSession.registerImage(rowIdx, colIdx, thumb.bytes());
} else {
spooled = imageSpool.put(trimmedUrl, thumb);
excelCellImageSession.registerImage(rowIdx, colIdx, spooled.path());
} else {
ResizedImage thumb = downloadAndResize(trimmedUrl, taskImageCache);
if (thumb == null) {
log.warn("[similar-asin][image] cell-image-miss url={} colIdx={} (download failed), leave blank",
trimmedUrl, colIdx);
return null;
}
imageId = excelCellImageSession.registerImage(thumb.bytes());
taskImageCache.remove(trimmedUrl);
return spooled == null
? new ImageDim(thumb.width(), thumb.height())
: new ImageDim(spooled.width(), spooled.height());
} catch (IOException | RuntimeException ex) {
taskImageCache.remove(trimmedUrl);
log.warn("[similar-asin][image] excel-cell-image-fail url={} err={}", trimmedUrl, ex.getMessage());
row.createCell(colIdx).setCellValue(trimmedUrl);
return null;
dim = new ImageDim(thumb.width(), thumb.height());
}
if (imageId != null) {
row.createCell(colIdx).setCellFormula(WpsCellImageWriter.dispImgFormula(imageId));
}
return dim;
}
private boolean hasLocalCachedThumb(String url) {
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/util/WpsCellImageWriter.java
+31 -1
View File
@@ -136,7 +136,7 @@ public final class WpsCellImageWriter {
buildCellImagesRelsXml(session).getBytes(StandardCharsets.UTF_8));
int idx = 1;
for (RegisteredImage img : session.images) {
writeEntry(zout, "xl/media/cellimage" + idx + ".jpeg", img.bytes);
writeEntry(zout, "xl/media/cellimage" + idx + ".jpeg", img.bytes());
idx++;
}
}
@@ -299,6 +299,8 @@ public final class WpsCellImageWriter {
final List<RegisteredImage> images = new ArrayList<>();
// 字节级去重：相同图片只写一次 cellimage{N}.jpeg，节省 xlsx 体积。
private final Map<String, String> bytesHashToId = new LinkedHashMap<>();
// 磁盘路径去重：同一任务内相同源文件（预取 spool 同一份）只写一次 media，避免内存/体积膨胀。
private final Map<String, String> pathHashToId = new LinkedHashMap<>();
public synchronized String registerImage(byte[] jpegBytes) {
if (jpegBytes == null || jpegBytes.length == 0) {
@@ -315,6 +317,22 @@ public final class WpsCellImageWriter {
return id;
}
/** 按磁盘路径注册（预取 spool 已落盘，避免整份字节驻留内存）；按绝对路径去重，重复路径复用同一 media。 */
public synchronized String registerImageByPath(Path path) {
if (path == null) {
return null;
}
String normalized = path.toAbsolutePath().normalize().toString();
String existingId = pathHashToId.get(normalized);
if (existingId != null) {
return existingId;
}
String id = newImageId();
images.add(new RegisteredImage(id, path));
pathHashToId.put(normalized, id);
return id;
}
public synchronized boolean isEmpty() {
return images.isEmpty();
}
@@ -347,10 +365,22 @@ public final class WpsCellImageWriter {
static final class RegisteredImage {
final String id;
final byte[] bytes;
final Path sourcePath;
RegisteredImage(String id, byte[] bytes) {
this.id = id;
this.bytes = bytes;
this.sourcePath = null;
}
RegisteredImage(String id, Path sourcePath) {
this.id = id;
this.bytes = null;
this.sourcePath = sourcePath;
}
byte[] bytes() throws IOException {
return sourcePath != null ? Files.readAllBytes(sourcePath) : bytes;
}
}
}
backend-java/src/main/resources/db/V110__image_history_query_indexes.sql
+4
View File
@@ -0,0 +1,4 @@
-- image_history 后台分页查询提速：按创建时间倒序取 id + 按用户过滤
ALTER TABLE image_history
ADD INDEX idx_image_history_created_at (created_at),
ADD INDEX idx_image_history_user_id (user_id);
backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/util/SimilarAsinImageEmbedderTest.java
+2 -2
View File
@@ -229,12 +229,12 @@ class SimilarAsinImageEmbedderTest {
var row = workbook.createSheet().createRow(1);
SimilarAsinImageEmbedder.ImageDim result = failedEmbedder.embedAsExcelCellImage(
1, 9, url, row, new HashMap<>(), ExcelCellImageWriter.createSession(), spool);
1, 9, url, row, new HashMap<>(), WpsCellImageWriter.createSession(), spool);
assertEquals(SimilarAsinImageEmbedder.DOWNLOAD_MAX_RETRY + 1, callsAfterPrefetch);
assertEquals(callsAfterPrefetch, networkCalls.get());
assertNull(result);
assertEquals(url, row.getCell(9).getStringCellValue());
assertNull(row.getCell(9), "预取失败不应把 URL 文本写进单元格，应留空");
} finally {
failedEmbedder.shutdown();
}