diff --git a/app/.env b/app/.env index 152d342..ef77969 100644 --- a/app/.env +++ b/app/.env @@ -11,8 +11,8 @@ zn_username=%E8%87%AA%E5%8A%A8%E5%8C%96_Robot client_name=ShuFuAI -java_api_base=http://47.111.163.154:18080 -# java_api_base=http://127.0.0.1:18080 +# java_api_base=http://47.111.163.154:18080 +java_api_base=http://127.0.0.1:18080 # java_api_base=http://8.136.19.173:18080 diff --git a/app/blueprints/main.py b/app/blueprints/main.py index 6881dc0..0ab1d93 100644 --- a/app/blueprints/main.py +++ b/app/blueprints/main.py @@ -195,10 +195,13 @@ def proxy(path): if key.lower() in ['host', 'content-length', 'connection']: continue headers[key] = value - ignore_url = [f"{JAVA_API_BASE}/api/delete-brand/tasks/batch", - f"{JAVA_API_BASE}/api/delete-brand/history", - f"{JAVA_API_BASE}/api/product-risk-resolve/tasks/batch", - ] + ignore_url = [f"{JAVA_API_BASE}/api/delete-brand/tasks/batch", + f"{JAVA_API_BASE}/api/delete-brand/tasks/progress/batch", + f"{JAVA_API_BASE}/api/delete-brand/history", + f"{JAVA_API_BASE}/api/product-risk-resolve/tasks/batch", + f"{JAVA_API_BASE}/api/product-risk-resolve/tasks/progress/batch", + ] + proxy_timeout = 180 if target_url.endswith("/api/delete-brand/run") else 30 if target_url not in ignore_url: try: print("=============================") @@ -219,7 +222,7 @@ def proxy(path): data=request.get_data() if request.get_data() else None, cookies=request.cookies, stream=True, # 启用流式传输 - timeout=30 + timeout=proxy_timeout ) # 流式响应 diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/TransientStorageProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/TransientStorageProperties.java index 8458f3e..e819631 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/TransientStorageProperties.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/TransientStorageProperties.java @@ -12,4 +12,8 @@ public class TransientStorageProperties { private String accessKeyId; private String accessKeySecret; private String region = "us-east-1"; + private int connectTimeoutSeconds = 10; + private int readTimeoutSeconds = 60; + private int writeTimeoutSeconds = 60; + private int uploadMaxRetries = 3; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskStorageService.java index 64388ff..40a59cf 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskStorageService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskStorageService.java @@ -240,7 +240,10 @@ public class DeleteBrandTaskStorageService { DeleteBrandResultFileDto dto = objectMapper.readValue(payloadJson, DeleteBrandResultFileDto.class); grouped.computeIfAbsent(entity.getScopeKey(), ignored -> new ArrayList<>()).add(dto); } catch (Exception ex) { - throw new BusinessException("failed to load delete-brand result chunks"); + log.warn("[delete-brand-storage] failed to load result chunk taskId={} scopeKey={} chunkIndex={} payload={}", + taskId, entity.getScopeKey(), entity.getChunkIndex(), entity.getPayloadJson(), ex); + throw new BusinessException("failed to load delete-brand result chunks: taskId=" + + taskId + ", scopeKey=" + entity.getScopeKey() + ", chunkIndex=" + entity.getChunkIndex()); } } for (List chunks : grouped.values()) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/object/RustfsObjectStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/object/RustfsObjectStorageService.java index b2a9f07..8940235 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/object/RustfsObjectStorageService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/object/RustfsObjectStorageService.java @@ -5,18 +5,24 @@ import io.minio.GetObjectArgs; import io.minio.MinioClient; import io.minio.PutObjectArgs; import io.minio.RemoveObjectArgs; +import io.minio.StatObjectArgs; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import okhttp3.OkHttpClient; import org.springframework.stereotype.Service; import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; import java.util.Objects; +import java.util.concurrent.TimeUnit; @Service @RequiredArgsConstructor +@Slf4j public class RustfsObjectStorageService { private final TransientStorageProperties properties; + private volatile OkHttpClient httpClient; public boolean isConfigured() { return notBlank(properties.getEndpoint()) @@ -29,18 +35,28 @@ public class RustfsObjectStorageService { if (!isConfigured()) { throw new IllegalStateException("transient storage is not configured"); } - try (ByteArrayInputStream stream = new ByteArrayInputStream( - Objects.requireNonNullElse(content, "").getBytes(StandardCharsets.UTF_8))) { - buildClient().putObject(PutObjectArgs.builder() - .bucket(properties.getBucket()) - .object(objectKey) - .stream(stream, stream.available(), -1) - .contentType("application/json") - .build()); - return objectKey; - } catch (Exception ex) { - throw new IllegalStateException("failed to upload payload to transient storage", ex); + byte[] bytes = Objects.requireNonNullElse(content, "").getBytes(StandardCharsets.UTF_8); + Exception last = null; + int maxRetries = Math.max(1, properties.getUploadMaxRetries()); + for (int attempt = 1; attempt <= maxRetries; attempt++) { + try (ByteArrayInputStream stream = new ByteArrayInputStream(bytes)) { + buildClient().putObject(PutObjectArgs.builder() + .bucket(properties.getBucket()) + .object(objectKey) + .stream(stream, bytes.length, -1) + .contentType("application/json") + .build()); + verifyObjectVisible(objectKey); + return objectKey; + } catch (Exception ex) { + last = ex; + if (attempt < maxRetries) { + log.warn("[rustfs] upload failed, retrying objectKey={} attempt={}/{}", objectKey, attempt, maxRetries, ex); + sleepQuietly(500L * attempt); + } + } } + throw new IllegalStateException("failed to upload payload to transient storage", last); } public String readObjectAsString(String objectKey) { @@ -76,9 +92,57 @@ public class RustfsObjectStorageService { .endpoint(properties.getEndpoint()) .credentials(properties.getAccessKeyId(), properties.getAccessKeySecret()) .region(properties.getRegion()) + .httpClient(getHttpClient()) .build(); } + private OkHttpClient getHttpClient() { + OkHttpClient current = httpClient; + if (current != null) { + return current; + } + synchronized (this) { + if (httpClient == null) { + httpClient = new OkHttpClient.Builder() + .connectTimeout(Math.max(1, properties.getConnectTimeoutSeconds()), TimeUnit.SECONDS) + .readTimeout(Math.max(1, properties.getReadTimeoutSeconds()), TimeUnit.SECONDS) + .writeTimeout(Math.max(1, properties.getWriteTimeoutSeconds()), TimeUnit.SECONDS) + .build(); + } + return httpClient; + } + } + + private void verifyObjectVisible(String objectKey) { + RuntimeException last = null; + for (int attempt = 1; attempt <= 3; attempt++) { + try { + buildClient().statObject(StatObjectArgs.builder() + .bucket(properties.getBucket()) + .object(objectKey) + .build()); + return; + } catch (Exception ex) { + last = new IllegalStateException("transient payload is not visible after upload: " + objectKey, ex); + if (attempt < 3) { + sleepQuietly(100L * attempt); + } + } + } + throw last == null + ? new IllegalStateException("transient payload is not visible after upload: " + objectKey) + : last; + } + + private void sleepQuietly(long millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted while verifying transient payload upload", ex); + } + } + private boolean notBlank(String value) { return value != null && !value.isBlank(); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/controller/SkipPriceAsinController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/controller/SkipPriceAsinController.java index 8b9e088..65044b4 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/controller/SkipPriceAsinController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/controller/SkipPriceAsinController.java @@ -3,6 +3,8 @@ package com.nanri.aiimage.modules.shopkey.controller; import com.nanri.aiimage.common.api.ApiResponse; import com.nanri.aiimage.modules.shopkey.model.dto.SkipPriceAsinCountryUpdateRequest; import com.nanri.aiimage.modules.shopkey.model.dto.SkipPriceAsinCreateRequest; +import com.nanri.aiimage.modules.shopkey.model.vo.QueryAsinImportProgressVo; +import com.nanri.aiimage.modules.shopkey.model.vo.QueryAsinImportStartVo; import com.nanri.aiimage.modules.shopkey.model.vo.SkipPriceAsinItemVo; import com.nanri.aiimage.modules.shopkey.model.vo.SkipPriceAsinPageVo; import com.nanri.aiimage.modules.shopkey.service.SkipPriceAsinService; @@ -20,6 +22,7 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; @RestController @RequiredArgsConstructor @@ -59,6 +62,40 @@ public class SkipPriceAsinController { skipPriceAsinService.createOrUpdate(request, operatorId, Boolean.TRUE.equals(superAdmin))); } + @PostMapping("/import") + @Operation(summary = "导入新增跳过跟价 ASIN") + public ApiResponse importExcel( + @Parameter(description = "xlsx/xls 文件，文件名必须等于店铺名", required = true) @RequestParam("file") MultipartFile file, + @Parameter(description = "页面选择的分组ID", required = true) @RequestParam Long groupId, + @Parameter(description = "当前操作人用户ID") @RequestParam Long operatorId, + @Parameter(description = "是否超级管理员") @RequestParam(defaultValue = "false") Boolean superAdmin) { + return ApiResponse.success("开始导入", + skipPriceAsinService.startImport(file, groupId, operatorId, Boolean.TRUE.equals(superAdmin))); + } + + @GetMapping("/import/{importId}") + @Operation(summary = "查询导入新增进度") + public ApiResponse importProgress(@PathVariable String importId) { + return ApiResponse.success(skipPriceAsinService.getImportProgress(importId)); + } + + @PostMapping("/delete-import") + @Operation(summary = "导入删除跳过跟价 ASIN") + public ApiResponse deleteImportExcel( + @Parameter(description = "xlsx/xls 文件，文件名必须等于店铺名", required = true) @RequestParam("file") MultipartFile file, + @Parameter(description = "页面选择的分组ID", required = true) @RequestParam Long groupId, + @Parameter(description = "当前操作人用户ID") @RequestParam Long operatorId, + @Parameter(description = "是否超级管理员") @RequestParam(defaultValue = "false") Boolean superAdmin) { + return ApiResponse.success("开始删除", + skipPriceAsinService.startDeleteImport(file, groupId, operatorId, Boolean.TRUE.equals(superAdmin))); + } + + @GetMapping("/delete-import/{importId}") + @Operation(summary = "查询导入删除进度") + public ApiResponse deleteImportProgress(@PathVariable String importId) { + return ApiResponse.success(skipPriceAsinService.getDeleteImportProgress(importId)); + } + @PutMapping("/{id}/countries/{country}") @Operation(summary = "编辑指定国家的跳过跟价 ASIN") public ApiResponse updateCountry( diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/entity/SkipPriceAsinEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/entity/SkipPriceAsinEntity.java index 874cb71..cc2e5aa 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/entity/SkipPriceAsinEntity.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/entity/SkipPriceAsinEntity.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.modules.shopkey.model.entity; import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.FieldStrategy; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; @@ -25,31 +26,31 @@ public class SkipPriceAsinEntity { @TableField("asin_de") private String asinDe; - @TableField("minimum_price_de") + @TableField(value = "minimum_price_de", updateStrategy = FieldStrategy.ALWAYS) private BigDecimal minimumPriceDe; @TableField("asin_uk") private String asinUk; - @TableField("minimum_price_uk") + @TableField(value = "minimum_price_uk", updateStrategy = FieldStrategy.ALWAYS) private BigDecimal minimumPriceUk; @TableField("asin_fr") private String asinFr; - @TableField("minimum_price_fr") + @TableField(value = "minimum_price_fr", updateStrategy = FieldStrategy.ALWAYS) private BigDecimal minimumPriceFr; @TableField("asin_it") private String asinIt; - @TableField("minimum_price_it") + @TableField(value = "minimum_price_it", updateStrategy = FieldStrategy.ALWAYS) private BigDecimal minimumPriceIt; @TableField("asin_es") private String asinEs; - @TableField("minimum_price_es") + @TableField(value = "minimum_price_es", updateStrategy = FieldStrategy.ALWAYS) private BigDecimal minimumPriceEs; @TableField("created_at") diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/SkipPriceAsinService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/SkipPriceAsinService.java index 5499804..11c3bde 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/SkipPriceAsinService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/SkipPriceAsinService.java @@ -1,17 +1,33 @@ package com.nanri.aiimage.modules.shopkey.service; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import cn.hutool.core.util.IdUtil; import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.modules.shopkey.mapper.SkipPriceAsinMapper; +import com.nanri.aiimage.modules.shopkey.mapper.ShopManageMapper; import com.nanri.aiimage.modules.shopkey.model.dto.SkipPriceAsinCreateRequest; +import com.nanri.aiimage.modules.shopkey.model.entity.ShopManageEntity; import com.nanri.aiimage.modules.shopkey.model.entity.ShopManageGroupEntity; import com.nanri.aiimage.modules.shopkey.model.entity.SkipPriceAsinEntity; +import com.nanri.aiimage.modules.shopkey.model.vo.QueryAsinImportProgressVo; +import com.nanri.aiimage.modules.shopkey.model.vo.QueryAsinImportStartVo; import com.nanri.aiimage.modules.shopkey.model.vo.SkipPriceAsinItemVo; import com.nanri.aiimage.modules.shopkey.model.vo.SkipPriceAsinPageVo; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.DataFormatter; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.ss.usermodel.WorkbookFactory; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.multipart.MultipartFile; +import java.io.File; +import java.io.FileInputStream; +import java.nio.file.Files; import java.math.BigDecimal; import java.math.RoundingMode; import java.time.LocalDateTime; @@ -21,15 +37,20 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; @Service @RequiredArgsConstructor +@Slf4j public class SkipPriceAsinService { private static final List SUPPORTED_COUNTRIES = List.of("DE", "UK", "FR", "IT", "ES"); private final SkipPriceAsinMapper skipPriceAsinMapper; + private final ShopManageMapper shopManageMapper; private final ShopManageGroupService shopManageGroupService; + private final Map importProgressMap = new ConcurrentHashMap<>(); + private final Map deleteImportProgressMap = new ConcurrentHashMap<>(); public SkipPriceAsinPageVo page(long page, long pageSize, Long groupId, String shopName, String asin, Long operatorId, boolean superAdmin) { @@ -141,6 +162,486 @@ public class SkipPriceAsinService { return toItemVo(saved, groupName); } + public QueryAsinImportStartVo startImport(MultipartFile file, Long groupId, Long operatorId, boolean superAdmin) { + return startImportTask(file, groupId, operatorId, superAdmin, false); + } + + public QueryAsinImportProgressVo getImportProgress(String importId) { + QueryAsinImportProgressVo progress = importProgressMap.get(importId); + if (progress == null) { + throw new BusinessException("导入任务不存在"); + } + return progress; + } + + public QueryAsinImportStartVo startDeleteImport(MultipartFile file, Long groupId, Long operatorId, boolean superAdmin) { + return startImportTask(file, groupId, operatorId, superAdmin, true); + } + + public QueryAsinImportProgressVo getDeleteImportProgress(String importId) { + QueryAsinImportProgressVo progress = deleteImportProgressMap.get(importId); + if (progress == null) { + throw new BusinessException("删除任务不存在"); + } + return progress; + } + + private QueryAsinImportStartVo startImportTask(MultipartFile file, Long groupId, Long operatorId, + boolean superAdmin, boolean deleteMode) { + if (file == null || file.isEmpty()) { + throw new BusinessException("请上传 xlsx 或 xls 文件"); + } + validateExcelFilename(file.getOriginalFilename()); + if (groupId == null || groupId <= 0) { + throw new BusinessException("请先选择分组"); + } + shopManageGroupService.getAccessibleById(groupId, operatorId, superAdmin); + String shopName = normalizeShopNameFromFilename(file.getOriginalFilename()); + ensureShopExistsInGroup(groupId, shopName); + + String importId = IdUtil.fastSimpleUUID(); + QueryAsinImportProgressVo progress = newImportProgress(); + Map progressMap = deleteMode ? deleteImportProgressMap : importProgressMap; + progressMap.put(importId, progress); + try { + File tempFile = saveMultipartToTempFile(file, deleteMode ? "skip-price-asin-delete-" : "skip-price-asin-import-"); + Thread.ofVirtual().start(() -> runImportTask(importId, tempFile, file.getOriginalFilename(), groupId, + shopName, deleteMode)); + } catch (Exception ex) { + progressMap.remove(importId); + throw new BusinessException("读取上传文件失败"); + } + + QueryAsinImportStartVo vo = new QueryAsinImportStartVo(); + vo.setImportId(importId); + return vo; + } + + private QueryAsinImportProgressVo newImportProgress() { + QueryAsinImportProgressVo progress = new QueryAsinImportProgressVo(); + progress.setStatus("pending"); + progress.setTotalRows(0); + progress.setProcessedRows(0); + progress.setAsinCount(0); + progress.setInsertedCount(0); + progress.setDeletedCount(0); + progress.setSkippedCount(0); + return progress; + } + + private void runImportTask(String importId, File tempFile, String filename, Long groupId, + String shopName, boolean deleteMode) { + Map progressMap = deleteMode ? deleteImportProgressMap : importProgressMap; + QueryAsinImportProgressVo progress = progressMap.get(importId); + if (progress == null) { + deleteQuietly(tempFile); + return; + } + progress.setStatus("running"); + progress.setErrorMessage(null); + try { + processImportFile(tempFile, filename, groupId, shopName, deleteMode, progress); + progress.setStatus("success"); + } catch (Exception ex) { + log.warn("[skip-price-asin-import] failed importId={} deleteMode={} msg={}", + importId, deleteMode, ex.getMessage()); + progress.setStatus("failed"); + progress.setErrorMessage(ex instanceof BusinessException ? ex.getMessage() + : (deleteMode ? "导入删除 Excel 失败" : "导入新增 Excel 失败")); + } finally { + deleteQuietly(tempFile); + } + } + + private void processImportFile(File tempFile, String filename, Long groupId, String shopName, + boolean deleteMode, QueryAsinImportProgressVo progress) throws Exception { + validateExcelFilename(filename); + try (FileInputStream inputStream = new FileInputStream(tempFile); + Workbook workbook = WorkbookFactory.create(inputStream)) { + Sheet sheet = workbook.getNumberOfSheets() == 0 ? null : workbook.getSheetAt(0); + if (sheet == null) { + throw new BusinessException("Excel 为空"); + } + HeaderMapping mapping = resolveHeaderMapping(sheet); + if (mapping.asinColumns().isEmpty()) { + throw new BusinessException("Excel 至少需要包含一个国家的 ASIN 列"); + } + + int firstDataRow = mapping.firstDataRowIndex(); + int lastRow = Math.max(sheet.getLastRowNum(), firstDataRow - 1); + int totalRows = Math.max(0, lastRow - firstDataRow + 1); + progress.setTotalRows(totalRows); + + DataFormatter formatter = new DataFormatter(); + ImportCounters counters = new ImportCounters(); + for (int rowIndex = firstDataRow; rowIndex <= lastRow; rowIndex++) { + Row row = sheet.getRow(rowIndex); + if (row == null || isBlankRow(row, formatter)) { + counters.skippedCount++; + updateImportProgress(progress, counters, rowIndex - firstDataRow + 1); + continue; + } + if (deleteMode) { + consumeDeleteImportRow(row, formatter, mapping, groupId, shopName, counters); + } else { + consumeAddImportRow(row, formatter, mapping, groupId, shopName, counters); + } + updateImportProgress(progress, counters, rowIndex - firstDataRow + 1); + } + } + } + + private void consumeAddImportRow(Row row, DataFormatter formatter, HeaderMapping mapping, + Long groupId, String shopName, ImportCounters counters) { + boolean acceptedAny = false; + boolean sawAnyAsin = false; + SkipPriceAsinEntity entity = new SkipPriceAsinEntity(); + entity.setGroupId(groupId); + entity.setShopName(shopName); + for (Map.Entry entry : mapping.asinColumns().entrySet()) { + String country = entry.getKey(); + String asin = normalizeBlank(cellText(row, entry.getValue(), formatter)).toUpperCase(Locale.ROOT); + if (asin.isEmpty()) { + continue; + } + sawAnyAsin = true; + counters.asinCount++; + if (asin.length() > 64) { + counters.skippedCount++; + continue; + } + BigDecimal minimumPrice = parseOptionalMinimumPrice( + cellText(row, mapping.priceColumns().get(country), formatter)); + if (existsCountryAsin(groupId, shopName, country, asin)) { + counters.skippedCount++; + continue; + } + setCountryData(entity, country, asin, minimumPrice); + counters.insertedCount++; + acceptedAny = true; + } + if (!acceptedAny) { + if (!sawAnyAsin) { + counters.skippedCount++; + } + return; + } + skipPriceAsinMapper.insert(entity); + } + + private void consumeDeleteImportRow(Row row, DataFormatter formatter, HeaderMapping mapping, + Long groupId, String shopName, ImportCounters counters) { + boolean acceptedAny = false; + boolean sawAnyAsin = false; + for (Map.Entry entry : mapping.asinColumns().entrySet()) { + String country = entry.getKey(); + String asin = normalizeBlank(cellText(row, entry.getValue(), formatter)).toUpperCase(Locale.ROOT); + if (asin.isEmpty()) { + continue; + } + sawAnyAsin = true; + counters.asinCount++; + if (asin.length() > 64) { + counters.skippedCount++; + continue; + } + SkipPriceAsinEntity entity = findCountryAsin(groupId, shopName, country, asin); + if (entity == null) { + counters.skippedCount++; + continue; + } + setCountryData(entity, country, "", null); + if (isEmptyRow(entity)) { + skipPriceAsinMapper.deleteById(entity.getId()); + } else { + skipPriceAsinMapper.updateById(entity); + } + counters.deletedCount++; + acceptedAny = true; + } + if (!acceptedAny && !sawAnyAsin) { + counters.skippedCount++; + } + } + + private boolean existsCountryAsin(Long groupId, String shopName, String country, String asin) { + return findCountryAsin(groupId, shopName, country, asin) != null; + } + + private SkipPriceAsinEntity findCountryAsin(Long groupId, String shopName, String country, String asin) { + LambdaQueryWrapper query = new LambdaQueryWrapper() + .eq(SkipPriceAsinEntity::getGroupId, groupId) + .eq(SkipPriceAsinEntity::getShopName, shopName); + switch (country) { + case "DE" -> query.eq(SkipPriceAsinEntity::getAsinDe, asin); + case "UK" -> query.eq(SkipPriceAsinEntity::getAsinUk, asin); + case "FR" -> query.eq(SkipPriceAsinEntity::getAsinFr, asin); + case "IT" -> query.eq(SkipPriceAsinEntity::getAsinIt, asin); + case "ES" -> query.eq(SkipPriceAsinEntity::getAsinEs, asin); + default -> throw new BusinessException("鍥藉鍙傛暟涓嶆敮鎸? " + country); + } + return skipPriceAsinMapper.selectOne(query.last("LIMIT 1")); + } + + private HeaderMapping resolveHeaderMapping(Sheet sheet) { + DataFormatter formatter = new DataFormatter(); + Row firstHeader = sheet.getRow(0); + Row secondHeader = sheet.getRow(1); + if (firstHeader == null) { + throw new BusinessException("Excel 表头为空"); + } + + int maxColumn = Math.max(lastCellNum(firstHeader), lastCellNum(secondHeader)); + Map asinColumns = new LinkedHashMap<>(); + Map priceColumns = new LinkedHashMap<>(); + for (int columnIndex = 0; columnIndex < maxColumn; columnIndex++) { + String countryText = cellTextWithMerged(sheet, firstHeader, columnIndex, formatter); + String fieldText = cellText(secondHeader, columnIndex, formatter); + String country = normalizeCountryAlias(countryText); + String fieldKey = normalizeHeaderKey(fieldText); + if (country.isEmpty()) { + country = normalizeCountryAlias(fieldText); + fieldKey = normalizeHeaderKey(countryText); + } + if (country.isEmpty() || !SUPPORTED_COUNTRIES.contains(country)) { + continue; + } + if (isAsinHeader(fieldKey)) { + asinColumns.putIfAbsent(country, columnIndex); + } else if (isPriceHeader(fieldKey)) { + priceColumns.putIfAbsent(country, columnIndex); + } + } + if (asinColumns.isEmpty()) { + for (int columnIndex = 0; columnIndex < maxColumn; columnIndex++) { + String headerText = cellText(firstHeader, columnIndex, formatter); + String key = normalizeHeaderKey(headerText); + String country = normalizeCountryAlias(headerText); + if (country.isEmpty()) { + country = countryFromHeaderKey(key); + } + if (!country.isEmpty() && isAsinHeader(key)) { + asinColumns.putIfAbsent(country, columnIndex); + } + } + return new HeaderMapping(asinColumns, priceColumns, 1); + } + return new HeaderMapping(asinColumns, priceColumns, 2); + } + + private int lastCellNum(Row row) { + return row == null || row.getLastCellNum() < 0 ? 0 : row.getLastCellNum(); + } + + private boolean isAsinHeader(String key) { + return key.contains("asin"); + } + + private boolean isPriceHeader(String key) { + return key.contains("最低价") || key.contains("最低价格") || key.contains("minimumprice") + || key.contains("minprice") || key.equals("price"); + } + + private String cellTextWithMerged(Sheet sheet, Row row, int columnIndex, DataFormatter formatter) { + if (row == null) { + return ""; + } + int rowIndex = row.getRowNum(); + for (int i = 0; i < sheet.getNumMergedRegions(); i++) { + var region = sheet.getMergedRegion(i); + if (region.isInRange(rowIndex, columnIndex)) { + Row firstRow = sheet.getRow(region.getFirstRow()); + return cellText(firstRow, region.getFirstColumn(), formatter); + } + } + return cellText(row, columnIndex, formatter); + } + + private String cellText(Row row, Integer columnIndex, DataFormatter formatter) { + if (row == null || columnIndex == null || columnIndex < 0) { + return ""; + } + Cell cell = row.getCell(columnIndex); + return cell == null ? "" : normalizeExcelText(formatter.formatCellValue(cell)); + } + + private String normalizeExcelText(String value) { + return value == null ? "" : value.replace(String.valueOf((char) 0xFEFF), "").trim(); + } + + private String normalizeHeaderKey(String value) { + return normalizeExcelText(value) + .toLowerCase(Locale.ROOT) + .replace(" ", "") + .replace("_", "") + .replace("-", "") + .replace("/", "") + .replace("\\", "") + .replace("：", "") + .replace(":", "") + .replace("（", "") + .replace("）", "") + .replace("(", "") + .replace(")", ""); + } + + private String normalizeCountryAlias(String country) { + String normalized = normalizeBlank(country); + if (normalized.isEmpty()) { + return ""; + } + String upper = normalized.toUpperCase(Locale.ROOT); + if (SUPPORTED_COUNTRIES.contains(upper)) { + return upper; + } + return switch (normalizeHeaderKey(normalized)) { + case "德国", "德", "de", "germany", "german", "deutschland" -> "DE"; + case "英国", "英", "uk", "gb", "greatbritain", "britain", "unitedkingdom" -> "UK"; + case "法国", "法", "fr", "france", "french" -> "FR"; + case "意大利", "意", "it", "italy", "italian" -> "IT"; + case "西班牙", "西", "es", "spain", "spanish", "espana" -> "ES"; + default -> ""; + }; + } + + private String countryFromHeaderKey(String key) { + if (key.contains("asinde") || key.contains("deasin") || key.contains("德国")) return "DE"; + if (key.contains("asinuk") || key.contains("ukasin") || key.contains("英国")) return "UK"; + if (key.contains("asinfr") || key.contains("frasin") || key.contains("法国")) return "FR"; + if (key.contains("asinit") || key.contains("itasin") || key.contains("意大利")) return "IT"; + if (key.contains("asines") || key.contains("esasin") || key.contains("西班牙")) return "ES"; + return ""; + } + + private BigDecimal parseOptionalMinimumPrice(String value) { + String normalized = normalizeBlank(value); + if (normalized.isEmpty()) { + return null; + } + try { + return normalizeMinimumPrice(new BigDecimal(normalized.replace(",", ""))); + } catch (Exception ex) { + throw new BusinessException("最低价格式不正确: " + value); + } + } + + private boolean priceEquals(BigDecimal left, BigDecimal right) { + if (left == null && right == null) { + return true; + } + if (left == null || right == null) { + return false; + } + return left.compareTo(right) == 0; + } + + private boolean isBlankRow(Row row, DataFormatter formatter) { + if (row == null) { + return true; + } + for (int i = 0; i < lastCellNum(row); i++) { + if (!cellText(row, i, formatter).isEmpty()) { + return false; + } + } + return true; + } + + private void updateImportProgress(QueryAsinImportProgressVo progress, ImportCounters counters, int processedRows) { + progress.setProcessedRows(processedRows); + progress.setAsinCount(counters.asinCount); + progress.setInsertedCount(counters.insertedCount); + progress.setDeletedCount(counters.deletedCount); + progress.setSkippedCount(counters.skippedCount); + } + + private File saveMultipartToTempFile(MultipartFile file, String prefix) throws Exception { + String suffix = ".xlsx"; + String name = file.getOriginalFilename(); + if (name != null) { + int idx = name.lastIndexOf('.'); + if (idx >= 0) { + suffix = name.substring(idx); + } + } + File tempFile = File.createTempFile(prefix, suffix); + file.transferTo(tempFile); + return tempFile; + } + + private void deleteQuietly(File file) { + if (file == null) { + return; + } + try { + Files.deleteIfExists(file.toPath()); + } catch (Exception ignored) { + } + } + + private void validateExcelFilename(String filename) { + String lowerFilename = filename == null ? "" : filename.toLowerCase(Locale.ROOT); + if (!(lowerFilename.endsWith(".xlsx") || lowerFilename.endsWith(".xls"))) { + throw new BusinessException("仅支持 .xlsx 或 .xls 文件"); + } + } + + private String normalizeShopNameFromFilename(String filename) { + String normalized = normalizeBlank(filename); + int slashIndex = Math.max(normalized.lastIndexOf('/'), normalized.lastIndexOf('\\')); + if (slashIndex >= 0) { + normalized = normalized.substring(slashIndex + 1); + } + int dotIndex = normalized.lastIndexOf('.'); + if (dotIndex > 0) { + normalized = normalized.substring(0, dotIndex); + } + return normalizeRequired(normalized, "文件名必须是店铺名"); + } + + private void ensureShopExistsInGroup(Long groupId, String shopName) { + Long count = shopManageMapper.selectCount(new LambdaQueryWrapper() + .eq(ShopManageEntity::getGroupId, groupId) + .eq(ShopManageEntity::getShopName, shopName)); + if (count == null || count <= 0) { + throw new BusinessException("当前分组下不存在文件名对应的店铺: " + shopName); + } + } + + private String getCountryAsin(SkipPriceAsinEntity entity, String country) { + return switch (country) { + case "DE" -> normalizeBlank(entity.getAsinDe()); + case "UK" -> normalizeBlank(entity.getAsinUk()); + case "FR" -> normalizeBlank(entity.getAsinFr()); + case "IT" -> normalizeBlank(entity.getAsinIt()); + case "ES" -> normalizeBlank(entity.getAsinEs()); + default -> ""; + }; + } + + private BigDecimal getCountryMinimumPrice(SkipPriceAsinEntity entity, String country) { + return switch (country) { + case "DE" -> entity.getMinimumPriceDe(); + case "UK" -> entity.getMinimumPriceUk(); + case "FR" -> entity.getMinimumPriceFr(); + case "IT" -> entity.getMinimumPriceIt(); + case "ES" -> entity.getMinimumPriceEs(); + default -> null; + }; + } + + private record HeaderMapping(Map asinColumns, + Map priceColumns, + int firstDataRowIndex) { + } + + private static final class ImportCounters { + private int asinCount; + private int insertedCount; + private int deletedCount; + private int skippedCount; + } + private SkipPriceAsinEntity getById(Long id) { SkipPriceAsinEntity entity = skipPriceAsinMapper.selectById(id); if (entity == null) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TransientPayloadStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TransientPayloadStorageService.java index 022fcab..4282fdf 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TransientPayloadStorageService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TransientPayloadStorageService.java @@ -60,7 +60,7 @@ public class TransientPayloadStorageService { return ossStorageService.readObjectAsString(pointer.substring(OSS_POINTER_PREFIX.length())); } } catch (Exception ex) { - throw new IllegalStateException(errorMessage, ex); + throw new IllegalStateException(errorMessage + ": " + pointer, ex); } return value; } diff --git a/backend-java/src/main/resources/application.yml b/backend-java/src/main/resources/application.yml index f1f6719..06fe730 100644 --- a/backend-java/src/main/resources/application.yml +++ b/backend-java/src/main/resources/application.yml @@ -84,6 +84,10 @@ aiimage: access-key-id: ${AIIMAGE_TRANSIENT_STORAGE_ACCESS_KEY_ID:} access-key-secret: ${AIIMAGE_TRANSIENT_STORAGE_ACCESS_KEY_SECRET:} region: ${AIIMAGE_TRANSIENT_STORAGE_REGION:us-east-1} + connect-timeout-seconds: ${AIIMAGE_TRANSIENT_STORAGE_CONNECT_TIMEOUT_SECONDS:10} + read-timeout-seconds: ${AIIMAGE_TRANSIENT_STORAGE_READ_TIMEOUT_SECONDS:60} + write-timeout-seconds: ${AIIMAGE_TRANSIENT_STORAGE_WRITE_TIMEOUT_SECONDS:60} + upload-max-retries: ${AIIMAGE_TRANSIENT_STORAGE_UPLOAD_MAX_RETRIES:3} storage: local-temp-dir: ${AIIMAGE_STORAGE_LOCAL_TEMP_DIR:./data/tmp} cleanup-enabled: ${AIIMAGE_STORAGE_CLEANUP_ENABLED:true} diff --git a/backend-java/src/main/resources/db/V39__allow_multiple_skip_price_asins_per_shop.sql b/backend-java/src/main/resources/db/V39__allow_multiple_skip_price_asins_per_shop.sql new file mode 100644 index 0000000..81b502e --- /dev/null +++ b/backend-java/src/main/resources/db/V39__allow_multiple_skip_price_asins_per_shop.sql @@ -0,0 +1,106 @@ +SET @db_name = DATABASE(); + +SET @idx_exists := ( + SELECT COUNT(*) + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = @db_name + AND TABLE_NAME = 'biz_skip_price_asin' + AND INDEX_NAME = 'uk_group_shop_name' +); +SET @sql := IF(@idx_exists > 0, + 'ALTER TABLE biz_skip_price_asin DROP INDEX uk_group_shop_name', + 'SELECT 1' +); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @idx_exists := ( + SELECT COUNT(*) + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = @db_name + AND TABLE_NAME = 'biz_skip_price_asin' + AND INDEX_NAME = 'idx_group_shop_name' +); +SET @sql := IF(@idx_exists = 0, + 'ALTER TABLE biz_skip_price_asin ADD INDEX idx_group_shop_name (group_id, shop_name)', + 'SELECT 1' +); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @idx_exists := ( + SELECT COUNT(*) + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = @db_name + AND TABLE_NAME = 'biz_skip_price_asin' + AND INDEX_NAME = 'idx_group_shop_asin_de' +); +SET @sql := IF(@idx_exists = 0, + 'ALTER TABLE biz_skip_price_asin ADD INDEX idx_group_shop_asin_de (group_id, shop_name, asin_de)', + 'SELECT 1' +); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @idx_exists := ( + SELECT COUNT(*) + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = @db_name + AND TABLE_NAME = 'biz_skip_price_asin' + AND INDEX_NAME = 'idx_group_shop_asin_uk' +); +SET @sql := IF(@idx_exists = 0, + 'ALTER TABLE biz_skip_price_asin ADD INDEX idx_group_shop_asin_uk (group_id, shop_name, asin_uk)', + 'SELECT 1' +); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @idx_exists := ( + SELECT COUNT(*) + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = @db_name + AND TABLE_NAME = 'biz_skip_price_asin' + AND INDEX_NAME = 'idx_group_shop_asin_fr' +); +SET @sql := IF(@idx_exists = 0, + 'ALTER TABLE biz_skip_price_asin ADD INDEX idx_group_shop_asin_fr (group_id, shop_name, asin_fr)', + 'SELECT 1' +); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @idx_exists := ( + SELECT COUNT(*) + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = @db_name + AND TABLE_NAME = 'biz_skip_price_asin' + AND INDEX_NAME = 'idx_group_shop_asin_it' +); +SET @sql := IF(@idx_exists = 0, + 'ALTER TABLE biz_skip_price_asin ADD INDEX idx_group_shop_asin_it (group_id, shop_name, asin_it)', + 'SELECT 1' +); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @idx_exists := ( + SELECT COUNT(*) + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = @db_name + AND TABLE_NAME = 'biz_skip_price_asin' + AND INDEX_NAME = 'idx_group_shop_asin_es' +); +SET @sql := IF(@idx_exists = 0, + 'ALTER TABLE biz_skip_price_asin ADD INDEX idx_group_shop_asin_es (group_id, shop_name, asin_es)', + 'SELECT 1' +); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; diff --git a/backend/blueprints/admin_api.py b/backend/blueprints/admin_api.py index ecda053..960f22d 100644 --- a/backend/blueprints/admin_api.py +++ b/backend/blueprints/admin_api.py @@ -1990,6 +1990,116 @@ def update_skip_price_asin_country(item_id, country): }) +@admin_api.route('/skip-price-asins/import/') +@login_required +def skip_price_asin_import_progress(import_id): + role, current_row, denied = _ensure_backend_menu_access('skip-price-asin') + if denied: + return denied + result, error_response, status = _proxy_backend_java( + 'GET', + f'/api/admin/skip-price-asins/import/{import_id}', + ) + if error_response is not None: + return error_response, status + return jsonify({ + 'success': True, + 'progress': _format_query_asin_import_progress(result.get('data') or {}), + }) + + +@admin_api.route('/skip-price-asins/import', methods=['POST']) +@login_required +def import_skip_price_asins(): + role, current_row, denied = _ensure_backend_menu_access('skip-price-asin') + if denied: + return denied + file_storage = request.files.get('file') + if not file_storage or file_storage.filename == '': + return jsonify({'success': False, 'error': '请选择 Excel 文件'}) + group_id_raw = (request.form.get('group_id') or '').strip() + if not group_id_raw: + return jsonify({'success': False, 'error': '请先选择分组'}) + params = { + 'groupId': group_id_raw, + 'operatorId': current_row.get('id') if current_row else None, + 'superAdmin': 'true' if role == 'super_admin' else 'false', + } + files = { + 'file': (file_storage.filename, file_storage.stream, file_storage.mimetype or 'application/octet-stream') + } + result, error_response, status = _proxy_backend_java( + 'POST', + '/api/admin/skip-price-asins/import', + params=params, + files=files, + timeout=60, + ) + if error_response is not None: + return error_response, status + summary = result.get('data') or {} + return jsonify({ + 'success': True, + 'msg': result.get('message') or '开始导入', + 'import_id': summary.get('importId') or '', + }) + + +@admin_api.route('/skip-price-asins/delete-import/') +@login_required +def skip_price_asin_delete_import_progress(import_id): + role, current_row, denied = _ensure_backend_menu_access('skip-price-asin') + if denied: + return denied + result, error_response, status = _proxy_backend_java( + 'GET', + f'/api/admin/skip-price-asins/delete-import/{import_id}', + ) + if error_response is not None: + return error_response, status + return jsonify({ + 'success': True, + 'progress': _format_query_asin_import_progress(result.get('data') or {}), + }) + + +@admin_api.route('/skip-price-asins/delete-import', methods=['POST']) +@login_required +def delete_import_skip_price_asins(): + role, current_row, denied = _ensure_backend_menu_access('skip-price-asin') + if denied: + return denied + file_storage = request.files.get('file') + if not file_storage or file_storage.filename == '': + return jsonify({'success': False, 'error': '请选择 Excel 文件'}) + group_id_raw = (request.form.get('group_id') or '').strip() + if not group_id_raw: + return jsonify({'success': False, 'error': '请先选择分组'}) + params = { + 'groupId': group_id_raw, + 'operatorId': current_row.get('id') if current_row else None, + 'superAdmin': 'true' if role == 'super_admin' else 'false', + } + files = { + 'file': (file_storage.filename, file_storage.stream, file_storage.mimetype or 'application/octet-stream') + } + result, error_response, status = _proxy_backend_java( + 'POST', + '/api/admin/skip-price-asins/delete-import', + params=params, + files=files, + timeout=60, + ) + if error_response is not None: + return error_response, status + summary = result.get('data') or {} + return jsonify({ + 'success': True, + 'msg': result.get('message') or '开始删除', + 'import_id': summary.get('importId') or '', + }) + + def _format_query_asin_item(item): return { 'id': item.get('id'), diff --git a/backend/config.py b/backend/config.py index ccb1747..db46cef 100644 --- a/backend/config.py +++ b/backend/config.py @@ -26,6 +26,7 @@ import os backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://127.0.0.1:18080').rstrip('/') # backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://8.136.19.173:18080').rstrip('/') # backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://47.111.163.154:18080').rstrip('/') +# backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://121.196.149.225:18080').rstrip('/') os.environ['OSS_ACCESS_KEY_ID'] = accessKeyId os.environ['OSS_ACCESS_KEY_SECRET'] = accessKeySecret os.environ['SECRET_KEY'] = "ddffc7c1d02121d9554d7b080b2511b6" diff --git a/backend/web_source/admin.html b/backend/web_source/admin.html index 04c7c99..cc8d4f7 100644 --- a/backend/web_source/admin.html +++ b/backend/web_source/admin.html @@ -227,6 +227,65 @@ color: #666; } + .request-loading-bar { + position: fixed; + left: 0; + top: 0; + width: 0; + height: 3px; + background: #667eea; + opacity: 0; + z-index: 3000; + transition: width 0.2s ease, opacity 0.2s ease; + } + + .request-loading-bar.show { + width: 74%; + opacity: 1; + } + + .request-loading-bar.finishing { + width: 100%; + opacity: 0; + } + + .request-loading { + position: fixed; + right: 24px; + top: 72px; + display: none; + align-items: center; + gap: 10px; + padding: 10px 14px; + color: #333; + background: rgba(255, 255, 255, 0.96); + border: 1px solid #e6eaf2; + border-radius: 8px; + box-shadow: 0 8px 24px rgba(0, 0, 0, 0.12); + font-size: 13px; + z-index: 3000; + pointer-events: none; + } + + .request-loading.show { + display: flex; + } + + .request-spinner { + width: 16px; + height: 16px; + border: 2px solid #dce2ff; + border-top-color: #667eea; + border-radius: 50%; + animation: request-spin 0.8s linear infinite; + } + + @keyframes request-spin { + to { + transform: rotate(360deg); + } + } + table { width: 100%; border-collapse: collapse; @@ -661,6 +720,11 @@ +
+
+ + 请求处理中... +

管理后台

@@ -1042,6 +1106,42 @@

+
+
+

导入文件新增

+
+
+ + +
+ +
+

+ +
+
+

导入文件删除

+
+
+ + +
+ +
+

+ +
+

店铺列表

@@ -1578,6 +1678,100 @@