diff --git a/app/.env b/app/.env index 8a4270d..a8104b3 100644 --- a/app/.env +++ b/app/.env @@ -13,7 +13,7 @@ client_name=ShuFuAI # java_api_base=http://47.111.163.154:18080 -# java_api_base=http://127.0.0.1:18080 -java_api_base=http://8.136.19.173:18080 +java_api_base=http://127.0.0.1:18080 +# java_api_base=http://8.136.19.173:18080 diff --git a/app/__pycache__/app_common.cpython-312.pyc b/app/__pycache__/app_common.cpython-312.pyc index eac4be7..5b2db77 100644 Binary files a/app/__pycache__/app_common.cpython-312.pyc and b/app/__pycache__/app_common.cpython-312.pyc differ diff --git a/app/__pycache__/config.cpython-312.pyc b/app/__pycache__/config.cpython-312.pyc index ed70141..921936d 100644 Binary files a/app/__pycache__/config.cpython-312.pyc and b/app/__pycache__/config.cpython-312.pyc differ diff --git a/app/amazon/__pycache__/approve.cpython-312.pyc b/app/amazon/__pycache__/approve.cpython-312.pyc index 383dcdf..7ad524b 100644 Binary files a/app/amazon/__pycache__/approve.cpython-312.pyc and b/app/amazon/__pycache__/approve.cpython-312.pyc differ diff --git a/app/amazon/__pycache__/main.cpython-312.pyc b/app/amazon/__pycache__/main.cpython-312.pyc index 1a9e82f..cb7f75a 100644 Binary files a/app/amazon/__pycache__/main.cpython-312.pyc and b/app/amazon/__pycache__/main.cpython-312.pyc differ diff --git a/app/amazon/__pycache__/price_match.cpython-312.pyc b/app/amazon/__pycache__/price_match.cpython-312.pyc index 56c0306..8fbcf05 100644 Binary files a/app/amazon/__pycache__/price_match.cpython-312.pyc and b/app/amazon/__pycache__/price_match.cpython-312.pyc differ diff --git a/app/amazon/__pycache__/tool.cpython-312.pyc b/app/amazon/__pycache__/tool.cpython-312.pyc index eceb0cd..9e0d13e 100644 Binary files a/app/amazon/__pycache__/tool.cpython-312.pyc and b/app/amazon/__pycache__/tool.cpython-312.pyc differ diff --git a/app/app_common.py b/app/app_common.py index 341da46..db0e20d 100644 --- a/app/app_common.py +++ b/app/app_common.py @@ -12,9 +12,14 @@ from flask import request, redirect, url_for, session, jsonify, render_template, from config import mysql_host, mysql_user, mysql_password, mysql_database -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -STATIC_DIR = os.path.join(BASE_DIR, 'static') -ASSETS_DIR = os.path.join(BASE_DIR, 'assets') +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +STATIC_DIR = os.path.join(BASE_DIR, 'static') +ASSETS_DIR = os.path.join(BASE_DIR, 'assets') +WEB_SOURCE_DIR = os.path.join(BASE_DIR, 'web_source') +TEMPLATE_FALLBACK_DIRS = ( + WEB_SOURCE_DIR, + os.path.join(WEB_SOURCE_DIR, 'templates_backup'), +) def get_db(): @@ -30,9 +35,13 @@ def get_db(): def _render_html(template_name: str, **context): """读取 HTML 模板：若为加密文件则先解密，再渲染。未加密或解密失败时按明文渲染。""" - path = os.path.join(BASE_DIR, "web_source", template_name) - if not os.path.isfile(path): - return render_template(template_name, **context) + path = next( + (candidate for candidate in (os.path.join(base_path, template_name) for base_path in TEMPLATE_FALLBACK_DIRS) + if os.path.isfile(candidate)), + None, + ) + if path is None: + return render_template(template_name, **context) with open(path, "rb") as f: raw = f.read() try: @@ -45,9 +54,13 @@ def _render_html(template_name: str, **context): def _render_html_new(template_name: str, **context): """读取 HTML 模板：若为加密文件则先解密，再渲染。未加密或解密失败时按明文渲染。""" - path = os.path.join(BASE_DIR, "web_source", template_name) - if not os.path.isfile(path): - return render_template(template_name, **context) + path = next( + (candidate for candidate in (os.path.join(base_path, template_name) for base_path in TEMPLATE_FALLBACK_DIRS) + if os.path.isfile(candidate)), + None, + ) + if path is None: + return render_template(template_name, **context) with open(path, "rb") as f: raw = f.read() try: diff --git a/app/blueprints/__pycache__/main.cpython-312.pyc b/app/blueprints/__pycache__/main.cpython-312.pyc index 25e36e2..61aa0d3 100644 Binary files a/app/blueprints/__pycache__/main.cpython-312.pyc and b/app/blueprints/__pycache__/main.cpython-312.pyc differ diff --git a/app/blueprints/main.py b/app/blueprints/main.py index b9313fd..8b3d19e 100644 --- a/app/blueprints/main.py +++ b/app/blueprints/main.py @@ -2,7 +2,7 @@ 主页面蓝图：首页、home、图片工作台、品牌页、静态文件、Logo """ import os -from flask import Blueprint, send_file +from flask import Blueprint, send_file, render_template_string from app_common import ( get_db, @@ -20,7 +20,39 @@ import requests from config import base_url,version,JAVA_API_BASE -main_bp = Blueprint('main', __name__) +main_bp = Blueprint('main', __name__) + + +def _resolve_asset_filename(filename): + """Resolve hashed Vite assets without maintaining hardcoded alias tables.""" + safe_name = os.path.basename(filename) + exact_path = os.path.join(ASSETS_DIR, safe_name) + if os.path.isfile(exact_path): + return safe_name + + base_name, ext = os.path.splitext(safe_name) + if not base_name or not ext: + return safe_name + + parts = base_name.split('-') + if len(parts) < 2: + return safe_name + + try: + asset_names = os.listdir(ASSETS_DIR) + except OSError: + return safe_name + + for prefix_length in range(len(parts) - 1, 0, -1): + prefix = '-'.join(parts[:prefix_length]) + matches = [ + asset_name for asset_name in asset_names + if asset_name.endswith(ext) and asset_name.startswith(prefix) + ] + if len(matches) == 1: + return matches[0] + + return safe_name @main_bp.route('/') @@ -50,10 +82,42 @@ def wb(): return _render_html('index.html') -@main_bp.route('/brand') -@login_required -def brand_page(): - return _render_html('brand.html', user_id=session.get('user_id')) +@main_bp.route('/brand') +@login_required +def brand_page(): + repo_brand_path = os.path.abspath(os.path.join(BASE_DIR, '..', 'web_source', 'brand.html')) + if os.path.isfile(repo_brand_path): + try: + with open(repo_brand_path, 'rb') as f: + raw = f.read() + try: + from html_crypto import decrypt + content = decrypt(raw).decode('utf-8') + except Exception: + if raw[:7] == b'gAAAAAB': + raise + content = raw.decode('utf-8', errors='replace') + return render_template_string(content, user_id=session.get('user_id')) + except Exception: + pass + return _render_html('brand.html', user_id=session.get('user_id')) + + +@main_bp.route('/brand/legacy') +@login_required +def brand_page_legacy(): + legacy_path = os.path.join(BASE_DIR, 'web_source', 'templates_backup', 'brand.html') + if not os.path.isfile(legacy_path): + return '', 404 + with open(legacy_path, 'rb') as f: + content = f.read().decode('utf-8', errors='replace') + content = content.replace( + '
', + '
', + 1, + ) + content = content.replace('height: calc(100vh - 56px);', 'height: 100vh;', 1) + return render_template_string(content, user_id=session.get('user_id')) @@ -69,10 +133,11 @@ def serve_static(filename): return send_file(file_abs, as_attachment=False) -@main_bp.route('/assets/') -def serve_assets(filename): - """提供 static 目录及子目录下的静态文件访问。""" - filepath = os.path.normpath(os.path.join(ASSETS_DIR, filename)) +@main_bp.route('/assets/') +def serve_assets(filename): + """提供 static 目录及子目录下的静态文件访问。""" + filename = _resolve_asset_filename(filename) + filepath = os.path.normpath(os.path.join(ASSETS_DIR, filename)) static_abs = os.path.abspath(ASSETS_DIR) file_abs = os.path.abspath(filepath) if not file_abs.startswith(static_abs) or not os.path.isfile(file_abs): @@ -82,14 +147,19 @@ def serve_assets(filename): @main_bp.route('/new_web_source/') def serve_new_web_source(filename): - """提供 static 目录及子目录下的静态文件访问。""" - filepath = os.path.normpath(os.path.join("new_web_source", filename)) - static_abs = os.path.abspath("new_web_source") - file_abs = os.path.abspath(filepath) - if not file_abs.startswith(static_abs) or not os.path.isfile(file_abs): - return '', 404 - return send_file(file_abs, as_attachment=False) - + """提供 new_web_source 目录下的静态页面文件访问。""" + candidate_dirs = [ + os.path.abspath(os.path.join(BASE_DIR, 'new_web_source')), + os.path.abspath(os.path.join(BASE_DIR, '..', 'new_web_source')), + ] + for static_abs in candidate_dirs: + filepath = os.path.normpath(os.path.join(static_abs, filename)) + file_abs = os.path.abspath(filepath) + if not file_abs.startswith(static_abs): + continue + if os.path.isfile(file_abs): + return send_file(file_abs, as_attachment=False) + return '', 404 @main_bp.route('/logo.jpg', methods=['GET']) @@ -161,4 +231,4 @@ def proxy(path): return Response("无法连接到后端服务", status=502) except Exception as e: print(f"代理请求失败: {e}") - return Response(f"代理错误: {str(e)}", status=500) + return Response(f"代理错误: {str(e)}", status=500) diff --git a/app/config.py b/app/config.py index 137cad8..db6b0e8 100644 --- a/app/config.py +++ b/app/config.py @@ -3,11 +3,12 @@ coze_token = "sat_12nW40INoJxArrDXbY4lSCoudbqkOYTcphC99BP2efWyzxmsk4q81WDX3ezWgq workflow_id = "7608812635877900322" STITCH_WORKFLOW_ID = "7608813873483300907" -import os -from dotenv import load_dotenv -from queue import Queue -load_dotenv() -_base_url = os.getenv("base_url","http://159.75.121.33:15124") +import os +from dotenv import load_dotenv +from queue import Queue +load_dotenv() +_base_url = os.getenv("base_url","http://159.75.121.33:15124") +base_dir = os.path.dirname(os.path.abspath(__file__)) # MySQL 配置 mysql_host = os.getenv("mysql_host") @@ -15,10 +16,11 @@ mysql_user = os.getenv("mysql_user") mysql_password = os.getenv("mysql_password","WTFrb5y6hNLz6hNy") mysql_database = os.getenv("mysql_database","aiimage") proxy_url = os.getenv("proxy_url") -proxy_mode = int(os.getenv("proxy_mode",1)) - -client_name=os.getenv("client_name") + ".exe" -JAVA_API_BASE = os.getenv("java_api_base", "http://127.0.0.1:18080") +proxy_mode = int(os.getenv("proxy_mode",1)) + +_client_name = os.getenv("client_name", "").strip() +client_name = f"{_client_name}.exe" if _client_name else "client.exe" +JAVA_API_BASE = os.getenv("java_api_base", "http://127.0.0.1:18080") # 紫鸟浏览器配置 from urllib.parse import unquote diff --git a/app/main.py b/app/main.py index be4fba7..f8bdcdd 100644 --- a/app/main.py +++ b/app/main.py @@ -433,7 +433,7 @@ def main(): webview.start( debug=True, storage_path=cache_path, - private_mode=False + private_mode=True ) diff --git a/backend-java/src/main/java/com/nanri/aiimage/config/TaskPressureProperties.java b/backend-java/src/main/java/com/nanri/aiimage/config/TaskPressureProperties.java index 5c7fdf8..3a157f0 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/config/TaskPressureProperties.java +++ b/backend-java/src/main/java/com/nanri/aiimage/config/TaskPressureProperties.java @@ -8,4 +8,8 @@ import org.springframework.boot.context.properties.ConfigurationProperties; public class TaskPressureProperties { private long localTaskEntityCacheMillis = 3000; private int dbSelectBatchSize = 200; + private long scopePayloadFlushIntervalMillis = 15000; + private long scopePayloadBufferRetentionHours = 24; + private int scopePayloadRecoveryMaxFiles = 200; + private String scopePayloadCleanupCron = "15 */30 * * * *"; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskProgressCacheService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskProgressCacheService.java index b7d0612..be8759c 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskProgressCacheService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskProgressCacheService.java @@ -1,25 +1,13 @@ package com.nanri.aiimage.modules.brand.service; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.config.BrandProgressProperties; -import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultFileDto; -import com.nanri.aiimage.modules.brand.model.dto.BrandFileAggregateCacheDto; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.stereotype.Service; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.security.MessageDigest; import java.time.Duration; import java.time.Instant; -import java.util.ArrayList; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; @Service @@ -33,6 +21,7 @@ public class BrandTaskProgressCacheService { private final StringRedisTemplate stringRedisTemplate; private final BrandProgressProperties brandProgressProperties; + @SuppressWarnings("unused") private final ObjectMapper objectMapper; public BrandTaskProgressCacheService(StringRedisTemplate stringRedisTemplate, @@ -96,121 +85,9 @@ public class BrandTaskProgressCacheService { return stringRedisTemplate.opsForHash().entries(buildKey(taskId)); } - public void saveParsedPayload(Long taskId, Object payload) { - try { - Files.createDirectories(buildTaskDir(taskId)); - Files.writeString( - buildPayloadFile(taskId), - objectMapper.writeValueAsString(payload), - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING, - StandardOpenOption.WRITE - ); - } catch (Exception ex) { - throw new BusinessException("鏆傚瓨鍝佺墝鍘熷鏁版嵁澶辫触"); - } - } - - public T getParsedPayload(Long taskId, TypeReference typeReference) { - Path payloadFile = buildPayloadFile(taskId); - if (!Files.isRegularFile(payloadFile)) { - return null; - } - try { - return objectMapper.readValue(Files.readString(payloadFile), typeReference); - } catch (Exception ex) { - throw new BusinessException("璇诲彇鍝佺墝鍘熷鏁版嵁澶辫触"); - } - } - - public ChunkStoreResult storeChunk(Long taskId, BrandCrawlResultFileDto file) { - validateChunk(file); - String fileUrl = normalizeFileUrl(file.getFileUrl()); - String chunkDigest = digestChunk(file); - String digestField = String.valueOf(file.getChunkIndex()); - String digestKey = buildChunkDigestKey(taskId, fileUrl); - Object existingDigest = stringRedisTemplate.opsForHash().get(digestKey, digestField); - if (existingDigest instanceof String existing && !existing.isBlank()) { - refreshFileKeys(taskId, fileUrl); - if (existing.equals(chunkDigest)) { - BrandFileAggregateCacheDto aggregate = getFileAggregate(taskId, fileUrl); - int finishedFiles = countCompletedFiles(taskId); - return new ChunkStoreResult(false, false, finishedFiles, aggregate); - } - throw new BusinessException("鍚屼竴鍒嗙墖閲嶅鎻愪氦浣嗗唴瀹逛笉涓€鑷? " + fileUrl + "#" + file.getChunkIndex()); - } - - BrandFileAggregateCacheDto aggregate = getFileAggregate(taskId, fileUrl); - if (aggregate == null) { - aggregate = createAggregate(file); - } else { - ensureChunkConsistency(aggregate, file); - } - mergeAggregate(aggregate, file); - - stringRedisTemplate.opsForSet().add(buildChunkReceiptKey(taskId, fileUrl), digestField); - Long receivedSize = stringRedisTemplate.opsForSet().size(buildChunkReceiptKey(taskId, fileUrl)); - int receivedCount = receivedSize == null ? 0 : receivedSize.intValue(); - if (receivedCount <= 0) { - receivedCount = aggregate.getReceivedChunkCount() == null ? 0 : aggregate.getReceivedChunkCount(); - } - aggregate.setReceivedChunkCount(Math.max(receivedCount, aggregate.getReceivedChunkCount() == null ? 0 : aggregate.getReceivedChunkCount())); - boolean fileCompleted = aggregate.getChunkTotal() != null - && aggregate.getChunkTotal() > 0 - && aggregate.getReceivedChunkCount() >= aggregate.getChunkTotal(); - aggregate.setCompleted(fileCompleted); - saveFileAggregate(taskId, aggregate); - stringRedisTemplate.opsForHash().put(digestKey, digestField, chunkDigest); - refreshFileKeys(taskId, fileUrl); - - boolean newlyCompleted = false; - int finishedFiles = countCompletedFiles(taskId); - if (fileCompleted) { - Long added = stringRedisTemplate.opsForSet().add(buildCompletedFilesKey(taskId), fileUrl); - refreshTaskKey(buildCompletedFilesKey(taskId)); - newlyCompleted = added != null && added > 0; - if (newlyCompleted) { - finishedFiles = countCompletedFiles(taskId); - } - } - return new ChunkStoreResult(true, newlyCompleted, finishedFiles, aggregate); - } - - public BrandFileAggregateCacheDto getFileAggregate(Long taskId, String fileUrl) { - String normalizedFileUrl = normalizeFileUrl(fileUrl); - Object raw = stringRedisTemplate.opsForHash().get(buildFileAggregateKey(taskId), normalizedFileUrl); - if (!(raw instanceof String json) || json.isBlank()) { - return null; - } - try { - return objectMapper.readValue(json, BrandFileAggregateCacheDto.class); - } catch (Exception ex) { - throw new BusinessException("璇诲彇鍝佺墝鏂囦欢鑱氬悎缂撳瓨澶辫触"); - } - } - - public Map getAllFileAggregates(Long taskId) { - Map stored = stringRedisTemplate.opsForHash().entries(buildFileAggregateKey(taskId)); - Map result = new LinkedHashMap<>(); - for (Map.Entry entry : stored.entrySet()) { - if (!(entry.getKey() instanceof String fileUrl) || !(entry.getValue() instanceof String json) || json.isBlank()) { - continue; - } - try { - result.put(fileUrl, objectMapper.readValue(json, BrandFileAggregateCacheDto.class)); - } catch (Exception ignored) { - } - } - return result; - } - - public int countCompletedFiles(Long taskId) { - Long count = stringRedisTemplate.opsForSet().size(buildCompletedFilesKey(taskId)); - return count == null ? 0 : count.intValue(); - } - public boolean acquireFinalizeLock(Long taskId) { - Boolean ok = stringRedisTemplate.opsForValue().setIfAbsent(buildFinalizeLockKey(taskId), String.valueOf(Instant.now().toEpochMilli()), FINALIZE_LOCK_TTL); + Boolean ok = stringRedisTemplate.opsForValue() + .setIfAbsent(buildFinalizeLockKey(taskId), String.valueOf(Instant.now().toEpochMilli()), FINALIZE_LOCK_TTL); return Boolean.TRUE.equals(ok); } @@ -220,18 +97,7 @@ public class BrandTaskProgressCacheService { public void delete(Long taskId) { stringRedisTemplate.delete(buildKey(taskId)); - stringRedisTemplate.delete(buildFileAggregateKey(taskId)); - stringRedisTemplate.delete(buildCompletedFilesKey(taskId)); stringRedisTemplate.delete(buildFinalizeLockKey(taskId)); - stringRedisTemplate.delete(buildLegacyResultKey(taskId)); - deleteTaskDirQuietly(taskId); - } - - public void deleteFileState(Long taskId, String fileUrl) { - String normalizedFileUrl = normalizeFileUrl(fileUrl); - stringRedisTemplate.opsForHash().delete(buildFileAggregateKey(taskId), normalizedFileUrl); - stringRedisTemplate.delete(buildChunkReceiptKey(taskId, normalizedFileUrl)); - stringRedisTemplate.delete(buildChunkDigestKey(taskId, normalizedFileUrl)); } public String buildKey(Long taskId) { @@ -242,182 +108,14 @@ public class BrandTaskProgressCacheService { return brandProgressProperties.getHeartbeatTimeoutMinutes(); } - private void saveFileAggregate(Long taskId, BrandFileAggregateCacheDto aggregate) { - try { - stringRedisTemplate.opsForHash().put(buildFileAggregateKey(taskId), normalizeFileUrl(aggregate.getFileUrl()), objectMapper.writeValueAsString(aggregate)); - refreshTaskKey(buildFileAggregateKey(taskId)); - } catch (Exception ex) { - throw new BusinessException("鏆傚瓨鍝佺墝鏂囦欢鑱氬悎缁撴灉澶辫触"); - } - } - - private void validateChunk(BrandCrawlResultFileDto file) { - if (file == null) { - throw new BusinessException("鍒嗙墖涓嶈兘涓虹┖"); - } - if (file.getChunkIndex() == null || file.getChunkTotal() == null || file.getChunkIndex() <= 0 || file.getChunkTotal() <= 0 || file.getChunkIndex() > file.getChunkTotal()) { - throw new BusinessException("鍒嗙墖鍙傛暟涓嶅悎娉?"); - } - if (file.getFileUrl() == null || file.getFileUrl().isBlank()) { - throw new BusinessException("fileUrl 涓嶈兘涓虹┖"); - } - } - - private BrandFileAggregateCacheDto createAggregate(BrandCrawlResultFileDto file) { - BrandFileAggregateCacheDto aggregate = new BrandFileAggregateCacheDto(); - aggregate.setFileUrl(normalizeFileUrl(file.getFileUrl())); - aggregate.setOriginalFilename(file.getOriginalFilename()); - aggregate.setRelativePath(file.getRelativePath()); - aggregate.setMainSheetName(file.getMainSheetName()); - aggregate.setChunkTotal(file.getChunkTotal()); - aggregate.setTotalLines(safePositive(file.getTotalLines())); - aggregate.setReceivedChunkCount(0); - aggregate.setProcessedLineCount(0); - aggregate.setCompleted(false); - aggregate.setInvalidBrands(new ArrayList<>()); - aggregate.setQueryFailedBrands(new ArrayList<>()); - return aggregate; - } - - private void ensureChunkConsistency(BrandFileAggregateCacheDto aggregate, BrandCrawlResultFileDto file) { - if (aggregate.getChunkTotal() != null && file.getChunkTotal() != null && !aggregate.getChunkTotal().equals(file.getChunkTotal())) { - throw new BusinessException("鍚屼竴鏂囦欢鐨?chunkTotal 涓嶄竴鑷? " + aggregate.getFileUrl()); - } - if (aggregate.getChunkTotal() == null) { - aggregate.setChunkTotal(file.getChunkTotal()); - } - int incomingTotalLines = safePositive(file.getTotalLines()); - if (incomingTotalLines > 0) { - aggregate.setTotalLines(Math.max(safePositive(aggregate.getTotalLines()), incomingTotalLines)); - } - if ((aggregate.getOriginalFilename() == null || aggregate.getOriginalFilename().isBlank()) && file.getOriginalFilename() != null && !file.getOriginalFilename().isBlank()) { - aggregate.setOriginalFilename(file.getOriginalFilename()); - } - if ((aggregate.getRelativePath() == null || aggregate.getRelativePath().isBlank()) && file.getRelativePath() != null && !file.getRelativePath().isBlank()) { - aggregate.setRelativePath(file.getRelativePath()); - } - if ((aggregate.getMainSheetName() == null || aggregate.getMainSheetName().isBlank()) && file.getMainSheetName() != null && !file.getMainSheetName().isBlank()) { - aggregate.setMainSheetName(file.getMainSheetName()); - } - } - - private void mergeAggregate(BrandFileAggregateCacheDto aggregate, BrandCrawlResultFileDto file) { - int processed = safePositive(aggregate.getProcessedLineCount()) - + sizeOf(file.getKeptRows()) - + sizeOf(file.getInvalidBrands()) - + sizeOf(file.getQueryFailedBrands()); - aggregate.setProcessedLineCount(processed); - if (file.getInvalidBrands() != null && !file.getInvalidBrands().isEmpty()) { - aggregate.getInvalidBrands().addAll(file.getInvalidBrands()); - } - if (file.getQueryFailedBrands() != null && !file.getQueryFailedBrands().isEmpty()) { - aggregate.getQueryFailedBrands().addAll(file.getQueryFailedBrands()); - } - } - - private int sizeOf(List list) { - return list == null ? 0 : list.size(); - } - - private int safePositive(Integer value) { - return value == null || value <= 0 ? 0 : value; - } - - private String digestChunk(BrandCrawlResultFileDto file) { - try { - MessageDigest digest = MessageDigest.getInstance("SHA-256"); - byte[] bytes = digest.digest(objectMapper.writeValueAsString(file).getBytes(StandardCharsets.UTF_8)); - StringBuilder sb = new StringBuilder(bytes.length * 2); - for (byte b : bytes) { - sb.append(String.format("%02x", b)); - } - return sb.toString(); - } catch (Exception ex) { - throw new BusinessException("璁＄畻鍝佺墝缁撴灉鍒嗙墖鎽樿澶辫触"); - } - } - - private void refreshFileKeys(Long taskId, String fileUrl) { - refreshTaskKey(buildChunkReceiptKey(taskId, fileUrl)); - refreshTaskKey(buildChunkDigestKey(taskId, fileUrl)); - refreshTaskKey(buildFileAggregateKey(taskId)); - } - - private void refreshTaskKey(String key) { - stringRedisTemplate.expire(key, ttl()); - } - private Duration ttl() { return Duration.ofHours(brandProgressProperties.getTtlHours()); } - private Path buildTaskDir(Long taskId) { - return Path.of(System.getProperty("java.io.tmpdir"), "brand-task-cache", String.valueOf(taskId)); - } - - private Path buildPayloadFile(Long taskId) { - return buildTaskDir(taskId).resolve("parsed-payload.json"); - } - - private void deleteTaskDirQuietly(Long taskId) { - Path taskDir = buildTaskDir(taskId); - if (!Files.exists(taskDir)) { - return; - } - try (var walk = Files.walk(taskDir)) { - walk.sorted(java.util.Comparator.reverseOrder()).forEach(path -> { - try { - Files.deleteIfExists(path); - } catch (IOException ignored) { - } - }); - } catch (IOException ignored) { - } - } - - private String buildFileAggregateKey(Long taskId) { - return "brand:task:file-aggregate:" + taskId; - } - - private String buildChunkReceiptKey(Long taskId, String fileUrl) { - return "brand:task:file-chunks:" + taskId + ":" + hashFileUrl(fileUrl); - } - - private String buildChunkDigestKey(Long taskId, String fileUrl) { - return "brand:task:file-chunk-digests:" + taskId + ":" + hashFileUrl(fileUrl); - } - - private String buildCompletedFilesKey(Long taskId) { - return "brand:task:completed-files:" + taskId; - } - private String buildFinalizeLockKey(Long taskId) { return "brand:task:finalizing:" + taskId; } - private String buildLegacyResultKey(Long taskId) { - return "brand:task:result-chunks:" + taskId; - } - - private String normalizeFileUrl(String fileUrl) { - return fileUrl == null ? "" : fileUrl.trim(); - } - - private String hashFileUrl(String fileUrl) { - String normalized = normalizeFileUrl(fileUrl); - try { - MessageDigest digest = MessageDigest.getInstance("SHA-256"); - byte[] bytes = digest.digest(normalized.getBytes(StandardCharsets.UTF_8)); - StringBuilder sb = new StringBuilder(bytes.length * 2); - for (byte b : bytes) { - sb.append(String.format("%02x", b)); - } - return sb.toString(); - } catch (Exception ex) { - throw new BusinessException("鐢熸垚鏂囦欢缂撳瓨閿け璐?"); - } - } - private String normalizePhase(String phase) { if (phase == null || phase.isBlank()) { return PHASE_CRAWLING; @@ -432,7 +130,4 @@ public class BrandTaskProgressCacheService { private String blankToEmpty(String value) { return value == null ? "" : value.trim(); } - - public record ChunkStoreResult(boolean stored, boolean newlyCompletedFile, int finishedFiles, BrandFileAggregateCacheDto aggregate) { - } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java index f9e7067..7d3b590 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java @@ -39,7 +39,7 @@ import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.ss.usermodel.WorkbookFactory; -import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -83,6 +83,7 @@ public class BrandTaskService { private final StorageProperties storageProperties; private final BrandProgressProperties brandProgressProperties; private final BrandTaskProgressCacheService brandTaskProgressCacheService; + private final BrandTaskStorageService brandTaskStorageService; private final DistributedJobLockService distributedJobLockService; private final ObjectMapper objectMapper; @@ -193,7 +194,7 @@ public class BrandTaskService { cachedFiles.add(cacheDto); } - brandTaskProgressCacheService.saveParsedPayload(taskId, cachedFiles); + brandTaskStorageService.saveParsedPayload(taskId, cachedFiles); brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper() .eq(BrandCrawlTaskEntity::getId, taskId) @@ -226,9 +227,7 @@ public class BrandTaskService { sourceByUrl.put(file.getFileUrl(), file); sourceIndexByUrl.put(file.getFileUrl(), i + 1); } - List cachedFiles = brandTaskProgressCacheService.getParsedPayload(taskId, - new TypeReference>() { - }); + List cachedFiles = brandTaskStorageService.getParsedPayload(taskId); if (cachedFiles == null || cachedFiles.isEmpty()) { throw new BusinessException("任务原始数据已过期，请重新创建任务"); } @@ -252,7 +251,7 @@ public class BrandTaskService { totalCount, Thread.currentThread().getName()); - int finishedCount = brandTaskProgressCacheService.countCompletedFiles(taskId); + int finishedCount = brandTaskStorageService.countCompletedFiles(taskId); for (BrandCrawlResultFileDto resultFile : resultFiles) { String fileUrl = blankToNull(resultFile.getFileUrl()); if (fileUrl == null) { @@ -272,7 +271,7 @@ public class BrandTaskService { } else if (totalLines > 0 && resultFile.getTotalLines() > totalLines) { throw new BusinessException("totalLines 超过原始文件行数: " + fileUrl); } - BrandTaskProgressCacheService.ChunkStoreResult storeResult = brandTaskProgressCacheService.storeChunk(taskId, resultFile); + BrandTaskStorageService.ChunkStoreResult storeResult = brandTaskStorageService.storeChunk(taskId, resultFile); finishedCount = Math.max(finishedCount, storeResult.finishedFiles()); log.info("[brand-submit] taskId={} fileUrl={} chunk={}/{} stored={} newlyCompletedFile={} finishedFiles={}", taskId, @@ -316,6 +315,7 @@ public class BrandTaskService { if (updated == 0) { throw new BusinessException("任务不存在或无法取消"); } + brandTaskStorageService.deleteTaskData(taskId); brandTaskProgressCacheService.delete(taskId); } @@ -327,6 +327,7 @@ public class BrandTaskService { if (deleted == 0) { throw new BusinessException("任务不存在或正在执行中无法删除"); } + brandTaskStorageService.deleteTaskData(taskId); brandTaskProgressCacheService.delete(taskId); } @@ -555,7 +556,7 @@ public class BrandTaskService { if (STATUS_FAILED.equalsIgnoreCase(freshStatus)) { throw new BusinessException("任务已结束，不能继续组装结果"); } - Map aggregates = brandTaskProgressCacheService.getAllFileAggregates(taskId); + Map aggregates = brandTaskStorageService.getAllFileAggregates(taskId); log.info("[brand-finalize] taskId={} aggregateFiles={} expectedFiles={} thread={}", taskId, aggregates.size(), @@ -600,9 +601,7 @@ public class BrandTaskService { if (updated == 0) { throw new BusinessException("任务已取消"); } - for (BrandSourceFileDto sourceFile : sourceFiles) { - brandTaskProgressCacheService.deleteFileState(taskId, sourceFile.getFileUrl()); - } + brandTaskStorageService.deleteTaskData(taskId); brandTaskProgressCacheService.delete(taskId); log.info("[brand-finalize] taskId={} finalized success files={} elapsedMs={}", taskId, @@ -614,7 +613,7 @@ public class BrandTaskService { .eq(BrandCrawlTaskEntity::getId, taskId) .ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED) .set(BrandCrawlTaskEntity::getStatus, STATUS_FAILED) - .set(BrandCrawlTaskEntity::getProgressCurrent, brandTaskProgressCacheService.countCompletedFiles(taskId)) + .set(BrandCrawlTaskEntity::getProgressCurrent, brandTaskStorageService.countCompletedFiles(taskId)) .set(BrandCrawlTaskEntity::getProgressTotal, totalCount) .set(BrandCrawlTaskEntity::getErrorMessage, ex.getMessage())); brandTaskProgressCacheService.markFailed(taskId, ex.getMessage()); @@ -622,6 +621,8 @@ public class BrandTaskService { throw businessException; } throw new BusinessException(ex.getMessage()); + } finally { + cleanupBrandResultTempFiles(outputEntries, outputDir); } } @@ -746,7 +747,7 @@ public class BrandTaskService { String value = index == null ? "" : normalizeCellText(formatter.formatCellValue(row.getCell(index))); rowData.put(column, value); } - String brand = normalizeCellText(Objects.toString(rowData.getOrDefault("\u54c1\u724c", ""), "")); + String brand = normalizeCellText(Objects.toString(rowData.getOrDefault("品牌", ""), "")); if (brand.isBlank()) { rowData.put("__rowIndex", rowNum + 1); rows.add(rowData); @@ -827,7 +828,9 @@ public class BrandTaskService { BrandParsedFileCacheDto cachedFile, BrandFileAggregateCacheDto resultFile) throws IOException { String actualStrategy = normalizeStrategy(strategy); - try (XSSFWorkbook workbook = new XSSFWorkbook()) { + SXSSFWorkbook workbook = new SXSSFWorkbook(200); + workbook.setCompressTempFiles(true); + try { String mainSheetName = blankToDefault(cachedFile.getSheetName(), "Sheet1"); var mainSheet = workbook.createSheet(mainSheetName); var headerRow = mainSheet.createRow(0); @@ -878,8 +881,35 @@ public class BrandTaskService { } try (FileOutputStream outputStream = new FileOutputStream(outputFile)) { + applyBrandSheetWidths(mainSheet, columns.size()); + applyFixedColumnWidths(invalidSheet, new int[]{20, 12, 18}); + applyFixedColumnWidths(queryFailedSheet, new int[]{20, 18}); workbook.write(outputStream); } + } finally { + try { + workbook.close(); + } catch (Exception ignored) { + } + workbook.dispose(); + } + } + + private void applyBrandSheetWidths(Sheet sheet, int columnCount) { + if (sheet == null || columnCount <= 0) { + return; + } + for (int i = 0; i < columnCount; i++) { + sheet.setColumnWidth(i, 20 * 256); + } + } + + private void applyFixedColumnWidths(Sheet sheet, int[] widths) { + if (sheet == null || widths == null) { + return; + } + for (int i = 0; i < widths.length; i++) { + sheet.setColumnWidth(i, widths[i] * 256); } } @@ -946,6 +976,43 @@ public class BrandTaskService { return zipFile; } + private void cleanupBrandResultTempFiles(List entries, File outputDir) { + if (entries != null) { + for (OutputEntry entry : entries) { + if (entry == null) { + continue; + } + deleteQuietly(entry.resultFile()); + if (isManagedBrandSourceTempFile(entry.sourceFile())) { + deleteQuietly(entry.sourceFile()); + } + } + } + deleteQuietly(outputDir); + } + + private boolean isManagedBrandSourceTempFile(File file) { + if (file == null) { + return false; + } + File managedDir = FileUtil.file(storageProperties.getLocalTempDir(), "brand-source-download"); + try { + return file.getCanonicalPath().startsWith(managedDir.getCanonicalPath()); + } catch (IOException ex) { + return false; + } + } + + private void deleteQuietly(File file) { + if (file == null || !file.exists()) { + return; + } + try { + FileUtil.del(file); + } catch (Exception ignored) { + } + } + private File buildNamedOutputFile(File outputDir, String filename) { File candidate = FileUtil.file(outputDir, filename); if (!candidate.exists()) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskStorageService.java new file mode 100644 index 0000000..9deb686 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskStorageService.java @@ -0,0 +1,475 @@ +package com.nanri.aiimage.modules.brand.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultFileDto; +import com.nanri.aiimage.modules.brand.model.dto.BrandFileAggregateCacheDto; +import com.nanri.aiimage.modules.brand.model.dto.BrandParsedFileCacheDto; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; +import lombok.RequiredArgsConstructor; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +@Service +@RequiredArgsConstructor +public class BrandTaskStorageService { + + private static final String MODULE_TYPE = "BRAND"; + private static final String OSS_POINTER_PREFIX = "oss:"; + + private final TaskChunkMapper taskChunkMapper; + private final TaskScopeStateMapper taskScopeStateMapper; + private final ObjectMapper objectMapper; + private final OssStorageService ossStorageService; + + @Transactional + public void saveParsedPayload(Long taskId, List payload) { + if (taskId == null || taskId <= 0 || payload == null || payload.isEmpty()) { + return; + } + LocalDateTime now = LocalDateTime.now(); + for (BrandParsedFileCacheDto item : payload) { + if (item == null || isBlank(item.getFileUrl())) { + continue; + } + String scopeKey = normalize(item.getFileUrl()); + String scopeHash = hash(scopeKey); + String parsedJson = storeParsedPayload(taskId, scopeHash, writeJson(item, "Save brand task parsed payload failed")); + TaskScopeStateEntity state = getScopeState(taskId, scopeHash); + if (state == null) { + TaskScopeStateEntity entity = new TaskScopeStateEntity(); + entity.setTaskId(taskId); + entity.setModuleType(MODULE_TYPE); + entity.setScopeKey(scopeKey); + entity.setScopeHash(scopeHash); + entity.setParsedPayloadJson(parsedJson); + entity.setChunkTotal(0); + entity.setReceivedChunkCount(0); + entity.setCompleted(0); + entity.setCreatedAt(now); + entity.setUpdatedAt(now); + try { + taskScopeStateMapper.insert(entity); + continue; + } catch (DuplicateKeyException ignored) { + state = getScopeState(taskId, scopeHash); + } + } + if (state == null) { + throw new BusinessException("Save brand task parsed payload failed"); + } + deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), parsedJson); + taskScopeStateMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskScopeStateEntity::getId, state.getId()) + .set(TaskScopeStateEntity::getScopeKey, scopeKey) + .set(TaskScopeStateEntity::getParsedPayloadJson, parsedJson) + .set(TaskScopeStateEntity::getUpdatedAt, now)); + } + } + + public List getParsedPayload(Long taskId) { + if (taskId == null || taskId <= 0) { + return List.of(); + } + List states = taskScopeStateMapper.selectList(new LambdaQueryWrapper() + .select(TaskScopeStateEntity::getParsedPayloadJson) + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .isNotNull(TaskScopeStateEntity::getParsedPayloadJson)); + if (states == null || states.isEmpty()) { + return List.of(); + } + List result = new ArrayList<>(); + for (TaskScopeStateEntity state : states) { + String payloadJson = resolveParsedPayload(state.getParsedPayloadJson()); + if (isBlank(payloadJson)) { + continue; + } + try { + result.add(objectMapper.readValue(payloadJson, BrandParsedFileCacheDto.class)); + } catch (Exception ex) { + throw new BusinessException("Read brand task parsed payload failed"); + } + } + return result; + } + + @Transactional + public ChunkStoreResult storeChunk(Long taskId, BrandCrawlResultFileDto file) { + validateChunk(file); + String scopeKey = normalize(file.getFileUrl()); + String scopeHash = hash(scopeKey); + String payloadJson = writeJson(file, "Save brand task chunk failed"); + String payloadHash = hash(payloadJson); + LocalDateTime now = LocalDateTime.now(); + + TaskChunkEntity existingChunk = taskChunkMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, scopeHash) + .eq(TaskChunkEntity::getChunkIndex, file.getChunkIndex()) + .last("limit 1")); + if (existingChunk != null) { + if (payloadHash.equals(existingChunk.getPayloadHash())) { + BrandFileAggregateCacheDto aggregate = rebuildAggregate(taskId, scopeKey, scopeHash); + return new ChunkStoreResult(false, false, countCompletedFiles(taskId), aggregate); + } + throw new BusinessException("Brand task chunk conflict: " + scopeKey + "#" + file.getChunkIndex()); + } + + TaskChunkEntity entity = new TaskChunkEntity(); + entity.setTaskId(taskId); + entity.setModuleType(MODULE_TYPE); + entity.setScopeKey(scopeKey); + entity.setScopeHash(scopeHash); + entity.setChunkIndex(file.getChunkIndex()); + entity.setChunkTotal(file.getChunkTotal()); + entity.setPayloadJson(payloadJson); + entity.setPayloadHash(payloadHash); + entity.setCreatedAt(now); + entity.setUpdatedAt(now); + try { + taskChunkMapper.insert(entity); + } catch (DuplicateKeyException ex) { + TaskChunkEntity concurrentChunk = taskChunkMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, scopeHash) + .eq(TaskChunkEntity::getChunkIndex, file.getChunkIndex()) + .last("limit 1")); + if (concurrentChunk != null && payloadHash.equals(concurrentChunk.getPayloadHash())) { + BrandFileAggregateCacheDto aggregate = rebuildAggregate(taskId, scopeKey, scopeHash); + return new ChunkStoreResult(false, false, countCompletedFiles(taskId), aggregate); + } + throw new BusinessException("Brand task chunk conflict: " + scopeKey + "#" + file.getChunkIndex()); + } + + BrandFileAggregateCacheDto aggregate = rebuildAggregate(taskId, scopeKey, scopeHash); + boolean completed = Boolean.TRUE.equals(aggregate.getCompleted()); + saveAggregate(taskId, scopeKey, scopeHash, aggregate, now); + return new ChunkStoreResult(true, completed, countCompletedFiles(taskId), aggregate); + } + + public BrandFileAggregateCacheDto getFileAggregate(Long taskId, String fileUrl) { + TaskScopeStateEntity state = getScopeState(taskId, hash(normalize(fileUrl))); + if (state == null || isBlank(state.getStateJson())) { + return null; + } + try { + return objectMapper.readValue(state.getStateJson(), BrandFileAggregateCacheDto.class); + } catch (Exception ex) { + throw new BusinessException("Read brand task aggregate failed"); + } + } + + public Map getAllFileAggregates(Long taskId) { + if (taskId == null || taskId <= 0) { + return Map.of(); + } + List states = taskScopeStateMapper.selectList(new LambdaQueryWrapper() + .select(TaskScopeStateEntity::getScopeKey, TaskScopeStateEntity::getStateJson) + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .isNotNull(TaskScopeStateEntity::getStateJson)); + if (states == null || states.isEmpty()) { + return Map.of(); + } + Map result = new LinkedHashMap<>(); + for (TaskScopeStateEntity state : states) { + if (isBlank(state.getStateJson()) || isBlank(state.getScopeKey())) { + continue; + } + try { + result.put(state.getScopeKey(), objectMapper.readValue(state.getStateJson(), BrandFileAggregateCacheDto.class)); + } catch (Exception ex) { + throw new BusinessException("Read brand task aggregate failed"); + } + } + return result; + } + + public int countCompletedFiles(Long taskId) { + if (taskId == null || taskId <= 0) { + return 0; + } + Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .eq(TaskScopeStateEntity::getCompleted, 1)); + return count == null ? 0 : count.intValue(); + } + + @Transactional + public void deleteTaskData(Long taskId) { + if (taskId == null || taskId <= 0) { + return; + } + List states = taskScopeStateMapper.selectList(new LambdaQueryWrapper() + .select(TaskScopeStateEntity::getParsedPayloadJson) + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); + if (states != null) { + for (TaskScopeStateEntity state : states) { + deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), null); + } + } + taskChunkMapper.delete(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); + taskScopeStateMapper.delete(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); + } + + private void saveAggregate(Long taskId, + String scopeKey, + String scopeHash, + BrandFileAggregateCacheDto aggregate, + LocalDateTime now) { + String aggregateJson = writeJson(aggregate, "Save brand task aggregate failed"); + TaskScopeStateEntity state = getScopeState(taskId, scopeHash); + if (state == null) { + throw new BusinessException("Brand task aggregate state missing: " + scopeKey); + } + int chunkTotal = aggregate.getChunkTotal() == null ? 0 : aggregate.getChunkTotal(); + int receivedCount = aggregate.getReceivedChunkCount() == null ? 0 : aggregate.getReceivedChunkCount(); + taskScopeStateMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskScopeStateEntity::getId, state.getId()) + .set(TaskScopeStateEntity::getScopeKey, scopeKey) + .set(TaskScopeStateEntity::getStateJson, aggregateJson) + .set(TaskScopeStateEntity::getChunkTotal, chunkTotal) + .set(TaskScopeStateEntity::getReceivedChunkCount, receivedCount) + .set(TaskScopeStateEntity::getCompleted, Boolean.TRUE.equals(aggregate.getCompleted()) ? 1 : 0) + .set(TaskScopeStateEntity::getLastChunkAt, now) + .set(TaskScopeStateEntity::getLastError, null) + .set(TaskScopeStateEntity::getUpdatedAt, now)); + } + + private TaskScopeStateEntity getScopeState(Long taskId, String scopeHash) { + return taskScopeStateMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .eq(TaskScopeStateEntity::getScopeHash, scopeHash) + .last("limit 1")); + } + + private BrandFileAggregateCacheDto createAggregate(BrandCrawlResultFileDto file) { + BrandFileAggregateCacheDto aggregate = new BrandFileAggregateCacheDto(); + aggregate.setFileUrl(normalize(file.getFileUrl())); + aggregate.setOriginalFilename(file.getOriginalFilename()); + aggregate.setRelativePath(file.getRelativePath()); + aggregate.setMainSheetName(file.getMainSheetName()); + aggregate.setChunkTotal(file.getChunkTotal()); + aggregate.setTotalLines(safePositive(file.getTotalLines())); + aggregate.setReceivedChunkCount(0); + aggregate.setProcessedLineCount(0); + aggregate.setCompleted(false); + aggregate.setInvalidBrands(new ArrayList<>()); + aggregate.setQueryFailedBrands(new ArrayList<>()); + return aggregate; + } + + private void ensureChunkConsistency(BrandFileAggregateCacheDto aggregate, BrandCrawlResultFileDto file) { + if (aggregate.getChunkTotal() != null && file.getChunkTotal() != null && !aggregate.getChunkTotal().equals(file.getChunkTotal())) { + throw new BusinessException("Brand task chunkTotal mismatch: " + aggregate.getFileUrl()); + } + if (aggregate.getChunkTotal() == null) { + aggregate.setChunkTotal(file.getChunkTotal()); + } + int incomingTotalLines = safePositive(file.getTotalLines()); + if (incomingTotalLines > 0) { + aggregate.setTotalLines(Math.max(safePositive(aggregate.getTotalLines()), incomingTotalLines)); + } + if (isBlank(aggregate.getOriginalFilename()) && !isBlank(file.getOriginalFilename())) { + aggregate.setOriginalFilename(file.getOriginalFilename()); + } + if (isBlank(aggregate.getRelativePath()) && !isBlank(file.getRelativePath())) { + aggregate.setRelativePath(file.getRelativePath()); + } + if (isBlank(aggregate.getMainSheetName()) && !isBlank(file.getMainSheetName())) { + aggregate.setMainSheetName(file.getMainSheetName()); + } + } + + private void mergeAggregate(BrandFileAggregateCacheDto aggregate, BrandCrawlResultFileDto file) { + int processed = safePositive(aggregate.getProcessedLineCount()) + + sizeOf(file.getKeptRows()) + + sizeOf(file.getInvalidBrands()) + + sizeOf(file.getQueryFailedBrands()); + aggregate.setProcessedLineCount(processed); + if (file.getInvalidBrands() != null && !file.getInvalidBrands().isEmpty()) { + aggregate.getInvalidBrands().addAll(file.getInvalidBrands()); + } + if (file.getQueryFailedBrands() != null && !file.getQueryFailedBrands().isEmpty()) { + aggregate.getQueryFailedBrands().addAll(file.getQueryFailedBrands()); + } + } + + private BrandFileAggregateCacheDto rebuildAggregate(Long taskId, String scopeKey, String scopeHash) { + List chunks = taskChunkMapper.selectList(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, scopeHash) + .orderByAsc(TaskChunkEntity::getChunkIndex, TaskChunkEntity::getId)); + if (chunks == null || chunks.isEmpty()) { + BrandFileAggregateCacheDto aggregate = getFileAggregate(taskId, scopeKey); + if (aggregate != null) { + aggregate.setFileUrl(scopeKey); + aggregate.setReceivedChunkCount(0); + aggregate.setProcessedLineCount(0); + aggregate.setCompleted(false); + aggregate.setInvalidBrands(new ArrayList<>()); + aggregate.setQueryFailedBrands(new ArrayList<>()); + } + return aggregate; + } + + BrandFileAggregateCacheDto aggregate = null; + for (TaskChunkEntity chunk : chunks) { + BrandCrawlResultFileDto chunkPayload = readChunkPayload(chunk.getPayloadJson()); + if (aggregate == null) { + aggregate = createAggregate(chunkPayload); + } else { + ensureChunkConsistency(aggregate, chunkPayload); + } + mergeAggregate(aggregate, chunkPayload); + } + if (aggregate == null) { + throw new BusinessException("Read brand task aggregate data failed"); + } + aggregate.setFileUrl(scopeKey); + aggregate.setReceivedChunkCount(chunks.size()); + aggregate.setCompleted(aggregate.getChunkTotal() != null + && aggregate.getChunkTotal() > 0 + && chunks.size() >= aggregate.getChunkTotal()); + return aggregate; + } + + private BrandCrawlResultFileDto readChunkPayload(String payloadJson) { + try { + return objectMapper.readValue(payloadJson, BrandCrawlResultFileDto.class); + } catch (Exception ex) { + throw new BusinessException("Read brand task chunk failed"); + } + } + + private void validateChunk(BrandCrawlResultFileDto file) { + if (file == null) { + throw new BusinessException("Brand task chunk must not be null"); + } + if (file.getChunkIndex() == null || file.getChunkTotal() == null || file.getChunkIndex() <= 0 + || file.getChunkTotal() <= 0 || file.getChunkIndex() > file.getChunkTotal()) { + throw new BusinessException("Brand task chunk parameters are invalid"); + } + if (isBlank(file.getFileUrl())) { + throw new BusinessException("fileUrl must not be blank"); + } + } + + private String storeParsedPayload(Long taskId, String scopeHash, String payloadJson) { + try { + String pointer = OSS_POINTER_PREFIX + ossStorageService.uploadTaskParsedPayload(MODULE_TYPE, taskId, scopeHash, payloadJson); + return objectMapper.writeValueAsString(pointer); + } catch (Exception ex) { + throw new BusinessException("Save brand task parsed payload failed"); + } + } + + private String resolveParsedPayload(String value) { + if (isBlank(value)) { + return value; + } + String ossPointer = extractOssPointer(value); + if (ossPointer == null) { + return value; + } + try { + return ossStorageService.readObjectAsString(ossPointer.substring(OSS_POINTER_PREFIX.length())); + } catch (Exception ex) { + throw new BusinessException("Read brand task parsed payload failed"); + } + } + + private void deleteParsedPayloadIfNeeded(String oldValue, String newValue) { + String oldPointer = extractOssPointer(oldValue); + String newPointer = extractOssPointer(newValue); + if (oldPointer == null || oldPointer.equals(newPointer)) { + return; + } + try { + ossStorageService.deleteObject(oldPointer.substring(OSS_POINTER_PREFIX.length())); + } catch (Exception ignored) { + } + } + + private String extractOssPointer(String value) { + if (isBlank(value)) { + return null; + } + if (value.startsWith(OSS_POINTER_PREFIX)) { + return value; + } + try { + String decoded = objectMapper.readValue(value, String.class); + return decoded != null && decoded.startsWith(OSS_POINTER_PREFIX) ? decoded : null; + } catch (Exception ignored) { + return null; + } + } + + private String writeJson(Object value, String message) { + try { + return objectMapper.writeValueAsString(value); + } catch (Exception ex) { + throw new BusinessException(message); + } + } + + private String normalize(String value) { + return value == null ? "" : value.trim(); + } + + private boolean isBlank(String value) { + return value == null || value.isBlank(); + } + + private int sizeOf(List list) { + return list == null ? 0 : list.size(); + } + + private int safePositive(Integer value) { + return value == null || value <= 0 ? 0 : value; + } + + private String hash(String value) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] bytes = digest.digest(normalize(value).getBytes(StandardCharsets.UTF_8)); + StringBuilder sb = new StringBuilder(bytes.length * 2); + for (byte b : bytes) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); + } catch (Exception ex) { + throw new IllegalStateException("failed to hash brand scope", ex); + } + } + + public record ChunkStoreResult(boolean stored, boolean newlyCompletedFile, int finishedFiles, BrandFileAggregateCacheDto aggregate) { + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertRunService.java index 65a01e6..b8d3fd2 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertRunService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertRunService.java @@ -44,11 +44,11 @@ public class ConvertRunService { private static final String MODULE_TYPE = "CONVERT"; private static final String TEMPLATE_CODE_FIVE_COUNTRIES = "uk_offer"; private static final List FIVE_COUNTRY_OUTPUT_FILES = List.of( - "\u82f1\u56fd.txt", - "\u6cd5\u56fd.txt", - "\u5fb7\u56fd.txt", - "\u897f\u73ed\u7259.txt", - "\u610f\u5927\u5229.txt" + "英国.txt", + "法国.txt", + "德国.txt", + "西班牙.txt", + "意大利.txt" ); private final FileTaskMapper fileTaskMapper; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java index 11a676a..b5f5bb0 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java @@ -461,8 +461,8 @@ public class DedupeRunService { if (value == null) { return ""; } - return value.replace("\ufeff", "") - .replace("\u3000", " ") + return value.replace("", "") + .replace("　", " ") .replace("\r\n", " ") .replace("\r", " ") .replace("\n", " ") diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java index 763b53e..0516837 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java @@ -229,7 +229,7 @@ public class DedupeTotalDataService { progress.setStatus("success"); } catch (Exception e) { progress.setStatus("failed"); - progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "鍒犻櫎 Excel 鍖归厤鏁版嵁澶辫触"); + progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "删除 Excel 匹配数据失败"); } finally { deleteQuietly(tempFile); } @@ -251,7 +251,7 @@ public class DedupeTotalDataService { progress.setStatus("success"); } catch (Exception e) { progress.setStatus("failed"); - progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "瀵煎叆 Excel 澶辫触"); + progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "导入 Excel 失败"); } finally { deleteQuietly(tempFile); } @@ -590,8 +590,8 @@ public class DedupeTotalDataService { if (value == null) { return ""; } - return value.replace("\ufeff", "") - .replace("\u3000", " ") + return value.replace("", "") + .replace("　", " ") .replace("\r\n", " ") .replace("\r", " ") .replace("\n", " ") diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/controller/DeleteBrandRunController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/controller/DeleteBrandRunController.java index 94a7ade..b259a9b 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/controller/DeleteBrandRunController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/controller/DeleteBrandRunController.java @@ -135,9 +135,10 @@ public class DeleteBrandRunController { try { String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20"); + String asciiFilename = buildAsciiDownloadFilename(filename, taskId); response.setContentType("application/octet-stream"); response.setHeader(HttpHeaders.CONTENT_DISPOSITION, - "attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename); + "attachment; filename=\"" + asciiFilename + "\"; filename*=UTF-8''" + encodedFilename); try (InputStream in = URI.create(url).toURL().openStream()) { byte[] buffer = new byte[65536]; int read; @@ -150,4 +151,24 @@ public class DeleteBrandRunController { throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "下载失败"); } } + + private String buildAsciiDownloadFilename(String filename, Long taskId) { + String sanitized = filename == null ? "" : filename.replaceAll("[^A-Za-z0-9._-]", "_"); + sanitized = sanitized.replaceAll("_+", "_"); + sanitized = sanitized.replaceAll("^[_\\.]+|[_\\.]+$", ""); + if (!sanitized.isBlank() && sanitized.contains(".")) { + return sanitized; + } + String ext = "xlsx"; + if (filename != null) { + int dotIndex = filename.lastIndexOf('.'); + if (dotIndex >= 0 && dotIndex < filename.length() - 1) { + String rawExt = filename.substring(dotIndex + 1).replaceAll("[^A-Za-z0-9]", ""); + if (!rawExt.isBlank()) { + ext = rawExt; + } + } + } + return "delete-brand_" + taskId + "." + ext; + } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandRunService.java index 4a0805c..f7d5a63 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandRunService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandRunService.java @@ -44,7 +44,7 @@ import org.apache.poi.ss.usermodel.DataFormatter; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; -import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -74,6 +74,7 @@ public class DeleteBrandRunService { private final FileResultMapper fileResultMapper; private final LocalFileStorageService localFileStorageService; private final DeleteBrandTaskCacheService deleteBrandTaskCacheService; + private final DeleteBrandTaskStorageService deleteBrandTaskStorageService; private final ZiniaoShopSwitchService ziniaoShopSwitchService; private final OssStorageService ossStorageService; private final ObjectMapper objectMapper; @@ -221,9 +222,7 @@ public class DeleteBrandRunService { deleteBrandTaskCacheService.saveTaskCache(task); } - if (!parsedPayloadByFileIdentity.isEmpty()) { - deleteBrandTaskCacheService.saveParsedPayload(task.getId(), parsedPayloadByFileIdentity); - } + deleteBrandTaskStorageService.saveParsedPayload(task.getId(), parsedPayloadByFileIdentity); DeleteBrandRunVo vo = new DeleteBrandRunVo(); vo.setTotal(request.getFiles().size()); @@ -277,7 +276,7 @@ public class DeleteBrandRunService { item.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1); item.setError(entity.getErrorMessage()); - // 以 task.resultJson 中的项为准补全 matched/shopId/platform/openStoreUrl/错误信息 + // 以 task.resultJson 中的项为准补充 matched/shopId/platform/openStoreUrl/错误信息 // 真实补充 if (entity.getTaskId() != null) { item.setTaskStatus(statusByTaskId.get(entity.getTaskId())); @@ -291,7 +290,7 @@ public class DeleteBrandRunService { item.setMatched(candidate.isMatched()); item.setMatchStatus(candidate.getMatchStatus()); item.setMatchMessage(candidate.getMatchMessage()); - // 如果物理结果已经是成功了，说明已经处理过了，强制置为匹配成功 + // 如果物理结果已经成功，说明已经处理过，强制标记为匹配成功 if (item.isSuccess() && candidate.getShopId() != null && !candidate.getShopId().isBlank()) { item.setMatched(true); item.setMatchStatus(ZiniaoShopIndexService.MATCH_STATUS_MATCHED); @@ -303,8 +302,8 @@ public class DeleteBrandRunService { item.setPlatform(candidate.getPlatform()); item.setOpenStoreUrl(candidate.getOpenStoreUrl()); - // 历史表可能只有通用 errorMessage；优先返回任务当时的真实 error - // 但是，如果物理结果是成功的，就不应该显示所谓的“未匹配”报错 + // 历史表里可能只有通用 errorMessage，优先返回任务当时的真实 error + // 但如果物理结果已成功，就不应该再显示所谓“未匹配”的报错 if (item.isSuccess()) { item.setError(null); } else if ((item.getError() == null || item.getError().isBlank()) && candidate.getError() != null && !candidate.getError().isBlank()) { @@ -344,7 +343,16 @@ public class DeleteBrandRunService { if (entity == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) { throw new BusinessException("记录不存在"); } + Long taskId = entity.getTaskId(); fileResultMapper.deleteById(resultId); + if (taskId != null && taskId > 0) { + Long remaining = fileResultMapper.selectCount(new LambdaQueryWrapper() + .eq(FileResultEntity::getTaskId, taskId) + .eq(FileResultEntity::getModuleType, MODULE_TYPE)); + if (remaining == null || remaining <= 0) { + deleteBrandTaskStorageService.deleteTaskData(taskId); + } + } } private ParsedDeleteBrandFile parseDeleteBrandFile(File inputFile) { @@ -468,8 +476,8 @@ public class DeleteBrandRunService { if (value == null) { return ""; } - return value.replace("\ufeff", "") - .replace("\u3000", " ") + return value.replace("", "") + .replace("　", " ") .replace("\r\n", " ") .replace("\r", " ") .replace("\n", " ") @@ -640,7 +648,7 @@ public class DeleteBrandRunService { taskVo.setCreatedAt(task.getCreatedAt()); taskVo.setUpdatedAt(task.getUpdatedAt()); taskVo.setFinishedAt(task.getFinishedAt()); - // 轮询期避免每次都查 biz_file_result（会放大 DB 压力）；仅任务成功后再提供下载信息 + // 轮询期间避免每次都查 biz_file_result，放大 DB 压力；仅任务成功后再提供下载信息 if ("SUCCESS".equals(task.getStatus())) { taskVo.setDownloadUrl(resolveTaskDownloadUrl(task.getId())); taskVo.setDownloadFilename(resolveTaskDownloadFilename(task.getId())); @@ -996,9 +1004,7 @@ public class DeleteBrandRunService { throw new BusinessException("任务已结束，拒绝继续提交结果"); } - Map parsedPayload = deleteBrandTaskCacheService.getParsedPayload(taskId, - new TypeReference>() { - }); + Map parsedPayload = deleteBrandTaskStorageService.loadParsedPayload(taskId); if (parsedPayload == null || parsedPayload.isEmpty()) { throw new BusinessException("任务原始数据已过期，请重新执行解析"); } @@ -1029,10 +1035,10 @@ public class DeleteBrandRunService { throw new BusinessException("文件标识不匹配: " + fileIdentity); } - // 更早的 duplicate fast-path：尽量在较重的校验/进度写入之前直接短路 + // 更早执行 duplicate fast-path，尽量在较重校验和进度写入前直接短路 validateChunk(fileDto, parsedFile); validateResultFileAgainstParsed(fileDto, parsedFile); - boolean stored = deleteBrandTaskCacheService.storeResultChunkIfChanged(taskId, fileIdentity, fileDto.getChunkIndex(), fileDto); + boolean stored = deleteBrandTaskStorageService.storeResultChunkIfChanged(taskId, fileIdentity, fileDto.getChunkIndex(), fileDto); if (!stored) { log.info("[DeleteBrand] Duplicate chunk ignored for taskId: {}, file: {}, chunkIndex: {}", taskId, fileIdentity, fileDto.getChunkIndex()); continue; @@ -1051,12 +1057,12 @@ public class DeleteBrandRunService { progress.put("last_heartbeat_at", String.valueOf(System.currentTimeMillis())); deleteBrandTaskCacheService.saveProgress(taskId, progress, true); - // 如果该文件已完成，立即更新成品计数的 Redis 缓存「提示」，让后续 tryFinalizeTask 能更精确 + // 如果该文件已完成，立即更新成品计数的 Redis 提示，让后续 tryFinalizeTask 更准确 if (isFileCompleted(fileDto)) { log.info("[DeleteBrand] File chunk transmission fully reached via chunkTotal for taskId: {}, file: {}", taskId, fileDto.getSourceFilename()); shouldTryFinalize = true; - // 这里我们暂不直接 increment，而是标记需要重算或在 tryFinalizeTask 中重新计算 - // 为了保险，我们直接在 tryFinalizeTask 里重新遍历分片状态 + // 这里暂不直接 increment，而是标记需要在 tryFinalizeTask 中重新计算 + // 为了保险，直接在 tryFinalizeTask 里重新遍历分片状态 } } @@ -1079,26 +1085,24 @@ public class DeleteBrandRunService { return; } - Map parsedPayload = deleteBrandTaskCacheService.getParsedPayload(taskId, - new TypeReference>() { - }); + Map parsedPayload = deleteBrandTaskStorageService.loadParsedPayload(taskId); if (parsedPayload == null || parsedPayload.isEmpty()) { return; } int expectedFiles = parsedPayload.size(); - // 移除 finishedFilesHint 强行返回逻辑，因为 Redis 里的 finished_files 可能是旧的， - // 既然进入了 tryFinalizeTask，就说明有新片到达，应该以 loadMergedChunks 为准 + // 移除 finishedFilesHint 的强依赖，因为 Redis 里的 finished_files 可能是旧值 + // 既然进入 tryFinalizeTask，就说明有新分片到达，应该以 loadMergedChunks 为准 - Map> mergedByFile = loadMergedChunks(taskId); + Map> mergedByFile = deleteBrandTaskStorageService.loadMergedChunks(taskId); int finishedFiles = countCompletedFiles(mergedByFile); if (finishedFiles < expectedFiles) { Map progress = new LinkedHashMap<>(); progress.put("finished_files", String.valueOf(finishedFiles)); - // 定时补偿只负责“再试一次 finalize”，不能把缺片任务重新续命， - // 否则 stale-check 永远看不到超时任务。 + // 定时补偿只负责“再试一次 finalize”，不能把缺片任务重新续命 + // 否则 stale-check 会一直看不到超时任务 if (!fromCompensation) { progress.put("updated_at", String.valueOf(System.currentTimeMillis())); } @@ -1133,24 +1137,6 @@ public class DeleteBrandRunService { finalizeTask(task, parsedPayload, mergedByFile); } - private Map> loadMergedChunks(Long taskId) { - Map> groupedJson = deleteBrandTaskCacheService.groupResultChunkJsonByFile(taskId); - Map> merged = new LinkedHashMap<>(); - for (Map.Entry> entry : groupedJson.entrySet()) { - List chunks = new ArrayList<>(); - for (String raw : entry.getValue()) { - try { - chunks.add(objectMapper.readValue(raw, DeleteBrandResultFileDto.class)); - } catch (Exception ex) { - throw new BusinessException("读取删除品牌结果分片失败"); - } - } - chunks.sort(Comparator.comparing(DeleteBrandResultFileDto::getChunkIndex, Comparator.nullsLast(Integer::compareTo))); - merged.put(entry.getKey(), chunks); - } - return merged; - } - private int countCompletedFiles(Map> mergedByFile) { int finishedFiles = 0; for (List chunks : mergedByFile.values()) { @@ -1187,6 +1173,7 @@ public class DeleteBrandRunService { private void finalizeTask(FileTaskEntity task, Map parsedPayload, Map> mergedByFile) { + File outputDir = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "delete-brand-result", String.valueOf(task.getId()))); try { deleteBrandTaskCacheService.saveProgress(task.getId(), Map.of( "phase", DeleteBrandTaskCacheService.PHASE_ASSEMBLING, @@ -1259,6 +1246,8 @@ public class DeleteBrandRunService { item.setPlatform(parsedFile.getPlatform()); item.setOpenStoreUrl(parsedFile.getOpenStoreUrl()); item.setMatched(parsedFile.getShopId() != null && !parsedFile.getShopId().isBlank()); + item.setSuccess(true); + item.setError(null); item.setMatchStatus(item.isMatched() ? ZiniaoShopIndexService.MATCH_STATUS_MATCHED : ZiniaoShopIndexService.MATCH_STATUS_PENDING); item.setMatchMessage(item.isMatched() ? null : "店铺索引信息缺失，请重新匹配索引"); finalItems.add(item); @@ -1267,12 +1256,14 @@ public class DeleteBrandRunService { task.setStatus("SUCCESS"); task.setSuccessFileCount(successCount); + task.setFailedFileCount(Math.max(0, parsedPayload.size() - successCount)); task.setErrorMessage(null); task.setResultJson(toCompactTaskResultJsonForDb(finalItems)); task.setUpdatedAt(LocalDateTime.now()); task.setFinishedAt(LocalDateTime.now()); fileTaskMapper.updateById(task); deleteBrandTaskCacheService.saveTaskCache(task); + deleteBrandTaskStorageService.deleteTaskData(task.getId()); deleteBrandTaskCacheService.saveProgress(task.getId(), Map.of( "phase", "success", @@ -1297,6 +1288,8 @@ public class DeleteBrandRunService { throw businessException; } throw new BusinessException("删除品牌结果组装失败"); + } finally { + deleteQuietly(outputDir); } } @@ -1355,7 +1348,9 @@ public class DeleteBrandRunService { File outputDir = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "delete-brand-result", String.valueOf(taskId))); File outputFile = buildNamedOutputFile(outputDir, blankToDefault(parsedFile.getSourceFilename(), "delete-brand-result.xlsx")); - try (Workbook workbook = new XSSFWorkbook(); FileOutputStream outputStream = new FileOutputStream(outputFile)) { + try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); + FileOutputStream outputStream = new FileOutputStream(outputFile)) { + workbook.setCompressTempFiles(true); Sheet sheet = workbook.createSheet("删除品牌结果"); Row headerRow = sheet.createRow(0); headerRow.createCell(0).setCellValue("国家"); @@ -1371,9 +1366,7 @@ public class DeleteBrandRunService { createTextCell(row, 2, item.getStatus()); } } - for (int i = 0; i < 3; i++) { - sheet.autoSizeColumn(i); - } + applyDeleteBrandColumnWidths(sheet, 3); workbook.write(outputStream); return outputFile; } catch (Exception ex) { @@ -1387,7 +1380,9 @@ public class DeleteBrandRunService { File outputDir = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "delete-brand-result", String.valueOf(taskId))); File outputFile = buildNamedOutputFile(outputDir, blankToDefault(parsedFile.getSourceFilename(), "delete-brand-result.xlsx")); - try (Workbook workbook = new XSSFWorkbook(); FileOutputStream outputStream = new FileOutputStream(outputFile)) { + try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); + FileOutputStream outputStream = new FileOutputStream(outputFile)) { + workbook.setCompressTempFiles(true); Sheet sheet = workbook.createSheet("删除品牌结果"); Row countryRow = sheet.createRow(0); Row headerRow = sheet.createRow(1); @@ -1417,9 +1412,7 @@ public class DeleteBrandRunService { } } - for (int i = 0; i < mergedFile.countries().size() * 2; i++) { - sheet.autoSizeColumn(i); - } + applyDeleteBrandColumnWidths(sheet, mergedFile.countries().size() * 2); workbook.write(outputStream); return outputFile; } catch (Exception ex) { @@ -1432,6 +1425,22 @@ public class DeleteBrandRunService { cell.setCellValue(value == null ? "" : value); } + private void applyDeleteBrandColumnWidths(Sheet sheet, int columnCount) { + for (int i = 0; i < columnCount; i++) { + sheet.setColumnWidth(i, 18 * 256); + } + } + + private void deleteQuietly(File file) { + if (file == null || !file.exists()) { + return; + } + try { + FileUtil.del(file); + } catch (Exception ignored) { + } + } + private File buildNamedOutputFile(File outputDir, String filename) { File candidate = FileUtil.file(outputDir, filename); if (!candidate.exists()) { @@ -1478,10 +1487,9 @@ public class DeleteBrandRunService { String normalized = blankToEmpty(value); return normalized.isBlank() ? defaultValue : normalized; } - /** * 落库用 result_json：去掉国家分组、预览行等大字段，保留匹配摘要与计数。 - * 索引常命中后单次任务 JSON 体积会陡增，易触发 max_allowed_packet / 更新失败，进而前端 unwrap 报「请求失败」。 + * 索引命中后单次任务 JSON 体积会明显膨胀，容易触发 max_allowed_packet 或更新失败。 */ private String toCompactTaskResultJsonForDb(List items) { if (items == null || items.isEmpty()) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskCacheService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskCacheService.java index e13a50d..2552055 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskCacheService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskCacheService.java @@ -1,16 +1,11 @@ package com.nanri.aiimage.modules.deletebrand.service; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.config.TaskPressureProperties; import lombok.RequiredArgsConstructor; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.stereotype.Service; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; import java.time.Duration; import java.util.concurrent.ConcurrentHashMap; @@ -24,63 +19,16 @@ public class DeleteBrandTaskCacheService { public static final String PHASE_FAILED = "failed"; private static final long PAYLOAD_TTL_HOURS = 24; - private static final long LOCAL_PARSED_PAYLOAD_CACHE_MILLIS = 3000; private static final long LOCAL_PROGRESS_CACHE_MILLIS = 1500; private static final long MIN_PROGRESS_REDIS_FLUSH_MILLIS = 1200; private final StringRedisTemplate stringRedisTemplate; private final ObjectMapper objectMapper; private final TaskPressureProperties taskPressureProperties; - private final ConcurrentHashMap parsedPayloadLocalCache = new ConcurrentHashMap<>(); private final ConcurrentHashMap progressLocalCache = new ConcurrentHashMap<>(); private final ConcurrentHashMap progressRedisFlushAt = new ConcurrentHashMap<>(); private final ConcurrentHashMap taskEntityLocalCache = new ConcurrentHashMap<>(); - public void saveParsedPayload(Long taskId, Object payload) { - try { - Files.createDirectories(buildTaskDir(taskId)); - Files.writeString( - buildPayloadFile(taskId), - objectMapper.writeValueAsString(payload), - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING, - StandardOpenOption.WRITE - ); - parsedPayloadLocalCache.put(taskId, new LocalParsedPayloadCacheEntry(System.currentTimeMillis(), payload)); - } catch (Exception ex) { - throw new BusinessException("鏆傚瓨鍒犻櫎鍝佺墝鍘熷鏁版嵁澶辫触"); - } - } - - public T getParsedPayload(Long taskId, TypeReference typeReference) { - if (taskId == null || taskId <= 0) { - return null; - } - - long now = System.currentTimeMillis(); - LocalParsedPayloadCacheEntry cached = parsedPayloadLocalCache.get(taskId); - if (cached != null && cached.isFresh(now) && cached.value() != null) { - try { - return objectMapper.convertValue(cached.value(), typeReference); - } catch (Exception ignored) { - // fall through to file - } - } - - Path payloadFile = buildPayloadFile(taskId); - if (!Files.isRegularFile(payloadFile)) { - parsedPayloadLocalCache.remove(taskId); - return null; - } - try { - T parsed = objectMapper.readValue(Files.readString(payloadFile), typeReference); - parsedPayloadLocalCache.put(taskId, new LocalParsedPayloadCacheEntry(now, parsed)); - return parsed; - } catch (Exception ex) { - throw new BusinessException("璇诲彇鍒犻櫎鍝佺墝鍘熷鏁版嵁澶辫触"); - } - } - public void saveProgress(Long taskId, java.util.Map values) { saveProgress(taskId, values, false); } @@ -150,13 +98,15 @@ public class DeleteBrandTaskCacheService { return result; } - java.util.List pipelineResults = stringRedisTemplate.executePipelined((org.springframework.data.redis.core.RedisCallback) connection -> { - org.springframework.data.redis.serializer.RedisSerializer serializer = stringRedisTemplate.getStringSerializer(); - for (Long taskId : missingTaskIds) { - connection.hGetAll(serializer.serialize(buildProgressKey(taskId))); - } - return null; - }); + java.util.List pipelineResults = stringRedisTemplate.executePipelined( + (org.springframework.data.redis.core.RedisCallback) connection -> { + org.springframework.data.redis.serializer.RedisSerializer serializer = + stringRedisTemplate.getStringSerializer(); + for (Long taskId : missingTaskIds) { + connection.hGetAll(serializer.serialize(buildProgressKey(taskId))); + } + return null; + }); for (int i = 0; i < missingTaskIds.size(); i++) { Long taskId = missingTaskIds.get(i); @@ -175,83 +125,12 @@ public class DeleteBrandTaskCacheService { return result; } - public boolean hasSameResultChunk(Long taskId, String fileIdentity, Integer chunkIndex, Object chunk) { - if (chunkIndex == null || chunkIndex <= 0) { - throw new BusinessException("chunkIndex 涓嶅悎娉?"); - } - if (fileIdentity == null || fileIdentity.isBlank()) { - throw new BusinessException("fileIdentity 涓嶈兘涓虹┖"); - } - String field = fileIdentity.trim() + "#" + chunkIndex; - String chunkJson; - try { - chunkJson = objectMapper.writeValueAsString(chunk); - } catch (Exception ex) { - throw new BusinessException("璇诲彇鍒犻櫎鍝佺墝缁撴灉鍒嗙墖澶辫触"); - } - Object existing = stringRedisTemplate.opsForHash().get(buildResultChunksKey(taskId), field); - if (chunkJson.equals(existing)) { - stringRedisTemplate.expire(buildResultChunksKey(taskId), Duration.ofHours(PAYLOAD_TTL_HOURS)); - return true; - } - return false; - } - - public boolean mergeResultChunk(Long taskId, String fileIdentity, Integer chunkIndex, Object chunk) { - String resultKey = buildResultChunksKey(taskId); - if (chunkIndex == null || chunkIndex <= 0) { - throw new BusinessException("chunkIndex 涓嶅悎娉?"); - } - if (fileIdentity == null || fileIdentity.isBlank()) { - throw new BusinessException("fileIdentity 涓嶈兘涓虹┖"); - } - String field = fileIdentity.trim() + "#" + chunkIndex; - String chunkJson; - try { - chunkJson = objectMapper.writeValueAsString(chunk); - } catch (Exception ex) { - throw new BusinessException("鏆傚瓨鍒犻櫎鍝佺墝缁撴灉鍒嗙墖澶辫触"); - } - Object existing = stringRedisTemplate.opsForHash().get(resultKey, field); - if (chunkJson.equals(existing)) { - stringRedisTemplate.expire(resultKey, Duration.ofHours(PAYLOAD_TTL_HOURS)); - return false; - } - stringRedisTemplate.opsForHash().put(resultKey, field, chunkJson); - stringRedisTemplate.expire(resultKey, Duration.ofHours(PAYLOAD_TTL_HOURS)); - return true; - } - - public boolean storeResultChunkIfChanged(Long taskId, String fileIdentity, Integer chunkIndex, Object chunk) { - return mergeResultChunk(taskId, fileIdentity, chunkIndex, chunk); - } - - public java.util.Map> groupResultChunkJsonByFile(Long taskId) { - java.util.Map stored = stringRedisTemplate.opsForHash().entries(buildResultChunksKey(taskId)); - java.util.Map> grouped = new java.util.LinkedHashMap<>(); - for (java.util.Map.Entry entry : stored.entrySet()) { - if (!(entry.getKey() instanceof String field) || !(entry.getValue() instanceof String raw) || raw.isBlank()) { - continue; - } - int sep = field.lastIndexOf('#'); - if (sep <= 0) { - continue; - } - String fileIdentity = field.substring(0, sep); - grouped.computeIfAbsent(fileIdentity, ignored -> new java.util.ArrayList<>()).add(raw); - } - return grouped; - } - public void delete(Long taskId) { - parsedPayloadLocalCache.remove(taskId); progressLocalCache.remove(taskId); progressRedisFlushAt.remove(taskId); taskEntityLocalCache.remove(taskId); stringRedisTemplate.delete(buildProgressKey(taskId)); - stringRedisTemplate.delete(buildResultChunksKey(taskId)); stringRedisTemplate.delete(buildTaskEntityKey(taskId)); - deleteTaskDirQuietly(taskId); } public void saveTaskCache(com.nanri.aiimage.modules.task.model.entity.FileTaskEntity task) { @@ -264,13 +143,18 @@ public class DeleteBrandTaskCacheService { objectMapper.convertValue(task, com.nanri.aiimage.modules.task.model.entity.FileTaskEntity.class) )); try { - stringRedisTemplate.opsForValue().set(buildTaskEntityKey(task.getId()), objectMapper.writeValueAsString(task), Duration.ofHours(PAYLOAD_TTL_HOURS)); + stringRedisTemplate.opsForValue().set( + buildTaskEntityKey(task.getId()), + objectMapper.writeValueAsString(task), + Duration.ofHours(PAYLOAD_TTL_HOURS)); } catch (Exception ignored) { } } - public java.util.Map getTaskCacheBatch(java.util.List taskIds) { - java.util.Map result = new java.util.LinkedHashMap<>(); + public java.util.Map getTaskCacheBatch( + java.util.List taskIds) { + java.util.Map result = + new java.util.LinkedHashMap<>(); if (taskIds == null || taskIds.isEmpty()) { return result; } @@ -285,7 +169,8 @@ public class DeleteBrandTaskCacheService { for (Long taskId : normalized) { LocalTaskEntityCacheEntry cached = taskEntityLocalCache.get(taskId); if (isLocalTaskEntityCacheFresh(cached, now)) { - result.put(taskId, objectMapper.convertValue(cached.task(), com.nanri.aiimage.modules.task.model.entity.FileTaskEntity.class)); + result.put(taskId, objectMapper.convertValue( + cached.task(), com.nanri.aiimage.modules.task.model.entity.FileTaskEntity.class)); } else { missingIds.add(taskId); } @@ -313,40 +198,10 @@ public class DeleteBrandTaskCacheService { return result; } - private Path buildTaskDir(Long taskId) { - return Path.of(System.getProperty("java.io.tmpdir"), "delete-brand-task-cache", String.valueOf(taskId)); - } - - private Path buildPayloadFile(Long taskId) { - return buildTaskDir(taskId).resolve("parsed-payload.json"); - } - - private void deleteTaskDirQuietly(Long taskId) { - Path taskDir = buildTaskDir(taskId); - if (!Files.exists(taskDir)) { - return; - } - try (var walk = Files.walk(taskDir)) { - walk.sorted(java.util.Comparator.reverseOrder()).forEach(path -> { - try { - Files.deleteIfExists(path); - } catch (Exception ignored) { - } - }); - } catch (Exception ignored) { - } - } - private String buildTaskEntityKey(Long taskId) { return "delete-brand:task:entity:" + taskId; } - private record LocalParsedPayloadCacheEntry(long cachedAtMillis, Object value) { - private boolean isFresh(long now) { - return now - cachedAtMillis <= LOCAL_PARSED_PAYLOAD_CACHE_MILLIS; - } - } - private record LocalProgressCacheEntry(long cachedAtMillis, java.util.Map values) { private boolean isFresh(long now) { return now - cachedAtMillis <= LOCAL_PROGRESS_CACHE_MILLIS; @@ -377,8 +232,4 @@ public class DeleteBrandTaskCacheService { private String buildProgressKey(Long taskId) { return "delete-brand:task:progress:" + taskId; } - - private String buildResultChunksKey(Long taskId) { - return "delete-brand:task:result-chunks:" + taskId; - } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskStorageService.java new file mode 100644 index 0000000..3cc4857 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskStorageService.java @@ -0,0 +1,404 @@ +package com.nanri.aiimage.modules.deletebrand.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.deletebrand.model.cache.DeleteBrandParsedFileCacheDto; +import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandResultFileDto; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +@Service +@RequiredArgsConstructor +@Slf4j +public class DeleteBrandTaskStorageService { + + private static final String MODULE_TYPE = "DELETE_BRAND"; + private static final String OSS_POINTER_PREFIX = "oss:"; + + private final TaskChunkMapper taskChunkMapper; + private final TaskScopeStateMapper taskScopeStateMapper; + private final ObjectMapper objectMapper; + private final OssStorageService ossStorageService; + + @Transactional + public void saveParsedPayload(Long taskId, Map parsedPayloadByScope) { + if (taskId == null || taskId <= 0 || parsedPayloadByScope == null || parsedPayloadByScope.isEmpty()) { + return; + } + LocalDateTime now = LocalDateTime.now(); + Map existingByHash = loadScopeStateByHash(taskId); + for (Map.Entry entry : parsedPayloadByScope.entrySet()) { + String scopeKey = normalizeScopeKey(entry.getKey()); + if (scopeKey.isBlank() || entry.getValue() == null) { + continue; + } + String scopeHash = hash(scopeKey); + TaskScopeStateEntity existing = existingByHash.get(scopeHash); + String payloadJson = writeJson(entry.getValue(), "failed to save delete-brand parsed payload"); + String parsedPayloadPointer = storeParsedPayload(taskId, scopeHash, payloadJson); + if (existing == null) { + TaskScopeStateEntity entity = new TaskScopeStateEntity(); + entity.setTaskId(taskId); + entity.setModuleType(MODULE_TYPE); + entity.setScopeKey(scopeKey); + entity.setScopeHash(scopeHash); + entity.setParsedPayloadJson(parsedPayloadPointer); + entity.setChunkTotal(0); + entity.setReceivedChunkCount(0); + entity.setCompleted(0); + entity.setLastChunkAt(null); + entity.setLastError(null); + entity.setCreatedAt(now); + entity.setUpdatedAt(now); + try { + taskScopeStateMapper.insert(entity); + existingByHash.put(scopeHash, entity); + continue; + } catch (DuplicateKeyException ex) { + log.info("[delete-brand-storage] parsed payload scope inserted concurrently taskId={} scopeKey={}", taskId, scopeKey); + existing = getScopeState(taskId, scopeHash); + } + } + if (existing == null) { + throw new BusinessException("failed to save delete-brand parsed payload"); + } + deleteParsedPayloadIfNeeded(existing.getParsedPayloadJson(), parsedPayloadPointer); + taskScopeStateMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskScopeStateEntity::getId, existing.getId()) + .set(TaskScopeStateEntity::getScopeKey, scopeKey) + .set(TaskScopeStateEntity::getParsedPayloadJson, parsedPayloadPointer) + .set(TaskScopeStateEntity::getUpdatedAt, now)); + } + } + + public Map loadParsedPayload(Long taskId) { + if (taskId == null || taskId <= 0) { + return Map.of(); + } + List states = taskScopeStateMapper.selectList(new LambdaQueryWrapper() + .select(TaskScopeStateEntity::getScopeKey, TaskScopeStateEntity::getParsedPayloadJson) + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .isNotNull(TaskScopeStateEntity::getParsedPayloadJson)); + if (states == null || states.isEmpty()) { + return Map.of(); + } + Map result = new LinkedHashMap<>(); + for (TaskScopeStateEntity state : states) { + if (isBlank(state.getScopeKey())) { + continue; + } + String payloadJson = resolveParsedPayload(state.getParsedPayloadJson()); + if (isBlank(payloadJson)) { + continue; + } + try { + result.put(state.getScopeKey(), objectMapper.readValue(payloadJson, DeleteBrandParsedFileCacheDto.class)); + } catch (Exception ex) { + throw new BusinessException("failed to load delete-brand parsed payload"); + } + } + return result; + } + + @Transactional + public boolean storeResultChunkIfChanged(Long taskId, String scopeKey, Integer chunkIndex, DeleteBrandResultFileDto chunk) { + if (taskId == null || taskId <= 0) { + throw new BusinessException("taskId is invalid"); + } + if (chunkIndex == null || chunkIndex <= 0) { + throw new BusinessException("chunkIndex is invalid"); + } + String normalizedScopeKey = normalizeScopeKey(scopeKey); + if (normalizedScopeKey.isBlank()) { + throw new BusinessException("scopeKey must not be blank"); + } + String scopeHash = hash(normalizedScopeKey); + String payloadJson = writeJson(chunk, "failed to save delete-brand result chunk"); + String payloadHash = hash(payloadJson); + LocalDateTime now = LocalDateTime.now(); + + TaskChunkEntity existingChunk = taskChunkMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, scopeHash) + .eq(TaskChunkEntity::getChunkIndex, chunkIndex) + .last("limit 1")); + boolean changed = true; + if (existingChunk == null) { + TaskChunkEntity entity = new TaskChunkEntity(); + entity.setTaskId(taskId); + entity.setModuleType(MODULE_TYPE); + entity.setScopeKey(normalizedScopeKey); + entity.setScopeHash(scopeHash); + entity.setChunkIndex(chunkIndex); + entity.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal()); + entity.setPayloadJson(payloadJson); + entity.setPayloadHash(payloadHash); + entity.setCreatedAt(now); + entity.setUpdatedAt(now); + try { + taskChunkMapper.insert(entity); + } catch (DuplicateKeyException ex) { + TaskChunkEntity concurrentChunk = taskChunkMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, scopeHash) + .eq(TaskChunkEntity::getChunkIndex, chunkIndex) + .last("limit 1")); + if (concurrentChunk != null) { + changed = !payloadHash.equals(concurrentChunk.getPayloadHash()); + concurrentChunk.setScopeKey(normalizedScopeKey); + concurrentChunk.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal()); + concurrentChunk.setPayloadJson(payloadJson); + concurrentChunk.setPayloadHash(payloadHash); + concurrentChunk.setUpdatedAt(now); + taskChunkMapper.updateById(concurrentChunk); + } else { + throw ex; + } + } + } else { + changed = !payloadHash.equals(existingChunk.getPayloadHash()); + existingChunk.setScopeKey(normalizedScopeKey); + existingChunk.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal()); + existingChunk.setPayloadJson(payloadJson); + existingChunk.setPayloadHash(payloadHash); + existingChunk.setUpdatedAt(now); + taskChunkMapper.updateById(existingChunk); + } + + refreshScopeState(taskId, normalizedScopeKey, scopeHash, chunk.getChunkTotal(), now); + return changed; + } + + public Map> loadMergedChunks(Long taskId) { + if (taskId == null || taskId <= 0) { + return Map.of(); + } + List entities = taskChunkMapper.selectList(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .orderByAsc(TaskChunkEntity::getScopeHash) + .orderByAsc(TaskChunkEntity::getChunkIndex) + .orderByAsc(TaskChunkEntity::getId)); + if (entities == null || entities.isEmpty()) { + return Map.of(); + } + Map> grouped = new LinkedHashMap<>(); + for (TaskChunkEntity entity : entities) { + if (entity.getPayloadJson() == null || entity.getPayloadJson().isBlank()) { + continue; + } + try { + DeleteBrandResultFileDto dto = objectMapper.readValue(entity.getPayloadJson(), DeleteBrandResultFileDto.class); + grouped.computeIfAbsent(entity.getScopeKey(), ignored -> new ArrayList<>()).add(dto); + } catch (Exception ex) { + throw new BusinessException("failed to load delete-brand result chunks"); + } + } + for (List chunks : grouped.values()) { + chunks.sort(Comparator.comparing(DeleteBrandResultFileDto::getChunkIndex, Comparator.nullsLast(Integer::compareTo))); + } + return grouped; + } + + @Transactional + public void deleteTaskData(Long taskId) { + if (taskId == null || taskId <= 0) { + return; + } + List states = taskScopeStateMapper.selectList(new LambdaQueryWrapper() + .select(TaskScopeStateEntity::getParsedPayloadJson) + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); + if (states != null) { + for (TaskScopeStateEntity state : states) { + deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), null); + } + } + taskChunkMapper.delete(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE)); + taskScopeStateMapper.delete(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); + } + + private void refreshScopeState(Long taskId, String scopeKey, String scopeHash, Integer chunkTotal, LocalDateTime now) { + Long receivedCountRaw = taskChunkMapper.selectCount(new LambdaQueryWrapper() + .eq(TaskChunkEntity::getTaskId, taskId) + .eq(TaskChunkEntity::getModuleType, MODULE_TYPE) + .eq(TaskChunkEntity::getScopeHash, scopeHash)); + int receivedCount = receivedCountRaw == null ? 0 : receivedCountRaw.intValue(); + int normalizedChunkTotal = chunkTotal == null || chunkTotal < 0 ? 0 : chunkTotal; + + TaskScopeStateEntity state = getScopeState(taskId, scopeHash); + if (state == null) { + state = new TaskScopeStateEntity(); + state.setTaskId(taskId); + state.setModuleType(MODULE_TYPE); + state.setScopeKey(scopeKey); + state.setScopeHash(scopeHash); + state.setParsedPayloadJson(null); + state.setChunkTotal(normalizedChunkTotal); + state.setReceivedChunkCount(receivedCount); + state.setCompleted(normalizedChunkTotal > 0 && receivedCount >= normalizedChunkTotal ? 1 : 0); + state.setLastChunkAt(now); + state.setLastError(null); + state.setCreatedAt(now); + state.setUpdatedAt(now); + try { + taskScopeStateMapper.insert(state); + } catch (DuplicateKeyException ex) { + taskScopeStateMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .eq(TaskScopeStateEntity::getScopeHash, scopeHash) + .set(TaskScopeStateEntity::getScopeKey, scopeKey) + .set(TaskScopeStateEntity::getChunkTotal, normalizedChunkTotal) + .set(TaskScopeStateEntity::getReceivedChunkCount, receivedCount) + .set(TaskScopeStateEntity::getCompleted, normalizedChunkTotal > 0 && receivedCount >= normalizedChunkTotal ? 1 : 0) + .set(TaskScopeStateEntity::getLastChunkAt, now) + .set(TaskScopeStateEntity::getLastError, null) + .set(TaskScopeStateEntity::getUpdatedAt, now)); + } + return; + } + + int mergedChunkTotal = Math.max(state.getChunkTotal() == null ? 0 : state.getChunkTotal(), normalizedChunkTotal); + taskScopeStateMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskScopeStateEntity::getId, state.getId()) + .set(TaskScopeStateEntity::getScopeKey, scopeKey) + .set(TaskScopeStateEntity::getChunkTotal, mergedChunkTotal) + .set(TaskScopeStateEntity::getReceivedChunkCount, receivedCount) + .set(TaskScopeStateEntity::getCompleted, mergedChunkTotal > 0 && receivedCount >= mergedChunkTotal ? 1 : 0) + .set(TaskScopeStateEntity::getLastChunkAt, now) + .set(TaskScopeStateEntity::getLastError, null) + .set(TaskScopeStateEntity::getUpdatedAt, now)); + } + + private TaskScopeStateEntity getScopeState(Long taskId, String scopeHash) { + return taskScopeStateMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE) + .eq(TaskScopeStateEntity::getScopeHash, scopeHash) + .last("limit 1")); + } + + private Map loadScopeStateByHash(Long taskId) { + List states = taskScopeStateMapper.selectList(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)); + Map result = new LinkedHashMap<>(); + if (states == null) { + return result; + } + for (TaskScopeStateEntity state : states) { + if (!isBlank(state.getScopeHash())) { + result.put(state.getScopeHash(), state); + } + } + return result; + } + + private String storeParsedPayload(Long taskId, String scopeHash, String payloadJson) { + try { + String pointer = OSS_POINTER_PREFIX + ossStorageService.uploadTaskParsedPayload(MODULE_TYPE, taskId, scopeHash, payloadJson); + return objectMapper.writeValueAsString(pointer); + } catch (Exception ex) { + throw new BusinessException("failed to save delete-brand parsed payload"); + } + } + + private String resolveParsedPayload(String value) { + if (isBlank(value)) { + return value; + } + String ossPointer = extractOssPointer(value); + if (ossPointer == null) { + return value; + } + try { + return ossStorageService.readObjectAsString(ossPointer.substring(OSS_POINTER_PREFIX.length())); + } catch (Exception ex) { + throw new BusinessException("failed to load delete-brand parsed payload"); + } + } + + private void deleteParsedPayloadIfNeeded(String oldValue, String newValue) { + String oldPointer = extractOssPointer(oldValue); + String newPointer = extractOssPointer(newValue); + if (oldPointer == null || oldPointer.equals(newPointer)) { + return; + } + try { + ossStorageService.deleteObject(oldPointer.substring(OSS_POINTER_PREFIX.length())); + } catch (Exception ignored) { + } + } + + private String extractOssPointer(String value) { + if (isBlank(value)) { + return null; + } + if (value.startsWith(OSS_POINTER_PREFIX)) { + return value; + } + try { + String decoded = objectMapper.readValue(value, String.class); + return decoded != null && decoded.startsWith(OSS_POINTER_PREFIX) ? decoded : null; + } catch (Exception ignored) { + return null; + } + } + + private String writeJson(Object value, String message) { + try { + return objectMapper.writeValueAsString(value); + } catch (Exception ex) { + throw new BusinessException(message); + } + } + + private String normalizeScopeKey(String scopeKey) { + return scopeKey == null ? "" : scopeKey.trim(); + } + + private boolean isBlank(String value) { + return value == null || value.isBlank(); + } + + private String hash(String value) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] bytes = digest.digest(normalizeScopeKey(value).getBytes(StandardCharsets.UTF_8)); + StringBuilder sb = new StringBuilder(bytes.length * 2); + for (byte b : bytes) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); + } catch (Exception ex) { + throw new IllegalStateException("failed to hash delete-brand scope", ex); + } + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java index 3b767bf..5fc49f3 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java @@ -2,14 +2,18 @@ package com.nanri.aiimage.modules.file.service.oss; import com.aliyun.oss.OSS; import com.aliyun.oss.OSSClientBuilder; +import com.aliyun.oss.model.OSSObject; import com.nanri.aiimage.config.OssProperties; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; +import java.io.ByteArrayInputStream; import java.io.File; import java.net.URI; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.util.Date; +import java.util.Objects; import java.util.UUID; @Service @@ -33,6 +37,68 @@ public class OssStorageService { } } + public String uploadText(String objectKey, String content) { + if (objectKey == null || objectKey.isBlank()) { + throw new IllegalArgumentException("objectKey must not be blank"); + } + OSS ossClient = buildClient(); + try (ByteArrayInputStream stream = new ByteArrayInputStream( + Objects.requireNonNullElse(content, "").getBytes(StandardCharsets.UTF_8))) { + ossClient.putObject(ossProperties.getBucket(), objectKey, stream); + return objectKey; + } catch (Exception ex) { + throw new IllegalStateException("failed to upload text to oss", ex); + } finally { + ossClient.shutdown(); + } + } + + public String uploadTaskScopePayload(String moduleType, Long taskId, String scopeHash, String content) { + String normalizedModuleType = moduleType == null || moduleType.isBlank() + ? "unknown" + : moduleType.trim().toLowerCase(); + String normalizedScopeHash = scopeHash == null || scopeHash.isBlank() ? UUID.randomUUID().toString() : scopeHash; + String objectKey = String.format("task-scope/%s/%s/%s.json", normalizedModuleType, taskId, normalizedScopeHash); + return uploadText(objectKey, content); + } + + public String uploadTaskParsedPayload(String moduleType, Long taskId, String scopeHash, String content) { + String normalizedModuleType = moduleType == null || moduleType.isBlank() + ? "unknown" + : moduleType.trim().toLowerCase(); + String normalizedScopeHash = scopeHash == null || scopeHash.isBlank() ? UUID.randomUUID().toString() : scopeHash; + String objectKey = String.format("task-parsed/%s/%s/%s.json", normalizedModuleType, taskId, normalizedScopeHash); + return uploadText(objectKey, content); + } + + public String readObjectAsString(String value) { + String objectKey = resolveObjectKey(value); + if (objectKey == null || objectKey.isBlank()) { + return null; + } + OSS ossClient = buildClient(); + try (OSSObject ossObject = ossClient.getObject(ossProperties.getBucket(), objectKey)) { + return new String(ossObject.getObjectContent().readAllBytes(), StandardCharsets.UTF_8); + } catch (Exception ex) { + throw new IllegalStateException("failed to read object from oss", ex); + } finally { + ossClient.shutdown(); + } + } + + public void deleteObject(String value) { + String objectKey = resolveObjectKey(value); + if (objectKey == null || objectKey.isBlank()) { + return; + } + OSS ossClient = buildClient(); + try { + ossClient.deleteObject(ossProperties.getBucket(), objectKey); + } finally { + ossClient.shutdown(); + } + } + /** * 根据 objectKey 生成预签名下载 URL（1小时有效）。 */ diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteExcelAssemblyService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteExcelAssemblyService.java index 90313e6..6a399a4 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteExcelAssemblyService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteExcelAssemblyService.java @@ -8,7 +8,7 @@ import com.nanri.aiimage.modules.patroldelete.model.vo.PatrolDeleteResultItemVo; import lombok.extern.slf4j.Slf4j; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; -import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.springframework.stereotype.Service; import java.io.File; @@ -40,8 +40,9 @@ public class PatrolDeleteExcelAssemblyService { }; public void writeWorkbook(File outputXlsx, List items) { - try (XSSFWorkbook workbook = new XSSFWorkbook(); - FileOutputStream outputStream = new FileOutputStream(outputXlsx)) { + SXSSFWorkbook workbook = new SXSSFWorkbook(200); + workbook.setCompressTempFiles(true); + try (FileOutputStream outputStream = new FileOutputStream(outputXlsx)) { Sheet sheet = workbook.createSheet("巡店删除结果"); Row headerRow = sheet.createRow(0); for (int columnIndex = 0; columnIndex < HEADER.length; columnIndex++) { @@ -73,13 +74,17 @@ public class PatrolDeleteExcelAssemblyService { rowIndex++; } - for (int columnIndex = 0; columnIndex < HEADER.length; columnIndex++) { - sheet.autoSizeColumn(columnIndex); - } + applyDefaultColumnWidths(sheet); workbook.write(outputStream); } catch (Exception ex) { log.warn("[patrol-delete] write workbook failed: {}", ex.getMessage()); - throw new BusinessException("生成巡店删除结果 Excel 失败: " + ex.getMessage()); + throw new BusinessException("generate patrol-delete excel failed: " + ex.getMessage()); + } finally { + try { + workbook.close(); + } catch (Exception ignored) { + } + workbook.dispose(); } } @@ -141,4 +146,23 @@ public class PatrolDeleteExcelAssemblyService { private String safe(String value) { return value == null ? "" : value; } + + private void applyDefaultColumnWidths(Sheet sheet) { + int[] widths = { + 20, + 14, 10, 12, 16, + 14, 10, 12, 16, + 14, 10, 12, 16, + 14, 10, 12, 16, + 14, 10, 12, 16, + 10, 16, + 10, 16, + 10, 16, + 10, 16, + 10, 16 + }; + for (int i = 0; i < widths.length; i++) { + sheet.setColumnWidth(i, widths[i] * 256); + } + } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskCacheService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskCacheService.java index c48d0ee..19212f1 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskCacheService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskCacheService.java @@ -1,10 +1,10 @@ package com.nanri.aiimage.modules.patroldelete.service; import com.fasterxml.jackson.databind.ObjectMapper; -import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.config.TaskPressureProperties; import com.nanri.aiimage.modules.patroldelete.model.dto.PatrolDeleteShopPayloadDto; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService; import lombok.RequiredArgsConstructor; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.stereotype.Service; @@ -20,80 +20,40 @@ import java.util.concurrent.ConcurrentHashMap; @RequiredArgsConstructor public class PatrolDeleteTaskCacheService { + private static final String MODULE_TYPE = "PATROL_DELETE"; private static final long PAYLOAD_TTL_HOURS = 24; private final StringRedisTemplate stringRedisTemplate; private final ObjectMapper objectMapper; private final TaskPressureProperties taskPressureProperties; + private final TaskScopePayloadStorageService taskScopePayloadStorageService; private final ConcurrentHashMap taskEntityLocalCache = new ConcurrentHashMap<>(); public PatrolDeleteShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) { - if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) { - return null; - } - Object raw = stringRedisTemplate.opsForHash().get(buildShopPayloadKey(taskId), shopKey); - if (!(raw instanceof String json) || json.isBlank()) { - return null; - } - try { - return objectMapper.readValue(json, PatrolDeleteShopPayloadDto.class); - } catch (Exception ex) { - throw new BusinessException("读取巡店删除店铺缓存失败"); - } + return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, PatrolDeleteShopPayloadDto.class); } public void saveShopMergedPayload(Long taskId, String shopKey, PatrolDeleteShopPayloadDto payload) { if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank() || payload == null) { return; } - try { - stringRedisTemplate.opsForHash().put(buildShopPayloadKey(taskId), shopKey, objectMapper.writeValueAsString(payload)); - stringRedisTemplate.expire(buildShopPayloadKey(taskId), Duration.ofHours(PAYLOAD_TTL_HOURS)); - touchTaskHeartbeat(taskId); - } catch (Exception ex) { - throw new BusinessException("暂存巡店删除店铺缓存失败"); - } + taskScopePayloadStorageService.saveScopePayload(taskId, MODULE_TYPE, shopKey, payload); + touchTaskHeartbeat(taskId); } public void removeShopMergedPayload(Long taskId, String shopKey) { - if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) { - return; - } - stringRedisTemplate.opsForHash().delete(buildShopPayloadKey(taskId), shopKey); + taskScopePayloadStorageService.removeScopePayload(taskId, MODULE_TYPE, shopKey); } public Map getAllShopMergedPayload(Long taskId) { - try { - Map raw = stringRedisTemplate.opsForHash().entries(buildShopPayloadKey(taskId)); - if (raw == null || raw.isEmpty()) { - return Map.of(); - } - Map out = new LinkedHashMap<>(); - for (Map.Entry entry : raw.entrySet()) { - if (!(entry.getKey() instanceof String key) || !(entry.getValue() instanceof String val) || val.isBlank()) { - continue; - } - out.put(key, objectMapper.readValue(val, PatrolDeleteShopPayloadDto.class)); - } - return out; - } catch (Exception ex) { - throw new BusinessException("读取巡店删除缓存失败"); - } + return taskScopePayloadStorageService.getAllScopePayload(taskId, MODULE_TYPE, PatrolDeleteShopPayloadDto.class); } public boolean hasAnyShopMergedPayload(Long taskId) { - if (taskId == null || taskId <= 0) { - return false; - } - Long size = stringRedisTemplate.opsForHash().size(buildShopPayloadKey(taskId)); - return size != null && size > 0; + return taskScopePayloadStorageService.hasAnyScopePayload(taskId, MODULE_TYPE); } public long countShopMergedPayload(Long taskId) { - if (taskId == null || taskId <= 0) { - return 0L; - } - Long size = stringRedisTemplate.opsForHash().size(buildShopPayloadKey(taskId)); - return size == null ? 0L : size; + return taskScopePayloadStorageService.countScopePayload(taskId, MODULE_TYPE); } public long getTaskHeartbeatMillis(Long taskId) { @@ -123,9 +83,9 @@ public class PatrolDeleteTaskCacheService { return; } taskEntityLocalCache.remove(taskId); - stringRedisTemplate.delete(buildShopPayloadKey(taskId)); stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId)); stringRedisTemplate.delete(buildTaskEntityKey(taskId)); + taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE); } public void saveTaskCache(FileTaskEntity task) { @@ -190,10 +150,6 @@ public class PatrolDeleteTaskCacheService { return result; } - private String buildShopPayloadKey(Long taskId) { - return "patrol-delete:task:shop-payload:" + taskId; - } - private String buildTaskHeartbeatKey(Long taskId) { return "patrol-delete:task:heartbeat:" + taskId; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/dto/PriceTrackSubmitResultRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/dto/PriceTrackSubmitResultRequest.java index d850686..c9f8762 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/dto/PriceTrackSubmitResultRequest.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/dto/PriceTrackSubmitResultRequest.java @@ -93,5 +93,14 @@ public class PriceTrackSubmitResultRequest { @Schema(description = "状态", example = "SUCCESS") private String status; + + @Schema(description = "Python 确认是否需要从 skip ASIN 表删除当前 ASIN", example = "true") + private Boolean deleteSkipAsin; + + @Schema(description = "需要删除的 skip ASIN；不传时默认使用 asin 字段", example = "B0ABCDE123") + private String removeAsin; + + @Schema(description = "删除确认原因，便于排查日志", example = "CURRENT_PRICE_BELOW_MINIMUM") + private String deleteReason; } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/vo/PriceTrackCreateTaskVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/vo/PriceTrackCreateTaskVo.java index d9887ae..50f998c 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/vo/PriceTrackCreateTaskVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/vo/PriceTrackCreateTaskVo.java @@ -18,6 +18,9 @@ public class PriceTrackCreateTaskVo { @Schema(description = "all skip asins grouped by country") private Map> skipAsinsByCountry; + @Schema(description = "all skip asin details grouped by country code, each item includes asin and minimumPrice") + private Map>> skipAsinDetailsByCountry; + @Schema(description = "parsed asin rows by country") private Map>> asinRowsByCountry; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/vo/PriceTrackMatchShopsVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/vo/PriceTrackMatchShopsVo.java index 426e06d..8475179 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/vo/PriceTrackMatchShopsVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/vo/PriceTrackMatchShopsVo.java @@ -17,6 +17,9 @@ public class PriceTrackMatchShopsVo { @Schema(description = "All skip ASINs grouped by country code") private Map> skipAsinsByCountry; + @Schema(description = "All skip ASIN details grouped by country code, each item includes asin and minimumPrice") + private Map>> skipAsinDetailsByCountry; + @Schema(description = "Parsed asin rows grouped by country code") private Map>> asinRowsByCountry; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackExcelAssemblyService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackExcelAssemblyService.java index f1adb09..2741df9 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackExcelAssemblyService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackExcelAssemblyService.java @@ -6,7 +6,7 @@ import com.nanri.aiimage.modules.productrisk.model.enums.ProductRiskCountryCode; import lombok.extern.slf4j.Slf4j; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; -import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.springframework.stereotype.Service; import java.io.File; @@ -20,25 +20,27 @@ import java.util.Map; public class PriceTrackExcelAssemblyService { private static final String[] RESULT_HEADER = { - "\u5e97\u94fa\u5546\u57ce\u540d\u79f0", + "店铺商城名称", "ASIN", - "\u4ef7\u683c", - "\u63a8\u8350\u4ef7", - "\u6700\u4f4e\u4ef7", - "\u7b2c\u4e00\u540d", - "\u7b2c\u4e8c\u540d", - "\u8d2d\u7269\u8f66\u5e97\u94fa\u540d", - "\u6539\u4ef7\u60c5\u51b5", - "\u4fee\u6539\u6b21\u6570", - "\u72b6\u6001" + "价格", + "推荐价", + "最低价", + "第一名", + "第二名", + "购物车店铺名", + "改价情况", + "修改次数", + "状态" }; public void writeWorkbook(File outputXlsx, Map> countries) { Map> safe = countries == null ? Map.of() : countries; - try (XSSFWorkbook wb = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(outputXlsx)) { + SXSSFWorkbook workbook = new SXSSFWorkbook(200); + workbook.setCompressTempFiles(true); + try (FileOutputStream fos = new FileOutputStream(outputXlsx)) { for (ProductRiskCountryCode code : ProductRiskCountryCode.values()) { String sheetName = code.name(); - Sheet sheet = wb.createSheet(sheetName); + Sheet sheet = workbook.createSheet(sheetName); Row headerRow = sheet.createRow(0); for (int i = 0; i < RESULT_HEADER.length; i++) { headerRow.createCell(i).setCellValue(RESULT_HEADER[i]); @@ -62,14 +64,18 @@ public class PriceTrackExcelAssemblyService { row.createCell(9).setCellValue(valueOf(item.getModifyCount())); row.createCell(10).setCellValue(valueOf(item.getStatus())); } - for (int i = 0; i < RESULT_HEADER.length; i++) { - sheet.autoSizeColumn(i); - } + applyDefaultColumnWidths(sheet); } - wb.write(fos); + workbook.write(fos); } catch (Exception ex) { log.warn("[price-track] write workbook failed: {}", ex.getMessage()); - throw new BusinessException("鐢熸垚 Excel 澶辫触: " + ex.getMessage()); + throw new BusinessException("generate price-track excel failed: " + ex.getMessage()); + } finally { + try { + workbook.close(); + } catch (Exception ignored) { + } + workbook.dispose(); } } @@ -105,4 +111,10 @@ public class PriceTrackExcelAssemblyService { return value == null ? "" : value; } + private void applyDefaultColumnWidths(Sheet sheet) { + int[] widths = {22, 18, 12, 12, 12, 20, 20, 22, 18, 12, 12}; + for (int i = 0; i < widths.length; i++) { + sheet.setColumnWidth(i, widths[i] * 256); + } + } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackService.java index 7caade6..d9210b6 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackService.java @@ -148,6 +148,9 @@ public class PriceTrackService { Map> skipAsinsByCountry = asinMode ? new LinkedHashMap<>() : skipPriceAsinService.listAllSkipAsinsByCountry(); + Map>> skipAsinDetailsByCountry = asinMode + ? new LinkedHashMap<>() + : skipPriceAsinService.listAllSkipAsinDetailsByCountry(); Map>> asinRowsByCountry = !asinMode ? new LinkedHashMap<>() @@ -157,6 +160,7 @@ public class PriceTrackService { PriceTrackMatchShopsVo vo = new PriceTrackMatchShopsVo(); vo.setSkipAsinsByCountry(skipAsinsByCountry); + vo.setSkipAsinDetailsByCountry(skipAsinDetailsByCountry); vo.setAsinRowsByCountry(asinRowsByCountry); vo.setMinimumPriceByCountryAndAsin(priceTrackTaskService.buildMinimumPriceLookupForMatch(asinRowsByCountry)); for (String shopName : ordered) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskCacheService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskCacheService.java index 432c18e..6570567 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskCacheService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskCacheService.java @@ -1,10 +1,10 @@ package com.nanri.aiimage.modules.pricetrack.service; import com.fasterxml.jackson.databind.ObjectMapper; -import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.config.TaskPressureProperties; import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackSubmitResultRequest; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService; import lombok.RequiredArgsConstructor; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.stereotype.Service; @@ -20,10 +20,12 @@ import java.util.concurrent.ConcurrentHashMap; @RequiredArgsConstructor public class PriceTrackTaskCacheService { + private static final String MODULE_TYPE = "PRICE_TRACK"; private static final long HEARTBEAT_TTL_HOURS = 24; private final StringRedisTemplate stringRedisTemplate; private final ObjectMapper objectMapper; private final TaskPressureProperties taskPressureProperties; + private final TaskScopePayloadStorageService taskScopePayloadStorageService; private final ConcurrentHashMap taskEntityLocalCache = new ConcurrentHashMap<>(); public void touchTaskHeartbeat(Long taskId) { @@ -52,65 +54,27 @@ public class PriceTrackTaskCacheService { } public PriceTrackSubmitResultRequest.ShopResult getShopMergedPayload(Long taskId, String shopKey) { - if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) { - return null; - } - Object raw = stringRedisTemplate.opsForHash().get(buildShopPayloadKey(taskId), shopKey); - if (!(raw instanceof String json) || json.isBlank()) { - return null; - } - try { - return objectMapper.readValue(json, PriceTrackSubmitResultRequest.ShopResult.class); - } catch (Exception ex) { - throw new BusinessException("read price-track cache failed"); - } + return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, PriceTrackSubmitResultRequest.ShopResult.class); } public void saveShopMergedPayload(Long taskId, String shopKey, PriceTrackSubmitResultRequest.ShopResult payload) { if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank() || payload == null) { return; } - try { - stringRedisTemplate.opsForHash().put(buildShopPayloadKey(taskId), shopKey, objectMapper.writeValueAsString(payload)); - stringRedisTemplate.expire(buildShopPayloadKey(taskId), Duration.ofHours(HEARTBEAT_TTL_HOURS)); - touchTaskHeartbeat(taskId); - } catch (Exception ex) { - throw new BusinessException("save price-track cache failed"); - } + taskScopePayloadStorageService.saveScopePayload(taskId, MODULE_TYPE, shopKey, payload); + touchTaskHeartbeat(taskId); } public void removeShopMergedPayload(Long taskId, String shopKey) { - if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) { - return; - } - stringRedisTemplate.opsForHash().delete(buildShopPayloadKey(taskId), shopKey); + taskScopePayloadStorageService.removeScopePayload(taskId, MODULE_TYPE, shopKey); } public Map getAllShopMergedPayload(Long taskId) { - try { - Map raw = stringRedisTemplate.opsForHash().entries(buildShopPayloadKey(taskId)); - if (raw == null || raw.isEmpty()) { - return Map.of(); - } - LinkedHashMap out = new LinkedHashMap<>(); - for (Map.Entry entry : raw.entrySet()) { - if (!(entry.getKey() instanceof String key) || !(entry.getValue() instanceof String val) || val.isBlank()) { - continue; - } - out.put(key, objectMapper.readValue(val, PriceTrackSubmitResultRequest.ShopResult.class)); - } - return out; - } catch (Exception ex) { - throw new BusinessException("read price-track cache failed"); - } + return taskScopePayloadStorageService.getAllScopePayload(taskId, MODULE_TYPE, PriceTrackSubmitResultRequest.ShopResult.class); } public boolean hasAnyShopMergedPayload(Long taskId) { - if (taskId == null || taskId <= 0) { - return false; - } - Long size = stringRedisTemplate.opsForHash().size(buildShopPayloadKey(taskId)); - return size != null && size > 0; + return taskScopePayloadStorageService.hasAnyScopePayload(taskId, MODULE_TYPE); } public void deleteTaskCache(Long taskId) { @@ -118,9 +82,9 @@ public class PriceTrackTaskCacheService { return; } taskEntityLocalCache.remove(taskId); - stringRedisTemplate.delete(buildShopPayloadKey(taskId)); stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId)); stringRedisTemplate.delete(buildTaskEntityKey(taskId)); + taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE); } public void saveTaskCache(FileTaskEntity task) { @@ -185,10 +149,6 @@ public class PriceTrackTaskCacheService { return result; } - private String buildShopPayloadKey(Long taskId) { - return "price-track:task:shop-payload:" + taskId; - } - private String buildTaskHeartbeatKey(Long taskId) { return "price-track:task:heartbeat:" + taskId; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java index 75c07d0..43fc570 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java @@ -206,7 +206,7 @@ public class PriceTrackTaskService { @Transactional public PriceTrackPendingDeleteVo deletePendingShopResult(Long userId, String shopName) { if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法"); - if (shopName == null || shopName.isBlank()) throw new BusinessException("shop_name 涓嶈兘涓虹┖"); + if (shopName == null || shopName.isBlank()) throw new BusinessException("shop_name 不能为空"); String norm = ziniaoShopSwitchService.normalizeShopName(shopName); if (norm.isBlank()) throw new BusinessException("店铺名称无效"); @@ -293,6 +293,9 @@ public class PriceTrackTaskService { Map> skipAsinsByCountry = request.isAsinMode() ? new LinkedHashMap<>() : skipPriceAsinService.listAllSkipAsinsByCountry(); + Map>> skipAsinDetailsByCountry = request.isAsinMode() + ? new LinkedHashMap<>() + : skipPriceAsinService.listAllSkipAsinDetailsByCountry(); Map>> asinRowsByCountry = request.isAsinMode() ? parseAsinRowsByCountry(request.getAsinFiles(), request.getCountryCodes()) : new LinkedHashMap<>(); @@ -348,6 +351,7 @@ public class PriceTrackTaskService { ctx.put("asinMode", request.isAsinMode()); ctx.put("countryCodes", request.getCountryCodes()); ctx.put("skipAsinsByCountry", skipAsinsByCountry); + ctx.put("skipAsinDetailsByCountry", skipAsinDetailsByCountry); ctx.put("asinRowsByCountry", asinRowsByCountry); ctx.put("minimumPriceByCountryAndAsin", minimumPriceByCountryAndAsin); ctx.put("items", uniqueItems); @@ -368,6 +372,7 @@ public class PriceTrackTaskService { vo.setTaskId(task.getId()); vo.setItems(snapshot); vo.setSkipAsinsByCountry(skipAsinsByCountry); + vo.setSkipAsinDetailsByCountry(skipAsinDetailsByCountry); vo.setAsinRowsByCountry(asinRowsByCountry); vo.setMinimumPriceByCountryAndAsin(minimumPriceByCountryAndAsin); return vo; @@ -419,6 +424,7 @@ public class PriceTrackTaskService { priceTrackTaskCacheService.removeShopMergedPayload(taskId, shopKey); continue; } + handleSkipAsinDeletionSignals(shopKey, payload); PriceTrackSubmitResultRequest.ShopResult merged = mergeShopPayload(taskId, shopKey, payload); if (!Boolean.TRUE.equals(merged.getSuccess())) { continue; @@ -833,8 +839,8 @@ public class PriceTrackTaskService { if (value == null) { return ""; } - return value.replace("\ufeff", "") - .replace("\u3000", " ") + return value.replace("", "") + .replace("　", " ") .replace("\r", " ") .replace("\n", " ") .trim() @@ -1075,6 +1081,11 @@ public class PriceTrackTaskService { merged.setPriceChangeStatus(firstNonBlank(incoming.getPriceChangeStatus(), merged.getPriceChangeStatus())); merged.setModifyCount(firstNonBlank(incoming.getModifyCount(), merged.getModifyCount())); merged.setStatus(firstNonBlank(incoming.getStatus(), merged.getStatus())); + if (incoming.getDeleteSkipAsin() != null) { + merged.setDeleteSkipAsin(incoming.getDeleteSkipAsin()); + } + merged.setRemoveAsin(firstNonBlank(incoming.getRemoveAsin(), merged.getRemoveAsin())); + merged.setDeleteReason(firstNonBlank(incoming.getDeleteReason(), merged.getDeleteReason())); return merged; } private PriceTrackSubmitResultRequest.AsinResult cloneRow(PriceTrackSubmitResultRequest.AsinResult row) { @@ -1093,8 +1104,36 @@ public class PriceTrackTaskService { out.setPriceChangeStatus(row.getPriceChangeStatus()); out.setModifyCount(row.getModifyCount()); out.setStatus(row.getStatus()); + out.setDeleteSkipAsin(row.getDeleteSkipAsin()); + out.setRemoveAsin(row.getRemoveAsin()); + out.setDeleteReason(row.getDeleteReason()); return out; } + private void handleSkipAsinDeletionSignals(String shopKey, + PriceTrackSubmitResultRequest.ShopResult payload) { + if (payload == null || payload.getCountries() == null || payload.getCountries().isEmpty()) { + return; + } + for (Map.Entry> entry : payload.getCountries().entrySet()) { + String countryCode = entry.getKey(); + List rows = entry.getValue(); + if (countryCode == null || countryCode.isBlank() || rows == null || rows.isEmpty()) { + continue; + } + for (PriceTrackSubmitResultRequest.AsinResult row : rows) { + if (row == null || !Boolean.TRUE.equals(row.getDeleteSkipAsin())) { + continue; + } + String targetAsin = firstNonBlank(row.getRemoveAsin(), row.getAsin()); + if (targetAsin == null || targetAsin.isBlank()) { + continue; + } + boolean removed = skipPriceAsinService.removeByShopCountryAndAsin(shopKey, countryCode, targetAsin); + log.info("[price-track] skip asin delete signal taskShop={} country={} asin={} removed={} reason={}", + shopKey, countryCode, targetAsin, removed, row.getDeleteReason()); + } + } + } private int countPayloadRows(PriceTrackSubmitResultRequest.ShopResult payload) { if (payload == null) { return 0; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskExcelAssemblyService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskExcelAssemblyService.java index 301ee7c..f678dc5 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskExcelAssemblyService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskExcelAssemblyService.java @@ -7,7 +7,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; -import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.springframework.stereotype.Service; import java.io.File; @@ -35,9 +35,11 @@ public class ProductRiskExcelAssemblyService { public void writeWorkbook(File outputXlsx, String shopDisplayName, Map> countries) { Map> safe = countries == null ? Map.of() : countries; - try (XSSFWorkbook wb = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(outputXlsx)) { + SXSSFWorkbook workbook = new SXSSFWorkbook(200); + workbook.setCompressTempFiles(true); + try (FileOutputStream fos = new FileOutputStream(outputXlsx)) { for (String sheetName : SHEET_ORDER) { - Sheet sheet = wb.createSheet(sheetName); + Sheet sheet = workbook.createSheet(sheetName); Row headerRow = sheet.createRow(0); for (int c = 0; c < HEADER.length; c++) { headerRow.createCell(c).setCellValue(HEADER[c]); @@ -53,16 +55,20 @@ public class ProductRiskExcelAssemblyService { createTextCell(row, 4, dto == null ? null : dto.getRemoveAsin()); createTextCell(row, 5, dto == null ? null : dto.getRemoveStatus()); } - for (int i = 0; i < HEADER.length; i++) { - sheet.autoSizeColumn(i); - } + applyDefaultColumnWidths(sheet); } - wb.write(fos); + workbook.write(fos); } catch (BusinessException ex) { throw ex; } catch (Exception ex) { log.warn("[product-risk] write workbook failed: {}", ex.getMessage()); - throw new BusinessException("生成 Excel 失败: " + ex.getMessage()); + throw new BusinessException("generate product-risk excel failed: " + ex.getMessage()); + } finally { + try { + workbook.close(); + } catch (Exception ignored) { + } + workbook.dispose(); } } @@ -79,9 +85,6 @@ public class ProductRiskExcelAssemblyService { return n; } - /** - * 将接口中的 {@link ProductRiskCountryCode} 键转为 Excel 中文工作表名 → 行列表。 - */ public Map> normalizeCountriesMap(Map> raw) { if (raw == null || raw.isEmpty()) { return new LinkedHashMap<>(); @@ -107,4 +110,11 @@ public class ProductRiskExcelAssemblyService { } return fallback == null ? "" : fallback; } + + private void applyDefaultColumnWidths(Sheet sheet) { + int[] widths = {20, 22, 18, 10, 18, 18}; + for (int i = 0; i < widths.length; i++) { + sheet.setColumnWidth(i, widths[i] * 256); + } + } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskResolveService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskResolveService.java index 002ac48..76d5492 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskResolveService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskResolveService.java @@ -34,7 +34,7 @@ import java.util.Objects; public class ProductRiskResolveService { /** - * 默认处理顺序：德国 → 英国 → 法国 → 意大利 → 西班牙（与前端一致）。 + * 默认国家偏好顺序：德国 -> 英国 -> 法国 -> 意大利 -> 西班牙，与前端保持一致。 */ public static final List DEFAULT_COUNTRY_PREFERENCE_ORDER = List.of("DE", "UK", "FR", "IT", "ES"); @@ -78,7 +78,7 @@ public class ProductRiskResolveService { if (indexHit == null || !indexHit.isMatched()) { String hint = indexHit != null && indexHit.getMatchMessage() != null && !indexHit.getMatchMessage().isBlank() ? indexHit.getMatchMessage() - : "店铺索引未命中，无法加入备选"; + : "店铺索引未命中，无法加入待处理列表"; throw new BusinessException(hint); } if (ZiniaoShopIndexService.MATCH_STATUS_CONFLICT.equals(indexHit.getMatchStatus())) { @@ -135,7 +135,7 @@ public class ProductRiskResolveService { } } if (ordered.isEmpty()) { - throw new BusinessException("shop_names 无有效店铺名"); + throw new BusinessException("shop_names 没有有效店铺名"); } ProductRiskMatchShopsVo vo = new ProductRiskMatchShopsVo(); for (String shopName : ordered) { @@ -227,7 +227,7 @@ public class ProductRiskResolveService { } /** - * 仅保留合法枚举名，去重并保持顺序；非法项静默丢弃（用于读取库内数据）。 + * 仅保留合法国家代码，去重并保持顺序；非法值静默忽略，用于读取库存量数据。 */ private static List parseValidCountryCodes(List raw) { if (raw == null || raw.isEmpty()) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskCacheService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskCacheService.java index aac342b..db0e5c9 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskCacheService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskCacheService.java @@ -1,10 +1,10 @@ package com.nanri.aiimage.modules.productrisk.service; import com.fasterxml.jackson.databind.ObjectMapper; -import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.config.TaskPressureProperties; import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskShopPayloadDto; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService; import lombok.RequiredArgsConstructor; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.stereotype.Service; @@ -20,80 +20,40 @@ import java.util.concurrent.ConcurrentHashMap; @RequiredArgsConstructor public class ProductRiskTaskCacheService { + private static final String MODULE_TYPE = "PRODUCT_RISK_RESOLVE"; private static final long PAYLOAD_TTL_HOURS = 24; private final StringRedisTemplate stringRedisTemplate; private final ObjectMapper objectMapper; private final TaskPressureProperties taskPressureProperties; + private final TaskScopePayloadStorageService taskScopePayloadStorageService; private final ConcurrentHashMap taskEntityLocalCache = new ConcurrentHashMap<>(); public ProductRiskShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) { - if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) { - return null; - } - Object raw = stringRedisTemplate.opsForHash().get(buildShopPayloadKey(taskId), shopKey); - if (!(raw instanceof String json) || json.isBlank()) { - return null; - } - try { - return objectMapper.readValue(json, ProductRiskShopPayloadDto.class); - } catch (Exception ex) { - throw new BusinessException("读取商品风险店铺缓存失败"); - } + return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, ProductRiskShopPayloadDto.class); } public void saveShopMergedPayload(Long taskId, String shopKey, ProductRiskShopPayloadDto payload) { if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank() || payload == null) { return; } - try { - stringRedisTemplate.opsForHash().put(buildShopPayloadKey(taskId), shopKey, objectMapper.writeValueAsString(payload)); - stringRedisTemplate.expire(buildShopPayloadKey(taskId), Duration.ofHours(PAYLOAD_TTL_HOURS)); - touchTaskHeartbeat(taskId); - } catch (Exception ex) { - throw new BusinessException("暂存商品风险店铺缓存失败"); - } + taskScopePayloadStorageService.saveScopePayload(taskId, MODULE_TYPE, shopKey, payload); + touchTaskHeartbeat(taskId); } public void removeShopMergedPayload(Long taskId, String shopKey) { - if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) { - return; - } - stringRedisTemplate.opsForHash().delete(buildShopPayloadKey(taskId), shopKey); + taskScopePayloadStorageService.removeScopePayload(taskId, MODULE_TYPE, shopKey); } public Map getAllShopMergedPayload(Long taskId) { - try { - Map raw = stringRedisTemplate.opsForHash().entries(buildShopPayloadKey(taskId)); - if (raw == null || raw.isEmpty()) { - return Map.of(); - } - java.util.LinkedHashMap out = new java.util.LinkedHashMap<>(); - for (Map.Entry entry : raw.entrySet()) { - if (!(entry.getKey() instanceof String key) || !(entry.getValue() instanceof String val) || val.isBlank()) { - continue; - } - out.put(key, objectMapper.readValue(val, ProductRiskShopPayloadDto.class)); - } - return out; - } catch (Exception ex) { - throw new BusinessException("读取商品风险缓存失败"); - } + return taskScopePayloadStorageService.getAllScopePayload(taskId, MODULE_TYPE, ProductRiskShopPayloadDto.class); } public boolean hasAnyShopMergedPayload(Long taskId) { - if (taskId == null || taskId <= 0) { - return false; - } - Long size = stringRedisTemplate.opsForHash().size(buildShopPayloadKey(taskId)); - return size != null && size > 0; + return taskScopePayloadStorageService.hasAnyScopePayload(taskId, MODULE_TYPE); } public long countShopMergedPayload(Long taskId) { - if (taskId == null || taskId <= 0) { - return 0L; - } - Long size = stringRedisTemplate.opsForHash().size(buildShopPayloadKey(taskId)); - return size == null ? 0L : size; + return taskScopePayloadStorageService.countScopePayload(taskId, MODULE_TYPE); } public long getTaskHeartbeatMillis(Long taskId) { @@ -116,9 +76,9 @@ public class ProductRiskTaskCacheService { return; } taskEntityLocalCache.remove(taskId); - stringRedisTemplate.delete(buildShopPayloadKey(taskId)); stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId)); stringRedisTemplate.delete(buildTaskEntityKey(taskId)); + taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE); } public void saveTaskCache(FileTaskEntity task) { @@ -183,10 +143,6 @@ public class ProductRiskTaskCacheService { return result; } - private String buildShopPayloadKey(Long taskId) { - return "product-risk:task:shop-payload:" + taskId; - } - public void touchTaskHeartbeat(Long taskId) { stringRedisTemplate.opsForValue().set( buildTaskHeartbeatKey(taskId), diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskService.java index a7cf6d5..bb00826 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskService.java @@ -189,7 +189,7 @@ public class ProductRiskTaskService { } /** - * 删除整条商品风险任务及其下所有店铺结果（运行中或已结束均可，需归属当前用户）。 + * 删除整条商品风险任务及其下所有店铺结果，运行中和已结束任务都允许删除，但必须归属当前用户。 */ @Transactional public void deleteTask(Long taskId, Long userId) { @@ -208,8 +208,8 @@ public class ProductRiskTaskService { } /** - * 从用户当前「运行中」的商品风险任务里，按规范化店铺名删除一条结果（用于前端移除匹配列表后同步后端）。 - * 若无对应记录则 removed=false。 + * 从用户当前“运行中”的商品风险任务里，按规范化店铺名删除一条结果，用于前端移除匹配列表后同步后端。 + * 如果没有对应记录，则返回 removed=false。 */ @Transactional public ProductRiskPendingDeleteVo deletePendingShopResult(Long userId, String shopName) { @@ -252,7 +252,7 @@ public class ProductRiskTaskService { } /** - * 删除一条 file_result 后重算父任务状态；若无剩余结果则删除任务。 + * 删除一条 file_result 后重算父任务状态；如果没有剩余结果则删除任务。 */ private void reconcileTaskAfterResultRemoval(Long taskId) { FileTaskEntity task = loadTaskForExecution(taskId); @@ -303,11 +303,11 @@ public class ProductRiskTaskService { task.setFinishedAt(LocalDateTime.now()); } else if (ok > 0) { task.setStatus("SUCCESS"); - task.setErrorMessage(String.join("；", allErrors)); + task.setErrorMessage(String.join("; ", allErrors)); task.setFinishedAt(LocalDateTime.now()); } else { task.setStatus("FAILED"); - task.setErrorMessage(allErrors.isEmpty() ? "全部店铺处理失败" : String.join("；", allErrors)); + task.setErrorMessage(allErrors.isEmpty() ? "全部店铺处理失败" : String.join("; ", allErrors)); task.setFinishedAt(LocalDateTime.now()); } try { @@ -507,7 +507,7 @@ public class ProductRiskTaskService { String shopKey = fr.getSourceFilename(); ProductRiskShopPayloadDto payload = shopKey == null ? null : payloadByShop.get(shopKey); - // 兼容单店任务：若规范化店名偶发不一致（空格/符号差异）导致未命中，且任务与回传都只有 1 个店时按唯一店回填。 + // 兼容单店任务：如果规范化店名偶发不一致导致未命中，且任务与回传都只有 1 个店时按唯一店铺回填。 if (payload == null && resultRows.size() == 1 && payloadByShop.size() == 1) { payload = payloadByShop.values().iterator().next(); fallbackMatchedCount++; @@ -517,7 +517,7 @@ public class ProductRiskTaskService { if (payload == null) { skippedUnmatchedCount++; - // 与删除品牌一致：支持队列逐条处理、分次回传，未在本次 payload 中的店铺保持待处理 + // 与删除品牌一致：支持队列逐条处理、分次回传，本次 payload 未包含的店铺保持待处理。 continue; } matchedShopCount++; @@ -602,11 +602,11 @@ public class ProductRiskTaskService { task.setFinishedAt(LocalDateTime.now()); } else if (ok > 0) { task.setStatus("SUCCESS"); - task.setErrorMessage(String.join("；", allErrors.isEmpty() ? batchErrors : allErrors)); + task.setErrorMessage(String.join("; ", allErrors.isEmpty() ? batchErrors : allErrors)); task.setFinishedAt(LocalDateTime.now()); } else { task.setStatus("FAILED"); - task.setErrorMessage(allErrors.isEmpty() ? "全部店铺处理失败" : String.join("；", allErrors)); + task.setErrorMessage(allErrors.isEmpty() ? "全部店铺处理失败" : String.join("; ", allErrors)); task.setFinishedAt(LocalDateTime.now()); } try { @@ -1034,7 +1034,7 @@ public class ProductRiskTaskService { continue; } - // 兼容 Python 末次“空行 done=true”心跳：视为当前国家已有行全部完成。 + // 兼容 Python 末次“空行 + done=true”心跳：视为当前国家已有行全部完成。 if (isDoneValue(row.getDone()) && isEmptyBusinessRow(row) && !byKey.isEmpty()) { for (Map.Entry entry : byKey.entrySet()) { ProductRiskRowDto existed = entry.getValue(); @@ -1103,8 +1103,8 @@ public class ProductRiskTaskService { if (row == null) { return true; } - // 注意：shopName 只是上下文信息，不作为业务行是否为空的判定条件。 - // Python 末次心跳行通常只带 shopName + done=true，其他业务字段为空。 + // 注意，shopName 只是上下文信息，不作为业务行是否为空的判定条件。 + // Python 末次心跳行通常只带 shopName + done=true，其它业务字段为空。 return isBlank(row.getProductAsinSku()) && isBlank(row.getRemoveAsin()) && isBlank(row.getStatus()) diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/controller/SkipPriceAsinController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/controller/SkipPriceAsinController.java index 74c951b..8b9e088 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/controller/SkipPriceAsinController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/controller/SkipPriceAsinController.java @@ -71,6 +71,7 @@ public class SkipPriceAsinController { id, country, request.getAsin(), + request.getMinimumPrice(), operatorId, Boolean.TRUE.equals(superAdmin))); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/dto/SkipPriceAsinCountryUpdateRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/dto/SkipPriceAsinCountryUpdateRequest.java index be37835..049cd53 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/dto/SkipPriceAsinCountryUpdateRequest.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/dto/SkipPriceAsinCountryUpdateRequest.java @@ -4,10 +4,15 @@ import io.swagger.v3.oas.annotations.media.Schema; import jakarta.validation.constraints.NotBlank; import lombok.Data; +import java.math.BigDecimal; + @Data public class SkipPriceAsinCountryUpdateRequest { @Schema(description = "ASIN", requiredMode = Schema.RequiredMode.REQUIRED) @NotBlank(message = "ASIN 不能为空") private String asin; + + @Schema(description = "最低价", example = "18.50") + private BigDecimal minimumPrice; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/dto/SkipPriceAsinCreateRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/dto/SkipPriceAsinCreateRequest.java index 22d666d..c9e72b6 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/dto/SkipPriceAsinCreateRequest.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/dto/SkipPriceAsinCreateRequest.java @@ -5,6 +5,7 @@ import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; import lombok.Data; +import java.math.BigDecimal; import java.util.List; import java.util.Map; @@ -29,4 +30,8 @@ public class SkipPriceAsinCreateRequest { * 新请求: 每个国家对应独立 ASIN，例如 {"DE":"B0...", "UK":"B0..."}。 */ private Map asinMappings; + + private BigDecimal minimumPrice; + + private Map minimumPriceMappings; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/entity/SkipPriceAsinEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/entity/SkipPriceAsinEntity.java index 63cc68d..874cb71 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/entity/SkipPriceAsinEntity.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/entity/SkipPriceAsinEntity.java @@ -6,6 +6,7 @@ import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import lombok.Data; +import java.math.BigDecimal; import java.time.LocalDateTime; @Data @@ -24,18 +25,33 @@ public class SkipPriceAsinEntity { @TableField("asin_de") private String asinDe; + @TableField("minimum_price_de") + private BigDecimal minimumPriceDe; + @TableField("asin_uk") private String asinUk; + @TableField("minimum_price_uk") + private BigDecimal minimumPriceUk; + @TableField("asin_fr") private String asinFr; + @TableField("minimum_price_fr") + private BigDecimal minimumPriceFr; + @TableField("asin_it") private String asinIt; + @TableField("minimum_price_it") + private BigDecimal minimumPriceIt; + @TableField("asin_es") private String asinEs; + @TableField("minimum_price_es") + private BigDecimal minimumPriceEs; + @TableField("created_at") private LocalDateTime createdAt; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/vo/SkipPriceAsinItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/vo/SkipPriceAsinItemVo.java index 963db9a..bc07309 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/vo/SkipPriceAsinItemVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/model/vo/SkipPriceAsinItemVo.java @@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.shopkey.model.vo; import lombok.Data; +import java.math.BigDecimal; import java.time.LocalDateTime; @Data @@ -12,10 +13,15 @@ public class SkipPriceAsinItemVo { private String groupName; private String shopName; private String asinDe; + private BigDecimal minimumPriceDe; private String asinUk; + private BigDecimal minimumPriceUk; private String asinFr; + private BigDecimal minimumPriceFr; private String asinIt; + private BigDecimal minimumPriceIt; private String asinEs; + private BigDecimal minimumPriceEs; private LocalDateTime createdAt; private LocalDateTime updatedAt; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/SkipPriceAsinService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/SkipPriceAsinService.java index 0407766..db38556 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/SkipPriceAsinService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopkey/service/SkipPriceAsinService.java @@ -12,6 +12,9 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.LocalDateTime; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -79,6 +82,7 @@ public class SkipPriceAsinService { String shopName = normalizeRequired(request.getShopName(), "店铺名不能为空"); Set countries = normalizeCountries(request.getCountries()); Map countryAsinMap = normalizeCountryAsinMap(countries, request); + Map countryMinimumPriceMap = normalizeCountryMinimumPriceMap(countries, request); SkipPriceAsinEntity entity = skipPriceAsinMapper.selectOne(new LambdaQueryWrapper() .eq(SkipPriceAsinEntity::getGroupId, group.getId()) @@ -91,7 +95,8 @@ public class SkipPriceAsinService { } for (Map.Entry entry : countryAsinMap.entrySet()) { - setCountryAsin(entity, entry.getKey(), entry.getValue()); + String country = entry.getKey(); + setCountryData(entity, country, entry.getValue(), countryMinimumPriceMap.get(country)); } if (entity.getId() == null) { @@ -108,7 +113,7 @@ public class SkipPriceAsinService { SkipPriceAsinEntity entity = getById(id); validateGroupAccess(entity, operatorId, superAdmin); String normalizedCountry = normalizeCountry(country); - setCountryAsin(entity, normalizedCountry, ""); + setCountryData(entity, normalizedCountry, "", null); if (isEmptyRow(entity)) { skipPriceAsinMapper.deleteById(entity.getId()); return; @@ -117,12 +122,13 @@ public class SkipPriceAsinService { } @Transactional - public SkipPriceAsinItemVo updateCountry(Long id, String country, String asin, Long operatorId, boolean superAdmin) { + public SkipPriceAsinItemVo updateCountry(Long id, String country, String asin, BigDecimal minimumPrice, + Long operatorId, boolean superAdmin) { SkipPriceAsinEntity entity = getById(id); validateGroupAccess(entity, operatorId, superAdmin); String normalizedCountry = normalizeCountry(country); String normalizedAsin = normalizeAsin(asin); - setCountryAsin(entity, normalizedCountry, normalizedAsin); + setCountryData(entity, normalizedCountry, normalizedAsin, normalizeMinimumPrice(minimumPrice)); skipPriceAsinMapper.updateById(entity); SkipPriceAsinEntity saved = getById(entity.getId()); String groupName = ""; @@ -154,10 +160,15 @@ public class SkipPriceAsinService { vo.setGroupName(groupName == null ? "" : groupName); vo.setShopName(entity.getShopName()); vo.setAsinDe(blankToEmpty(entity.getAsinDe())); + vo.setMinimumPriceDe(entity.getMinimumPriceDe()); vo.setAsinUk(blankToEmpty(entity.getAsinUk())); + vo.setMinimumPriceUk(entity.getMinimumPriceUk()); vo.setAsinFr(blankToEmpty(entity.getAsinFr())); + vo.setMinimumPriceFr(entity.getMinimumPriceFr()); vo.setAsinIt(blankToEmpty(entity.getAsinIt())); + vo.setMinimumPriceIt(entity.getMinimumPriceIt()); vo.setAsinEs(blankToEmpty(entity.getAsinEs())); + vo.setMinimumPriceEs(entity.getMinimumPriceEs()); vo.setCreatedAt(entity.getCreatedAt()); vo.setUpdatedAt(entity.getUpdatedAt()); return vo; @@ -213,6 +224,27 @@ public class SkipPriceAsinService { return normalized; } + private Map normalizeCountryMinimumPriceMap(Set countries, SkipPriceAsinCreateRequest request) { + LinkedHashMap normalized = new LinkedHashMap<>(); + Map requestMappings = request.getMinimumPriceMappings(); + if (requestMappings != null && !requestMappings.isEmpty()) { + for (String country : countries) { + BigDecimal minimumPrice = requestMappings.get(country); + if (minimumPrice == null) { + minimumPrice = requestMappings.get(country.toLowerCase(Locale.ROOT)); + } + normalized.put(country, normalizeMinimumPrice(minimumPrice)); + } + return normalized; + } + + BigDecimal minimumPrice = normalizeMinimumPrice(request.getMinimumPrice()); + for (String country : countries) { + normalized.put(country, minimumPrice); + } + return normalized; + } + private String normalizeCountry(String country) { String normalized = normalizeBlank(country).toUpperCase(Locale.ROOT); if (!SUPPORTED_COUNTRIES.contains(normalized)) { @@ -229,6 +261,16 @@ public class SkipPriceAsinService { return normalized; } + private BigDecimal normalizeMinimumPrice(BigDecimal minimumPrice) { + if (minimumPrice == null) { + return null; + } + if (minimumPrice.compareTo(BigDecimal.ZERO) < 0) { + throw new BusinessException("最低价不能小于 0"); + } + return minimumPrice.setScale(2, RoundingMode.HALF_UP); + } + private String normalizeRequired(String value, String message) { String normalized = normalizeBlank(value); if (normalized.isEmpty()) { @@ -245,14 +287,29 @@ public class SkipPriceAsinService { return value == null ? "" : value; } - private void setCountryAsin(SkipPriceAsinEntity entity, String country, String asin) { + private void setCountryData(SkipPriceAsinEntity entity, String country, String asin, BigDecimal minimumPrice) { String value = normalizeBlank(asin); switch (country) { - case "DE" -> entity.setAsinDe(value); - case "UK" -> entity.setAsinUk(value); - case "FR" -> entity.setAsinFr(value); - case "IT" -> entity.setAsinIt(value); - case "ES" -> entity.setAsinEs(value); + case "DE" -> { + entity.setAsinDe(value); + entity.setMinimumPriceDe(minimumPrice); + } + case "UK" -> { + entity.setAsinUk(value); + entity.setMinimumPriceUk(minimumPrice); + } + case "FR" -> { + entity.setAsinFr(value); + entity.setMinimumPriceFr(minimumPrice); + } + case "IT" -> { + entity.setAsinIt(value); + entity.setMinimumPriceIt(minimumPrice); + } + case "ES" -> { + entity.setAsinEs(value); + entity.setMinimumPriceEs(minimumPrice); + } default -> throw new BusinessException("国家参数不支持: " + country); } } @@ -262,7 +319,12 @@ public class SkipPriceAsinService { && normalizeBlank(entity.getAsinUk()).isEmpty() && normalizeBlank(entity.getAsinFr()).isEmpty() && normalizeBlank(entity.getAsinIt()).isEmpty() - && normalizeBlank(entity.getAsinEs()).isEmpty(); + && normalizeBlank(entity.getAsinEs()).isEmpty() + && entity.getMinimumPriceDe() == null + && entity.getMinimumPriceUk() == null + && entity.getMinimumPriceFr() == null + && entity.getMinimumPriceIt() == null + && entity.getMinimumPriceEs() == null; } public Map> listSkipAsinsByShops(List shopNames) { @@ -312,10 +374,78 @@ public class SkipPriceAsinService { return result; } + public Map>> listAllSkipAsinDetailsByCountry() { + List rows = skipPriceAsinMapper.selectList( + new LambdaQueryWrapper() + .orderByDesc(SkipPriceAsinEntity::getId)); + Map>> grouped = new LinkedHashMap<>(); + grouped.put("DE", new java.util.ArrayList<>()); + grouped.put("UK", new java.util.ArrayList<>()); + grouped.put("FR", new java.util.ArrayList<>()); + grouped.put("IT", new java.util.ArrayList<>()); + grouped.put("ES", new java.util.ArrayList<>()); + + for (SkipPriceAsinEntity row : rows) { + addCountryAsinDetail(grouped.get("DE"), row.getAsinDe(), row.getMinimumPriceDe()); + addCountryAsinDetail(grouped.get("UK"), row.getAsinUk(), row.getMinimumPriceUk()); + addCountryAsinDetail(grouped.get("FR"), row.getAsinFr(), row.getMinimumPriceFr()); + addCountryAsinDetail(grouped.get("IT"), row.getAsinIt(), row.getMinimumPriceIt()); + addCountryAsinDetail(grouped.get("ES"), row.getAsinEs(), row.getMinimumPriceEs()); + } + return grouped; + } + + @Transactional + public boolean removeByShopCountryAndAsin(String shopName, String country, String asin) { + String normalizedShopName = normalizeBlank(shopName); + String normalizedCountry = normalizeCountry(country); + String normalizedAsin = normalizeAsin(asin); + if (normalizedShopName.isEmpty() || normalizedAsin.isEmpty()) { + return false; + } + List rows = skipPriceAsinMapper.selectList( + new LambdaQueryWrapper() + .eq(SkipPriceAsinEntity::getShopName, normalizedShopName)); + boolean changed = false; + for (SkipPriceAsinEntity row : rows) { + String existingAsin = switch (normalizedCountry) { + case "DE" -> normalizeBlank(row.getAsinDe()); + case "UK" -> normalizeBlank(row.getAsinUk()); + case "FR" -> normalizeBlank(row.getAsinFr()); + case "IT" -> normalizeBlank(row.getAsinIt()); + case "ES" -> normalizeBlank(row.getAsinEs()); + default -> ""; + }; + if (!normalizedAsin.equals(existingAsin)) { + continue; + } + setCountryData(row, normalizedCountry, "", null); + row.setUpdatedAt(LocalDateTime.now()); + if (isEmptyRow(row)) { + skipPriceAsinMapper.deleteById(row.getId()); + } else { + skipPriceAsinMapper.updateById(row); + } + changed = true; + } + return changed; + } + private void addCountryAsin(Set bucket, String asin) { String normalized = normalizeBlank(asin); if (!normalized.isEmpty()) { bucket.add(normalized); } } + + private void addCountryAsinDetail(List> bucket, String asin, BigDecimal minimumPrice) { + String normalizedAsin = normalizeBlank(asin); + if (normalizedAsin.isEmpty()) { + return; + } + Map detail = new LinkedHashMap<>(); + detail.put("asin", normalizedAsin); + detail.put("minimumPrice", minimumPrice == null ? "" : minimumPrice.toPlainString()); + bucket.add(detail); + } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchExcelAssemblyService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchExcelAssemblyService.java index e58159e..ecb3e2d 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchExcelAssemblyService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchExcelAssemblyService.java @@ -6,7 +6,7 @@ import com.nanri.aiimage.modules.shopmatch.model.dto.ShopMatchRowDto; import lombok.extern.slf4j.Slf4j; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; -import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.springframework.stereotype.Service; import java.io.File; @@ -23,10 +23,12 @@ public class ShopMatchExcelAssemblyService { public void writeWorkbook(File outputXlsx, Map> countries) { Map> safe = countries == null ? Map.of() : countries; - try (XSSFWorkbook wb = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(outputXlsx)) { + SXSSFWorkbook workbook = new SXSSFWorkbook(200); + workbook.setCompressTempFiles(true); + try (FileOutputStream fos = new FileOutputStream(outputXlsx)) { for (ProductRiskCountryCode code : ProductRiskCountryCode.values()) { String sheetName = code.getSheetName(); - Sheet sheet = wb.createSheet(sheetName); + Sheet sheet = workbook.createSheet(sheetName); Row header = sheet.createRow(0); header.createCell(0).setCellValue(HEADER[0]); header.createCell(1).setCellValue(HEADER[1]); @@ -40,13 +42,19 @@ public class ShopMatchExcelAssemblyService { row.createCell(0).setCellValue(item.getAsin() == null ? "" : item.getAsin()); row.createCell(1).setCellValue(item.getStatus() == null ? "" : item.getStatus()); } - sheet.autoSizeColumn(0); - sheet.autoSizeColumn(1); + sheet.setColumnWidth(0, 20 * 256); + sheet.setColumnWidth(1, 18 * 256); } - wb.write(fos); + workbook.write(fos); } catch (Exception ex) { log.warn("[shop-match] write workbook failed: {}", ex.getMessage()); - throw new BusinessException("生成 Excel 失败: " + ex.getMessage()); + throw new BusinessException("generate shop-match excel failed: " + ex.getMessage()); + } finally { + try { + workbook.close(); + } catch (Exception ignored) { + } + workbook.dispose(); } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskCacheService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskCacheService.java index 9925eae..4a10295 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskCacheService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskCacheService.java @@ -1,24 +1,20 @@ package com.nanri.aiimage.modules.shopmatch.service; import com.fasterxml.jackson.databind.ObjectMapper; -import com.nanri.aiimage.common.exception.BusinessException; -import com.nanri.aiimage.config.TaskPressureProperties; import com.nanri.aiimage.modules.shopmatch.model.dto.ShopMatchShopPayloadDto; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService; +import com.nanri.aiimage.config.TaskPressureProperties; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.security.MessageDigest; import java.time.Duration; import java.util.ArrayList; -import java.util.Comparator; -import java.util.HexFormat; import java.util.LinkedHashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -28,104 +24,34 @@ import java.util.concurrent.ConcurrentHashMap; @Slf4j public class ShopMatchTaskCacheService { + private static final String MODULE_TYPE = "SHOP_MATCH"; private static final long PAYLOAD_TTL_HOURS = 24; private final ObjectMapper objectMapper; private final TaskPressureProperties taskPressureProperties; + private final TaskScopePayloadStorageService taskScopePayloadStorageService; private final ConcurrentHashMap taskEntityLocalCache = new ConcurrentHashMap<>(); public ShopMatchShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) { - if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) { - return null; - } - Path file = buildShopPayloadFile(taskId, shopKey); - if (!Files.isRegularFile(file)) { - return null; - } - try { - PayloadFile payloadFile = objectMapper.readValue(Files.readString(file), PayloadFile.class); - if (payloadFile == null || payloadFile.payload == null) { - return null; - } - if (payloadFile.shopKey == null || !shopKey.equals(payloadFile.shopKey)) { - log.warn("[shop-match] payload file shop key mismatch taskId={} shopKey={} storedKey={}", - taskId, shopKey, payloadFile.shopKey); - return null; - } - return payloadFile.payload; - } catch (Exception ex) { - throw new BusinessException("璇诲彇鍖归厤搴楅摵缂撳瓨澶辫触"); - } + return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, ShopMatchShopPayloadDto.class); } public void saveShopMergedPayload(Long taskId, String shopKey, ShopMatchShopPayloadDto payload) { if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank() || payload == null) { return; } - try { - Files.createDirectories(buildTaskDir(taskId)); - Files.writeString( - buildShopPayloadFile(taskId, shopKey), - objectMapper.writeValueAsString(new PayloadFile(shopKey, payload)), - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING, - StandardOpenOption.WRITE - ); - } catch (Exception ex) { - throw new BusinessException("鏆傚瓨鍖归厤搴楅摵缁撴灉澶辫触"); - } + taskScopePayloadStorageService.saveScopePayload(taskId, MODULE_TYPE, shopKey, payload); } public void removeShopMergedPayload(Long taskId, String shopKey) { - if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) { - return; - } - try { - Files.deleteIfExists(buildShopPayloadFile(taskId, shopKey)); - } catch (IOException ex) { - log.warn("[shop-match] delete shop payload failed taskId={} shopKey={} msg={}", taskId, shopKey, ex.getMessage()); - } + taskScopePayloadStorageService.removeScopePayload(taskId, MODULE_TYPE, shopKey); } public Map getAllShopMergedPayload(Long taskId) { - try { - Path taskDir = buildTaskDir(taskId); - if (!Files.isDirectory(taskDir)) { - return Map.of(); - } - LinkedHashMap out = new LinkedHashMap<>(); - try (var stream = Files.list(taskDir)) { - for (Path path : stream - .filter(file -> file.getFileName().toString().endsWith(".json")) - .sorted(Comparator.comparing(file -> file.getFileName().toString())) - .toList()) { - PayloadFile payloadFile = objectMapper.readValue(Files.readString(path), PayloadFile.class); - if (payloadFile == null || payloadFile.shopKey == null || payloadFile.shopKey.isBlank() || payloadFile.payload == null) { - continue; - } - out.put(payloadFile.shopKey, payloadFile.payload); - } - } - return out; - } catch (Exception ex) { - throw new BusinessException("璇诲彇鍖归厤搴楅摵缂撳瓨澶辫触"); - } + return taskScopePayloadStorageService.getAllScopePayload(taskId, MODULE_TYPE, ShopMatchShopPayloadDto.class); } public boolean hasAnyShopMergedPayload(Long taskId) { - if (taskId == null || taskId <= 0) { - return false; - } - Path taskDir = buildTaskDir(taskId); - if (!Files.isDirectory(taskDir)) { - return false; - } - try (var stream = Files.list(taskDir)) { - return stream.anyMatch(file -> Files.isRegularFile(file) - && file.getFileName().toString().endsWith(".json")); - } catch (IOException ex) { - log.warn("[shop-match] scan task cache failed taskId={} msg={}", taskId, ex.getMessage()); - return false; - } + return taskScopePayloadStorageService.hasAnyScopePayload(taskId, MODULE_TYPE); } public void deleteTaskCache(Long taskId) { @@ -133,6 +59,7 @@ public class ShopMatchTaskCacheService { return; } taskEntityLocalCache.remove(taskId); + taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE); try { Path taskDir = buildTaskDir(taskId); if (!Files.exists(taskDir)) { @@ -140,7 +67,7 @@ public class ShopMatchTaskCacheService { return; } try (var walk = Files.walk(taskDir)) { - walk.sorted(Comparator.reverseOrder()).forEach(path -> { + walk.sorted(java.util.Comparator.reverseOrder()).forEach(path -> { try { Files.deleteIfExists(path); } catch (IOException ex) { @@ -225,37 +152,10 @@ public class ShopMatchTaskCacheService { return Path.of(System.getProperty("java.io.tmpdir"), "shop-match-cache", String.valueOf(taskId)); } - private Path buildShopPayloadFile(Long taskId, String shopKey) { - return buildTaskDir(taskId).resolve(hashShopKey(shopKey) + ".json"); - } - private Path buildTaskEntityFile(Long taskId) { return buildTaskDir(taskId).resolve("_task-entity.json"); } - private String hashShopKey(String shopKey) { - try { - MessageDigest digest = MessageDigest.getInstance("SHA-256"); - byte[] bytes = digest.digest(shopKey.trim().getBytes(StandardCharsets.UTF_8)); - return HexFormat.of().formatHex(bytes); - } catch (Exception ex) { - throw new IllegalStateException("failed to hash shop key", ex); - } - } - - private static class PayloadFile { - public String shopKey; - public ShopMatchShopPayloadDto payload; - - public PayloadFile() { - } - - public PayloadFile(String shopKey, ShopMatchShopPayloadDto payload) { - this.shopKey = shopKey; - this.payload = payload; - } - } - private boolean isLocalCacheFresh(LocalTaskEntityCacheEntry cached, long now) { return cached != null && now - cached.cachedAtMillis() <= Math.max(0L, taskPressureProperties.getLocalTaskEntityCacheMillis()); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/split/service/SplitRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/split/service/SplitRunService.java index ad2477e..c47f882 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/split/service/SplitRunService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/split/service/SplitRunService.java @@ -44,8 +44,8 @@ public class SplitRunService { private static final String ZIP_CONTENT_TYPE = "application/zip"; private static final String SPLIT_MODE_ROWS_PER_FILE = "rows_per_file"; private static final String SPLIT_MODE_PARTS = "parts"; - private static final String HEADER_SUFFIX_MARKER = "idASIN\u56fd\u5bb6\u72b6\u6001\u4ef7\u683c\u53d8\u4f53\u6570\u91cf"; - private static final String HEADER_STOP_COLUMN = "\u7f29\u7565\u56fe\u5730\u57408"; + private static final String HEADER_SUFFIX_MARKER = "idASIN国家状态价格变体数量"; + private static final String HEADER_STOP_COLUMN = "缩略图地址8"; private static final DateTimeFormatter ZIP_NAME_DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd"); private final FileTaskMapper fileTaskMapper; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskChunkMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskChunkMapper.java new file mode 100644 index 0000000..23c0895 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskChunkMapper.java @@ -0,0 +1,9 @@ +package com.nanri.aiimage.modules.task.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface TaskChunkMapper extends BaseMapper { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskScopeStateMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskScopeStateMapper.java new file mode 100644 index 0000000..c4c4961 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskScopeStateMapper.java @@ -0,0 +1,9 @@ +package com.nanri.aiimage.modules.task.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface TaskScopeStateMapper extends BaseMapper { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskChunkEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskChunkEntity.java new file mode 100644 index 0000000..dc9963e --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskChunkEntity.java @@ -0,0 +1,26 @@ +package com.nanri.aiimage.modules.task.model.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +@TableName("biz_task_chunk") +public class TaskChunkEntity { + + @TableId(type = IdType.AUTO) + private Long id; + private Long taskId; + private String moduleType; + private String scopeKey; + private String scopeHash; + private Integer chunkIndex; + private Integer chunkTotal; + private String payloadJson; + private String payloadHash; + private LocalDateTime createdAt; + private LocalDateTime updatedAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskScopeStateEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskScopeStateEntity.java new file mode 100644 index 0000000..5d31c85 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskScopeStateEntity.java @@ -0,0 +1,29 @@ +package com.nanri.aiimage.modules.task.model.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +@TableName("biz_task_scope_state") +public class TaskScopeStateEntity { + + @TableId(type = IdType.AUTO) + private Long id; + private Long taskId; + private String moduleType; + private String scopeKey; + private String scopeHash; + private String parsedPayloadJson; + private String stateJson; + private Integer chunkTotal; + private Integer receivedChunkCount; + private Integer completed; + private LocalDateTime lastChunkAt; + private String lastError; + private LocalDateTime createdAt; + private LocalDateTime updatedAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskScopePayloadBufferMaintenanceService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskScopePayloadBufferMaintenanceService.java new file mode 100644 index 0000000..cca7c59 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskScopePayloadBufferMaintenanceService.java @@ -0,0 +1,172 @@ +package com.nanri.aiimage.modules.task.service; + +import com.nanri.aiimage.common.service.DistributedJobLockService; +import com.nanri.aiimage.config.TaskPressureProperties; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileTime; +import java.time.Duration; +import java.time.Instant; +import java.util.Comparator; +import java.util.List; + +@Service +@RequiredArgsConstructor +@Slf4j +public class TaskScopePayloadBufferMaintenanceService { + + private static final Duration RECOVERY_LOCK_TTL = Duration.ofMinutes(10); + private static final Duration CLEANUP_LOCK_TTL = Duration.ofMinutes(10); + + private final TaskScopePayloadStorageService taskScopePayloadStorageService; + private final TaskPressureProperties taskPressureProperties; + private final DistributedJobLockService distributedJobLockService; + + @EventListener(ApplicationReadyEvent.class) + public void recoverBufferedPayloadsOnStartup() { + DistributedJobLockService.LockHandle lockHandle = + distributedJobLockService.tryLock("task-scope-buffer:recover", RECOVERY_LOCK_TTL); + if (lockHandle == null) { + log.info("[task-scope-buffer] skip startup recovery because another instance holds the lock"); + return; + } + try (lockHandle) { + Path root = taskScopePayloadStorageService.getBufferedPayloadRoot(); + if (!Files.isDirectory(root)) { + return; + } + int limit = Math.max(0, taskPressureProperties.getScopePayloadRecoveryMaxFiles()); + if (limit == 0) { + return; + } + int recovered = 0; + int scanned = 0; + for (Path file : listBufferedPayloadFiles(root, limit)) { + scanned++; + try { + BufferFileRef ref = parseBufferFile(root, file); + if (ref == null) { + Files.deleteIfExists(file); + continue; + } + if (taskScopePayloadStorageService.recoverBufferedScopePayload(ref.taskId(), ref.moduleType(), ref.scopeHash())) { + recovered++; + } + } catch (Exception ex) { + log.warn("[task-scope-buffer] startup recovery failed path={} msg={}", file, ex.getMessage()); + } + } + if (scanned > 0) { + log.info("[task-scope-buffer] startup recovery finished scanned={} recovered={} limit={}", scanned, recovered, limit); + } + } + } + + @Scheduled(cron = "${aiimage.task-pressure.scope-payload-cleanup-cron:15 */30 * * * *}") + public void cleanupExpiredBufferedPayloads() { + DistributedJobLockService.LockHandle lockHandle = + distributedJobLockService.tryLock("task-scope-buffer:cleanup", CLEANUP_LOCK_TTL); + if (lockHandle == null) { + log.info("[task-scope-buffer] skip cleanup because another instance holds the lock"); + return; + } + try (lockHandle) { + Path root = taskScopePayloadStorageService.getBufferedPayloadRoot(); + if (!Files.isDirectory(root)) { + return; + } + Instant cutoff = Instant.now().minus(Duration.ofHours(Math.max(1L, taskPressureProperties.getScopePayloadBufferRetentionHours()))); + int deleted = 0; + for (Path file : listBufferedPayloadFiles(root, Integer.MAX_VALUE)) { + try { + FileTime lastModified = Files.getLastModifiedTime(file); + if (lastModified.toInstant().isAfter(cutoff)) { + continue; + } + Files.deleteIfExists(file); + deleted++; + deleteEmptyParents(file.getParent(), root); + } catch (Exception ex) { + log.warn("[task-scope-buffer] cleanup failed path={} msg={}", file, ex.getMessage()); + } + } + if (deleted > 0) { + log.info("[task-scope-buffer] cleanup finished deleted={} root={}", deleted, root); + } + } + } + + private List listBufferedPayloadFiles(Path root, int limit) { + try (var walk = Files.walk(root, 4)) { + return walk + .filter(Files::isRegularFile) + .filter(path -> path.getFileName().toString().endsWith(".json")) + .sorted(Comparator.comparing(this::safeLastModified)) + .limit(limit) + .toList(); + } catch (IOException ex) { + log.warn("[task-scope-buffer] scan failed root={} msg={}", root, ex.getMessage()); + return List.of(); + } + } + + private Instant safeLastModified(Path path) { + try { + return Files.getLastModifiedTime(path).toInstant(); + } catch (IOException ex) { + return Instant.EPOCH; + } + } + + private BufferFileRef parseBufferFile(Path root, Path file) { + Path relative = root.relativize(file); + if (relative.getNameCount() != 3) { + return null; + } + String moduleType = relative.getName(0).toString().toUpperCase(); + String taskIdRaw = relative.getName(1).toString(); + String fileName = relative.getName(2).toString(); + if (!fileName.endsWith(".json")) { + return null; + } + try { + long taskId = Long.parseLong(taskIdRaw); + String scopeHash = fileName.substring(0, fileName.length() - 5); + if (scopeHash.isBlank()) { + return null; + } + return new BufferFileRef(moduleType, taskId, scopeHash); + } catch (NumberFormatException ex) { + return null; + } + } + + private void deleteEmptyParents(Path start, Path root) { + Path current = start; + while (current != null && !current.equals(root)) { + try (var list = Files.list(current)) { + if (list.findAny().isPresent()) { + return; + } + } catch (IOException ex) { + return; + } + try { + Files.deleteIfExists(current); + } catch (IOException ex) { + return; + } + current = current.getParent(); + } + } + + private record BufferFileRef(String moduleType, Long taskId, String scopeHash) {} +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskScopePayloadStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskScopePayloadStorageService.java new file mode 100644 index 0000000..f9d4971 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskScopePayloadStorageService.java @@ -0,0 +1,542 @@ +package com.nanri.aiimage.modules.task.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nanri.aiimage.config.TaskPressureProperties; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; +import lombok.RequiredArgsConstructor; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.time.LocalDateTime; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@Service +@RequiredArgsConstructor +public class TaskScopePayloadStorageService { + + private static final Set OSS_BACKED_STATE_MODULES = new HashSet<>(Set.of( + "SHOP_MATCH", + "PRODUCT_RISK_RESOLVE", + "PRICE_TRACK", + "PATROL_DELETE" + )); + private static final String OSS_POINTER_PREFIX = "oss:"; + private static final String LOCAL_BUFFER_DIR = "task-scope-buffer"; + + private final TaskScopeStateMapper taskScopeStateMapper; + private final ObjectMapper objectMapper; + private final OssStorageService ossStorageService; + private final TaskPressureProperties taskPressureProperties; + + @Transactional + public void saveScopePayload(Long taskId, String moduleType, String scopeKey, T payload) { + if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey) || payload == null) { + return; + } + String normalizedScopeKey = normalize(scopeKey); + String scopeHash = hash(normalizedScopeKey); + String payloadJson = writeJson(payload, "写入任务范围载荷失败"); + LocalDateTime now = LocalDateTime.now(); + TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash); + if (shouldStoreStatePayloadInOss(moduleType)) { + saveOssBackedScopePayload(taskId, moduleType, normalizedScopeKey, scopeHash, payloadJson, now, state); + return; + } + String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson); + if (state == null) { + TaskScopeStateEntity entity = new TaskScopeStateEntity(); + entity.setTaskId(taskId); + entity.setModuleType(moduleType); + entity.setScopeKey(normalizedScopeKey); + entity.setScopeHash(scopeHash); + entity.setStateJson(stateJson); + entity.setChunkTotal(0); + entity.setReceivedChunkCount(0); + entity.setCompleted(0); + entity.setLastChunkAt(now); + entity.setLastError(null); + entity.setCreatedAt(now); + entity.setUpdatedAt(now); + try { + taskScopeStateMapper.insert(entity); + return; + } catch (DuplicateKeyException ignored) { + state = getScopeState(taskId, moduleType, scopeHash); + } + } + if (state == null) { + throw new BusinessException("写入任务范围载荷失败"); + } + deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson); + taskScopeStateMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskScopeStateEntity::getId, state.getId()) + .set(TaskScopeStateEntity::getScopeKey, normalizedScopeKey) + .set(TaskScopeStateEntity::getStateJson, stateJson) + .set(TaskScopeStateEntity::getLastChunkAt, now) + .set(TaskScopeStateEntity::getUpdatedAt, now)); + } + + public T getScopePayload(Long taskId, String moduleType, String scopeKey, Class clazz) { + if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey)) { + return null; + } + TaskScopeStateEntity state = getScopeState(taskId, moduleType, hash(normalize(scopeKey))); + String payloadJson = resolveStatePayload(state); + if (payloadJson == null) { + return null; + } + try { + return objectMapper.readValue(payloadJson, clazz); + } catch (Exception ex) { + throw new BusinessException("读取任务范围载荷失败"); + } + } + + public Map getAllScopePayload(Long taskId, String moduleType, Class clazz) { + if (taskId == null || taskId <= 0 || isBlank(moduleType)) { + return Map.of(); + } + List states = taskScopeStateMapper.selectList(new LambdaQueryWrapper() + .select(TaskScopeStateEntity::getScopeKey, TaskScopeStateEntity::getScopeHash, TaskScopeStateEntity::getStateJson) + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, moduleType) + .isNotNull(TaskScopeStateEntity::getStateJson)); + if (states == null || states.isEmpty()) { + return Map.of(); + } + Map result = new LinkedHashMap<>(); + for (TaskScopeStateEntity state : states) { + String payloadJson = resolveStatePayload(state); + if (payloadJson == null || isBlank(state.getScopeKey())) { + continue; + } + try { + result.put(state.getScopeKey(), objectMapper.readValue(payloadJson, clazz)); + } catch (Exception ex) { + throw new BusinessException("读取任务范围载荷失败"); + } + } + return result; + } + + @Transactional + public void removeScopePayload(Long taskId, String moduleType, String scopeKey) { + if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey)) { + return; + } + TaskScopeStateEntity state = getScopeState(taskId, moduleType, hash(normalize(scopeKey))); + if (state == null) { + return; + } + deleteBufferedPayload(taskId, moduleType, state.getScopeHash()); + deleteStatePayloadIfNeeded(state.getStateJson()); + boolean hasParsedPayload = !isBlank(state.getParsedPayloadJson()); + if (hasParsedPayload) { + taskScopeStateMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskScopeStateEntity::getId, state.getId()) + .set(TaskScopeStateEntity::getStateJson, null) + .set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now())); + return; + } + taskScopeStateMapper.deleteById(state.getId()); + } + + public boolean hasAnyScopePayload(Long taskId, String moduleType) { + if (taskId == null || taskId <= 0 || isBlank(moduleType)) { + return false; + } + Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, moduleType) + .isNotNull(TaskScopeStateEntity::getStateJson)); + return count != null && count > 0; + } + + public long countScopePayload(Long taskId, String moduleType) { + if (taskId == null || taskId <= 0 || isBlank(moduleType)) { + return 0L; + } + Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, moduleType) + .isNotNull(TaskScopeStateEntity::getStateJson)); + return count == null ? 0L : count; + } + + @Transactional + public void deleteTaskScopePayloads(Long taskId, String moduleType) { + if (taskId == null || taskId <= 0 || isBlank(moduleType)) { + return; + } + List states = taskScopeStateMapper.selectList(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, moduleType)); + if (states == null || states.isEmpty()) { + return; + } + for (TaskScopeStateEntity state : states) { + if (state == null || state.getId() == null) { + continue; + } + deleteBufferedPayload(taskId, moduleType, state.getScopeHash()); + deleteStatePayloadIfNeeded(state.getStateJson()); + if (!isBlank(state.getParsedPayloadJson())) { + taskScopeStateMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskScopeStateEntity::getId, state.getId()) + .set(TaskScopeStateEntity::getStateJson, null) + .set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now())); + } else { + taskScopeStateMapper.deleteById(state.getId()); + } + } + } + + @Transactional + public void saveParsedPayloadMap(Long taskId, String moduleType, Map payloadByScope) { + if (taskId == null || taskId <= 0 || isBlank(moduleType) || payloadByScope == null || payloadByScope.isEmpty()) { + return; + } + LocalDateTime now = LocalDateTime.now(); + for (Map.Entry entry : payloadByScope.entrySet()) { + if (isBlank(entry.getKey()) || entry.getValue() == null) { + continue; + } + String scopeKey = normalize(entry.getKey()); + String scopeHash = hash(scopeKey); + String parsedPayloadJson = writeJson(entry.getValue(), "写入任务解析载荷失败"); + TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash); + if (state == null) { + TaskScopeStateEntity entity = new TaskScopeStateEntity(); + entity.setTaskId(taskId); + entity.setModuleType(moduleType); + entity.setScopeKey(scopeKey); + entity.setScopeHash(scopeHash); + entity.setParsedPayloadJson(parsedPayloadJson); + entity.setChunkTotal(0); + entity.setReceivedChunkCount(0); + entity.setCompleted(0); + entity.setCreatedAt(now); + entity.setUpdatedAt(now); + try { + taskScopeStateMapper.insert(entity); + continue; + } catch (DuplicateKeyException ignored) { + state = getScopeState(taskId, moduleType, scopeHash); + } + } + if (state == null) { + throw new BusinessException("写入任务解析载荷失败"); + } + deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), parsedPayloadJson); + taskScopeStateMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskScopeStateEntity::getId, state.getId()) + .set(TaskScopeStateEntity::getScopeKey, scopeKey) + .set(TaskScopeStateEntity::getParsedPayloadJson, parsedPayloadJson) + .set(TaskScopeStateEntity::getUpdatedAt, now)); + } + } + + public Map getParsedPayloadMap(Long taskId, String moduleType, Class clazz) { + if (taskId == null || taskId <= 0 || isBlank(moduleType)) { + return Map.of(); + } + List states = taskScopeStateMapper.selectList(new LambdaQueryWrapper() + .select(TaskScopeStateEntity::getScopeKey, TaskScopeStateEntity::getParsedPayloadJson) + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, moduleType) + .isNotNull(TaskScopeStateEntity::getParsedPayloadJson)); + if (states == null || states.isEmpty()) { + return Map.of(); + } + Map result = new LinkedHashMap<>(); + for (TaskScopeStateEntity state : states) { + if (isBlank(state.getParsedPayloadJson()) || isBlank(state.getScopeKey())) { + continue; + } + try { + result.put(state.getScopeKey(), objectMapper.readValue(resolveParsedPayload(state.getParsedPayloadJson()), clazz)); + } catch (Exception ex) { + throw new BusinessException("读取任务原始载荷失败"); + } + } + return result; + } + + @Transactional + public boolean recoverBufferedScopePayload(Long taskId, String moduleType, String scopeHash) { + if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeHash)) { + return false; + } + String payloadJson = readBufferedPayload(taskId, moduleType, scopeHash); + if (payloadJson == null) { + return false; + } + TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash); + if (state == null) { + deleteBufferedPayload(taskId, moduleType, scopeHash); + return false; + } + if (!shouldStoreStatePayloadInOss(moduleType)) { + deleteBufferedPayload(taskId, moduleType, scopeHash); + return false; + } + LocalDateTime now = LocalDateTime.now(); + String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson); + deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson); + taskScopeStateMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskScopeStateEntity::getId, state.getId()) + .set(TaskScopeStateEntity::getStateJson, stateJson) + .set(TaskScopeStateEntity::getLastChunkAt, now) + .set(TaskScopeStateEntity::getUpdatedAt, now)); + deleteBufferedPayload(taskId, moduleType, scopeHash); + return true; + } + + public Path getBufferedPayloadRoot() { + return Path.of(System.getProperty("java.io.tmpdir"), LOCAL_BUFFER_DIR); + } + + private TaskScopeStateEntity getScopeState(Long taskId, String moduleType, String scopeHash) { + return taskScopeStateMapper.selectOne(new LambdaQueryWrapper() + .eq(TaskScopeStateEntity::getTaskId, taskId) + .eq(TaskScopeStateEntity::getModuleType, moduleType) + .eq(TaskScopeStateEntity::getScopeHash, scopeHash) + .last("limit 1")); + } + + private String storeStatePayload(String moduleType, Long taskId, String scopeHash, String payloadJson) { + if (!shouldStoreStatePayloadInOss(moduleType)) { + return payloadJson; + } + try { + String pointer = OSS_POINTER_PREFIX + ossStorageService.uploadTaskScopePayload(moduleType, taskId, scopeHash, payloadJson); + return objectMapper.writeValueAsString(pointer); + } catch (Exception ex) { + throw new BusinessException("写入任务范围载荷失败"); + } + } + + private String resolveStatePayload(TaskScopeStateEntity state) { + if (state == null || isBlank(state.getStateJson())) { + return null; + } + String bufferedPayload = readBufferedPayload(state.getTaskId(), state.getModuleType(), state.getScopeHash()); + if (bufferedPayload != null) { + return bufferedPayload; + } + String ossPointer = extractOssPointer(state.getStateJson()); + if (ossPointer == null) { + return state.getStateJson(); + } + try { + return ossStorageService.readObjectAsString(stripOssPointerPrefix(ossPointer)); + } catch (Exception ex) { + throw new BusinessException("读取任务范围载荷失败"); + } + } + + private void deleteStatePayloadIfNeeded(String value) { + String ossPointer = extractOssPointer(value); + if (ossPointer == null) { + return; + } + try { + ossStorageService.deleteObject(stripOssPointerPrefix(ossPointer)); + } catch (Exception ignored) { + } + } + + private String resolveParsedPayload(String value) { + if (isBlank(value) || !isOssPointer(value)) { + return value; + } + try { + return ossStorageService.readObjectAsString(stripOssPointerPrefix(value)); + } catch (Exception ex) { + throw new BusinessException("读取任务解析载荷失败"); + } + } + + private void deleteParsedPayloadIfNeeded(String oldValue, String newValue) { + if (isBlank(oldValue) || oldValue.equals(newValue) || !isOssPointer(oldValue)) { + return; + } + try { + ossStorageService.deleteObject(stripOssPointerPrefix(oldValue)); + } catch (Exception ignored) { + } + } + + private void deleteReplacedStatePayloadIfNeeded(String oldValue, String newValue) { + if (isBlank(oldValue) || oldValue.equals(newValue)) { + return; + } + deleteStatePayloadIfNeeded(oldValue); + } + + private void saveOssBackedScopePayload(Long taskId, + String moduleType, + String normalizedScopeKey, + String scopeHash, + String payloadJson, + LocalDateTime now, + TaskScopeStateEntity state) { + if (state == null) { + String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson); + TaskScopeStateEntity entity = new TaskScopeStateEntity(); + entity.setTaskId(taskId); + entity.setModuleType(moduleType); + entity.setScopeKey(normalizedScopeKey); + entity.setScopeHash(scopeHash); + entity.setStateJson(stateJson); + entity.setChunkTotal(0); + entity.setReceivedChunkCount(0); + entity.setCompleted(0); + entity.setLastChunkAt(now); + entity.setLastError(null); + entity.setCreatedAt(now); + entity.setUpdatedAt(now); + try { + taskScopeStateMapper.insert(entity); + deleteBufferedPayload(taskId, moduleType, scopeHash); + return; + } catch (DuplicateKeyException ignored) { + state = getScopeState(taskId, moduleType, scopeHash); + } + } + if (state == null) { + throw new BusinessException("刷新任务范围缓冲载荷失败"); + } + + writeBufferedPayload(taskId, moduleType, scopeHash, payloadJson); + if (!shouldFlushBufferedPayload(state, now)) { + return; + } + + String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson); + deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson); + taskScopeStateMapper.update(null, new LambdaUpdateWrapper() + .eq(TaskScopeStateEntity::getId, state.getId()) + .set(TaskScopeStateEntity::getScopeKey, normalizedScopeKey) + .set(TaskScopeStateEntity::getStateJson, stateJson) + .set(TaskScopeStateEntity::getLastChunkAt, now) + .set(TaskScopeStateEntity::getUpdatedAt, now)); + deleteBufferedPayload(taskId, moduleType, scopeHash); + } + + private String writeJson(Object value, String message) { + try { + return objectMapper.writeValueAsString(value); + } catch (Exception ex) { + throw new BusinessException(message); + } + } + + private String normalize(String value) { + return value == null ? "" : value.trim(); + } + + private boolean isBlank(String value) { + return value == null || value.isBlank(); + } + + private boolean shouldStoreStatePayloadInOss(String moduleType) { + return OSS_BACKED_STATE_MODULES.contains(normalize(moduleType).toUpperCase()); + } + + private boolean isOssPointer(String value) { + return !isBlank(value) && value.startsWith(OSS_POINTER_PREFIX); + } + + private String extractOssPointer(String value) { + if (isBlank(value)) { + return null; + } + if (isOssPointer(value)) { + return value; + } + try { + String decoded = objectMapper.readValue(value, String.class); + return isOssPointer(decoded) ? decoded : null; + } catch (Exception ignored) { + return null; + } + } + + private String stripOssPointerPrefix(String value) { + return value.substring(OSS_POINTER_PREFIX.length()); + } + + private boolean shouldFlushBufferedPayload(TaskScopeStateEntity state, LocalDateTime now) { + long flushIntervalMillis = Math.max(0L, taskPressureProperties.getScopePayloadFlushIntervalMillis()); + if (flushIntervalMillis <= 0L || state.getUpdatedAt() == null) { + return true; + } + return java.time.Duration.between(state.getUpdatedAt(), now).toMillis() >= flushIntervalMillis; + } + + private void writeBufferedPayload(Long taskId, String moduleType, String scopeHash, String payloadJson) { + try { + Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash); + Files.createDirectories(path.getParent()); + Files.writeString(path, payloadJson, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); + } catch (IOException ex) { + throw new BusinessException("写入任务范围缓冲载荷失败"); + } + } + + private String readBufferedPayload(Long taskId, String moduleType, String scopeHash) { + try { + Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash); + if (!Files.isRegularFile(path)) { + return null; + } + return Files.readString(path); + } catch (IOException ex) { + return null; + } + } + + private void deleteBufferedPayload(Long taskId, String moduleType, String scopeHash) { + try { + Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash); + Files.deleteIfExists(path); + } catch (IOException ignored) { + } + } + + private Path buildBufferedPayloadPath(Long taskId, String moduleType, String scopeHash) { + String normalizedModuleType = normalize(moduleType).toLowerCase(); + return Path.of(System.getProperty("java.io.tmpdir"), LOCAL_BUFFER_DIR, normalizedModuleType, String.valueOf(taskId), scopeHash + ".json"); + } + + private String hash(String value) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] bytes = digest.digest(normalize(value).getBytes(StandardCharsets.UTF_8)); + StringBuilder sb = new StringBuilder(bytes.length * 2); + for (byte b : bytes) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); + } catch (Exception ex) { + throw new IllegalStateException("failed to hash scope key", ex); + } + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoShopSwitchService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoShopSwitchService.java index cf9629b..250a60d 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoShopSwitchService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoShopSwitchService.java @@ -50,7 +50,7 @@ public class ZiniaoShopSwitchService { if (value == null) { return ""; } - return value.replace("\u3000", " ").trim(); + return value.replace("　", " ").trim(); } public boolean isSkippableMatchError(BusinessException ex) { @@ -59,9 +59,9 @@ public class ZiniaoShopSwitchService { return false; } return (message.contains("code=40004") - || message.contains("userId\u5b58\u5728\u65e0\u6548\u7684\u53c2\u6570\u503c") - || message.contains("\u65e0\u6743\u9650") - || message.contains("\u6ca1\u6709\u6743\u9650")) - && !message.contains("\u767d\u540d\u5355"); + || message.contains("userId存在无效的参数值") + || message.contains("无权限") + || message.contains("没有权限")) + && !message.contains("白名单"); } } diff --git a/backend-java/src/main/resources/application.yml b/backend-java/src/main/resources/application.yml index faf0ee3..b740648 100644 --- a/backend-java/src/main/resources/application.yml +++ b/backend-java/src/main/resources/application.yml @@ -99,6 +99,10 @@ aiimage: task-pressure: local-task-entity-cache-millis: ${AIIMAGE_TASK_LOCAL_ENTITY_CACHE_MILLIS:3000} db-select-batch-size: ${AIIMAGE_TASK_DB_SELECT_BATCH_SIZE:200} + scope-payload-flush-interval-millis: ${AIIMAGE_TASK_SCOPE_PAYLOAD_FLUSH_INTERVAL_MILLIS:15000} + scope-payload-buffer-retention-hours: ${AIIMAGE_TASK_SCOPE_PAYLOAD_BUFFER_RETENTION_HOURS:24} + scope-payload-recovery-max-files: ${AIIMAGE_TASK_SCOPE_PAYLOAD_RECOVERY_MAX_FILES:200} + scope-payload-cleanup-cron: ${AIIMAGE_TASK_SCOPE_PAYLOAD_CLEANUP_CRON:15 */30 * * * *} security: shop-credential-key: ${AIIMAGE_SHOP_CREDENTIAL_KEY:change-me-shop-credential-key} internal-token: ${AIIMAGE_INTERNAL_TOKEN:} diff --git a/backend-java/src/main/resources/db/V29__skip_price_asin_minimum_price.sql b/backend-java/src/main/resources/db/V29__skip_price_asin_minimum_price.sql new file mode 100644 index 0000000..2a2d5bc --- /dev/null +++ b/backend-java/src/main/resources/db/V29__skip_price_asin_minimum_price.sql @@ -0,0 +1,76 @@ +SET @schema_name = DATABASE(); + +SET @sql = IF( + EXISTS ( + SELECT 1 + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = @schema_name + AND TABLE_NAME = 'biz_skip_price_asin' + AND COLUMN_NAME = 'minimum_price_de' + ), + 'SELECT 1', + 'ALTER TABLE biz_skip_price_asin ADD COLUMN minimum_price_de DECIMAL(10,2) NULL DEFAULT NULL COMMENT ''DE minimum price'' AFTER asin_de' +); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @sql = IF( + EXISTS ( + SELECT 1 + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = @schema_name + AND TABLE_NAME = 'biz_skip_price_asin' + AND COLUMN_NAME = 'minimum_price_uk' + ), + 'SELECT 1', + 'ALTER TABLE biz_skip_price_asin ADD COLUMN minimum_price_uk DECIMAL(10,2) NULL DEFAULT NULL COMMENT ''UK minimum price'' AFTER asin_uk' +); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @sql = IF( + EXISTS ( + SELECT 1 + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = @schema_name + AND TABLE_NAME = 'biz_skip_price_asin' + AND COLUMN_NAME = 'minimum_price_fr' + ), + 'SELECT 1', + 'ALTER TABLE biz_skip_price_asin ADD COLUMN minimum_price_fr DECIMAL(10,2) NULL DEFAULT NULL COMMENT ''FR minimum price'' AFTER asin_fr' +); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @sql = IF( + EXISTS ( + SELECT 1 + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = @schema_name + AND TABLE_NAME = 'biz_skip_price_asin' + AND COLUMN_NAME = 'minimum_price_it' + ), + 'SELECT 1', + 'ALTER TABLE biz_skip_price_asin ADD COLUMN minimum_price_it DECIMAL(10,2) NULL DEFAULT NULL COMMENT ''IT minimum price'' AFTER asin_it' +); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @sql = IF( + EXISTS ( + SELECT 1 + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = @schema_name + AND TABLE_NAME = 'biz_skip_price_asin' + AND COLUMN_NAME = 'minimum_price_es' + ), + 'SELECT 1', + 'ALTER TABLE biz_skip_price_asin ADD COLUMN minimum_price_es DECIMAL(10,2) NULL DEFAULT NULL COMMENT ''ES minimum price'' AFTER asin_es' +); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; diff --git a/backend-java/src/main/resources/db/V30__task_chunk_storage.sql b/backend-java/src/main/resources/db/V30__task_chunk_storage.sql new file mode 100644 index 0000000..3b339be --- /dev/null +++ b/backend-java/src/main/resources/db/V30__task_chunk_storage.sql @@ -0,0 +1,36 @@ +CREATE TABLE IF NOT EXISTS biz_task_scope_state ( + id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key', + task_id BIGINT NOT NULL COMMENT 'task id', + module_type VARCHAR(32) NOT NULL COMMENT 'module type', + scope_key VARCHAR(1000) NOT NULL COMMENT 'scope identifier', + scope_hash CHAR(64) NOT NULL COMMENT 'hash of scope key', + parsed_payload_json JSON NULL COMMENT 'parsed source payload', + state_json JSON NULL COMMENT 'merged intermediate state', + chunk_total INT NOT NULL DEFAULT 0 COMMENT 'expected chunk count', + received_chunk_count INT NOT NULL DEFAULT 0 COMMENT 'received chunk count', + completed TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'whether scope is complete', + last_chunk_at DATETIME NULL COMMENT 'last chunk received time', + last_error VARCHAR(1000) NULL COMMENT 'last error message', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time', + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time', + UNIQUE KEY uk_task_scope (task_id, module_type, scope_hash), + KEY idx_task_completed (task_id, module_type, completed), + KEY idx_scope_hash (scope_hash) +) COMMENT='task scope state'; + +CREATE TABLE IF NOT EXISTS biz_task_chunk ( + id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key', + task_id BIGINT NOT NULL COMMENT 'task id', + module_type VARCHAR(32) NOT NULL COMMENT 'module type', + scope_key VARCHAR(1000) NOT NULL COMMENT 'scope identifier', + scope_hash CHAR(64) NOT NULL COMMENT 'hash of scope key', + chunk_index INT NOT NULL COMMENT 'chunk index starting from 1', + chunk_total INT NOT NULL DEFAULT 0 COMMENT 'total chunk count', + payload_json JSON NOT NULL COMMENT 'chunk payload', + payload_hash CHAR(64) NOT NULL COMMENT 'hash of payload json', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time', + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time', + UNIQUE KEY uk_task_scope_chunk (task_id, module_type, scope_hash, chunk_index), + KEY idx_task_scope (task_id, module_type, scope_hash), + KEY idx_task_module (task_id, module_type) +) COMMENT='task chunk detail'; diff --git a/backend/__pycache__/config.cpython-312.pyc b/backend/__pycache__/config.cpython-312.pyc index a42abff..7a86814 100644 Binary files a/backend/__pycache__/config.cpython-312.pyc and b/backend/__pycache__/config.cpython-312.pyc differ diff --git a/backend/blueprints/__pycache__/admin_api.cpython-312.pyc b/backend/blueprints/__pycache__/admin_api.cpython-312.pyc index da6ab9a..eeba27b 100644 Binary files a/backend/blueprints/__pycache__/admin_api.cpython-312.pyc and b/backend/blueprints/__pycache__/admin_api.cpython-312.pyc differ diff --git a/backend/blueprints/__pycache__/auth.cpython-312.pyc b/backend/blueprints/__pycache__/auth.cpython-312.pyc index 20f6081..00d50f0 100644 Binary files a/backend/blueprints/__pycache__/auth.cpython-312.pyc and b/backend/blueprints/__pycache__/auth.cpython-312.pyc differ diff --git a/backend/blueprints/__pycache__/main.cpython-312.pyc b/backend/blueprints/__pycache__/main.cpython-312.pyc index 35db3ff..05dca58 100644 Binary files a/backend/blueprints/__pycache__/main.cpython-312.pyc and b/backend/blueprints/__pycache__/main.cpython-312.pyc differ diff --git a/backend/blueprints/admin_api.py b/backend/blueprints/admin_api.py index 66b6633..8826a2a 100644 --- a/backend/blueprints/admin_api.py +++ b/backend/blueprints/admin_api.py @@ -12,7 +12,7 @@ import pymysql from werkzeug.security import generate_password_hash from utils.db import get_db -from utils.auth import admin_required, get_current_admin_role +from utils.auth import admin_required, login_required, get_current_admin_role ALLOWED_DEDUPE_TOTAL_DATA_ADMIN_USERNAMES = {'刘丽泓'} @@ -353,8 +353,82 @@ def _load_current_admin_menu_items(): return role, current_row, items, None +def _load_current_backend_menu_items(): + role, current_row = get_current_admin_role() + if not role or not current_row: + return None, None, None, (jsonify({'success': False, 'error': '需要登录'}), 403) + user_id = _get_current_admin_id(current_row) + if not user_id: + return role, current_row, None, (jsonify({'success': False, 'error': '当前登录用户缺少有效ID'}), 400) + result, error_response, status = _proxy_backend_java( + 'GET', + f'/api/admin/permission-users/{user_id}/column-permissions', + params={'menuType': 'admin'}, + ) + if error_response is not None: + return role, current_row, None, (error_response, status) + items = [_format_permission_item(item) for item in (result.get('data') or [])] + return role, current_row, items, None + + +def _ensure_backend_menu_access(*menu_names): + role, current_row = get_current_admin_role() + if role == 'super_admin': + return role, current_row, None + if not role or not current_row: + return role, current_row, (jsonify({'success': False, 'error': '需要登录'}), 403) + key_set, route_set = _get_current_admin_permission_sets() + for menu_name in menu_names: + config = ADMIN_MENU_ACCESS_CONFIG.get(menu_name) or {} + column_key = (config.get('column_key') or '').strip() + route_path = (config.get('route_path') or '').strip() + if (column_key and column_key in key_set) or (route_path and route_path in route_set): + return role, current_row, None + fallback_menu = menu_names[0] if menu_names else '' + fallback_config = ADMIN_MENU_ACCESS_CONFIG.get(fallback_menu) or {} + error_message = fallback_config.get('error') or '无权访问当前模块' + return role, current_row, (jsonify({'success': False, 'error': error_message}), 403) + + +def _get_column_ids_by_route_paths(route_paths, menu_type='admin'): + normalized_paths = [(path or '').strip() for path in (route_paths or []) if (path or '').strip()] + if not normalized_paths: + return [] + conn = get_db() + try: + with conn.cursor() as cur: + placeholders = ','.join(['%s'] * len(normalized_paths)) + cur.execute( + f"SELECT id FROM columns WHERE menu_type = %s AND route_path IN ({placeholders}) ORDER BY id ASC", + [menu_type] + normalized_paths, + ) + rows = cur.fetchall() + return [int(row['id']) for row in rows if row.get('id') is not None] + finally: + conn.close() + + +def _grant_backend_menu_permissions(user_ids, route_paths): + normalized_user_ids = [int(uid) for uid in (user_ids or []) if str(uid).isdigit() and int(uid) > 0] + column_ids = _get_column_ids_by_route_paths(route_paths, 'admin') + if not normalized_user_ids or not column_ids: + return + conn = get_db() + try: + with conn.cursor() as cur: + for user_id in normalized_user_ids: + for column_id in column_ids: + cur.execute( + "INSERT IGNORE INTO user_column_permission (user_id, column_id) VALUES (%s, %s)", + (user_id, column_id), + ) + conn.commit() + finally: + conn.close() + + @admin_api.route('/current-user') -@admin_required +@login_required def get_admin_current_user(): role, current_row = get_current_admin_role() if not role or not current_row: @@ -370,16 +444,16 @@ def get_admin_current_user(): @admin_api.route('/current-user/menus') -@admin_required +@login_required def get_admin_current_user_menus(): - _, _, items, denied = _load_current_admin_menu_items() + _, _, items, denied = _load_current_backend_menu_items() if denied: return denied return jsonify({'success': True, 'items': items}) @admin_api.route('/logout', methods=['POST']) -@admin_required +@login_required def admin_logout(): session.clear() return jsonify({'success': True, 'msg': '退出成功', 'redirect': '/login'}) @@ -388,17 +462,14 @@ def admin_logout(): # ---------- 用户管理 ---------- @admin_api.route('/users') -@admin_required +@login_required def list_users(): - """分页获取用户列表，支持用户名模糊搜索和按管理员归属筛选普通用户""" + """分页获取用户列表，支持用户名模糊搜索和按管理员归属筛选普通用户。""" role, current_row = get_current_admin_role() if not role: - return jsonify({'success': False, 'error': '需要管理员权限'}), 403 + return jsonify({'success': False, 'error': '需要登录'}), 403 page = max(1, int(request.args.get('page', 1))) - _, _, denied = _ensure_admin_menu_access('users', 'history', 'shop-manage', 'skip-price-asin') - if denied: - return denied - _, _, denied = _ensure_admin_menu_access('users', 'history', 'shop-manage', 'skip-price-asin') + _, _, denied = _ensure_backend_menu_access('users', 'history', 'shop-manage', 'skip-price-asin') if denied: return denied page_size = min(999, max(5, int(request.args.get('page_size', 15)))) @@ -407,6 +478,7 @@ def list_users(): created_by_id_arg = request.args.get('created_by_id') or request.args.get('admin_id') created_by_id = int(created_by_id_arg) if created_by_id_arg and str(created_by_id_arg).isdigit() else None can_view_all_users = role == 'super_admin' or (current_row and (current_row.get('username') or '').strip().lower() == 'admin') + can_view_child_users = role == 'admin' and not can_view_all_users if not can_view_all_users: created_by_id = None try: @@ -435,7 +507,7 @@ def list_users(): total = cur.fetchone()['total'] cur.execute("SELECT id, username FROM users WHERE role = 'admin' ORDER BY id") admins = [{'id': r['id'], 'username': r['username']} for r in cur.fetchall()] - else: + elif can_view_child_users: admin_id = current_row['id'] where_parts = ["(u.id = %s OR (u.role = 'normal' AND u.created_by_id = %s))"] params = [admin_id, admin_id] @@ -458,6 +530,28 @@ def list_users(): ) total = cur.fetchone()['total'] admins = [] + else: + where_parts = ["u.id = %s"] + params = [current_row['id']] + if search_username: + where_parts.append("u.username LIKE %s") + params.append("%" + search_username + "%") + where_sql = " AND ".join(where_parts) + cur.execute( + """SELECT u.id, u.username, u.is_admin, u.role, u.created_at, u.created_by_id, + creator.username AS creator_username + FROM users u + LEFT JOIN users creator ON creator.id = u.created_by_id + WHERE """ + where_sql + """ ORDER BY u.id LIMIT %s OFFSET %s""", + tuple(params) + (page_size, offset), + ) + rows = cur.fetchall() + cur.execute( + "SELECT COUNT(*) as total FROM users u WHERE " + where_sql, + tuple(params), + ) + total = cur.fetchone()['total'] + admins = [] items = [ { 'id': r['id'], @@ -485,7 +579,6 @@ def list_users(): except Exception as e: return jsonify({'success': False, 'error': str(e)}) - @admin_api.route('/user', methods=['POST']) @admin_required def create_user(): @@ -1341,9 +1434,9 @@ def delete_dedupe_total_data(item_id): # ---------- 店铺管理 ---------- @admin_api.route('/shop-manages') -@admin_required +@login_required def list_shop_manages(): - role, current_row, denied = _ensure_admin_menu_access('shop-manage') + role, current_row, denied = _ensure_backend_menu_access('shop-manage') if denied: return denied page = max(1, int(request.args.get('page', 1))) @@ -1397,9 +1490,9 @@ def list_shop_manages(): @admin_api.route('/shop-manage', methods=['POST']) -@admin_required +@login_required def create_shop_manage(): - role, current_row, denied = _ensure_admin_menu_access('shop-manage') + role, current_row, denied = _ensure_backend_menu_access('shop-manage') if denied: return denied data = request.get_json() or {} @@ -1441,9 +1534,9 @@ def create_shop_manage(): @admin_api.route('/shop-manage/', methods=['PUT']) -@admin_required +@login_required def update_shop_manage(item_id): - role, current_row, denied = _ensure_admin_menu_access('shop-manage') + role, current_row, denied = _ensure_backend_menu_access('shop-manage') if denied: return denied data = request.get_json() or {} @@ -1483,9 +1576,9 @@ def update_shop_manage(item_id): @admin_api.route('/shop-manage/', methods=['DELETE']) -@admin_required +@login_required def delete_shop_manage(item_id): - role, current_row, denied = _ensure_admin_menu_access('shop-manage') + role, current_row, denied = _ensure_backend_menu_access('shop-manage') if denied: return denied result, error_response, status = _proxy_backend_java( @@ -1505,9 +1598,9 @@ def delete_shop_manage(item_id): @admin_api.route('/shop-manage-groups') -@admin_required +@login_required def list_shop_manage_groups(): - role, current_row, denied = _ensure_admin_menu_access('shop-manage', 'skip-price-asin') + role, current_row, denied = _ensure_backend_menu_access('shop-manage', 'skip-price-asin') if denied: return denied params = {} @@ -1543,12 +1636,13 @@ def list_shop_manage_groups(): @admin_api.route('/shop-manage-group', methods=['POST']) -@admin_required +@login_required def create_shop_manage_group(): - role, current_row, denied = _ensure_admin_menu_access('shop-manage', 'skip-price-asin') + role, current_row, denied = _ensure_backend_menu_access('shop-manage', 'skip-price-asin') if denied: return denied data = request.get_json() or {} + grant_menu_routes = data.get('grant_menu_routes') or [] payload = { 'groupName': (data.get('group_name') or '').strip(), 'memberUserIds': data.get('member_user_ids') or [], @@ -1565,6 +1659,7 @@ def create_shop_manage_group(): ) if error_response is not None: return error_response, status + _grant_backend_menu_permissions(data.get('member_user_ids') or [], grant_menu_routes) item = result.get('data') or {} return jsonify({ 'success': True, @@ -1586,12 +1681,13 @@ def create_shop_manage_group(): @admin_api.route('/shop-manage-group/', methods=['PUT']) -@admin_required +@login_required def update_shop_manage_group(item_id): - role, current_row, denied = _ensure_admin_menu_access('shop-manage', 'skip-price-asin') + role, current_row, denied = _ensure_backend_menu_access('shop-manage', 'skip-price-asin') if denied: return denied data = request.get_json() or {} + grant_menu_routes = data.get('grant_menu_routes') or [] payload = { 'groupName': (data.get('group_name') or '').strip(), 'memberUserIds': data.get('member_user_ids') or [], @@ -1607,6 +1703,7 @@ def update_shop_manage_group(item_id): ) if error_response is not None: return error_response, status + _grant_backend_menu_permissions(data.get('member_user_ids') or [], grant_menu_routes) item = result.get('data') or {} return jsonify({ 'success': True, @@ -1628,9 +1725,9 @@ def update_shop_manage_group(item_id): @admin_api.route('/skip-price-asins') -@admin_required +@login_required def list_skip_price_asins(): - role, current_row, denied = _ensure_admin_menu_access('skip-price-asin') + role, current_row, denied = _ensure_backend_menu_access('skip-price-asin') if denied: return denied page = max(1, int(request.args.get('page', 1))) @@ -1668,10 +1765,15 @@ def list_skip_price_asins(): 'group_name': item.get('groupName') or '', 'shop_name': item.get('shopName') or '', 'asin_de': item.get('asinDe') or '', + 'minimum_price_de': item.get('minimumPriceDe'), 'asin_uk': item.get('asinUk') or '', + 'minimum_price_uk': item.get('minimumPriceUk'), 'asin_fr': item.get('asinFr') or '', + 'minimum_price_fr': item.get('minimumPriceFr'), 'asin_it': item.get('asinIt') or '', + 'minimum_price_it': item.get('minimumPriceIt'), 'asin_es': item.get('asinEs') or '', + 'minimum_price_es': item.get('minimumPriceEs'), 'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16], 'updated_at': (item.get('updatedAt') or '').replace('T', ' ')[:16], } @@ -1687,25 +1789,34 @@ def list_skip_price_asins(): @admin_api.route('/skip-price-asin', methods=['POST']) -@admin_required +@login_required def create_skip_price_asin(): - role, current_row, denied = _ensure_admin_menu_access('skip-price-asin') + role, current_row, denied = _ensure_backend_menu_access('skip-price-asin') if denied: return denied data = request.get_json() or {} asin_mappings = data.get('asin_mappings') or {} + minimum_price_mappings = data.get('minimum_price_mappings') or {} fallback_asin = (data.get('asin') or '').strip() if not fallback_asin and isinstance(asin_mappings, dict): for value in asin_mappings.values(): fallback_asin = (value or '').strip() if fallback_asin: break + fallback_minimum_price = data.get('minimum_price') + if fallback_minimum_price in ('', None) and isinstance(minimum_price_mappings, dict): + for value in minimum_price_mappings.values(): + if value not in ('', None): + fallback_minimum_price = value + break payload = { 'groupId': data.get('group_id'), 'shopName': (data.get('shop_name') or '').strip(), 'countries': data.get('countries') or [], 'asin': fallback_asin, 'asinMappings': asin_mappings, + 'minimumPrice': fallback_minimum_price, + 'minimumPriceMappings': minimum_price_mappings, } result, error_response, status = _proxy_backend_java( 'POST', @@ -1728,10 +1839,15 @@ def create_skip_price_asin(): 'group_name': item.get('groupName') or '', 'shop_name': item.get('shopName') or '', 'asin_de': item.get('asinDe') or '', + 'minimum_price_de': item.get('minimumPriceDe'), 'asin_uk': item.get('asinUk') or '', + 'minimum_price_uk': item.get('minimumPriceUk'), 'asin_fr': item.get('asinFr') or '', + 'minimum_price_fr': item.get('minimumPriceFr'), 'asin_it': item.get('asinIt') or '', + 'minimum_price_it': item.get('minimumPriceIt'), 'asin_es': item.get('asinEs') or '', + 'minimum_price_es': item.get('minimumPriceEs'), 'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16], 'updated_at': (item.get('updatedAt') or '').replace('T', ' ')[:16], }, @@ -1739,9 +1855,9 @@ def create_skip_price_asin(): @admin_api.route('/skip-price-asin//country/', methods=['DELETE']) -@admin_required +@login_required def delete_skip_price_asin_country(item_id, country): - role, current_row, denied = _ensure_admin_menu_access('skip-price-asin') + role, current_row, denied = _ensure_backend_menu_access('skip-price-asin') if denied: return denied result, error_response, status = _proxy_backend_java( @@ -1758,9 +1874,9 @@ def delete_skip_price_asin_country(item_id, country): @admin_api.route('/shop-manage-group/', methods=['DELETE']) -@admin_required +@login_required def delete_shop_manage_group(item_id): - role, current_row, denied = _ensure_admin_menu_access('shop-manage', 'skip-price-asin') + role, current_row, denied = _ensure_backend_menu_access('shop-manage', 'skip-price-asin') if denied: return denied result, error_response, status = _proxy_backend_java( @@ -1777,13 +1893,16 @@ def delete_shop_manage_group(item_id): @admin_api.route('/skip-price-asin//country/', methods=['PUT']) -@admin_required +@login_required def update_skip_price_asin_country(item_id, country): - role, current_row, denied = _ensure_admin_menu_access('skip-price-asin') + role, current_row, denied = _ensure_backend_menu_access('skip-price-asin') if denied: return denied data = request.get_json() or {} - payload = {'asin': (data.get('asin') or '').strip()} + payload = { + 'asin': (data.get('asin') or '').strip(), + 'minimumPrice': data.get('minimum_price'), + } result, error_response, status = _proxy_backend_java( 'PUT', f'/api/admin/skip-price-asins/{item_id}/countries/{country}', @@ -1805,10 +1924,15 @@ def update_skip_price_asin_country(item_id, country): 'group_name': item.get('groupName') or '', 'shop_name': item.get('shopName') or '', 'asin_de': item.get('asinDe') or '', + 'minimum_price_de': item.get('minimumPriceDe'), 'asin_uk': item.get('asinUk') or '', + 'minimum_price_uk': item.get('minimumPriceUk'), 'asin_fr': item.get('asinFr') or '', + 'minimum_price_fr': item.get('minimumPriceFr'), 'asin_it': item.get('asinIt') or '', + 'minimum_price_it': item.get('minimumPriceIt'), 'asin_es': item.get('asinEs') or '', + 'minimum_price_es': item.get('minimumPriceEs'), 'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16], 'updated_at': (item.get('updatedAt') or '').replace('T', ' ')[:16], }, diff --git a/backend/blueprints/auth.py b/backend/blueprints/auth.py index 479918b..4d1e5ff 100644 --- a/backend/blueprints/auth.py +++ b/backend/blueprints/auth.py @@ -5,7 +5,7 @@ from flask import Blueprint, request, redirect, url_for, session, jsonify from werkzeug.security import check_password_hash from utils.db import get_db -from utils.auth import login_required, is_session_user_valid, is_current_user_admin +from utils.auth import login_required, is_session_user_valid from utils.render import render_html auth = Blueprint('auth', __name__, url_prefix='') @@ -13,7 +13,7 @@ auth = Blueprint('auth', __name__, url_prefix='') @auth.route('/login', methods=['GET', 'POST']) def login(): - if session.get('user_id') and is_session_user_valid() and is_current_user_admin(): + if session.get('user_id') and is_session_user_valid(): return redirect(url_for('main.admin_page')) if request.method == 'POST': data = request.get_json() if request.is_json else request.form @@ -31,22 +31,17 @@ def login(): (username,) ) row = cur.fetchone() + conn.close() if row and check_password_hash(row['password_hash'], password): session.permanent = True session['user_id'] = row['id'] session['username'] = username - if not row.get('is_admin'): - session.clear() - if request.is_json: - return jsonify({'success': False, 'error': '需要管理员权限'}), 403 - return render_html('login.html', error='需要管理员权限') if request.is_json: return jsonify({'success': True, 'redirect': url_for('main.admin_page')}) return redirect(url_for('main.admin_page')) - conn.close() - except Exception as e: + except Exception as exc: if request.is_json: - return jsonify({'success': False, 'error': str(e)}) + return jsonify({'success': False, 'error': str(exc)}) return render_html('login.html', error='登录失败，请稍后重试') if request.is_json: return jsonify({'success': False, 'error': '用户名或密码错误'}) @@ -57,7 +52,7 @@ def login(): @auth.route('/api/auth/check') @login_required def api_auth_check(): - """校验登录状态，用于页面加载时判断是否已登录""" + """校验登录状态，用于页面加载时判断是否已登录。""" if not session.get('user_id'): return jsonify({'logged_in': False}) try: @@ -66,7 +61,7 @@ def api_auth_check(): cur.execute("SELECT machine, is_admin FROM users WHERE id = %s", (session['user_id'],)) row = cur.fetchone() conn.close() - if not row or not row.get('is_admin'): + if not row: return jsonify({'logged_in': False}) except Exception: return jsonify({'logged_in': False}) diff --git a/backend/blueprints/main.py b/backend/blueprints/main.py index 1781403..f7d572e 100644 --- a/backend/blueprints/main.py +++ b/backend/blueprints/main.py @@ -4,7 +4,7 @@ import os from flask import Blueprint, redirect, url_for, send_file, session -from utils.auth import login_required, admin_required, is_session_user_valid, is_current_user_admin +from utils.auth import login_required, is_session_user_valid from utils.render import render_html main = Blueprint('main', __name__, url_prefix='') @@ -15,14 +15,13 @@ STATIC_DIR = os.path.join(BASE_DIR, 'static') @main.route('/') def index(): - if session.get('user_id') and is_session_user_valid() and is_current_user_admin(): + if session.get('user_id') and is_session_user_valid(): return redirect(url_for('main.admin_page')) return redirect(url_for('auth.login')) @main.route('/admin') @login_required -@admin_required def admin_page(): return render_html('admin.html') diff --git a/backend/config.py b/backend/config.py index 6692838..97a3eb7 100644 --- a/backend/config.py +++ b/backend/config.py @@ -23,8 +23,8 @@ bucket_path = "nanri-image/" file_url_pre = f"https://{bucket}.oss-cn-hangzhou.aliyuncs.com/" import os -# backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://127.0.0.1:18080').rstrip('/') -backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://8.136.19.173:18080').rstrip('/') +backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://127.0.0.1:18080').rstrip('/') +# backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://8.136.19.173:18080').rstrip('/') os.environ['OSS_ACCESS_KEY_ID'] = accessKeyId os.environ['OSS_ACCESS_KEY_SECRET'] = accessKeySecret os.environ['SECRET_KEY'] = "ddffc7c1d02121d9554d7b080b2511b6" diff --git a/backend/run.sh b/backend/run.sh new file mode 100644 index 0000000..83112e6 --- /dev/null +++ b/backend/run.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash + +set -euo pipefail + +PORT="${PORT:-15124}" +HOST="${HOST:-0.0.0.0}" +APP_MODULE="${APP_MODULE:-app.py}" +PID_FILE="${PID_FILE:-backend.pid}" +LOG_FILE="${LOG_FILE:-backend.log}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +echo "检查端口 ${PORT} ..." +if command -v ss >/dev/null 2>&1; then + if ss -ltn | awk '{print $4}' | grep -Eq "(^|:)${PORT}$"; then + echo "错误: 端口 ${PORT} 已被占用，请先释放后再启动。" + exit 1 + fi +elif command -v netstat >/dev/null 2>&1; then + if netstat -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "(^|:)${PORT}$"; then + echo "错误: 端口 ${PORT} 已被占用，请先释放后再启动。" + exit 1 + fi +else + echo "警告: 未找到 ss 或 netstat，跳过端口占用检查。" +fi +echo "端口 ${PORT} 未被占用。" +echo "工作目录已切换至: ${SCRIPT_DIR}" + +PYTHON_BIN="" +if [[ -x "${SCRIPT_DIR}/venv/bin/python" ]]; then + PYTHON_BIN="${SCRIPT_DIR}/venv/bin/python" +elif [[ -x "${SCRIPT_DIR}/.venv/bin/python" ]]; then + PYTHON_BIN="${SCRIPT_DIR}/.venv/bin/python" +elif command -v python3 >/dev/null 2>&1; then + PYTHON_BIN="$(command -v python3)" +elif command -v python >/dev/null 2>&1; then + PYTHON_BIN="$(command -v python)" +else + echo "错误: 未找到可用的 Python，请先安装 Python 3.9+。" + exit 1 +fi + +echo "使用 Python: ${PYTHON_BIN}" +"${PYTHON_BIN}" - <<'PY' +import sys +major, minor = sys.version_info[:2] +if (major, minor) < (3, 9): + print(f"错误: 当前 Python 版本为 {major}.{minor}，项目至少需要 Python 3.9。") + print("原因: requirement.txt 中的 Flask 3.x、pandas 2.x 等依赖不支持 Python 3.6。") + raise SystemExit(1) +print(f"Python 版本检查通过: {major}.{minor}") +PY + +if [[ ! -f "requirement.txt" ]]; then + echo "错误: 未找到 requirement.txt。" + exit 1 +fi + +echo "检查关键依赖 ..." +if ! "${PYTHON_BIN}" - <<'PY' +import importlib.util +modules = ("flask", "flask_cors", "pymysql", "werkzeug") +missing = [name for name in modules if importlib.util.find_spec(name) is None] +if missing: + print("缺少依赖: " + ", ".join(missing)) + raise SystemExit(1) +print("关键依赖已安装。") +PY +then + echo "开始安装 requirement.txt ..." + "${PYTHON_BIN}" -m pip install --upgrade pip + "${PYTHON_BIN}" -m pip install -r requirement.txt +fi + +if [[ -f "${PID_FILE}" ]]; then + OLD_PID="$(cat "${PID_FILE}" 2>/dev/null || true)" + if [[ -n "${OLD_PID}" ]] && kill -0 "${OLD_PID}" 2>/dev/null; then + echo "错误: 检测到服务已在运行，PID=${OLD_PID}" + exit 1 + fi + rm -f "${PID_FILE}" +fi + +echo "启动 Flask 服务 ${HOST}:${PORT} ..." +nohup "${PYTHON_BIN}" "${APP_MODULE}" >> "${LOG_FILE}" 2>&1 & +NEW_PID=$! +echo "${NEW_PID}" > "${PID_FILE}" + +sleep 2 +if kill -0 "${NEW_PID}" 2>/dev/null; then + echo "启动成功，PID=${NEW_PID}" + echo "日志文件: ${SCRIPT_DIR}/${LOG_FILE}" +else + echo "错误: 服务启动失败，请检查日志 ${SCRIPT_DIR}/${LOG_FILE}" + exit 1 +fi diff --git a/backend/utils/__pycache__/auth.cpython-312.pyc b/backend/utils/__pycache__/auth.cpython-312.pyc index b681092..f8172f6 100644 Binary files a/backend/utils/__pycache__/auth.cpython-312.pyc and b/backend/utils/__pycache__/auth.cpython-312.pyc differ diff --git a/backend/utils/auth.py b/backend/utils/auth.py index d81fff1..772cd13 100644 --- a/backend/utils/auth.py +++ b/backend/utils/auth.py @@ -42,9 +42,13 @@ def get_current_admin_role(): ) row = cur.fetchone() conn.close() - if not row or not row.get('is_admin'): - return None, row - role = row.get('role') or ('super_admin' if row.get('created_by_id') is None else 'admin') + if not row: + return None, None + role = (row.get('role') or '').strip().lower() + if not role: + role = 'super_admin' if row.get('is_admin') and row.get('created_by_id') is None else ( + 'admin' if row.get('is_admin') else 'normal' + ) return role, row except Exception: return None, None @@ -56,7 +60,14 @@ def is_current_user_admin(): def _is_ajax_request(): - return request.headers.get('X-Requested-With') == 'XMLHttpRequest' + if request.headers.get('X-Requested-With') == 'XMLHttpRequest': + return True + if request.path.startswith('/api/'): + return True + accept = (request.headers.get('Accept') or '').lower() + if 'application/json' in accept: + return True + return False def login_required(f): diff --git a/backend/web_source/admin.html b/backend/web_source/admin.html index 2aa0b12..875370d 100644 --- a/backend/web_source/admin.html +++ b/backend/web_source/admin.html @@ -769,10 +769,10 @@ -
- +
+
-
请选择国家后输入对应 ASIN
+
请选择国家后输入对应 ASIN 和最低价
@@ -966,7 +966,7 @@ -
可添加当前组长创建的普通员工账号，按住 Ctrl 或 Command 可多选。
+
可添加当前组长创建的普通员工账号，按住 Ctrl 或 Command 可多选。
@@ -1034,6 +1034,35 @@
+ +