完成后端架构重构等

This commit is contained in:
super
2026-04-23 15:25:41 +08:00
parent 6894f9cc57
commit 0391cb223f
86 changed files with 10843 additions and 1757 deletions

6468
aiimage-backend (2).log Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -13,7 +13,7 @@ client_name=ShuFuAI
# java_api_base=http://47.111.163.154:18080
# java_api_base=http://127.0.0.1:18080
java_api_base=http://8.136.19.173:18080
java_api_base=http://127.0.0.1:18080
# java_api_base=http://8.136.19.173:18080

View File

@@ -12,9 +12,14 @@ from flask import request, redirect, url_for, session, jsonify, render_template,
from config import mysql_host, mysql_user, mysql_password, mysql_database
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
STATIC_DIR = os.path.join(BASE_DIR, 'static')
ASSETS_DIR = os.path.join(BASE_DIR, 'assets')
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
STATIC_DIR = os.path.join(BASE_DIR, 'static')
ASSETS_DIR = os.path.join(BASE_DIR, 'assets')
WEB_SOURCE_DIR = os.path.join(BASE_DIR, 'web_source')
TEMPLATE_FALLBACK_DIRS = (
WEB_SOURCE_DIR,
os.path.join(WEB_SOURCE_DIR, 'templates_backup'),
)
def get_db():
@@ -30,9 +35,13 @@ def get_db():
def _render_html(template_name: str, **context):
"""读取 HTML 模板:若为加密文件则先解密,再渲染。未加密或解密失败时按明文渲染。"""
path = os.path.join(BASE_DIR, "web_source", template_name)
if not os.path.isfile(path):
return render_template(template_name, **context)
path = next(
(candidate for candidate in (os.path.join(base_path, template_name) for base_path in TEMPLATE_FALLBACK_DIRS)
if os.path.isfile(candidate)),
None,
)
if path is None:
return render_template(template_name, **context)
with open(path, "rb") as f:
raw = f.read()
try:
@@ -45,9 +54,13 @@ def _render_html(template_name: str, **context):
def _render_html_new(template_name: str, **context):
"""读取 HTML 模板:若为加密文件则先解密,再渲染。未加密或解密失败时按明文渲染。"""
path = os.path.join(BASE_DIR, "web_source", template_name)
if not os.path.isfile(path):
return render_template(template_name, **context)
path = next(
(candidate for candidate in (os.path.join(base_path, template_name) for base_path in TEMPLATE_FALLBACK_DIRS)
if os.path.isfile(candidate)),
None,
)
if path is None:
return render_template(template_name, **context)
with open(path, "rb") as f:
raw = f.read()
try:

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -2,7 +2,7 @@
主页面蓝图首页、home、图片工作台、品牌页、静态文件、Logo
"""
import os
from flask import Blueprint, send_file
from flask import Blueprint, send_file, render_template_string
from app_common import (
get_db,
@@ -20,7 +20,39 @@ import requests
from config import base_url,version,JAVA_API_BASE
main_bp = Blueprint('main', __name__)
main_bp = Blueprint('main', __name__)
def _resolve_asset_filename(filename):
"""Resolve hashed Vite assets without maintaining hardcoded alias tables."""
safe_name = os.path.basename(filename)
exact_path = os.path.join(ASSETS_DIR, safe_name)
if os.path.isfile(exact_path):
return safe_name
base_name, ext = os.path.splitext(safe_name)
if not base_name or not ext:
return safe_name
parts = base_name.split('-')
if len(parts) < 2:
return safe_name
try:
asset_names = os.listdir(ASSETS_DIR)
except OSError:
return safe_name
for prefix_length in range(len(parts) - 1, 0, -1):
prefix = '-'.join(parts[:prefix_length])
matches = [
asset_name for asset_name in asset_names
if asset_name.endswith(ext) and asset_name.startswith(prefix)
]
if len(matches) == 1:
return matches[0]
return safe_name
@main_bp.route('/')
@@ -50,10 +82,42 @@ def wb():
return _render_html('index.html')
@main_bp.route('/brand')
@login_required
def brand_page():
return _render_html('brand.html', user_id=session.get('user_id'))
@main_bp.route('/brand')
@login_required
def brand_page():
repo_brand_path = os.path.abspath(os.path.join(BASE_DIR, '..', 'web_source', 'brand.html'))
if os.path.isfile(repo_brand_path):
try:
with open(repo_brand_path, 'rb') as f:
raw = f.read()
try:
from html_crypto import decrypt
content = decrypt(raw).decode('utf-8')
except Exception:
if raw[:7] == b'gAAAAAB':
raise
content = raw.decode('utf-8', errors='replace')
return render_template_string(content, user_id=session.get('user_id'))
except Exception:
pass
return _render_html('brand.html', user_id=session.get('user_id'))
@main_bp.route('/brand/legacy')
@login_required
def brand_page_legacy():
legacy_path = os.path.join(BASE_DIR, 'web_source', 'templates_backup', 'brand.html')
if not os.path.isfile(legacy_path):
return '', 404
with open(legacy_path, 'rb') as f:
content = f.read().decode('utf-8', errors='replace')
content = content.replace(
'<header class="top-bar">',
'<header class="top-bar" style="display:none !important;">',
1,
)
content = content.replace('height: calc(100vh - 56px);', 'height: 100vh;', 1)
return render_template_string(content, user_id=session.get('user_id'))
@@ -69,10 +133,11 @@ def serve_static(filename):
return send_file(file_abs, as_attachment=False)
@main_bp.route('/assets/<path:filename>')
def serve_assets(filename):
"""提供 static 目录及子目录下的静态文件访问。"""
filepath = os.path.normpath(os.path.join(ASSETS_DIR, filename))
@main_bp.route('/assets/<path:filename>')
def serve_assets(filename):
"""提供 static 目录及子目录下的静态文件访问。"""
filename = _resolve_asset_filename(filename)
filepath = os.path.normpath(os.path.join(ASSETS_DIR, filename))
static_abs = os.path.abspath(ASSETS_DIR)
file_abs = os.path.abspath(filepath)
if not file_abs.startswith(static_abs) or not os.path.isfile(file_abs):
@@ -82,14 +147,19 @@ def serve_assets(filename):
@main_bp.route('/new_web_source/<path:filename>')
def serve_new_web_source(filename):
"""提供 static 目录及子目录下的静态文件访问。"""
filepath = os.path.normpath(os.path.join("new_web_source", filename))
static_abs = os.path.abspath("new_web_source")
file_abs = os.path.abspath(filepath)
if not file_abs.startswith(static_abs) or not os.path.isfile(file_abs):
return '', 404
return send_file(file_abs, as_attachment=False)
"""提供 new_web_source 目录下的静态页面文件访问。"""
candidate_dirs = [
os.path.abspath(os.path.join(BASE_DIR, 'new_web_source')),
os.path.abspath(os.path.join(BASE_DIR, '..', 'new_web_source')),
]
for static_abs in candidate_dirs:
filepath = os.path.normpath(os.path.join(static_abs, filename))
file_abs = os.path.abspath(filepath)
if not file_abs.startswith(static_abs):
continue
if os.path.isfile(file_abs):
return send_file(file_abs, as_attachment=False)
return '', 404
@main_bp.route('/logo.jpg', methods=['GET'])
@@ -161,4 +231,4 @@ def proxy(path):
return Response("无法连接到后端服务", status=502)
except Exception as e:
print(f"代理请求失败: {e}")
return Response(f"代理错误: {str(e)}", status=500)
return Response(f"代理错误: {str(e)}", status=500)

View File

@@ -3,11 +3,12 @@ coze_token = "sat_12nW40INoJxArrDXbY4lSCoudbqkOYTcphC99BP2efWyzxmsk4q81WDX3ezWgq
workflow_id = "7608812635877900322"
STITCH_WORKFLOW_ID = "7608813873483300907"
import os
from dotenv import load_dotenv
from queue import Queue
load_dotenv()
_base_url = os.getenv("base_url","http://159.75.121.33:15124")
import os
from dotenv import load_dotenv
from queue import Queue
load_dotenv()
_base_url = os.getenv("base_url","http://159.75.121.33:15124")
base_dir = os.path.dirname(os.path.abspath(__file__))
# MySQL 配置
mysql_host = os.getenv("mysql_host")
@@ -15,10 +16,11 @@ mysql_user = os.getenv("mysql_user")
mysql_password = os.getenv("mysql_password","WTFrb5y6hNLz6hNy")
mysql_database = os.getenv("mysql_database","aiimage")
proxy_url = os.getenv("proxy_url")
proxy_mode = int(os.getenv("proxy_mode",1))
client_name=os.getenv("client_name") + ".exe"
JAVA_API_BASE = os.getenv("java_api_base", "http://127.0.0.1:18080")
proxy_mode = int(os.getenv("proxy_mode",1))
_client_name = os.getenv("client_name", "").strip()
client_name = f"{_client_name}.exe" if _client_name else "client.exe"
JAVA_API_BASE = os.getenv("java_api_base", "http://127.0.0.1:18080")
# 紫鸟浏览器配置
from urllib.parse import unquote

View File

@@ -433,7 +433,7 @@ def main():
webview.start(
debug=True,
storage_path=cache_path,
private_mode=False
private_mode=True
)

View File

@@ -5,8 +5,8 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>格式转换 - 数富AI</title>
<script type="module" crossorigin src="/assets/convert.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-DuyK2jB1.js">
<link rel="modulepreload" crossorigin href="/assets/brand-CKgrwtni.js">
<link rel="modulepreload" crossorigin href="/assets/pywebview-9YBa--7x.js">
<link rel="modulepreload" crossorigin href="/assets/brand-CRmRvyGD.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-D74jYk6P.css">
<link rel="stylesheet" crossorigin href="/assets/convert-7wWJ02Tw.css">
</head>

View File

@@ -5,8 +5,8 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>数据去重 - 数富AI</title>
<script type="module" crossorigin src="/assets/dedupe.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-DuyK2jB1.js">
<link rel="modulepreload" crossorigin href="/assets/brand-CKgrwtni.js">
<link rel="modulepreload" crossorigin href="/assets/pywebview-9YBa--7x.js">
<link rel="modulepreload" crossorigin href="/assets/brand-CRmRvyGD.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-D74jYk6P.css">
<link rel="stylesheet" crossorigin href="/assets/dedupe-BpNHwt51.css">
</head>

View File

@@ -5,8 +5,8 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>删除品牌 - 数富AI</title>
<script type="module" crossorigin src="/assets/delete-brand.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-DuyK2jB1.js">
<link rel="modulepreload" crossorigin href="/assets/brand-CKgrwtni.js">
<link rel="modulepreload" crossorigin href="/assets/pywebview-9YBa--7x.js">
<link rel="modulepreload" crossorigin href="/assets/brand-CRmRvyGD.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-D74jYk6P.css">
<link rel="stylesheet" crossorigin href="/assets/delete-brand-CWLpe7lu.css">
</head>

View File

@@ -5,8 +5,8 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>数据拆分 - 数富AI</title>
<script type="module" crossorigin src="/assets/split.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-DuyK2jB1.js">
<link rel="modulepreload" crossorigin href="/assets/brand-CKgrwtni.js">
<link rel="modulepreload" crossorigin href="/assets/pywebview-9YBa--7x.js">
<link rel="modulepreload" crossorigin href="/assets/brand-CRmRvyGD.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-D74jYk6P.css">
<link rel="stylesheet" crossorigin href="/assets/split-CRUIYKS6.css">
</head>

File diff suppressed because it is too large Load Diff

View File

@@ -8,4 +8,8 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
public class TaskPressureProperties {
private long localTaskEntityCacheMillis = 3000;
private int dbSelectBatchSize = 200;
private long scopePayloadFlushIntervalMillis = 15000;
private long scopePayloadBufferRetentionHours = 24;
private int scopePayloadRecoveryMaxFiles = 200;
private String scopePayloadCleanupCron = "15 */30 * * * *";
}

View File

@@ -1,25 +1,13 @@
package com.nanri.aiimage.modules.brand.service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.BrandProgressProperties;
import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultFileDto;
import com.nanri.aiimage.modules.brand.model.dto.BrandFileAggregateCacheDto;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.security.MessageDigest;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@Service
@@ -33,6 +21,7 @@ public class BrandTaskProgressCacheService {
private final StringRedisTemplate stringRedisTemplate;
private final BrandProgressProperties brandProgressProperties;
@SuppressWarnings("unused")
private final ObjectMapper objectMapper;
public BrandTaskProgressCacheService(StringRedisTemplate stringRedisTemplate,
@@ -96,121 +85,9 @@ public class BrandTaskProgressCacheService {
return stringRedisTemplate.opsForHash().entries(buildKey(taskId));
}
public void saveParsedPayload(Long taskId, Object payload) {
try {
Files.createDirectories(buildTaskDir(taskId));
Files.writeString(
buildPayloadFile(taskId),
objectMapper.writeValueAsString(payload),
StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.WRITE
);
} catch (Exception ex) {
throw new BusinessException("鏆傚瓨鍝佺墝鍘熷鏁版嵁澶辫触");
}
}
public <T> T getParsedPayload(Long taskId, TypeReference<T> typeReference) {
Path payloadFile = buildPayloadFile(taskId);
if (!Files.isRegularFile(payloadFile)) {
return null;
}
try {
return objectMapper.readValue(Files.readString(payloadFile), typeReference);
} catch (Exception ex) {
throw new BusinessException("璇诲彇鍝佺墝鍘熷鏁版嵁澶辫触");
}
}
public ChunkStoreResult storeChunk(Long taskId, BrandCrawlResultFileDto file) {
validateChunk(file);
String fileUrl = normalizeFileUrl(file.getFileUrl());
String chunkDigest = digestChunk(file);
String digestField = String.valueOf(file.getChunkIndex());
String digestKey = buildChunkDigestKey(taskId, fileUrl);
Object existingDigest = stringRedisTemplate.opsForHash().get(digestKey, digestField);
if (existingDigest instanceof String existing && !existing.isBlank()) {
refreshFileKeys(taskId, fileUrl);
if (existing.equals(chunkDigest)) {
BrandFileAggregateCacheDto aggregate = getFileAggregate(taskId, fileUrl);
int finishedFiles = countCompletedFiles(taskId);
return new ChunkStoreResult(false, false, finishedFiles, aggregate);
}
throw new BusinessException("鍚屼竴鍒嗙墖閲嶅鎻愪氦浣嗗唴瀹逛笉涓€鑷? " + fileUrl + "#" + file.getChunkIndex());
}
BrandFileAggregateCacheDto aggregate = getFileAggregate(taskId, fileUrl);
if (aggregate == null) {
aggregate = createAggregate(file);
} else {
ensureChunkConsistency(aggregate, file);
}
mergeAggregate(aggregate, file);
stringRedisTemplate.opsForSet().add(buildChunkReceiptKey(taskId, fileUrl), digestField);
Long receivedSize = stringRedisTemplate.opsForSet().size(buildChunkReceiptKey(taskId, fileUrl));
int receivedCount = receivedSize == null ? 0 : receivedSize.intValue();
if (receivedCount <= 0) {
receivedCount = aggregate.getReceivedChunkCount() == null ? 0 : aggregate.getReceivedChunkCount();
}
aggregate.setReceivedChunkCount(Math.max(receivedCount, aggregate.getReceivedChunkCount() == null ? 0 : aggregate.getReceivedChunkCount()));
boolean fileCompleted = aggregate.getChunkTotal() != null
&& aggregate.getChunkTotal() > 0
&& aggregate.getReceivedChunkCount() >= aggregate.getChunkTotal();
aggregate.setCompleted(fileCompleted);
saveFileAggregate(taskId, aggregate);
stringRedisTemplate.opsForHash().put(digestKey, digestField, chunkDigest);
refreshFileKeys(taskId, fileUrl);
boolean newlyCompleted = false;
int finishedFiles = countCompletedFiles(taskId);
if (fileCompleted) {
Long added = stringRedisTemplate.opsForSet().add(buildCompletedFilesKey(taskId), fileUrl);
refreshTaskKey(buildCompletedFilesKey(taskId));
newlyCompleted = added != null && added > 0;
if (newlyCompleted) {
finishedFiles = countCompletedFiles(taskId);
}
}
return new ChunkStoreResult(true, newlyCompleted, finishedFiles, aggregate);
}
public BrandFileAggregateCacheDto getFileAggregate(Long taskId, String fileUrl) {
String normalizedFileUrl = normalizeFileUrl(fileUrl);
Object raw = stringRedisTemplate.opsForHash().get(buildFileAggregateKey(taskId), normalizedFileUrl);
if (!(raw instanceof String json) || json.isBlank()) {
return null;
}
try {
return objectMapper.readValue(json, BrandFileAggregateCacheDto.class);
} catch (Exception ex) {
throw new BusinessException("璇诲彇鍝佺墝鏂囦欢鑱氬悎缂撳瓨澶辫触");
}
}
public Map<String, BrandFileAggregateCacheDto> getAllFileAggregates(Long taskId) {
Map<Object, Object> stored = stringRedisTemplate.opsForHash().entries(buildFileAggregateKey(taskId));
Map<String, BrandFileAggregateCacheDto> result = new LinkedHashMap<>();
for (Map.Entry<Object, Object> entry : stored.entrySet()) {
if (!(entry.getKey() instanceof String fileUrl) || !(entry.getValue() instanceof String json) || json.isBlank()) {
continue;
}
try {
result.put(fileUrl, objectMapper.readValue(json, BrandFileAggregateCacheDto.class));
} catch (Exception ignored) {
}
}
return result;
}
public int countCompletedFiles(Long taskId) {
Long count = stringRedisTemplate.opsForSet().size(buildCompletedFilesKey(taskId));
return count == null ? 0 : count.intValue();
}
public boolean acquireFinalizeLock(Long taskId) {
Boolean ok = stringRedisTemplate.opsForValue().setIfAbsent(buildFinalizeLockKey(taskId), String.valueOf(Instant.now().toEpochMilli()), FINALIZE_LOCK_TTL);
Boolean ok = stringRedisTemplate.opsForValue()
.setIfAbsent(buildFinalizeLockKey(taskId), String.valueOf(Instant.now().toEpochMilli()), FINALIZE_LOCK_TTL);
return Boolean.TRUE.equals(ok);
}
@@ -220,18 +97,7 @@ public class BrandTaskProgressCacheService {
public void delete(Long taskId) {
stringRedisTemplate.delete(buildKey(taskId));
stringRedisTemplate.delete(buildFileAggregateKey(taskId));
stringRedisTemplate.delete(buildCompletedFilesKey(taskId));
stringRedisTemplate.delete(buildFinalizeLockKey(taskId));
stringRedisTemplate.delete(buildLegacyResultKey(taskId));
deleteTaskDirQuietly(taskId);
}
public void deleteFileState(Long taskId, String fileUrl) {
String normalizedFileUrl = normalizeFileUrl(fileUrl);
stringRedisTemplate.opsForHash().delete(buildFileAggregateKey(taskId), normalizedFileUrl);
stringRedisTemplate.delete(buildChunkReceiptKey(taskId, normalizedFileUrl));
stringRedisTemplate.delete(buildChunkDigestKey(taskId, normalizedFileUrl));
}
public String buildKey(Long taskId) {
@@ -242,182 +108,14 @@ public class BrandTaskProgressCacheService {
return brandProgressProperties.getHeartbeatTimeoutMinutes();
}
private void saveFileAggregate(Long taskId, BrandFileAggregateCacheDto aggregate) {
try {
stringRedisTemplate.opsForHash().put(buildFileAggregateKey(taskId), normalizeFileUrl(aggregate.getFileUrl()), objectMapper.writeValueAsString(aggregate));
refreshTaskKey(buildFileAggregateKey(taskId));
} catch (Exception ex) {
throw new BusinessException("鏆傚瓨鍝佺墝鏂囦欢鑱氬悎缁撴灉澶辫触");
}
}
private void validateChunk(BrandCrawlResultFileDto file) {
if (file == null) {
throw new BusinessException("鍒嗙墖涓嶈兘涓虹┖");
}
if (file.getChunkIndex() == null || file.getChunkTotal() == null || file.getChunkIndex() <= 0 || file.getChunkTotal() <= 0 || file.getChunkIndex() > file.getChunkTotal()) {
throw new BusinessException("鍒嗙墖鍙傛暟涓嶅悎娉?");
}
if (file.getFileUrl() == null || file.getFileUrl().isBlank()) {
throw new BusinessException("fileUrl 涓嶈兘涓虹┖");
}
}
private BrandFileAggregateCacheDto createAggregate(BrandCrawlResultFileDto file) {
BrandFileAggregateCacheDto aggregate = new BrandFileAggregateCacheDto();
aggregate.setFileUrl(normalizeFileUrl(file.getFileUrl()));
aggregate.setOriginalFilename(file.getOriginalFilename());
aggregate.setRelativePath(file.getRelativePath());
aggregate.setMainSheetName(file.getMainSheetName());
aggregate.setChunkTotal(file.getChunkTotal());
aggregate.setTotalLines(safePositive(file.getTotalLines()));
aggregate.setReceivedChunkCount(0);
aggregate.setProcessedLineCount(0);
aggregate.setCompleted(false);
aggregate.setInvalidBrands(new ArrayList<>());
aggregate.setQueryFailedBrands(new ArrayList<>());
return aggregate;
}
private void ensureChunkConsistency(BrandFileAggregateCacheDto aggregate, BrandCrawlResultFileDto file) {
if (aggregate.getChunkTotal() != null && file.getChunkTotal() != null && !aggregate.getChunkTotal().equals(file.getChunkTotal())) {
throw new BusinessException("鍚屼竴鏂囦欢鐨?chunkTotal 涓嶄竴鑷? " + aggregate.getFileUrl());
}
if (aggregate.getChunkTotal() == null) {
aggregate.setChunkTotal(file.getChunkTotal());
}
int incomingTotalLines = safePositive(file.getTotalLines());
if (incomingTotalLines > 0) {
aggregate.setTotalLines(Math.max(safePositive(aggregate.getTotalLines()), incomingTotalLines));
}
if ((aggregate.getOriginalFilename() == null || aggregate.getOriginalFilename().isBlank()) && file.getOriginalFilename() != null && !file.getOriginalFilename().isBlank()) {
aggregate.setOriginalFilename(file.getOriginalFilename());
}
if ((aggregate.getRelativePath() == null || aggregate.getRelativePath().isBlank()) && file.getRelativePath() != null && !file.getRelativePath().isBlank()) {
aggregate.setRelativePath(file.getRelativePath());
}
if ((aggregate.getMainSheetName() == null || aggregate.getMainSheetName().isBlank()) && file.getMainSheetName() != null && !file.getMainSheetName().isBlank()) {
aggregate.setMainSheetName(file.getMainSheetName());
}
}
private void mergeAggregate(BrandFileAggregateCacheDto aggregate, BrandCrawlResultFileDto file) {
int processed = safePositive(aggregate.getProcessedLineCount())
+ sizeOf(file.getKeptRows())
+ sizeOf(file.getInvalidBrands())
+ sizeOf(file.getQueryFailedBrands());
aggregate.setProcessedLineCount(processed);
if (file.getInvalidBrands() != null && !file.getInvalidBrands().isEmpty()) {
aggregate.getInvalidBrands().addAll(file.getInvalidBrands());
}
if (file.getQueryFailedBrands() != null && !file.getQueryFailedBrands().isEmpty()) {
aggregate.getQueryFailedBrands().addAll(file.getQueryFailedBrands());
}
}
private int sizeOf(List<?> list) {
return list == null ? 0 : list.size();
}
private int safePositive(Integer value) {
return value == null || value <= 0 ? 0 : value;
}
private String digestChunk(BrandCrawlResultFileDto file) {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest(objectMapper.writeValueAsString(file).getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
} catch (Exception ex) {
throw new BusinessException("璁$畻鍝佺墝缁撴灉鍒嗙墖鎽樿澶辫触");
}
}
private void refreshFileKeys(Long taskId, String fileUrl) {
refreshTaskKey(buildChunkReceiptKey(taskId, fileUrl));
refreshTaskKey(buildChunkDigestKey(taskId, fileUrl));
refreshTaskKey(buildFileAggregateKey(taskId));
}
private void refreshTaskKey(String key) {
stringRedisTemplate.expire(key, ttl());
}
private Duration ttl() {
return Duration.ofHours(brandProgressProperties.getTtlHours());
}
private Path buildTaskDir(Long taskId) {
return Path.of(System.getProperty("java.io.tmpdir"), "brand-task-cache", String.valueOf(taskId));
}
private Path buildPayloadFile(Long taskId) {
return buildTaskDir(taskId).resolve("parsed-payload.json");
}
private void deleteTaskDirQuietly(Long taskId) {
Path taskDir = buildTaskDir(taskId);
if (!Files.exists(taskDir)) {
return;
}
try (var walk = Files.walk(taskDir)) {
walk.sorted(java.util.Comparator.reverseOrder()).forEach(path -> {
try {
Files.deleteIfExists(path);
} catch (IOException ignored) {
}
});
} catch (IOException ignored) {
}
}
private String buildFileAggregateKey(Long taskId) {
return "brand:task:file-aggregate:" + taskId;
}
private String buildChunkReceiptKey(Long taskId, String fileUrl) {
return "brand:task:file-chunks:" + taskId + ":" + hashFileUrl(fileUrl);
}
private String buildChunkDigestKey(Long taskId, String fileUrl) {
return "brand:task:file-chunk-digests:" + taskId + ":" + hashFileUrl(fileUrl);
}
private String buildCompletedFilesKey(Long taskId) {
return "brand:task:completed-files:" + taskId;
}
private String buildFinalizeLockKey(Long taskId) {
return "brand:task:finalizing:" + taskId;
}
private String buildLegacyResultKey(Long taskId) {
return "brand:task:result-chunks:" + taskId;
}
private String normalizeFileUrl(String fileUrl) {
return fileUrl == null ? "" : fileUrl.trim();
}
private String hashFileUrl(String fileUrl) {
String normalized = normalizeFileUrl(fileUrl);
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest(normalized.getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
} catch (Exception ex) {
throw new BusinessException("鐢熸垚鏂囦欢缂撳瓨閿け璐?");
}
}
private String normalizePhase(String phase) {
if (phase == null || phase.isBlank()) {
return PHASE_CRAWLING;
@@ -432,7 +130,4 @@ public class BrandTaskProgressCacheService {
private String blankToEmpty(String value) {
return value == null ? "" : value.trim();
}
public record ChunkStoreResult(boolean stored, boolean newlyCompletedFile, int finishedFiles, BrandFileAggregateCacheDto aggregate) {
}
}

View File

@@ -39,7 +39,7 @@ import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@@ -83,6 +83,7 @@ public class BrandTaskService {
private final StorageProperties storageProperties;
private final BrandProgressProperties brandProgressProperties;
private final BrandTaskProgressCacheService brandTaskProgressCacheService;
private final BrandTaskStorageService brandTaskStorageService;
private final DistributedJobLockService distributedJobLockService;
private final ObjectMapper objectMapper;
@@ -193,7 +194,7 @@ public class BrandTaskService {
cachedFiles.add(cacheDto);
}
brandTaskProgressCacheService.saveParsedPayload(taskId, cachedFiles);
brandTaskStorageService.saveParsedPayload(taskId, cachedFiles);
brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
.eq(BrandCrawlTaskEntity::getId, taskId)
@@ -226,9 +227,7 @@ public class BrandTaskService {
sourceByUrl.put(file.getFileUrl(), file);
sourceIndexByUrl.put(file.getFileUrl(), i + 1);
}
List<BrandParsedFileCacheDto> cachedFiles = brandTaskProgressCacheService.getParsedPayload(taskId,
new TypeReference<List<BrandParsedFileCacheDto>>() {
});
List<BrandParsedFileCacheDto> cachedFiles = brandTaskStorageService.getParsedPayload(taskId);
if (cachedFiles == null || cachedFiles.isEmpty()) {
throw new BusinessException("任务原始数据已过期,请重新创建任务");
}
@@ -252,7 +251,7 @@ public class BrandTaskService {
totalCount,
Thread.currentThread().getName());
int finishedCount = brandTaskProgressCacheService.countCompletedFiles(taskId);
int finishedCount = brandTaskStorageService.countCompletedFiles(taskId);
for (BrandCrawlResultFileDto resultFile : resultFiles) {
String fileUrl = blankToNull(resultFile.getFileUrl());
if (fileUrl == null) {
@@ -272,7 +271,7 @@ public class BrandTaskService {
} else if (totalLines > 0 && resultFile.getTotalLines() > totalLines) {
throw new BusinessException("totalLines 超过原始文件行数: " + fileUrl);
}
BrandTaskProgressCacheService.ChunkStoreResult storeResult = brandTaskProgressCacheService.storeChunk(taskId, resultFile);
BrandTaskStorageService.ChunkStoreResult storeResult = brandTaskStorageService.storeChunk(taskId, resultFile);
finishedCount = Math.max(finishedCount, storeResult.finishedFiles());
log.info("[brand-submit] taskId={} fileUrl={} chunk={}/{} stored={} newlyCompletedFile={} finishedFiles={}",
taskId,
@@ -316,6 +315,7 @@ public class BrandTaskService {
if (updated == 0) {
throw new BusinessException("任务不存在或无法取消");
}
brandTaskStorageService.deleteTaskData(taskId);
brandTaskProgressCacheService.delete(taskId);
}
@@ -327,6 +327,7 @@ public class BrandTaskService {
if (deleted == 0) {
throw new BusinessException("任务不存在或正在执行中无法删除");
}
brandTaskStorageService.deleteTaskData(taskId);
brandTaskProgressCacheService.delete(taskId);
}
@@ -555,7 +556,7 @@ public class BrandTaskService {
if (STATUS_FAILED.equalsIgnoreCase(freshStatus)) {
throw new BusinessException("任务已结束,不能继续组装结果");
}
Map<String, BrandFileAggregateCacheDto> aggregates = brandTaskProgressCacheService.getAllFileAggregates(taskId);
Map<String, BrandFileAggregateCacheDto> aggregates = brandTaskStorageService.getAllFileAggregates(taskId);
log.info("[brand-finalize] taskId={} aggregateFiles={} expectedFiles={} thread={}",
taskId,
aggregates.size(),
@@ -600,9 +601,7 @@ public class BrandTaskService {
if (updated == 0) {
throw new BusinessException("任务已取消");
}
for (BrandSourceFileDto sourceFile : sourceFiles) {
brandTaskProgressCacheService.deleteFileState(taskId, sourceFile.getFileUrl());
}
brandTaskStorageService.deleteTaskData(taskId);
brandTaskProgressCacheService.delete(taskId);
log.info("[brand-finalize] taskId={} finalized success files={} elapsedMs={}",
taskId,
@@ -614,7 +613,7 @@ public class BrandTaskService {
.eq(BrandCrawlTaskEntity::getId, taskId)
.ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED)
.set(BrandCrawlTaskEntity::getStatus, STATUS_FAILED)
.set(BrandCrawlTaskEntity::getProgressCurrent, brandTaskProgressCacheService.countCompletedFiles(taskId))
.set(BrandCrawlTaskEntity::getProgressCurrent, brandTaskStorageService.countCompletedFiles(taskId))
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
.set(BrandCrawlTaskEntity::getErrorMessage, ex.getMessage()));
brandTaskProgressCacheService.markFailed(taskId, ex.getMessage());
@@ -622,6 +621,8 @@ public class BrandTaskService {
throw businessException;
}
throw new BusinessException(ex.getMessage());
} finally {
cleanupBrandResultTempFiles(outputEntries, outputDir);
}
}
@@ -746,7 +747,7 @@ public class BrandTaskService {
String value = index == null ? "" : normalizeCellText(formatter.formatCellValue(row.getCell(index)));
rowData.put(column, value);
}
String brand = normalizeCellText(Objects.toString(rowData.getOrDefault("\u54c1\u724c", ""), ""));
String brand = normalizeCellText(Objects.toString(rowData.getOrDefault("品牌", ""), ""));
if (brand.isBlank()) {
rowData.put("__rowIndex", rowNum + 1);
rows.add(rowData);
@@ -827,7 +828,9 @@ public class BrandTaskService {
BrandParsedFileCacheDto cachedFile,
BrandFileAggregateCacheDto resultFile) throws IOException {
String actualStrategy = normalizeStrategy(strategy);
try (XSSFWorkbook workbook = new XSSFWorkbook()) {
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
workbook.setCompressTempFiles(true);
try {
String mainSheetName = blankToDefault(cachedFile.getSheetName(), "Sheet1");
var mainSheet = workbook.createSheet(mainSheetName);
var headerRow = mainSheet.createRow(0);
@@ -878,8 +881,35 @@ public class BrandTaskService {
}
try (FileOutputStream outputStream = new FileOutputStream(outputFile)) {
applyBrandSheetWidths(mainSheet, columns.size());
applyFixedColumnWidths(invalidSheet, new int[]{20, 12, 18});
applyFixedColumnWidths(queryFailedSheet, new int[]{20, 18});
workbook.write(outputStream);
}
} finally {
try {
workbook.close();
} catch (Exception ignored) {
}
workbook.dispose();
}
}
private void applyBrandSheetWidths(Sheet sheet, int columnCount) {
if (sheet == null || columnCount <= 0) {
return;
}
for (int i = 0; i < columnCount; i++) {
sheet.setColumnWidth(i, 20 * 256);
}
}
private void applyFixedColumnWidths(Sheet sheet, int[] widths) {
if (sheet == null || widths == null) {
return;
}
for (int i = 0; i < widths.length; i++) {
sheet.setColumnWidth(i, widths[i] * 256);
}
}
@@ -946,6 +976,43 @@ public class BrandTaskService {
return zipFile;
}
private void cleanupBrandResultTempFiles(List<OutputEntry> entries, File outputDir) {
if (entries != null) {
for (OutputEntry entry : entries) {
if (entry == null) {
continue;
}
deleteQuietly(entry.resultFile());
if (isManagedBrandSourceTempFile(entry.sourceFile())) {
deleteQuietly(entry.sourceFile());
}
}
}
deleteQuietly(outputDir);
}
private boolean isManagedBrandSourceTempFile(File file) {
if (file == null) {
return false;
}
File managedDir = FileUtil.file(storageProperties.getLocalTempDir(), "brand-source-download");
try {
return file.getCanonicalPath().startsWith(managedDir.getCanonicalPath());
} catch (IOException ex) {
return false;
}
}
private void deleteQuietly(File file) {
if (file == null || !file.exists()) {
return;
}
try {
FileUtil.del(file);
} catch (Exception ignored) {
}
}
private File buildNamedOutputFile(File outputDir, String filename) {
File candidate = FileUtil.file(outputDir, filename);
if (!candidate.exists()) {

View File

@@ -0,0 +1,475 @@
package com.nanri.aiimage.modules.brand.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultFileDto;
import com.nanri.aiimage.modules.brand.model.dto.BrandFileAggregateCacheDto;
import com.nanri.aiimage.modules.brand.model.dto.BrandParsedFileCacheDto;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import lombok.RequiredArgsConstructor;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@Service
@RequiredArgsConstructor
public class BrandTaskStorageService {
private static final String MODULE_TYPE = "BRAND";
private static final String OSS_POINTER_PREFIX = "oss:";
private final TaskChunkMapper taskChunkMapper;
private final TaskScopeStateMapper taskScopeStateMapper;
private final ObjectMapper objectMapper;
private final OssStorageService ossStorageService;
@Transactional
public void saveParsedPayload(Long taskId, List<BrandParsedFileCacheDto> payload) {
if (taskId == null || taskId <= 0 || payload == null || payload.isEmpty()) {
return;
}
LocalDateTime now = LocalDateTime.now();
for (BrandParsedFileCacheDto item : payload) {
if (item == null || isBlank(item.getFileUrl())) {
continue;
}
String scopeKey = normalize(item.getFileUrl());
String scopeHash = hash(scopeKey);
String parsedJson = storeParsedPayload(taskId, scopeHash, writeJson(item, "Save brand task parsed payload failed"));
TaskScopeStateEntity state = getScopeState(taskId, scopeHash);
if (state == null) {
TaskScopeStateEntity entity = new TaskScopeStateEntity();
entity.setTaskId(taskId);
entity.setModuleType(MODULE_TYPE);
entity.setScopeKey(scopeKey);
entity.setScopeHash(scopeHash);
entity.setParsedPayloadJson(parsedJson);
entity.setChunkTotal(0);
entity.setReceivedChunkCount(0);
entity.setCompleted(0);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskScopeStateMapper.insert(entity);
continue;
} catch (DuplicateKeyException ignored) {
state = getScopeState(taskId, scopeHash);
}
}
if (state == null) {
throw new BusinessException("Save brand task parsed payload failed");
}
deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), parsedJson);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
.set(TaskScopeStateEntity::getParsedPayloadJson, parsedJson)
.set(TaskScopeStateEntity::getUpdatedAt, now));
}
}
public List<BrandParsedFileCacheDto> getParsedPayload(Long taskId) {
if (taskId == null || taskId <= 0) {
return List.of();
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.select(TaskScopeStateEntity::getParsedPayloadJson)
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.isNotNull(TaskScopeStateEntity::getParsedPayloadJson));
if (states == null || states.isEmpty()) {
return List.of();
}
List<BrandParsedFileCacheDto> result = new ArrayList<>();
for (TaskScopeStateEntity state : states) {
String payloadJson = resolveParsedPayload(state.getParsedPayloadJson());
if (isBlank(payloadJson)) {
continue;
}
try {
result.add(objectMapper.readValue(payloadJson, BrandParsedFileCacheDto.class));
} catch (Exception ex) {
throw new BusinessException("Read brand task parsed payload failed");
}
}
return result;
}
@Transactional
public ChunkStoreResult storeChunk(Long taskId, BrandCrawlResultFileDto file) {
validateChunk(file);
String scopeKey = normalize(file.getFileUrl());
String scopeHash = hash(scopeKey);
String payloadJson = writeJson(file, "Save brand task chunk failed");
String payloadHash = hash(payloadJson);
LocalDateTime now = LocalDateTime.now();
TaskChunkEntity existingChunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash)
.eq(TaskChunkEntity::getChunkIndex, file.getChunkIndex())
.last("limit 1"));
if (existingChunk != null) {
if (payloadHash.equals(existingChunk.getPayloadHash())) {
BrandFileAggregateCacheDto aggregate = rebuildAggregate(taskId, scopeKey, scopeHash);
return new ChunkStoreResult(false, false, countCompletedFiles(taskId), aggregate);
}
throw new BusinessException("Brand task chunk conflict: " + scopeKey + "#" + file.getChunkIndex());
}
TaskChunkEntity entity = new TaskChunkEntity();
entity.setTaskId(taskId);
entity.setModuleType(MODULE_TYPE);
entity.setScopeKey(scopeKey);
entity.setScopeHash(scopeHash);
entity.setChunkIndex(file.getChunkIndex());
entity.setChunkTotal(file.getChunkTotal());
entity.setPayloadJson(payloadJson);
entity.setPayloadHash(payloadHash);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskChunkMapper.insert(entity);
} catch (DuplicateKeyException ex) {
TaskChunkEntity concurrentChunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash)
.eq(TaskChunkEntity::getChunkIndex, file.getChunkIndex())
.last("limit 1"));
if (concurrentChunk != null && payloadHash.equals(concurrentChunk.getPayloadHash())) {
BrandFileAggregateCacheDto aggregate = rebuildAggregate(taskId, scopeKey, scopeHash);
return new ChunkStoreResult(false, false, countCompletedFiles(taskId), aggregate);
}
throw new BusinessException("Brand task chunk conflict: " + scopeKey + "#" + file.getChunkIndex());
}
BrandFileAggregateCacheDto aggregate = rebuildAggregate(taskId, scopeKey, scopeHash);
boolean completed = Boolean.TRUE.equals(aggregate.getCompleted());
saveAggregate(taskId, scopeKey, scopeHash, aggregate, now);
return new ChunkStoreResult(true, completed, countCompletedFiles(taskId), aggregate);
}
public BrandFileAggregateCacheDto getFileAggregate(Long taskId, String fileUrl) {
TaskScopeStateEntity state = getScopeState(taskId, hash(normalize(fileUrl)));
if (state == null || isBlank(state.getStateJson())) {
return null;
}
try {
return objectMapper.readValue(state.getStateJson(), BrandFileAggregateCacheDto.class);
} catch (Exception ex) {
throw new BusinessException("Read brand task aggregate failed");
}
}
public Map<String, BrandFileAggregateCacheDto> getAllFileAggregates(Long taskId) {
if (taskId == null || taskId <= 0) {
return Map.of();
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.select(TaskScopeStateEntity::getScopeKey, TaskScopeStateEntity::getStateJson)
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.isNotNull(TaskScopeStateEntity::getStateJson));
if (states == null || states.isEmpty()) {
return Map.of();
}
Map<String, BrandFileAggregateCacheDto> result = new LinkedHashMap<>();
for (TaskScopeStateEntity state : states) {
if (isBlank(state.getStateJson()) || isBlank(state.getScopeKey())) {
continue;
}
try {
result.put(state.getScopeKey(), objectMapper.readValue(state.getStateJson(), BrandFileAggregateCacheDto.class));
} catch (Exception ex) {
throw new BusinessException("Read brand task aggregate failed");
}
}
return result;
}
public int countCompletedFiles(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0;
}
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.eq(TaskScopeStateEntity::getCompleted, 1));
return count == null ? 0 : count.intValue();
}
@Transactional
public void deleteTaskData(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.select(TaskScopeStateEntity::getParsedPayloadJson)
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
if (states != null) {
for (TaskScopeStateEntity state : states) {
deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), null);
}
}
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
}
private void saveAggregate(Long taskId,
String scopeKey,
String scopeHash,
BrandFileAggregateCacheDto aggregate,
LocalDateTime now) {
String aggregateJson = writeJson(aggregate, "Save brand task aggregate failed");
TaskScopeStateEntity state = getScopeState(taskId, scopeHash);
if (state == null) {
throw new BusinessException("Brand task aggregate state missing: " + scopeKey);
}
int chunkTotal = aggregate.getChunkTotal() == null ? 0 : aggregate.getChunkTotal();
int receivedCount = aggregate.getReceivedChunkCount() == null ? 0 : aggregate.getReceivedChunkCount();
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
.set(TaskScopeStateEntity::getStateJson, aggregateJson)
.set(TaskScopeStateEntity::getChunkTotal, chunkTotal)
.set(TaskScopeStateEntity::getReceivedChunkCount, receivedCount)
.set(TaskScopeStateEntity::getCompleted, Boolean.TRUE.equals(aggregate.getCompleted()) ? 1 : 0)
.set(TaskScopeStateEntity::getLastChunkAt, now)
.set(TaskScopeStateEntity::getLastError, null)
.set(TaskScopeStateEntity::getUpdatedAt, now));
}
private TaskScopeStateEntity getScopeState(Long taskId, String scopeHash) {
return taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.eq(TaskScopeStateEntity::getScopeHash, scopeHash)
.last("limit 1"));
}
private BrandFileAggregateCacheDto createAggregate(BrandCrawlResultFileDto file) {
BrandFileAggregateCacheDto aggregate = new BrandFileAggregateCacheDto();
aggregate.setFileUrl(normalize(file.getFileUrl()));
aggregate.setOriginalFilename(file.getOriginalFilename());
aggregate.setRelativePath(file.getRelativePath());
aggregate.setMainSheetName(file.getMainSheetName());
aggregate.setChunkTotal(file.getChunkTotal());
aggregate.setTotalLines(safePositive(file.getTotalLines()));
aggregate.setReceivedChunkCount(0);
aggregate.setProcessedLineCount(0);
aggregate.setCompleted(false);
aggregate.setInvalidBrands(new ArrayList<>());
aggregate.setQueryFailedBrands(new ArrayList<>());
return aggregate;
}
private void ensureChunkConsistency(BrandFileAggregateCacheDto aggregate, BrandCrawlResultFileDto file) {
if (aggregate.getChunkTotal() != null && file.getChunkTotal() != null && !aggregate.getChunkTotal().equals(file.getChunkTotal())) {
throw new BusinessException("Brand task chunkTotal mismatch: " + aggregate.getFileUrl());
}
if (aggregate.getChunkTotal() == null) {
aggregate.setChunkTotal(file.getChunkTotal());
}
int incomingTotalLines = safePositive(file.getTotalLines());
if (incomingTotalLines > 0) {
aggregate.setTotalLines(Math.max(safePositive(aggregate.getTotalLines()), incomingTotalLines));
}
if (isBlank(aggregate.getOriginalFilename()) && !isBlank(file.getOriginalFilename())) {
aggregate.setOriginalFilename(file.getOriginalFilename());
}
if (isBlank(aggregate.getRelativePath()) && !isBlank(file.getRelativePath())) {
aggregate.setRelativePath(file.getRelativePath());
}
if (isBlank(aggregate.getMainSheetName()) && !isBlank(file.getMainSheetName())) {
aggregate.setMainSheetName(file.getMainSheetName());
}
}
private void mergeAggregate(BrandFileAggregateCacheDto aggregate, BrandCrawlResultFileDto file) {
int processed = safePositive(aggregate.getProcessedLineCount())
+ sizeOf(file.getKeptRows())
+ sizeOf(file.getInvalidBrands())
+ sizeOf(file.getQueryFailedBrands());
aggregate.setProcessedLineCount(processed);
if (file.getInvalidBrands() != null && !file.getInvalidBrands().isEmpty()) {
aggregate.getInvalidBrands().addAll(file.getInvalidBrands());
}
if (file.getQueryFailedBrands() != null && !file.getQueryFailedBrands().isEmpty()) {
aggregate.getQueryFailedBrands().addAll(file.getQueryFailedBrands());
}
}
private BrandFileAggregateCacheDto rebuildAggregate(Long taskId, String scopeKey, String scopeHash) {
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash)
.orderByAsc(TaskChunkEntity::getChunkIndex, TaskChunkEntity::getId));
if (chunks == null || chunks.isEmpty()) {
BrandFileAggregateCacheDto aggregate = getFileAggregate(taskId, scopeKey);
if (aggregate != null) {
aggregate.setFileUrl(scopeKey);
aggregate.setReceivedChunkCount(0);
aggregate.setProcessedLineCount(0);
aggregate.setCompleted(false);
aggregate.setInvalidBrands(new ArrayList<>());
aggregate.setQueryFailedBrands(new ArrayList<>());
}
return aggregate;
}
BrandFileAggregateCacheDto aggregate = null;
for (TaskChunkEntity chunk : chunks) {
BrandCrawlResultFileDto chunkPayload = readChunkPayload(chunk.getPayloadJson());
if (aggregate == null) {
aggregate = createAggregate(chunkPayload);
} else {
ensureChunkConsistency(aggregate, chunkPayload);
}
mergeAggregate(aggregate, chunkPayload);
}
if (aggregate == null) {
throw new BusinessException("Read brand task aggregate data failed");
}
aggregate.setFileUrl(scopeKey);
aggregate.setReceivedChunkCount(chunks.size());
aggregate.setCompleted(aggregate.getChunkTotal() != null
&& aggregate.getChunkTotal() > 0
&& chunks.size() >= aggregate.getChunkTotal());
return aggregate;
}
private BrandCrawlResultFileDto readChunkPayload(String payloadJson) {
try {
return objectMapper.readValue(payloadJson, BrandCrawlResultFileDto.class);
} catch (Exception ex) {
throw new BusinessException("Read brand task chunk failed");
}
}
private void validateChunk(BrandCrawlResultFileDto file) {
if (file == null) {
throw new BusinessException("Brand task chunk must not be null");
}
if (file.getChunkIndex() == null || file.getChunkTotal() == null || file.getChunkIndex() <= 0
|| file.getChunkTotal() <= 0 || file.getChunkIndex() > file.getChunkTotal()) {
throw new BusinessException("Brand task chunk parameters are invalid");
}
if (isBlank(file.getFileUrl())) {
throw new BusinessException("fileUrl must not be blank");
}
}
private String storeParsedPayload(Long taskId, String scopeHash, String payloadJson) {
try {
String pointer = OSS_POINTER_PREFIX + ossStorageService.uploadTaskParsedPayload(MODULE_TYPE, taskId, scopeHash, payloadJson);
return objectMapper.writeValueAsString(pointer);
} catch (Exception ex) {
throw new BusinessException("Save brand task parsed payload failed");
}
}
private String resolveParsedPayload(String value) {
if (isBlank(value)) {
return value;
}
String ossPointer = extractOssPointer(value);
if (ossPointer == null) {
return value;
}
try {
return ossStorageService.readObjectAsString(ossPointer.substring(OSS_POINTER_PREFIX.length()));
} catch (Exception ex) {
throw new BusinessException("Read brand task parsed payload failed");
}
}
private void deleteParsedPayloadIfNeeded(String oldValue, String newValue) {
String oldPointer = extractOssPointer(oldValue);
String newPointer = extractOssPointer(newValue);
if (oldPointer == null || oldPointer.equals(newPointer)) {
return;
}
try {
ossStorageService.deleteObject(oldPointer.substring(OSS_POINTER_PREFIX.length()));
} catch (Exception ignored) {
}
}
private String extractOssPointer(String value) {
if (isBlank(value)) {
return null;
}
if (value.startsWith(OSS_POINTER_PREFIX)) {
return value;
}
try {
String decoded = objectMapper.readValue(value, String.class);
return decoded != null && decoded.startsWith(OSS_POINTER_PREFIX) ? decoded : null;
} catch (Exception ignored) {
return null;
}
}
private String writeJson(Object value, String message) {
try {
return objectMapper.writeValueAsString(value);
} catch (Exception ex) {
throw new BusinessException(message);
}
}
private String normalize(String value) {
return value == null ? "" : value.trim();
}
private boolean isBlank(String value) {
return value == null || value.isBlank();
}
private int sizeOf(List<?> list) {
return list == null ? 0 : list.size();
}
private int safePositive(Integer value) {
return value == null || value <= 0 ? 0 : value;
}
private String hash(String value) {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest(normalize(value).getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
} catch (Exception ex) {
throw new IllegalStateException("failed to hash brand scope", ex);
}
}
public record ChunkStoreResult(boolean stored, boolean newlyCompletedFile, int finishedFiles, BrandFileAggregateCacheDto aggregate) {
}
}

View File

@@ -44,11 +44,11 @@ public class ConvertRunService {
private static final String MODULE_TYPE = "CONVERT";
private static final String TEMPLATE_CODE_FIVE_COUNTRIES = "uk_offer";
private static final List<String> FIVE_COUNTRY_OUTPUT_FILES = List.of(
"\u82f1\u56fd.txt",
"\u6cd5\u56fd.txt",
"\u5fb7\u56fd.txt",
"\u897f\u73ed\u7259.txt",
"\u610f\u5927\u5229.txt"
"英国.txt",
"法国.txt",
"德国.txt",
"西班牙.txt",
"意大利.txt"
);
private final FileTaskMapper fileTaskMapper;

View File

@@ -461,8 +461,8 @@ public class DedupeRunService {
if (value == null) {
return "";
}
return value.replace("\ufeff", "")
.replace("\u3000", " ")
return value.replace("", "")
.replace(" ", " ")
.replace("\r\n", " ")
.replace("\r", " ")
.replace("\n", " ")

View File

@@ -229,7 +229,7 @@ public class DedupeTotalDataService {
progress.setStatus("success");
} catch (Exception e) {
progress.setStatus("failed");
progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "鍒犻櫎 Excel 鍖归厤鏁版嵁澶辫触");
progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "删除 Excel 匹配数据失败");
} finally {
deleteQuietly(tempFile);
}
@@ -251,7 +251,7 @@ public class DedupeTotalDataService {
progress.setStatus("success");
} catch (Exception e) {
progress.setStatus("failed");
progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "瀵煎叆 Excel 澶辫触");
progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "导入 Excel 失败");
} finally {
deleteQuietly(tempFile);
}
@@ -590,8 +590,8 @@ public class DedupeTotalDataService {
if (value == null) {
return "";
}
return value.replace("\ufeff", "")
.replace("\u3000", " ")
return value.replace("", "")
.replace(" ", " ")
.replace("\r\n", " ")
.replace("\r", " ")
.replace("\n", " ")

View File

@@ -135,9 +135,10 @@ public class DeleteBrandRunController {
try {
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
String asciiFilename = buildAsciiDownloadFilename(filename, taskId);
response.setContentType("application/octet-stream");
response.setHeader(HttpHeaders.CONTENT_DISPOSITION,
"attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename);
"attachment; filename=\"" + asciiFilename + "\"; filename*=UTF-8''" + encodedFilename);
try (InputStream in = URI.create(url).toURL().openStream()) {
byte[] buffer = new byte[65536];
int read;
@@ -150,4 +151,24 @@ public class DeleteBrandRunController {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "下载失败");
}
}
private String buildAsciiDownloadFilename(String filename, Long taskId) {
String sanitized = filename == null ? "" : filename.replaceAll("[^A-Za-z0-9._-]", "_");
sanitized = sanitized.replaceAll("_+", "_");
sanitized = sanitized.replaceAll("^[_\\.]+|[_\\.]+$", "");
if (!sanitized.isBlank() && sanitized.contains(".")) {
return sanitized;
}
String ext = "xlsx";
if (filename != null) {
int dotIndex = filename.lastIndexOf('.');
if (dotIndex >= 0 && dotIndex < filename.length() - 1) {
String rawExt = filename.substring(dotIndex + 1).replaceAll("[^A-Za-z0-9]", "");
if (!rawExt.isBlank()) {
ext = rawExt;
}
}
}
return "delete-brand_" + taskId + "." + ext;
}
}

View File

@@ -44,7 +44,7 @@ import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@@ -74,6 +74,7 @@ public class DeleteBrandRunService {
private final FileResultMapper fileResultMapper;
private final LocalFileStorageService localFileStorageService;
private final DeleteBrandTaskCacheService deleteBrandTaskCacheService;
private final DeleteBrandTaskStorageService deleteBrandTaskStorageService;
private final ZiniaoShopSwitchService ziniaoShopSwitchService;
private final OssStorageService ossStorageService;
private final ObjectMapper objectMapper;
@@ -221,9 +222,7 @@ public class DeleteBrandRunService {
deleteBrandTaskCacheService.saveTaskCache(task);
}
if (!parsedPayloadByFileIdentity.isEmpty()) {
deleteBrandTaskCacheService.saveParsedPayload(task.getId(), parsedPayloadByFileIdentity);
}
deleteBrandTaskStorageService.saveParsedPayload(task.getId(), parsedPayloadByFileIdentity);
DeleteBrandRunVo vo = new DeleteBrandRunVo();
vo.setTotal(request.getFiles().size());
@@ -277,7 +276,7 @@ public class DeleteBrandRunService {
item.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
item.setError(entity.getErrorMessage());
// 以 task.resultJson 中的项为准补 matched/shopId/platform/openStoreUrl/错误信息
// 以 task.resultJson 中的项为准补 matched/shopId/platform/openStoreUrl/错误信息
// 真实补充
if (entity.getTaskId() != null) {
item.setTaskStatus(statusByTaskId.get(entity.getTaskId()));
@@ -291,7 +290,7 @@ public class DeleteBrandRunService {
item.setMatched(candidate.isMatched());
item.setMatchStatus(candidate.getMatchStatus());
item.setMatchMessage(candidate.getMatchMessage());
// 如果物理结果已经成功,说明已经处理过,强制为匹配成功
// 如果物理结果已经成功,说明已经处理过,强制标记为匹配成功
if (item.isSuccess() && candidate.getShopId() != null && !candidate.getShopId().isBlank()) {
item.setMatched(true);
item.setMatchStatus(ZiniaoShopIndexService.MATCH_STATUS_MATCHED);
@@ -303,8 +302,8 @@ public class DeleteBrandRunService {
item.setPlatform(candidate.getPlatform());
item.setOpenStoreUrl(candidate.getOpenStoreUrl());
// 历史表可能只有通用 errorMessage优先返回任务当时的真实 error
// 但是,如果物理结果成功,就不应该显示所谓“未匹配”报错
// 历史表可能只有通用 errorMessage优先返回任务当时的真实 error
// 但如果物理结果成功,就不应该显示所谓“未匹配”报错
if (item.isSuccess()) {
item.setError(null);
} else if ((item.getError() == null || item.getError().isBlank()) && candidate.getError() != null && !candidate.getError().isBlank()) {
@@ -344,7 +343,16 @@ public class DeleteBrandRunService {
if (entity == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) {
throw new BusinessException("记录不存在");
}
Long taskId = entity.getTaskId();
fileResultMapper.deleteById(resultId);
if (taskId != null && taskId > 0) {
Long remaining = fileResultMapper.selectCount(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getTaskId, taskId)
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
if (remaining == null || remaining <= 0) {
deleteBrandTaskStorageService.deleteTaskData(taskId);
}
}
}
private ParsedDeleteBrandFile parseDeleteBrandFile(File inputFile) {
@@ -468,8 +476,8 @@ public class DeleteBrandRunService {
if (value == null) {
return "";
}
return value.replace("\ufeff", "")
.replace("\u3000", " ")
return value.replace("", "")
.replace(" ", " ")
.replace("\r\n", " ")
.replace("\r", " ")
.replace("\n", " ")
@@ -640,7 +648,7 @@ public class DeleteBrandRunService {
taskVo.setCreatedAt(task.getCreatedAt());
taskVo.setUpdatedAt(task.getUpdatedAt());
taskVo.setFinishedAt(task.getFinishedAt());
// 轮询期避免每次都查 biz_file_result(会放大 DB 压力;仅任务成功后再提供下载信息
// 轮询期避免每次都查 biz_file_result放大 DB 压力;仅任务成功后再提供下载信息
if ("SUCCESS".equals(task.getStatus())) {
taskVo.setDownloadUrl(resolveTaskDownloadUrl(task.getId()));
taskVo.setDownloadFilename(resolveTaskDownloadFilename(task.getId()));
@@ -996,9 +1004,7 @@ public class DeleteBrandRunService {
throw new BusinessException("任务已结束,拒绝继续提交结果");
}
Map<String, DeleteBrandParsedFileCacheDto> parsedPayload = deleteBrandTaskCacheService.getParsedPayload(taskId,
new TypeReference<Map<String, DeleteBrandParsedFileCacheDto>>() {
});
Map<String, DeleteBrandParsedFileCacheDto> parsedPayload = deleteBrandTaskStorageService.loadParsedPayload(taskId);
if (parsedPayload == null || parsedPayload.isEmpty()) {
throw new BusinessException("任务原始数据已过期,请重新执行解析");
}
@@ -1029,10 +1035,10 @@ public class DeleteBrandRunService {
throw new BusinessException("文件标识不匹配: " + fileIdentity);
}
// 更早 duplicate fast-path尽量在较重校验/进度写入前直接短路
// 更早执行 duplicate fast-path尽量在较重校验进度写入前直接短路
validateChunk(fileDto, parsedFile);
validateResultFileAgainstParsed(fileDto, parsedFile);
boolean stored = deleteBrandTaskCacheService.storeResultChunkIfChanged(taskId, fileIdentity, fileDto.getChunkIndex(), fileDto);
boolean stored = deleteBrandTaskStorageService.storeResultChunkIfChanged(taskId, fileIdentity, fileDto.getChunkIndex(), fileDto);
if (!stored) {
log.info("[DeleteBrand] Duplicate chunk ignored for taskId: {}, file: {}, chunkIndex: {}", taskId, fileIdentity, fileDto.getChunkIndex());
continue;
@@ -1051,12 +1057,12 @@ public class DeleteBrandRunService {
progress.put("last_heartbeat_at", String.valueOf(System.currentTimeMillis()));
deleteBrandTaskCacheService.saveProgress(taskId, progress, true);
// 如果该文件已完成,立即更新成品计数的 Redis 缓存「提示,让后续 tryFinalizeTask 能更精
// 如果该文件已完成,立即更新成品计数的 Redis 提示,让后续 tryFinalizeTask 更准
if (isFileCompleted(fileDto)) {
log.info("[DeleteBrand] File chunk transmission fully reached via chunkTotal for taskId: {}, file: {}", taskId, fileDto.getSourceFilename());
shouldTryFinalize = true;
// 这里我们暂不直接 increment而是标记需要重算或在 tryFinalizeTask 中重新计算
// 为了保险,我们直接在 tryFinalizeTask 里重新遍历分片状态
// 这里暂不直接 increment而是标记需要在 tryFinalizeTask 中重新计算
// 为了保险,直接在 tryFinalizeTask 里重新遍历分片状态
}
}
@@ -1079,26 +1085,24 @@ public class DeleteBrandRunService {
return;
}
Map<String, DeleteBrandParsedFileCacheDto> parsedPayload = deleteBrandTaskCacheService.getParsedPayload(taskId,
new TypeReference<Map<String, DeleteBrandParsedFileCacheDto>>() {
});
Map<String, DeleteBrandParsedFileCacheDto> parsedPayload = deleteBrandTaskStorageService.loadParsedPayload(taskId);
if (parsedPayload == null || parsedPayload.isEmpty()) {
return;
}
int expectedFiles = parsedPayload.size();
// 移除 finishedFilesHint 强行返回逻辑,因为 Redis 里的 finished_files 可能是旧的,
// 既然进入 tryFinalizeTask就说明有新片到达应该以 loadMergedChunks 为准
// 移除 finishedFilesHint 的强依赖,因为 Redis 里的 finished_files 可能是旧
// 既然进入 tryFinalizeTask就说明有新片到达,应该以 loadMergedChunks 为准
Map<String, List<DeleteBrandResultFileDto>> mergedByFile = loadMergedChunks(taskId);
Map<String, List<DeleteBrandResultFileDto>> mergedByFile = deleteBrandTaskStorageService.loadMergedChunks(taskId);
int finishedFiles = countCompletedFiles(mergedByFile);
if (finishedFiles < expectedFiles) {
Map<String, String> progress = new LinkedHashMap<>();
progress.put("finished_files", String.valueOf(finishedFiles));
// 定时补偿只负责“再试一次 finalize”不能把缺片任务重新续命
// 否则 stale-check 永远看不到超时任务
// 定时补偿只负责“再试一次 finalize”不能把缺片任务重新续命
// 否则 stale-check 会一直看不到超时任务
if (!fromCompensation) {
progress.put("updated_at", String.valueOf(System.currentTimeMillis()));
}
@@ -1133,24 +1137,6 @@ public class DeleteBrandRunService {
finalizeTask(task, parsedPayload, mergedByFile);
}
private Map<String, List<DeleteBrandResultFileDto>> loadMergedChunks(Long taskId) {
Map<String, List<String>> groupedJson = deleteBrandTaskCacheService.groupResultChunkJsonByFile(taskId);
Map<String, List<DeleteBrandResultFileDto>> merged = new LinkedHashMap<>();
for (Map.Entry<String, List<String>> entry : groupedJson.entrySet()) {
List<DeleteBrandResultFileDto> chunks = new ArrayList<>();
for (String raw : entry.getValue()) {
try {
chunks.add(objectMapper.readValue(raw, DeleteBrandResultFileDto.class));
} catch (Exception ex) {
throw new BusinessException("读取删除品牌结果分片失败");
}
}
chunks.sort(Comparator.comparing(DeleteBrandResultFileDto::getChunkIndex, Comparator.nullsLast(Integer::compareTo)));
merged.put(entry.getKey(), chunks);
}
return merged;
}
private int countCompletedFiles(Map<String, List<DeleteBrandResultFileDto>> mergedByFile) {
int finishedFiles = 0;
for (List<DeleteBrandResultFileDto> chunks : mergedByFile.values()) {
@@ -1187,6 +1173,7 @@ public class DeleteBrandRunService {
private void finalizeTask(FileTaskEntity task,
Map<String, DeleteBrandParsedFileCacheDto> parsedPayload,
Map<String, List<DeleteBrandResultFileDto>> mergedByFile) {
File outputDir = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "delete-brand-result", String.valueOf(task.getId())));
try {
deleteBrandTaskCacheService.saveProgress(task.getId(), Map.of(
"phase", DeleteBrandTaskCacheService.PHASE_ASSEMBLING,
@@ -1259,6 +1246,8 @@ public class DeleteBrandRunService {
item.setPlatform(parsedFile.getPlatform());
item.setOpenStoreUrl(parsedFile.getOpenStoreUrl());
item.setMatched(parsedFile.getShopId() != null && !parsedFile.getShopId().isBlank());
item.setSuccess(true);
item.setError(null);
item.setMatchStatus(item.isMatched() ? ZiniaoShopIndexService.MATCH_STATUS_MATCHED : ZiniaoShopIndexService.MATCH_STATUS_PENDING);
item.setMatchMessage(item.isMatched() ? null : "店铺索引信息缺失,请重新匹配索引");
finalItems.add(item);
@@ -1267,12 +1256,14 @@ public class DeleteBrandRunService {
task.setStatus("SUCCESS");
task.setSuccessFileCount(successCount);
task.setFailedFileCount(Math.max(0, parsedPayload.size() - successCount));
task.setErrorMessage(null);
task.setResultJson(toCompactTaskResultJsonForDb(finalItems));
task.setUpdatedAt(LocalDateTime.now());
task.setFinishedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
deleteBrandTaskCacheService.saveTaskCache(task);
deleteBrandTaskStorageService.deleteTaskData(task.getId());
deleteBrandTaskCacheService.saveProgress(task.getId(), Map.of(
"phase", "success",
@@ -1297,6 +1288,8 @@ public class DeleteBrandRunService {
throw businessException;
}
throw new BusinessException("删除品牌结果组装失败");
} finally {
deleteQuietly(outputDir);
}
}
@@ -1355,7 +1348,9 @@ public class DeleteBrandRunService {
File outputDir = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "delete-brand-result", String.valueOf(taskId)));
File outputFile = buildNamedOutputFile(outputDir, blankToDefault(parsedFile.getSourceFilename(), "delete-brand-result.xlsx"));
try (Workbook workbook = new XSSFWorkbook(); FileOutputStream outputStream = new FileOutputStream(outputFile)) {
try (SXSSFWorkbook workbook = new SXSSFWorkbook(200);
FileOutputStream outputStream = new FileOutputStream(outputFile)) {
workbook.setCompressTempFiles(true);
Sheet sheet = workbook.createSheet("删除品牌结果");
Row headerRow = sheet.createRow(0);
headerRow.createCell(0).setCellValue("国家");
@@ -1371,9 +1366,7 @@ public class DeleteBrandRunService {
createTextCell(row, 2, item.getStatus());
}
}
for (int i = 0; i < 3; i++) {
sheet.autoSizeColumn(i);
}
applyDeleteBrandColumnWidths(sheet, 3);
workbook.write(outputStream);
return outputFile;
} catch (Exception ex) {
@@ -1387,7 +1380,9 @@ public class DeleteBrandRunService {
File outputDir = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "delete-brand-result", String.valueOf(taskId)));
File outputFile = buildNamedOutputFile(outputDir, blankToDefault(parsedFile.getSourceFilename(), "delete-brand-result.xlsx"));
try (Workbook workbook = new XSSFWorkbook(); FileOutputStream outputStream = new FileOutputStream(outputFile)) {
try (SXSSFWorkbook workbook = new SXSSFWorkbook(200);
FileOutputStream outputStream = new FileOutputStream(outputFile)) {
workbook.setCompressTempFiles(true);
Sheet sheet = workbook.createSheet("删除品牌结果");
Row countryRow = sheet.createRow(0);
Row headerRow = sheet.createRow(1);
@@ -1417,9 +1412,7 @@ public class DeleteBrandRunService {
}
}
for (int i = 0; i < mergedFile.countries().size() * 2; i++) {
sheet.autoSizeColumn(i);
}
applyDeleteBrandColumnWidths(sheet, mergedFile.countries().size() * 2);
workbook.write(outputStream);
return outputFile;
} catch (Exception ex) {
@@ -1432,6 +1425,22 @@ public class DeleteBrandRunService {
cell.setCellValue(value == null ? "" : value);
}
private void applyDeleteBrandColumnWidths(Sheet sheet, int columnCount) {
for (int i = 0; i < columnCount; i++) {
sheet.setColumnWidth(i, 18 * 256);
}
}
private void deleteQuietly(File file) {
if (file == null || !file.exists()) {
return;
}
try {
FileUtil.del(file);
} catch (Exception ignored) {
}
}
private File buildNamedOutputFile(File outputDir, String filename) {
File candidate = FileUtil.file(outputDir, filename);
if (!candidate.exists()) {
@@ -1478,10 +1487,9 @@ public class DeleteBrandRunService {
String normalized = blankToEmpty(value);
return normalized.isBlank() ? defaultValue : normalized;
}
/**
* 落库用 result_json去掉国家分组、预览行等大字段保留匹配摘要与计数。
* 索引命中后单次任务 JSON 体积会陡增,易触发 max_allowed_packet / 更新失败,进而前端 unwrap 报「请求失败」
* 索引命中后单次任务 JSON 体积会明显膨胀,容易触发 max_allowed_packet 更新失败。
*/
private String toCompactTaskResultJsonForDb(List<DeleteBrandResultItemVo> items) {
if (items == null || items.isEmpty()) {

View File

@@ -1,16 +1,11 @@
package com.nanri.aiimage.modules.deletebrand.service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.TaskPressureProperties;
import lombok.RequiredArgsConstructor;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.time.Duration;
import java.util.concurrent.ConcurrentHashMap;
@@ -24,63 +19,16 @@ public class DeleteBrandTaskCacheService {
public static final String PHASE_FAILED = "failed";
private static final long PAYLOAD_TTL_HOURS = 24;
private static final long LOCAL_PARSED_PAYLOAD_CACHE_MILLIS = 3000;
private static final long LOCAL_PROGRESS_CACHE_MILLIS = 1500;
private static final long MIN_PROGRESS_REDIS_FLUSH_MILLIS = 1200;
private final StringRedisTemplate stringRedisTemplate;
private final ObjectMapper objectMapper;
private final TaskPressureProperties taskPressureProperties;
private final ConcurrentHashMap<Long, LocalParsedPayloadCacheEntry> parsedPayloadLocalCache = new ConcurrentHashMap<>();
private final ConcurrentHashMap<Long, LocalProgressCacheEntry> progressLocalCache = new ConcurrentHashMap<>();
private final ConcurrentHashMap<Long, Long> progressRedisFlushAt = new ConcurrentHashMap<>();
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
public void saveParsedPayload(Long taskId, Object payload) {
try {
Files.createDirectories(buildTaskDir(taskId));
Files.writeString(
buildPayloadFile(taskId),
objectMapper.writeValueAsString(payload),
StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.WRITE
);
parsedPayloadLocalCache.put(taskId, new LocalParsedPayloadCacheEntry(System.currentTimeMillis(), payload));
} catch (Exception ex) {
throw new BusinessException("鏆傚瓨鍒犻櫎鍝佺墝鍘熷鏁版嵁澶辫触");
}
}
public <T> T getParsedPayload(Long taskId, TypeReference<T> typeReference) {
if (taskId == null || taskId <= 0) {
return null;
}
long now = System.currentTimeMillis();
LocalParsedPayloadCacheEntry cached = parsedPayloadLocalCache.get(taskId);
if (cached != null && cached.isFresh(now) && cached.value() != null) {
try {
return objectMapper.convertValue(cached.value(), typeReference);
} catch (Exception ignored) {
// fall through to file
}
}
Path payloadFile = buildPayloadFile(taskId);
if (!Files.isRegularFile(payloadFile)) {
parsedPayloadLocalCache.remove(taskId);
return null;
}
try {
T parsed = objectMapper.readValue(Files.readString(payloadFile), typeReference);
parsedPayloadLocalCache.put(taskId, new LocalParsedPayloadCacheEntry(now, parsed));
return parsed;
} catch (Exception ex) {
throw new BusinessException("璇诲彇鍒犻櫎鍝佺墝鍘熷鏁版嵁澶辫触");
}
}
public void saveProgress(Long taskId, java.util.Map<String, String> values) {
saveProgress(taskId, values, false);
}
@@ -150,13 +98,15 @@ public class DeleteBrandTaskCacheService {
return result;
}
java.util.List<Object> pipelineResults = stringRedisTemplate.executePipelined((org.springframework.data.redis.core.RedisCallback<Object>) connection -> {
org.springframework.data.redis.serializer.RedisSerializer<String> serializer = stringRedisTemplate.getStringSerializer();
for (Long taskId : missingTaskIds) {
connection.hGetAll(serializer.serialize(buildProgressKey(taskId)));
}
return null;
});
java.util.List<Object> pipelineResults = stringRedisTemplate.executePipelined(
(org.springframework.data.redis.core.RedisCallback<Object>) connection -> {
org.springframework.data.redis.serializer.RedisSerializer<String> serializer =
stringRedisTemplate.getStringSerializer();
for (Long taskId : missingTaskIds) {
connection.hGetAll(serializer.serialize(buildProgressKey(taskId)));
}
return null;
});
for (int i = 0; i < missingTaskIds.size(); i++) {
Long taskId = missingTaskIds.get(i);
@@ -175,83 +125,12 @@ public class DeleteBrandTaskCacheService {
return result;
}
public boolean hasSameResultChunk(Long taskId, String fileIdentity, Integer chunkIndex, Object chunk) {
if (chunkIndex == null || chunkIndex <= 0) {
throw new BusinessException("chunkIndex 涓嶅悎娉?");
}
if (fileIdentity == null || fileIdentity.isBlank()) {
throw new BusinessException("fileIdentity 涓嶈兘涓虹┖");
}
String field = fileIdentity.trim() + "#" + chunkIndex;
String chunkJson;
try {
chunkJson = objectMapper.writeValueAsString(chunk);
} catch (Exception ex) {
throw new BusinessException("璇诲彇鍒犻櫎鍝佺墝缁撴灉鍒嗙墖澶辫触");
}
Object existing = stringRedisTemplate.opsForHash().get(buildResultChunksKey(taskId), field);
if (chunkJson.equals(existing)) {
stringRedisTemplate.expire(buildResultChunksKey(taskId), Duration.ofHours(PAYLOAD_TTL_HOURS));
return true;
}
return false;
}
public boolean mergeResultChunk(Long taskId, String fileIdentity, Integer chunkIndex, Object chunk) {
String resultKey = buildResultChunksKey(taskId);
if (chunkIndex == null || chunkIndex <= 0) {
throw new BusinessException("chunkIndex 涓嶅悎娉?");
}
if (fileIdentity == null || fileIdentity.isBlank()) {
throw new BusinessException("fileIdentity 涓嶈兘涓虹┖");
}
String field = fileIdentity.trim() + "#" + chunkIndex;
String chunkJson;
try {
chunkJson = objectMapper.writeValueAsString(chunk);
} catch (Exception ex) {
throw new BusinessException("鏆傚瓨鍒犻櫎鍝佺墝缁撴灉鍒嗙墖澶辫触");
}
Object existing = stringRedisTemplate.opsForHash().get(resultKey, field);
if (chunkJson.equals(existing)) {
stringRedisTemplate.expire(resultKey, Duration.ofHours(PAYLOAD_TTL_HOURS));
return false;
}
stringRedisTemplate.opsForHash().put(resultKey, field, chunkJson);
stringRedisTemplate.expire(resultKey, Duration.ofHours(PAYLOAD_TTL_HOURS));
return true;
}
public boolean storeResultChunkIfChanged(Long taskId, String fileIdentity, Integer chunkIndex, Object chunk) {
return mergeResultChunk(taskId, fileIdentity, chunkIndex, chunk);
}
public java.util.Map<String, java.util.List<String>> groupResultChunkJsonByFile(Long taskId) {
java.util.Map<Object, Object> stored = stringRedisTemplate.opsForHash().entries(buildResultChunksKey(taskId));
java.util.Map<String, java.util.List<String>> grouped = new java.util.LinkedHashMap<>();
for (java.util.Map.Entry<Object, Object> entry : stored.entrySet()) {
if (!(entry.getKey() instanceof String field) || !(entry.getValue() instanceof String raw) || raw.isBlank()) {
continue;
}
int sep = field.lastIndexOf('#');
if (sep <= 0) {
continue;
}
String fileIdentity = field.substring(0, sep);
grouped.computeIfAbsent(fileIdentity, ignored -> new java.util.ArrayList<>()).add(raw);
}
return grouped;
}
public void delete(Long taskId) {
parsedPayloadLocalCache.remove(taskId);
progressLocalCache.remove(taskId);
progressRedisFlushAt.remove(taskId);
taskEntityLocalCache.remove(taskId);
stringRedisTemplate.delete(buildProgressKey(taskId));
stringRedisTemplate.delete(buildResultChunksKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
deleteTaskDirQuietly(taskId);
}
public void saveTaskCache(com.nanri.aiimage.modules.task.model.entity.FileTaskEntity task) {
@@ -264,13 +143,18 @@ public class DeleteBrandTaskCacheService {
objectMapper.convertValue(task, com.nanri.aiimage.modules.task.model.entity.FileTaskEntity.class)
));
try {
stringRedisTemplate.opsForValue().set(buildTaskEntityKey(task.getId()), objectMapper.writeValueAsString(task), Duration.ofHours(PAYLOAD_TTL_HOURS));
stringRedisTemplate.opsForValue().set(
buildTaskEntityKey(task.getId()),
objectMapper.writeValueAsString(task),
Duration.ofHours(PAYLOAD_TTL_HOURS));
} catch (Exception ignored) {
}
}
public java.util.Map<Long, com.nanri.aiimage.modules.task.model.entity.FileTaskEntity> getTaskCacheBatch(java.util.List<Long> taskIds) {
java.util.Map<Long, com.nanri.aiimage.modules.task.model.entity.FileTaskEntity> result = new java.util.LinkedHashMap<>();
public java.util.Map<Long, com.nanri.aiimage.modules.task.model.entity.FileTaskEntity> getTaskCacheBatch(
java.util.List<Long> taskIds) {
java.util.Map<Long, com.nanri.aiimage.modules.task.model.entity.FileTaskEntity> result =
new java.util.LinkedHashMap<>();
if (taskIds == null || taskIds.isEmpty()) {
return result;
}
@@ -285,7 +169,8 @@ public class DeleteBrandTaskCacheService {
for (Long taskId : normalized) {
LocalTaskEntityCacheEntry cached = taskEntityLocalCache.get(taskId);
if (isLocalTaskEntityCacheFresh(cached, now)) {
result.put(taskId, objectMapper.convertValue(cached.task(), com.nanri.aiimage.modules.task.model.entity.FileTaskEntity.class));
result.put(taskId, objectMapper.convertValue(
cached.task(), com.nanri.aiimage.modules.task.model.entity.FileTaskEntity.class));
} else {
missingIds.add(taskId);
}
@@ -313,40 +198,10 @@ public class DeleteBrandTaskCacheService {
return result;
}
private Path buildTaskDir(Long taskId) {
return Path.of(System.getProperty("java.io.tmpdir"), "delete-brand-task-cache", String.valueOf(taskId));
}
private Path buildPayloadFile(Long taskId) {
return buildTaskDir(taskId).resolve("parsed-payload.json");
}
private void deleteTaskDirQuietly(Long taskId) {
Path taskDir = buildTaskDir(taskId);
if (!Files.exists(taskDir)) {
return;
}
try (var walk = Files.walk(taskDir)) {
walk.sorted(java.util.Comparator.reverseOrder()).forEach(path -> {
try {
Files.deleteIfExists(path);
} catch (Exception ignored) {
}
});
} catch (Exception ignored) {
}
}
private String buildTaskEntityKey(Long taskId) {
return "delete-brand:task:entity:" + taskId;
}
private record LocalParsedPayloadCacheEntry(long cachedAtMillis, Object value) {
private boolean isFresh(long now) {
return now - cachedAtMillis <= LOCAL_PARSED_PAYLOAD_CACHE_MILLIS;
}
}
private record LocalProgressCacheEntry(long cachedAtMillis, java.util.Map<String, String> values) {
private boolean isFresh(long now) {
return now - cachedAtMillis <= LOCAL_PROGRESS_CACHE_MILLIS;
@@ -377,8 +232,4 @@ public class DeleteBrandTaskCacheService {
private String buildProgressKey(Long taskId) {
return "delete-brand:task:progress:" + taskId;
}
private String buildResultChunksKey(Long taskId) {
return "delete-brand:task:result-chunks:" + taskId;
}
}

View File

@@ -0,0 +1,404 @@
package com.nanri.aiimage.modules.deletebrand.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.deletebrand.model.cache.DeleteBrandParsedFileCacheDto;
import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandResultFileDto;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@Service
@RequiredArgsConstructor
@Slf4j
public class DeleteBrandTaskStorageService {
private static final String MODULE_TYPE = "DELETE_BRAND";
private static final String OSS_POINTER_PREFIX = "oss:";
private final TaskChunkMapper taskChunkMapper;
private final TaskScopeStateMapper taskScopeStateMapper;
private final ObjectMapper objectMapper;
private final OssStorageService ossStorageService;
@Transactional
public void saveParsedPayload(Long taskId, Map<String, DeleteBrandParsedFileCacheDto> parsedPayloadByScope) {
if (taskId == null || taskId <= 0 || parsedPayloadByScope == null || parsedPayloadByScope.isEmpty()) {
return;
}
LocalDateTime now = LocalDateTime.now();
Map<String, TaskScopeStateEntity> existingByHash = loadScopeStateByHash(taskId);
for (Map.Entry<String, DeleteBrandParsedFileCacheDto> entry : parsedPayloadByScope.entrySet()) {
String scopeKey = normalizeScopeKey(entry.getKey());
if (scopeKey.isBlank() || entry.getValue() == null) {
continue;
}
String scopeHash = hash(scopeKey);
TaskScopeStateEntity existing = existingByHash.get(scopeHash);
String payloadJson = writeJson(entry.getValue(), "failed to save delete-brand parsed payload");
String parsedPayloadPointer = storeParsedPayload(taskId, scopeHash, payloadJson);
if (existing == null) {
TaskScopeStateEntity entity = new TaskScopeStateEntity();
entity.setTaskId(taskId);
entity.setModuleType(MODULE_TYPE);
entity.setScopeKey(scopeKey);
entity.setScopeHash(scopeHash);
entity.setParsedPayloadJson(parsedPayloadPointer);
entity.setChunkTotal(0);
entity.setReceivedChunkCount(0);
entity.setCompleted(0);
entity.setLastChunkAt(null);
entity.setLastError(null);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskScopeStateMapper.insert(entity);
existingByHash.put(scopeHash, entity);
continue;
} catch (DuplicateKeyException ex) {
log.info("[delete-brand-storage] parsed payload scope inserted concurrently taskId={} scopeKey={}", taskId, scopeKey);
existing = getScopeState(taskId, scopeHash);
}
}
if (existing == null) {
throw new BusinessException("failed to save delete-brand parsed payload");
}
deleteParsedPayloadIfNeeded(existing.getParsedPayloadJson(), parsedPayloadPointer);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, existing.getId())
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
.set(TaskScopeStateEntity::getParsedPayloadJson, parsedPayloadPointer)
.set(TaskScopeStateEntity::getUpdatedAt, now));
}
}
public Map<String, DeleteBrandParsedFileCacheDto> loadParsedPayload(Long taskId) {
if (taskId == null || taskId <= 0) {
return Map.of();
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.select(TaskScopeStateEntity::getScopeKey, TaskScopeStateEntity::getParsedPayloadJson)
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.isNotNull(TaskScopeStateEntity::getParsedPayloadJson));
if (states == null || states.isEmpty()) {
return Map.of();
}
Map<String, DeleteBrandParsedFileCacheDto> result = new LinkedHashMap<>();
for (TaskScopeStateEntity state : states) {
if (isBlank(state.getScopeKey())) {
continue;
}
String payloadJson = resolveParsedPayload(state.getParsedPayloadJson());
if (isBlank(payloadJson)) {
continue;
}
try {
result.put(state.getScopeKey(), objectMapper.readValue(payloadJson, DeleteBrandParsedFileCacheDto.class));
} catch (Exception ex) {
throw new BusinessException("failed to load delete-brand parsed payload");
}
}
return result;
}
@Transactional
public boolean storeResultChunkIfChanged(Long taskId, String scopeKey, Integer chunkIndex, DeleteBrandResultFileDto chunk) {
if (taskId == null || taskId <= 0) {
throw new BusinessException("taskId is invalid");
}
if (chunkIndex == null || chunkIndex <= 0) {
throw new BusinessException("chunkIndex is invalid");
}
String normalizedScopeKey = normalizeScopeKey(scopeKey);
if (normalizedScopeKey.isBlank()) {
throw new BusinessException("scopeKey must not be blank");
}
String scopeHash = hash(normalizedScopeKey);
String payloadJson = writeJson(chunk, "failed to save delete-brand result chunk");
String payloadHash = hash(payloadJson);
LocalDateTime now = LocalDateTime.now();
TaskChunkEntity existingChunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash)
.eq(TaskChunkEntity::getChunkIndex, chunkIndex)
.last("limit 1"));
boolean changed = true;
if (existingChunk == null) {
TaskChunkEntity entity = new TaskChunkEntity();
entity.setTaskId(taskId);
entity.setModuleType(MODULE_TYPE);
entity.setScopeKey(normalizedScopeKey);
entity.setScopeHash(scopeHash);
entity.setChunkIndex(chunkIndex);
entity.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal());
entity.setPayloadJson(payloadJson);
entity.setPayloadHash(payloadHash);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskChunkMapper.insert(entity);
} catch (DuplicateKeyException ex) {
TaskChunkEntity concurrentChunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash)
.eq(TaskChunkEntity::getChunkIndex, chunkIndex)
.last("limit 1"));
if (concurrentChunk != null) {
changed = !payloadHash.equals(concurrentChunk.getPayloadHash());
concurrentChunk.setScopeKey(normalizedScopeKey);
concurrentChunk.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal());
concurrentChunk.setPayloadJson(payloadJson);
concurrentChunk.setPayloadHash(payloadHash);
concurrentChunk.setUpdatedAt(now);
taskChunkMapper.updateById(concurrentChunk);
} else {
throw ex;
}
}
} else {
changed = !payloadHash.equals(existingChunk.getPayloadHash());
existingChunk.setScopeKey(normalizedScopeKey);
existingChunk.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal());
existingChunk.setPayloadJson(payloadJson);
existingChunk.setPayloadHash(payloadHash);
existingChunk.setUpdatedAt(now);
taskChunkMapper.updateById(existingChunk);
}
refreshScopeState(taskId, normalizedScopeKey, scopeHash, chunk.getChunkTotal(), now);
return changed;
}
public Map<String, List<DeleteBrandResultFileDto>> loadMergedChunks(Long taskId) {
if (taskId == null || taskId <= 0) {
return Map.of();
}
List<TaskChunkEntity> entities = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.orderByAsc(TaskChunkEntity::getScopeHash)
.orderByAsc(TaskChunkEntity::getChunkIndex)
.orderByAsc(TaskChunkEntity::getId));
if (entities == null || entities.isEmpty()) {
return Map.of();
}
Map<String, List<DeleteBrandResultFileDto>> grouped = new LinkedHashMap<>();
for (TaskChunkEntity entity : entities) {
if (entity.getPayloadJson() == null || entity.getPayloadJson().isBlank()) {
continue;
}
try {
DeleteBrandResultFileDto dto = objectMapper.readValue(entity.getPayloadJson(), DeleteBrandResultFileDto.class);
grouped.computeIfAbsent(entity.getScopeKey(), ignored -> new ArrayList<>()).add(dto);
} catch (Exception ex) {
throw new BusinessException("failed to load delete-brand result chunks");
}
}
for (List<DeleteBrandResultFileDto> chunks : grouped.values()) {
chunks.sort(Comparator.comparing(DeleteBrandResultFileDto::getChunkIndex, Comparator.nullsLast(Integer::compareTo)));
}
return grouped;
}
@Transactional
public void deleteTaskData(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.select(TaskScopeStateEntity::getParsedPayloadJson)
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
if (states != null) {
for (TaskScopeStateEntity state : states) {
deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), null);
}
}
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
}
private void refreshScopeState(Long taskId, String scopeKey, String scopeHash, Integer chunkTotal, LocalDateTime now) {
Long receivedCountRaw = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
.eq(TaskChunkEntity::getScopeHash, scopeHash));
int receivedCount = receivedCountRaw == null ? 0 : receivedCountRaw.intValue();
int normalizedChunkTotal = chunkTotal == null || chunkTotal < 0 ? 0 : chunkTotal;
TaskScopeStateEntity state = getScopeState(taskId, scopeHash);
if (state == null) {
state = new TaskScopeStateEntity();
state.setTaskId(taskId);
state.setModuleType(MODULE_TYPE);
state.setScopeKey(scopeKey);
state.setScopeHash(scopeHash);
state.setParsedPayloadJson(null);
state.setChunkTotal(normalizedChunkTotal);
state.setReceivedChunkCount(receivedCount);
state.setCompleted(normalizedChunkTotal > 0 && receivedCount >= normalizedChunkTotal ? 1 : 0);
state.setLastChunkAt(now);
state.setLastError(null);
state.setCreatedAt(now);
state.setUpdatedAt(now);
try {
taskScopeStateMapper.insert(state);
} catch (DuplicateKeyException ex) {
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.eq(TaskScopeStateEntity::getScopeHash, scopeHash)
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
.set(TaskScopeStateEntity::getChunkTotal, normalizedChunkTotal)
.set(TaskScopeStateEntity::getReceivedChunkCount, receivedCount)
.set(TaskScopeStateEntity::getCompleted, normalizedChunkTotal > 0 && receivedCount >= normalizedChunkTotal ? 1 : 0)
.set(TaskScopeStateEntity::getLastChunkAt, now)
.set(TaskScopeStateEntity::getLastError, null)
.set(TaskScopeStateEntity::getUpdatedAt, now));
}
return;
}
int mergedChunkTotal = Math.max(state.getChunkTotal() == null ? 0 : state.getChunkTotal(), normalizedChunkTotal);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
.set(TaskScopeStateEntity::getChunkTotal, mergedChunkTotal)
.set(TaskScopeStateEntity::getReceivedChunkCount, receivedCount)
.set(TaskScopeStateEntity::getCompleted, mergedChunkTotal > 0 && receivedCount >= mergedChunkTotal ? 1 : 0)
.set(TaskScopeStateEntity::getLastChunkAt, now)
.set(TaskScopeStateEntity::getLastError, null)
.set(TaskScopeStateEntity::getUpdatedAt, now));
}
private TaskScopeStateEntity getScopeState(Long taskId, String scopeHash) {
return taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.eq(TaskScopeStateEntity::getScopeHash, scopeHash)
.last("limit 1"));
}
private Map<String, TaskScopeStateEntity> loadScopeStateByHash(Long taskId) {
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
Map<String, TaskScopeStateEntity> result = new LinkedHashMap<>();
if (states == null) {
return result;
}
for (TaskScopeStateEntity state : states) {
if (!isBlank(state.getScopeHash())) {
result.put(state.getScopeHash(), state);
}
}
return result;
}
private String storeParsedPayload(Long taskId, String scopeHash, String payloadJson) {
try {
String pointer = OSS_POINTER_PREFIX + ossStorageService.uploadTaskParsedPayload(MODULE_TYPE, taskId, scopeHash, payloadJson);
return objectMapper.writeValueAsString(pointer);
} catch (Exception ex) {
throw new BusinessException("failed to save delete-brand parsed payload");
}
}
private String resolveParsedPayload(String value) {
if (isBlank(value)) {
return value;
}
String ossPointer = extractOssPointer(value);
if (ossPointer == null) {
return value;
}
try {
return ossStorageService.readObjectAsString(ossPointer.substring(OSS_POINTER_PREFIX.length()));
} catch (Exception ex) {
throw new BusinessException("failed to load delete-brand parsed payload");
}
}
private void deleteParsedPayloadIfNeeded(String oldValue, String newValue) {
String oldPointer = extractOssPointer(oldValue);
String newPointer = extractOssPointer(newValue);
if (oldPointer == null || oldPointer.equals(newPointer)) {
return;
}
try {
ossStorageService.deleteObject(oldPointer.substring(OSS_POINTER_PREFIX.length()));
} catch (Exception ignored) {
}
}
private String extractOssPointer(String value) {
if (isBlank(value)) {
return null;
}
if (value.startsWith(OSS_POINTER_PREFIX)) {
return value;
}
try {
String decoded = objectMapper.readValue(value, String.class);
return decoded != null && decoded.startsWith(OSS_POINTER_PREFIX) ? decoded : null;
} catch (Exception ignored) {
return null;
}
}
private String writeJson(Object value, String message) {
try {
return objectMapper.writeValueAsString(value);
} catch (Exception ex) {
throw new BusinessException(message);
}
}
private String normalizeScopeKey(String scopeKey) {
return scopeKey == null ? "" : scopeKey.trim();
}
private boolean isBlank(String value) {
return value == null || value.isBlank();
}
private String hash(String value) {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest(normalizeScopeKey(value).getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
} catch (Exception ex) {
throw new IllegalStateException("failed to hash delete-brand scope", ex);
}
}
}

View File

@@ -2,14 +2,18 @@ package com.nanri.aiimage.modules.file.service.oss;
import com.aliyun.oss.OSS;
import com.aliyun.oss.OSSClientBuilder;
import com.aliyun.oss.model.OSSObject;
import com.nanri.aiimage.config.OssProperties;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.net.URI;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.Date;
import java.util.Objects;
import java.util.UUID;
@Service
@@ -33,6 +37,68 @@ public class OssStorageService {
}
}
public String uploadText(String objectKey, String content) {
if (objectKey == null || objectKey.isBlank()) {
throw new IllegalArgumentException("objectKey must not be blank");
}
OSS ossClient = buildClient();
try (ByteArrayInputStream stream = new ByteArrayInputStream(
Objects.requireNonNullElse(content, "").getBytes(StandardCharsets.UTF_8))) {
ossClient.putObject(ossProperties.getBucket(), objectKey, stream);
return objectKey;
} catch (Exception ex) {
throw new IllegalStateException("failed to upload text to oss", ex);
} finally {
ossClient.shutdown();
}
}
public String uploadTaskScopePayload(String moduleType, Long taskId, String scopeHash, String content) {
String normalizedModuleType = moduleType == null || moduleType.isBlank()
? "unknown"
: moduleType.trim().toLowerCase();
String normalizedScopeHash = scopeHash == null || scopeHash.isBlank() ? UUID.randomUUID().toString() : scopeHash;
String objectKey = String.format("task-scope/%s/%s/%s.json", normalizedModuleType, taskId, normalizedScopeHash);
return uploadText(objectKey, content);
}
public String uploadTaskParsedPayload(String moduleType, Long taskId, String scopeHash, String content) {
String normalizedModuleType = moduleType == null || moduleType.isBlank()
? "unknown"
: moduleType.trim().toLowerCase();
String normalizedScopeHash = scopeHash == null || scopeHash.isBlank() ? UUID.randomUUID().toString() : scopeHash;
String objectKey = String.format("task-parsed/%s/%s/%s.json", normalizedModuleType, taskId, normalizedScopeHash);
return uploadText(objectKey, content);
}
public String readObjectAsString(String value) {
String objectKey = resolveObjectKey(value);
if (objectKey == null || objectKey.isBlank()) {
return null;
}
OSS ossClient = buildClient();
try (OSSObject ossObject = ossClient.getObject(ossProperties.getBucket(), objectKey)) {
return new String(ossObject.getObjectContent().readAllBytes(), StandardCharsets.UTF_8);
} catch (Exception ex) {
throw new IllegalStateException("failed to read object from oss", ex);
} finally {
ossClient.shutdown();
}
}
public void deleteObject(String value) {
String objectKey = resolveObjectKey(value);
if (objectKey == null || objectKey.isBlank()) {
return;
}
OSS ossClient = buildClient();
try {
ossClient.deleteObject(ossProperties.getBucket(), objectKey);
} finally {
ossClient.shutdown();
}
}
/**
* 根据 objectKey 生成预签名下载 URL1小时有效
*/

View File

@@ -8,7 +8,7 @@ import com.nanri.aiimage.modules.patroldelete.model.vo.PatrolDeleteResultItemVo;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.stereotype.Service;
import java.io.File;
@@ -40,8 +40,9 @@ public class PatrolDeleteExcelAssemblyService {
};
public void writeWorkbook(File outputXlsx, List<PatrolDeleteResultItemVo> items) {
try (XSSFWorkbook workbook = new XSSFWorkbook();
FileOutputStream outputStream = new FileOutputStream(outputXlsx)) {
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
workbook.setCompressTempFiles(true);
try (FileOutputStream outputStream = new FileOutputStream(outputXlsx)) {
Sheet sheet = workbook.createSheet("巡店删除结果");
Row headerRow = sheet.createRow(0);
for (int columnIndex = 0; columnIndex < HEADER.length; columnIndex++) {
@@ -73,13 +74,17 @@ public class PatrolDeleteExcelAssemblyService {
rowIndex++;
}
for (int columnIndex = 0; columnIndex < HEADER.length; columnIndex++) {
sheet.autoSizeColumn(columnIndex);
}
applyDefaultColumnWidths(sheet);
workbook.write(outputStream);
} catch (Exception ex) {
log.warn("[patrol-delete] write workbook failed: {}", ex.getMessage());
throw new BusinessException("生成巡店删除结果 Excel 失败: " + ex.getMessage());
throw new BusinessException("generate patrol-delete excel failed: " + ex.getMessage());
} finally {
try {
workbook.close();
} catch (Exception ignored) {
}
workbook.dispose();
}
}
@@ -141,4 +146,23 @@ public class PatrolDeleteExcelAssemblyService {
private String safe(String value) {
return value == null ? "" : value;
}
private void applyDefaultColumnWidths(Sheet sheet) {
int[] widths = {
20,
14, 10, 12, 16,
14, 10, 12, 16,
14, 10, 12, 16,
14, 10, 12, 16,
14, 10, 12, 16,
10, 16,
10, 16,
10, 16,
10, 16,
10, 16
};
for (int i = 0; i < widths.length; i++) {
sheet.setColumnWidth(i, widths[i] * 256);
}
}
}

View File

@@ -1,10 +1,10 @@
package com.nanri.aiimage.modules.patroldelete.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.patroldelete.model.dto.PatrolDeleteShopPayloadDto;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
import lombok.RequiredArgsConstructor;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
@@ -20,80 +20,40 @@ import java.util.concurrent.ConcurrentHashMap;
@RequiredArgsConstructor
public class PatrolDeleteTaskCacheService {
private static final String MODULE_TYPE = "PATROL_DELETE";
private static final long PAYLOAD_TTL_HOURS = 24;
private final StringRedisTemplate stringRedisTemplate;
private final ObjectMapper objectMapper;
private final TaskPressureProperties taskPressureProperties;
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
public PatrolDeleteShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
return null;
}
Object raw = stringRedisTemplate.opsForHash().get(buildShopPayloadKey(taskId), shopKey);
if (!(raw instanceof String json) || json.isBlank()) {
return null;
}
try {
return objectMapper.readValue(json, PatrolDeleteShopPayloadDto.class);
} catch (Exception ex) {
throw new BusinessException("读取巡店删除店铺缓存失败");
}
return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, PatrolDeleteShopPayloadDto.class);
}
public void saveShopMergedPayload(Long taskId, String shopKey, PatrolDeleteShopPayloadDto payload) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank() || payload == null) {
return;
}
try {
stringRedisTemplate.opsForHash().put(buildShopPayloadKey(taskId), shopKey, objectMapper.writeValueAsString(payload));
stringRedisTemplate.expire(buildShopPayloadKey(taskId), Duration.ofHours(PAYLOAD_TTL_HOURS));
touchTaskHeartbeat(taskId);
} catch (Exception ex) {
throw new BusinessException("暂存巡店删除店铺缓存失败");
}
taskScopePayloadStorageService.saveScopePayload(taskId, MODULE_TYPE, shopKey, payload);
touchTaskHeartbeat(taskId);
}
public void removeShopMergedPayload(Long taskId, String shopKey) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
return;
}
stringRedisTemplate.opsForHash().delete(buildShopPayloadKey(taskId), shopKey);
taskScopePayloadStorageService.removeScopePayload(taskId, MODULE_TYPE, shopKey);
}
public Map<String, PatrolDeleteShopPayloadDto> getAllShopMergedPayload(Long taskId) {
try {
Map<Object, Object> raw = stringRedisTemplate.opsForHash().entries(buildShopPayloadKey(taskId));
if (raw == null || raw.isEmpty()) {
return Map.of();
}
Map<String, PatrolDeleteShopPayloadDto> out = new LinkedHashMap<>();
for (Map.Entry<Object, Object> entry : raw.entrySet()) {
if (!(entry.getKey() instanceof String key) || !(entry.getValue() instanceof String val) || val.isBlank()) {
continue;
}
out.put(key, objectMapper.readValue(val, PatrolDeleteShopPayloadDto.class));
}
return out;
} catch (Exception ex) {
throw new BusinessException("读取巡店删除缓存失败");
}
return taskScopePayloadStorageService.getAllScopePayload(taskId, MODULE_TYPE, PatrolDeleteShopPayloadDto.class);
}
public boolean hasAnyShopMergedPayload(Long taskId) {
if (taskId == null || taskId <= 0) {
return false;
}
Long size = stringRedisTemplate.opsForHash().size(buildShopPayloadKey(taskId));
return size != null && size > 0;
return taskScopePayloadStorageService.hasAnyScopePayload(taskId, MODULE_TYPE);
}
public long countShopMergedPayload(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0L;
}
Long size = stringRedisTemplate.opsForHash().size(buildShopPayloadKey(taskId));
return size == null ? 0L : size;
return taskScopePayloadStorageService.countScopePayload(taskId, MODULE_TYPE);
}
public long getTaskHeartbeatMillis(Long taskId) {
@@ -123,9 +83,9 @@ public class PatrolDeleteTaskCacheService {
return;
}
taskEntityLocalCache.remove(taskId);
stringRedisTemplate.delete(buildShopPayloadKey(taskId));
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
}
public void saveTaskCache(FileTaskEntity task) {
@@ -190,10 +150,6 @@ public class PatrolDeleteTaskCacheService {
return result;
}
private String buildShopPayloadKey(Long taskId) {
return "patrol-delete:task:shop-payload:" + taskId;
}
private String buildTaskHeartbeatKey(Long taskId) {
return "patrol-delete:task:heartbeat:" + taskId;
}

View File

@@ -93,5 +93,14 @@ public class PriceTrackSubmitResultRequest {
@Schema(description = "状态", example = "SUCCESS")
private String status;
@Schema(description = "Python 确认是否需要从 skip ASIN 表删除当前 ASIN", example = "true")
private Boolean deleteSkipAsin;
@Schema(description = "需要删除的 skip ASIN不传时默认使用 asin 字段", example = "B0ABCDE123")
private String removeAsin;
@Schema(description = "删除确认原因,便于排查日志", example = "CURRENT_PRICE_BELOW_MINIMUM")
private String deleteReason;
}
}

View File

@@ -18,6 +18,9 @@ public class PriceTrackCreateTaskVo {
@Schema(description = "all skip asins grouped by country")
private Map<String, List<String>> skipAsinsByCountry;
@Schema(description = "all skip asin details grouped by country code, each item includes asin and minimumPrice")
private Map<String, List<Map<String, String>>> skipAsinDetailsByCountry;
@Schema(description = "parsed asin rows by country")
private Map<String, List<Map<String, String>>> asinRowsByCountry;

View File

@@ -17,6 +17,9 @@ public class PriceTrackMatchShopsVo {
@Schema(description = "All skip ASINs grouped by country code")
private Map<String, List<String>> skipAsinsByCountry;
@Schema(description = "All skip ASIN details grouped by country code, each item includes asin and minimumPrice")
private Map<String, List<Map<String, String>>> skipAsinDetailsByCountry;
@Schema(description = "Parsed asin rows grouped by country code")
private Map<String, List<Map<String, String>>> asinRowsByCountry;

View File

@@ -6,7 +6,7 @@ import com.nanri.aiimage.modules.productrisk.model.enums.ProductRiskCountryCode;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.stereotype.Service;
import java.io.File;
@@ -20,25 +20,27 @@ import java.util.Map;
public class PriceTrackExcelAssemblyService {
private static final String[] RESULT_HEADER = {
"\u5e97\u94fa\u5546\u57ce\u540d\u79f0",
"店铺商城名称",
"ASIN",
"\u4ef7\u683c",
"\u63a8\u8350\u4ef7",
"\u6700\u4f4e\u4ef7",
"\u7b2c\u4e00\u540d",
"\u7b2c\u4e8c\u540d",
"\u8d2d\u7269\u8f66\u5e97\u94fa\u540d",
"\u6539\u4ef7\u60c5\u51b5",
"\u4fee\u6539\u6b21\u6570",
"\u72b6\u6001"
"价格",
"推荐价",
"最低价",
"第一名",
"第二名",
"购物车店铺名",
"改价情况",
"修改次数",
"状态"
};
public void writeWorkbook(File outputXlsx, Map<String, List<PriceTrackSubmitResultRequest.AsinResult>> countries) {
Map<String, List<PriceTrackSubmitResultRequest.AsinResult>> safe = countries == null ? Map.of() : countries;
try (XSSFWorkbook wb = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(outputXlsx)) {
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
workbook.setCompressTempFiles(true);
try (FileOutputStream fos = new FileOutputStream(outputXlsx)) {
for (ProductRiskCountryCode code : ProductRiskCountryCode.values()) {
String sheetName = code.name();
Sheet sheet = wb.createSheet(sheetName);
Sheet sheet = workbook.createSheet(sheetName);
Row headerRow = sheet.createRow(0);
for (int i = 0; i < RESULT_HEADER.length; i++) {
headerRow.createCell(i).setCellValue(RESULT_HEADER[i]);
@@ -62,14 +64,18 @@ public class PriceTrackExcelAssemblyService {
row.createCell(9).setCellValue(valueOf(item.getModifyCount()));
row.createCell(10).setCellValue(valueOf(item.getStatus()));
}
for (int i = 0; i < RESULT_HEADER.length; i++) {
sheet.autoSizeColumn(i);
}
applyDefaultColumnWidths(sheet);
}
wb.write(fos);
workbook.write(fos);
} catch (Exception ex) {
log.warn("[price-track] write workbook failed: {}", ex.getMessage());
throw new BusinessException("鐢熸垚 Excel 澶辫触: " + ex.getMessage());
throw new BusinessException("generate price-track excel failed: " + ex.getMessage());
} finally {
try {
workbook.close();
} catch (Exception ignored) {
}
workbook.dispose();
}
}
@@ -105,4 +111,10 @@ public class PriceTrackExcelAssemblyService {
return value == null ? "" : value;
}
private void applyDefaultColumnWidths(Sheet sheet) {
int[] widths = {22, 18, 12, 12, 12, 20, 20, 22, 18, 12, 12};
for (int i = 0; i < widths.length; i++) {
sheet.setColumnWidth(i, widths[i] * 256);
}
}
}

View File

@@ -148,6 +148,9 @@ public class PriceTrackService {
Map<String, List<String>> skipAsinsByCountry = asinMode
? new LinkedHashMap<>()
: skipPriceAsinService.listAllSkipAsinsByCountry();
Map<String, List<Map<String, String>>> skipAsinDetailsByCountry = asinMode
? new LinkedHashMap<>()
: skipPriceAsinService.listAllSkipAsinDetailsByCountry();
Map<String, List<Map<String, String>>> asinRowsByCountry =
!asinMode
? new LinkedHashMap<>()
@@ -157,6 +160,7 @@ public class PriceTrackService {
PriceTrackMatchShopsVo vo = new PriceTrackMatchShopsVo();
vo.setSkipAsinsByCountry(skipAsinsByCountry);
vo.setSkipAsinDetailsByCountry(skipAsinDetailsByCountry);
vo.setAsinRowsByCountry(asinRowsByCountry);
vo.setMinimumPriceByCountryAndAsin(priceTrackTaskService.buildMinimumPriceLookupForMatch(asinRowsByCountry));
for (String shopName : ordered) {

View File

@@ -1,10 +1,10 @@
package com.nanri.aiimage.modules.pricetrack.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackSubmitResultRequest;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
import lombok.RequiredArgsConstructor;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
@@ -20,10 +20,12 @@ import java.util.concurrent.ConcurrentHashMap;
@RequiredArgsConstructor
public class PriceTrackTaskCacheService {
private static final String MODULE_TYPE = "PRICE_TRACK";
private static final long HEARTBEAT_TTL_HOURS = 24;
private final StringRedisTemplate stringRedisTemplate;
private final ObjectMapper objectMapper;
private final TaskPressureProperties taskPressureProperties;
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
public void touchTaskHeartbeat(Long taskId) {
@@ -52,65 +54,27 @@ public class PriceTrackTaskCacheService {
}
public PriceTrackSubmitResultRequest.ShopResult getShopMergedPayload(Long taskId, String shopKey) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
return null;
}
Object raw = stringRedisTemplate.opsForHash().get(buildShopPayloadKey(taskId), shopKey);
if (!(raw instanceof String json) || json.isBlank()) {
return null;
}
try {
return objectMapper.readValue(json, PriceTrackSubmitResultRequest.ShopResult.class);
} catch (Exception ex) {
throw new BusinessException("read price-track cache failed");
}
return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, PriceTrackSubmitResultRequest.ShopResult.class);
}
public void saveShopMergedPayload(Long taskId, String shopKey, PriceTrackSubmitResultRequest.ShopResult payload) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank() || payload == null) {
return;
}
try {
stringRedisTemplate.opsForHash().put(buildShopPayloadKey(taskId), shopKey, objectMapper.writeValueAsString(payload));
stringRedisTemplate.expire(buildShopPayloadKey(taskId), Duration.ofHours(HEARTBEAT_TTL_HOURS));
touchTaskHeartbeat(taskId);
} catch (Exception ex) {
throw new BusinessException("save price-track cache failed");
}
taskScopePayloadStorageService.saveScopePayload(taskId, MODULE_TYPE, shopKey, payload);
touchTaskHeartbeat(taskId);
}
public void removeShopMergedPayload(Long taskId, String shopKey) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
return;
}
stringRedisTemplate.opsForHash().delete(buildShopPayloadKey(taskId), shopKey);
taskScopePayloadStorageService.removeScopePayload(taskId, MODULE_TYPE, shopKey);
}
public Map<String, PriceTrackSubmitResultRequest.ShopResult> getAllShopMergedPayload(Long taskId) {
try {
Map<Object, Object> raw = stringRedisTemplate.opsForHash().entries(buildShopPayloadKey(taskId));
if (raw == null || raw.isEmpty()) {
return Map.of();
}
LinkedHashMap<String, PriceTrackSubmitResultRequest.ShopResult> out = new LinkedHashMap<>();
for (Map.Entry<Object, Object> entry : raw.entrySet()) {
if (!(entry.getKey() instanceof String key) || !(entry.getValue() instanceof String val) || val.isBlank()) {
continue;
}
out.put(key, objectMapper.readValue(val, PriceTrackSubmitResultRequest.ShopResult.class));
}
return out;
} catch (Exception ex) {
throw new BusinessException("read price-track cache failed");
}
return taskScopePayloadStorageService.getAllScopePayload(taskId, MODULE_TYPE, PriceTrackSubmitResultRequest.ShopResult.class);
}
public boolean hasAnyShopMergedPayload(Long taskId) {
if (taskId == null || taskId <= 0) {
return false;
}
Long size = stringRedisTemplate.opsForHash().size(buildShopPayloadKey(taskId));
return size != null && size > 0;
return taskScopePayloadStorageService.hasAnyScopePayload(taskId, MODULE_TYPE);
}
public void deleteTaskCache(Long taskId) {
@@ -118,9 +82,9 @@ public class PriceTrackTaskCacheService {
return;
}
taskEntityLocalCache.remove(taskId);
stringRedisTemplate.delete(buildShopPayloadKey(taskId));
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
}
public void saveTaskCache(FileTaskEntity task) {
@@ -185,10 +149,6 @@ public class PriceTrackTaskCacheService {
return result;
}
private String buildShopPayloadKey(Long taskId) {
return "price-track:task:shop-payload:" + taskId;
}
private String buildTaskHeartbeatKey(Long taskId) {
return "price-track:task:heartbeat:" + taskId;
}

View File

@@ -206,7 +206,7 @@ public class PriceTrackTaskService {
@Transactional
public PriceTrackPendingDeleteVo deletePendingShopResult(Long userId, String shopName) {
if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法");
if (shopName == null || shopName.isBlank()) throw new BusinessException("shop_name 涓嶈兘涓虹┖");
if (shopName == null || shopName.isBlank()) throw new BusinessException("shop_name 不能为空");
String norm = ziniaoShopSwitchService.normalizeShopName(shopName);
if (norm.isBlank()) throw new BusinessException("店铺名称无效");
@@ -293,6 +293,9 @@ public class PriceTrackTaskService {
Map<String, List<String>> skipAsinsByCountry = request.isAsinMode()
? new LinkedHashMap<>()
: skipPriceAsinService.listAllSkipAsinsByCountry();
Map<String, List<Map<String, String>>> skipAsinDetailsByCountry = request.isAsinMode()
? new LinkedHashMap<>()
: skipPriceAsinService.listAllSkipAsinDetailsByCountry();
Map<String, List<Map<String, String>>> asinRowsByCountry = request.isAsinMode()
? parseAsinRowsByCountry(request.getAsinFiles(), request.getCountryCodes())
: new LinkedHashMap<>();
@@ -348,6 +351,7 @@ public class PriceTrackTaskService {
ctx.put("asinMode", request.isAsinMode());
ctx.put("countryCodes", request.getCountryCodes());
ctx.put("skipAsinsByCountry", skipAsinsByCountry);
ctx.put("skipAsinDetailsByCountry", skipAsinDetailsByCountry);
ctx.put("asinRowsByCountry", asinRowsByCountry);
ctx.put("minimumPriceByCountryAndAsin", minimumPriceByCountryAndAsin);
ctx.put("items", uniqueItems);
@@ -368,6 +372,7 @@ public class PriceTrackTaskService {
vo.setTaskId(task.getId());
vo.setItems(snapshot);
vo.setSkipAsinsByCountry(skipAsinsByCountry);
vo.setSkipAsinDetailsByCountry(skipAsinDetailsByCountry);
vo.setAsinRowsByCountry(asinRowsByCountry);
vo.setMinimumPriceByCountryAndAsin(minimumPriceByCountryAndAsin);
return vo;
@@ -419,6 +424,7 @@ public class PriceTrackTaskService {
priceTrackTaskCacheService.removeShopMergedPayload(taskId, shopKey);
continue;
}
handleSkipAsinDeletionSignals(shopKey, payload);
PriceTrackSubmitResultRequest.ShopResult merged = mergeShopPayload(taskId, shopKey, payload);
if (!Boolean.TRUE.equals(merged.getSuccess())) {
continue;
@@ -833,8 +839,8 @@ public class PriceTrackTaskService {
if (value == null) {
return "";
}
return value.replace("\ufeff", "")
.replace("\u3000", " ")
return value.replace("", "")
.replace(" ", " ")
.replace("\r", " ")
.replace("\n", " ")
.trim()
@@ -1075,6 +1081,11 @@ public class PriceTrackTaskService {
merged.setPriceChangeStatus(firstNonBlank(incoming.getPriceChangeStatus(), merged.getPriceChangeStatus()));
merged.setModifyCount(firstNonBlank(incoming.getModifyCount(), merged.getModifyCount()));
merged.setStatus(firstNonBlank(incoming.getStatus(), merged.getStatus()));
if (incoming.getDeleteSkipAsin() != null) {
merged.setDeleteSkipAsin(incoming.getDeleteSkipAsin());
}
merged.setRemoveAsin(firstNonBlank(incoming.getRemoveAsin(), merged.getRemoveAsin()));
merged.setDeleteReason(firstNonBlank(incoming.getDeleteReason(), merged.getDeleteReason()));
return merged;
}
private PriceTrackSubmitResultRequest.AsinResult cloneRow(PriceTrackSubmitResultRequest.AsinResult row) {
@@ -1093,8 +1104,36 @@ public class PriceTrackTaskService {
out.setPriceChangeStatus(row.getPriceChangeStatus());
out.setModifyCount(row.getModifyCount());
out.setStatus(row.getStatus());
out.setDeleteSkipAsin(row.getDeleteSkipAsin());
out.setRemoveAsin(row.getRemoveAsin());
out.setDeleteReason(row.getDeleteReason());
return out;
}
private void handleSkipAsinDeletionSignals(String shopKey,
PriceTrackSubmitResultRequest.ShopResult payload) {
if (payload == null || payload.getCountries() == null || payload.getCountries().isEmpty()) {
return;
}
for (Map.Entry<String, List<PriceTrackSubmitResultRequest.AsinResult>> entry : payload.getCountries().entrySet()) {
String countryCode = entry.getKey();
List<PriceTrackSubmitResultRequest.AsinResult> rows = entry.getValue();
if (countryCode == null || countryCode.isBlank() || rows == null || rows.isEmpty()) {
continue;
}
for (PriceTrackSubmitResultRequest.AsinResult row : rows) {
if (row == null || !Boolean.TRUE.equals(row.getDeleteSkipAsin())) {
continue;
}
String targetAsin = firstNonBlank(row.getRemoveAsin(), row.getAsin());
if (targetAsin == null || targetAsin.isBlank()) {
continue;
}
boolean removed = skipPriceAsinService.removeByShopCountryAndAsin(shopKey, countryCode, targetAsin);
log.info("[price-track] skip asin delete signal taskShop={} country={} asin={} removed={} reason={}",
shopKey, countryCode, targetAsin, removed, row.getDeleteReason());
}
}
}
private int countPayloadRows(PriceTrackSubmitResultRequest.ShopResult payload) {
if (payload == null) {
return 0;

View File

@@ -7,7 +7,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.stereotype.Service;
import java.io.File;
@@ -35,9 +35,11 @@ public class ProductRiskExcelAssemblyService {
public void writeWorkbook(File outputXlsx, String shopDisplayName, Map<String, List<ProductRiskRowDto>> countries) {
Map<String, List<ProductRiskRowDto>> safe = countries == null ? Map.of() : countries;
try (XSSFWorkbook wb = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(outputXlsx)) {
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
workbook.setCompressTempFiles(true);
try (FileOutputStream fos = new FileOutputStream(outputXlsx)) {
for (String sheetName : SHEET_ORDER) {
Sheet sheet = wb.createSheet(sheetName);
Sheet sheet = workbook.createSheet(sheetName);
Row headerRow = sheet.createRow(0);
for (int c = 0; c < HEADER.length; c++) {
headerRow.createCell(c).setCellValue(HEADER[c]);
@@ -53,16 +55,20 @@ public class ProductRiskExcelAssemblyService {
createTextCell(row, 4, dto == null ? null : dto.getRemoveAsin());
createTextCell(row, 5, dto == null ? null : dto.getRemoveStatus());
}
for (int i = 0; i < HEADER.length; i++) {
sheet.autoSizeColumn(i);
}
applyDefaultColumnWidths(sheet);
}
wb.write(fos);
workbook.write(fos);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
log.warn("[product-risk] write workbook failed: {}", ex.getMessage());
throw new BusinessException("生成 Excel 失败: " + ex.getMessage());
throw new BusinessException("generate product-risk excel failed: " + ex.getMessage());
} finally {
try {
workbook.close();
} catch (Exception ignored) {
}
workbook.dispose();
}
}
@@ -79,9 +85,6 @@ public class ProductRiskExcelAssemblyService {
return n;
}
/**
* 将接口中的 {@link ProductRiskCountryCode} 键转为 Excel 中文工作表名 → 行列表。
*/
public Map<String, List<ProductRiskRowDto>> normalizeCountriesMap(Map<ProductRiskCountryCode, List<ProductRiskRowDto>> raw) {
if (raw == null || raw.isEmpty()) {
return new LinkedHashMap<>();
@@ -107,4 +110,11 @@ public class ProductRiskExcelAssemblyService {
}
return fallback == null ? "" : fallback;
}
private void applyDefaultColumnWidths(Sheet sheet) {
int[] widths = {20, 22, 18, 10, 18, 18};
for (int i = 0; i < widths.length; i++) {
sheet.setColumnWidth(i, widths[i] * 256);
}
}
}

View File

@@ -34,7 +34,7 @@ import java.util.Objects;
public class ProductRiskResolveService {
/**
* 默认处理顺序:德国 英国 法国 意大利 西班牙与前端一致
* 默认国家偏好顺序:德国 -> 英国 -> 法国 -> 意大利 -> 西班牙与前端保持一致。
*/
public static final List<String> DEFAULT_COUNTRY_PREFERENCE_ORDER = List.of("DE", "UK", "FR", "IT", "ES");
@@ -78,7 +78,7 @@ public class ProductRiskResolveService {
if (indexHit == null || !indexHit.isMatched()) {
String hint = indexHit != null && indexHit.getMatchMessage() != null && !indexHit.getMatchMessage().isBlank()
? indexHit.getMatchMessage()
: "店铺索引未命中,无法加入备选";
: "店铺索引未命中,无法加入待处理列表";
throw new BusinessException(hint);
}
if (ZiniaoShopIndexService.MATCH_STATUS_CONFLICT.equals(indexHit.getMatchStatus())) {
@@ -135,7 +135,7 @@ public class ProductRiskResolveService {
}
}
if (ordered.isEmpty()) {
throw new BusinessException("shop_names 有效店铺名");
throw new BusinessException("shop_names 没有有效店铺名");
}
ProductRiskMatchShopsVo vo = new ProductRiskMatchShopsVo();
for (String shopName : ordered) {
@@ -227,7 +227,7 @@ public class ProductRiskResolveService {
}
/**
* 仅保留合法枚举名,去重并保持顺序;非法静默丢弃(用于读取库数据
* 仅保留合法国家代码,去重并保持顺序;非法静默忽略,用于读取库存量数据。
*/
private static List<String> parseValidCountryCodes(List<String> raw) {
if (raw == null || raw.isEmpty()) {

View File

@@ -1,10 +1,10 @@
package com.nanri.aiimage.modules.productrisk.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskShopPayloadDto;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
import lombok.RequiredArgsConstructor;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
@@ -20,80 +20,40 @@ import java.util.concurrent.ConcurrentHashMap;
@RequiredArgsConstructor
public class ProductRiskTaskCacheService {
private static final String MODULE_TYPE = "PRODUCT_RISK_RESOLVE";
private static final long PAYLOAD_TTL_HOURS = 24;
private final StringRedisTemplate stringRedisTemplate;
private final ObjectMapper objectMapper;
private final TaskPressureProperties taskPressureProperties;
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
public ProductRiskShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
return null;
}
Object raw = stringRedisTemplate.opsForHash().get(buildShopPayloadKey(taskId), shopKey);
if (!(raw instanceof String json) || json.isBlank()) {
return null;
}
try {
return objectMapper.readValue(json, ProductRiskShopPayloadDto.class);
} catch (Exception ex) {
throw new BusinessException("读取商品风险店铺缓存失败");
}
return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, ProductRiskShopPayloadDto.class);
}
public void saveShopMergedPayload(Long taskId, String shopKey, ProductRiskShopPayloadDto payload) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank() || payload == null) {
return;
}
try {
stringRedisTemplate.opsForHash().put(buildShopPayloadKey(taskId), shopKey, objectMapper.writeValueAsString(payload));
stringRedisTemplate.expire(buildShopPayloadKey(taskId), Duration.ofHours(PAYLOAD_TTL_HOURS));
touchTaskHeartbeat(taskId);
} catch (Exception ex) {
throw new BusinessException("暂存商品风险店铺缓存失败");
}
taskScopePayloadStorageService.saveScopePayload(taskId, MODULE_TYPE, shopKey, payload);
touchTaskHeartbeat(taskId);
}
public void removeShopMergedPayload(Long taskId, String shopKey) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
return;
}
stringRedisTemplate.opsForHash().delete(buildShopPayloadKey(taskId), shopKey);
taskScopePayloadStorageService.removeScopePayload(taskId, MODULE_TYPE, shopKey);
}
public Map<String, ProductRiskShopPayloadDto> getAllShopMergedPayload(Long taskId) {
try {
Map<Object, Object> raw = stringRedisTemplate.opsForHash().entries(buildShopPayloadKey(taskId));
if (raw == null || raw.isEmpty()) {
return Map.of();
}
java.util.LinkedHashMap<String, ProductRiskShopPayloadDto> out = new java.util.LinkedHashMap<>();
for (Map.Entry<Object, Object> entry : raw.entrySet()) {
if (!(entry.getKey() instanceof String key) || !(entry.getValue() instanceof String val) || val.isBlank()) {
continue;
}
out.put(key, objectMapper.readValue(val, ProductRiskShopPayloadDto.class));
}
return out;
} catch (Exception ex) {
throw new BusinessException("读取商品风险缓存失败");
}
return taskScopePayloadStorageService.getAllScopePayload(taskId, MODULE_TYPE, ProductRiskShopPayloadDto.class);
}
public boolean hasAnyShopMergedPayload(Long taskId) {
if (taskId == null || taskId <= 0) {
return false;
}
Long size = stringRedisTemplate.opsForHash().size(buildShopPayloadKey(taskId));
return size != null && size > 0;
return taskScopePayloadStorageService.hasAnyScopePayload(taskId, MODULE_TYPE);
}
public long countShopMergedPayload(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0L;
}
Long size = stringRedisTemplate.opsForHash().size(buildShopPayloadKey(taskId));
return size == null ? 0L : size;
return taskScopePayloadStorageService.countScopePayload(taskId, MODULE_TYPE);
}
public long getTaskHeartbeatMillis(Long taskId) {
@@ -116,9 +76,9 @@ public class ProductRiskTaskCacheService {
return;
}
taskEntityLocalCache.remove(taskId);
stringRedisTemplate.delete(buildShopPayloadKey(taskId));
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
}
public void saveTaskCache(FileTaskEntity task) {
@@ -183,10 +143,6 @@ public class ProductRiskTaskCacheService {
return result;
}
private String buildShopPayloadKey(Long taskId) {
return "product-risk:task:shop-payload:" + taskId;
}
public void touchTaskHeartbeat(Long taskId) {
stringRedisTemplate.opsForValue().set(
buildTaskHeartbeatKey(taskId),

View File

@@ -189,7 +189,7 @@ public class ProductRiskTaskService {
}
/**
* 删除整条商品风险任务及其下所有店铺结果运行中已结束均可,需归属当前用户
* 删除整条商品风险任务及其下所有店铺结果运行中已结束任务都允许删除,但必须归属当前用户。
*/
@Transactional
public void deleteTask(Long taskId, Long userId) {
@@ -208,8 +208,8 @@ public class ProductRiskTaskService {
}
/**
* 从用户当前运行中的商品风险任务里,按规范化店铺名删除一条结果用于前端移除匹配列表后同步后端
* 若无对应记录则 removed=false。
* 从用户当前运行中的商品风险任务里,按规范化店铺名删除一条结果用于前端移除匹配列表后同步后端。
* 如果没有对应记录,则返回 removed=false。
*/
@Transactional
public ProductRiskPendingDeleteVo deletePendingShopResult(Long userId, String shopName) {
@@ -252,7 +252,7 @@ public class ProductRiskTaskService {
}
/**
* 删除一条 file_result 后重算父任务状态;若无剩余结果则删除任务。
* 删除一条 file_result 后重算父任务状态;如果没有剩余结果则删除任务。
*/
private void reconcileTaskAfterResultRemoval(Long taskId) {
FileTaskEntity task = loadTaskForExecution(taskId);
@@ -303,11 +303,11 @@ public class ProductRiskTaskService {
task.setFinishedAt(LocalDateTime.now());
} else if (ok > 0) {
task.setStatus("SUCCESS");
task.setErrorMessage(String.join("", allErrors));
task.setErrorMessage(String.join("; ", allErrors));
task.setFinishedAt(LocalDateTime.now());
} else {
task.setStatus("FAILED");
task.setErrorMessage(allErrors.isEmpty() ? "全部店铺处理失败" : String.join("", allErrors));
task.setErrorMessage(allErrors.isEmpty() ? "全部店铺处理失败" : String.join("; ", allErrors));
task.setFinishedAt(LocalDateTime.now());
}
try {
@@ -507,7 +507,7 @@ public class ProductRiskTaskService {
String shopKey = fr.getSourceFilename();
ProductRiskShopPayloadDto payload = shopKey == null ? null : payloadByShop.get(shopKey);
// 兼容单店任务:规范化店名偶发不一致(空格/符号差异)导致未命中,且任务与回传都只有 1 个店时按唯一店回填。
// 兼容单店任务:如果规范化店名偶发不一致导致未命中,且任务与回传都只有 1 个店时按唯一店回填。
if (payload == null && resultRows.size() == 1 && payloadByShop.size() == 1) {
payload = payloadByShop.values().iterator().next();
fallbackMatchedCount++;
@@ -517,7 +517,7 @@ public class ProductRiskTaskService {
if (payload == null) {
skippedUnmatchedCount++;
// 与删除品牌一致:支持队列逐条处理、分次回传,未在本次 payload 的店铺保持待处理
// 与删除品牌一致:支持队列逐条处理、分次回传,本次 payload 未包含的店铺保持待处理
continue;
}
matchedShopCount++;
@@ -602,11 +602,11 @@ public class ProductRiskTaskService {
task.setFinishedAt(LocalDateTime.now());
} else if (ok > 0) {
task.setStatus("SUCCESS");
task.setErrorMessage(String.join("", allErrors.isEmpty() ? batchErrors : allErrors));
task.setErrorMessage(String.join("; ", allErrors.isEmpty() ? batchErrors : allErrors));
task.setFinishedAt(LocalDateTime.now());
} else {
task.setStatus("FAILED");
task.setErrorMessage(allErrors.isEmpty() ? "全部店铺处理失败" : String.join("", allErrors));
task.setErrorMessage(allErrors.isEmpty() ? "全部店铺处理失败" : String.join("; ", allErrors));
task.setFinishedAt(LocalDateTime.now());
}
try {
@@ -1034,7 +1034,7 @@ public class ProductRiskTaskService {
continue;
}
// 兼容 Python 末次“空行 done=true”心跳视为当前国家已有行全部完成。
// 兼容 Python 末次“空行 + done=true”心跳视为当前国家已有行全部完成。
if (isDoneValue(row.getDone()) && isEmptyBusinessRow(row) && !byKey.isEmpty()) {
for (Map.Entry<String, ProductRiskRowDto> entry : byKey.entrySet()) {
ProductRiskRowDto existed = entry.getValue();
@@ -1103,8 +1103,8 @@ public class ProductRiskTaskService {
if (row == null) {
return true;
}
// 注意shopName 只是上下文信息,不作为业务行是否为空的判定条件。
// Python 末次心跳行通常只带 shopName + done=true业务字段为空。
// 注意shopName 只是上下文信息,不作为业务行是否为空的判定条件。
// Python 末次心跳行通常只带 shopName + done=true业务字段为空。
return isBlank(row.getProductAsinSku())
&& isBlank(row.getRemoveAsin())
&& isBlank(row.getStatus())

View File

@@ -71,6 +71,7 @@ public class SkipPriceAsinController {
id,
country,
request.getAsin(),
request.getMinimumPrice(),
operatorId,
Boolean.TRUE.equals(superAdmin)));
}

View File

@@ -4,10 +4,15 @@ import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotBlank;
import lombok.Data;
import java.math.BigDecimal;
@Data
public class SkipPriceAsinCountryUpdateRequest {
@Schema(description = "ASIN", requiredMode = Schema.RequiredMode.REQUIRED)
@NotBlank(message = "ASIN 不能为空")
private String asin;
@Schema(description = "最低价", example = "18.50")
private BigDecimal minimumPrice;
}

View File

@@ -5,6 +5,7 @@ import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
import java.math.BigDecimal;
import java.util.List;
import java.util.Map;
@@ -29,4 +30,8 @@ public class SkipPriceAsinCreateRequest {
* 新请求: 每个国家对应独立 ASIN例如 {"DE":"B0...", "UK":"B0..."}。
*/
private Map<String, String> asinMappings;
private BigDecimal minimumPrice;
private Map<String, BigDecimal> minimumPriceMappings;
}

View File

@@ -6,6 +6,7 @@ import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.math.BigDecimal;
import java.time.LocalDateTime;
@Data
@@ -24,18 +25,33 @@ public class SkipPriceAsinEntity {
@TableField("asin_de")
private String asinDe;
@TableField("minimum_price_de")
private BigDecimal minimumPriceDe;
@TableField("asin_uk")
private String asinUk;
@TableField("minimum_price_uk")
private BigDecimal minimumPriceUk;
@TableField("asin_fr")
private String asinFr;
@TableField("minimum_price_fr")
private BigDecimal minimumPriceFr;
@TableField("asin_it")
private String asinIt;
@TableField("minimum_price_it")
private BigDecimal minimumPriceIt;
@TableField("asin_es")
private String asinEs;
@TableField("minimum_price_es")
private BigDecimal minimumPriceEs;
@TableField("created_at")
private LocalDateTime createdAt;

View File

@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.shopkey.model.vo;
import lombok.Data;
import java.math.BigDecimal;
import java.time.LocalDateTime;
@Data
@@ -12,10 +13,15 @@ public class SkipPriceAsinItemVo {
private String groupName;
private String shopName;
private String asinDe;
private BigDecimal minimumPriceDe;
private String asinUk;
private BigDecimal minimumPriceUk;
private String asinFr;
private BigDecimal minimumPriceFr;
private String asinIt;
private BigDecimal minimumPriceIt;
private String asinEs;
private BigDecimal minimumPriceEs;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}

View File

@@ -12,6 +12,9 @@ import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.time.LocalDateTime;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
@@ -79,6 +82,7 @@ public class SkipPriceAsinService {
String shopName = normalizeRequired(request.getShopName(), "店铺名不能为空");
Set<String> countries = normalizeCountries(request.getCountries());
Map<String, String> countryAsinMap = normalizeCountryAsinMap(countries, request);
Map<String, BigDecimal> countryMinimumPriceMap = normalizeCountryMinimumPriceMap(countries, request);
SkipPriceAsinEntity entity = skipPriceAsinMapper.selectOne(new LambdaQueryWrapper<SkipPriceAsinEntity>()
.eq(SkipPriceAsinEntity::getGroupId, group.getId())
@@ -91,7 +95,8 @@ public class SkipPriceAsinService {
}
for (Map.Entry<String, String> entry : countryAsinMap.entrySet()) {
setCountryAsin(entity, entry.getKey(), entry.getValue());
String country = entry.getKey();
setCountryData(entity, country, entry.getValue(), countryMinimumPriceMap.get(country));
}
if (entity.getId() == null) {
@@ -108,7 +113,7 @@ public class SkipPriceAsinService {
SkipPriceAsinEntity entity = getById(id);
validateGroupAccess(entity, operatorId, superAdmin);
String normalizedCountry = normalizeCountry(country);
setCountryAsin(entity, normalizedCountry, "");
setCountryData(entity, normalizedCountry, "", null);
if (isEmptyRow(entity)) {
skipPriceAsinMapper.deleteById(entity.getId());
return;
@@ -117,12 +122,13 @@ public class SkipPriceAsinService {
}
@Transactional
public SkipPriceAsinItemVo updateCountry(Long id, String country, String asin, Long operatorId, boolean superAdmin) {
public SkipPriceAsinItemVo updateCountry(Long id, String country, String asin, BigDecimal minimumPrice,
Long operatorId, boolean superAdmin) {
SkipPriceAsinEntity entity = getById(id);
validateGroupAccess(entity, operatorId, superAdmin);
String normalizedCountry = normalizeCountry(country);
String normalizedAsin = normalizeAsin(asin);
setCountryAsin(entity, normalizedCountry, normalizedAsin);
setCountryData(entity, normalizedCountry, normalizedAsin, normalizeMinimumPrice(minimumPrice));
skipPriceAsinMapper.updateById(entity);
SkipPriceAsinEntity saved = getById(entity.getId());
String groupName = "";
@@ -154,10 +160,15 @@ public class SkipPriceAsinService {
vo.setGroupName(groupName == null ? "" : groupName);
vo.setShopName(entity.getShopName());
vo.setAsinDe(blankToEmpty(entity.getAsinDe()));
vo.setMinimumPriceDe(entity.getMinimumPriceDe());
vo.setAsinUk(blankToEmpty(entity.getAsinUk()));
vo.setMinimumPriceUk(entity.getMinimumPriceUk());
vo.setAsinFr(blankToEmpty(entity.getAsinFr()));
vo.setMinimumPriceFr(entity.getMinimumPriceFr());
vo.setAsinIt(blankToEmpty(entity.getAsinIt()));
vo.setMinimumPriceIt(entity.getMinimumPriceIt());
vo.setAsinEs(blankToEmpty(entity.getAsinEs()));
vo.setMinimumPriceEs(entity.getMinimumPriceEs());
vo.setCreatedAt(entity.getCreatedAt());
vo.setUpdatedAt(entity.getUpdatedAt());
return vo;
@@ -213,6 +224,27 @@ public class SkipPriceAsinService {
return normalized;
}
private Map<String, BigDecimal> normalizeCountryMinimumPriceMap(Set<String> countries, SkipPriceAsinCreateRequest request) {
LinkedHashMap<String, BigDecimal> normalized = new LinkedHashMap<>();
Map<String, BigDecimal> requestMappings = request.getMinimumPriceMappings();
if (requestMappings != null && !requestMappings.isEmpty()) {
for (String country : countries) {
BigDecimal minimumPrice = requestMappings.get(country);
if (minimumPrice == null) {
minimumPrice = requestMappings.get(country.toLowerCase(Locale.ROOT));
}
normalized.put(country, normalizeMinimumPrice(minimumPrice));
}
return normalized;
}
BigDecimal minimumPrice = normalizeMinimumPrice(request.getMinimumPrice());
for (String country : countries) {
normalized.put(country, minimumPrice);
}
return normalized;
}
private String normalizeCountry(String country) {
String normalized = normalizeBlank(country).toUpperCase(Locale.ROOT);
if (!SUPPORTED_COUNTRIES.contains(normalized)) {
@@ -229,6 +261,16 @@ public class SkipPriceAsinService {
return normalized;
}
private BigDecimal normalizeMinimumPrice(BigDecimal minimumPrice) {
if (minimumPrice == null) {
return null;
}
if (minimumPrice.compareTo(BigDecimal.ZERO) < 0) {
throw new BusinessException("最低价不能小于 0");
}
return minimumPrice.setScale(2, RoundingMode.HALF_UP);
}
private String normalizeRequired(String value, String message) {
String normalized = normalizeBlank(value);
if (normalized.isEmpty()) {
@@ -245,14 +287,29 @@ public class SkipPriceAsinService {
return value == null ? "" : value;
}
private void setCountryAsin(SkipPriceAsinEntity entity, String country, String asin) {
private void setCountryData(SkipPriceAsinEntity entity, String country, String asin, BigDecimal minimumPrice) {
String value = normalizeBlank(asin);
switch (country) {
case "DE" -> entity.setAsinDe(value);
case "UK" -> entity.setAsinUk(value);
case "FR" -> entity.setAsinFr(value);
case "IT" -> entity.setAsinIt(value);
case "ES" -> entity.setAsinEs(value);
case "DE" -> {
entity.setAsinDe(value);
entity.setMinimumPriceDe(minimumPrice);
}
case "UK" -> {
entity.setAsinUk(value);
entity.setMinimumPriceUk(minimumPrice);
}
case "FR" -> {
entity.setAsinFr(value);
entity.setMinimumPriceFr(minimumPrice);
}
case "IT" -> {
entity.setAsinIt(value);
entity.setMinimumPriceIt(minimumPrice);
}
case "ES" -> {
entity.setAsinEs(value);
entity.setMinimumPriceEs(minimumPrice);
}
default -> throw new BusinessException("国家参数不支持: " + country);
}
}
@@ -262,7 +319,12 @@ public class SkipPriceAsinService {
&& normalizeBlank(entity.getAsinUk()).isEmpty()
&& normalizeBlank(entity.getAsinFr()).isEmpty()
&& normalizeBlank(entity.getAsinIt()).isEmpty()
&& normalizeBlank(entity.getAsinEs()).isEmpty();
&& normalizeBlank(entity.getAsinEs()).isEmpty()
&& entity.getMinimumPriceDe() == null
&& entity.getMinimumPriceUk() == null
&& entity.getMinimumPriceFr() == null
&& entity.getMinimumPriceIt() == null
&& entity.getMinimumPriceEs() == null;
}
public Map<String, Map<String, String>> listSkipAsinsByShops(List<String> shopNames) {
@@ -312,10 +374,78 @@ public class SkipPriceAsinService {
return result;
}
public Map<String, List<Map<String, String>>> listAllSkipAsinDetailsByCountry() {
List<SkipPriceAsinEntity> rows = skipPriceAsinMapper.selectList(
new LambdaQueryWrapper<SkipPriceAsinEntity>()
.orderByDesc(SkipPriceAsinEntity::getId));
Map<String, List<Map<String, String>>> grouped = new LinkedHashMap<>();
grouped.put("DE", new java.util.ArrayList<>());
grouped.put("UK", new java.util.ArrayList<>());
grouped.put("FR", new java.util.ArrayList<>());
grouped.put("IT", new java.util.ArrayList<>());
grouped.put("ES", new java.util.ArrayList<>());
for (SkipPriceAsinEntity row : rows) {
addCountryAsinDetail(grouped.get("DE"), row.getAsinDe(), row.getMinimumPriceDe());
addCountryAsinDetail(grouped.get("UK"), row.getAsinUk(), row.getMinimumPriceUk());
addCountryAsinDetail(grouped.get("FR"), row.getAsinFr(), row.getMinimumPriceFr());
addCountryAsinDetail(grouped.get("IT"), row.getAsinIt(), row.getMinimumPriceIt());
addCountryAsinDetail(grouped.get("ES"), row.getAsinEs(), row.getMinimumPriceEs());
}
return grouped;
}
@Transactional
public boolean removeByShopCountryAndAsin(String shopName, String country, String asin) {
String normalizedShopName = normalizeBlank(shopName);
String normalizedCountry = normalizeCountry(country);
String normalizedAsin = normalizeAsin(asin);
if (normalizedShopName.isEmpty() || normalizedAsin.isEmpty()) {
return false;
}
List<SkipPriceAsinEntity> rows = skipPriceAsinMapper.selectList(
new LambdaQueryWrapper<SkipPriceAsinEntity>()
.eq(SkipPriceAsinEntity::getShopName, normalizedShopName));
boolean changed = false;
for (SkipPriceAsinEntity row : rows) {
String existingAsin = switch (normalizedCountry) {
case "DE" -> normalizeBlank(row.getAsinDe());
case "UK" -> normalizeBlank(row.getAsinUk());
case "FR" -> normalizeBlank(row.getAsinFr());
case "IT" -> normalizeBlank(row.getAsinIt());
case "ES" -> normalizeBlank(row.getAsinEs());
default -> "";
};
if (!normalizedAsin.equals(existingAsin)) {
continue;
}
setCountryData(row, normalizedCountry, "", null);
row.setUpdatedAt(LocalDateTime.now());
if (isEmptyRow(row)) {
skipPriceAsinMapper.deleteById(row.getId());
} else {
skipPriceAsinMapper.updateById(row);
}
changed = true;
}
return changed;
}
private void addCountryAsin(Set<String> bucket, String asin) {
String normalized = normalizeBlank(asin);
if (!normalized.isEmpty()) {
bucket.add(normalized);
}
}
private void addCountryAsinDetail(List<Map<String, String>> bucket, String asin, BigDecimal minimumPrice) {
String normalizedAsin = normalizeBlank(asin);
if (normalizedAsin.isEmpty()) {
return;
}
Map<String, String> detail = new LinkedHashMap<>();
detail.put("asin", normalizedAsin);
detail.put("minimumPrice", minimumPrice == null ? "" : minimumPrice.toPlainString());
bucket.add(detail);
}
}

View File

@@ -6,7 +6,7 @@ import com.nanri.aiimage.modules.shopmatch.model.dto.ShopMatchRowDto;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.stereotype.Service;
import java.io.File;
@@ -23,10 +23,12 @@ public class ShopMatchExcelAssemblyService {
public void writeWorkbook(File outputXlsx, Map<String, List<ShopMatchRowDto>> countries) {
Map<String, List<ShopMatchRowDto>> safe = countries == null ? Map.of() : countries;
try (XSSFWorkbook wb = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(outputXlsx)) {
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
workbook.setCompressTempFiles(true);
try (FileOutputStream fos = new FileOutputStream(outputXlsx)) {
for (ProductRiskCountryCode code : ProductRiskCountryCode.values()) {
String sheetName = code.getSheetName();
Sheet sheet = wb.createSheet(sheetName);
Sheet sheet = workbook.createSheet(sheetName);
Row header = sheet.createRow(0);
header.createCell(0).setCellValue(HEADER[0]);
header.createCell(1).setCellValue(HEADER[1]);
@@ -40,13 +42,19 @@ public class ShopMatchExcelAssemblyService {
row.createCell(0).setCellValue(item.getAsin() == null ? "" : item.getAsin());
row.createCell(1).setCellValue(item.getStatus() == null ? "" : item.getStatus());
}
sheet.autoSizeColumn(0);
sheet.autoSizeColumn(1);
sheet.setColumnWidth(0, 20 * 256);
sheet.setColumnWidth(1, 18 * 256);
}
wb.write(fos);
workbook.write(fos);
} catch (Exception ex) {
log.warn("[shop-match] write workbook failed: {}", ex.getMessage());
throw new BusinessException("生成 Excel 失败: " + ex.getMessage());
throw new BusinessException("generate shop-match excel failed: " + ex.getMessage());
} finally {
try {
workbook.close();
} catch (Exception ignored) {
}
workbook.dispose();
}
}

View File

@@ -1,24 +1,20 @@
package com.nanri.aiimage.modules.shopmatch.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.shopmatch.model.dto.ShopMatchShopPayloadDto;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
import com.nanri.aiimage.config.TaskPressureProperties;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.security.MessageDigest;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HexFormat;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -28,104 +24,34 @@ import java.util.concurrent.ConcurrentHashMap;
@Slf4j
public class ShopMatchTaskCacheService {
private static final String MODULE_TYPE = "SHOP_MATCH";
private static final long PAYLOAD_TTL_HOURS = 24;
private final ObjectMapper objectMapper;
private final TaskPressureProperties taskPressureProperties;
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
public ShopMatchShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
return null;
}
Path file = buildShopPayloadFile(taskId, shopKey);
if (!Files.isRegularFile(file)) {
return null;
}
try {
PayloadFile payloadFile = objectMapper.readValue(Files.readString(file), PayloadFile.class);
if (payloadFile == null || payloadFile.payload == null) {
return null;
}
if (payloadFile.shopKey == null || !shopKey.equals(payloadFile.shopKey)) {
log.warn("[shop-match] payload file shop key mismatch taskId={} shopKey={} storedKey={}",
taskId, shopKey, payloadFile.shopKey);
return null;
}
return payloadFile.payload;
} catch (Exception ex) {
throw new BusinessException("璇诲彇鍖归厤搴楅摵缂撳瓨澶辫触");
}
return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, ShopMatchShopPayloadDto.class);
}
public void saveShopMergedPayload(Long taskId, String shopKey, ShopMatchShopPayloadDto payload) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank() || payload == null) {
return;
}
try {
Files.createDirectories(buildTaskDir(taskId));
Files.writeString(
buildShopPayloadFile(taskId, shopKey),
objectMapper.writeValueAsString(new PayloadFile(shopKey, payload)),
StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.WRITE
);
} catch (Exception ex) {
throw new BusinessException("鏆傚瓨鍖归厤搴楅摵缁撴灉澶辫触");
}
taskScopePayloadStorageService.saveScopePayload(taskId, MODULE_TYPE, shopKey, payload);
}
public void removeShopMergedPayload(Long taskId, String shopKey) {
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
return;
}
try {
Files.deleteIfExists(buildShopPayloadFile(taskId, shopKey));
} catch (IOException ex) {
log.warn("[shop-match] delete shop payload failed taskId={} shopKey={} msg={}", taskId, shopKey, ex.getMessage());
}
taskScopePayloadStorageService.removeScopePayload(taskId, MODULE_TYPE, shopKey);
}
public Map<String, ShopMatchShopPayloadDto> getAllShopMergedPayload(Long taskId) {
try {
Path taskDir = buildTaskDir(taskId);
if (!Files.isDirectory(taskDir)) {
return Map.of();
}
LinkedHashMap<String, ShopMatchShopPayloadDto> out = new LinkedHashMap<>();
try (var stream = Files.list(taskDir)) {
for (Path path : stream
.filter(file -> file.getFileName().toString().endsWith(".json"))
.sorted(Comparator.comparing(file -> file.getFileName().toString()))
.toList()) {
PayloadFile payloadFile = objectMapper.readValue(Files.readString(path), PayloadFile.class);
if (payloadFile == null || payloadFile.shopKey == null || payloadFile.shopKey.isBlank() || payloadFile.payload == null) {
continue;
}
out.put(payloadFile.shopKey, payloadFile.payload);
}
}
return out;
} catch (Exception ex) {
throw new BusinessException("璇诲彇鍖归厤搴楅摵缂撳瓨澶辫触");
}
return taskScopePayloadStorageService.getAllScopePayload(taskId, MODULE_TYPE, ShopMatchShopPayloadDto.class);
}
public boolean hasAnyShopMergedPayload(Long taskId) {
if (taskId == null || taskId <= 0) {
return false;
}
Path taskDir = buildTaskDir(taskId);
if (!Files.isDirectory(taskDir)) {
return false;
}
try (var stream = Files.list(taskDir)) {
return stream.anyMatch(file -> Files.isRegularFile(file)
&& file.getFileName().toString().endsWith(".json"));
} catch (IOException ex) {
log.warn("[shop-match] scan task cache failed taskId={} msg={}", taskId, ex.getMessage());
return false;
}
return taskScopePayloadStorageService.hasAnyScopePayload(taskId, MODULE_TYPE);
}
public void deleteTaskCache(Long taskId) {
@@ -133,6 +59,7 @@ public class ShopMatchTaskCacheService {
return;
}
taskEntityLocalCache.remove(taskId);
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
try {
Path taskDir = buildTaskDir(taskId);
if (!Files.exists(taskDir)) {
@@ -140,7 +67,7 @@ public class ShopMatchTaskCacheService {
return;
}
try (var walk = Files.walk(taskDir)) {
walk.sorted(Comparator.reverseOrder()).forEach(path -> {
walk.sorted(java.util.Comparator.reverseOrder()).forEach(path -> {
try {
Files.deleteIfExists(path);
} catch (IOException ex) {
@@ -225,37 +152,10 @@ public class ShopMatchTaskCacheService {
return Path.of(System.getProperty("java.io.tmpdir"), "shop-match-cache", String.valueOf(taskId));
}
private Path buildShopPayloadFile(Long taskId, String shopKey) {
return buildTaskDir(taskId).resolve(hashShopKey(shopKey) + ".json");
}
private Path buildTaskEntityFile(Long taskId) {
return buildTaskDir(taskId).resolve("_task-entity.json");
}
private String hashShopKey(String shopKey) {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest(shopKey.trim().getBytes(StandardCharsets.UTF_8));
return HexFormat.of().formatHex(bytes);
} catch (Exception ex) {
throw new IllegalStateException("failed to hash shop key", ex);
}
}
private static class PayloadFile {
public String shopKey;
public ShopMatchShopPayloadDto payload;
public PayloadFile() {
}
public PayloadFile(String shopKey, ShopMatchShopPayloadDto payload) {
this.shopKey = shopKey;
this.payload = payload;
}
}
private boolean isLocalCacheFresh(LocalTaskEntityCacheEntry cached, long now) {
return cached != null
&& now - cached.cachedAtMillis() <= Math.max(0L, taskPressureProperties.getLocalTaskEntityCacheMillis());

View File

@@ -44,8 +44,8 @@ public class SplitRunService {
private static final String ZIP_CONTENT_TYPE = "application/zip";
private static final String SPLIT_MODE_ROWS_PER_FILE = "rows_per_file";
private static final String SPLIT_MODE_PARTS = "parts";
private static final String HEADER_SUFFIX_MARKER = "idASIN\u56fd\u5bb6\u72b6\u6001\u4ef7\u683c\u53d8\u4f53\u6570\u91cf";
private static final String HEADER_STOP_COLUMN = "\u7f29\u7565\u56fe\u5730\u57408";
private static final String HEADER_SUFFIX_MARKER = "idASIN国家状态价格变体数量";
private static final String HEADER_STOP_COLUMN = "缩略图地址8";
private static final DateTimeFormatter ZIP_NAME_DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd");
private final FileTaskMapper fileTaskMapper;

View File

@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.task.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TaskChunkMapper extends BaseMapper<TaskChunkEntity> {
}

View File

@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.task.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TaskScopeStateMapper extends BaseMapper<TaskScopeStateEntity> {
}

View File

@@ -0,0 +1,26 @@
package com.nanri.aiimage.modules.task.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("biz_task_chunk")
public class TaskChunkEntity {
@TableId(type = IdType.AUTO)
private Long id;
private Long taskId;
private String moduleType;
private String scopeKey;
private String scopeHash;
private Integer chunkIndex;
private Integer chunkTotal;
private String payloadJson;
private String payloadHash;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}

View File

@@ -0,0 +1,29 @@
package com.nanri.aiimage.modules.task.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("biz_task_scope_state")
public class TaskScopeStateEntity {
@TableId(type = IdType.AUTO)
private Long id;
private Long taskId;
private String moduleType;
private String scopeKey;
private String scopeHash;
private String parsedPayloadJson;
private String stateJson;
private Integer chunkTotal;
private Integer receivedChunkCount;
private Integer completed;
private LocalDateTime lastChunkAt;
private String lastError;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}

View File

@@ -0,0 +1,172 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.config.TaskPressureProperties;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.FileTime;
import java.time.Duration;
import java.time.Instant;
import java.util.Comparator;
import java.util.List;
@Service
@RequiredArgsConstructor
@Slf4j
public class TaskScopePayloadBufferMaintenanceService {
private static final Duration RECOVERY_LOCK_TTL = Duration.ofMinutes(10);
private static final Duration CLEANUP_LOCK_TTL = Duration.ofMinutes(10);
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
private final TaskPressureProperties taskPressureProperties;
private final DistributedJobLockService distributedJobLockService;
@EventListener(ApplicationReadyEvent.class)
public void recoverBufferedPayloadsOnStartup() {
DistributedJobLockService.LockHandle lockHandle =
distributedJobLockService.tryLock("task-scope-buffer:recover", RECOVERY_LOCK_TTL);
if (lockHandle == null) {
log.info("[task-scope-buffer] skip startup recovery because another instance holds the lock");
return;
}
try (lockHandle) {
Path root = taskScopePayloadStorageService.getBufferedPayloadRoot();
if (!Files.isDirectory(root)) {
return;
}
int limit = Math.max(0, taskPressureProperties.getScopePayloadRecoveryMaxFiles());
if (limit == 0) {
return;
}
int recovered = 0;
int scanned = 0;
for (Path file : listBufferedPayloadFiles(root, limit)) {
scanned++;
try {
BufferFileRef ref = parseBufferFile(root, file);
if (ref == null) {
Files.deleteIfExists(file);
continue;
}
if (taskScopePayloadStorageService.recoverBufferedScopePayload(ref.taskId(), ref.moduleType(), ref.scopeHash())) {
recovered++;
}
} catch (Exception ex) {
log.warn("[task-scope-buffer] startup recovery failed path={} msg={}", file, ex.getMessage());
}
}
if (scanned > 0) {
log.info("[task-scope-buffer] startup recovery finished scanned={} recovered={} limit={}", scanned, recovered, limit);
}
}
}
@Scheduled(cron = "${aiimage.task-pressure.scope-payload-cleanup-cron:15 */30 * * * *}")
public void cleanupExpiredBufferedPayloads() {
DistributedJobLockService.LockHandle lockHandle =
distributedJobLockService.tryLock("task-scope-buffer:cleanup", CLEANUP_LOCK_TTL);
if (lockHandle == null) {
log.info("[task-scope-buffer] skip cleanup because another instance holds the lock");
return;
}
try (lockHandle) {
Path root = taskScopePayloadStorageService.getBufferedPayloadRoot();
if (!Files.isDirectory(root)) {
return;
}
Instant cutoff = Instant.now().minus(Duration.ofHours(Math.max(1L, taskPressureProperties.getScopePayloadBufferRetentionHours())));
int deleted = 0;
for (Path file : listBufferedPayloadFiles(root, Integer.MAX_VALUE)) {
try {
FileTime lastModified = Files.getLastModifiedTime(file);
if (lastModified.toInstant().isAfter(cutoff)) {
continue;
}
Files.deleteIfExists(file);
deleted++;
deleteEmptyParents(file.getParent(), root);
} catch (Exception ex) {
log.warn("[task-scope-buffer] cleanup failed path={} msg={}", file, ex.getMessage());
}
}
if (deleted > 0) {
log.info("[task-scope-buffer] cleanup finished deleted={} root={}", deleted, root);
}
}
}
private List<Path> listBufferedPayloadFiles(Path root, int limit) {
try (var walk = Files.walk(root, 4)) {
return walk
.filter(Files::isRegularFile)
.filter(path -> path.getFileName().toString().endsWith(".json"))
.sorted(Comparator.comparing(this::safeLastModified))
.limit(limit)
.toList();
} catch (IOException ex) {
log.warn("[task-scope-buffer] scan failed root={} msg={}", root, ex.getMessage());
return List.of();
}
}
private Instant safeLastModified(Path path) {
try {
return Files.getLastModifiedTime(path).toInstant();
} catch (IOException ex) {
return Instant.EPOCH;
}
}
private BufferFileRef parseBufferFile(Path root, Path file) {
Path relative = root.relativize(file);
if (relative.getNameCount() != 3) {
return null;
}
String moduleType = relative.getName(0).toString().toUpperCase();
String taskIdRaw = relative.getName(1).toString();
String fileName = relative.getName(2).toString();
if (!fileName.endsWith(".json")) {
return null;
}
try {
long taskId = Long.parseLong(taskIdRaw);
String scopeHash = fileName.substring(0, fileName.length() - 5);
if (scopeHash.isBlank()) {
return null;
}
return new BufferFileRef(moduleType, taskId, scopeHash);
} catch (NumberFormatException ex) {
return null;
}
}
private void deleteEmptyParents(Path start, Path root) {
Path current = start;
while (current != null && !current.equals(root)) {
try (var list = Files.list(current)) {
if (list.findAny().isPresent()) {
return;
}
} catch (IOException ex) {
return;
}
try {
Files.deleteIfExists(current);
} catch (IOException ex) {
return;
}
current = current.getParent();
}
}
private record BufferFileRef(String moduleType, Long taskId, String scopeHash) {}
}

View File

@@ -0,0 +1,542 @@
package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import lombok.RequiredArgsConstructor;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.time.LocalDateTime;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@Service
@RequiredArgsConstructor
public class TaskScopePayloadStorageService {
private static final Set<String> OSS_BACKED_STATE_MODULES = new HashSet<>(Set.of(
"SHOP_MATCH",
"PRODUCT_RISK_RESOLVE",
"PRICE_TRACK",
"PATROL_DELETE"
));
private static final String OSS_POINTER_PREFIX = "oss:";
private static final String LOCAL_BUFFER_DIR = "task-scope-buffer";
private final TaskScopeStateMapper taskScopeStateMapper;
private final ObjectMapper objectMapper;
private final OssStorageService ossStorageService;
private final TaskPressureProperties taskPressureProperties;
@Transactional
public <T> void saveScopePayload(Long taskId, String moduleType, String scopeKey, T payload) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey) || payload == null) {
return;
}
String normalizedScopeKey = normalize(scopeKey);
String scopeHash = hash(normalizedScopeKey);
String payloadJson = writeJson(payload, "写入任务范围载荷失败");
LocalDateTime now = LocalDateTime.now();
TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash);
if (shouldStoreStatePayloadInOss(moduleType)) {
saveOssBackedScopePayload(taskId, moduleType, normalizedScopeKey, scopeHash, payloadJson, now, state);
return;
}
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
if (state == null) {
TaskScopeStateEntity entity = new TaskScopeStateEntity();
entity.setTaskId(taskId);
entity.setModuleType(moduleType);
entity.setScopeKey(normalizedScopeKey);
entity.setScopeHash(scopeHash);
entity.setStateJson(stateJson);
entity.setChunkTotal(0);
entity.setReceivedChunkCount(0);
entity.setCompleted(0);
entity.setLastChunkAt(now);
entity.setLastError(null);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskScopeStateMapper.insert(entity);
return;
} catch (DuplicateKeyException ignored) {
state = getScopeState(taskId, moduleType, scopeHash);
}
}
if (state == null) {
throw new BusinessException("写入任务范围载荷失败");
}
deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getScopeKey, normalizedScopeKey)
.set(TaskScopeStateEntity::getStateJson, stateJson)
.set(TaskScopeStateEntity::getLastChunkAt, now)
.set(TaskScopeStateEntity::getUpdatedAt, now));
}
public <T> T getScopePayload(Long taskId, String moduleType, String scopeKey, Class<T> clazz) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey)) {
return null;
}
TaskScopeStateEntity state = getScopeState(taskId, moduleType, hash(normalize(scopeKey)));
String payloadJson = resolveStatePayload(state);
if (payloadJson == null) {
return null;
}
try {
return objectMapper.readValue(payloadJson, clazz);
} catch (Exception ex) {
throw new BusinessException("读取任务范围载荷失败");
}
}
public <T> Map<String, T> getAllScopePayload(Long taskId, String moduleType, Class<T> clazz) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return Map.of();
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.select(TaskScopeStateEntity::getScopeKey, TaskScopeStateEntity::getScopeHash, TaskScopeStateEntity::getStateJson)
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, moduleType)
.isNotNull(TaskScopeStateEntity::getStateJson));
if (states == null || states.isEmpty()) {
return Map.of();
}
Map<String, T> result = new LinkedHashMap<>();
for (TaskScopeStateEntity state : states) {
String payloadJson = resolveStatePayload(state);
if (payloadJson == null || isBlank(state.getScopeKey())) {
continue;
}
try {
result.put(state.getScopeKey(), objectMapper.readValue(payloadJson, clazz));
} catch (Exception ex) {
throw new BusinessException("读取任务范围载荷失败");
}
}
return result;
}
@Transactional
public void removeScopePayload(Long taskId, String moduleType, String scopeKey) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey)) {
return;
}
TaskScopeStateEntity state = getScopeState(taskId, moduleType, hash(normalize(scopeKey)));
if (state == null) {
return;
}
deleteBufferedPayload(taskId, moduleType, state.getScopeHash());
deleteStatePayloadIfNeeded(state.getStateJson());
boolean hasParsedPayload = !isBlank(state.getParsedPayloadJson());
if (hasParsedPayload) {
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getStateJson, null)
.set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now()));
return;
}
taskScopeStateMapper.deleteById(state.getId());
}
public boolean hasAnyScopePayload(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return false;
}
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, moduleType)
.isNotNull(TaskScopeStateEntity::getStateJson));
return count != null && count > 0;
}
public long countScopePayload(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return 0L;
}
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, moduleType)
.isNotNull(TaskScopeStateEntity::getStateJson));
return count == null ? 0L : count;
}
@Transactional
public void deleteTaskScopePayloads(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return;
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, moduleType));
if (states == null || states.isEmpty()) {
return;
}
for (TaskScopeStateEntity state : states) {
if (state == null || state.getId() == null) {
continue;
}
deleteBufferedPayload(taskId, moduleType, state.getScopeHash());
deleteStatePayloadIfNeeded(state.getStateJson());
if (!isBlank(state.getParsedPayloadJson())) {
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getStateJson, null)
.set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now()));
} else {
taskScopeStateMapper.deleteById(state.getId());
}
}
}
@Transactional
public <T> void saveParsedPayloadMap(Long taskId, String moduleType, Map<String, T> payloadByScope) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || payloadByScope == null || payloadByScope.isEmpty()) {
return;
}
LocalDateTime now = LocalDateTime.now();
for (Map.Entry<String, T> entry : payloadByScope.entrySet()) {
if (isBlank(entry.getKey()) || entry.getValue() == null) {
continue;
}
String scopeKey = normalize(entry.getKey());
String scopeHash = hash(scopeKey);
String parsedPayloadJson = writeJson(entry.getValue(), "写入任务解析载荷失败");
TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash);
if (state == null) {
TaskScopeStateEntity entity = new TaskScopeStateEntity();
entity.setTaskId(taskId);
entity.setModuleType(moduleType);
entity.setScopeKey(scopeKey);
entity.setScopeHash(scopeHash);
entity.setParsedPayloadJson(parsedPayloadJson);
entity.setChunkTotal(0);
entity.setReceivedChunkCount(0);
entity.setCompleted(0);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskScopeStateMapper.insert(entity);
continue;
} catch (DuplicateKeyException ignored) {
state = getScopeState(taskId, moduleType, scopeHash);
}
}
if (state == null) {
throw new BusinessException("写入任务解析载荷失败");
}
deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), parsedPayloadJson);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
.set(TaskScopeStateEntity::getParsedPayloadJson, parsedPayloadJson)
.set(TaskScopeStateEntity::getUpdatedAt, now));
}
}
public <T> Map<String, T> getParsedPayloadMap(Long taskId, String moduleType, Class<T> clazz) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return Map.of();
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.select(TaskScopeStateEntity::getScopeKey, TaskScopeStateEntity::getParsedPayloadJson)
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, moduleType)
.isNotNull(TaskScopeStateEntity::getParsedPayloadJson));
if (states == null || states.isEmpty()) {
return Map.of();
}
Map<String, T> result = new LinkedHashMap<>();
for (TaskScopeStateEntity state : states) {
if (isBlank(state.getParsedPayloadJson()) || isBlank(state.getScopeKey())) {
continue;
}
try {
result.put(state.getScopeKey(), objectMapper.readValue(resolveParsedPayload(state.getParsedPayloadJson()), clazz));
} catch (Exception ex) {
throw new BusinessException("读取任务原始载荷失败");
}
}
return result;
}
@Transactional
public boolean recoverBufferedScopePayload(Long taskId, String moduleType, String scopeHash) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeHash)) {
return false;
}
String payloadJson = readBufferedPayload(taskId, moduleType, scopeHash);
if (payloadJson == null) {
return false;
}
TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash);
if (state == null) {
deleteBufferedPayload(taskId, moduleType, scopeHash);
return false;
}
if (!shouldStoreStatePayloadInOss(moduleType)) {
deleteBufferedPayload(taskId, moduleType, scopeHash);
return false;
}
LocalDateTime now = LocalDateTime.now();
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getStateJson, stateJson)
.set(TaskScopeStateEntity::getLastChunkAt, now)
.set(TaskScopeStateEntity::getUpdatedAt, now));
deleteBufferedPayload(taskId, moduleType, scopeHash);
return true;
}
public Path getBufferedPayloadRoot() {
return Path.of(System.getProperty("java.io.tmpdir"), LOCAL_BUFFER_DIR);
}
private TaskScopeStateEntity getScopeState(Long taskId, String moduleType, String scopeHash) {
return taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, moduleType)
.eq(TaskScopeStateEntity::getScopeHash, scopeHash)
.last("limit 1"));
}
private String storeStatePayload(String moduleType, Long taskId, String scopeHash, String payloadJson) {
if (!shouldStoreStatePayloadInOss(moduleType)) {
return payloadJson;
}
try {
String pointer = OSS_POINTER_PREFIX + ossStorageService.uploadTaskScopePayload(moduleType, taskId, scopeHash, payloadJson);
return objectMapper.writeValueAsString(pointer);
} catch (Exception ex) {
throw new BusinessException("写入任务范围载荷失败");
}
}
private String resolveStatePayload(TaskScopeStateEntity state) {
if (state == null || isBlank(state.getStateJson())) {
return null;
}
String bufferedPayload = readBufferedPayload(state.getTaskId(), state.getModuleType(), state.getScopeHash());
if (bufferedPayload != null) {
return bufferedPayload;
}
String ossPointer = extractOssPointer(state.getStateJson());
if (ossPointer == null) {
return state.getStateJson();
}
try {
return ossStorageService.readObjectAsString(stripOssPointerPrefix(ossPointer));
} catch (Exception ex) {
throw new BusinessException("读取任务范围载荷失败");
}
}
private void deleteStatePayloadIfNeeded(String value) {
String ossPointer = extractOssPointer(value);
if (ossPointer == null) {
return;
}
try {
ossStorageService.deleteObject(stripOssPointerPrefix(ossPointer));
} catch (Exception ignored) {
}
}
private String resolveParsedPayload(String value) {
if (isBlank(value) || !isOssPointer(value)) {
return value;
}
try {
return ossStorageService.readObjectAsString(stripOssPointerPrefix(value));
} catch (Exception ex) {
throw new BusinessException("读取任务解析载荷失败");
}
}
private void deleteParsedPayloadIfNeeded(String oldValue, String newValue) {
if (isBlank(oldValue) || oldValue.equals(newValue) || !isOssPointer(oldValue)) {
return;
}
try {
ossStorageService.deleteObject(stripOssPointerPrefix(oldValue));
} catch (Exception ignored) {
}
}
private void deleteReplacedStatePayloadIfNeeded(String oldValue, String newValue) {
if (isBlank(oldValue) || oldValue.equals(newValue)) {
return;
}
deleteStatePayloadIfNeeded(oldValue);
}
private void saveOssBackedScopePayload(Long taskId,
String moduleType,
String normalizedScopeKey,
String scopeHash,
String payloadJson,
LocalDateTime now,
TaskScopeStateEntity state) {
if (state == null) {
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
TaskScopeStateEntity entity = new TaskScopeStateEntity();
entity.setTaskId(taskId);
entity.setModuleType(moduleType);
entity.setScopeKey(normalizedScopeKey);
entity.setScopeHash(scopeHash);
entity.setStateJson(stateJson);
entity.setChunkTotal(0);
entity.setReceivedChunkCount(0);
entity.setCompleted(0);
entity.setLastChunkAt(now);
entity.setLastError(null);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskScopeStateMapper.insert(entity);
deleteBufferedPayload(taskId, moduleType, scopeHash);
return;
} catch (DuplicateKeyException ignored) {
state = getScopeState(taskId, moduleType, scopeHash);
}
}
if (state == null) {
throw new BusinessException("刷新任务范围缓冲载荷失败");
}
writeBufferedPayload(taskId, moduleType, scopeHash, payloadJson);
if (!shouldFlushBufferedPayload(state, now)) {
return;
}
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getScopeKey, normalizedScopeKey)
.set(TaskScopeStateEntity::getStateJson, stateJson)
.set(TaskScopeStateEntity::getLastChunkAt, now)
.set(TaskScopeStateEntity::getUpdatedAt, now));
deleteBufferedPayload(taskId, moduleType, scopeHash);
}
private String writeJson(Object value, String message) {
try {
return objectMapper.writeValueAsString(value);
} catch (Exception ex) {
throw new BusinessException(message);
}
}
private String normalize(String value) {
return value == null ? "" : value.trim();
}
private boolean isBlank(String value) {
return value == null || value.isBlank();
}
private boolean shouldStoreStatePayloadInOss(String moduleType) {
return OSS_BACKED_STATE_MODULES.contains(normalize(moduleType).toUpperCase());
}
private boolean isOssPointer(String value) {
return !isBlank(value) && value.startsWith(OSS_POINTER_PREFIX);
}
private String extractOssPointer(String value) {
if (isBlank(value)) {
return null;
}
if (isOssPointer(value)) {
return value;
}
try {
String decoded = objectMapper.readValue(value, String.class);
return isOssPointer(decoded) ? decoded : null;
} catch (Exception ignored) {
return null;
}
}
private String stripOssPointerPrefix(String value) {
return value.substring(OSS_POINTER_PREFIX.length());
}
private boolean shouldFlushBufferedPayload(TaskScopeStateEntity state, LocalDateTime now) {
long flushIntervalMillis = Math.max(0L, taskPressureProperties.getScopePayloadFlushIntervalMillis());
if (flushIntervalMillis <= 0L || state.getUpdatedAt() == null) {
return true;
}
return java.time.Duration.between(state.getUpdatedAt(), now).toMillis() >= flushIntervalMillis;
}
private void writeBufferedPayload(Long taskId, String moduleType, String scopeHash, String payloadJson) {
try {
Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash);
Files.createDirectories(path.getParent());
Files.writeString(path, payloadJson, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE);
} catch (IOException ex) {
throw new BusinessException("写入任务范围缓冲载荷失败");
}
}
private String readBufferedPayload(Long taskId, String moduleType, String scopeHash) {
try {
Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash);
if (!Files.isRegularFile(path)) {
return null;
}
return Files.readString(path);
} catch (IOException ex) {
return null;
}
}
private void deleteBufferedPayload(Long taskId, String moduleType, String scopeHash) {
try {
Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash);
Files.deleteIfExists(path);
} catch (IOException ignored) {
}
}
private Path buildBufferedPayloadPath(Long taskId, String moduleType, String scopeHash) {
String normalizedModuleType = normalize(moduleType).toLowerCase();
return Path.of(System.getProperty("java.io.tmpdir"), LOCAL_BUFFER_DIR, normalizedModuleType, String.valueOf(taskId), scopeHash + ".json");
}
private String hash(String value) {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest(normalize(value).getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
} catch (Exception ex) {
throw new IllegalStateException("failed to hash scope key", ex);
}
}
}

View File

@@ -50,7 +50,7 @@ public class ZiniaoShopSwitchService {
if (value == null) {
return "";
}
return value.replace("\u3000", " ").trim();
return value.replace(" ", " ").trim();
}
public boolean isSkippableMatchError(BusinessException ex) {
@@ -59,9 +59,9 @@ public class ZiniaoShopSwitchService {
return false;
}
return (message.contains("code=40004")
|| message.contains("userId\u5b58\u5728\u65e0\u6548\u7684\u53c2\u6570\u503c")
|| message.contains("\u65e0\u6743\u9650")
|| message.contains("\u6ca1\u6709\u6743\u9650"))
&& !message.contains("\u767d\u540d\u5355");
|| message.contains("userId存在无效的参数值")
|| message.contains("无权限")
|| message.contains("没有权限"))
&& !message.contains("白名单");
}
}

View File

@@ -99,6 +99,10 @@ aiimage:
task-pressure:
local-task-entity-cache-millis: ${AIIMAGE_TASK_LOCAL_ENTITY_CACHE_MILLIS:3000}
db-select-batch-size: ${AIIMAGE_TASK_DB_SELECT_BATCH_SIZE:200}
scope-payload-flush-interval-millis: ${AIIMAGE_TASK_SCOPE_PAYLOAD_FLUSH_INTERVAL_MILLIS:15000}
scope-payload-buffer-retention-hours: ${AIIMAGE_TASK_SCOPE_PAYLOAD_BUFFER_RETENTION_HOURS:24}
scope-payload-recovery-max-files: ${AIIMAGE_TASK_SCOPE_PAYLOAD_RECOVERY_MAX_FILES:200}
scope-payload-cleanup-cron: ${AIIMAGE_TASK_SCOPE_PAYLOAD_CLEANUP_CRON:15 */30 * * * *}
security:
shop-credential-key: ${AIIMAGE_SHOP_CREDENTIAL_KEY:change-me-shop-credential-key}
internal-token: ${AIIMAGE_INTERNAL_TOKEN:}

View File

@@ -0,0 +1,76 @@
SET @schema_name = DATABASE();
SET @sql = IF(
EXISTS (
SELECT 1
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = @schema_name
AND TABLE_NAME = 'biz_skip_price_asin'
AND COLUMN_NAME = 'minimum_price_de'
),
'SELECT 1',
'ALTER TABLE biz_skip_price_asin ADD COLUMN minimum_price_de DECIMAL(10,2) NULL DEFAULT NULL COMMENT ''DE minimum price'' AFTER asin_de'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
SET @sql = IF(
EXISTS (
SELECT 1
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = @schema_name
AND TABLE_NAME = 'biz_skip_price_asin'
AND COLUMN_NAME = 'minimum_price_uk'
),
'SELECT 1',
'ALTER TABLE biz_skip_price_asin ADD COLUMN minimum_price_uk DECIMAL(10,2) NULL DEFAULT NULL COMMENT ''UK minimum price'' AFTER asin_uk'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
SET @sql = IF(
EXISTS (
SELECT 1
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = @schema_name
AND TABLE_NAME = 'biz_skip_price_asin'
AND COLUMN_NAME = 'minimum_price_fr'
),
'SELECT 1',
'ALTER TABLE biz_skip_price_asin ADD COLUMN minimum_price_fr DECIMAL(10,2) NULL DEFAULT NULL COMMENT ''FR minimum price'' AFTER asin_fr'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
SET @sql = IF(
EXISTS (
SELECT 1
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = @schema_name
AND TABLE_NAME = 'biz_skip_price_asin'
AND COLUMN_NAME = 'minimum_price_it'
),
'SELECT 1',
'ALTER TABLE biz_skip_price_asin ADD COLUMN minimum_price_it DECIMAL(10,2) NULL DEFAULT NULL COMMENT ''IT minimum price'' AFTER asin_it'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
SET @sql = IF(
EXISTS (
SELECT 1
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = @schema_name
AND TABLE_NAME = 'biz_skip_price_asin'
AND COLUMN_NAME = 'minimum_price_es'
),
'SELECT 1',
'ALTER TABLE biz_skip_price_asin ADD COLUMN minimum_price_es DECIMAL(10,2) NULL DEFAULT NULL COMMENT ''ES minimum price'' AFTER asin_es'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;

View File

@@ -0,0 +1,36 @@
CREATE TABLE IF NOT EXISTS biz_task_scope_state (
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key',
task_id BIGINT NOT NULL COMMENT 'task id',
module_type VARCHAR(32) NOT NULL COMMENT 'module type',
scope_key VARCHAR(1000) NOT NULL COMMENT 'scope identifier',
scope_hash CHAR(64) NOT NULL COMMENT 'hash of scope key',
parsed_payload_json JSON NULL COMMENT 'parsed source payload',
state_json JSON NULL COMMENT 'merged intermediate state',
chunk_total INT NOT NULL DEFAULT 0 COMMENT 'expected chunk count',
received_chunk_count INT NOT NULL DEFAULT 0 COMMENT 'received chunk count',
completed TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'whether scope is complete',
last_chunk_at DATETIME NULL COMMENT 'last chunk received time',
last_error VARCHAR(1000) NULL COMMENT 'last error message',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
UNIQUE KEY uk_task_scope (task_id, module_type, scope_hash),
KEY idx_task_completed (task_id, module_type, completed),
KEY idx_scope_hash (scope_hash)
) COMMENT='task scope state';
CREATE TABLE IF NOT EXISTS biz_task_chunk (
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key',
task_id BIGINT NOT NULL COMMENT 'task id',
module_type VARCHAR(32) NOT NULL COMMENT 'module type',
scope_key VARCHAR(1000) NOT NULL COMMENT 'scope identifier',
scope_hash CHAR(64) NOT NULL COMMENT 'hash of scope key',
chunk_index INT NOT NULL COMMENT 'chunk index starting from 1',
chunk_total INT NOT NULL DEFAULT 0 COMMENT 'total chunk count',
payload_json JSON NOT NULL COMMENT 'chunk payload',
payload_hash CHAR(64) NOT NULL COMMENT 'hash of payload json',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
UNIQUE KEY uk_task_scope_chunk (task_id, module_type, scope_hash, chunk_index),
KEY idx_task_scope (task_id, module_type, scope_hash),
KEY idx_task_module (task_id, module_type)
) COMMENT='task chunk detail';

View File

@@ -12,7 +12,7 @@ import pymysql
from werkzeug.security import generate_password_hash
from utils.db import get_db
from utils.auth import admin_required, get_current_admin_role
from utils.auth import admin_required, login_required, get_current_admin_role
ALLOWED_DEDUPE_TOTAL_DATA_ADMIN_USERNAMES = {'刘丽泓'}
@@ -353,8 +353,82 @@ def _load_current_admin_menu_items():
return role, current_row, items, None
def _load_current_backend_menu_items():
role, current_row = get_current_admin_role()
if not role or not current_row:
return None, None, None, (jsonify({'success': False, 'error': '需要登录'}), 403)
user_id = _get_current_admin_id(current_row)
if not user_id:
return role, current_row, None, (jsonify({'success': False, 'error': '当前登录用户缺少有效ID'}), 400)
result, error_response, status = _proxy_backend_java(
'GET',
f'/api/admin/permission-users/{user_id}/column-permissions',
params={'menuType': 'admin'},
)
if error_response is not None:
return role, current_row, None, (error_response, status)
items = [_format_permission_item(item) for item in (result.get('data') or [])]
return role, current_row, items, None
def _ensure_backend_menu_access(*menu_names):
role, current_row = get_current_admin_role()
if role == 'super_admin':
return role, current_row, None
if not role or not current_row:
return role, current_row, (jsonify({'success': False, 'error': '需要登录'}), 403)
key_set, route_set = _get_current_admin_permission_sets()
for menu_name in menu_names:
config = ADMIN_MENU_ACCESS_CONFIG.get(menu_name) or {}
column_key = (config.get('column_key') or '').strip()
route_path = (config.get('route_path') or '').strip()
if (column_key and column_key in key_set) or (route_path and route_path in route_set):
return role, current_row, None
fallback_menu = menu_names[0] if menu_names else ''
fallback_config = ADMIN_MENU_ACCESS_CONFIG.get(fallback_menu) or {}
error_message = fallback_config.get('error') or '无权访问当前模块'
return role, current_row, (jsonify({'success': False, 'error': error_message}), 403)
def _get_column_ids_by_route_paths(route_paths, menu_type='admin'):
normalized_paths = [(path or '').strip() for path in (route_paths or []) if (path or '').strip()]
if not normalized_paths:
return []
conn = get_db()
try:
with conn.cursor() as cur:
placeholders = ','.join(['%s'] * len(normalized_paths))
cur.execute(
f"SELECT id FROM columns WHERE menu_type = %s AND route_path IN ({placeholders}) ORDER BY id ASC",
[menu_type] + normalized_paths,
)
rows = cur.fetchall()
return [int(row['id']) for row in rows if row.get('id') is not None]
finally:
conn.close()
def _grant_backend_menu_permissions(user_ids, route_paths):
normalized_user_ids = [int(uid) for uid in (user_ids or []) if str(uid).isdigit() and int(uid) > 0]
column_ids = _get_column_ids_by_route_paths(route_paths, 'admin')
if not normalized_user_ids or not column_ids:
return
conn = get_db()
try:
with conn.cursor() as cur:
for user_id in normalized_user_ids:
for column_id in column_ids:
cur.execute(
"INSERT IGNORE INTO user_column_permission (user_id, column_id) VALUES (%s, %s)",
(user_id, column_id),
)
conn.commit()
finally:
conn.close()
@admin_api.route('/current-user')
@admin_required
@login_required
def get_admin_current_user():
role, current_row = get_current_admin_role()
if not role or not current_row:
@@ -370,16 +444,16 @@ def get_admin_current_user():
@admin_api.route('/current-user/menus')
@admin_required
@login_required
def get_admin_current_user_menus():
_, _, items, denied = _load_current_admin_menu_items()
_, _, items, denied = _load_current_backend_menu_items()
if denied:
return denied
return jsonify({'success': True, 'items': items})
@admin_api.route('/logout', methods=['POST'])
@admin_required
@login_required
def admin_logout():
session.clear()
return jsonify({'success': True, 'msg': '退出成功', 'redirect': '/login'})
@@ -388,17 +462,14 @@ def admin_logout():
# ---------- 用户管理 ----------
@admin_api.route('/users')
@admin_required
@login_required
def list_users():
"""分页获取用户列表,支持用户名模糊搜索和按管理员归属筛选普通用户"""
"""分页获取用户列表,支持用户名模糊搜索和按管理员归属筛选普通用户"""
role, current_row = get_current_admin_role()
if not role:
return jsonify({'success': False, 'error': '需要管理员权限'}), 403
return jsonify({'success': False, 'error': '需要登录'}), 403
page = max(1, int(request.args.get('page', 1)))
_, _, denied = _ensure_admin_menu_access('users', 'history', 'shop-manage', 'skip-price-asin')
if denied:
return denied
_, _, denied = _ensure_admin_menu_access('users', 'history', 'shop-manage', 'skip-price-asin')
_, _, denied = _ensure_backend_menu_access('users', 'history', 'shop-manage', 'skip-price-asin')
if denied:
return denied
page_size = min(999, max(5, int(request.args.get('page_size', 15))))
@@ -407,6 +478,7 @@ def list_users():
created_by_id_arg = request.args.get('created_by_id') or request.args.get('admin_id')
created_by_id = int(created_by_id_arg) if created_by_id_arg and str(created_by_id_arg).isdigit() else None
can_view_all_users = role == 'super_admin' or (current_row and (current_row.get('username') or '').strip().lower() == 'admin')
can_view_child_users = role == 'admin' and not can_view_all_users
if not can_view_all_users:
created_by_id = None
try:
@@ -435,7 +507,7 @@ def list_users():
total = cur.fetchone()['total']
cur.execute("SELECT id, username FROM users WHERE role = 'admin' ORDER BY id")
admins = [{'id': r['id'], 'username': r['username']} for r in cur.fetchall()]
else:
elif can_view_child_users:
admin_id = current_row['id']
where_parts = ["(u.id = %s OR (u.role = 'normal' AND u.created_by_id = %s))"]
params = [admin_id, admin_id]
@@ -458,6 +530,28 @@ def list_users():
)
total = cur.fetchone()['total']
admins = []
else:
where_parts = ["u.id = %s"]
params = [current_row['id']]
if search_username:
where_parts.append("u.username LIKE %s")
params.append("%" + search_username + "%")
where_sql = " AND ".join(where_parts)
cur.execute(
"""SELECT u.id, u.username, u.is_admin, u.role, u.created_at, u.created_by_id,
creator.username AS creator_username
FROM users u
LEFT JOIN users creator ON creator.id = u.created_by_id
WHERE """ + where_sql + """ ORDER BY u.id LIMIT %s OFFSET %s""",
tuple(params) + (page_size, offset),
)
rows = cur.fetchall()
cur.execute(
"SELECT COUNT(*) as total FROM users u WHERE " + where_sql,
tuple(params),
)
total = cur.fetchone()['total']
admins = []
items = [
{
'id': r['id'],
@@ -485,7 +579,6 @@ def list_users():
except Exception as e:
return jsonify({'success': False, 'error': str(e)})
@admin_api.route('/user', methods=['POST'])
@admin_required
def create_user():
@@ -1341,9 +1434,9 @@ def delete_dedupe_total_data(item_id):
# ---------- 店铺管理 ----------
@admin_api.route('/shop-manages')
@admin_required
@login_required
def list_shop_manages():
role, current_row, denied = _ensure_admin_menu_access('shop-manage')
role, current_row, denied = _ensure_backend_menu_access('shop-manage')
if denied:
return denied
page = max(1, int(request.args.get('page', 1)))
@@ -1397,9 +1490,9 @@ def list_shop_manages():
@admin_api.route('/shop-manage', methods=['POST'])
@admin_required
@login_required
def create_shop_manage():
role, current_row, denied = _ensure_admin_menu_access('shop-manage')
role, current_row, denied = _ensure_backend_menu_access('shop-manage')
if denied:
return denied
data = request.get_json() or {}
@@ -1441,9 +1534,9 @@ def create_shop_manage():
@admin_api.route('/shop-manage/<int:item_id>', methods=['PUT'])
@admin_required
@login_required
def update_shop_manage(item_id):
role, current_row, denied = _ensure_admin_menu_access('shop-manage')
role, current_row, denied = _ensure_backend_menu_access('shop-manage')
if denied:
return denied
data = request.get_json() or {}
@@ -1483,9 +1576,9 @@ def update_shop_manage(item_id):
@admin_api.route('/shop-manage/<int:item_id>', methods=['DELETE'])
@admin_required
@login_required
def delete_shop_manage(item_id):
role, current_row, denied = _ensure_admin_menu_access('shop-manage')
role, current_row, denied = _ensure_backend_menu_access('shop-manage')
if denied:
return denied
result, error_response, status = _proxy_backend_java(
@@ -1505,9 +1598,9 @@ def delete_shop_manage(item_id):
@admin_api.route('/shop-manage-groups')
@admin_required
@login_required
def list_shop_manage_groups():
role, current_row, denied = _ensure_admin_menu_access('shop-manage', 'skip-price-asin')
role, current_row, denied = _ensure_backend_menu_access('shop-manage', 'skip-price-asin')
if denied:
return denied
params = {}
@@ -1543,12 +1636,13 @@ def list_shop_manage_groups():
@admin_api.route('/shop-manage-group', methods=['POST'])
@admin_required
@login_required
def create_shop_manage_group():
role, current_row, denied = _ensure_admin_menu_access('shop-manage', 'skip-price-asin')
role, current_row, denied = _ensure_backend_menu_access('shop-manage', 'skip-price-asin')
if denied:
return denied
data = request.get_json() or {}
grant_menu_routes = data.get('grant_menu_routes') or []
payload = {
'groupName': (data.get('group_name') or '').strip(),
'memberUserIds': data.get('member_user_ids') or [],
@@ -1565,6 +1659,7 @@ def create_shop_manage_group():
)
if error_response is not None:
return error_response, status
_grant_backend_menu_permissions(data.get('member_user_ids') or [], grant_menu_routes)
item = result.get('data') or {}
return jsonify({
'success': True,
@@ -1586,12 +1681,13 @@ def create_shop_manage_group():
@admin_api.route('/shop-manage-group/<int:item_id>', methods=['PUT'])
@admin_required
@login_required
def update_shop_manage_group(item_id):
role, current_row, denied = _ensure_admin_menu_access('shop-manage', 'skip-price-asin')
role, current_row, denied = _ensure_backend_menu_access('shop-manage', 'skip-price-asin')
if denied:
return denied
data = request.get_json() or {}
grant_menu_routes = data.get('grant_menu_routes') or []
payload = {
'groupName': (data.get('group_name') or '').strip(),
'memberUserIds': data.get('member_user_ids') or [],
@@ -1607,6 +1703,7 @@ def update_shop_manage_group(item_id):
)
if error_response is not None:
return error_response, status
_grant_backend_menu_permissions(data.get('member_user_ids') or [], grant_menu_routes)
item = result.get('data') or {}
return jsonify({
'success': True,
@@ -1628,9 +1725,9 @@ def update_shop_manage_group(item_id):
@admin_api.route('/skip-price-asins')
@admin_required
@login_required
def list_skip_price_asins():
role, current_row, denied = _ensure_admin_menu_access('skip-price-asin')
role, current_row, denied = _ensure_backend_menu_access('skip-price-asin')
if denied:
return denied
page = max(1, int(request.args.get('page', 1)))
@@ -1668,10 +1765,15 @@ def list_skip_price_asins():
'group_name': item.get('groupName') or '',
'shop_name': item.get('shopName') or '',
'asin_de': item.get('asinDe') or '',
'minimum_price_de': item.get('minimumPriceDe'),
'asin_uk': item.get('asinUk') or '',
'minimum_price_uk': item.get('minimumPriceUk'),
'asin_fr': item.get('asinFr') or '',
'minimum_price_fr': item.get('minimumPriceFr'),
'asin_it': item.get('asinIt') or '',
'minimum_price_it': item.get('minimumPriceIt'),
'asin_es': item.get('asinEs') or '',
'minimum_price_es': item.get('minimumPriceEs'),
'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16],
'updated_at': (item.get('updatedAt') or '').replace('T', ' ')[:16],
}
@@ -1687,25 +1789,34 @@ def list_skip_price_asins():
@admin_api.route('/skip-price-asin', methods=['POST'])
@admin_required
@login_required
def create_skip_price_asin():
role, current_row, denied = _ensure_admin_menu_access('skip-price-asin')
role, current_row, denied = _ensure_backend_menu_access('skip-price-asin')
if denied:
return denied
data = request.get_json() or {}
asin_mappings = data.get('asin_mappings') or {}
minimum_price_mappings = data.get('minimum_price_mappings') or {}
fallback_asin = (data.get('asin') or '').strip()
if not fallback_asin and isinstance(asin_mappings, dict):
for value in asin_mappings.values():
fallback_asin = (value or '').strip()
if fallback_asin:
break
fallback_minimum_price = data.get('minimum_price')
if fallback_minimum_price in ('', None) and isinstance(minimum_price_mappings, dict):
for value in minimum_price_mappings.values():
if value not in ('', None):
fallback_minimum_price = value
break
payload = {
'groupId': data.get('group_id'),
'shopName': (data.get('shop_name') or '').strip(),
'countries': data.get('countries') or [],
'asin': fallback_asin,
'asinMappings': asin_mappings,
'minimumPrice': fallback_minimum_price,
'minimumPriceMappings': minimum_price_mappings,
}
result, error_response, status = _proxy_backend_java(
'POST',
@@ -1728,10 +1839,15 @@ def create_skip_price_asin():
'group_name': item.get('groupName') or '',
'shop_name': item.get('shopName') or '',
'asin_de': item.get('asinDe') or '',
'minimum_price_de': item.get('minimumPriceDe'),
'asin_uk': item.get('asinUk') or '',
'minimum_price_uk': item.get('minimumPriceUk'),
'asin_fr': item.get('asinFr') or '',
'minimum_price_fr': item.get('minimumPriceFr'),
'asin_it': item.get('asinIt') or '',
'minimum_price_it': item.get('minimumPriceIt'),
'asin_es': item.get('asinEs') or '',
'minimum_price_es': item.get('minimumPriceEs'),
'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16],
'updated_at': (item.get('updatedAt') or '').replace('T', ' ')[:16],
},
@@ -1739,9 +1855,9 @@ def create_skip_price_asin():
@admin_api.route('/skip-price-asin/<int:item_id>/country/<country>', methods=['DELETE'])
@admin_required
@login_required
def delete_skip_price_asin_country(item_id, country):
role, current_row, denied = _ensure_admin_menu_access('skip-price-asin')
role, current_row, denied = _ensure_backend_menu_access('skip-price-asin')
if denied:
return denied
result, error_response, status = _proxy_backend_java(
@@ -1758,9 +1874,9 @@ def delete_skip_price_asin_country(item_id, country):
@admin_api.route('/shop-manage-group/<int:item_id>', methods=['DELETE'])
@admin_required
@login_required
def delete_shop_manage_group(item_id):
role, current_row, denied = _ensure_admin_menu_access('shop-manage', 'skip-price-asin')
role, current_row, denied = _ensure_backend_menu_access('shop-manage', 'skip-price-asin')
if denied:
return denied
result, error_response, status = _proxy_backend_java(
@@ -1777,13 +1893,16 @@ def delete_shop_manage_group(item_id):
@admin_api.route('/skip-price-asin/<int:item_id>/country/<country>', methods=['PUT'])
@admin_required
@login_required
def update_skip_price_asin_country(item_id, country):
role, current_row, denied = _ensure_admin_menu_access('skip-price-asin')
role, current_row, denied = _ensure_backend_menu_access('skip-price-asin')
if denied:
return denied
data = request.get_json() or {}
payload = {'asin': (data.get('asin') or '').strip()}
payload = {
'asin': (data.get('asin') or '').strip(),
'minimumPrice': data.get('minimum_price'),
}
result, error_response, status = _proxy_backend_java(
'PUT',
f'/api/admin/skip-price-asins/{item_id}/countries/{country}',
@@ -1805,10 +1924,15 @@ def update_skip_price_asin_country(item_id, country):
'group_name': item.get('groupName') or '',
'shop_name': item.get('shopName') or '',
'asin_de': item.get('asinDe') or '',
'minimum_price_de': item.get('minimumPriceDe'),
'asin_uk': item.get('asinUk') or '',
'minimum_price_uk': item.get('minimumPriceUk'),
'asin_fr': item.get('asinFr') or '',
'minimum_price_fr': item.get('minimumPriceFr'),
'asin_it': item.get('asinIt') or '',
'minimum_price_it': item.get('minimumPriceIt'),
'asin_es': item.get('asinEs') or '',
'minimum_price_es': item.get('minimumPriceEs'),
'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16],
'updated_at': (item.get('updatedAt') or '').replace('T', ' ')[:16],
},

View File

@@ -5,7 +5,7 @@ from flask import Blueprint, request, redirect, url_for, session, jsonify
from werkzeug.security import check_password_hash
from utils.db import get_db
from utils.auth import login_required, is_session_user_valid, is_current_user_admin
from utils.auth import login_required, is_session_user_valid
from utils.render import render_html
auth = Blueprint('auth', __name__, url_prefix='')
@@ -13,7 +13,7 @@ auth = Blueprint('auth', __name__, url_prefix='')
@auth.route('/login', methods=['GET', 'POST'])
def login():
if session.get('user_id') and is_session_user_valid() and is_current_user_admin():
if session.get('user_id') and is_session_user_valid():
return redirect(url_for('main.admin_page'))
if request.method == 'POST':
data = request.get_json() if request.is_json else request.form
@@ -31,22 +31,17 @@ def login():
(username,)
)
row = cur.fetchone()
conn.close()
if row and check_password_hash(row['password_hash'], password):
session.permanent = True
session['user_id'] = row['id']
session['username'] = username
if not row.get('is_admin'):
session.clear()
if request.is_json:
return jsonify({'success': False, 'error': '需要管理员权限'}), 403
return render_html('login.html', error='需要管理员权限')
if request.is_json:
return jsonify({'success': True, 'redirect': url_for('main.admin_page')})
return redirect(url_for('main.admin_page'))
conn.close()
except Exception as e:
except Exception as exc:
if request.is_json:
return jsonify({'success': False, 'error': str(e)})
return jsonify({'success': False, 'error': str(exc)})
return render_html('login.html', error='登录失败,请稍后重试')
if request.is_json:
return jsonify({'success': False, 'error': '用户名或密码错误'})
@@ -57,7 +52,7 @@ def login():
@auth.route('/api/auth/check')
@login_required
def api_auth_check():
"""校验登录状态,用于页面加载时判断是否已登录"""
"""校验登录状态,用于页面加载时判断是否已登录"""
if not session.get('user_id'):
return jsonify({'logged_in': False})
try:
@@ -66,7 +61,7 @@ def api_auth_check():
cur.execute("SELECT machine, is_admin FROM users WHERE id = %s", (session['user_id'],))
row = cur.fetchone()
conn.close()
if not row or not row.get('is_admin'):
if not row:
return jsonify({'logged_in': False})
except Exception:
return jsonify({'logged_in': False})

View File

@@ -4,7 +4,7 @@
import os
from flask import Blueprint, redirect, url_for, send_file, session
from utils.auth import login_required, admin_required, is_session_user_valid, is_current_user_admin
from utils.auth import login_required, is_session_user_valid
from utils.render import render_html
main = Blueprint('main', __name__, url_prefix='')
@@ -15,14 +15,13 @@ STATIC_DIR = os.path.join(BASE_DIR, 'static')
@main.route('/')
def index():
if session.get('user_id') and is_session_user_valid() and is_current_user_admin():
if session.get('user_id') and is_session_user_valid():
return redirect(url_for('main.admin_page'))
return redirect(url_for('auth.login'))
@main.route('/admin')
@login_required
@admin_required
def admin_page():
return render_html('admin.html')

View File

@@ -23,8 +23,8 @@ bucket_path = "nanri-image/"
file_url_pre = f"https://{bucket}.oss-cn-hangzhou.aliyuncs.com/"
import os
# backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://127.0.0.1:18080').rstrip('/')
backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://8.136.19.173:18080').rstrip('/')
backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://127.0.0.1:18080').rstrip('/')
# backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://8.136.19.173:18080').rstrip('/')
os.environ['OSS_ACCESS_KEY_ID'] = accessKeyId
os.environ['OSS_ACCESS_KEY_SECRET'] = accessKeySecret
os.environ['SECRET_KEY'] = "ddffc7c1d02121d9554d7b080b2511b6"

98
backend/run.sh Normal file
View File

@@ -0,0 +1,98 @@
#!/usr/bin/env bash
set -euo pipefail
PORT="${PORT:-15124}"
HOST="${HOST:-0.0.0.0}"
APP_MODULE="${APP_MODULE:-app.py}"
PID_FILE="${PID_FILE:-backend.pid}"
LOG_FILE="${LOG_FILE:-backend.log}"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$SCRIPT_DIR"
echo "检查端口 ${PORT} ..."
if command -v ss >/dev/null 2>&1; then
if ss -ltn | awk '{print $4}' | grep -Eq "(^|:)${PORT}$"; then
echo "错误: 端口 ${PORT} 已被占用,请先释放后再启动。"
exit 1
fi
elif command -v netstat >/dev/null 2>&1; then
if netstat -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "(^|:)${PORT}$"; then
echo "错误: 端口 ${PORT} 已被占用,请先释放后再启动。"
exit 1
fi
else
echo "警告: 未找到 ss 或 netstat跳过端口占用检查。"
fi
echo "端口 ${PORT} 未被占用。"
echo "工作目录已切换至: ${SCRIPT_DIR}"
PYTHON_BIN=""
if [[ -x "${SCRIPT_DIR}/venv/bin/python" ]]; then
PYTHON_BIN="${SCRIPT_DIR}/venv/bin/python"
elif [[ -x "${SCRIPT_DIR}/.venv/bin/python" ]]; then
PYTHON_BIN="${SCRIPT_DIR}/.venv/bin/python"
elif command -v python3 >/dev/null 2>&1; then
PYTHON_BIN="$(command -v python3)"
elif command -v python >/dev/null 2>&1; then
PYTHON_BIN="$(command -v python)"
else
echo "错误: 未找到可用的 Python请先安装 Python 3.9+。"
exit 1
fi
echo "使用 Python: ${PYTHON_BIN}"
"${PYTHON_BIN}" - <<'PY'
import sys
major, minor = sys.version_info[:2]
if (major, minor) < (3, 9):
print(f"错误: 当前 Python 版本为 {major}.{minor},项目至少需要 Python 3.9。")
print("原因: requirement.txt 中的 Flask 3.x、pandas 2.x 等依赖不支持 Python 3.6。")
raise SystemExit(1)
print(f"Python 版本检查通过: {major}.{minor}")
PY
if [[ ! -f "requirement.txt" ]]; then
echo "错误: 未找到 requirement.txt。"
exit 1
fi
echo "检查关键依赖 ..."
if ! "${PYTHON_BIN}" - <<'PY'
import importlib.util
modules = ("flask", "flask_cors", "pymysql", "werkzeug")
missing = [name for name in modules if importlib.util.find_spec(name) is None]
if missing:
print("缺少依赖: " + ", ".join(missing))
raise SystemExit(1)
print("关键依赖已安装。")
PY
then
echo "开始安装 requirement.txt ..."
"${PYTHON_BIN}" -m pip install --upgrade pip
"${PYTHON_BIN}" -m pip install -r requirement.txt
fi
if [[ -f "${PID_FILE}" ]]; then
OLD_PID="$(cat "${PID_FILE}" 2>/dev/null || true)"
if [[ -n "${OLD_PID}" ]] && kill -0 "${OLD_PID}" 2>/dev/null; then
echo "错误: 检测到服务已在运行PID=${OLD_PID}"
exit 1
fi
rm -f "${PID_FILE}"
fi
echo "启动 Flask 服务 ${HOST}:${PORT} ..."
nohup "${PYTHON_BIN}" "${APP_MODULE}" >> "${LOG_FILE}" 2>&1 &
NEW_PID=$!
echo "${NEW_PID}" > "${PID_FILE}"
sleep 2
if kill -0 "${NEW_PID}" 2>/dev/null; then
echo "启动成功PID=${NEW_PID}"
echo "日志文件: ${SCRIPT_DIR}/${LOG_FILE}"
else
echo "错误: 服务启动失败,请检查日志 ${SCRIPT_DIR}/${LOG_FILE}"
exit 1
fi

View File

@@ -42,9 +42,13 @@ def get_current_admin_role():
)
row = cur.fetchone()
conn.close()
if not row or not row.get('is_admin'):
return None, row
role = row.get('role') or ('super_admin' if row.get('created_by_id') is None else 'admin')
if not row:
return None, None
role = (row.get('role') or '').strip().lower()
if not role:
role = 'super_admin' if row.get('is_admin') and row.get('created_by_id') is None else (
'admin' if row.get('is_admin') else 'normal'
)
return role, row
except Exception:
return None, None
@@ -56,7 +60,14 @@ def is_current_user_admin():
def _is_ajax_request():
return request.headers.get('X-Requested-With') == 'XMLHttpRequest'
if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
return True
if request.path.startswith('/api/'):
return True
accept = (request.headers.get('Accept') or '').lower()
if 'application/json' in accept:
return True
return False
def login_required(f):

View File

@@ -769,10 +769,10 @@
<option value="ES">西班牙</option>
</select>
</div>
<div class="form-group" style="min-width:260px;flex:1;">
<label>ASIN</label>
<div class="form-group" style="min-width:320px;flex:1;">
<label>ASIN / 最低价</label>
<div id="skipPriceAsinInputs" style="display:flex;flex-direction:column;gap:8px;">
<div style="color:#999;font-size:13px;">请选择国家后输入对应 ASIN</div>
<div style="color:#999;font-size:13px;">请选择国家后输入对应 ASIN 和最低价</div>
</div>
</div>
<button class="btn" id="btnCreateSkipPriceAsin">新增 ASIN</button>
@@ -966,7 +966,7 @@
<label>组员</label>
<select id="shopManageGroupMemberSelect" multiple size="6" style="min-height:140px;">
</select>
<div style="color:#888;font-size:12px;margin-top:6px;">可添加当前组长创建的普通员工账号,按住 Ctrl 或 Command 可多选。</div>
<div id="shopManageGroupMemberHelp" style="color:#888;font-size:12px;margin-top:6px;">可添加当前组长创建的普通员工账号,按住 Ctrl 或 Command 可多选。</div>
</div>
</div>
<div style="display:flex;gap:8px;justify-content:flex-end;margin-bottom:12px;">
@@ -1034,6 +1034,35 @@
</div>
</div>
<div class="modal-mask" id="editSkipPriceAsinModal">
<div class="modal">
<h3>编辑跳过跟价 ASIN</h3>
<input type="hidden" id="editSkipPriceAsinId">
<input type="hidden" id="editSkipPriceAsinCountry">
<div class="form-group">
<label>店铺名</label>
<input type="text" id="editSkipPriceAsinShopName" readonly style="background:#f5f5f5;">
</div>
<div class="form-group">
<label>国家</label>
<input type="text" id="editSkipPriceAsinCountryLabel" readonly style="background:#f5f5f5;">
</div>
<div class="form-group">
<label>ASIN</label>
<input type="text" id="editSkipPriceAsinValue" placeholder="请输入 ASIN">
</div>
<div class="form-group">
<label>最低价</label>
<input type="number" id="editSkipPriceMinimumPrice" min="0" step="0.01" placeholder="请输入最低价,可留空">
</div>
<p class="msg" id="msgEditSkipPriceAsin"></p>
<div style="margin-top:16px;display:flex;gap:8px;">
<button class="btn" id="btnSaveSkipPriceAsin">保存</button>
<button class="btn btn-secondary" id="btnCloseEditSkipPriceAsin">取消</button>
</div>
</div>
</div>
<div class="modal-mask" id="editShopManageModal">
<div class="modal">
<h3>编辑店铺</h3>
@@ -1655,22 +1684,6 @@
document.getElementById('historyListBody').innerHTML = '<tr><td colspan="5" class="empty-tip">请求失败</td></tr>';
});
}
function loadUserOptions() {
fetch('/api/admin/users?page=1&page_size=999')
.then(function (r) { return r.json(); })
.then(function (res) {
var sel = document.getElementById('filterUser');
var cur = sel.value;
sel.innerHTML = '<option value="">全部用户</option>';
(res.items || []).forEach(function (u) {
var opt = document.createElement('option');
opt.value = u.id;
opt.textContent = u.username + ' (' + roleLabel(u.role || 'normal') + ')';
sel.appendChild(opt);
});
sel.value = cur || '';
});
}
var shopManageAllUsers = [];
function getEligibleShopManageGroupUsers(leaderUserId) {
var canViewAllMembers = currentUserRole === 'super_admin';
@@ -1683,15 +1696,29 @@
}
function refreshShopManageGroupMemberSelect(leaderUserId, selectedUserIds) {
var sel = document.getElementById('shopManageGroupMemberSelect');
var helpEl = document.getElementById('shopManageGroupMemberHelp');
if (!sel) return;
var selectedMap = {};
(selectedUserIds || []).forEach(function (id) {
selectedMap[String(id)] = true;
});
var options = getEligibleShopManageGroupUsers(leaderUserId).map(function (u) {
var eligibleUsers = getEligibleShopManageGroupUsers(leaderUserId);
var options = eligibleUsers.map(function (u) {
return '<option value="' + u.id + '"' + (selectedMap[String(u.id)] ? ' selected' : '') + '>' + (u.username || '') + '</option>';
});
sel.innerHTML = options.join('');
if (options.length) {
sel.innerHTML = options.join('');
if (helpEl) {
helpEl.textContent = '可添加当前组长创建的普通员工账号,按住 Ctrl 或 Command 可多选。';
}
return;
}
sel.innerHTML = '<option value="" disabled>当前组长暂无可添加组员</option>';
if (helpEl) {
helpEl.textContent = currentUserRole === 'normal'
? '普通账号没有下属普通员工时,这里会为空;当前账号只能作为组长使用。'
: '当前组长名下暂无可添加的普通员工账号。';
}
}
function setShopManageGroupLeader(leaderUserId, leaderUsername, selectedUserIds) {
var leaderIdEl = document.getElementById('shopManageGroupLeaderUserId');
@@ -1712,6 +1739,9 @@
return fetch('/api/admin/users?page=1&page_size=999')
.then(function (r) { return r.json(); })
.then(function (res) {
if (!res.success) {
throw new Error(res.error || '加载用户失败');
}
var items = res.items || [];
shopManageAllUsers = items.map(function (u) {
return {
@@ -1736,6 +1766,17 @@
sel.appendChild(opt);
});
sel.value = cur || '';
})
.catch(function () {
shopManageAllUsers = [];
var sel = document.getElementById('filterUser');
if (sel) {
sel.innerHTML = '<option value="">全部用户</option>';
}
refreshShopManageGroupMemberSelect(
document.getElementById('shopManageGroupLeaderUserId') ? document.getElementById('shopManageGroupLeaderUserId').value : '',
[]
);
});
}
document.getElementById('btnFilterHistory').onclick = function () { loadHistory(1); };
@@ -2184,6 +2225,7 @@
// ========== 店铺管理 ==========
var shopManagePage = 1, shopManagePageSize = 15;
var shopManageGroups = [];
var currentShopManageGroupGrantRoutes = [];
function buildShopManageQuery(page) {
var query = 'page=' + (page || 1) + '&page_size=' + shopManagePageSize;
@@ -2313,6 +2355,18 @@
refreshShopManageGroupMemberSelect(currentUserId, []);
}
function setShopManageGroupGrantRoutes(routes) {
currentShopManageGroupGrantRoutes = Array.isArray(routes) ? routes.slice() : [];
}
function updateShopManageGroupButtonsAccess() {
var canManageGroups = !!currentUserId;
['btnManageShopGroups', 'btnManageShopGroupsFromEdit', 'btnManageSkipPriceAsinGroups'].forEach(function (id) {
var btn = document.getElementById(id);
if (btn) btn.style.display = canManageGroups ? '' : 'none';
});
}
function openShopManageGroupModal() {
resetShopManageGroupForm();
loadUserOptions()
@@ -2331,10 +2385,13 @@
}
tbody.innerHTML = shopManageGroups.map(function (item, index) {
var memberNames = Array.isArray(item.member_usernames) ? item.member_usernames.join('、') : '';
var canEdit = currentUserRole === 'super_admin' || String(item.leader_user_id || '') === String(currentUserId || '');
var actionHtml = canEdit
? ('<button class="btn btn-sm" data-shop-group-edit="' + item.id + '">编辑</button> ' +
'<button class="btn btn-sm btn-danger" data-shop-group-delete="' + item.id + '" data-shop-group-name="' + (item.group_name || '').replace(/"/g, '&quot;') + '">删除</button>')
: '<span style="color:#999;">-</span>';
return '<tr><td>' + (index + 1) + '</td><td>' + (item.group_name || '') + '</td><td>' + (item.leader_username || '') + '</td><td>' + (item.member_count || 0) + '</td><td>' + (memberNames || '-') + '</td><td>' + (item.created_at || '') + '</td><td>' + (item.updated_at || '') + '</td><td>' +
'<button class="btn btn-sm" data-shop-group-edit="' + item.id + '">编辑</button> ' +
'<button class="btn btn-sm btn-danger" data-shop-group-delete="' + item.id + '" data-shop-group-name="' + (item.group_name || '').replace(/"/g, '&quot;') + '">删除</button>' +
'</td></tr>';
actionHtml + '</td></tr>';
}).join('');
document.querySelectorAll('[data-shop-group-edit]').forEach(function (btn) {
@@ -2378,9 +2435,18 @@
});
}
document.getElementById('btnManageShopGroups').onclick = openShopManageGroupModal;
document.getElementById('btnManageShopGroupsFromEdit').onclick = openShopManageGroupModal;
document.getElementById('btnManageSkipPriceAsinGroups').onclick = openShopManageGroupModal;
document.getElementById('btnManageShopGroups').onclick = function () {
setShopManageGroupGrantRoutes(['shop-manage']);
openShopManageGroupModal();
};
document.getElementById('btnManageShopGroupsFromEdit').onclick = function () {
setShopManageGroupGrantRoutes(['shop-manage']);
openShopManageGroupModal();
};
document.getElementById('btnManageSkipPriceAsinGroups').onclick = function () {
setShopManageGroupGrantRoutes(['skip-price-asin']);
openShopManageGroupModal();
};
document.getElementById('btnSearchShopManage').onclick = function () {
loadShopManage(1);
};
@@ -2411,7 +2477,11 @@
fetch(url, {
method: method,
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ group_name: groupName, member_user_ids: memberUserIds })
body: JSON.stringify({
group_name: groupName,
member_user_ids: memberUserIds,
grant_menu_routes: currentShopManageGroupGrantRoutes
})
})
.then(function (r) { return r.json(); })
.then(function (res) {
@@ -2520,11 +2590,11 @@
var skipPriceAsinPage = 1, skipPriceAsinPageSize = 15;
var chooseSkipPriceAsinShopPage = 1, chooseSkipPriceAsinShopPageSize = 10;
var skipPriceCountryColumns = [
{ code: 'DE', field: 'asin_de', label: '德国' },
{ code: 'UK', field: 'asin_uk', label: '英国' },
{ code: 'FR', field: 'asin_fr', label: '法国' },
{ code: 'IT', field: 'asin_it', label: '意大利' },
{ code: 'ES', field: 'asin_es', label: '西班牙' }
{ code: 'DE', field: 'asin_de', minimumPriceField: 'minimum_price_de', label: '德国' },
{ code: 'UK', field: 'asin_uk', minimumPriceField: 'minimum_price_uk', label: '英国' },
{ code: 'FR', field: 'asin_fr', minimumPriceField: 'minimum_price_fr', label: '法国' },
{ code: 'IT', field: 'asin_it', minimumPriceField: 'minimum_price_it', label: '意大利' },
{ code: 'ES', field: 'asin_es', minimumPriceField: 'minimum_price_es', label: '西班牙' }
];
function buildChooseSkipPriceAsinShopQuery(page) {
var query = 'page=' + (page || 1) + '&page_size=' + chooseSkipPriceAsinShopPageSize;
@@ -2607,41 +2677,68 @@
return option.value;
});
}
function formatSkipPriceMinimumPrice(value) {
if (value === null || value === undefined || value === '') return '';
var num = Number(value);
if (!isFinite(num)) return String(value);
return num.toFixed(2);
}
function getSkipPriceCountryLabel(countryCode) {
var country = skipPriceCountryColumns.find(function (item) { return item.code === countryCode; });
return country ? country.label : countryCode;
}
function renderSkipPriceAsinInputs() {
var container = document.getElementById('skipPriceAsinInputs');
var selectedCountries = getSelectedSkipPriceCountries();
var existingValues = {};
var existingMinimumPrices = {};
container.querySelectorAll('[data-skip-price-country-input]').forEach(function (input) {
existingValues[input.getAttribute('data-skip-price-country-input')] = input.value;
});
container.querySelectorAll('[data-skip-price-country-minimum-price-input]').forEach(function (input) {
existingMinimumPrices[input.getAttribute('data-skip-price-country-minimum-price-input')] = input.value;
});
if (!selectedCountries.length) {
container.innerHTML = '<div style="color:#999;font-size:13px;">请选择国家后输入 ASIN</div>';
container.innerHTML = '<div style="color:#999;font-size:13px;">请选择国家后输入 ASIN 和最低价</div>';
return;
}
container.innerHTML = selectedCountries.map(function (countryCode) {
var country = skipPriceCountryColumns.find(function (item) { return item.code === countryCode; });
var label = country ? country.label : countryCode;
var label = getSkipPriceCountryLabel(countryCode);
var value = existingValues[countryCode] || '';
var minimumPrice = existingMinimumPrices[countryCode] || '';
return '<div style="display:flex;align-items:center;gap:8px;flex-wrap:wrap;">' +
'<span style="min-width:56px;color:#555;">' + label + '</span>' +
'<input type="text" data-skip-price-country-input="' + countryCode + '" value="' + value.replace(/"/g, '&quot;') + '" placeholder="请输入' + label + ' ASIN" style="flex:1;min-width:180px;">' +
'<input type="number" data-skip-price-country-minimum-price-input="' + countryCode + '" value="' + minimumPrice.replace(/"/g, '&quot;') + '" min="0" step="0.01" placeholder="最低价" style="width:140px;">' +
'</div>';
}).join('');
}
function collectSkipPriceAsinMappings(countries) {
var mappings = {};
var asinMappings = {};
var minimumPriceMappings = {};
for (var i = 0; i < countries.length; i++) {
var countryCode = countries[i];
var input = document.querySelector('[data-skip-price-country-input="' + countryCode + '"]');
var minimumPriceInput = document.querySelector('[data-skip-price-country-minimum-price-input="' + countryCode + '"]');
var asin = input ? (input.value || '').trim().toUpperCase() : '';
if (!asin) {
var country = skipPriceCountryColumns.find(function (item) { return item.code === countryCode; });
var label = country ? country.label : countryCode;
var label = getSkipPriceCountryLabel(countryCode);
throw new Error(label + ' ASIN 不能为空');
}
mappings[countryCode] = asin;
var minimumPrice = minimumPriceInput ? (minimumPriceInput.value || '').trim() : '';
if (minimumPrice) {
var minimumPriceNumber = Number(minimumPrice);
if (!isFinite(minimumPriceNumber) || minimumPriceNumber < 0) {
throw new Error(getSkipPriceCountryLabel(countryCode) + ' 最低价格式不正确');
}
minimumPriceMappings[countryCode] = minimumPrice;
}
asinMappings[countryCode] = asin;
}
return mappings;
return {
asinMappings: asinMappings,
minimumPriceMappings: minimumPriceMappings
};
}
function buildSkipPriceAsinQuery(page) {
var query = 'page=' + (page || 1) + '&page_size=' + skipPriceAsinPageSize;
@@ -2654,37 +2751,49 @@
return query;
}
function renderSkipPriceAsinCell(item, country) {
var value = item[country.field] || '';
if (!value) return '<span style="color:#999;">-</span>';
var asinValue = item[country.field] || '';
var minimumPriceValue = formatSkipPriceMinimumPrice(item[country.minimumPriceField]);
var hasValue = !!asinValue || !!minimumPriceValue;
var infoHtml = hasValue
? ('<div style="display:flex;flex-direction:column;gap:4px;min-width:0;">' +
'<span>' + (asinValue || '<span style="color:#999;">-</span>') + '</span>' +
'<span style="color:#666;font-size:12px;">最低价:' + (minimumPriceValue || '-') + '</span>' +
'</div>')
: '<span style="color:#999;">-</span>';
var deleteHtml = hasValue
? ('<button class="btn btn-sm btn-danger" data-skip-price-asin-delete="' + item.id + '" data-country="' + country.code + '" data-shop-name="' + (item.shop_name || '').replace(/"/g, '&quot;') + '">删除</button>')
: '';
return '<div style="display:flex;align-items:center;gap:8px;flex-wrap:wrap;">' +
'<span>' + value + '</span>' +
'<button class="btn btn-sm" data-skip-price-asin-edit="' + item.id + '" data-country="' + country.code + '" data-asin="' + value.replace(/"/g, '&quot;') + '" data-shop-name="' + (item.shop_name || '').replace(/"/g, '&quot;') + '">编辑</button>' +
'<button class="btn btn-sm btn-danger" data-skip-price-asin-delete="' + item.id + '" data-country="' + country.code + '" data-shop-name="' + (item.shop_name || '').replace(/"/g, '&quot;') + '">删除</button>' +
infoHtml +
'<button class="btn btn-sm" data-skip-price-asin-edit="' + item.id + '" data-country="' + country.code + '" data-asin="' + asinValue.replace(/"/g, '&quot;') + '" data-minimum-price="' + minimumPriceValue.replace(/"/g, '&quot;') + '" data-shop-name="' + (item.shop_name || '').replace(/"/g, '&quot;') + '">编辑</button>' +
deleteHtml +
'</div>';
}
function openEditSkipPriceAsinModal(itemId, countryCode, shopName, asinValue, minimumPriceValue) {
document.getElementById('editSkipPriceAsinId').value = itemId || '';
document.getElementById('editSkipPriceAsinCountry').value = countryCode || '';
document.getElementById('editSkipPriceAsinShopName').value = shopName || '';
document.getElementById('editSkipPriceAsinCountryLabel').value = getSkipPriceCountryLabel(countryCode || '');
document.getElementById('editSkipPriceAsinValue').value = asinValue || '';
document.getElementById('editSkipPriceMinimumPrice').value = minimumPriceValue || '';
document.getElementById('msgEditSkipPriceAsin').textContent = '';
document.getElementById('msgEditSkipPriceAsin').className = 'msg';
document.getElementById('editSkipPriceAsinModal').classList.add('show');
}
function bindSkipPriceAsinActions() {
document.querySelectorAll('[data-skip-price-asin-edit]').forEach(function (btn) {
btn.onclick = function () {
var shopName = (btn.dataset.shopName || '').replace(/&quot;/g, '"');
var countryCode = btn.dataset.country || '';
var currentAsin = (btn.dataset.asin || '').replace(/&quot;/g, '"');
var nextAsin = prompt('请输入店铺“' + shopName + '”在 ' + countryCode + ' 的 ASIN', currentAsin);
if (nextAsin === null) return;
nextAsin = (nextAsin || '').trim();
if (!nextAsin) {
alert('ASIN 不能为空');
return;
}
fetch('/api/admin/skip-price-asin/' + btn.dataset.skipPriceAsinEdit + '/country/' + countryCode, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ asin: nextAsin })
})
.then(function (r) { return r.json(); })
.then(function (res) {
if (res.success) loadSkipPriceAsin(skipPriceAsinPage);
else alert(res.error || '保存失败');
});
var currentMinimumPrice = (btn.dataset.minimumPrice || '').replace(/&quot;/g, '"');
openEditSkipPriceAsinModal(
btn.dataset.skipPriceAsinEdit,
countryCode,
shopName,
currentAsin,
currentMinimumPrice
);
};
});
document.querySelectorAll('[data-skip-price-asin-delete]').forEach(function (btn) {
@@ -2733,7 +2842,6 @@
document.getElementById('skipPriceAsinListBody').innerHTML = '<tr><td colspan="8" class="empty-tip">请求失败</td></tr>';
});
}
document.getElementById('btnManageSkipPriceAsinGroups').onclick = openShopManageGroupModal;
document.getElementById('btnChooseSkipPriceAsinShop').onclick = openChooseSkipPriceAsinShopModal;
document.getElementById('btnSearchChooseSkipPriceAsinShop').onclick = function () {
loadChooseSkipPriceAsinShops(1);
@@ -2750,6 +2858,58 @@
document.getElementById('btnCloseChooseSkipPriceAsinShopModal').onclick = function () {
document.getElementById('chooseSkipPriceAsinShopModal').classList.remove('show');
};
document.getElementById('btnCloseEditSkipPriceAsin').onclick = function () {
document.getElementById('editSkipPriceAsinModal').classList.remove('show');
};
document.getElementById('btnSaveSkipPriceAsin').onclick = function () {
var itemId = (document.getElementById('editSkipPriceAsinId').value || '').trim();
var countryCode = (document.getElementById('editSkipPriceAsinCountry').value || '').trim();
var asin = (document.getElementById('editSkipPriceAsinValue').value || '').trim().toUpperCase();
var minimumPrice = (document.getElementById('editSkipPriceMinimumPrice').value || '').trim();
var msgEl = document.getElementById('msgEditSkipPriceAsin');
msgEl.textContent = '';
msgEl.className = 'msg';
if (!itemId || !countryCode) {
msgEl.textContent = '缺少编辑记录';
msgEl.className = 'msg err';
return;
}
if (!asin) {
msgEl.textContent = 'ASIN 不能为空';
msgEl.className = 'msg err';
return;
}
if (minimumPrice) {
var minimumPriceNumber = Number(minimumPrice);
if (!isFinite(minimumPriceNumber) || minimumPriceNumber < 0) {
msgEl.textContent = '最低价格式不正确';
msgEl.className = 'msg err';
return;
}
}
fetch('/api/admin/skip-price-asin/' + itemId + '/country/' + countryCode, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
asin: asin,
minimum_price: minimumPrice || null
})
})
.then(function (r) { return r.json(); })
.then(function (res) {
if (!res.success) {
msgEl.textContent = res.error || '保存失败';
msgEl.className = 'msg err';
return;
}
document.getElementById('editSkipPriceAsinModal').classList.remove('show');
loadSkipPriceAsin(skipPriceAsinPage);
})
.catch(function () {
msgEl.textContent = '请求失败';
msgEl.className = 'msg err';
});
};
document.getElementById('skipPriceAsinCountries').addEventListener('change', renderSkipPriceAsinInputs);
document.getElementById('btnSearchSkipPriceAsin').onclick = function () {
loadSkipPriceAsin(1);
@@ -2779,8 +2939,11 @@
return;
}
var asinMappings = {};
var minimumPriceMappings = {};
try {
asinMappings = collectSkipPriceAsinMappings(countries);
var mappings = collectSkipPriceAsinMappings(countries);
asinMappings = mappings.asinMappings || {};
minimumPriceMappings = mappings.minimumPriceMappings || {};
} catch (err) {
msgEl.textContent = err.message || '请输入 ASIN';
msgEl.className = 'msg err';
@@ -2799,7 +2962,8 @@
shop_name: shopName,
countries: countries,
asin: fallbackAsin,
asin_mappings: asinMappings
asin_mappings: asinMappings,
minimum_price_mappings: minimumPriceMappings
})
})
.then(function (r) { return r.json(); })
@@ -3245,13 +3409,19 @@
return;
}
var item = res.item || {};
currentUserId = item.id || currentUserId;
currentUserRole = item.role || currentUserRole;
currentUserUsername = item.username || currentUserUsername;
nameEl.textContent = item.username || '管理员';
if (item.role) {
roleEl.textContent = item.role === 'super_admin' ? '超级管理员' : '管理员';
roleEl.textContent = item.role === 'super_admin'
? '超级管理员'
: (item.role === 'admin' ? '管理员' : '普通账号');
roleEl.style.display = 'inline-block';
} else {
roleEl.style.display = 'none';
}
updateShopManageGroupButtonsAccess();
})
.catch(function () {
fetch('/api/auth/check')
@@ -3259,10 +3429,12 @@
.then(function (res) {
document.getElementById('adminCurrentUsername').textContent = res && res.logged_in ? '当前用户' : '未登录';
document.getElementById('adminCurrentUserRole').style.display = 'none';
updateShopManageGroupButtonsAccess();
})
.catch(function () {
document.getElementById('adminCurrentUsername').textContent = '当前用户';
document.getElementById('adminCurrentUserRole').style.display = 'none';
updateShopManageGroupButtonsAccess();
});
});
}

View File

@@ -358,6 +358,7 @@ const pendingQueue = ref<PatrolDeleteShopQueueItem[]>([]);
const activeTaskId = ref<number | null>(null);
const activeQueueItem = ref<PatrolDeleteShopQueueItem | null>(null);
const queueWorkerRunning = ref(false);
const autoQueueEnabled = ref(false);
let historyPollTimer: number | null = null;
const matchedRunnableItems = computed(() =>
@@ -774,6 +775,11 @@ async function runMatch() {
const data = await matchPatrolDeleteShops(names);
matchedItems.value = mergeMatchedItems(matchedItems.value, data.items || []);
saveMatchedItems();
if (autoQueueEnabled.value) {
pendingQueue.value = mergeQueueItems(pendingQueue.value, matchedRunnableItems.value);
saveQueueState();
void processQueue();
}
ElMessage.success(`本次返回 ${data.items?.length || 0} 条匹配结果`);
} catch (error) {
ElMessage.error(error instanceof Error ? error.message : "匹配失败");
@@ -963,6 +969,7 @@ async function processQueue() {
}
async function pushToPythonQueue() {
autoQueueEnabled.value = true;
const runnable = matchedRunnableItems.value;
if (!runnable.length) {
ElMessage.warning("请先匹配可用店铺");
@@ -1016,6 +1023,7 @@ onMounted(async () => {
await Promise.all([loadCandidates(), loadDashboard(), loadHistory()]);
if (activeTaskId.value || pendingQueue.value.length) {
autoQueueEnabled.value = true;
queuePushResult.value =
activeTaskId.value != null
? `检测到未完成队列,继续等待任务 ${activeTaskId.value} 完成并自动接续后续店铺`

View File

@@ -312,6 +312,8 @@ const matchedItems = ref<PriceTrackShopQueueItem[]>([])
const adding = ref(false)
const matching = ref(false)
const pushing = ref(false)
const queueWorkerRunning = ref(false)
const autoQueueEnabled = ref(false)
const queuePushResult = ref('')
const queuePayloadText = ref('')
const matchAsinRowsByCountry = ref<Record<string, PriceTrackAsinParsedRow[]>>({})
@@ -787,6 +789,9 @@ async function runMatch() {
}
matchedItems.value = [...nextByShop.values()]
saveMatchedItemsToStorage()
if (autoQueueEnabled.value) {
void processMatchedQueue()
}
ElMessage.success(`匹配 ${batch.length} 个店铺`)
} catch (e) {
ElMessage.error(e instanceof Error ? e.message : '匹配失败')
@@ -813,6 +818,19 @@ function rowKeyForMatch(row: { shopName?: string; shopId?: number | string | nul
return `${(row.shopName || '').trim()}\u0001${row.shopId ?? ''}`
}
function buildSkipAsinDeletePolicy() {
const enabled = statusModeEnabled.value && !asinModeEnabled.value
return {
enabled,
mode: enabled ? 'DELETE_WHEN_PRICE_BELOW_MINIMUM' : 'NONE',
deleteWhenPriceBelowMinimum: enabled,
compareField: 'price',
thresholdField: 'minimumPrice',
target: 'skip_asin',
source: 'frontend-price-track',
}
}
function removeMatchedRowsLocally(rows: Array<{ shopName?: string; shopId?: number | string | null }>) {
if (!rows.length) return
const keys = new Set(rows.map((row) => rowKeyForMatch(row)))
@@ -903,6 +921,9 @@ async function pushToPythonQueueLegacy() {
// 店铺简要信息(用于展示)
shop_names: matchedRows.map((i) => i.shopName).filter(Boolean),
country_codes: [...orderedCountryCodes.value],
skip_asin_delete_policy: buildSkipAsinDeletePolicy(),
delete_skip_asin_when_price_below_minimum: statusModeEnabled.value && !asinModeEnabled.value,
deleteSkipAsinWhenPriceBelowMinimum: statusModeEnabled.value && !asinModeEnabled.value,
},
}
queuePayloadText.value = JSON.stringify(queuePayload, null, 2)
@@ -930,6 +951,9 @@ function buildQueuePayload(taskVo: PriceTrackCreateTaskVo, row: PriceTrackShopQu
const taskItem = taskVo.items?.[0]
const shopName = (row.shopName || '').trim()
const skipAsinsByCountry = row.skipAsins || taskVo.skipAsinsByCountry || {}
const skipAsinDetailsByCountry = taskVo.skipAsinDetailsByCountry || {}
const minimumPriceByCountryAndAsin = taskVo.minimumPriceByCountryAndAsin || {}
const skipAsinDeletePolicy = buildSkipAsinDeletePolicy()
const resultId = taskItem?.resultId ?? null
const loopRunId = taskItem?.loopRunId ?? null
const roundIndex = taskItem?.roundIndex ?? null
@@ -963,6 +987,11 @@ function buildQueuePayload(taskVo: PriceTrackCreateTaskVo, row: PriceTrackShopQu
mode: statusModeEnabled.value ? 'status' : 'asin',
skip_asins: skipAsinsByCountry,
skip_asins_by_country: skipAsinsByCountry,
skip_asin_details_by_country: skipAsinDetailsByCountry,
minimum_price_by_country_and_asin: minimumPriceByCountryAndAsin,
skip_asin_delete_policy: skipAsinDeletePolicy,
delete_skip_asin_when_price_below_minimum: skipAsinDeletePolicy.deleteWhenPriceBelowMinimum,
deleteSkipAsinWhenPriceBelowMinimum: skipAsinDeletePolicy.deleteWhenPriceBelowMinimum,
asin_rows_by_country: Object.keys(matchAsinRowsByCountry.value).length
? matchAsinRowsByCountry.value
: (taskVo.asinRowsByCountry || {}),
@@ -1039,6 +1068,18 @@ async function waitForTaskTerminal(taskId: number) {
}
async function pushToPythonQueue() {
autoQueueEnabled.value = true
await processMatchedQueue()
}
function nextMatchedQueueItem() {
return matchedItems.value.find((item) => item.matched)
}
async function processMatchedQueue() {
if (queueWorkerRunning.value) {
return
}
if (!hasValidMode.value) {
ElMessage.warning('请至少选择一个跟价模式')
return
@@ -1057,13 +1098,19 @@ async function pushToPythonQueue() {
ElMessage.warning('无可用匹配店铺')
return
}
queueWorkerRunning.value = true
pushing.value = true
queuePayloadText.value = ''
let successCount = 0
let failedCount = 0
try {
for (let index = 0; index < matchedRows.length; index += 1) {
const row = matchedRows[index]
let index = 0
while (autoQueueEnabled.value) {
const row = nextMatchedQueueItem()
if (!row) {
break
}
index += 1
try {
let attempt = 0
const taskVo = await (async () => {
@@ -1107,37 +1154,38 @@ async function pushToPythonQueue() {
failedCount += 1
queuePushResult.value = '任务 ' + taskVo.taskId + ' 推送失败,已自动继续下一条:' + (pushResult?.error || '未知错误')
ElMessage.error(queuePushResult.value)
removeMatchedRowsLocally([row])
continue
}
addPollingTask(taskVo.taskId)
scheduleNextPoll(true)
removeMatchedRowsLocally([row])
queuePushResult.value = matchedRows.length > 1
? '任务 ' + taskVo.taskId + ' 已入队,等待完成后继续下一条(' + (index + 1) + '/' + matchedRows.length + ''
: '任务 ' + taskVo.taskId + ' 已入队,等待执行完成'
queuePushResult.value = '任务 ' + taskVo.taskId + ' 已入队,等待执行完成'
const finalStatus = await waitForTaskTerminal(taskVo.taskId)
if (finalStatus !== 'SUCCESS') {
failedCount += 1
queuePushResult.value = '任务 ' + taskVo.taskId + ' 执行失败,已自动继续下一条' + (index + 1) + '/' + matchedRows.length + ''
queuePushResult.value = '任务 ' + taskVo.taskId + ' 执行失败,已自动继续下一条'
ElMessage.error(queuePushResult.value)
continue
}
successCount += 1
queuePushResult.value = index + 1 < matchedRows.length
? '任务 ' + taskVo.taskId + ' 已完成,继续推送下一条(' + (index + 1) + '/' + matchedRows.length + ''
: '任务 ' + taskVo.taskId + ' 已完成'
queuePushResult.value = '任务 ' + taskVo.taskId + ' 已完成'
} catch (error) {
failedCount += 1
queuePushResult.value = '第 ' + (index + 1) + '/' + matchedRows.length + ' 条任务处理失败,已自动继续下一条:' + (error instanceof Error ? error.message : '未知错误')
queuePushResult.value = '第 ' + index + ' 条任务处理失败,已自动继续下一条:' + (error instanceof Error ? error.message : '未知错误')
ElMessage.error(queuePushResult.value)
removeMatchedRowsLocally([row])
continue
}
}
ElMessage.success('店铺任务推送已完成:成功 ' + successCount + ' 条,失败 ' + failedCount + ' 条')
if (successCount > 0 || failedCount > 0) {
ElMessage.success('店铺任务推送已完成:成功 ' + successCount + ' 条,失败 ' + failedCount + ' 条')
}
} catch (e) {
queuePushResult.value = e instanceof Error ? e.message : '推送异常'
ElMessage.error(queuePushResult.value)
} finally {
queueWorkerRunning.value = false
pushing.value = false
}
}

View File

@@ -229,6 +229,8 @@ const matchedItems = ref<ProductRiskShopQueueItem[]>([])
const adding = ref(false)
const matching = ref(false)
const pushing = ref(false)
const queueWorkerRunning = ref(false)
const autoQueueEnabled = ref(false)
const queuePushResult = ref('')
const queuePayloadText = ref('')
@@ -897,6 +899,9 @@ async function runMatch() {
const batch = res.items || []
matchedItems.value = mergeMatchedItems(matchedItems.value, batch)
saveMatchedItemsToStorage()
if (autoQueueEnabled.value) {
void processMatchedQueue()
}
if (!batch.length) {
ElMessage.warning('未返回匹配结果')
} else {
@@ -982,6 +987,18 @@ function formatMatchRemark(row: ProductRiskShopQueueItem) {
}
async function pushToPythonQueue() {
autoQueueEnabled.value = true
await processMatchedQueue()
}
function nextMatchedQueueItem() {
return matchedItems.value.find((item) => item.matched)
}
async function processMatchedQueue() {
if (queueWorkerRunning.value) {
return
}
if (!matchedItems.value.length) {
ElMessage.warning('请先匹配店铺')
return
@@ -996,13 +1013,19 @@ async function pushToPythonQueue() {
ElMessage.warning('没有已匹配的店铺可推送')
return
}
queueWorkerRunning.value = true
pushing.value = true
queuePayloadText.value = ''
let successCount = 0
let failedCount = 0
try {
for (let i = 0; i < toPush.length; i++) {
const item = toPush[i]
let index = 0
while (autoQueueEnabled.value) {
const item = nextMatchedQueueItem()
if (!item) {
break
}
index += 1
try {
const created = await createProductRiskTaskWithRetry([item])
const taskId = created.taskId
@@ -1031,39 +1054,40 @@ async function pushToPythonQueue() {
if (!pushResult?.success) {
removePollingTask(taskId)
failedCount += 1
queuePushResult.value = '第 ' + (i + 1) + '/' + toPush.length + ' 条推送失败:' + (pushResult?.error || '未知错误') + ',已自动继续下一条'
queuePushResult.value = '第 ' + index + ' 条推送失败:' + (pushResult?.error || '未知错误') + ',已自动继续下一条'
ElMessage.error(queuePushResult.value)
removeMatchedRowsLocally([item])
continue
}
removeMatchedRowsLocally([item])
queuePushResult.value = toPush.length > 1
? '任务 ' + taskId + ':第 ' + (i + 1) + '/' + toPush.length + ' 条店铺已入队,等待执行完成...'
: '任务 ' + taskId + ' 已入队,等待执行完成...'
queuePushResult.value = '任务 ' + taskId + ' 已入队,等待执行完成...'
const finalStatus = await waitForTaskTerminal(taskId)
if (finalStatus !== 'SUCCESS') {
failedCount += 1
queuePushResult.value = '任务 ' + taskId + ' 执行失败,已自动继续下一条' + (i + 1) + '/' + toPush.length + ''
queuePushResult.value = '任务 ' + taskId + ' 执行失败,已自动继续下一条'
ElMessage.error(queuePushResult.value)
continue
}
successCount += 1
queuePushResult.value = i + 1 < toPush.length
? '任务 ' + taskId + ' 已完成,继续推送下一条(' + (i + 1) + '/' + toPush.length + ''
: '任务 ' + taskId + ' 已完成'
queuePushResult.value = '任务 ' + taskId + ' 已完成'
} catch (error) {
failedCount += 1
queuePushResult.value = '第 ' + (i + 1) + '/' + toPush.length + ' 条任务处理失败,已自动继续下一条:' + (error instanceof Error ? error.message : '未知错误')
queuePushResult.value = '第 ' + index + ' 条任务处理失败,已自动继续下一条:' + (error instanceof Error ? error.message : '未知错误')
ElMessage.error(queuePushResult.value)
removeMatchedRowsLocally([item])
continue
}
}
ElMessage.success('店铺推送已完成:成功 ' + successCount + ' 条,失败 ' + failedCount + ' 条')
if (successCount > 0 || failedCount > 0) {
ElMessage.success('店铺推送已完成:成功 ' + successCount + ' 条,失败 ' + failedCount + ' 条')
}
await loadHistory()
ensurePolling(true)
} catch (e) {
queuePushResult.value = e instanceof Error ? e.message : '推送异常'
ElMessage.error(queuePushResult.value)
} finally {
queueWorkerRunning.value = false
pushing.value = false
}
}

View File

@@ -121,6 +121,8 @@ const shopMatchListingFilter = ref<ListingFilterValue>('Active')
const adding = ref(false)
const matching = ref(false)
const pushing = ref(false)
const queueWorkerRunning = ref(false)
const autoQueueEnabled = ref(false)
const queuePushResult = ref('')
const queuePayloadText = ref('')
const orderedCountryCodes = ref<string[]>(['DE', 'UK', 'FR', 'IT', 'ES'])
@@ -241,7 +243,7 @@ function addScheduleValue() { const last = schedulePickerValues.value[schedulePi
function removeScheduleValue(index: number) { if (schedulePickerValues.value.length <= 1) return; schedulePickerValues.value = schedulePickerValues.value.filter((_, idx) => idx !== index) }
function updateScheduleValue(index: number, value: string | null) { const list = [...schedulePickerValues.value]; list[index] = value || ''; schedulePickerValues.value = sanitizeSchedulePickerValues(list, false) }
function mergeMatchedItems(base: ShopMatchShopQueueItem[], incoming: ShopMatchShopQueueItem[]) { const map = new Map<string, ShopMatchShopQueueItem>(); for (const item of base) map.set(rowKeyForMatch(item), item); for (const item of incoming) map.set(rowKeyForMatch(item), item); return Array.from(map.values()) }
async function runMatch() { const names = selectedCandidates.value.map((item) => item.shop_name).filter(Boolean); if (!names.length) { ElMessage.warning('请先选择候选店铺'); return } matching.value = true; try { const data = await matchShopMatchShops(names); matchedItems.value = mergeMatchedItems(matchedItems.value, data.items || []); saveMatchedItemsToStorage(); ElMessage.success(`本次返回 ${data.items?.length || 0} 条匹配结果`) } catch (error) { ElMessage.error(error instanceof Error ? error.message : '匹配失败') } finally { matching.value = false } }
async function runMatch() { const names = selectedCandidates.value.map((item) => item.shop_name).filter(Boolean); if (!names.length) { ElMessage.warning('请先选择候选店铺'); return } matching.value = true; try { const data = await matchShopMatchShops(names); matchedItems.value = mergeMatchedItems(matchedItems.value, data.items || []); saveMatchedItemsToStorage(); if (autoQueueEnabled.value) void processMatchedQueue(); ElMessage.success(`本次返回 ${data.items?.length || 0} 条匹配结果`) } catch (error) { ElMessage.error(error instanceof Error ? error.message : '匹配失败') } finally { matching.value = false } }
function removeMatchedRow(row: ShopMatchShopQueueItem) { matchedItems.value = matchedItems.value.filter((item) => rowKeyForMatch(item) !== rowKeyForMatch(row)); saveMatchedItemsToStorage() }
function removeMatchedRowsLocally(rows: ShopMatchHistoryItem[] | ShopMatchShopQueueItem[]) { const keys = new Set(rows.map((row) => rowKeyForMatch(row as ShopMatchShopQueueItem))); matchedItems.value = matchedItems.value.filter((item) => !keys.has(rowKeyForMatch(item))); saveMatchedItemsToStorage() }
function parseScheduleValues() {
@@ -289,10 +291,9 @@ function sanitizeSchedulePickerValues(values: string[], ensureOne: boolean) {
function buildQueuePayload(taskId: number, item: Pick<ShopMatchHistoryItem, 'shopName' | 'shopId' | 'platform' | 'companyName' | 'matchStatus' | 'matchMessage'>, countryCodes: string[], stageIndex?: number, finalStage = true) { return { type: 'shop-match-run', ts: Date.now(), data: { taskId, user_id: currentUserId(), items: [{ shopName: item.shopName, shopId: item.shopId, platform: item.platform, companyName: item.companyName, matched: true, matchStatus: item.matchStatus, matchMessage: item.matchMessage }], country_codes: [...countryCodes], risk_listing_filter: shopMatchListingFilter.value, stage_index: stageIndex, final_stage: finalStage } } }
function nextScheduleStage(taskId: number) { const task = taskSnapshots.value[taskId]?.task; const stages = task?.scheduleStages || []; const currentStageIndex = task?.currentStageIndex; if (typeof currentStageIndex !== 'number') return null; const stage = stages.find((item) => item.stageIndex === currentStageIndex); if (!stage?.scheduledAt) return null; return { stageIndex: currentStageIndex, scheduledAt: stage.scheduledAt, finalStage: currentStageIndex === stages.length - 1 } }
async function dispatchScheduledTask(taskId: number, item: ShopMatchHistoryItem) { const api = getPywebviewApi(); if (!api?.enqueue_json) throw new Error('当前客户端未提供 enqueue_json'); const stage = nextScheduleStage(taskId); const snapshot = taskSnapshots.value[taskId]; const countryCodes = snapshot?.task?.countryCodes || orderedCountryCodes.value; if (!stage) throw new Error(`任务 ${taskId} 缺少待执行阶段`); clearDispatchTimer(taskId); await withTransientRetry(() => activateShopMatchTask(taskId, stage.stageIndex), (attempt, maxAttempts) => { queuePushResult.value = `任务 ${taskId} 等待后端恢复后再激活执行(${attempt}/${maxAttempts}...` }); taskDetails.value = { ...taskDetails.value, [taskId]: 'RUNNING' }; if (snapshot?.task) taskSnapshots.value = { ...taskSnapshots.value, [taskId]: { ...snapshot, task: { ...snapshot.task, status: 'RUNNING', activeStageIndex: stage.stageIndex } } }; saveTaskSnapshotsToStorage(); saveTaskDetailsToStorage(); const payload = buildQueuePayload(taskId, item, countryCodes, stage.stageIndex, stage.finalStage); queuePayloadText.value = JSON.stringify(payload, null, 2); const result = await api.enqueue_json(payload); if (!result?.success) throw new Error(result?.error || `任务 ${taskId} 推送失败`); addPollingTask(taskId); queuePushResult.value = `任务 ${taskId} 已推送第 ${stage.stageIndex + 1}`; ensurePolling(true) }
function hasRunningTask() { return Object.values(taskDetails.value).some((status) => status === 'RUNNING') || Object.values(taskSnapshots.value).some((detail) => detail.task?.status === 'RUNNING') }
function scheduledTimestamp(value?: string) { const date = parseScheduleDateTime(value); return date ? date.getTime() : Number.NaN }
function findNextReadyScheduledTask() { const taskIds = new Set<number>(pollingTaskIds.value); for (const key of Object.keys(taskSnapshots.value)) { const id = normalizeTaskId(key); if (id > 0) taskIds.add(id) } for (const item of historyItems.value) { const id = normalizeTaskId(item.taskId); if (id > 0) taskIds.add(id) } const ready = Array.from(taskIds).map((taskId) => { const snapshot = taskSnapshots.value[taskId]; const taskStatus = taskDetails.value[taskId] || snapshot?.task?.status; const item = snapshot?.items?.[0] || historyItems.value.find((row) => normalizeTaskId(row.taskId) === taskId); const stage = nextScheduleStage(taskId); if (taskStatus !== 'SCHEDULED' || !item || !stage?.scheduledAt) return null; const scheduledTime = scheduledTimestamp(stage.scheduledAt); if (!Number.isFinite(scheduledTime) || scheduledTime > Date.now()) return null; return { taskId, item, scheduledTime } }).filter((item): item is { taskId: number; item: ShopMatchHistoryItem; scheduledTime: number } => !!item); ready.sort((a, b) => a.scheduledTime - b.scheduledTime || a.taskId - b.taskId); return ready[0] || null }
async function pumpScheduledQueue() { if (scheduledDispatchInFlight.value || hasRunningTask()) return; const readyTask = findNextReadyScheduledTask(); if (!readyTask) return; scheduledDispatchInFlight.value = true; try { await dispatchScheduledTask(readyTask.taskId, readyTask.item) } catch (error) { const message = error instanceof Error ? error.message : `任务 ${readyTask.taskId} 调度失败`; const key = `${readyTask.taskId}:${message}`; const now = Date.now(); queuePushResult.value = message; if (lastScheduledErrorKey !== key || now - lastScheduledErrorAt > 10000) { lastScheduledErrorKey = key; lastScheduledErrorAt = now; ElMessage.error(message) } await Promise.allSettled([loadHistory(), refreshTaskBatch()]) } finally { scheduledDispatchInFlight.value = false } }
function findReadyScheduledTasks() { const taskIds = new Set<number>(pollingTaskIds.value); for (const key of Object.keys(taskSnapshots.value)) { const id = normalizeTaskId(key); if (id > 0) taskIds.add(id) } for (const item of historyItems.value) { const id = normalizeTaskId(item.taskId); if (id > 0) taskIds.add(id) } const ready = Array.from(taskIds).map((taskId) => { const snapshot = taskSnapshots.value[taskId]; const taskStatus = taskDetails.value[taskId] || snapshot?.task?.status; const item = snapshot?.items?.[0] || historyItems.value.find((row) => normalizeTaskId(row.taskId) === taskId); const stage = nextScheduleStage(taskId); if (taskStatus !== 'SCHEDULED' || !item || !stage?.scheduledAt) return null; const scheduledTime = scheduledTimestamp(stage.scheduledAt); if (!Number.isFinite(scheduledTime) || scheduledTime > Date.now()) return null; return { taskId, item, scheduledTime } }).filter((item): item is { taskId: number; item: ShopMatchHistoryItem; scheduledTime: number } => !!item); ready.sort((a, b) => a.scheduledTime - b.scheduledTime || a.taskId - b.taskId); return ready }
async function pumpScheduledQueue() { if (scheduledDispatchInFlight.value) return; const readyTasks = findReadyScheduledTasks(); if (!readyTasks.length) return; scheduledDispatchInFlight.value = true; try { for (const readyTask of readyTasks) { try { await dispatchScheduledTask(readyTask.taskId, readyTask.item) } catch (error) { const message = error instanceof Error ? error.message : `任务 ${readyTask.taskId} 调度失败`; const key = `${readyTask.taskId}:${message}`; const now = Date.now(); queuePushResult.value = message; if (lastScheduledErrorKey !== key || now - lastScheduledErrorAt > 10000) { lastScheduledErrorKey = key; lastScheduledErrorAt = now; ElMessage.error(message) } await Promise.allSettled([loadHistory(), refreshTaskBatch()]) } } } finally { scheduledDispatchInFlight.value = false } }
function scheduleDispatch(taskId: number, scheduledAt?: string) { if (!scheduledAt) return; clearDispatchTimer(taskId); const time = scheduledTimestamp(scheduledAt); if (!Number.isFinite(time)) return; const delay = Math.max(0, time - Date.now()); const timer = window.setTimeout(() => { dispatchTimers.delete(taskId); void pumpScheduledQueue() }, delay); dispatchTimers.set(taskId, timer) }
function restoreScheduledDispatches() { const taskIds = new Set<number>(pollingTaskIds.value); for (const key of Object.keys(taskSnapshots.value)) { const id = normalizeTaskId(key); if (id > 0) taskIds.add(id) } for (const item of historyItems.value) { const id = normalizeTaskId(item.taskId); if (id > 0) taskIds.add(id) } for (const taskId of taskIds) { const snapshot = taskSnapshots.value[taskId]; const taskStatus = snapshot?.task?.status || taskDetails.value[taskId]; const stage = nextScheduleStage(taskId); if (taskStatus === 'SCHEDULED' && stage?.scheduledAt) scheduleDispatch(taskId, stage.scheduledAt); else clearDispatchTimer(taskId) } void pumpScheduledQueue() }
function startScheduleHeartbeat() { if (scheduleHeartbeatTimer) return; scheduleHeartbeatTimer = window.setInterval(() => { void pumpScheduledQueue() }, 1000) }
@@ -344,7 +345,9 @@ async function deleteTaskRecord(item: ShopMatchHistoryItem) {
}
function formatMatchStatus(status?: string) { const value = (status || '').trim(); return { MATCHED: '已匹配', PENDING: '待匹配', CONFLICT: '需人工确认', INDEX_STALE: '索引过期' }[value] || value || '—' }
function formatMatchRemark(row: ShopMatchShopQueueItem) { const message = (row.matchMessage || '').trim(); if (message) return message; if (row.matched && row.matchStatus === 'MATCHED') return '紫鸟索引已命中,可推送队列'; if (row.matched) return '已关联索引,请结合状态列查看是否可推送'; return '未命中或未就绪,请检查店铺名与索引刷新' }
async function pushToPythonQueue() { const api = getPywebviewApi(); if (!api?.enqueue_json) { ElMessage.error('当前客户端未提供 enqueue_json'); return } const matched = matchedItems.value.filter((item) => item.matched); if (!matched.length) { ElMessage.warning('请先匹配可用店铺'); return } if (!orderedCountryCodes.value.length) { ElMessage.warning('请至少勾选一个国家'); return } let scheduleValues: string[] | undefined; try { scheduleValues = parseScheduleValues() } catch (error) { ElMessage.error(error instanceof Error ? error.message : '定时配置无效'); return } pushing.value = true; queuePayloadText.value = ''; let successCount = 0; let failedCount = 0; try { for (let index = 0; index < matched.length; index += 1) { const item = matched[index]; try { const created = await withTransientRetry(() => createShopMatchTask([item], orderedCountryCodes.value, scheduleValues), (attempt, maxAttempts) => { queuePushResult.value = `后端服务暂时不可用,正在重试创建任务(${attempt}/${maxAttempts}...` }); const taskId = created.taskId; const initialStatus = scheduleValues?.length ? 'SCHEDULED' : 'RUNNING'; const snapshot: ShopMatchTaskDetailVo = { task: { id: taskId, status: initialStatus, scheduledAt: scheduleValues?.[0], countryCodes: [...orderedCountryCodes.value], currentStageIndex: scheduleValues?.length ? 0 : undefined, activeStageIndex: undefined, scheduleStages: (scheduleValues || []).map((value, stageIndex) => ({ stageIndex, scheduledAt: value, status: stageIndex === 0 ? 'SCHEDULED' : 'PENDING' })) }, items: created.items }; taskSnapshots.value = { ...taskSnapshots.value, [taskId]: snapshot }; taskDetails.value = { ...taskDetails.value, [taskId]: initialStatus }; saveTaskSnapshotsToStorage(); saveTaskDetailsToStorage(); const createdItem = created.items?.[0]; if (!createdItem) continue; if (scheduleValues?.length) { successCount += 1; restoreScheduledDispatches(); queuePushResult.value = `任务 ${taskId} 已创建,共 ${scheduleValues.length} 个执行时间点`; removeMatchedRowsLocally([item]); continue } const payload = buildQueuePayload(taskId, item, orderedCountryCodes.value); queuePayloadText.value = JSON.stringify(payload, null, 2); const result = await api.enqueue_json(payload); if (!result?.success) { failedCount += 1; queuePushResult.value = `任务 ${taskId} 推送失败,已自动继续下一条:${result?.error || '未知错误'}`; ElMessage.error(queuePushResult.value); continue } addPollingTask(taskId); ensurePolling(true); removeMatchedRowsLocally([item]); queuePushResult.value = matched.length > 1 ? `任务 ${taskId} 已入队,等待完成后继续推送下一条(${index + 1}/${matched.length}` : `任务 ${taskId} 已入队,等待执行完成`; const finalStatus = await waitForTaskTerminal(taskId); if (finalStatus !== 'SUCCESS') { failedCount += 1; queuePushResult.value = `任务 ${taskId} 执行失败,已自动继续下一条(${index + 1}/${matched.length}`; ElMessage.error(queuePushResult.value); continue } successCount += 1; queuePushResult.value = index + 1 < matched.length ? `任务 ${taskId} 已完成,继续推送下一条(${index + 1}/${matched.length}` : `任务 ${taskId} 已完成` } catch (error) { failedCount += 1; queuePushResult.value = `${index + 1}/${matched.length} 条任务处理失败,已自动继续下一条:${error instanceof Error ? error.message : '未知错误'}`; ElMessage.error(queuePushResult.value); continue } } restoreScheduledDispatches(); if (!scheduleValues?.length) ensurePolling(true); ElMessage.success(scheduleValues?.length ? `定时任务已创建:成功 ${successCount} 条,失败 ${failedCount}` : `店铺推送已完成:成功 ${successCount} 条,失败 ${failedCount}`) } catch (error) { const message = error instanceof Error ? error.message : '推送失败'; queuePushResult.value = message; ElMessage.error(message) } finally { pushing.value = false } }
async function pushToPythonQueue() { autoQueueEnabled.value = true; await processMatchedQueue() }
function nextMatchedQueueItem() { return matchedItems.value.find((item) => item.matched) }
async function processMatchedQueue() { if (queueWorkerRunning.value) return; const api = getPywebviewApi(); if (!api?.enqueue_json) { ElMessage.error('当前客户端未提供 enqueue_json'); return } const matched = matchedItems.value.filter((item) => item.matched); if (!matched.length) { ElMessage.warning('请先匹配可用店铺'); return } if (!orderedCountryCodes.value.length) { ElMessage.warning('请至少勾选一个国家'); return } let scheduleValues: string[] | undefined; try { scheduleValues = parseScheduleValues() } catch (error) { ElMessage.error(error instanceof Error ? error.message : '定时配置无效'); return } queueWorkerRunning.value = true; pushing.value = true; queuePayloadText.value = ''; let successCount = 0; let failedCount = 0; let index = 0; try { while (autoQueueEnabled.value) { const item = nextMatchedQueueItem(); if (!item) break; index += 1; try { const created = await withTransientRetry(() => createShopMatchTask([item], orderedCountryCodes.value, scheduleValues), (attempt, maxAttempts) => { queuePushResult.value = `后端服务暂时不可用,正在重试创建任务(${attempt}/${maxAttempts}...` }); const taskId = created.taskId; const initialStatus = scheduleValues?.length ? 'SCHEDULED' : 'RUNNING'; const snapshot: ShopMatchTaskDetailVo = { task: { id: taskId, status: initialStatus, scheduledAt: scheduleValues?.[0], countryCodes: [...orderedCountryCodes.value], currentStageIndex: scheduleValues?.length ? 0 : undefined, activeStageIndex: undefined, scheduleStages: (scheduleValues || []).map((value, stageIndex) => ({ stageIndex, scheduledAt: value, status: stageIndex === 0 ? 'SCHEDULED' : 'PENDING' })) }, items: created.items }; taskSnapshots.value = { ...taskSnapshots.value, [taskId]: snapshot }; taskDetails.value = { ...taskDetails.value, [taskId]: initialStatus }; saveTaskSnapshotsToStorage(); saveTaskDetailsToStorage(); const createdItem = created.items?.[0]; if (!createdItem) { removeMatchedRowsLocally([item]); continue } if (scheduleValues?.length) { successCount += 1; restoreScheduledDispatches(); queuePushResult.value = `任务 ${taskId} 已创建,共 ${scheduleValues.length} 个执行时间点`; removeMatchedRowsLocally([item]); continue } const payload = buildQueuePayload(taskId, item, orderedCountryCodes.value); queuePayloadText.value = JSON.stringify(payload, null, 2); const result = await api.enqueue_json(payload); if (!result?.success) { failedCount += 1; queuePushResult.value = `任务 ${taskId} 推送失败,已自动继续下一条:${result?.error || '未知错误'}`; ElMessage.error(queuePushResult.value); removeMatchedRowsLocally([item]); continue } addPollingTask(taskId); ensurePolling(true); removeMatchedRowsLocally([item]); queuePushResult.value = `任务 ${taskId} 已入队,等待执行完成`; const finalStatus = await waitForTaskTerminal(taskId); if (finalStatus !== 'SUCCESS') { failedCount += 1; queuePushResult.value = `任务 ${taskId} 执行失败,已自动继续下一条`; ElMessage.error(queuePushResult.value); continue } successCount += 1; queuePushResult.value = `任务 ${taskId} 已完成` } catch (error) { failedCount += 1; queuePushResult.value = `${index} 条任务处理失败,已自动继续下一条:${error instanceof Error ? error.message : '未知错误'}`; ElMessage.error(queuePushResult.value); removeMatchedRowsLocally([item]); continue } } restoreScheduledDispatches(); if (!scheduleValues?.length) ensurePolling(true); if (successCount > 0 || failedCount > 0) ElMessage.success(scheduleValues?.length ? `定时任务已创建:成功 ${successCount} 条,失败 ${failedCount}` : `店铺推送已完成:成功 ${successCount} 条,失败 ${failedCount}`) } catch (error) { const message = error instanceof Error ? error.message : '推送失败'; queuePushResult.value = message; ElMessage.error(message) } finally { queueWorkerRunning.value = false; pushing.value = false } }
onMounted(async () => { loadPollingIdsFromStorage(); loadTaskDetailsFromStorage(); loadTaskSnapshotsFromStorage(); loadMatchedItemsFromStorage(); schedulePickerValues.value = sanitizeSchedulePickerValues(schedulePickerValues.value, scheduleEnabled.value); syncPollingIdsWithTaskState(); await Promise.allSettled([loadCandidates(), loadCountryPreference(), loadHistory(), loadDashboard()]); startScheduleHeartbeat(); window.addEventListener('focus', handleScheduleRecovery); document.addEventListener('visibilitychange', handleScheduleRecovery); restoreScheduledDispatches(); if (pollingTaskIds.value.length) ensurePolling(true) })
onUnmounted(() => { stopPolling(); stopScheduleHeartbeat(); window.removeEventListener('focus', handleScheduleRecovery); document.removeEventListener('visibilitychange', handleScheduleRecovery); if (countryPrefSaveTimer) clearTimeout(countryPrefSaveTimer); for (const timer of dispatchTimers.values()) window.clearTimeout(timer); dispatchTimers.clear() })
</script>

View File

@@ -1211,7 +1211,9 @@ export interface PriceTrackShopQueueItem {
export interface PriceTrackMatchShopsVo {
items: PriceTrackShopQueueItem[];
skipAsinsByCountry?: Record<string, string[]>;
skipAsinDetailsByCountry?: Record<string, Array<{ asin?: string; minimumPrice?: string }>>;
asinRowsByCountry?: Record<string, PriceTrackAsinParsedRow[]>;
minimumPriceByCountryAndAsin?: Record<string, Record<string, string>>;
}
export interface PriceTrackCountryPreferenceVo {
@@ -1268,7 +1270,9 @@ export interface PriceTrackCreateTaskVo {
taskId: number;
items: PriceTrackHistoryItem[];
skipAsinsByCountry?: Record<string, string[]>;
skipAsinDetailsByCountry?: Record<string, Array<{ asin?: string; minimumPrice?: string }>>;
asinRowsByCountry?: Record<string, PriceTrackAsinParsedRow[]>;
minimumPriceByCountryAndAsin?: Record<string, Record<string, string>>;
}
export interface PriceTrackTaskSummary {

View File

@@ -0,0 +1,17 @@
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>巡店删除 - 数富AI</title>
<script type="module" crossorigin src="/assets/patrol-delete.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-DuyK2jB1.js">
<link rel="modulepreload" crossorigin href="/assets/zh-cn-Z9PBnl-n.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-D74jYk6P.css">
<link rel="stylesheet" crossorigin href="/assets/patrol-delete-DIvFZUTh.css">
<link rel="stylesheet" crossorigin href="/assets/el-input-CezJelDw.css">
</head>
<body>
<div id="app"></div>
</body>
</html>

View File

@@ -5,9 +5,10 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>跟价 - 数富AI</title>
<script type="module" crossorigin src="/assets/price-track.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-DiP0HdY6.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-D-mMH8F6.css">
<link rel="stylesheet" crossorigin href="/assets/price-track-BO5PtL0W.css">
<link rel="modulepreload" crossorigin href="/assets/pywebview-DuyK2jB1.js">
<link rel="modulepreload" crossorigin href="/assets/brand-CKgrwtni.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-D74jYk6P.css">
<link rel="stylesheet" crossorigin href="/assets/price-track-BccJT4Qr.css">
<link rel="stylesheet" crossorigin href="/assets/el-input-CezJelDw.css">
</head>
<body>

View File

@@ -5,11 +5,12 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>匹配店铺 - 数富AI</title>
<script type="module" crossorigin src="/assets/shop-match.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-DiP0HdY6.js">
<link rel="modulepreload" crossorigin href="/assets/pywebview-DuyK2jB1.js">
<link rel="modulepreload" crossorigin href="/assets/zh-cn-Z9PBnl-n.js">
<link rel="modulepreload" crossorigin href="/assets/listingFilters-BpGOU_pJ.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-D-mMH8F6.css">
<link rel="stylesheet" crossorigin href="/assets/pywebview-D74jYk6P.css">
<link rel="stylesheet" crossorigin href="/assets/listingFilters-CK58rX4v.css">
<link rel="stylesheet" crossorigin href="/assets/shop-match-MsxXMG_G.css">
<link rel="stylesheet" crossorigin href="/assets/shop-match-DNFA8ceR.css">
<link rel="stylesheet" crossorigin href="/assets/el-input-CezJelDw.css">
</head>
<body>