Merge branch 'master' into dev/koko
This commit is contained in:
4
app/.env
4
app/.env
@@ -13,7 +13,7 @@ client_name=ShuFuAI
|
|||||||
|
|
||||||
|
|
||||||
# java_api_base=http://47.111.163.154:18080
|
# java_api_base=http://47.111.163.154:18080
|
||||||
# java_api_base=http://127.0.0.1:18080
|
java_api_base=http://127.0.0.1:18080
|
||||||
java_api_base=http://8.136.19.173:18080
|
# java_api_base=http://8.136.19.173:18080
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -15,6 +15,11 @@ from config import mysql_host, mysql_user, mysql_password, mysql_database
|
|||||||
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
|
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||||
STATIC_DIR = os.path.join(BASE_DIR, 'static')
|
STATIC_DIR = os.path.join(BASE_DIR, 'static')
|
||||||
ASSETS_DIR = os.path.join(BASE_DIR, 'assets')
|
ASSETS_DIR = os.path.join(BASE_DIR, 'assets')
|
||||||
|
WEB_SOURCE_DIR = os.path.join(BASE_DIR, 'web_source')
|
||||||
|
TEMPLATE_FALLBACK_DIRS = (
|
||||||
|
WEB_SOURCE_DIR,
|
||||||
|
os.path.join(WEB_SOURCE_DIR, 'templates_backup'),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def get_db():
|
def get_db():
|
||||||
@@ -30,8 +35,12 @@ def get_db():
|
|||||||
|
|
||||||
def _render_html(template_name: str, **context):
|
def _render_html(template_name: str, **context):
|
||||||
"""读取 HTML 模板:若为加密文件则先解密,再渲染。未加密或解密失败时按明文渲染。"""
|
"""读取 HTML 模板:若为加密文件则先解密,再渲染。未加密或解密失败时按明文渲染。"""
|
||||||
path = os.path.join(BASE_DIR, "web_source", template_name)
|
path = next(
|
||||||
if not os.path.isfile(path):
|
(candidate for candidate in (os.path.join(base_path, template_name) for base_path in TEMPLATE_FALLBACK_DIRS)
|
||||||
|
if os.path.isfile(candidate)),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if path is None:
|
||||||
return render_template(template_name, **context)
|
return render_template(template_name, **context)
|
||||||
with open(path, "rb") as f:
|
with open(path, "rb") as f:
|
||||||
raw = f.read()
|
raw = f.read()
|
||||||
@@ -45,8 +54,12 @@ def _render_html(template_name: str, **context):
|
|||||||
|
|
||||||
def _render_html_new(template_name: str, **context):
|
def _render_html_new(template_name: str, **context):
|
||||||
"""读取 HTML 模板:若为加密文件则先解密,再渲染。未加密或解密失败时按明文渲染。"""
|
"""读取 HTML 模板:若为加密文件则先解密,再渲染。未加密或解密失败时按明文渲染。"""
|
||||||
path = os.path.join(BASE_DIR, "web_source", template_name)
|
path = next(
|
||||||
if not os.path.isfile(path):
|
(candidate for candidate in (os.path.join(base_path, template_name) for base_path in TEMPLATE_FALLBACK_DIRS)
|
||||||
|
if os.path.isfile(candidate)),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if path is None:
|
||||||
return render_template(template_name, **context)
|
return render_template(template_name, **context)
|
||||||
with open(path, "rb") as f:
|
with open(path, "rb") as f:
|
||||||
raw = f.read()
|
raw = f.read()
|
||||||
|
|||||||
Binary file not shown.
@@ -2,7 +2,7 @@
|
|||||||
主页面蓝图:首页、home、图片工作台、品牌页、静态文件、Logo
|
主页面蓝图:首页、home、图片工作台、品牌页、静态文件、Logo
|
||||||
"""
|
"""
|
||||||
import os
|
import os
|
||||||
from flask import Blueprint, send_file
|
from flask import Blueprint, send_file, render_template_string
|
||||||
|
|
||||||
from app_common import (
|
from app_common import (
|
||||||
get_db,
|
get_db,
|
||||||
@@ -23,6 +23,38 @@ from config import base_url,version,JAVA_API_BASE
|
|||||||
main_bp = Blueprint('main', __name__)
|
main_bp = Blueprint('main', __name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_asset_filename(filename):
|
||||||
|
"""Resolve hashed Vite assets without maintaining hardcoded alias tables."""
|
||||||
|
safe_name = os.path.basename(filename)
|
||||||
|
exact_path = os.path.join(ASSETS_DIR, safe_name)
|
||||||
|
if os.path.isfile(exact_path):
|
||||||
|
return safe_name
|
||||||
|
|
||||||
|
base_name, ext = os.path.splitext(safe_name)
|
||||||
|
if not base_name or not ext:
|
||||||
|
return safe_name
|
||||||
|
|
||||||
|
parts = base_name.split('-')
|
||||||
|
if len(parts) < 2:
|
||||||
|
return safe_name
|
||||||
|
|
||||||
|
try:
|
||||||
|
asset_names = os.listdir(ASSETS_DIR)
|
||||||
|
except OSError:
|
||||||
|
return safe_name
|
||||||
|
|
||||||
|
for prefix_length in range(len(parts) - 1, 0, -1):
|
||||||
|
prefix = '-'.join(parts[:prefix_length])
|
||||||
|
matches = [
|
||||||
|
asset_name for asset_name in asset_names
|
||||||
|
if asset_name.endswith(ext) and asset_name.startswith(prefix)
|
||||||
|
]
|
||||||
|
if len(matches) == 1:
|
||||||
|
return matches[0]
|
||||||
|
|
||||||
|
return safe_name
|
||||||
|
|
||||||
|
|
||||||
@main_bp.route('/')
|
@main_bp.route('/')
|
||||||
def index():
|
def index():
|
||||||
if session.get('user_id') and _is_session_user_valid():
|
if session.get('user_id') and _is_session_user_valid():
|
||||||
@@ -53,9 +85,41 @@ def wb():
|
|||||||
@main_bp.route('/brand')
|
@main_bp.route('/brand')
|
||||||
@login_required
|
@login_required
|
||||||
def brand_page():
|
def brand_page():
|
||||||
|
repo_brand_path = os.path.abspath(os.path.join(BASE_DIR, '..', 'web_source', 'brand.html'))
|
||||||
|
if os.path.isfile(repo_brand_path):
|
||||||
|
try:
|
||||||
|
with open(repo_brand_path, 'rb') as f:
|
||||||
|
raw = f.read()
|
||||||
|
try:
|
||||||
|
from html_crypto import decrypt
|
||||||
|
content = decrypt(raw).decode('utf-8')
|
||||||
|
except Exception:
|
||||||
|
if raw[:7] == b'gAAAAAB':
|
||||||
|
raise
|
||||||
|
content = raw.decode('utf-8', errors='replace')
|
||||||
|
return render_template_string(content, user_id=session.get('user_id'))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
return _render_html('brand.html', user_id=session.get('user_id'))
|
return _render_html('brand.html', user_id=session.get('user_id'))
|
||||||
|
|
||||||
|
|
||||||
|
@main_bp.route('/brand/legacy')
|
||||||
|
@login_required
|
||||||
|
def brand_page_legacy():
|
||||||
|
legacy_path = os.path.join(BASE_DIR, 'web_source', 'templates_backup', 'brand.html')
|
||||||
|
if not os.path.isfile(legacy_path):
|
||||||
|
return '', 404
|
||||||
|
with open(legacy_path, 'rb') as f:
|
||||||
|
content = f.read().decode('utf-8', errors='replace')
|
||||||
|
content = content.replace(
|
||||||
|
'<header class="top-bar">',
|
||||||
|
'<header class="top-bar" style="display:none !important;">',
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
content = content.replace('height: calc(100vh - 56px);', 'height: 100vh;', 1)
|
||||||
|
return render_template_string(content, user_id=session.get('user_id'))
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@main_bp.route('/static/<path:filename>')
|
@main_bp.route('/static/<path:filename>')
|
||||||
@@ -72,6 +136,7 @@ def serve_static(filename):
|
|||||||
@main_bp.route('/assets/<path:filename>')
|
@main_bp.route('/assets/<path:filename>')
|
||||||
def serve_assets(filename):
|
def serve_assets(filename):
|
||||||
"""提供 static 目录及子目录下的静态文件访问。"""
|
"""提供 static 目录及子目录下的静态文件访问。"""
|
||||||
|
filename = _resolve_asset_filename(filename)
|
||||||
filepath = os.path.normpath(os.path.join(ASSETS_DIR, filename))
|
filepath = os.path.normpath(os.path.join(ASSETS_DIR, filename))
|
||||||
static_abs = os.path.abspath(ASSETS_DIR)
|
static_abs = os.path.abspath(ASSETS_DIR)
|
||||||
file_abs = os.path.abspath(filepath)
|
file_abs = os.path.abspath(filepath)
|
||||||
@@ -82,14 +147,19 @@ def serve_assets(filename):
|
|||||||
|
|
||||||
@main_bp.route('/new_web_source/<path:filename>')
|
@main_bp.route('/new_web_source/<path:filename>')
|
||||||
def serve_new_web_source(filename):
|
def serve_new_web_source(filename):
|
||||||
"""提供 static 目录及子目录下的静态文件访问。"""
|
"""提供 new_web_source 目录下的静态页面文件访问。"""
|
||||||
filepath = os.path.normpath(os.path.join("new_web_source", filename))
|
candidate_dirs = [
|
||||||
static_abs = os.path.abspath("new_web_source")
|
os.path.abspath(os.path.join(BASE_DIR, 'new_web_source')),
|
||||||
|
os.path.abspath(os.path.join(BASE_DIR, '..', 'new_web_source')),
|
||||||
|
]
|
||||||
|
for static_abs in candidate_dirs:
|
||||||
|
filepath = os.path.normpath(os.path.join(static_abs, filename))
|
||||||
file_abs = os.path.abspath(filepath)
|
file_abs = os.path.abspath(filepath)
|
||||||
if not file_abs.startswith(static_abs) or not os.path.isfile(file_abs):
|
if not file_abs.startswith(static_abs):
|
||||||
return '', 404
|
continue
|
||||||
|
if os.path.isfile(file_abs):
|
||||||
return send_file(file_abs, as_attachment=False)
|
return send_file(file_abs, as_attachment=False)
|
||||||
|
return '', 404
|
||||||
|
|
||||||
|
|
||||||
@main_bp.route('/logo.jpg', methods=['GET'])
|
@main_bp.route('/logo.jpg', methods=['GET'])
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ from dotenv import load_dotenv
|
|||||||
from queue import Queue
|
from queue import Queue
|
||||||
load_dotenv()
|
load_dotenv()
|
||||||
_base_url = os.getenv("base_url","http://159.75.121.33:15124")
|
_base_url = os.getenv("base_url","http://159.75.121.33:15124")
|
||||||
|
base_dir = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
|
||||||
# MySQL 配置
|
# MySQL 配置
|
||||||
mysql_host = os.getenv("mysql_host")
|
mysql_host = os.getenv("mysql_host")
|
||||||
@@ -17,7 +18,8 @@ mysql_database = os.getenv("mysql_database","aiimage")
|
|||||||
proxy_url = os.getenv("proxy_url")
|
proxy_url = os.getenv("proxy_url")
|
||||||
proxy_mode = int(os.getenv("proxy_mode",1))
|
proxy_mode = int(os.getenv("proxy_mode",1))
|
||||||
|
|
||||||
client_name=os.getenv("client_name") + ".exe"
|
_client_name = os.getenv("client_name", "").strip()
|
||||||
|
client_name = f"{_client_name}.exe" if _client_name else "client.exe"
|
||||||
JAVA_API_BASE = os.getenv("java_api_base", "http://127.0.0.1:18080")
|
JAVA_API_BASE = os.getenv("java_api_base", "http://127.0.0.1:18080")
|
||||||
|
|
||||||
# 紫鸟浏览器配置
|
# 紫鸟浏览器配置
|
||||||
|
|||||||
@@ -433,7 +433,7 @@ def main():
|
|||||||
webview.start(
|
webview.start(
|
||||||
debug=True,
|
debug=True,
|
||||||
storage_path=cache_path,
|
storage_path=cache_path,
|
||||||
private_mode=False
|
private_mode=True
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -8,4 +8,8 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
|
|||||||
public class TaskPressureProperties {
|
public class TaskPressureProperties {
|
||||||
private long localTaskEntityCacheMillis = 3000;
|
private long localTaskEntityCacheMillis = 3000;
|
||||||
private int dbSelectBatchSize = 200;
|
private int dbSelectBatchSize = 200;
|
||||||
|
private long scopePayloadFlushIntervalMillis = 15000;
|
||||||
|
private long scopePayloadBufferRetentionHours = 24;
|
||||||
|
private int scopePayloadRecoveryMaxFiles = 200;
|
||||||
|
private String scopePayloadCleanupCron = "15 */30 * * * *";
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,25 +1,13 @@
|
|||||||
package com.nanri.aiimage.modules.brand.service;
|
package com.nanri.aiimage.modules.brand.service;
|
||||||
|
|
||||||
import com.fasterxml.jackson.core.type.TypeReference;
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.nanri.aiimage.common.exception.BusinessException;
|
|
||||||
import com.nanri.aiimage.config.BrandProgressProperties;
|
import com.nanri.aiimage.config.BrandProgressProperties;
|
||||||
import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultFileDto;
|
|
||||||
import com.nanri.aiimage.modules.brand.model.dto.BrandFileAggregateCacheDto;
|
|
||||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.nio.charset.StandardCharsets;
|
|
||||||
import java.nio.file.Files;
|
|
||||||
import java.nio.file.Path;
|
|
||||||
import java.nio.file.StandardOpenOption;
|
|
||||||
import java.security.MessageDigest;
|
|
||||||
import java.time.Duration;
|
import java.time.Duration;
|
||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
@@ -33,6 +21,7 @@ public class BrandTaskProgressCacheService {
|
|||||||
|
|
||||||
private final StringRedisTemplate stringRedisTemplate;
|
private final StringRedisTemplate stringRedisTemplate;
|
||||||
private final BrandProgressProperties brandProgressProperties;
|
private final BrandProgressProperties brandProgressProperties;
|
||||||
|
@SuppressWarnings("unused")
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
|
|
||||||
public BrandTaskProgressCacheService(StringRedisTemplate stringRedisTemplate,
|
public BrandTaskProgressCacheService(StringRedisTemplate stringRedisTemplate,
|
||||||
@@ -96,121 +85,9 @@ public class BrandTaskProgressCacheService {
|
|||||||
return stringRedisTemplate.opsForHash().entries(buildKey(taskId));
|
return stringRedisTemplate.opsForHash().entries(buildKey(taskId));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void saveParsedPayload(Long taskId, Object payload) {
|
|
||||||
try {
|
|
||||||
Files.createDirectories(buildTaskDir(taskId));
|
|
||||||
Files.writeString(
|
|
||||||
buildPayloadFile(taskId),
|
|
||||||
objectMapper.writeValueAsString(payload),
|
|
||||||
StandardOpenOption.CREATE,
|
|
||||||
StandardOpenOption.TRUNCATE_EXISTING,
|
|
||||||
StandardOpenOption.WRITE
|
|
||||||
);
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("鏆傚瓨鍝佺墝鍘熷鏁版嵁澶辫触");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public <T> T getParsedPayload(Long taskId, TypeReference<T> typeReference) {
|
|
||||||
Path payloadFile = buildPayloadFile(taskId);
|
|
||||||
if (!Files.isRegularFile(payloadFile)) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return objectMapper.readValue(Files.readString(payloadFile), typeReference);
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("璇诲彇鍝佺墝鍘熷鏁版嵁澶辫触");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public ChunkStoreResult storeChunk(Long taskId, BrandCrawlResultFileDto file) {
|
|
||||||
validateChunk(file);
|
|
||||||
String fileUrl = normalizeFileUrl(file.getFileUrl());
|
|
||||||
String chunkDigest = digestChunk(file);
|
|
||||||
String digestField = String.valueOf(file.getChunkIndex());
|
|
||||||
String digestKey = buildChunkDigestKey(taskId, fileUrl);
|
|
||||||
Object existingDigest = stringRedisTemplate.opsForHash().get(digestKey, digestField);
|
|
||||||
if (existingDigest instanceof String existing && !existing.isBlank()) {
|
|
||||||
refreshFileKeys(taskId, fileUrl);
|
|
||||||
if (existing.equals(chunkDigest)) {
|
|
||||||
BrandFileAggregateCacheDto aggregate = getFileAggregate(taskId, fileUrl);
|
|
||||||
int finishedFiles = countCompletedFiles(taskId);
|
|
||||||
return new ChunkStoreResult(false, false, finishedFiles, aggregate);
|
|
||||||
}
|
|
||||||
throw new BusinessException("鍚屼竴鍒嗙墖閲嶅鎻愪氦浣嗗唴瀹逛笉涓€鑷? " + fileUrl + "#" + file.getChunkIndex());
|
|
||||||
}
|
|
||||||
|
|
||||||
BrandFileAggregateCacheDto aggregate = getFileAggregate(taskId, fileUrl);
|
|
||||||
if (aggregate == null) {
|
|
||||||
aggregate = createAggregate(file);
|
|
||||||
} else {
|
|
||||||
ensureChunkConsistency(aggregate, file);
|
|
||||||
}
|
|
||||||
mergeAggregate(aggregate, file);
|
|
||||||
|
|
||||||
stringRedisTemplate.opsForSet().add(buildChunkReceiptKey(taskId, fileUrl), digestField);
|
|
||||||
Long receivedSize = stringRedisTemplate.opsForSet().size(buildChunkReceiptKey(taskId, fileUrl));
|
|
||||||
int receivedCount = receivedSize == null ? 0 : receivedSize.intValue();
|
|
||||||
if (receivedCount <= 0) {
|
|
||||||
receivedCount = aggregate.getReceivedChunkCount() == null ? 0 : aggregate.getReceivedChunkCount();
|
|
||||||
}
|
|
||||||
aggregate.setReceivedChunkCount(Math.max(receivedCount, aggregate.getReceivedChunkCount() == null ? 0 : aggregate.getReceivedChunkCount()));
|
|
||||||
boolean fileCompleted = aggregate.getChunkTotal() != null
|
|
||||||
&& aggregate.getChunkTotal() > 0
|
|
||||||
&& aggregate.getReceivedChunkCount() >= aggregate.getChunkTotal();
|
|
||||||
aggregate.setCompleted(fileCompleted);
|
|
||||||
saveFileAggregate(taskId, aggregate);
|
|
||||||
stringRedisTemplate.opsForHash().put(digestKey, digestField, chunkDigest);
|
|
||||||
refreshFileKeys(taskId, fileUrl);
|
|
||||||
|
|
||||||
boolean newlyCompleted = false;
|
|
||||||
int finishedFiles = countCompletedFiles(taskId);
|
|
||||||
if (fileCompleted) {
|
|
||||||
Long added = stringRedisTemplate.opsForSet().add(buildCompletedFilesKey(taskId), fileUrl);
|
|
||||||
refreshTaskKey(buildCompletedFilesKey(taskId));
|
|
||||||
newlyCompleted = added != null && added > 0;
|
|
||||||
if (newlyCompleted) {
|
|
||||||
finishedFiles = countCompletedFiles(taskId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return new ChunkStoreResult(true, newlyCompleted, finishedFiles, aggregate);
|
|
||||||
}
|
|
||||||
|
|
||||||
public BrandFileAggregateCacheDto getFileAggregate(Long taskId, String fileUrl) {
|
|
||||||
String normalizedFileUrl = normalizeFileUrl(fileUrl);
|
|
||||||
Object raw = stringRedisTemplate.opsForHash().get(buildFileAggregateKey(taskId), normalizedFileUrl);
|
|
||||||
if (!(raw instanceof String json) || json.isBlank()) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return objectMapper.readValue(json, BrandFileAggregateCacheDto.class);
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("璇诲彇鍝佺墝鏂囦欢鑱氬悎缂撳瓨澶辫触");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public Map<String, BrandFileAggregateCacheDto> getAllFileAggregates(Long taskId) {
|
|
||||||
Map<Object, Object> stored = stringRedisTemplate.opsForHash().entries(buildFileAggregateKey(taskId));
|
|
||||||
Map<String, BrandFileAggregateCacheDto> result = new LinkedHashMap<>();
|
|
||||||
for (Map.Entry<Object, Object> entry : stored.entrySet()) {
|
|
||||||
if (!(entry.getKey() instanceof String fileUrl) || !(entry.getValue() instanceof String json) || json.isBlank()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
result.put(fileUrl, objectMapper.readValue(json, BrandFileAggregateCacheDto.class));
|
|
||||||
} catch (Exception ignored) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
public int countCompletedFiles(Long taskId) {
|
|
||||||
Long count = stringRedisTemplate.opsForSet().size(buildCompletedFilesKey(taskId));
|
|
||||||
return count == null ? 0 : count.intValue();
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean acquireFinalizeLock(Long taskId) {
|
public boolean acquireFinalizeLock(Long taskId) {
|
||||||
Boolean ok = stringRedisTemplate.opsForValue().setIfAbsent(buildFinalizeLockKey(taskId), String.valueOf(Instant.now().toEpochMilli()), FINALIZE_LOCK_TTL);
|
Boolean ok = stringRedisTemplate.opsForValue()
|
||||||
|
.setIfAbsent(buildFinalizeLockKey(taskId), String.valueOf(Instant.now().toEpochMilli()), FINALIZE_LOCK_TTL);
|
||||||
return Boolean.TRUE.equals(ok);
|
return Boolean.TRUE.equals(ok);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -220,18 +97,7 @@ public class BrandTaskProgressCacheService {
|
|||||||
|
|
||||||
public void delete(Long taskId) {
|
public void delete(Long taskId) {
|
||||||
stringRedisTemplate.delete(buildKey(taskId));
|
stringRedisTemplate.delete(buildKey(taskId));
|
||||||
stringRedisTemplate.delete(buildFileAggregateKey(taskId));
|
|
||||||
stringRedisTemplate.delete(buildCompletedFilesKey(taskId));
|
|
||||||
stringRedisTemplate.delete(buildFinalizeLockKey(taskId));
|
stringRedisTemplate.delete(buildFinalizeLockKey(taskId));
|
||||||
stringRedisTemplate.delete(buildLegacyResultKey(taskId));
|
|
||||||
deleteTaskDirQuietly(taskId);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void deleteFileState(Long taskId, String fileUrl) {
|
|
||||||
String normalizedFileUrl = normalizeFileUrl(fileUrl);
|
|
||||||
stringRedisTemplate.opsForHash().delete(buildFileAggregateKey(taskId), normalizedFileUrl);
|
|
||||||
stringRedisTemplate.delete(buildChunkReceiptKey(taskId, normalizedFileUrl));
|
|
||||||
stringRedisTemplate.delete(buildChunkDigestKey(taskId, normalizedFileUrl));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public String buildKey(Long taskId) {
|
public String buildKey(Long taskId) {
|
||||||
@@ -242,182 +108,14 @@ public class BrandTaskProgressCacheService {
|
|||||||
return brandProgressProperties.getHeartbeatTimeoutMinutes();
|
return brandProgressProperties.getHeartbeatTimeoutMinutes();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void saveFileAggregate(Long taskId, BrandFileAggregateCacheDto aggregate) {
|
|
||||||
try {
|
|
||||||
stringRedisTemplate.opsForHash().put(buildFileAggregateKey(taskId), normalizeFileUrl(aggregate.getFileUrl()), objectMapper.writeValueAsString(aggregate));
|
|
||||||
refreshTaskKey(buildFileAggregateKey(taskId));
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("鏆傚瓨鍝佺墝鏂囦欢鑱氬悎缁撴灉澶辫触");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void validateChunk(BrandCrawlResultFileDto file) {
|
|
||||||
if (file == null) {
|
|
||||||
throw new BusinessException("鍒嗙墖涓嶈兘涓虹┖");
|
|
||||||
}
|
|
||||||
if (file.getChunkIndex() == null || file.getChunkTotal() == null || file.getChunkIndex() <= 0 || file.getChunkTotal() <= 0 || file.getChunkIndex() > file.getChunkTotal()) {
|
|
||||||
throw new BusinessException("鍒嗙墖鍙傛暟涓嶅悎娉?");
|
|
||||||
}
|
|
||||||
if (file.getFileUrl() == null || file.getFileUrl().isBlank()) {
|
|
||||||
throw new BusinessException("fileUrl 涓嶈兘涓虹┖");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private BrandFileAggregateCacheDto createAggregate(BrandCrawlResultFileDto file) {
|
|
||||||
BrandFileAggregateCacheDto aggregate = new BrandFileAggregateCacheDto();
|
|
||||||
aggregate.setFileUrl(normalizeFileUrl(file.getFileUrl()));
|
|
||||||
aggregate.setOriginalFilename(file.getOriginalFilename());
|
|
||||||
aggregate.setRelativePath(file.getRelativePath());
|
|
||||||
aggregate.setMainSheetName(file.getMainSheetName());
|
|
||||||
aggregate.setChunkTotal(file.getChunkTotal());
|
|
||||||
aggregate.setTotalLines(safePositive(file.getTotalLines()));
|
|
||||||
aggregate.setReceivedChunkCount(0);
|
|
||||||
aggregate.setProcessedLineCount(0);
|
|
||||||
aggregate.setCompleted(false);
|
|
||||||
aggregate.setInvalidBrands(new ArrayList<>());
|
|
||||||
aggregate.setQueryFailedBrands(new ArrayList<>());
|
|
||||||
return aggregate;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void ensureChunkConsistency(BrandFileAggregateCacheDto aggregate, BrandCrawlResultFileDto file) {
|
|
||||||
if (aggregate.getChunkTotal() != null && file.getChunkTotal() != null && !aggregate.getChunkTotal().equals(file.getChunkTotal())) {
|
|
||||||
throw new BusinessException("鍚屼竴鏂囦欢鐨?chunkTotal 涓嶄竴鑷? " + aggregate.getFileUrl());
|
|
||||||
}
|
|
||||||
if (aggregate.getChunkTotal() == null) {
|
|
||||||
aggregate.setChunkTotal(file.getChunkTotal());
|
|
||||||
}
|
|
||||||
int incomingTotalLines = safePositive(file.getTotalLines());
|
|
||||||
if (incomingTotalLines > 0) {
|
|
||||||
aggregate.setTotalLines(Math.max(safePositive(aggregate.getTotalLines()), incomingTotalLines));
|
|
||||||
}
|
|
||||||
if ((aggregate.getOriginalFilename() == null || aggregate.getOriginalFilename().isBlank()) && file.getOriginalFilename() != null && !file.getOriginalFilename().isBlank()) {
|
|
||||||
aggregate.setOriginalFilename(file.getOriginalFilename());
|
|
||||||
}
|
|
||||||
if ((aggregate.getRelativePath() == null || aggregate.getRelativePath().isBlank()) && file.getRelativePath() != null && !file.getRelativePath().isBlank()) {
|
|
||||||
aggregate.setRelativePath(file.getRelativePath());
|
|
||||||
}
|
|
||||||
if ((aggregate.getMainSheetName() == null || aggregate.getMainSheetName().isBlank()) && file.getMainSheetName() != null && !file.getMainSheetName().isBlank()) {
|
|
||||||
aggregate.setMainSheetName(file.getMainSheetName());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void mergeAggregate(BrandFileAggregateCacheDto aggregate, BrandCrawlResultFileDto file) {
|
|
||||||
int processed = safePositive(aggregate.getProcessedLineCount())
|
|
||||||
+ sizeOf(file.getKeptRows())
|
|
||||||
+ sizeOf(file.getInvalidBrands())
|
|
||||||
+ sizeOf(file.getQueryFailedBrands());
|
|
||||||
aggregate.setProcessedLineCount(processed);
|
|
||||||
if (file.getInvalidBrands() != null && !file.getInvalidBrands().isEmpty()) {
|
|
||||||
aggregate.getInvalidBrands().addAll(file.getInvalidBrands());
|
|
||||||
}
|
|
||||||
if (file.getQueryFailedBrands() != null && !file.getQueryFailedBrands().isEmpty()) {
|
|
||||||
aggregate.getQueryFailedBrands().addAll(file.getQueryFailedBrands());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private int sizeOf(List<?> list) {
|
|
||||||
return list == null ? 0 : list.size();
|
|
||||||
}
|
|
||||||
|
|
||||||
private int safePositive(Integer value) {
|
|
||||||
return value == null || value <= 0 ? 0 : value;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String digestChunk(BrandCrawlResultFileDto file) {
|
|
||||||
try {
|
|
||||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
|
||||||
byte[] bytes = digest.digest(objectMapper.writeValueAsString(file).getBytes(StandardCharsets.UTF_8));
|
|
||||||
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
|
||||||
for (byte b : bytes) {
|
|
||||||
sb.append(String.format("%02x", b));
|
|
||||||
}
|
|
||||||
return sb.toString();
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("璁$畻鍝佺墝缁撴灉鍒嗙墖鎽樿澶辫触");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void refreshFileKeys(Long taskId, String fileUrl) {
|
|
||||||
refreshTaskKey(buildChunkReceiptKey(taskId, fileUrl));
|
|
||||||
refreshTaskKey(buildChunkDigestKey(taskId, fileUrl));
|
|
||||||
refreshTaskKey(buildFileAggregateKey(taskId));
|
|
||||||
}
|
|
||||||
|
|
||||||
private void refreshTaskKey(String key) {
|
|
||||||
stringRedisTemplate.expire(key, ttl());
|
|
||||||
}
|
|
||||||
|
|
||||||
private Duration ttl() {
|
private Duration ttl() {
|
||||||
return Duration.ofHours(brandProgressProperties.getTtlHours());
|
return Duration.ofHours(brandProgressProperties.getTtlHours());
|
||||||
}
|
}
|
||||||
|
|
||||||
private Path buildTaskDir(Long taskId) {
|
|
||||||
return Path.of(System.getProperty("java.io.tmpdir"), "brand-task-cache", String.valueOf(taskId));
|
|
||||||
}
|
|
||||||
|
|
||||||
private Path buildPayloadFile(Long taskId) {
|
|
||||||
return buildTaskDir(taskId).resolve("parsed-payload.json");
|
|
||||||
}
|
|
||||||
|
|
||||||
private void deleteTaskDirQuietly(Long taskId) {
|
|
||||||
Path taskDir = buildTaskDir(taskId);
|
|
||||||
if (!Files.exists(taskDir)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try (var walk = Files.walk(taskDir)) {
|
|
||||||
walk.sorted(java.util.Comparator.reverseOrder()).forEach(path -> {
|
|
||||||
try {
|
|
||||||
Files.deleteIfExists(path);
|
|
||||||
} catch (IOException ignored) {
|
|
||||||
}
|
|
||||||
});
|
|
||||||
} catch (IOException ignored) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private String buildFileAggregateKey(Long taskId) {
|
|
||||||
return "brand:task:file-aggregate:" + taskId;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String buildChunkReceiptKey(Long taskId, String fileUrl) {
|
|
||||||
return "brand:task:file-chunks:" + taskId + ":" + hashFileUrl(fileUrl);
|
|
||||||
}
|
|
||||||
|
|
||||||
private String buildChunkDigestKey(Long taskId, String fileUrl) {
|
|
||||||
return "brand:task:file-chunk-digests:" + taskId + ":" + hashFileUrl(fileUrl);
|
|
||||||
}
|
|
||||||
|
|
||||||
private String buildCompletedFilesKey(Long taskId) {
|
|
||||||
return "brand:task:completed-files:" + taskId;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String buildFinalizeLockKey(Long taskId) {
|
private String buildFinalizeLockKey(Long taskId) {
|
||||||
return "brand:task:finalizing:" + taskId;
|
return "brand:task:finalizing:" + taskId;
|
||||||
}
|
}
|
||||||
|
|
||||||
private String buildLegacyResultKey(Long taskId) {
|
|
||||||
return "brand:task:result-chunks:" + taskId;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String normalizeFileUrl(String fileUrl) {
|
|
||||||
return fileUrl == null ? "" : fileUrl.trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
private String hashFileUrl(String fileUrl) {
|
|
||||||
String normalized = normalizeFileUrl(fileUrl);
|
|
||||||
try {
|
|
||||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
|
||||||
byte[] bytes = digest.digest(normalized.getBytes(StandardCharsets.UTF_8));
|
|
||||||
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
|
||||||
for (byte b : bytes) {
|
|
||||||
sb.append(String.format("%02x", b));
|
|
||||||
}
|
|
||||||
return sb.toString();
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("鐢熸垚鏂囦欢缂撳瓨閿け璐?");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private String normalizePhase(String phase) {
|
private String normalizePhase(String phase) {
|
||||||
if (phase == null || phase.isBlank()) {
|
if (phase == null || phase.isBlank()) {
|
||||||
return PHASE_CRAWLING;
|
return PHASE_CRAWLING;
|
||||||
@@ -432,7 +130,4 @@ public class BrandTaskProgressCacheService {
|
|||||||
private String blankToEmpty(String value) {
|
private String blankToEmpty(String value) {
|
||||||
return value == null ? "" : value.trim();
|
return value == null ? "" : value.trim();
|
||||||
}
|
}
|
||||||
|
|
||||||
public record ChunkStoreResult(boolean stored, boolean newlyCompletedFile, int finishedFiles, BrandFileAggregateCacheDto aggregate) {
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ import org.apache.poi.ss.usermodel.Row;
|
|||||||
import org.apache.poi.ss.usermodel.Sheet;
|
import org.apache.poi.ss.usermodel.Sheet;
|
||||||
import org.apache.poi.ss.usermodel.Workbook;
|
import org.apache.poi.ss.usermodel.Workbook;
|
||||||
import org.apache.poi.ss.usermodel.WorkbookFactory;
|
import org.apache.poi.ss.usermodel.WorkbookFactory;
|
||||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.transaction.annotation.Transactional;
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
|
||||||
@@ -83,6 +83,7 @@ public class BrandTaskService {
|
|||||||
private final StorageProperties storageProperties;
|
private final StorageProperties storageProperties;
|
||||||
private final BrandProgressProperties brandProgressProperties;
|
private final BrandProgressProperties brandProgressProperties;
|
||||||
private final BrandTaskProgressCacheService brandTaskProgressCacheService;
|
private final BrandTaskProgressCacheService brandTaskProgressCacheService;
|
||||||
|
private final BrandTaskStorageService brandTaskStorageService;
|
||||||
private final DistributedJobLockService distributedJobLockService;
|
private final DistributedJobLockService distributedJobLockService;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
|
|
||||||
@@ -193,7 +194,7 @@ public class BrandTaskService {
|
|||||||
cachedFiles.add(cacheDto);
|
cachedFiles.add(cacheDto);
|
||||||
}
|
}
|
||||||
|
|
||||||
brandTaskProgressCacheService.saveParsedPayload(taskId, cachedFiles);
|
brandTaskStorageService.saveParsedPayload(taskId, cachedFiles);
|
||||||
|
|
||||||
brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
|
brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
|
||||||
.eq(BrandCrawlTaskEntity::getId, taskId)
|
.eq(BrandCrawlTaskEntity::getId, taskId)
|
||||||
@@ -226,9 +227,7 @@ public class BrandTaskService {
|
|||||||
sourceByUrl.put(file.getFileUrl(), file);
|
sourceByUrl.put(file.getFileUrl(), file);
|
||||||
sourceIndexByUrl.put(file.getFileUrl(), i + 1);
|
sourceIndexByUrl.put(file.getFileUrl(), i + 1);
|
||||||
}
|
}
|
||||||
List<BrandParsedFileCacheDto> cachedFiles = brandTaskProgressCacheService.getParsedPayload(taskId,
|
List<BrandParsedFileCacheDto> cachedFiles = brandTaskStorageService.getParsedPayload(taskId);
|
||||||
new TypeReference<List<BrandParsedFileCacheDto>>() {
|
|
||||||
});
|
|
||||||
if (cachedFiles == null || cachedFiles.isEmpty()) {
|
if (cachedFiles == null || cachedFiles.isEmpty()) {
|
||||||
throw new BusinessException("任务原始数据已过期,请重新创建任务");
|
throw new BusinessException("任务原始数据已过期,请重新创建任务");
|
||||||
}
|
}
|
||||||
@@ -252,7 +251,7 @@ public class BrandTaskService {
|
|||||||
totalCount,
|
totalCount,
|
||||||
Thread.currentThread().getName());
|
Thread.currentThread().getName());
|
||||||
|
|
||||||
int finishedCount = brandTaskProgressCacheService.countCompletedFiles(taskId);
|
int finishedCount = brandTaskStorageService.countCompletedFiles(taskId);
|
||||||
for (BrandCrawlResultFileDto resultFile : resultFiles) {
|
for (BrandCrawlResultFileDto resultFile : resultFiles) {
|
||||||
String fileUrl = blankToNull(resultFile.getFileUrl());
|
String fileUrl = blankToNull(resultFile.getFileUrl());
|
||||||
if (fileUrl == null) {
|
if (fileUrl == null) {
|
||||||
@@ -272,7 +271,7 @@ public class BrandTaskService {
|
|||||||
} else if (totalLines > 0 && resultFile.getTotalLines() > totalLines) {
|
} else if (totalLines > 0 && resultFile.getTotalLines() > totalLines) {
|
||||||
throw new BusinessException("totalLines 超过原始文件行数: " + fileUrl);
|
throw new BusinessException("totalLines 超过原始文件行数: " + fileUrl);
|
||||||
}
|
}
|
||||||
BrandTaskProgressCacheService.ChunkStoreResult storeResult = brandTaskProgressCacheService.storeChunk(taskId, resultFile);
|
BrandTaskStorageService.ChunkStoreResult storeResult = brandTaskStorageService.storeChunk(taskId, resultFile);
|
||||||
finishedCount = Math.max(finishedCount, storeResult.finishedFiles());
|
finishedCount = Math.max(finishedCount, storeResult.finishedFiles());
|
||||||
log.info("[brand-submit] taskId={} fileUrl={} chunk={}/{} stored={} newlyCompletedFile={} finishedFiles={}",
|
log.info("[brand-submit] taskId={} fileUrl={} chunk={}/{} stored={} newlyCompletedFile={} finishedFiles={}",
|
||||||
taskId,
|
taskId,
|
||||||
@@ -316,6 +315,7 @@ public class BrandTaskService {
|
|||||||
if (updated == 0) {
|
if (updated == 0) {
|
||||||
throw new BusinessException("任务不存在或无法取消");
|
throw new BusinessException("任务不存在或无法取消");
|
||||||
}
|
}
|
||||||
|
brandTaskStorageService.deleteTaskData(taskId);
|
||||||
brandTaskProgressCacheService.delete(taskId);
|
brandTaskProgressCacheService.delete(taskId);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -327,6 +327,7 @@ public class BrandTaskService {
|
|||||||
if (deleted == 0) {
|
if (deleted == 0) {
|
||||||
throw new BusinessException("任务不存在或正在执行中无法删除");
|
throw new BusinessException("任务不存在或正在执行中无法删除");
|
||||||
}
|
}
|
||||||
|
brandTaskStorageService.deleteTaskData(taskId);
|
||||||
brandTaskProgressCacheService.delete(taskId);
|
brandTaskProgressCacheService.delete(taskId);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -555,7 +556,7 @@ public class BrandTaskService {
|
|||||||
if (STATUS_FAILED.equalsIgnoreCase(freshStatus)) {
|
if (STATUS_FAILED.equalsIgnoreCase(freshStatus)) {
|
||||||
throw new BusinessException("任务已结束,不能继续组装结果");
|
throw new BusinessException("任务已结束,不能继续组装结果");
|
||||||
}
|
}
|
||||||
Map<String, BrandFileAggregateCacheDto> aggregates = brandTaskProgressCacheService.getAllFileAggregates(taskId);
|
Map<String, BrandFileAggregateCacheDto> aggregates = brandTaskStorageService.getAllFileAggregates(taskId);
|
||||||
log.info("[brand-finalize] taskId={} aggregateFiles={} expectedFiles={} thread={}",
|
log.info("[brand-finalize] taskId={} aggregateFiles={} expectedFiles={} thread={}",
|
||||||
taskId,
|
taskId,
|
||||||
aggregates.size(),
|
aggregates.size(),
|
||||||
@@ -600,9 +601,7 @@ public class BrandTaskService {
|
|||||||
if (updated == 0) {
|
if (updated == 0) {
|
||||||
throw new BusinessException("任务已取消");
|
throw new BusinessException("任务已取消");
|
||||||
}
|
}
|
||||||
for (BrandSourceFileDto sourceFile : sourceFiles) {
|
brandTaskStorageService.deleteTaskData(taskId);
|
||||||
brandTaskProgressCacheService.deleteFileState(taskId, sourceFile.getFileUrl());
|
|
||||||
}
|
|
||||||
brandTaskProgressCacheService.delete(taskId);
|
brandTaskProgressCacheService.delete(taskId);
|
||||||
log.info("[brand-finalize] taskId={} finalized success files={} elapsedMs={}",
|
log.info("[brand-finalize] taskId={} finalized success files={} elapsedMs={}",
|
||||||
taskId,
|
taskId,
|
||||||
@@ -614,7 +613,7 @@ public class BrandTaskService {
|
|||||||
.eq(BrandCrawlTaskEntity::getId, taskId)
|
.eq(BrandCrawlTaskEntity::getId, taskId)
|
||||||
.ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED)
|
.ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED)
|
||||||
.set(BrandCrawlTaskEntity::getStatus, STATUS_FAILED)
|
.set(BrandCrawlTaskEntity::getStatus, STATUS_FAILED)
|
||||||
.set(BrandCrawlTaskEntity::getProgressCurrent, brandTaskProgressCacheService.countCompletedFiles(taskId))
|
.set(BrandCrawlTaskEntity::getProgressCurrent, brandTaskStorageService.countCompletedFiles(taskId))
|
||||||
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
|
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
|
||||||
.set(BrandCrawlTaskEntity::getErrorMessage, ex.getMessage()));
|
.set(BrandCrawlTaskEntity::getErrorMessage, ex.getMessage()));
|
||||||
brandTaskProgressCacheService.markFailed(taskId, ex.getMessage());
|
brandTaskProgressCacheService.markFailed(taskId, ex.getMessage());
|
||||||
@@ -622,6 +621,8 @@ public class BrandTaskService {
|
|||||||
throw businessException;
|
throw businessException;
|
||||||
}
|
}
|
||||||
throw new BusinessException(ex.getMessage());
|
throw new BusinessException(ex.getMessage());
|
||||||
|
} finally {
|
||||||
|
cleanupBrandResultTempFiles(outputEntries, outputDir);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -746,7 +747,7 @@ public class BrandTaskService {
|
|||||||
String value = index == null ? "" : normalizeCellText(formatter.formatCellValue(row.getCell(index)));
|
String value = index == null ? "" : normalizeCellText(formatter.formatCellValue(row.getCell(index)));
|
||||||
rowData.put(column, value);
|
rowData.put(column, value);
|
||||||
}
|
}
|
||||||
String brand = normalizeCellText(Objects.toString(rowData.getOrDefault("\u54c1\u724c", ""), ""));
|
String brand = normalizeCellText(Objects.toString(rowData.getOrDefault("品牌", ""), ""));
|
||||||
if (brand.isBlank()) {
|
if (brand.isBlank()) {
|
||||||
rowData.put("__rowIndex", rowNum + 1);
|
rowData.put("__rowIndex", rowNum + 1);
|
||||||
rows.add(rowData);
|
rows.add(rowData);
|
||||||
@@ -827,7 +828,9 @@ public class BrandTaskService {
|
|||||||
BrandParsedFileCacheDto cachedFile,
|
BrandParsedFileCacheDto cachedFile,
|
||||||
BrandFileAggregateCacheDto resultFile) throws IOException {
|
BrandFileAggregateCacheDto resultFile) throws IOException {
|
||||||
String actualStrategy = normalizeStrategy(strategy);
|
String actualStrategy = normalizeStrategy(strategy);
|
||||||
try (XSSFWorkbook workbook = new XSSFWorkbook()) {
|
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
|
||||||
|
workbook.setCompressTempFiles(true);
|
||||||
|
try {
|
||||||
String mainSheetName = blankToDefault(cachedFile.getSheetName(), "Sheet1");
|
String mainSheetName = blankToDefault(cachedFile.getSheetName(), "Sheet1");
|
||||||
var mainSheet = workbook.createSheet(mainSheetName);
|
var mainSheet = workbook.createSheet(mainSheetName);
|
||||||
var headerRow = mainSheet.createRow(0);
|
var headerRow = mainSheet.createRow(0);
|
||||||
@@ -878,8 +881,35 @@ public class BrandTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
try (FileOutputStream outputStream = new FileOutputStream(outputFile)) {
|
try (FileOutputStream outputStream = new FileOutputStream(outputFile)) {
|
||||||
|
applyBrandSheetWidths(mainSheet, columns.size());
|
||||||
|
applyFixedColumnWidths(invalidSheet, new int[]{20, 12, 18});
|
||||||
|
applyFixedColumnWidths(queryFailedSheet, new int[]{20, 18});
|
||||||
workbook.write(outputStream);
|
workbook.write(outputStream);
|
||||||
}
|
}
|
||||||
|
} finally {
|
||||||
|
try {
|
||||||
|
workbook.close();
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
}
|
||||||
|
workbook.dispose();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void applyBrandSheetWidths(Sheet sheet, int columnCount) {
|
||||||
|
if (sheet == null || columnCount <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < columnCount; i++) {
|
||||||
|
sheet.setColumnWidth(i, 20 * 256);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void applyFixedColumnWidths(Sheet sheet, int[] widths) {
|
||||||
|
if (sheet == null || widths == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < widths.length; i++) {
|
||||||
|
sheet.setColumnWidth(i, widths[i] * 256);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -946,6 +976,43 @@ public class BrandTaskService {
|
|||||||
return zipFile;
|
return zipFile;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void cleanupBrandResultTempFiles(List<OutputEntry> entries, File outputDir) {
|
||||||
|
if (entries != null) {
|
||||||
|
for (OutputEntry entry : entries) {
|
||||||
|
if (entry == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
deleteQuietly(entry.resultFile());
|
||||||
|
if (isManagedBrandSourceTempFile(entry.sourceFile())) {
|
||||||
|
deleteQuietly(entry.sourceFile());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
deleteQuietly(outputDir);
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isManagedBrandSourceTempFile(File file) {
|
||||||
|
if (file == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
File managedDir = FileUtil.file(storageProperties.getLocalTempDir(), "brand-source-download");
|
||||||
|
try {
|
||||||
|
return file.getCanonicalPath().startsWith(managedDir.getCanonicalPath());
|
||||||
|
} catch (IOException ex) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void deleteQuietly(File file) {
|
||||||
|
if (file == null || !file.exists()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
FileUtil.del(file);
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private File buildNamedOutputFile(File outputDir, String filename) {
|
private File buildNamedOutputFile(File outputDir, String filename) {
|
||||||
File candidate = FileUtil.file(outputDir, filename);
|
File candidate = FileUtil.file(outputDir, filename);
|
||||||
if (!candidate.exists()) {
|
if (!candidate.exists()) {
|
||||||
|
|||||||
@@ -0,0 +1,475 @@
|
|||||||
|
package com.nanri.aiimage.modules.brand.service;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultFileDto;
|
||||||
|
import com.nanri.aiimage.modules.brand.model.dto.BrandFileAggregateCacheDto;
|
||||||
|
import com.nanri.aiimage.modules.brand.model.dto.BrandParsedFileCacheDto;
|
||||||
|
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import org.springframework.dao.DuplicateKeyException;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.security.MessageDigest;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class BrandTaskStorageService {
|
||||||
|
|
||||||
|
private static final String MODULE_TYPE = "BRAND";
|
||||||
|
private static final String OSS_POINTER_PREFIX = "oss:";
|
||||||
|
|
||||||
|
private final TaskChunkMapper taskChunkMapper;
|
||||||
|
private final TaskScopeStateMapper taskScopeStateMapper;
|
||||||
|
private final ObjectMapper objectMapper;
|
||||||
|
private final OssStorageService ossStorageService;
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public void saveParsedPayload(Long taskId, List<BrandParsedFileCacheDto> payload) {
|
||||||
|
if (taskId == null || taskId <= 0 || payload == null || payload.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
LocalDateTime now = LocalDateTime.now();
|
||||||
|
for (BrandParsedFileCacheDto item : payload) {
|
||||||
|
if (item == null || isBlank(item.getFileUrl())) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
String scopeKey = normalize(item.getFileUrl());
|
||||||
|
String scopeHash = hash(scopeKey);
|
||||||
|
String parsedJson = storeParsedPayload(taskId, scopeHash, writeJson(item, "Save brand task parsed payload failed"));
|
||||||
|
TaskScopeStateEntity state = getScopeState(taskId, scopeHash);
|
||||||
|
if (state == null) {
|
||||||
|
TaskScopeStateEntity entity = new TaskScopeStateEntity();
|
||||||
|
entity.setTaskId(taskId);
|
||||||
|
entity.setModuleType(MODULE_TYPE);
|
||||||
|
entity.setScopeKey(scopeKey);
|
||||||
|
entity.setScopeHash(scopeHash);
|
||||||
|
entity.setParsedPayloadJson(parsedJson);
|
||||||
|
entity.setChunkTotal(0);
|
||||||
|
entity.setReceivedChunkCount(0);
|
||||||
|
entity.setCompleted(0);
|
||||||
|
entity.setCreatedAt(now);
|
||||||
|
entity.setUpdatedAt(now);
|
||||||
|
try {
|
||||||
|
taskScopeStateMapper.insert(entity);
|
||||||
|
continue;
|
||||||
|
} catch (DuplicateKeyException ignored) {
|
||||||
|
state = getScopeState(taskId, scopeHash);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (state == null) {
|
||||||
|
throw new BusinessException("Save brand task parsed payload failed");
|
||||||
|
}
|
||||||
|
deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), parsedJson);
|
||||||
|
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getId, state.getId())
|
||||||
|
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
|
||||||
|
.set(TaskScopeStateEntity::getParsedPayloadJson, parsedJson)
|
||||||
|
.set(TaskScopeStateEntity::getUpdatedAt, now));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<BrandParsedFileCacheDto> getParsedPayload(Long taskId) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.select(TaskScopeStateEntity::getParsedPayloadJson)
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.isNotNull(TaskScopeStateEntity::getParsedPayloadJson));
|
||||||
|
if (states == null || states.isEmpty()) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
List<BrandParsedFileCacheDto> result = new ArrayList<>();
|
||||||
|
for (TaskScopeStateEntity state : states) {
|
||||||
|
String payloadJson = resolveParsedPayload(state.getParsedPayloadJson());
|
||||||
|
if (isBlank(payloadJson)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
result.add(objectMapper.readValue(payloadJson, BrandParsedFileCacheDto.class));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException("Read brand task parsed payload failed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public ChunkStoreResult storeChunk(Long taskId, BrandCrawlResultFileDto file) {
|
||||||
|
validateChunk(file);
|
||||||
|
String scopeKey = normalize(file.getFileUrl());
|
||||||
|
String scopeHash = hash(scopeKey);
|
||||||
|
String payloadJson = writeJson(file, "Save brand task chunk failed");
|
||||||
|
String payloadHash = hash(payloadJson);
|
||||||
|
LocalDateTime now = LocalDateTime.now();
|
||||||
|
|
||||||
|
TaskChunkEntity existingChunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.eq(TaskChunkEntity::getScopeHash, scopeHash)
|
||||||
|
.eq(TaskChunkEntity::getChunkIndex, file.getChunkIndex())
|
||||||
|
.last("limit 1"));
|
||||||
|
if (existingChunk != null) {
|
||||||
|
if (payloadHash.equals(existingChunk.getPayloadHash())) {
|
||||||
|
BrandFileAggregateCacheDto aggregate = rebuildAggregate(taskId, scopeKey, scopeHash);
|
||||||
|
return new ChunkStoreResult(false, false, countCompletedFiles(taskId), aggregate);
|
||||||
|
}
|
||||||
|
throw new BusinessException("Brand task chunk conflict: " + scopeKey + "#" + file.getChunkIndex());
|
||||||
|
}
|
||||||
|
|
||||||
|
TaskChunkEntity entity = new TaskChunkEntity();
|
||||||
|
entity.setTaskId(taskId);
|
||||||
|
entity.setModuleType(MODULE_TYPE);
|
||||||
|
entity.setScopeKey(scopeKey);
|
||||||
|
entity.setScopeHash(scopeHash);
|
||||||
|
entity.setChunkIndex(file.getChunkIndex());
|
||||||
|
entity.setChunkTotal(file.getChunkTotal());
|
||||||
|
entity.setPayloadJson(payloadJson);
|
||||||
|
entity.setPayloadHash(payloadHash);
|
||||||
|
entity.setCreatedAt(now);
|
||||||
|
entity.setUpdatedAt(now);
|
||||||
|
try {
|
||||||
|
taskChunkMapper.insert(entity);
|
||||||
|
} catch (DuplicateKeyException ex) {
|
||||||
|
TaskChunkEntity concurrentChunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.eq(TaskChunkEntity::getScopeHash, scopeHash)
|
||||||
|
.eq(TaskChunkEntity::getChunkIndex, file.getChunkIndex())
|
||||||
|
.last("limit 1"));
|
||||||
|
if (concurrentChunk != null && payloadHash.equals(concurrentChunk.getPayloadHash())) {
|
||||||
|
BrandFileAggregateCacheDto aggregate = rebuildAggregate(taskId, scopeKey, scopeHash);
|
||||||
|
return new ChunkStoreResult(false, false, countCompletedFiles(taskId), aggregate);
|
||||||
|
}
|
||||||
|
throw new BusinessException("Brand task chunk conflict: " + scopeKey + "#" + file.getChunkIndex());
|
||||||
|
}
|
||||||
|
|
||||||
|
BrandFileAggregateCacheDto aggregate = rebuildAggregate(taskId, scopeKey, scopeHash);
|
||||||
|
boolean completed = Boolean.TRUE.equals(aggregate.getCompleted());
|
||||||
|
saveAggregate(taskId, scopeKey, scopeHash, aggregate, now);
|
||||||
|
return new ChunkStoreResult(true, completed, countCompletedFiles(taskId), aggregate);
|
||||||
|
}
|
||||||
|
|
||||||
|
public BrandFileAggregateCacheDto getFileAggregate(Long taskId, String fileUrl) {
|
||||||
|
TaskScopeStateEntity state = getScopeState(taskId, hash(normalize(fileUrl)));
|
||||||
|
if (state == null || isBlank(state.getStateJson())) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return objectMapper.readValue(state.getStateJson(), BrandFileAggregateCacheDto.class);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException("Read brand task aggregate failed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public Map<String, BrandFileAggregateCacheDto> getAllFileAggregates(Long taskId) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
return Map.of();
|
||||||
|
}
|
||||||
|
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.select(TaskScopeStateEntity::getScopeKey, TaskScopeStateEntity::getStateJson)
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.isNotNull(TaskScopeStateEntity::getStateJson));
|
||||||
|
if (states == null || states.isEmpty()) {
|
||||||
|
return Map.of();
|
||||||
|
}
|
||||||
|
Map<String, BrandFileAggregateCacheDto> result = new LinkedHashMap<>();
|
||||||
|
for (TaskScopeStateEntity state : states) {
|
||||||
|
if (isBlank(state.getStateJson()) || isBlank(state.getScopeKey())) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
result.put(state.getScopeKey(), objectMapper.readValue(state.getStateJson(), BrandFileAggregateCacheDto.class));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException("Read brand task aggregate failed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int countCompletedFiles(Long taskId) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.eq(TaskScopeStateEntity::getCompleted, 1));
|
||||||
|
return count == null ? 0 : count.intValue();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public void deleteTaskData(Long taskId) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.select(TaskScopeStateEntity::getParsedPayloadJson)
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
|
||||||
|
if (states != null) {
|
||||||
|
for (TaskScopeStateEntity state : states) {
|
||||||
|
deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), null);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||||
|
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
|
||||||
|
}
|
||||||
|
|
||||||
|
private void saveAggregate(Long taskId,
|
||||||
|
String scopeKey,
|
||||||
|
String scopeHash,
|
||||||
|
BrandFileAggregateCacheDto aggregate,
|
||||||
|
LocalDateTime now) {
|
||||||
|
String aggregateJson = writeJson(aggregate, "Save brand task aggregate failed");
|
||||||
|
TaskScopeStateEntity state = getScopeState(taskId, scopeHash);
|
||||||
|
if (state == null) {
|
||||||
|
throw new BusinessException("Brand task aggregate state missing: " + scopeKey);
|
||||||
|
}
|
||||||
|
int chunkTotal = aggregate.getChunkTotal() == null ? 0 : aggregate.getChunkTotal();
|
||||||
|
int receivedCount = aggregate.getReceivedChunkCount() == null ? 0 : aggregate.getReceivedChunkCount();
|
||||||
|
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getId, state.getId())
|
||||||
|
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
|
||||||
|
.set(TaskScopeStateEntity::getStateJson, aggregateJson)
|
||||||
|
.set(TaskScopeStateEntity::getChunkTotal, chunkTotal)
|
||||||
|
.set(TaskScopeStateEntity::getReceivedChunkCount, receivedCount)
|
||||||
|
.set(TaskScopeStateEntity::getCompleted, Boolean.TRUE.equals(aggregate.getCompleted()) ? 1 : 0)
|
||||||
|
.set(TaskScopeStateEntity::getLastChunkAt, now)
|
||||||
|
.set(TaskScopeStateEntity::getLastError, null)
|
||||||
|
.set(TaskScopeStateEntity::getUpdatedAt, now));
|
||||||
|
}
|
||||||
|
|
||||||
|
private TaskScopeStateEntity getScopeState(Long taskId, String scopeHash) {
|
||||||
|
return taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.eq(TaskScopeStateEntity::getScopeHash, scopeHash)
|
||||||
|
.last("limit 1"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private BrandFileAggregateCacheDto createAggregate(BrandCrawlResultFileDto file) {
|
||||||
|
BrandFileAggregateCacheDto aggregate = new BrandFileAggregateCacheDto();
|
||||||
|
aggregate.setFileUrl(normalize(file.getFileUrl()));
|
||||||
|
aggregate.setOriginalFilename(file.getOriginalFilename());
|
||||||
|
aggregate.setRelativePath(file.getRelativePath());
|
||||||
|
aggregate.setMainSheetName(file.getMainSheetName());
|
||||||
|
aggregate.setChunkTotal(file.getChunkTotal());
|
||||||
|
aggregate.setTotalLines(safePositive(file.getTotalLines()));
|
||||||
|
aggregate.setReceivedChunkCount(0);
|
||||||
|
aggregate.setProcessedLineCount(0);
|
||||||
|
aggregate.setCompleted(false);
|
||||||
|
aggregate.setInvalidBrands(new ArrayList<>());
|
||||||
|
aggregate.setQueryFailedBrands(new ArrayList<>());
|
||||||
|
return aggregate;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ensureChunkConsistency(BrandFileAggregateCacheDto aggregate, BrandCrawlResultFileDto file) {
|
||||||
|
if (aggregate.getChunkTotal() != null && file.getChunkTotal() != null && !aggregate.getChunkTotal().equals(file.getChunkTotal())) {
|
||||||
|
throw new BusinessException("Brand task chunkTotal mismatch: " + aggregate.getFileUrl());
|
||||||
|
}
|
||||||
|
if (aggregate.getChunkTotal() == null) {
|
||||||
|
aggregate.setChunkTotal(file.getChunkTotal());
|
||||||
|
}
|
||||||
|
int incomingTotalLines = safePositive(file.getTotalLines());
|
||||||
|
if (incomingTotalLines > 0) {
|
||||||
|
aggregate.setTotalLines(Math.max(safePositive(aggregate.getTotalLines()), incomingTotalLines));
|
||||||
|
}
|
||||||
|
if (isBlank(aggregate.getOriginalFilename()) && !isBlank(file.getOriginalFilename())) {
|
||||||
|
aggregate.setOriginalFilename(file.getOriginalFilename());
|
||||||
|
}
|
||||||
|
if (isBlank(aggregate.getRelativePath()) && !isBlank(file.getRelativePath())) {
|
||||||
|
aggregate.setRelativePath(file.getRelativePath());
|
||||||
|
}
|
||||||
|
if (isBlank(aggregate.getMainSheetName()) && !isBlank(file.getMainSheetName())) {
|
||||||
|
aggregate.setMainSheetName(file.getMainSheetName());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void mergeAggregate(BrandFileAggregateCacheDto aggregate, BrandCrawlResultFileDto file) {
|
||||||
|
int processed = safePositive(aggregate.getProcessedLineCount())
|
||||||
|
+ sizeOf(file.getKeptRows())
|
||||||
|
+ sizeOf(file.getInvalidBrands())
|
||||||
|
+ sizeOf(file.getQueryFailedBrands());
|
||||||
|
aggregate.setProcessedLineCount(processed);
|
||||||
|
if (file.getInvalidBrands() != null && !file.getInvalidBrands().isEmpty()) {
|
||||||
|
aggregate.getInvalidBrands().addAll(file.getInvalidBrands());
|
||||||
|
}
|
||||||
|
if (file.getQueryFailedBrands() != null && !file.getQueryFailedBrands().isEmpty()) {
|
||||||
|
aggregate.getQueryFailedBrands().addAll(file.getQueryFailedBrands());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private BrandFileAggregateCacheDto rebuildAggregate(Long taskId, String scopeKey, String scopeHash) {
|
||||||
|
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.eq(TaskChunkEntity::getScopeHash, scopeHash)
|
||||||
|
.orderByAsc(TaskChunkEntity::getChunkIndex, TaskChunkEntity::getId));
|
||||||
|
if (chunks == null || chunks.isEmpty()) {
|
||||||
|
BrandFileAggregateCacheDto aggregate = getFileAggregate(taskId, scopeKey);
|
||||||
|
if (aggregate != null) {
|
||||||
|
aggregate.setFileUrl(scopeKey);
|
||||||
|
aggregate.setReceivedChunkCount(0);
|
||||||
|
aggregate.setProcessedLineCount(0);
|
||||||
|
aggregate.setCompleted(false);
|
||||||
|
aggregate.setInvalidBrands(new ArrayList<>());
|
||||||
|
aggregate.setQueryFailedBrands(new ArrayList<>());
|
||||||
|
}
|
||||||
|
return aggregate;
|
||||||
|
}
|
||||||
|
|
||||||
|
BrandFileAggregateCacheDto aggregate = null;
|
||||||
|
for (TaskChunkEntity chunk : chunks) {
|
||||||
|
BrandCrawlResultFileDto chunkPayload = readChunkPayload(chunk.getPayloadJson());
|
||||||
|
if (aggregate == null) {
|
||||||
|
aggregate = createAggregate(chunkPayload);
|
||||||
|
} else {
|
||||||
|
ensureChunkConsistency(aggregate, chunkPayload);
|
||||||
|
}
|
||||||
|
mergeAggregate(aggregate, chunkPayload);
|
||||||
|
}
|
||||||
|
if (aggregate == null) {
|
||||||
|
throw new BusinessException("Read brand task aggregate data failed");
|
||||||
|
}
|
||||||
|
aggregate.setFileUrl(scopeKey);
|
||||||
|
aggregate.setReceivedChunkCount(chunks.size());
|
||||||
|
aggregate.setCompleted(aggregate.getChunkTotal() != null
|
||||||
|
&& aggregate.getChunkTotal() > 0
|
||||||
|
&& chunks.size() >= aggregate.getChunkTotal());
|
||||||
|
return aggregate;
|
||||||
|
}
|
||||||
|
|
||||||
|
private BrandCrawlResultFileDto readChunkPayload(String payloadJson) {
|
||||||
|
try {
|
||||||
|
return objectMapper.readValue(payloadJson, BrandCrawlResultFileDto.class);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException("Read brand task chunk failed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void validateChunk(BrandCrawlResultFileDto file) {
|
||||||
|
if (file == null) {
|
||||||
|
throw new BusinessException("Brand task chunk must not be null");
|
||||||
|
}
|
||||||
|
if (file.getChunkIndex() == null || file.getChunkTotal() == null || file.getChunkIndex() <= 0
|
||||||
|
|| file.getChunkTotal() <= 0 || file.getChunkIndex() > file.getChunkTotal()) {
|
||||||
|
throw new BusinessException("Brand task chunk parameters are invalid");
|
||||||
|
}
|
||||||
|
if (isBlank(file.getFileUrl())) {
|
||||||
|
throw new BusinessException("fileUrl must not be blank");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String storeParsedPayload(Long taskId, String scopeHash, String payloadJson) {
|
||||||
|
try {
|
||||||
|
String pointer = OSS_POINTER_PREFIX + ossStorageService.uploadTaskParsedPayload(MODULE_TYPE, taskId, scopeHash, payloadJson);
|
||||||
|
return objectMapper.writeValueAsString(pointer);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException("Save brand task parsed payload failed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveParsedPayload(String value) {
|
||||||
|
if (isBlank(value)) {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
String ossPointer = extractOssPointer(value);
|
||||||
|
if (ossPointer == null) {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return ossStorageService.readObjectAsString(ossPointer.substring(OSS_POINTER_PREFIX.length()));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException("Read brand task parsed payload failed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void deleteParsedPayloadIfNeeded(String oldValue, String newValue) {
|
||||||
|
String oldPointer = extractOssPointer(oldValue);
|
||||||
|
String newPointer = extractOssPointer(newValue);
|
||||||
|
if (oldPointer == null || oldPointer.equals(newPointer)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
ossStorageService.deleteObject(oldPointer.substring(OSS_POINTER_PREFIX.length()));
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String extractOssPointer(String value) {
|
||||||
|
if (isBlank(value)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (value.startsWith(OSS_POINTER_PREFIX)) {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
String decoded = objectMapper.readValue(value, String.class);
|
||||||
|
return decoded != null && decoded.startsWith(OSS_POINTER_PREFIX) ? decoded : null;
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String writeJson(Object value, String message) {
|
||||||
|
try {
|
||||||
|
return objectMapper.writeValueAsString(value);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException(message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalize(String value) {
|
||||||
|
return value == null ? "" : value.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isBlank(String value) {
|
||||||
|
return value == null || value.isBlank();
|
||||||
|
}
|
||||||
|
|
||||||
|
private int sizeOf(List<?> list) {
|
||||||
|
return list == null ? 0 : list.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
private int safePositive(Integer value) {
|
||||||
|
return value == null || value <= 0 ? 0 : value;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String hash(String value) {
|
||||||
|
try {
|
||||||
|
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||||
|
byte[] bytes = digest.digest(normalize(value).getBytes(StandardCharsets.UTF_8));
|
||||||
|
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
||||||
|
for (byte b : bytes) {
|
||||||
|
sb.append(String.format("%02x", b));
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new IllegalStateException("failed to hash brand scope", ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public record ChunkStoreResult(boolean stored, boolean newlyCompletedFile, int finishedFiles, BrandFileAggregateCacheDto aggregate) {
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -44,11 +44,11 @@ public class ConvertRunService {
|
|||||||
private static final String MODULE_TYPE = "CONVERT";
|
private static final String MODULE_TYPE = "CONVERT";
|
||||||
private static final String TEMPLATE_CODE_FIVE_COUNTRIES = "uk_offer";
|
private static final String TEMPLATE_CODE_FIVE_COUNTRIES = "uk_offer";
|
||||||
private static final List<String> FIVE_COUNTRY_OUTPUT_FILES = List.of(
|
private static final List<String> FIVE_COUNTRY_OUTPUT_FILES = List.of(
|
||||||
"\u82f1\u56fd.txt",
|
"英国.txt",
|
||||||
"\u6cd5\u56fd.txt",
|
"法国.txt",
|
||||||
"\u5fb7\u56fd.txt",
|
"德国.txt",
|
||||||
"\u897f\u73ed\u7259.txt",
|
"西班牙.txt",
|
||||||
"\u610f\u5927\u5229.txt"
|
"意大利.txt"
|
||||||
);
|
);
|
||||||
|
|
||||||
private final FileTaskMapper fileTaskMapper;
|
private final FileTaskMapper fileTaskMapper;
|
||||||
|
|||||||
@@ -461,8 +461,8 @@ public class DedupeRunService {
|
|||||||
if (value == null) {
|
if (value == null) {
|
||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
return value.replace("\ufeff", "")
|
return value.replace("", "")
|
||||||
.replace("\u3000", " ")
|
.replace(" ", " ")
|
||||||
.replace("\r\n", " ")
|
.replace("\r\n", " ")
|
||||||
.replace("\r", " ")
|
.replace("\r", " ")
|
||||||
.replace("\n", " ")
|
.replace("\n", " ")
|
||||||
|
|||||||
@@ -229,7 +229,7 @@ public class DedupeTotalDataService {
|
|||||||
progress.setStatus("success");
|
progress.setStatus("success");
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
progress.setStatus("failed");
|
progress.setStatus("failed");
|
||||||
progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "鍒犻櫎 Excel 鍖归厤鏁版嵁澶辫触");
|
progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "删除 Excel 匹配数据失败");
|
||||||
} finally {
|
} finally {
|
||||||
deleteQuietly(tempFile);
|
deleteQuietly(tempFile);
|
||||||
}
|
}
|
||||||
@@ -251,7 +251,7 @@ public class DedupeTotalDataService {
|
|||||||
progress.setStatus("success");
|
progress.setStatus("success");
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
progress.setStatus("failed");
|
progress.setStatus("failed");
|
||||||
progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "瀵煎叆 Excel 澶辫触");
|
progress.setErrorMessage(e instanceof BusinessException ? e.getMessage() : "导入 Excel 失败");
|
||||||
} finally {
|
} finally {
|
||||||
deleteQuietly(tempFile);
|
deleteQuietly(tempFile);
|
||||||
}
|
}
|
||||||
@@ -590,8 +590,8 @@ public class DedupeTotalDataService {
|
|||||||
if (value == null) {
|
if (value == null) {
|
||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
return value.replace("\ufeff", "")
|
return value.replace("", "")
|
||||||
.replace("\u3000", " ")
|
.replace(" ", " ")
|
||||||
.replace("\r\n", " ")
|
.replace("\r\n", " ")
|
||||||
.replace("\r", " ")
|
.replace("\r", " ")
|
||||||
.replace("\n", " ")
|
.replace("\n", " ")
|
||||||
|
|||||||
@@ -135,9 +135,10 @@ public class DeleteBrandRunController {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
|
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
|
||||||
|
String asciiFilename = buildAsciiDownloadFilename(filename, taskId);
|
||||||
response.setContentType("application/octet-stream");
|
response.setContentType("application/octet-stream");
|
||||||
response.setHeader(HttpHeaders.CONTENT_DISPOSITION,
|
response.setHeader(HttpHeaders.CONTENT_DISPOSITION,
|
||||||
"attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename);
|
"attachment; filename=\"" + asciiFilename + "\"; filename*=UTF-8''" + encodedFilename);
|
||||||
try (InputStream in = URI.create(url).toURL().openStream()) {
|
try (InputStream in = URI.create(url).toURL().openStream()) {
|
||||||
byte[] buffer = new byte[65536];
|
byte[] buffer = new byte[65536];
|
||||||
int read;
|
int read;
|
||||||
@@ -150,4 +151,24 @@ public class DeleteBrandRunController {
|
|||||||
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "下载失败");
|
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "下载失败");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private String buildAsciiDownloadFilename(String filename, Long taskId) {
|
||||||
|
String sanitized = filename == null ? "" : filename.replaceAll("[^A-Za-z0-9._-]", "_");
|
||||||
|
sanitized = sanitized.replaceAll("_+", "_");
|
||||||
|
sanitized = sanitized.replaceAll("^[_\\.]+|[_\\.]+$", "");
|
||||||
|
if (!sanitized.isBlank() && sanitized.contains(".")) {
|
||||||
|
return sanitized;
|
||||||
|
}
|
||||||
|
String ext = "xlsx";
|
||||||
|
if (filename != null) {
|
||||||
|
int dotIndex = filename.lastIndexOf('.');
|
||||||
|
if (dotIndex >= 0 && dotIndex < filename.length() - 1) {
|
||||||
|
String rawExt = filename.substring(dotIndex + 1).replaceAll("[^A-Za-z0-9]", "");
|
||||||
|
if (!rawExt.isBlank()) {
|
||||||
|
ext = rawExt;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "delete-brand_" + taskId + "." + ext;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ import org.apache.poi.ss.usermodel.DataFormatter;
|
|||||||
import org.apache.poi.ss.usermodel.Row;
|
import org.apache.poi.ss.usermodel.Row;
|
||||||
import org.apache.poi.ss.usermodel.Sheet;
|
import org.apache.poi.ss.usermodel.Sheet;
|
||||||
import org.apache.poi.ss.usermodel.Workbook;
|
import org.apache.poi.ss.usermodel.Workbook;
|
||||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.transaction.annotation.Transactional;
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
|
||||||
@@ -74,6 +74,7 @@ public class DeleteBrandRunService {
|
|||||||
private final FileResultMapper fileResultMapper;
|
private final FileResultMapper fileResultMapper;
|
||||||
private final LocalFileStorageService localFileStorageService;
|
private final LocalFileStorageService localFileStorageService;
|
||||||
private final DeleteBrandTaskCacheService deleteBrandTaskCacheService;
|
private final DeleteBrandTaskCacheService deleteBrandTaskCacheService;
|
||||||
|
private final DeleteBrandTaskStorageService deleteBrandTaskStorageService;
|
||||||
private final ZiniaoShopSwitchService ziniaoShopSwitchService;
|
private final ZiniaoShopSwitchService ziniaoShopSwitchService;
|
||||||
private final OssStorageService ossStorageService;
|
private final OssStorageService ossStorageService;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
@@ -221,9 +222,7 @@ public class DeleteBrandRunService {
|
|||||||
deleteBrandTaskCacheService.saveTaskCache(task);
|
deleteBrandTaskCacheService.saveTaskCache(task);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!parsedPayloadByFileIdentity.isEmpty()) {
|
deleteBrandTaskStorageService.saveParsedPayload(task.getId(), parsedPayloadByFileIdentity);
|
||||||
deleteBrandTaskCacheService.saveParsedPayload(task.getId(), parsedPayloadByFileIdentity);
|
|
||||||
}
|
|
||||||
|
|
||||||
DeleteBrandRunVo vo = new DeleteBrandRunVo();
|
DeleteBrandRunVo vo = new DeleteBrandRunVo();
|
||||||
vo.setTotal(request.getFiles().size());
|
vo.setTotal(request.getFiles().size());
|
||||||
@@ -277,7 +276,7 @@ public class DeleteBrandRunService {
|
|||||||
item.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
|
item.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
|
||||||
item.setError(entity.getErrorMessage());
|
item.setError(entity.getErrorMessage());
|
||||||
|
|
||||||
// 以 task.resultJson 中的项为准补全 matched/shopId/platform/openStoreUrl/错误信息
|
// 以 task.resultJson 中的项为准补充 matched/shopId/platform/openStoreUrl/错误信息
|
||||||
// 真实补充
|
// 真实补充
|
||||||
if (entity.getTaskId() != null) {
|
if (entity.getTaskId() != null) {
|
||||||
item.setTaskStatus(statusByTaskId.get(entity.getTaskId()));
|
item.setTaskStatus(statusByTaskId.get(entity.getTaskId()));
|
||||||
@@ -291,7 +290,7 @@ public class DeleteBrandRunService {
|
|||||||
item.setMatched(candidate.isMatched());
|
item.setMatched(candidate.isMatched());
|
||||||
item.setMatchStatus(candidate.getMatchStatus());
|
item.setMatchStatus(candidate.getMatchStatus());
|
||||||
item.setMatchMessage(candidate.getMatchMessage());
|
item.setMatchMessage(candidate.getMatchMessage());
|
||||||
// 如果物理结果已经是成功了,说明已经处理过了,强制置为匹配成功
|
// 如果物理结果已经成功,说明已经处理过,强制标记为匹配成功
|
||||||
if (item.isSuccess() && candidate.getShopId() != null && !candidate.getShopId().isBlank()) {
|
if (item.isSuccess() && candidate.getShopId() != null && !candidate.getShopId().isBlank()) {
|
||||||
item.setMatched(true);
|
item.setMatched(true);
|
||||||
item.setMatchStatus(ZiniaoShopIndexService.MATCH_STATUS_MATCHED);
|
item.setMatchStatus(ZiniaoShopIndexService.MATCH_STATUS_MATCHED);
|
||||||
@@ -303,8 +302,8 @@ public class DeleteBrandRunService {
|
|||||||
item.setPlatform(candidate.getPlatform());
|
item.setPlatform(candidate.getPlatform());
|
||||||
item.setOpenStoreUrl(candidate.getOpenStoreUrl());
|
item.setOpenStoreUrl(candidate.getOpenStoreUrl());
|
||||||
|
|
||||||
// 历史表可能只有通用 errorMessage;优先返回任务当时的真实 error
|
// 历史表里可能只有通用 errorMessage,优先返回任务当时的真实 error
|
||||||
// 但是,如果物理结果是成功的,就不应该显示所谓的“未匹配”报错
|
// 但如果物理结果已成功,就不应该再显示所谓“未匹配”的报错
|
||||||
if (item.isSuccess()) {
|
if (item.isSuccess()) {
|
||||||
item.setError(null);
|
item.setError(null);
|
||||||
} else if ((item.getError() == null || item.getError().isBlank()) && candidate.getError() != null && !candidate.getError().isBlank()) {
|
} else if ((item.getError() == null || item.getError().isBlank()) && candidate.getError() != null && !candidate.getError().isBlank()) {
|
||||||
@@ -344,7 +343,16 @@ public class DeleteBrandRunService {
|
|||||||
if (entity == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) {
|
if (entity == null || !MODULE_TYPE.equals(entity.getModuleType()) || !userId.equals(entity.getUserId())) {
|
||||||
throw new BusinessException("记录不存在");
|
throw new BusinessException("记录不存在");
|
||||||
}
|
}
|
||||||
|
Long taskId = entity.getTaskId();
|
||||||
fileResultMapper.deleteById(resultId);
|
fileResultMapper.deleteById(resultId);
|
||||||
|
if (taskId != null && taskId > 0) {
|
||||||
|
Long remaining = fileResultMapper.selectCount(new LambdaQueryWrapper<FileResultEntity>()
|
||||||
|
.eq(FileResultEntity::getTaskId, taskId)
|
||||||
|
.eq(FileResultEntity::getModuleType, MODULE_TYPE));
|
||||||
|
if (remaining == null || remaining <= 0) {
|
||||||
|
deleteBrandTaskStorageService.deleteTaskData(taskId);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private ParsedDeleteBrandFile parseDeleteBrandFile(File inputFile) {
|
private ParsedDeleteBrandFile parseDeleteBrandFile(File inputFile) {
|
||||||
@@ -468,8 +476,8 @@ public class DeleteBrandRunService {
|
|||||||
if (value == null) {
|
if (value == null) {
|
||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
return value.replace("\ufeff", "")
|
return value.replace("", "")
|
||||||
.replace("\u3000", " ")
|
.replace(" ", " ")
|
||||||
.replace("\r\n", " ")
|
.replace("\r\n", " ")
|
||||||
.replace("\r", " ")
|
.replace("\r", " ")
|
||||||
.replace("\n", " ")
|
.replace("\n", " ")
|
||||||
@@ -640,7 +648,7 @@ public class DeleteBrandRunService {
|
|||||||
taskVo.setCreatedAt(task.getCreatedAt());
|
taskVo.setCreatedAt(task.getCreatedAt());
|
||||||
taskVo.setUpdatedAt(task.getUpdatedAt());
|
taskVo.setUpdatedAt(task.getUpdatedAt());
|
||||||
taskVo.setFinishedAt(task.getFinishedAt());
|
taskVo.setFinishedAt(task.getFinishedAt());
|
||||||
// 轮询期避免每次都查 biz_file_result(会放大 DB 压力);仅任务成功后再提供下载信息
|
// 轮询期间避免每次都查 biz_file_result,放大 DB 压力;仅任务成功后再提供下载信息
|
||||||
if ("SUCCESS".equals(task.getStatus())) {
|
if ("SUCCESS".equals(task.getStatus())) {
|
||||||
taskVo.setDownloadUrl(resolveTaskDownloadUrl(task.getId()));
|
taskVo.setDownloadUrl(resolveTaskDownloadUrl(task.getId()));
|
||||||
taskVo.setDownloadFilename(resolveTaskDownloadFilename(task.getId()));
|
taskVo.setDownloadFilename(resolveTaskDownloadFilename(task.getId()));
|
||||||
@@ -996,9 +1004,7 @@ public class DeleteBrandRunService {
|
|||||||
throw new BusinessException("任务已结束,拒绝继续提交结果");
|
throw new BusinessException("任务已结束,拒绝继续提交结果");
|
||||||
}
|
}
|
||||||
|
|
||||||
Map<String, DeleteBrandParsedFileCacheDto> parsedPayload = deleteBrandTaskCacheService.getParsedPayload(taskId,
|
Map<String, DeleteBrandParsedFileCacheDto> parsedPayload = deleteBrandTaskStorageService.loadParsedPayload(taskId);
|
||||||
new TypeReference<Map<String, DeleteBrandParsedFileCacheDto>>() {
|
|
||||||
});
|
|
||||||
if (parsedPayload == null || parsedPayload.isEmpty()) {
|
if (parsedPayload == null || parsedPayload.isEmpty()) {
|
||||||
throw new BusinessException("任务原始数据已过期,请重新执行解析");
|
throw new BusinessException("任务原始数据已过期,请重新执行解析");
|
||||||
}
|
}
|
||||||
@@ -1029,10 +1035,10 @@ public class DeleteBrandRunService {
|
|||||||
throw new BusinessException("文件标识不匹配: " + fileIdentity);
|
throw new BusinessException("文件标识不匹配: " + fileIdentity);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 更早的 duplicate fast-path:尽量在较重的校验/进度写入之前直接短路
|
// 更早执行 duplicate fast-path,尽量在较重校验和进度写入前直接短路
|
||||||
validateChunk(fileDto, parsedFile);
|
validateChunk(fileDto, parsedFile);
|
||||||
validateResultFileAgainstParsed(fileDto, parsedFile);
|
validateResultFileAgainstParsed(fileDto, parsedFile);
|
||||||
boolean stored = deleteBrandTaskCacheService.storeResultChunkIfChanged(taskId, fileIdentity, fileDto.getChunkIndex(), fileDto);
|
boolean stored = deleteBrandTaskStorageService.storeResultChunkIfChanged(taskId, fileIdentity, fileDto.getChunkIndex(), fileDto);
|
||||||
if (!stored) {
|
if (!stored) {
|
||||||
log.info("[DeleteBrand] Duplicate chunk ignored for taskId: {}, file: {}, chunkIndex: {}", taskId, fileIdentity, fileDto.getChunkIndex());
|
log.info("[DeleteBrand] Duplicate chunk ignored for taskId: {}, file: {}, chunkIndex: {}", taskId, fileIdentity, fileDto.getChunkIndex());
|
||||||
continue;
|
continue;
|
||||||
@@ -1051,12 +1057,12 @@ public class DeleteBrandRunService {
|
|||||||
progress.put("last_heartbeat_at", String.valueOf(System.currentTimeMillis()));
|
progress.put("last_heartbeat_at", String.valueOf(System.currentTimeMillis()));
|
||||||
deleteBrandTaskCacheService.saveProgress(taskId, progress, true);
|
deleteBrandTaskCacheService.saveProgress(taskId, progress, true);
|
||||||
|
|
||||||
// 如果该文件已完成,立即更新成品计数的 Redis 缓存「提示」,让后续 tryFinalizeTask 能更精确
|
// 如果该文件已完成,立即更新成品计数的 Redis 提示,让后续 tryFinalizeTask 更准确
|
||||||
if (isFileCompleted(fileDto)) {
|
if (isFileCompleted(fileDto)) {
|
||||||
log.info("[DeleteBrand] File chunk transmission fully reached via chunkTotal for taskId: {}, file: {}", taskId, fileDto.getSourceFilename());
|
log.info("[DeleteBrand] File chunk transmission fully reached via chunkTotal for taskId: {}, file: {}", taskId, fileDto.getSourceFilename());
|
||||||
shouldTryFinalize = true;
|
shouldTryFinalize = true;
|
||||||
// 这里我们暂不直接 increment,而是标记需要重算或在 tryFinalizeTask 中重新计算
|
// 这里暂不直接 increment,而是标记需要在 tryFinalizeTask 中重新计算
|
||||||
// 为了保险,我们直接在 tryFinalizeTask 里重新遍历分片状态
|
// 为了保险,直接在 tryFinalizeTask 里重新遍历分片状态
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
@@ -1079,26 +1085,24 @@ public class DeleteBrandRunService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
Map<String, DeleteBrandParsedFileCacheDto> parsedPayload = deleteBrandTaskCacheService.getParsedPayload(taskId,
|
Map<String, DeleteBrandParsedFileCacheDto> parsedPayload = deleteBrandTaskStorageService.loadParsedPayload(taskId);
|
||||||
new TypeReference<Map<String, DeleteBrandParsedFileCacheDto>>() {
|
|
||||||
});
|
|
||||||
if (parsedPayload == null || parsedPayload.isEmpty()) {
|
if (parsedPayload == null || parsedPayload.isEmpty()) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
int expectedFiles = parsedPayload.size();
|
int expectedFiles = parsedPayload.size();
|
||||||
// 移除 finishedFilesHint 强行返回逻辑,因为 Redis 里的 finished_files 可能是旧的,
|
// 移除 finishedFilesHint 的强依赖,因为 Redis 里的 finished_files 可能是旧值
|
||||||
// 既然进入了 tryFinalizeTask,就说明有新片到达,应该以 loadMergedChunks 为准
|
// 既然进入 tryFinalizeTask,就说明有新分片到达,应该以 loadMergedChunks 为准
|
||||||
|
|
||||||
|
|
||||||
Map<String, List<DeleteBrandResultFileDto>> mergedByFile = loadMergedChunks(taskId);
|
Map<String, List<DeleteBrandResultFileDto>> mergedByFile = deleteBrandTaskStorageService.loadMergedChunks(taskId);
|
||||||
int finishedFiles = countCompletedFiles(mergedByFile);
|
int finishedFiles = countCompletedFiles(mergedByFile);
|
||||||
|
|
||||||
if (finishedFiles < expectedFiles) {
|
if (finishedFiles < expectedFiles) {
|
||||||
Map<String, String> progress = new LinkedHashMap<>();
|
Map<String, String> progress = new LinkedHashMap<>();
|
||||||
progress.put("finished_files", String.valueOf(finishedFiles));
|
progress.put("finished_files", String.valueOf(finishedFiles));
|
||||||
// 定时补偿只负责“再试一次 finalize”,不能把缺片任务重新续命,
|
// 定时补偿只负责“再试一次 finalize”,不能把缺片任务重新续命
|
||||||
// 否则 stale-check 永远看不到超时任务。
|
// 否则 stale-check 会一直看不到超时任务
|
||||||
if (!fromCompensation) {
|
if (!fromCompensation) {
|
||||||
progress.put("updated_at", String.valueOf(System.currentTimeMillis()));
|
progress.put("updated_at", String.valueOf(System.currentTimeMillis()));
|
||||||
}
|
}
|
||||||
@@ -1133,24 +1137,6 @@ public class DeleteBrandRunService {
|
|||||||
finalizeTask(task, parsedPayload, mergedByFile);
|
finalizeTask(task, parsedPayload, mergedByFile);
|
||||||
}
|
}
|
||||||
|
|
||||||
private Map<String, List<DeleteBrandResultFileDto>> loadMergedChunks(Long taskId) {
|
|
||||||
Map<String, List<String>> groupedJson = deleteBrandTaskCacheService.groupResultChunkJsonByFile(taskId);
|
|
||||||
Map<String, List<DeleteBrandResultFileDto>> merged = new LinkedHashMap<>();
|
|
||||||
for (Map.Entry<String, List<String>> entry : groupedJson.entrySet()) {
|
|
||||||
List<DeleteBrandResultFileDto> chunks = new ArrayList<>();
|
|
||||||
for (String raw : entry.getValue()) {
|
|
||||||
try {
|
|
||||||
chunks.add(objectMapper.readValue(raw, DeleteBrandResultFileDto.class));
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("读取删除品牌结果分片失败");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
chunks.sort(Comparator.comparing(DeleteBrandResultFileDto::getChunkIndex, Comparator.nullsLast(Integer::compareTo)));
|
|
||||||
merged.put(entry.getKey(), chunks);
|
|
||||||
}
|
|
||||||
return merged;
|
|
||||||
}
|
|
||||||
|
|
||||||
private int countCompletedFiles(Map<String, List<DeleteBrandResultFileDto>> mergedByFile) {
|
private int countCompletedFiles(Map<String, List<DeleteBrandResultFileDto>> mergedByFile) {
|
||||||
int finishedFiles = 0;
|
int finishedFiles = 0;
|
||||||
for (List<DeleteBrandResultFileDto> chunks : mergedByFile.values()) {
|
for (List<DeleteBrandResultFileDto> chunks : mergedByFile.values()) {
|
||||||
@@ -1187,6 +1173,7 @@ public class DeleteBrandRunService {
|
|||||||
private void finalizeTask(FileTaskEntity task,
|
private void finalizeTask(FileTaskEntity task,
|
||||||
Map<String, DeleteBrandParsedFileCacheDto> parsedPayload,
|
Map<String, DeleteBrandParsedFileCacheDto> parsedPayload,
|
||||||
Map<String, List<DeleteBrandResultFileDto>> mergedByFile) {
|
Map<String, List<DeleteBrandResultFileDto>> mergedByFile) {
|
||||||
|
File outputDir = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "delete-brand-result", String.valueOf(task.getId())));
|
||||||
try {
|
try {
|
||||||
deleteBrandTaskCacheService.saveProgress(task.getId(), Map.of(
|
deleteBrandTaskCacheService.saveProgress(task.getId(), Map.of(
|
||||||
"phase", DeleteBrandTaskCacheService.PHASE_ASSEMBLING,
|
"phase", DeleteBrandTaskCacheService.PHASE_ASSEMBLING,
|
||||||
@@ -1259,6 +1246,8 @@ public class DeleteBrandRunService {
|
|||||||
item.setPlatform(parsedFile.getPlatform());
|
item.setPlatform(parsedFile.getPlatform());
|
||||||
item.setOpenStoreUrl(parsedFile.getOpenStoreUrl());
|
item.setOpenStoreUrl(parsedFile.getOpenStoreUrl());
|
||||||
item.setMatched(parsedFile.getShopId() != null && !parsedFile.getShopId().isBlank());
|
item.setMatched(parsedFile.getShopId() != null && !parsedFile.getShopId().isBlank());
|
||||||
|
item.setSuccess(true);
|
||||||
|
item.setError(null);
|
||||||
item.setMatchStatus(item.isMatched() ? ZiniaoShopIndexService.MATCH_STATUS_MATCHED : ZiniaoShopIndexService.MATCH_STATUS_PENDING);
|
item.setMatchStatus(item.isMatched() ? ZiniaoShopIndexService.MATCH_STATUS_MATCHED : ZiniaoShopIndexService.MATCH_STATUS_PENDING);
|
||||||
item.setMatchMessage(item.isMatched() ? null : "店铺索引信息缺失,请重新匹配索引");
|
item.setMatchMessage(item.isMatched() ? null : "店铺索引信息缺失,请重新匹配索引");
|
||||||
finalItems.add(item);
|
finalItems.add(item);
|
||||||
@@ -1267,12 +1256,14 @@ public class DeleteBrandRunService {
|
|||||||
|
|
||||||
task.setStatus("SUCCESS");
|
task.setStatus("SUCCESS");
|
||||||
task.setSuccessFileCount(successCount);
|
task.setSuccessFileCount(successCount);
|
||||||
|
task.setFailedFileCount(Math.max(0, parsedPayload.size() - successCount));
|
||||||
task.setErrorMessage(null);
|
task.setErrorMessage(null);
|
||||||
task.setResultJson(toCompactTaskResultJsonForDb(finalItems));
|
task.setResultJson(toCompactTaskResultJsonForDb(finalItems));
|
||||||
task.setUpdatedAt(LocalDateTime.now());
|
task.setUpdatedAt(LocalDateTime.now());
|
||||||
task.setFinishedAt(LocalDateTime.now());
|
task.setFinishedAt(LocalDateTime.now());
|
||||||
fileTaskMapper.updateById(task);
|
fileTaskMapper.updateById(task);
|
||||||
deleteBrandTaskCacheService.saveTaskCache(task);
|
deleteBrandTaskCacheService.saveTaskCache(task);
|
||||||
|
deleteBrandTaskStorageService.deleteTaskData(task.getId());
|
||||||
|
|
||||||
deleteBrandTaskCacheService.saveProgress(task.getId(), Map.of(
|
deleteBrandTaskCacheService.saveProgress(task.getId(), Map.of(
|
||||||
"phase", "success",
|
"phase", "success",
|
||||||
@@ -1297,6 +1288,8 @@ public class DeleteBrandRunService {
|
|||||||
throw businessException;
|
throw businessException;
|
||||||
}
|
}
|
||||||
throw new BusinessException("删除品牌结果组装失败");
|
throw new BusinessException("删除品牌结果组装失败");
|
||||||
|
} finally {
|
||||||
|
deleteQuietly(outputDir);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1355,7 +1348,9 @@ public class DeleteBrandRunService {
|
|||||||
File outputDir = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "delete-brand-result", String.valueOf(taskId)));
|
File outputDir = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "delete-brand-result", String.valueOf(taskId)));
|
||||||
File outputFile = buildNamedOutputFile(outputDir, blankToDefault(parsedFile.getSourceFilename(), "delete-brand-result.xlsx"));
|
File outputFile = buildNamedOutputFile(outputDir, blankToDefault(parsedFile.getSourceFilename(), "delete-brand-result.xlsx"));
|
||||||
|
|
||||||
try (Workbook workbook = new XSSFWorkbook(); FileOutputStream outputStream = new FileOutputStream(outputFile)) {
|
try (SXSSFWorkbook workbook = new SXSSFWorkbook(200);
|
||||||
|
FileOutputStream outputStream = new FileOutputStream(outputFile)) {
|
||||||
|
workbook.setCompressTempFiles(true);
|
||||||
Sheet sheet = workbook.createSheet("删除品牌结果");
|
Sheet sheet = workbook.createSheet("删除品牌结果");
|
||||||
Row headerRow = sheet.createRow(0);
|
Row headerRow = sheet.createRow(0);
|
||||||
headerRow.createCell(0).setCellValue("国家");
|
headerRow.createCell(0).setCellValue("国家");
|
||||||
@@ -1371,9 +1366,7 @@ public class DeleteBrandRunService {
|
|||||||
createTextCell(row, 2, item.getStatus());
|
createTextCell(row, 2, item.getStatus());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for (int i = 0; i < 3; i++) {
|
applyDeleteBrandColumnWidths(sheet, 3);
|
||||||
sheet.autoSizeColumn(i);
|
|
||||||
}
|
|
||||||
workbook.write(outputStream);
|
workbook.write(outputStream);
|
||||||
return outputFile;
|
return outputFile;
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
@@ -1387,7 +1380,9 @@ public class DeleteBrandRunService {
|
|||||||
File outputDir = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "delete-brand-result", String.valueOf(taskId)));
|
File outputDir = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "delete-brand-result", String.valueOf(taskId)));
|
||||||
File outputFile = buildNamedOutputFile(outputDir, blankToDefault(parsedFile.getSourceFilename(), "delete-brand-result.xlsx"));
|
File outputFile = buildNamedOutputFile(outputDir, blankToDefault(parsedFile.getSourceFilename(), "delete-brand-result.xlsx"));
|
||||||
|
|
||||||
try (Workbook workbook = new XSSFWorkbook(); FileOutputStream outputStream = new FileOutputStream(outputFile)) {
|
try (SXSSFWorkbook workbook = new SXSSFWorkbook(200);
|
||||||
|
FileOutputStream outputStream = new FileOutputStream(outputFile)) {
|
||||||
|
workbook.setCompressTempFiles(true);
|
||||||
Sheet sheet = workbook.createSheet("删除品牌结果");
|
Sheet sheet = workbook.createSheet("删除品牌结果");
|
||||||
Row countryRow = sheet.createRow(0);
|
Row countryRow = sheet.createRow(0);
|
||||||
Row headerRow = sheet.createRow(1);
|
Row headerRow = sheet.createRow(1);
|
||||||
@@ -1417,9 +1412,7 @@ public class DeleteBrandRunService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for (int i = 0; i < mergedFile.countries().size() * 2; i++) {
|
applyDeleteBrandColumnWidths(sheet, mergedFile.countries().size() * 2);
|
||||||
sheet.autoSizeColumn(i);
|
|
||||||
}
|
|
||||||
workbook.write(outputStream);
|
workbook.write(outputStream);
|
||||||
return outputFile;
|
return outputFile;
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
@@ -1432,6 +1425,22 @@ public class DeleteBrandRunService {
|
|||||||
cell.setCellValue(value == null ? "" : value);
|
cell.setCellValue(value == null ? "" : value);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void applyDeleteBrandColumnWidths(Sheet sheet, int columnCount) {
|
||||||
|
for (int i = 0; i < columnCount; i++) {
|
||||||
|
sheet.setColumnWidth(i, 18 * 256);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void deleteQuietly(File file) {
|
||||||
|
if (file == null || !file.exists()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
FileUtil.del(file);
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private File buildNamedOutputFile(File outputDir, String filename) {
|
private File buildNamedOutputFile(File outputDir, String filename) {
|
||||||
File candidate = FileUtil.file(outputDir, filename);
|
File candidate = FileUtil.file(outputDir, filename);
|
||||||
if (!candidate.exists()) {
|
if (!candidate.exists()) {
|
||||||
@@ -1478,10 +1487,9 @@ public class DeleteBrandRunService {
|
|||||||
String normalized = blankToEmpty(value);
|
String normalized = blankToEmpty(value);
|
||||||
return normalized.isBlank() ? defaultValue : normalized;
|
return normalized.isBlank() ? defaultValue : normalized;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 落库用 result_json:去掉国家分组、预览行等大字段,保留匹配摘要与计数。
|
* 落库用 result_json:去掉国家分组、预览行等大字段,保留匹配摘要与计数。
|
||||||
* 索引常命中后单次任务 JSON 体积会陡增,易触发 max_allowed_packet / 更新失败,进而前端 unwrap 报「请求失败」。
|
* 索引命中后单次任务 JSON 体积会明显膨胀,容易触发 max_allowed_packet 或更新失败。
|
||||||
*/
|
*/
|
||||||
private String toCompactTaskResultJsonForDb(List<DeleteBrandResultItemVo> items) {
|
private String toCompactTaskResultJsonForDb(List<DeleteBrandResultItemVo> items) {
|
||||||
if (items == null || items.isEmpty()) {
|
if (items == null || items.isEmpty()) {
|
||||||
|
|||||||
@@ -1,16 +1,11 @@
|
|||||||
package com.nanri.aiimage.modules.deletebrand.service;
|
package com.nanri.aiimage.modules.deletebrand.service;
|
||||||
|
|
||||||
import com.fasterxml.jackson.core.type.TypeReference;
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.nanri.aiimage.common.exception.BusinessException;
|
|
||||||
import com.nanri.aiimage.config.TaskPressureProperties;
|
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
import java.nio.file.Files;
|
|
||||||
import java.nio.file.Path;
|
|
||||||
import java.nio.file.StandardOpenOption;
|
|
||||||
import java.time.Duration;
|
import java.time.Duration;
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
|
||||||
@@ -24,63 +19,16 @@ public class DeleteBrandTaskCacheService {
|
|||||||
public static final String PHASE_FAILED = "failed";
|
public static final String PHASE_FAILED = "failed";
|
||||||
|
|
||||||
private static final long PAYLOAD_TTL_HOURS = 24;
|
private static final long PAYLOAD_TTL_HOURS = 24;
|
||||||
private static final long LOCAL_PARSED_PAYLOAD_CACHE_MILLIS = 3000;
|
|
||||||
private static final long LOCAL_PROGRESS_CACHE_MILLIS = 1500;
|
private static final long LOCAL_PROGRESS_CACHE_MILLIS = 1500;
|
||||||
private static final long MIN_PROGRESS_REDIS_FLUSH_MILLIS = 1200;
|
private static final long MIN_PROGRESS_REDIS_FLUSH_MILLIS = 1200;
|
||||||
|
|
||||||
private final StringRedisTemplate stringRedisTemplate;
|
private final StringRedisTemplate stringRedisTemplate;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
private final TaskPressureProperties taskPressureProperties;
|
private final TaskPressureProperties taskPressureProperties;
|
||||||
private final ConcurrentHashMap<Long, LocalParsedPayloadCacheEntry> parsedPayloadLocalCache = new ConcurrentHashMap<>();
|
|
||||||
private final ConcurrentHashMap<Long, LocalProgressCacheEntry> progressLocalCache = new ConcurrentHashMap<>();
|
private final ConcurrentHashMap<Long, LocalProgressCacheEntry> progressLocalCache = new ConcurrentHashMap<>();
|
||||||
private final ConcurrentHashMap<Long, Long> progressRedisFlushAt = new ConcurrentHashMap<>();
|
private final ConcurrentHashMap<Long, Long> progressRedisFlushAt = new ConcurrentHashMap<>();
|
||||||
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
|
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
|
||||||
|
|
||||||
public void saveParsedPayload(Long taskId, Object payload) {
|
|
||||||
try {
|
|
||||||
Files.createDirectories(buildTaskDir(taskId));
|
|
||||||
Files.writeString(
|
|
||||||
buildPayloadFile(taskId),
|
|
||||||
objectMapper.writeValueAsString(payload),
|
|
||||||
StandardOpenOption.CREATE,
|
|
||||||
StandardOpenOption.TRUNCATE_EXISTING,
|
|
||||||
StandardOpenOption.WRITE
|
|
||||||
);
|
|
||||||
parsedPayloadLocalCache.put(taskId, new LocalParsedPayloadCacheEntry(System.currentTimeMillis(), payload));
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("鏆傚瓨鍒犻櫎鍝佺墝鍘熷鏁版嵁澶辫触");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public <T> T getParsedPayload(Long taskId, TypeReference<T> typeReference) {
|
|
||||||
if (taskId == null || taskId <= 0) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
long now = System.currentTimeMillis();
|
|
||||||
LocalParsedPayloadCacheEntry cached = parsedPayloadLocalCache.get(taskId);
|
|
||||||
if (cached != null && cached.isFresh(now) && cached.value() != null) {
|
|
||||||
try {
|
|
||||||
return objectMapper.convertValue(cached.value(), typeReference);
|
|
||||||
} catch (Exception ignored) {
|
|
||||||
// fall through to file
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Path payloadFile = buildPayloadFile(taskId);
|
|
||||||
if (!Files.isRegularFile(payloadFile)) {
|
|
||||||
parsedPayloadLocalCache.remove(taskId);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
T parsed = objectMapper.readValue(Files.readString(payloadFile), typeReference);
|
|
||||||
parsedPayloadLocalCache.put(taskId, new LocalParsedPayloadCacheEntry(now, parsed));
|
|
||||||
return parsed;
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("璇诲彇鍒犻櫎鍝佺墝鍘熷鏁版嵁澶辫触");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public void saveProgress(Long taskId, java.util.Map<String, String> values) {
|
public void saveProgress(Long taskId, java.util.Map<String, String> values) {
|
||||||
saveProgress(taskId, values, false);
|
saveProgress(taskId, values, false);
|
||||||
}
|
}
|
||||||
@@ -150,8 +98,10 @@ public class DeleteBrandTaskCacheService {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
java.util.List<Object> pipelineResults = stringRedisTemplate.executePipelined((org.springframework.data.redis.core.RedisCallback<Object>) connection -> {
|
java.util.List<Object> pipelineResults = stringRedisTemplate.executePipelined(
|
||||||
org.springframework.data.redis.serializer.RedisSerializer<String> serializer = stringRedisTemplate.getStringSerializer();
|
(org.springframework.data.redis.core.RedisCallback<Object>) connection -> {
|
||||||
|
org.springframework.data.redis.serializer.RedisSerializer<String> serializer =
|
||||||
|
stringRedisTemplate.getStringSerializer();
|
||||||
for (Long taskId : missingTaskIds) {
|
for (Long taskId : missingTaskIds) {
|
||||||
connection.hGetAll(serializer.serialize(buildProgressKey(taskId)));
|
connection.hGetAll(serializer.serialize(buildProgressKey(taskId)));
|
||||||
}
|
}
|
||||||
@@ -175,83 +125,12 @@ public class DeleteBrandTaskCacheService {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean hasSameResultChunk(Long taskId, String fileIdentity, Integer chunkIndex, Object chunk) {
|
|
||||||
if (chunkIndex == null || chunkIndex <= 0) {
|
|
||||||
throw new BusinessException("chunkIndex 涓嶅悎娉?");
|
|
||||||
}
|
|
||||||
if (fileIdentity == null || fileIdentity.isBlank()) {
|
|
||||||
throw new BusinessException("fileIdentity 涓嶈兘涓虹┖");
|
|
||||||
}
|
|
||||||
String field = fileIdentity.trim() + "#" + chunkIndex;
|
|
||||||
String chunkJson;
|
|
||||||
try {
|
|
||||||
chunkJson = objectMapper.writeValueAsString(chunk);
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("璇诲彇鍒犻櫎鍝佺墝缁撴灉鍒嗙墖澶辫触");
|
|
||||||
}
|
|
||||||
Object existing = stringRedisTemplate.opsForHash().get(buildResultChunksKey(taskId), field);
|
|
||||||
if (chunkJson.equals(existing)) {
|
|
||||||
stringRedisTemplate.expire(buildResultChunksKey(taskId), Duration.ofHours(PAYLOAD_TTL_HOURS));
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean mergeResultChunk(Long taskId, String fileIdentity, Integer chunkIndex, Object chunk) {
|
|
||||||
String resultKey = buildResultChunksKey(taskId);
|
|
||||||
if (chunkIndex == null || chunkIndex <= 0) {
|
|
||||||
throw new BusinessException("chunkIndex 涓嶅悎娉?");
|
|
||||||
}
|
|
||||||
if (fileIdentity == null || fileIdentity.isBlank()) {
|
|
||||||
throw new BusinessException("fileIdentity 涓嶈兘涓虹┖");
|
|
||||||
}
|
|
||||||
String field = fileIdentity.trim() + "#" + chunkIndex;
|
|
||||||
String chunkJson;
|
|
||||||
try {
|
|
||||||
chunkJson = objectMapper.writeValueAsString(chunk);
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("鏆傚瓨鍒犻櫎鍝佺墝缁撴灉鍒嗙墖澶辫触");
|
|
||||||
}
|
|
||||||
Object existing = stringRedisTemplate.opsForHash().get(resultKey, field);
|
|
||||||
if (chunkJson.equals(existing)) {
|
|
||||||
stringRedisTemplate.expire(resultKey, Duration.ofHours(PAYLOAD_TTL_HOURS));
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
stringRedisTemplate.opsForHash().put(resultKey, field, chunkJson);
|
|
||||||
stringRedisTemplate.expire(resultKey, Duration.ofHours(PAYLOAD_TTL_HOURS));
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean storeResultChunkIfChanged(Long taskId, String fileIdentity, Integer chunkIndex, Object chunk) {
|
|
||||||
return mergeResultChunk(taskId, fileIdentity, chunkIndex, chunk);
|
|
||||||
}
|
|
||||||
|
|
||||||
public java.util.Map<String, java.util.List<String>> groupResultChunkJsonByFile(Long taskId) {
|
|
||||||
java.util.Map<Object, Object> stored = stringRedisTemplate.opsForHash().entries(buildResultChunksKey(taskId));
|
|
||||||
java.util.Map<String, java.util.List<String>> grouped = new java.util.LinkedHashMap<>();
|
|
||||||
for (java.util.Map.Entry<Object, Object> entry : stored.entrySet()) {
|
|
||||||
if (!(entry.getKey() instanceof String field) || !(entry.getValue() instanceof String raw) || raw.isBlank()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
int sep = field.lastIndexOf('#');
|
|
||||||
if (sep <= 0) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
String fileIdentity = field.substring(0, sep);
|
|
||||||
grouped.computeIfAbsent(fileIdentity, ignored -> new java.util.ArrayList<>()).add(raw);
|
|
||||||
}
|
|
||||||
return grouped;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void delete(Long taskId) {
|
public void delete(Long taskId) {
|
||||||
parsedPayloadLocalCache.remove(taskId);
|
|
||||||
progressLocalCache.remove(taskId);
|
progressLocalCache.remove(taskId);
|
||||||
progressRedisFlushAt.remove(taskId);
|
progressRedisFlushAt.remove(taskId);
|
||||||
taskEntityLocalCache.remove(taskId);
|
taskEntityLocalCache.remove(taskId);
|
||||||
stringRedisTemplate.delete(buildProgressKey(taskId));
|
stringRedisTemplate.delete(buildProgressKey(taskId));
|
||||||
stringRedisTemplate.delete(buildResultChunksKey(taskId));
|
|
||||||
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
|
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
|
||||||
deleteTaskDirQuietly(taskId);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void saveTaskCache(com.nanri.aiimage.modules.task.model.entity.FileTaskEntity task) {
|
public void saveTaskCache(com.nanri.aiimage.modules.task.model.entity.FileTaskEntity task) {
|
||||||
@@ -264,13 +143,18 @@ public class DeleteBrandTaskCacheService {
|
|||||||
objectMapper.convertValue(task, com.nanri.aiimage.modules.task.model.entity.FileTaskEntity.class)
|
objectMapper.convertValue(task, com.nanri.aiimage.modules.task.model.entity.FileTaskEntity.class)
|
||||||
));
|
));
|
||||||
try {
|
try {
|
||||||
stringRedisTemplate.opsForValue().set(buildTaskEntityKey(task.getId()), objectMapper.writeValueAsString(task), Duration.ofHours(PAYLOAD_TTL_HOURS));
|
stringRedisTemplate.opsForValue().set(
|
||||||
|
buildTaskEntityKey(task.getId()),
|
||||||
|
objectMapper.writeValueAsString(task),
|
||||||
|
Duration.ofHours(PAYLOAD_TTL_HOURS));
|
||||||
} catch (Exception ignored) {
|
} catch (Exception ignored) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public java.util.Map<Long, com.nanri.aiimage.modules.task.model.entity.FileTaskEntity> getTaskCacheBatch(java.util.List<Long> taskIds) {
|
public java.util.Map<Long, com.nanri.aiimage.modules.task.model.entity.FileTaskEntity> getTaskCacheBatch(
|
||||||
java.util.Map<Long, com.nanri.aiimage.modules.task.model.entity.FileTaskEntity> result = new java.util.LinkedHashMap<>();
|
java.util.List<Long> taskIds) {
|
||||||
|
java.util.Map<Long, com.nanri.aiimage.modules.task.model.entity.FileTaskEntity> result =
|
||||||
|
new java.util.LinkedHashMap<>();
|
||||||
if (taskIds == null || taskIds.isEmpty()) {
|
if (taskIds == null || taskIds.isEmpty()) {
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
@@ -285,7 +169,8 @@ public class DeleteBrandTaskCacheService {
|
|||||||
for (Long taskId : normalized) {
|
for (Long taskId : normalized) {
|
||||||
LocalTaskEntityCacheEntry cached = taskEntityLocalCache.get(taskId);
|
LocalTaskEntityCacheEntry cached = taskEntityLocalCache.get(taskId);
|
||||||
if (isLocalTaskEntityCacheFresh(cached, now)) {
|
if (isLocalTaskEntityCacheFresh(cached, now)) {
|
||||||
result.put(taskId, objectMapper.convertValue(cached.task(), com.nanri.aiimage.modules.task.model.entity.FileTaskEntity.class));
|
result.put(taskId, objectMapper.convertValue(
|
||||||
|
cached.task(), com.nanri.aiimage.modules.task.model.entity.FileTaskEntity.class));
|
||||||
} else {
|
} else {
|
||||||
missingIds.add(taskId);
|
missingIds.add(taskId);
|
||||||
}
|
}
|
||||||
@@ -313,40 +198,10 @@ public class DeleteBrandTaskCacheService {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
private Path buildTaskDir(Long taskId) {
|
|
||||||
return Path.of(System.getProperty("java.io.tmpdir"), "delete-brand-task-cache", String.valueOf(taskId));
|
|
||||||
}
|
|
||||||
|
|
||||||
private Path buildPayloadFile(Long taskId) {
|
|
||||||
return buildTaskDir(taskId).resolve("parsed-payload.json");
|
|
||||||
}
|
|
||||||
|
|
||||||
private void deleteTaskDirQuietly(Long taskId) {
|
|
||||||
Path taskDir = buildTaskDir(taskId);
|
|
||||||
if (!Files.exists(taskDir)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try (var walk = Files.walk(taskDir)) {
|
|
||||||
walk.sorted(java.util.Comparator.reverseOrder()).forEach(path -> {
|
|
||||||
try {
|
|
||||||
Files.deleteIfExists(path);
|
|
||||||
} catch (Exception ignored) {
|
|
||||||
}
|
|
||||||
});
|
|
||||||
} catch (Exception ignored) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private String buildTaskEntityKey(Long taskId) {
|
private String buildTaskEntityKey(Long taskId) {
|
||||||
return "delete-brand:task:entity:" + taskId;
|
return "delete-brand:task:entity:" + taskId;
|
||||||
}
|
}
|
||||||
|
|
||||||
private record LocalParsedPayloadCacheEntry(long cachedAtMillis, Object value) {
|
|
||||||
private boolean isFresh(long now) {
|
|
||||||
return now - cachedAtMillis <= LOCAL_PARSED_PAYLOAD_CACHE_MILLIS;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private record LocalProgressCacheEntry(long cachedAtMillis, java.util.Map<String, String> values) {
|
private record LocalProgressCacheEntry(long cachedAtMillis, java.util.Map<String, String> values) {
|
||||||
private boolean isFresh(long now) {
|
private boolean isFresh(long now) {
|
||||||
return now - cachedAtMillis <= LOCAL_PROGRESS_CACHE_MILLIS;
|
return now - cachedAtMillis <= LOCAL_PROGRESS_CACHE_MILLIS;
|
||||||
@@ -377,8 +232,4 @@ public class DeleteBrandTaskCacheService {
|
|||||||
private String buildProgressKey(Long taskId) {
|
private String buildProgressKey(Long taskId) {
|
||||||
return "delete-brand:task:progress:" + taskId;
|
return "delete-brand:task:progress:" + taskId;
|
||||||
}
|
}
|
||||||
|
|
||||||
private String buildResultChunksKey(Long taskId) {
|
|
||||||
return "delete-brand:task:result-chunks:" + taskId;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,404 @@
|
|||||||
|
package com.nanri.aiimage.modules.deletebrand.service;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import com.nanri.aiimage.modules.deletebrand.model.cache.DeleteBrandParsedFileCacheDto;
|
||||||
|
import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandResultFileDto;
|
||||||
|
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.dao.DuplicateKeyException;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.security.MessageDigest;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Comparator;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
public class DeleteBrandTaskStorageService {
|
||||||
|
|
||||||
|
private static final String MODULE_TYPE = "DELETE_BRAND";
|
||||||
|
private static final String OSS_POINTER_PREFIX = "oss:";
|
||||||
|
|
||||||
|
private final TaskChunkMapper taskChunkMapper;
|
||||||
|
private final TaskScopeStateMapper taskScopeStateMapper;
|
||||||
|
private final ObjectMapper objectMapper;
|
||||||
|
private final OssStorageService ossStorageService;
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public void saveParsedPayload(Long taskId, Map<String, DeleteBrandParsedFileCacheDto> parsedPayloadByScope) {
|
||||||
|
if (taskId == null || taskId <= 0 || parsedPayloadByScope == null || parsedPayloadByScope.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
LocalDateTime now = LocalDateTime.now();
|
||||||
|
Map<String, TaskScopeStateEntity> existingByHash = loadScopeStateByHash(taskId);
|
||||||
|
for (Map.Entry<String, DeleteBrandParsedFileCacheDto> entry : parsedPayloadByScope.entrySet()) {
|
||||||
|
String scopeKey = normalizeScopeKey(entry.getKey());
|
||||||
|
if (scopeKey.isBlank() || entry.getValue() == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
String scopeHash = hash(scopeKey);
|
||||||
|
TaskScopeStateEntity existing = existingByHash.get(scopeHash);
|
||||||
|
String payloadJson = writeJson(entry.getValue(), "failed to save delete-brand parsed payload");
|
||||||
|
String parsedPayloadPointer = storeParsedPayload(taskId, scopeHash, payloadJson);
|
||||||
|
if (existing == null) {
|
||||||
|
TaskScopeStateEntity entity = new TaskScopeStateEntity();
|
||||||
|
entity.setTaskId(taskId);
|
||||||
|
entity.setModuleType(MODULE_TYPE);
|
||||||
|
entity.setScopeKey(scopeKey);
|
||||||
|
entity.setScopeHash(scopeHash);
|
||||||
|
entity.setParsedPayloadJson(parsedPayloadPointer);
|
||||||
|
entity.setChunkTotal(0);
|
||||||
|
entity.setReceivedChunkCount(0);
|
||||||
|
entity.setCompleted(0);
|
||||||
|
entity.setLastChunkAt(null);
|
||||||
|
entity.setLastError(null);
|
||||||
|
entity.setCreatedAt(now);
|
||||||
|
entity.setUpdatedAt(now);
|
||||||
|
try {
|
||||||
|
taskScopeStateMapper.insert(entity);
|
||||||
|
existingByHash.put(scopeHash, entity);
|
||||||
|
continue;
|
||||||
|
} catch (DuplicateKeyException ex) {
|
||||||
|
log.info("[delete-brand-storage] parsed payload scope inserted concurrently taskId={} scopeKey={}", taskId, scopeKey);
|
||||||
|
existing = getScopeState(taskId, scopeHash);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (existing == null) {
|
||||||
|
throw new BusinessException("failed to save delete-brand parsed payload");
|
||||||
|
}
|
||||||
|
deleteParsedPayloadIfNeeded(existing.getParsedPayloadJson(), parsedPayloadPointer);
|
||||||
|
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getId, existing.getId())
|
||||||
|
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
|
||||||
|
.set(TaskScopeStateEntity::getParsedPayloadJson, parsedPayloadPointer)
|
||||||
|
.set(TaskScopeStateEntity::getUpdatedAt, now));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public Map<String, DeleteBrandParsedFileCacheDto> loadParsedPayload(Long taskId) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
return Map.of();
|
||||||
|
}
|
||||||
|
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.select(TaskScopeStateEntity::getScopeKey, TaskScopeStateEntity::getParsedPayloadJson)
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.isNotNull(TaskScopeStateEntity::getParsedPayloadJson));
|
||||||
|
if (states == null || states.isEmpty()) {
|
||||||
|
return Map.of();
|
||||||
|
}
|
||||||
|
Map<String, DeleteBrandParsedFileCacheDto> result = new LinkedHashMap<>();
|
||||||
|
for (TaskScopeStateEntity state : states) {
|
||||||
|
if (isBlank(state.getScopeKey())) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
String payloadJson = resolveParsedPayload(state.getParsedPayloadJson());
|
||||||
|
if (isBlank(payloadJson)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
result.put(state.getScopeKey(), objectMapper.readValue(payloadJson, DeleteBrandParsedFileCacheDto.class));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException("failed to load delete-brand parsed payload");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public boolean storeResultChunkIfChanged(Long taskId, String scopeKey, Integer chunkIndex, DeleteBrandResultFileDto chunk) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
throw new BusinessException("taskId is invalid");
|
||||||
|
}
|
||||||
|
if (chunkIndex == null || chunkIndex <= 0) {
|
||||||
|
throw new BusinessException("chunkIndex is invalid");
|
||||||
|
}
|
||||||
|
String normalizedScopeKey = normalizeScopeKey(scopeKey);
|
||||||
|
if (normalizedScopeKey.isBlank()) {
|
||||||
|
throw new BusinessException("scopeKey must not be blank");
|
||||||
|
}
|
||||||
|
String scopeHash = hash(normalizedScopeKey);
|
||||||
|
String payloadJson = writeJson(chunk, "failed to save delete-brand result chunk");
|
||||||
|
String payloadHash = hash(payloadJson);
|
||||||
|
LocalDateTime now = LocalDateTime.now();
|
||||||
|
|
||||||
|
TaskChunkEntity existingChunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.eq(TaskChunkEntity::getScopeHash, scopeHash)
|
||||||
|
.eq(TaskChunkEntity::getChunkIndex, chunkIndex)
|
||||||
|
.last("limit 1"));
|
||||||
|
boolean changed = true;
|
||||||
|
if (existingChunk == null) {
|
||||||
|
TaskChunkEntity entity = new TaskChunkEntity();
|
||||||
|
entity.setTaskId(taskId);
|
||||||
|
entity.setModuleType(MODULE_TYPE);
|
||||||
|
entity.setScopeKey(normalizedScopeKey);
|
||||||
|
entity.setScopeHash(scopeHash);
|
||||||
|
entity.setChunkIndex(chunkIndex);
|
||||||
|
entity.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal());
|
||||||
|
entity.setPayloadJson(payloadJson);
|
||||||
|
entity.setPayloadHash(payloadHash);
|
||||||
|
entity.setCreatedAt(now);
|
||||||
|
entity.setUpdatedAt(now);
|
||||||
|
try {
|
||||||
|
taskChunkMapper.insert(entity);
|
||||||
|
} catch (DuplicateKeyException ex) {
|
||||||
|
TaskChunkEntity concurrentChunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.eq(TaskChunkEntity::getScopeHash, scopeHash)
|
||||||
|
.eq(TaskChunkEntity::getChunkIndex, chunkIndex)
|
||||||
|
.last("limit 1"));
|
||||||
|
if (concurrentChunk != null) {
|
||||||
|
changed = !payloadHash.equals(concurrentChunk.getPayloadHash());
|
||||||
|
concurrentChunk.setScopeKey(normalizedScopeKey);
|
||||||
|
concurrentChunk.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal());
|
||||||
|
concurrentChunk.setPayloadJson(payloadJson);
|
||||||
|
concurrentChunk.setPayloadHash(payloadHash);
|
||||||
|
concurrentChunk.setUpdatedAt(now);
|
||||||
|
taskChunkMapper.updateById(concurrentChunk);
|
||||||
|
} else {
|
||||||
|
throw ex;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
changed = !payloadHash.equals(existingChunk.getPayloadHash());
|
||||||
|
existingChunk.setScopeKey(normalizedScopeKey);
|
||||||
|
existingChunk.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal());
|
||||||
|
existingChunk.setPayloadJson(payloadJson);
|
||||||
|
existingChunk.setPayloadHash(payloadHash);
|
||||||
|
existingChunk.setUpdatedAt(now);
|
||||||
|
taskChunkMapper.updateById(existingChunk);
|
||||||
|
}
|
||||||
|
|
||||||
|
refreshScopeState(taskId, normalizedScopeKey, scopeHash, chunk.getChunkTotal(), now);
|
||||||
|
return changed;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Map<String, List<DeleteBrandResultFileDto>> loadMergedChunks(Long taskId) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
return Map.of();
|
||||||
|
}
|
||||||
|
List<TaskChunkEntity> entities = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.orderByAsc(TaskChunkEntity::getScopeHash)
|
||||||
|
.orderByAsc(TaskChunkEntity::getChunkIndex)
|
||||||
|
.orderByAsc(TaskChunkEntity::getId));
|
||||||
|
if (entities == null || entities.isEmpty()) {
|
||||||
|
return Map.of();
|
||||||
|
}
|
||||||
|
Map<String, List<DeleteBrandResultFileDto>> grouped = new LinkedHashMap<>();
|
||||||
|
for (TaskChunkEntity entity : entities) {
|
||||||
|
if (entity.getPayloadJson() == null || entity.getPayloadJson().isBlank()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
DeleteBrandResultFileDto dto = objectMapper.readValue(entity.getPayloadJson(), DeleteBrandResultFileDto.class);
|
||||||
|
grouped.computeIfAbsent(entity.getScopeKey(), ignored -> new ArrayList<>()).add(dto);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException("failed to load delete-brand result chunks");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (List<DeleteBrandResultFileDto> chunks : grouped.values()) {
|
||||||
|
chunks.sort(Comparator.comparing(DeleteBrandResultFileDto::getChunkIndex, Comparator.nullsLast(Integer::compareTo)));
|
||||||
|
}
|
||||||
|
return grouped;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public void deleteTaskData(Long taskId) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.select(TaskScopeStateEntity::getParsedPayloadJson)
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
|
||||||
|
if (states != null) {
|
||||||
|
for (TaskScopeStateEntity state : states) {
|
||||||
|
deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), null);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||||
|
taskScopeStateMapper.delete(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
|
||||||
|
}
|
||||||
|
|
||||||
|
private void refreshScopeState(Long taskId, String scopeKey, String scopeHash, Integer chunkTotal, LocalDateTime now) {
|
||||||
|
Long receivedCountRaw = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.eq(TaskChunkEntity::getScopeHash, scopeHash));
|
||||||
|
int receivedCount = receivedCountRaw == null ? 0 : receivedCountRaw.intValue();
|
||||||
|
int normalizedChunkTotal = chunkTotal == null || chunkTotal < 0 ? 0 : chunkTotal;
|
||||||
|
|
||||||
|
TaskScopeStateEntity state = getScopeState(taskId, scopeHash);
|
||||||
|
if (state == null) {
|
||||||
|
state = new TaskScopeStateEntity();
|
||||||
|
state.setTaskId(taskId);
|
||||||
|
state.setModuleType(MODULE_TYPE);
|
||||||
|
state.setScopeKey(scopeKey);
|
||||||
|
state.setScopeHash(scopeHash);
|
||||||
|
state.setParsedPayloadJson(null);
|
||||||
|
state.setChunkTotal(normalizedChunkTotal);
|
||||||
|
state.setReceivedChunkCount(receivedCount);
|
||||||
|
state.setCompleted(normalizedChunkTotal > 0 && receivedCount >= normalizedChunkTotal ? 1 : 0);
|
||||||
|
state.setLastChunkAt(now);
|
||||||
|
state.setLastError(null);
|
||||||
|
state.setCreatedAt(now);
|
||||||
|
state.setUpdatedAt(now);
|
||||||
|
try {
|
||||||
|
taskScopeStateMapper.insert(state);
|
||||||
|
} catch (DuplicateKeyException ex) {
|
||||||
|
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.eq(TaskScopeStateEntity::getScopeHash, scopeHash)
|
||||||
|
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
|
||||||
|
.set(TaskScopeStateEntity::getChunkTotal, normalizedChunkTotal)
|
||||||
|
.set(TaskScopeStateEntity::getReceivedChunkCount, receivedCount)
|
||||||
|
.set(TaskScopeStateEntity::getCompleted, normalizedChunkTotal > 0 && receivedCount >= normalizedChunkTotal ? 1 : 0)
|
||||||
|
.set(TaskScopeStateEntity::getLastChunkAt, now)
|
||||||
|
.set(TaskScopeStateEntity::getLastError, null)
|
||||||
|
.set(TaskScopeStateEntity::getUpdatedAt, now));
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
int mergedChunkTotal = Math.max(state.getChunkTotal() == null ? 0 : state.getChunkTotal(), normalizedChunkTotal);
|
||||||
|
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getId, state.getId())
|
||||||
|
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
|
||||||
|
.set(TaskScopeStateEntity::getChunkTotal, mergedChunkTotal)
|
||||||
|
.set(TaskScopeStateEntity::getReceivedChunkCount, receivedCount)
|
||||||
|
.set(TaskScopeStateEntity::getCompleted, mergedChunkTotal > 0 && receivedCount >= mergedChunkTotal ? 1 : 0)
|
||||||
|
.set(TaskScopeStateEntity::getLastChunkAt, now)
|
||||||
|
.set(TaskScopeStateEntity::getLastError, null)
|
||||||
|
.set(TaskScopeStateEntity::getUpdatedAt, now));
|
||||||
|
}
|
||||||
|
|
||||||
|
private TaskScopeStateEntity getScopeState(Long taskId, String scopeHash) {
|
||||||
|
return taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.eq(TaskScopeStateEntity::getScopeHash, scopeHash)
|
||||||
|
.last("limit 1"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, TaskScopeStateEntity> loadScopeStateByHash(Long taskId) {
|
||||||
|
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
|
||||||
|
Map<String, TaskScopeStateEntity> result = new LinkedHashMap<>();
|
||||||
|
if (states == null) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
for (TaskScopeStateEntity state : states) {
|
||||||
|
if (!isBlank(state.getScopeHash())) {
|
||||||
|
result.put(state.getScopeHash(), state);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String storeParsedPayload(Long taskId, String scopeHash, String payloadJson) {
|
||||||
|
try {
|
||||||
|
String pointer = OSS_POINTER_PREFIX + ossStorageService.uploadTaskParsedPayload(MODULE_TYPE, taskId, scopeHash, payloadJson);
|
||||||
|
return objectMapper.writeValueAsString(pointer);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException("failed to save delete-brand parsed payload");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveParsedPayload(String value) {
|
||||||
|
if (isBlank(value)) {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
String ossPointer = extractOssPointer(value);
|
||||||
|
if (ossPointer == null) {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return ossStorageService.readObjectAsString(ossPointer.substring(OSS_POINTER_PREFIX.length()));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException("failed to load delete-brand parsed payload");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void deleteParsedPayloadIfNeeded(String oldValue, String newValue) {
|
||||||
|
String oldPointer = extractOssPointer(oldValue);
|
||||||
|
String newPointer = extractOssPointer(newValue);
|
||||||
|
if (oldPointer == null || oldPointer.equals(newPointer)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
ossStorageService.deleteObject(oldPointer.substring(OSS_POINTER_PREFIX.length()));
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String extractOssPointer(String value) {
|
||||||
|
if (isBlank(value)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (value.startsWith(OSS_POINTER_PREFIX)) {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
String decoded = objectMapper.readValue(value, String.class);
|
||||||
|
return decoded != null && decoded.startsWith(OSS_POINTER_PREFIX) ? decoded : null;
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String writeJson(Object value, String message) {
|
||||||
|
try {
|
||||||
|
return objectMapper.writeValueAsString(value);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException(message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeScopeKey(String scopeKey) {
|
||||||
|
return scopeKey == null ? "" : scopeKey.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isBlank(String value) {
|
||||||
|
return value == null || value.isBlank();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String hash(String value) {
|
||||||
|
try {
|
||||||
|
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||||
|
byte[] bytes = digest.digest(normalizeScopeKey(value).getBytes(StandardCharsets.UTF_8));
|
||||||
|
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
||||||
|
for (byte b : bytes) {
|
||||||
|
sb.append(String.format("%02x", b));
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new IllegalStateException("failed to hash delete-brand scope", ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,14 +2,18 @@ package com.nanri.aiimage.modules.file.service.oss;
|
|||||||
|
|
||||||
import com.aliyun.oss.OSS;
|
import com.aliyun.oss.OSS;
|
||||||
import com.aliyun.oss.OSSClientBuilder;
|
import com.aliyun.oss.OSSClientBuilder;
|
||||||
|
import com.aliyun.oss.model.OSSObject;
|
||||||
import com.nanri.aiimage.config.OssProperties;
|
import com.nanri.aiimage.config.OssProperties;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.io.ByteArrayInputStream;
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.util.Date;
|
import java.util.Date;
|
||||||
|
import java.util.Objects;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
@@ -33,6 +37,68 @@ public class OssStorageService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public String uploadText(String objectKey, String content) {
|
||||||
|
if (objectKey == null || objectKey.isBlank()) {
|
||||||
|
throw new IllegalArgumentException("objectKey must not be blank");
|
||||||
|
}
|
||||||
|
OSS ossClient = buildClient();
|
||||||
|
try (ByteArrayInputStream stream = new ByteArrayInputStream(
|
||||||
|
Objects.requireNonNullElse(content, "").getBytes(StandardCharsets.UTF_8))) {
|
||||||
|
ossClient.putObject(ossProperties.getBucket(), objectKey, stream);
|
||||||
|
return objectKey;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new IllegalStateException("failed to upload text to oss", ex);
|
||||||
|
} finally {
|
||||||
|
ossClient.shutdown();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public String uploadTaskScopePayload(String moduleType, Long taskId, String scopeHash, String content) {
|
||||||
|
String normalizedModuleType = moduleType == null || moduleType.isBlank()
|
||||||
|
? "unknown"
|
||||||
|
: moduleType.trim().toLowerCase();
|
||||||
|
String normalizedScopeHash = scopeHash == null || scopeHash.isBlank() ? UUID.randomUUID().toString() : scopeHash;
|
||||||
|
String objectKey = String.format("task-scope/%s/%s/%s.json", normalizedModuleType, taskId, normalizedScopeHash);
|
||||||
|
return uploadText(objectKey, content);
|
||||||
|
}
|
||||||
|
|
||||||
|
public String uploadTaskParsedPayload(String moduleType, Long taskId, String scopeHash, String content) {
|
||||||
|
String normalizedModuleType = moduleType == null || moduleType.isBlank()
|
||||||
|
? "unknown"
|
||||||
|
: moduleType.trim().toLowerCase();
|
||||||
|
String normalizedScopeHash = scopeHash == null || scopeHash.isBlank() ? UUID.randomUUID().toString() : scopeHash;
|
||||||
|
String objectKey = String.format("task-parsed/%s/%s/%s.json", normalizedModuleType, taskId, normalizedScopeHash);
|
||||||
|
return uploadText(objectKey, content);
|
||||||
|
}
|
||||||
|
|
||||||
|
public String readObjectAsString(String value) {
|
||||||
|
String objectKey = resolveObjectKey(value);
|
||||||
|
if (objectKey == null || objectKey.isBlank()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
OSS ossClient = buildClient();
|
||||||
|
try (OSSObject ossObject = ossClient.getObject(ossProperties.getBucket(), objectKey)) {
|
||||||
|
return new String(ossObject.getObjectContent().readAllBytes(), StandardCharsets.UTF_8);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new IllegalStateException("failed to read object from oss", ex);
|
||||||
|
} finally {
|
||||||
|
ossClient.shutdown();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void deleteObject(String value) {
|
||||||
|
String objectKey = resolveObjectKey(value);
|
||||||
|
if (objectKey == null || objectKey.isBlank()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
OSS ossClient = buildClient();
|
||||||
|
try {
|
||||||
|
ossClient.deleteObject(ossProperties.getBucket(), objectKey);
|
||||||
|
} finally {
|
||||||
|
ossClient.shutdown();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 根据 objectKey 生成预签名下载 URL(1小时有效)。
|
* 根据 objectKey 生成预签名下载 URL(1小时有效)。
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import com.nanri.aiimage.modules.patroldelete.model.vo.PatrolDeleteResultItemVo;
|
|||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.poi.ss.usermodel.Row;
|
import org.apache.poi.ss.usermodel.Row;
|
||||||
import org.apache.poi.ss.usermodel.Sheet;
|
import org.apache.poi.ss.usermodel.Sheet;
|
||||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
@@ -40,8 +40,9 @@ public class PatrolDeleteExcelAssemblyService {
|
|||||||
};
|
};
|
||||||
|
|
||||||
public void writeWorkbook(File outputXlsx, List<PatrolDeleteResultItemVo> items) {
|
public void writeWorkbook(File outputXlsx, List<PatrolDeleteResultItemVo> items) {
|
||||||
try (XSSFWorkbook workbook = new XSSFWorkbook();
|
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
|
||||||
FileOutputStream outputStream = new FileOutputStream(outputXlsx)) {
|
workbook.setCompressTempFiles(true);
|
||||||
|
try (FileOutputStream outputStream = new FileOutputStream(outputXlsx)) {
|
||||||
Sheet sheet = workbook.createSheet("巡店删除结果");
|
Sheet sheet = workbook.createSheet("巡店删除结果");
|
||||||
Row headerRow = sheet.createRow(0);
|
Row headerRow = sheet.createRow(0);
|
||||||
for (int columnIndex = 0; columnIndex < HEADER.length; columnIndex++) {
|
for (int columnIndex = 0; columnIndex < HEADER.length; columnIndex++) {
|
||||||
@@ -73,13 +74,17 @@ public class PatrolDeleteExcelAssemblyService {
|
|||||||
rowIndex++;
|
rowIndex++;
|
||||||
}
|
}
|
||||||
|
|
||||||
for (int columnIndex = 0; columnIndex < HEADER.length; columnIndex++) {
|
applyDefaultColumnWidths(sheet);
|
||||||
sheet.autoSizeColumn(columnIndex);
|
|
||||||
}
|
|
||||||
workbook.write(outputStream);
|
workbook.write(outputStream);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
log.warn("[patrol-delete] write workbook failed: {}", ex.getMessage());
|
log.warn("[patrol-delete] write workbook failed: {}", ex.getMessage());
|
||||||
throw new BusinessException("生成巡店删除结果 Excel 失败: " + ex.getMessage());
|
throw new BusinessException("generate patrol-delete excel failed: " + ex.getMessage());
|
||||||
|
} finally {
|
||||||
|
try {
|
||||||
|
workbook.close();
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
}
|
||||||
|
workbook.dispose();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -141,4 +146,23 @@ public class PatrolDeleteExcelAssemblyService {
|
|||||||
private String safe(String value) {
|
private String safe(String value) {
|
||||||
return value == null ? "" : value;
|
return value == null ? "" : value;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void applyDefaultColumnWidths(Sheet sheet) {
|
||||||
|
int[] widths = {
|
||||||
|
20,
|
||||||
|
14, 10, 12, 16,
|
||||||
|
14, 10, 12, 16,
|
||||||
|
14, 10, 12, 16,
|
||||||
|
14, 10, 12, 16,
|
||||||
|
14, 10, 12, 16,
|
||||||
|
10, 16,
|
||||||
|
10, 16,
|
||||||
|
10, 16,
|
||||||
|
10, 16,
|
||||||
|
10, 16
|
||||||
|
};
|
||||||
|
for (int i = 0; i < widths.length; i++) {
|
||||||
|
sheet.setColumnWidth(i, widths[i] * 256);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
package com.nanri.aiimage.modules.patroldelete.service;
|
package com.nanri.aiimage.modules.patroldelete.service;
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.nanri.aiimage.common.exception.BusinessException;
|
|
||||||
import com.nanri.aiimage.config.TaskPressureProperties;
|
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||||
import com.nanri.aiimage.modules.patroldelete.model.dto.PatrolDeleteShopPayloadDto;
|
import com.nanri.aiimage.modules.patroldelete.model.dto.PatrolDeleteShopPayloadDto;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
@@ -20,80 +20,40 @@ import java.util.concurrent.ConcurrentHashMap;
|
|||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public class PatrolDeleteTaskCacheService {
|
public class PatrolDeleteTaskCacheService {
|
||||||
|
|
||||||
|
private static final String MODULE_TYPE = "PATROL_DELETE";
|
||||||
private static final long PAYLOAD_TTL_HOURS = 24;
|
private static final long PAYLOAD_TTL_HOURS = 24;
|
||||||
private final StringRedisTemplate stringRedisTemplate;
|
private final StringRedisTemplate stringRedisTemplate;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
private final TaskPressureProperties taskPressureProperties;
|
private final TaskPressureProperties taskPressureProperties;
|
||||||
|
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
|
||||||
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
|
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
|
||||||
|
|
||||||
public PatrolDeleteShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) {
|
public PatrolDeleteShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) {
|
||||||
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
|
return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, PatrolDeleteShopPayloadDto.class);
|
||||||
return null;
|
|
||||||
}
|
|
||||||
Object raw = stringRedisTemplate.opsForHash().get(buildShopPayloadKey(taskId), shopKey);
|
|
||||||
if (!(raw instanceof String json) || json.isBlank()) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return objectMapper.readValue(json, PatrolDeleteShopPayloadDto.class);
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("读取巡店删除店铺缓存失败");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void saveShopMergedPayload(Long taskId, String shopKey, PatrolDeleteShopPayloadDto payload) {
|
public void saveShopMergedPayload(Long taskId, String shopKey, PatrolDeleteShopPayloadDto payload) {
|
||||||
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank() || payload == null) {
|
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank() || payload == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
try {
|
taskScopePayloadStorageService.saveScopePayload(taskId, MODULE_TYPE, shopKey, payload);
|
||||||
stringRedisTemplate.opsForHash().put(buildShopPayloadKey(taskId), shopKey, objectMapper.writeValueAsString(payload));
|
|
||||||
stringRedisTemplate.expire(buildShopPayloadKey(taskId), Duration.ofHours(PAYLOAD_TTL_HOURS));
|
|
||||||
touchTaskHeartbeat(taskId);
|
touchTaskHeartbeat(taskId);
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("暂存巡店删除店铺缓存失败");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void removeShopMergedPayload(Long taskId, String shopKey) {
|
public void removeShopMergedPayload(Long taskId, String shopKey) {
|
||||||
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
|
taskScopePayloadStorageService.removeScopePayload(taskId, MODULE_TYPE, shopKey);
|
||||||
return;
|
|
||||||
}
|
|
||||||
stringRedisTemplate.opsForHash().delete(buildShopPayloadKey(taskId), shopKey);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public Map<String, PatrolDeleteShopPayloadDto> getAllShopMergedPayload(Long taskId) {
|
public Map<String, PatrolDeleteShopPayloadDto> getAllShopMergedPayload(Long taskId) {
|
||||||
try {
|
return taskScopePayloadStorageService.getAllScopePayload(taskId, MODULE_TYPE, PatrolDeleteShopPayloadDto.class);
|
||||||
Map<Object, Object> raw = stringRedisTemplate.opsForHash().entries(buildShopPayloadKey(taskId));
|
|
||||||
if (raw == null || raw.isEmpty()) {
|
|
||||||
return Map.of();
|
|
||||||
}
|
|
||||||
Map<String, PatrolDeleteShopPayloadDto> out = new LinkedHashMap<>();
|
|
||||||
for (Map.Entry<Object, Object> entry : raw.entrySet()) {
|
|
||||||
if (!(entry.getKey() instanceof String key) || !(entry.getValue() instanceof String val) || val.isBlank()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
out.put(key, objectMapper.readValue(val, PatrolDeleteShopPayloadDto.class));
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("读取巡店删除缓存失败");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean hasAnyShopMergedPayload(Long taskId) {
|
public boolean hasAnyShopMergedPayload(Long taskId) {
|
||||||
if (taskId == null || taskId <= 0) {
|
return taskScopePayloadStorageService.hasAnyScopePayload(taskId, MODULE_TYPE);
|
||||||
return false;
|
|
||||||
}
|
|
||||||
Long size = stringRedisTemplate.opsForHash().size(buildShopPayloadKey(taskId));
|
|
||||||
return size != null && size > 0;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public long countShopMergedPayload(Long taskId) {
|
public long countShopMergedPayload(Long taskId) {
|
||||||
if (taskId == null || taskId <= 0) {
|
return taskScopePayloadStorageService.countScopePayload(taskId, MODULE_TYPE);
|
||||||
return 0L;
|
|
||||||
}
|
|
||||||
Long size = stringRedisTemplate.opsForHash().size(buildShopPayloadKey(taskId));
|
|
||||||
return size == null ? 0L : size;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public long getTaskHeartbeatMillis(Long taskId) {
|
public long getTaskHeartbeatMillis(Long taskId) {
|
||||||
@@ -123,9 +83,9 @@ public class PatrolDeleteTaskCacheService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
taskEntityLocalCache.remove(taskId);
|
taskEntityLocalCache.remove(taskId);
|
||||||
stringRedisTemplate.delete(buildShopPayloadKey(taskId));
|
|
||||||
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
|
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
|
||||||
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
|
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
|
||||||
|
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void saveTaskCache(FileTaskEntity task) {
|
public void saveTaskCache(FileTaskEntity task) {
|
||||||
@@ -190,10 +150,6 @@ public class PatrolDeleteTaskCacheService {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
private String buildShopPayloadKey(Long taskId) {
|
|
||||||
return "patrol-delete:task:shop-payload:" + taskId;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String buildTaskHeartbeatKey(Long taskId) {
|
private String buildTaskHeartbeatKey(Long taskId) {
|
||||||
return "patrol-delete:task:heartbeat:" + taskId;
|
return "patrol-delete:task:heartbeat:" + taskId;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -93,5 +93,14 @@ public class PriceTrackSubmitResultRequest {
|
|||||||
|
|
||||||
@Schema(description = "状态", example = "SUCCESS")
|
@Schema(description = "状态", example = "SUCCESS")
|
||||||
private String status;
|
private String status;
|
||||||
|
|
||||||
|
@Schema(description = "Python 确认是否需要从 skip ASIN 表删除当前 ASIN", example = "true")
|
||||||
|
private Boolean deleteSkipAsin;
|
||||||
|
|
||||||
|
@Schema(description = "需要删除的 skip ASIN;不传时默认使用 asin 字段", example = "B0ABCDE123")
|
||||||
|
private String removeAsin;
|
||||||
|
|
||||||
|
@Schema(description = "删除确认原因,便于排查日志", example = "CURRENT_PRICE_BELOW_MINIMUM")
|
||||||
|
private String deleteReason;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,6 +18,9 @@ public class PriceTrackCreateTaskVo {
|
|||||||
@Schema(description = "all skip asins grouped by country")
|
@Schema(description = "all skip asins grouped by country")
|
||||||
private Map<String, List<String>> skipAsinsByCountry;
|
private Map<String, List<String>> skipAsinsByCountry;
|
||||||
|
|
||||||
|
@Schema(description = "all skip asin details grouped by country code, each item includes asin and minimumPrice")
|
||||||
|
private Map<String, List<Map<String, String>>> skipAsinDetailsByCountry;
|
||||||
|
|
||||||
@Schema(description = "parsed asin rows by country")
|
@Schema(description = "parsed asin rows by country")
|
||||||
private Map<String, List<Map<String, String>>> asinRowsByCountry;
|
private Map<String, List<Map<String, String>>> asinRowsByCountry;
|
||||||
|
|
||||||
|
|||||||
@@ -17,6 +17,9 @@ public class PriceTrackMatchShopsVo {
|
|||||||
@Schema(description = "All skip ASINs grouped by country code")
|
@Schema(description = "All skip ASINs grouped by country code")
|
||||||
private Map<String, List<String>> skipAsinsByCountry;
|
private Map<String, List<String>> skipAsinsByCountry;
|
||||||
|
|
||||||
|
@Schema(description = "All skip ASIN details grouped by country code, each item includes asin and minimumPrice")
|
||||||
|
private Map<String, List<Map<String, String>>> skipAsinDetailsByCountry;
|
||||||
|
|
||||||
@Schema(description = "Parsed asin rows grouped by country code")
|
@Schema(description = "Parsed asin rows grouped by country code")
|
||||||
private Map<String, List<Map<String, String>>> asinRowsByCountry;
|
private Map<String, List<Map<String, String>>> asinRowsByCountry;
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import com.nanri.aiimage.modules.productrisk.model.enums.ProductRiskCountryCode;
|
|||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.poi.ss.usermodel.Row;
|
import org.apache.poi.ss.usermodel.Row;
|
||||||
import org.apache.poi.ss.usermodel.Sheet;
|
import org.apache.poi.ss.usermodel.Sheet;
|
||||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
@@ -20,25 +20,27 @@ import java.util.Map;
|
|||||||
public class PriceTrackExcelAssemblyService {
|
public class PriceTrackExcelAssemblyService {
|
||||||
|
|
||||||
private static final String[] RESULT_HEADER = {
|
private static final String[] RESULT_HEADER = {
|
||||||
"\u5e97\u94fa\u5546\u57ce\u540d\u79f0",
|
"店铺商城名称",
|
||||||
"ASIN",
|
"ASIN",
|
||||||
"\u4ef7\u683c",
|
"价格",
|
||||||
"\u63a8\u8350\u4ef7",
|
"推荐价",
|
||||||
"\u6700\u4f4e\u4ef7",
|
"最低价",
|
||||||
"\u7b2c\u4e00\u540d",
|
"第一名",
|
||||||
"\u7b2c\u4e8c\u540d",
|
"第二名",
|
||||||
"\u8d2d\u7269\u8f66\u5e97\u94fa\u540d",
|
"购物车店铺名",
|
||||||
"\u6539\u4ef7\u60c5\u51b5",
|
"改价情况",
|
||||||
"\u4fee\u6539\u6b21\u6570",
|
"修改次数",
|
||||||
"\u72b6\u6001"
|
"状态"
|
||||||
};
|
};
|
||||||
|
|
||||||
public void writeWorkbook(File outputXlsx, Map<String, List<PriceTrackSubmitResultRequest.AsinResult>> countries) {
|
public void writeWorkbook(File outputXlsx, Map<String, List<PriceTrackSubmitResultRequest.AsinResult>> countries) {
|
||||||
Map<String, List<PriceTrackSubmitResultRequest.AsinResult>> safe = countries == null ? Map.of() : countries;
|
Map<String, List<PriceTrackSubmitResultRequest.AsinResult>> safe = countries == null ? Map.of() : countries;
|
||||||
try (XSSFWorkbook wb = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(outputXlsx)) {
|
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
|
||||||
|
workbook.setCompressTempFiles(true);
|
||||||
|
try (FileOutputStream fos = new FileOutputStream(outputXlsx)) {
|
||||||
for (ProductRiskCountryCode code : ProductRiskCountryCode.values()) {
|
for (ProductRiskCountryCode code : ProductRiskCountryCode.values()) {
|
||||||
String sheetName = code.name();
|
String sheetName = code.name();
|
||||||
Sheet sheet = wb.createSheet(sheetName);
|
Sheet sheet = workbook.createSheet(sheetName);
|
||||||
Row headerRow = sheet.createRow(0);
|
Row headerRow = sheet.createRow(0);
|
||||||
for (int i = 0; i < RESULT_HEADER.length; i++) {
|
for (int i = 0; i < RESULT_HEADER.length; i++) {
|
||||||
headerRow.createCell(i).setCellValue(RESULT_HEADER[i]);
|
headerRow.createCell(i).setCellValue(RESULT_HEADER[i]);
|
||||||
@@ -62,14 +64,18 @@ public class PriceTrackExcelAssemblyService {
|
|||||||
row.createCell(9).setCellValue(valueOf(item.getModifyCount()));
|
row.createCell(9).setCellValue(valueOf(item.getModifyCount()));
|
||||||
row.createCell(10).setCellValue(valueOf(item.getStatus()));
|
row.createCell(10).setCellValue(valueOf(item.getStatus()));
|
||||||
}
|
}
|
||||||
for (int i = 0; i < RESULT_HEADER.length; i++) {
|
applyDefaultColumnWidths(sheet);
|
||||||
sheet.autoSizeColumn(i);
|
|
||||||
}
|
}
|
||||||
}
|
workbook.write(fos);
|
||||||
wb.write(fos);
|
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
log.warn("[price-track] write workbook failed: {}", ex.getMessage());
|
log.warn("[price-track] write workbook failed: {}", ex.getMessage());
|
||||||
throw new BusinessException("鐢熸垚 Excel 澶辫触: " + ex.getMessage());
|
throw new BusinessException("generate price-track excel failed: " + ex.getMessage());
|
||||||
|
} finally {
|
||||||
|
try {
|
||||||
|
workbook.close();
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
}
|
||||||
|
workbook.dispose();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -105,4 +111,10 @@ public class PriceTrackExcelAssemblyService {
|
|||||||
return value == null ? "" : value;
|
return value == null ? "" : value;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void applyDefaultColumnWidths(Sheet sheet) {
|
||||||
|
int[] widths = {22, 18, 12, 12, 12, 20, 20, 22, 18, 12, 12};
|
||||||
|
for (int i = 0; i < widths.length; i++) {
|
||||||
|
sheet.setColumnWidth(i, widths[i] * 256);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -148,6 +148,9 @@ public class PriceTrackService {
|
|||||||
Map<String, List<String>> skipAsinsByCountry = asinMode
|
Map<String, List<String>> skipAsinsByCountry = asinMode
|
||||||
? new LinkedHashMap<>()
|
? new LinkedHashMap<>()
|
||||||
: skipPriceAsinService.listAllSkipAsinsByCountry();
|
: skipPriceAsinService.listAllSkipAsinsByCountry();
|
||||||
|
Map<String, List<Map<String, String>>> skipAsinDetailsByCountry = asinMode
|
||||||
|
? new LinkedHashMap<>()
|
||||||
|
: skipPriceAsinService.listAllSkipAsinDetailsByCountry();
|
||||||
Map<String, List<Map<String, String>>> asinRowsByCountry =
|
Map<String, List<Map<String, String>>> asinRowsByCountry =
|
||||||
!asinMode
|
!asinMode
|
||||||
? new LinkedHashMap<>()
|
? new LinkedHashMap<>()
|
||||||
@@ -157,6 +160,7 @@ public class PriceTrackService {
|
|||||||
|
|
||||||
PriceTrackMatchShopsVo vo = new PriceTrackMatchShopsVo();
|
PriceTrackMatchShopsVo vo = new PriceTrackMatchShopsVo();
|
||||||
vo.setSkipAsinsByCountry(skipAsinsByCountry);
|
vo.setSkipAsinsByCountry(skipAsinsByCountry);
|
||||||
|
vo.setSkipAsinDetailsByCountry(skipAsinDetailsByCountry);
|
||||||
vo.setAsinRowsByCountry(asinRowsByCountry);
|
vo.setAsinRowsByCountry(asinRowsByCountry);
|
||||||
vo.setMinimumPriceByCountryAndAsin(priceTrackTaskService.buildMinimumPriceLookupForMatch(asinRowsByCountry));
|
vo.setMinimumPriceByCountryAndAsin(priceTrackTaskService.buildMinimumPriceLookupForMatch(asinRowsByCountry));
|
||||||
for (String shopName : ordered) {
|
for (String shopName : ordered) {
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
package com.nanri.aiimage.modules.pricetrack.service;
|
package com.nanri.aiimage.modules.pricetrack.service;
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.nanri.aiimage.common.exception.BusinessException;
|
|
||||||
import com.nanri.aiimage.config.TaskPressureProperties;
|
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||||
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackSubmitResultRequest;
|
import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackSubmitResultRequest;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
@@ -20,10 +20,12 @@ import java.util.concurrent.ConcurrentHashMap;
|
|||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public class PriceTrackTaskCacheService {
|
public class PriceTrackTaskCacheService {
|
||||||
|
|
||||||
|
private static final String MODULE_TYPE = "PRICE_TRACK";
|
||||||
private static final long HEARTBEAT_TTL_HOURS = 24;
|
private static final long HEARTBEAT_TTL_HOURS = 24;
|
||||||
private final StringRedisTemplate stringRedisTemplate;
|
private final StringRedisTemplate stringRedisTemplate;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
private final TaskPressureProperties taskPressureProperties;
|
private final TaskPressureProperties taskPressureProperties;
|
||||||
|
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
|
||||||
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
|
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
|
||||||
|
|
||||||
public void touchTaskHeartbeat(Long taskId) {
|
public void touchTaskHeartbeat(Long taskId) {
|
||||||
@@ -52,65 +54,27 @@ public class PriceTrackTaskCacheService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public PriceTrackSubmitResultRequest.ShopResult getShopMergedPayload(Long taskId, String shopKey) {
|
public PriceTrackSubmitResultRequest.ShopResult getShopMergedPayload(Long taskId, String shopKey) {
|
||||||
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
|
return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, PriceTrackSubmitResultRequest.ShopResult.class);
|
||||||
return null;
|
|
||||||
}
|
|
||||||
Object raw = stringRedisTemplate.opsForHash().get(buildShopPayloadKey(taskId), shopKey);
|
|
||||||
if (!(raw instanceof String json) || json.isBlank()) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return objectMapper.readValue(json, PriceTrackSubmitResultRequest.ShopResult.class);
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("read price-track cache failed");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void saveShopMergedPayload(Long taskId, String shopKey, PriceTrackSubmitResultRequest.ShopResult payload) {
|
public void saveShopMergedPayload(Long taskId, String shopKey, PriceTrackSubmitResultRequest.ShopResult payload) {
|
||||||
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank() || payload == null) {
|
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank() || payload == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
try {
|
taskScopePayloadStorageService.saveScopePayload(taskId, MODULE_TYPE, shopKey, payload);
|
||||||
stringRedisTemplate.opsForHash().put(buildShopPayloadKey(taskId), shopKey, objectMapper.writeValueAsString(payload));
|
|
||||||
stringRedisTemplate.expire(buildShopPayloadKey(taskId), Duration.ofHours(HEARTBEAT_TTL_HOURS));
|
|
||||||
touchTaskHeartbeat(taskId);
|
touchTaskHeartbeat(taskId);
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("save price-track cache failed");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void removeShopMergedPayload(Long taskId, String shopKey) {
|
public void removeShopMergedPayload(Long taskId, String shopKey) {
|
||||||
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
|
taskScopePayloadStorageService.removeScopePayload(taskId, MODULE_TYPE, shopKey);
|
||||||
return;
|
|
||||||
}
|
|
||||||
stringRedisTemplate.opsForHash().delete(buildShopPayloadKey(taskId), shopKey);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public Map<String, PriceTrackSubmitResultRequest.ShopResult> getAllShopMergedPayload(Long taskId) {
|
public Map<String, PriceTrackSubmitResultRequest.ShopResult> getAllShopMergedPayload(Long taskId) {
|
||||||
try {
|
return taskScopePayloadStorageService.getAllScopePayload(taskId, MODULE_TYPE, PriceTrackSubmitResultRequest.ShopResult.class);
|
||||||
Map<Object, Object> raw = stringRedisTemplate.opsForHash().entries(buildShopPayloadKey(taskId));
|
|
||||||
if (raw == null || raw.isEmpty()) {
|
|
||||||
return Map.of();
|
|
||||||
}
|
|
||||||
LinkedHashMap<String, PriceTrackSubmitResultRequest.ShopResult> out = new LinkedHashMap<>();
|
|
||||||
for (Map.Entry<Object, Object> entry : raw.entrySet()) {
|
|
||||||
if (!(entry.getKey() instanceof String key) || !(entry.getValue() instanceof String val) || val.isBlank()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
out.put(key, objectMapper.readValue(val, PriceTrackSubmitResultRequest.ShopResult.class));
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("read price-track cache failed");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean hasAnyShopMergedPayload(Long taskId) {
|
public boolean hasAnyShopMergedPayload(Long taskId) {
|
||||||
if (taskId == null || taskId <= 0) {
|
return taskScopePayloadStorageService.hasAnyScopePayload(taskId, MODULE_TYPE);
|
||||||
return false;
|
|
||||||
}
|
|
||||||
Long size = stringRedisTemplate.opsForHash().size(buildShopPayloadKey(taskId));
|
|
||||||
return size != null && size > 0;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void deleteTaskCache(Long taskId) {
|
public void deleteTaskCache(Long taskId) {
|
||||||
@@ -118,9 +82,9 @@ public class PriceTrackTaskCacheService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
taskEntityLocalCache.remove(taskId);
|
taskEntityLocalCache.remove(taskId);
|
||||||
stringRedisTemplate.delete(buildShopPayloadKey(taskId));
|
|
||||||
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
|
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
|
||||||
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
|
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
|
||||||
|
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void saveTaskCache(FileTaskEntity task) {
|
public void saveTaskCache(FileTaskEntity task) {
|
||||||
@@ -185,10 +149,6 @@ public class PriceTrackTaskCacheService {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
private String buildShopPayloadKey(Long taskId) {
|
|
||||||
return "price-track:task:shop-payload:" + taskId;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String buildTaskHeartbeatKey(Long taskId) {
|
private String buildTaskHeartbeatKey(Long taskId) {
|
||||||
return "price-track:task:heartbeat:" + taskId;
|
return "price-track:task:heartbeat:" + taskId;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -206,7 +206,7 @@ public class PriceTrackTaskService {
|
|||||||
@Transactional
|
@Transactional
|
||||||
public PriceTrackPendingDeleteVo deletePendingShopResult(Long userId, String shopName) {
|
public PriceTrackPendingDeleteVo deletePendingShopResult(Long userId, String shopName) {
|
||||||
if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法");
|
if (userId == null || userId <= 0) throw new BusinessException("user_id 不合法");
|
||||||
if (shopName == null || shopName.isBlank()) throw new BusinessException("shop_name 涓嶈兘涓虹┖");
|
if (shopName == null || shopName.isBlank()) throw new BusinessException("shop_name 不能为空");
|
||||||
String norm = ziniaoShopSwitchService.normalizeShopName(shopName);
|
String norm = ziniaoShopSwitchService.normalizeShopName(shopName);
|
||||||
if (norm.isBlank()) throw new BusinessException("店铺名称无效");
|
if (norm.isBlank()) throw new BusinessException("店铺名称无效");
|
||||||
|
|
||||||
@@ -293,6 +293,9 @@ public class PriceTrackTaskService {
|
|||||||
Map<String, List<String>> skipAsinsByCountry = request.isAsinMode()
|
Map<String, List<String>> skipAsinsByCountry = request.isAsinMode()
|
||||||
? new LinkedHashMap<>()
|
? new LinkedHashMap<>()
|
||||||
: skipPriceAsinService.listAllSkipAsinsByCountry();
|
: skipPriceAsinService.listAllSkipAsinsByCountry();
|
||||||
|
Map<String, List<Map<String, String>>> skipAsinDetailsByCountry = request.isAsinMode()
|
||||||
|
? new LinkedHashMap<>()
|
||||||
|
: skipPriceAsinService.listAllSkipAsinDetailsByCountry();
|
||||||
Map<String, List<Map<String, String>>> asinRowsByCountry = request.isAsinMode()
|
Map<String, List<Map<String, String>>> asinRowsByCountry = request.isAsinMode()
|
||||||
? parseAsinRowsByCountry(request.getAsinFiles(), request.getCountryCodes())
|
? parseAsinRowsByCountry(request.getAsinFiles(), request.getCountryCodes())
|
||||||
: new LinkedHashMap<>();
|
: new LinkedHashMap<>();
|
||||||
@@ -348,6 +351,7 @@ public class PriceTrackTaskService {
|
|||||||
ctx.put("asinMode", request.isAsinMode());
|
ctx.put("asinMode", request.isAsinMode());
|
||||||
ctx.put("countryCodes", request.getCountryCodes());
|
ctx.put("countryCodes", request.getCountryCodes());
|
||||||
ctx.put("skipAsinsByCountry", skipAsinsByCountry);
|
ctx.put("skipAsinsByCountry", skipAsinsByCountry);
|
||||||
|
ctx.put("skipAsinDetailsByCountry", skipAsinDetailsByCountry);
|
||||||
ctx.put("asinRowsByCountry", asinRowsByCountry);
|
ctx.put("asinRowsByCountry", asinRowsByCountry);
|
||||||
ctx.put("minimumPriceByCountryAndAsin", minimumPriceByCountryAndAsin);
|
ctx.put("minimumPriceByCountryAndAsin", minimumPriceByCountryAndAsin);
|
||||||
ctx.put("items", uniqueItems);
|
ctx.put("items", uniqueItems);
|
||||||
@@ -368,6 +372,7 @@ public class PriceTrackTaskService {
|
|||||||
vo.setTaskId(task.getId());
|
vo.setTaskId(task.getId());
|
||||||
vo.setItems(snapshot);
|
vo.setItems(snapshot);
|
||||||
vo.setSkipAsinsByCountry(skipAsinsByCountry);
|
vo.setSkipAsinsByCountry(skipAsinsByCountry);
|
||||||
|
vo.setSkipAsinDetailsByCountry(skipAsinDetailsByCountry);
|
||||||
vo.setAsinRowsByCountry(asinRowsByCountry);
|
vo.setAsinRowsByCountry(asinRowsByCountry);
|
||||||
vo.setMinimumPriceByCountryAndAsin(minimumPriceByCountryAndAsin);
|
vo.setMinimumPriceByCountryAndAsin(minimumPriceByCountryAndAsin);
|
||||||
return vo;
|
return vo;
|
||||||
@@ -419,6 +424,7 @@ public class PriceTrackTaskService {
|
|||||||
priceTrackTaskCacheService.removeShopMergedPayload(taskId, shopKey);
|
priceTrackTaskCacheService.removeShopMergedPayload(taskId, shopKey);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
handleSkipAsinDeletionSignals(shopKey, payload);
|
||||||
PriceTrackSubmitResultRequest.ShopResult merged = mergeShopPayload(taskId, shopKey, payload);
|
PriceTrackSubmitResultRequest.ShopResult merged = mergeShopPayload(taskId, shopKey, payload);
|
||||||
if (!Boolean.TRUE.equals(merged.getSuccess())) {
|
if (!Boolean.TRUE.equals(merged.getSuccess())) {
|
||||||
continue;
|
continue;
|
||||||
@@ -833,8 +839,8 @@ public class PriceTrackTaskService {
|
|||||||
if (value == null) {
|
if (value == null) {
|
||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
return value.replace("\ufeff", "")
|
return value.replace("", "")
|
||||||
.replace("\u3000", " ")
|
.replace(" ", " ")
|
||||||
.replace("\r", " ")
|
.replace("\r", " ")
|
||||||
.replace("\n", " ")
|
.replace("\n", " ")
|
||||||
.trim()
|
.trim()
|
||||||
@@ -1075,6 +1081,11 @@ public class PriceTrackTaskService {
|
|||||||
merged.setPriceChangeStatus(firstNonBlank(incoming.getPriceChangeStatus(), merged.getPriceChangeStatus()));
|
merged.setPriceChangeStatus(firstNonBlank(incoming.getPriceChangeStatus(), merged.getPriceChangeStatus()));
|
||||||
merged.setModifyCount(firstNonBlank(incoming.getModifyCount(), merged.getModifyCount()));
|
merged.setModifyCount(firstNonBlank(incoming.getModifyCount(), merged.getModifyCount()));
|
||||||
merged.setStatus(firstNonBlank(incoming.getStatus(), merged.getStatus()));
|
merged.setStatus(firstNonBlank(incoming.getStatus(), merged.getStatus()));
|
||||||
|
if (incoming.getDeleteSkipAsin() != null) {
|
||||||
|
merged.setDeleteSkipAsin(incoming.getDeleteSkipAsin());
|
||||||
|
}
|
||||||
|
merged.setRemoveAsin(firstNonBlank(incoming.getRemoveAsin(), merged.getRemoveAsin()));
|
||||||
|
merged.setDeleteReason(firstNonBlank(incoming.getDeleteReason(), merged.getDeleteReason()));
|
||||||
return merged;
|
return merged;
|
||||||
}
|
}
|
||||||
private PriceTrackSubmitResultRequest.AsinResult cloneRow(PriceTrackSubmitResultRequest.AsinResult row) {
|
private PriceTrackSubmitResultRequest.AsinResult cloneRow(PriceTrackSubmitResultRequest.AsinResult row) {
|
||||||
@@ -1093,8 +1104,36 @@ public class PriceTrackTaskService {
|
|||||||
out.setPriceChangeStatus(row.getPriceChangeStatus());
|
out.setPriceChangeStatus(row.getPriceChangeStatus());
|
||||||
out.setModifyCount(row.getModifyCount());
|
out.setModifyCount(row.getModifyCount());
|
||||||
out.setStatus(row.getStatus());
|
out.setStatus(row.getStatus());
|
||||||
|
out.setDeleteSkipAsin(row.getDeleteSkipAsin());
|
||||||
|
out.setRemoveAsin(row.getRemoveAsin());
|
||||||
|
out.setDeleteReason(row.getDeleteReason());
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
private void handleSkipAsinDeletionSignals(String shopKey,
|
||||||
|
PriceTrackSubmitResultRequest.ShopResult payload) {
|
||||||
|
if (payload == null || payload.getCountries() == null || payload.getCountries().isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (Map.Entry<String, List<PriceTrackSubmitResultRequest.AsinResult>> entry : payload.getCountries().entrySet()) {
|
||||||
|
String countryCode = entry.getKey();
|
||||||
|
List<PriceTrackSubmitResultRequest.AsinResult> rows = entry.getValue();
|
||||||
|
if (countryCode == null || countryCode.isBlank() || rows == null || rows.isEmpty()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
for (PriceTrackSubmitResultRequest.AsinResult row : rows) {
|
||||||
|
if (row == null || !Boolean.TRUE.equals(row.getDeleteSkipAsin())) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
String targetAsin = firstNonBlank(row.getRemoveAsin(), row.getAsin());
|
||||||
|
if (targetAsin == null || targetAsin.isBlank()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
boolean removed = skipPriceAsinService.removeByShopCountryAndAsin(shopKey, countryCode, targetAsin);
|
||||||
|
log.info("[price-track] skip asin delete signal taskShop={} country={} asin={} removed={} reason={}",
|
||||||
|
shopKey, countryCode, targetAsin, removed, row.getDeleteReason());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
private int countPayloadRows(PriceTrackSubmitResultRequest.ShopResult payload) {
|
private int countPayloadRows(PriceTrackSubmitResultRequest.ShopResult payload) {
|
||||||
if (payload == null) {
|
if (payload == null) {
|
||||||
return 0;
|
return 0;
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import lombok.extern.slf4j.Slf4j;
|
|||||||
import org.apache.poi.ss.usermodel.Cell;
|
import org.apache.poi.ss.usermodel.Cell;
|
||||||
import org.apache.poi.ss.usermodel.Row;
|
import org.apache.poi.ss.usermodel.Row;
|
||||||
import org.apache.poi.ss.usermodel.Sheet;
|
import org.apache.poi.ss.usermodel.Sheet;
|
||||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
@@ -35,9 +35,11 @@ public class ProductRiskExcelAssemblyService {
|
|||||||
|
|
||||||
public void writeWorkbook(File outputXlsx, String shopDisplayName, Map<String, List<ProductRiskRowDto>> countries) {
|
public void writeWorkbook(File outputXlsx, String shopDisplayName, Map<String, List<ProductRiskRowDto>> countries) {
|
||||||
Map<String, List<ProductRiskRowDto>> safe = countries == null ? Map.of() : countries;
|
Map<String, List<ProductRiskRowDto>> safe = countries == null ? Map.of() : countries;
|
||||||
try (XSSFWorkbook wb = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(outputXlsx)) {
|
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
|
||||||
|
workbook.setCompressTempFiles(true);
|
||||||
|
try (FileOutputStream fos = new FileOutputStream(outputXlsx)) {
|
||||||
for (String sheetName : SHEET_ORDER) {
|
for (String sheetName : SHEET_ORDER) {
|
||||||
Sheet sheet = wb.createSheet(sheetName);
|
Sheet sheet = workbook.createSheet(sheetName);
|
||||||
Row headerRow = sheet.createRow(0);
|
Row headerRow = sheet.createRow(0);
|
||||||
for (int c = 0; c < HEADER.length; c++) {
|
for (int c = 0; c < HEADER.length; c++) {
|
||||||
headerRow.createCell(c).setCellValue(HEADER[c]);
|
headerRow.createCell(c).setCellValue(HEADER[c]);
|
||||||
@@ -53,16 +55,20 @@ public class ProductRiskExcelAssemblyService {
|
|||||||
createTextCell(row, 4, dto == null ? null : dto.getRemoveAsin());
|
createTextCell(row, 4, dto == null ? null : dto.getRemoveAsin());
|
||||||
createTextCell(row, 5, dto == null ? null : dto.getRemoveStatus());
|
createTextCell(row, 5, dto == null ? null : dto.getRemoveStatus());
|
||||||
}
|
}
|
||||||
for (int i = 0; i < HEADER.length; i++) {
|
applyDefaultColumnWidths(sheet);
|
||||||
sheet.autoSizeColumn(i);
|
|
||||||
}
|
}
|
||||||
}
|
workbook.write(fos);
|
||||||
wb.write(fos);
|
|
||||||
} catch (BusinessException ex) {
|
} catch (BusinessException ex) {
|
||||||
throw ex;
|
throw ex;
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
log.warn("[product-risk] write workbook failed: {}", ex.getMessage());
|
log.warn("[product-risk] write workbook failed: {}", ex.getMessage());
|
||||||
throw new BusinessException("生成 Excel 失败: " + ex.getMessage());
|
throw new BusinessException("generate product-risk excel failed: " + ex.getMessage());
|
||||||
|
} finally {
|
||||||
|
try {
|
||||||
|
workbook.close();
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
}
|
||||||
|
workbook.dispose();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -79,9 +85,6 @@ public class ProductRiskExcelAssemblyService {
|
|||||||
return n;
|
return n;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 将接口中的 {@link ProductRiskCountryCode} 键转为 Excel 中文工作表名 → 行列表。
|
|
||||||
*/
|
|
||||||
public Map<String, List<ProductRiskRowDto>> normalizeCountriesMap(Map<ProductRiskCountryCode, List<ProductRiskRowDto>> raw) {
|
public Map<String, List<ProductRiskRowDto>> normalizeCountriesMap(Map<ProductRiskCountryCode, List<ProductRiskRowDto>> raw) {
|
||||||
if (raw == null || raw.isEmpty()) {
|
if (raw == null || raw.isEmpty()) {
|
||||||
return new LinkedHashMap<>();
|
return new LinkedHashMap<>();
|
||||||
@@ -107,4 +110,11 @@ public class ProductRiskExcelAssemblyService {
|
|||||||
}
|
}
|
||||||
return fallback == null ? "" : fallback;
|
return fallback == null ? "" : fallback;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void applyDefaultColumnWidths(Sheet sheet) {
|
||||||
|
int[] widths = {20, 22, 18, 10, 18, 18};
|
||||||
|
for (int i = 0; i < widths.length; i++) {
|
||||||
|
sheet.setColumnWidth(i, widths[i] * 256);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ import java.util.Objects;
|
|||||||
public class ProductRiskResolveService {
|
public class ProductRiskResolveService {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 默认处理顺序:德国 → 英国 → 法国 → 意大利 → 西班牙(与前端一致)。
|
* 默认国家偏好顺序:德国 -> 英国 -> 法国 -> 意大利 -> 西班牙,与前端保持一致。
|
||||||
*/
|
*/
|
||||||
public static final List<String> DEFAULT_COUNTRY_PREFERENCE_ORDER = List.of("DE", "UK", "FR", "IT", "ES");
|
public static final List<String> DEFAULT_COUNTRY_PREFERENCE_ORDER = List.of("DE", "UK", "FR", "IT", "ES");
|
||||||
|
|
||||||
@@ -78,7 +78,7 @@ public class ProductRiskResolveService {
|
|||||||
if (indexHit == null || !indexHit.isMatched()) {
|
if (indexHit == null || !indexHit.isMatched()) {
|
||||||
String hint = indexHit != null && indexHit.getMatchMessage() != null && !indexHit.getMatchMessage().isBlank()
|
String hint = indexHit != null && indexHit.getMatchMessage() != null && !indexHit.getMatchMessage().isBlank()
|
||||||
? indexHit.getMatchMessage()
|
? indexHit.getMatchMessage()
|
||||||
: "店铺索引未命中,无法加入备选";
|
: "店铺索引未命中,无法加入待处理列表";
|
||||||
throw new BusinessException(hint);
|
throw new BusinessException(hint);
|
||||||
}
|
}
|
||||||
if (ZiniaoShopIndexService.MATCH_STATUS_CONFLICT.equals(indexHit.getMatchStatus())) {
|
if (ZiniaoShopIndexService.MATCH_STATUS_CONFLICT.equals(indexHit.getMatchStatus())) {
|
||||||
@@ -135,7 +135,7 @@ public class ProductRiskResolveService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (ordered.isEmpty()) {
|
if (ordered.isEmpty()) {
|
||||||
throw new BusinessException("shop_names 无有效店铺名");
|
throw new BusinessException("shop_names 没有有效店铺名");
|
||||||
}
|
}
|
||||||
ProductRiskMatchShopsVo vo = new ProductRiskMatchShopsVo();
|
ProductRiskMatchShopsVo vo = new ProductRiskMatchShopsVo();
|
||||||
for (String shopName : ordered) {
|
for (String shopName : ordered) {
|
||||||
@@ -227,7 +227,7 @@ public class ProductRiskResolveService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 仅保留合法枚举名,去重并保持顺序;非法项静默丢弃(用于读取库内数据)。
|
* 仅保留合法国家代码,去重并保持顺序;非法值静默忽略,用于读取库存量数据。
|
||||||
*/
|
*/
|
||||||
private static List<String> parseValidCountryCodes(List<String> raw) {
|
private static List<String> parseValidCountryCodes(List<String> raw) {
|
||||||
if (raw == null || raw.isEmpty()) {
|
if (raw == null || raw.isEmpty()) {
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
package com.nanri.aiimage.modules.productrisk.service;
|
package com.nanri.aiimage.modules.productrisk.service;
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.nanri.aiimage.common.exception.BusinessException;
|
|
||||||
import com.nanri.aiimage.config.TaskPressureProperties;
|
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||||
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskShopPayloadDto;
|
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskShopPayloadDto;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
@@ -20,80 +20,40 @@ import java.util.concurrent.ConcurrentHashMap;
|
|||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public class ProductRiskTaskCacheService {
|
public class ProductRiskTaskCacheService {
|
||||||
|
|
||||||
|
private static final String MODULE_TYPE = "PRODUCT_RISK_RESOLVE";
|
||||||
private static final long PAYLOAD_TTL_HOURS = 24;
|
private static final long PAYLOAD_TTL_HOURS = 24;
|
||||||
private final StringRedisTemplate stringRedisTemplate;
|
private final StringRedisTemplate stringRedisTemplate;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
private final TaskPressureProperties taskPressureProperties;
|
private final TaskPressureProperties taskPressureProperties;
|
||||||
|
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
|
||||||
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
|
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
|
||||||
|
|
||||||
public ProductRiskShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) {
|
public ProductRiskShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) {
|
||||||
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
|
return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, ProductRiskShopPayloadDto.class);
|
||||||
return null;
|
|
||||||
}
|
|
||||||
Object raw = stringRedisTemplate.opsForHash().get(buildShopPayloadKey(taskId), shopKey);
|
|
||||||
if (!(raw instanceof String json) || json.isBlank()) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return objectMapper.readValue(json, ProductRiskShopPayloadDto.class);
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("读取商品风险店铺缓存失败");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void saveShopMergedPayload(Long taskId, String shopKey, ProductRiskShopPayloadDto payload) {
|
public void saveShopMergedPayload(Long taskId, String shopKey, ProductRiskShopPayloadDto payload) {
|
||||||
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank() || payload == null) {
|
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank() || payload == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
try {
|
taskScopePayloadStorageService.saveScopePayload(taskId, MODULE_TYPE, shopKey, payload);
|
||||||
stringRedisTemplate.opsForHash().put(buildShopPayloadKey(taskId), shopKey, objectMapper.writeValueAsString(payload));
|
|
||||||
stringRedisTemplate.expire(buildShopPayloadKey(taskId), Duration.ofHours(PAYLOAD_TTL_HOURS));
|
|
||||||
touchTaskHeartbeat(taskId);
|
touchTaskHeartbeat(taskId);
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("暂存商品风险店铺缓存失败");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void removeShopMergedPayload(Long taskId, String shopKey) {
|
public void removeShopMergedPayload(Long taskId, String shopKey) {
|
||||||
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
|
taskScopePayloadStorageService.removeScopePayload(taskId, MODULE_TYPE, shopKey);
|
||||||
return;
|
|
||||||
}
|
|
||||||
stringRedisTemplate.opsForHash().delete(buildShopPayloadKey(taskId), shopKey);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public Map<String, ProductRiskShopPayloadDto> getAllShopMergedPayload(Long taskId) {
|
public Map<String, ProductRiskShopPayloadDto> getAllShopMergedPayload(Long taskId) {
|
||||||
try {
|
return taskScopePayloadStorageService.getAllScopePayload(taskId, MODULE_TYPE, ProductRiskShopPayloadDto.class);
|
||||||
Map<Object, Object> raw = stringRedisTemplate.opsForHash().entries(buildShopPayloadKey(taskId));
|
|
||||||
if (raw == null || raw.isEmpty()) {
|
|
||||||
return Map.of();
|
|
||||||
}
|
|
||||||
java.util.LinkedHashMap<String, ProductRiskShopPayloadDto> out = new java.util.LinkedHashMap<>();
|
|
||||||
for (Map.Entry<Object, Object> entry : raw.entrySet()) {
|
|
||||||
if (!(entry.getKey() instanceof String key) || !(entry.getValue() instanceof String val) || val.isBlank()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
out.put(key, objectMapper.readValue(val, ProductRiskShopPayloadDto.class));
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("读取商品风险缓存失败");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean hasAnyShopMergedPayload(Long taskId) {
|
public boolean hasAnyShopMergedPayload(Long taskId) {
|
||||||
if (taskId == null || taskId <= 0) {
|
return taskScopePayloadStorageService.hasAnyScopePayload(taskId, MODULE_TYPE);
|
||||||
return false;
|
|
||||||
}
|
|
||||||
Long size = stringRedisTemplate.opsForHash().size(buildShopPayloadKey(taskId));
|
|
||||||
return size != null && size > 0;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public long countShopMergedPayload(Long taskId) {
|
public long countShopMergedPayload(Long taskId) {
|
||||||
if (taskId == null || taskId <= 0) {
|
return taskScopePayloadStorageService.countScopePayload(taskId, MODULE_TYPE);
|
||||||
return 0L;
|
|
||||||
}
|
|
||||||
Long size = stringRedisTemplate.opsForHash().size(buildShopPayloadKey(taskId));
|
|
||||||
return size == null ? 0L : size;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public long getTaskHeartbeatMillis(Long taskId) {
|
public long getTaskHeartbeatMillis(Long taskId) {
|
||||||
@@ -116,9 +76,9 @@ public class ProductRiskTaskCacheService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
taskEntityLocalCache.remove(taskId);
|
taskEntityLocalCache.remove(taskId);
|
||||||
stringRedisTemplate.delete(buildShopPayloadKey(taskId));
|
|
||||||
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
|
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
|
||||||
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
|
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
|
||||||
|
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void saveTaskCache(FileTaskEntity task) {
|
public void saveTaskCache(FileTaskEntity task) {
|
||||||
@@ -183,10 +143,6 @@ public class ProductRiskTaskCacheService {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
private String buildShopPayloadKey(Long taskId) {
|
|
||||||
return "product-risk:task:shop-payload:" + taskId;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void touchTaskHeartbeat(Long taskId) {
|
public void touchTaskHeartbeat(Long taskId) {
|
||||||
stringRedisTemplate.opsForValue().set(
|
stringRedisTemplate.opsForValue().set(
|
||||||
buildTaskHeartbeatKey(taskId),
|
buildTaskHeartbeatKey(taskId),
|
||||||
|
|||||||
@@ -189,7 +189,7 @@ public class ProductRiskTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 删除整条商品风险任务及其下所有店铺结果(运行中或已结束均可,需归属当前用户)。
|
* 删除整条商品风险任务及其下所有店铺结果,运行中和已结束任务都允许删除,但必须归属当前用户。
|
||||||
*/
|
*/
|
||||||
@Transactional
|
@Transactional
|
||||||
public void deleteTask(Long taskId, Long userId) {
|
public void deleteTask(Long taskId, Long userId) {
|
||||||
@@ -208,8 +208,8 @@ public class ProductRiskTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 从用户当前「运行中」的商品风险任务里,按规范化店铺名删除一条结果(用于前端移除匹配列表后同步后端)。
|
* 从用户当前“运行中”的商品风险任务里,按规范化店铺名删除一条结果,用于前端移除匹配列表后同步后端。
|
||||||
* 若无对应记录则 removed=false。
|
* 如果没有对应记录,则返回 removed=false。
|
||||||
*/
|
*/
|
||||||
@Transactional
|
@Transactional
|
||||||
public ProductRiskPendingDeleteVo deletePendingShopResult(Long userId, String shopName) {
|
public ProductRiskPendingDeleteVo deletePendingShopResult(Long userId, String shopName) {
|
||||||
@@ -252,7 +252,7 @@ public class ProductRiskTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 删除一条 file_result 后重算父任务状态;若无剩余结果则删除任务。
|
* 删除一条 file_result 后重算父任务状态;如果没有剩余结果则删除任务。
|
||||||
*/
|
*/
|
||||||
private void reconcileTaskAfterResultRemoval(Long taskId) {
|
private void reconcileTaskAfterResultRemoval(Long taskId) {
|
||||||
FileTaskEntity task = loadTaskForExecution(taskId);
|
FileTaskEntity task = loadTaskForExecution(taskId);
|
||||||
@@ -303,11 +303,11 @@ public class ProductRiskTaskService {
|
|||||||
task.setFinishedAt(LocalDateTime.now());
|
task.setFinishedAt(LocalDateTime.now());
|
||||||
} else if (ok > 0) {
|
} else if (ok > 0) {
|
||||||
task.setStatus("SUCCESS");
|
task.setStatus("SUCCESS");
|
||||||
task.setErrorMessage(String.join(";", allErrors));
|
task.setErrorMessage(String.join("; ", allErrors));
|
||||||
task.setFinishedAt(LocalDateTime.now());
|
task.setFinishedAt(LocalDateTime.now());
|
||||||
} else {
|
} else {
|
||||||
task.setStatus("FAILED");
|
task.setStatus("FAILED");
|
||||||
task.setErrorMessage(allErrors.isEmpty() ? "全部店铺处理失败" : String.join(";", allErrors));
|
task.setErrorMessage(allErrors.isEmpty() ? "全部店铺处理失败" : String.join("; ", allErrors));
|
||||||
task.setFinishedAt(LocalDateTime.now());
|
task.setFinishedAt(LocalDateTime.now());
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
@@ -507,7 +507,7 @@ public class ProductRiskTaskService {
|
|||||||
String shopKey = fr.getSourceFilename();
|
String shopKey = fr.getSourceFilename();
|
||||||
ProductRiskShopPayloadDto payload = shopKey == null ? null : payloadByShop.get(shopKey);
|
ProductRiskShopPayloadDto payload = shopKey == null ? null : payloadByShop.get(shopKey);
|
||||||
|
|
||||||
// 兼容单店任务:若规范化店名偶发不一致(空格/符号差异)导致未命中,且任务与回传都只有 1 个店时按唯一店回填。
|
// 兼容单店任务:如果规范化店名偶发不一致导致未命中,且任务与回传都只有 1 个店时按唯一店铺回填。
|
||||||
if (payload == null && resultRows.size() == 1 && payloadByShop.size() == 1) {
|
if (payload == null && resultRows.size() == 1 && payloadByShop.size() == 1) {
|
||||||
payload = payloadByShop.values().iterator().next();
|
payload = payloadByShop.values().iterator().next();
|
||||||
fallbackMatchedCount++;
|
fallbackMatchedCount++;
|
||||||
@@ -517,7 +517,7 @@ public class ProductRiskTaskService {
|
|||||||
|
|
||||||
if (payload == null) {
|
if (payload == null) {
|
||||||
skippedUnmatchedCount++;
|
skippedUnmatchedCount++;
|
||||||
// 与删除品牌一致:支持队列逐条处理、分次回传,未在本次 payload 中的店铺保持待处理
|
// 与删除品牌一致:支持队列逐条处理、分次回传,本次 payload 未包含的店铺保持待处理。
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
matchedShopCount++;
|
matchedShopCount++;
|
||||||
@@ -602,11 +602,11 @@ public class ProductRiskTaskService {
|
|||||||
task.setFinishedAt(LocalDateTime.now());
|
task.setFinishedAt(LocalDateTime.now());
|
||||||
} else if (ok > 0) {
|
} else if (ok > 0) {
|
||||||
task.setStatus("SUCCESS");
|
task.setStatus("SUCCESS");
|
||||||
task.setErrorMessage(String.join(";", allErrors.isEmpty() ? batchErrors : allErrors));
|
task.setErrorMessage(String.join("; ", allErrors.isEmpty() ? batchErrors : allErrors));
|
||||||
task.setFinishedAt(LocalDateTime.now());
|
task.setFinishedAt(LocalDateTime.now());
|
||||||
} else {
|
} else {
|
||||||
task.setStatus("FAILED");
|
task.setStatus("FAILED");
|
||||||
task.setErrorMessage(allErrors.isEmpty() ? "全部店铺处理失败" : String.join(";", allErrors));
|
task.setErrorMessage(allErrors.isEmpty() ? "全部店铺处理失败" : String.join("; ", allErrors));
|
||||||
task.setFinishedAt(LocalDateTime.now());
|
task.setFinishedAt(LocalDateTime.now());
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
@@ -1034,7 +1034,7 @@ public class ProductRiskTaskService {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 兼容 Python 末次“空行 done=true”心跳:视为当前国家已有行全部完成。
|
// 兼容 Python 末次“空行 + done=true”心跳:视为当前国家已有行全部完成。
|
||||||
if (isDoneValue(row.getDone()) && isEmptyBusinessRow(row) && !byKey.isEmpty()) {
|
if (isDoneValue(row.getDone()) && isEmptyBusinessRow(row) && !byKey.isEmpty()) {
|
||||||
for (Map.Entry<String, ProductRiskRowDto> entry : byKey.entrySet()) {
|
for (Map.Entry<String, ProductRiskRowDto> entry : byKey.entrySet()) {
|
||||||
ProductRiskRowDto existed = entry.getValue();
|
ProductRiskRowDto existed = entry.getValue();
|
||||||
@@ -1103,8 +1103,8 @@ public class ProductRiskTaskService {
|
|||||||
if (row == null) {
|
if (row == null) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
// 注意:shopName 只是上下文信息,不作为业务行是否为空的判定条件。
|
// 注意,shopName 只是上下文信息,不作为业务行是否为空的判定条件。
|
||||||
// Python 末次心跳行通常只带 shopName + done=true,其他业务字段为空。
|
// Python 末次心跳行通常只带 shopName + done=true,其它业务字段为空。
|
||||||
return isBlank(row.getProductAsinSku())
|
return isBlank(row.getProductAsinSku())
|
||||||
&& isBlank(row.getRemoveAsin())
|
&& isBlank(row.getRemoveAsin())
|
||||||
&& isBlank(row.getStatus())
|
&& isBlank(row.getStatus())
|
||||||
|
|||||||
@@ -71,6 +71,7 @@ public class SkipPriceAsinController {
|
|||||||
id,
|
id,
|
||||||
country,
|
country,
|
||||||
request.getAsin(),
|
request.getAsin(),
|
||||||
|
request.getMinimumPrice(),
|
||||||
operatorId,
|
operatorId,
|
||||||
Boolean.TRUE.equals(superAdmin)));
|
Boolean.TRUE.equals(superAdmin)));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,10 +4,15 @@ import io.swagger.v3.oas.annotations.media.Schema;
|
|||||||
import jakarta.validation.constraints.NotBlank;
|
import jakarta.validation.constraints.NotBlank;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
|
||||||
@Data
|
@Data
|
||||||
public class SkipPriceAsinCountryUpdateRequest {
|
public class SkipPriceAsinCountryUpdateRequest {
|
||||||
|
|
||||||
@Schema(description = "ASIN", requiredMode = Schema.RequiredMode.REQUIRED)
|
@Schema(description = "ASIN", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||||
@NotBlank(message = "ASIN 不能为空")
|
@NotBlank(message = "ASIN 不能为空")
|
||||||
private String asin;
|
private String asin;
|
||||||
|
|
||||||
|
@Schema(description = "最低价", example = "18.50")
|
||||||
|
private BigDecimal minimumPrice;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import jakarta.validation.constraints.NotEmpty;
|
|||||||
import jakarta.validation.constraints.NotNull;
|
import jakarta.validation.constraints.NotNull;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
@@ -29,4 +30,8 @@ public class SkipPriceAsinCreateRequest {
|
|||||||
* 新请求: 每个国家对应独立 ASIN,例如 {"DE":"B0...", "UK":"B0..."}。
|
* 新请求: 每个国家对应独立 ASIN,例如 {"DE":"B0...", "UK":"B0..."}。
|
||||||
*/
|
*/
|
||||||
private Map<String, String> asinMappings;
|
private Map<String, String> asinMappings;
|
||||||
|
|
||||||
|
private BigDecimal minimumPrice;
|
||||||
|
|
||||||
|
private Map<String, BigDecimal> minimumPriceMappings;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import com.baomidou.mybatisplus.annotation.TableId;
|
|||||||
import com.baomidou.mybatisplus.annotation.TableName;
|
import com.baomidou.mybatisplus.annotation.TableName;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
import java.time.LocalDateTime;
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
@Data
|
@Data
|
||||||
@@ -24,18 +25,33 @@ public class SkipPriceAsinEntity {
|
|||||||
@TableField("asin_de")
|
@TableField("asin_de")
|
||||||
private String asinDe;
|
private String asinDe;
|
||||||
|
|
||||||
|
@TableField("minimum_price_de")
|
||||||
|
private BigDecimal minimumPriceDe;
|
||||||
|
|
||||||
@TableField("asin_uk")
|
@TableField("asin_uk")
|
||||||
private String asinUk;
|
private String asinUk;
|
||||||
|
|
||||||
|
@TableField("minimum_price_uk")
|
||||||
|
private BigDecimal minimumPriceUk;
|
||||||
|
|
||||||
@TableField("asin_fr")
|
@TableField("asin_fr")
|
||||||
private String asinFr;
|
private String asinFr;
|
||||||
|
|
||||||
|
@TableField("minimum_price_fr")
|
||||||
|
private BigDecimal minimumPriceFr;
|
||||||
|
|
||||||
@TableField("asin_it")
|
@TableField("asin_it")
|
||||||
private String asinIt;
|
private String asinIt;
|
||||||
|
|
||||||
|
@TableField("minimum_price_it")
|
||||||
|
private BigDecimal minimumPriceIt;
|
||||||
|
|
||||||
@TableField("asin_es")
|
@TableField("asin_es")
|
||||||
private String asinEs;
|
private String asinEs;
|
||||||
|
|
||||||
|
@TableField("minimum_price_es")
|
||||||
|
private BigDecimal minimumPriceEs;
|
||||||
|
|
||||||
@TableField("created_at")
|
@TableField("created_at")
|
||||||
private LocalDateTime createdAt;
|
private LocalDateTime createdAt;
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.shopkey.model.vo;
|
|||||||
|
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
import java.time.LocalDateTime;
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
@Data
|
@Data
|
||||||
@@ -12,10 +13,15 @@ public class SkipPriceAsinItemVo {
|
|||||||
private String groupName;
|
private String groupName;
|
||||||
private String shopName;
|
private String shopName;
|
||||||
private String asinDe;
|
private String asinDe;
|
||||||
|
private BigDecimal minimumPriceDe;
|
||||||
private String asinUk;
|
private String asinUk;
|
||||||
|
private BigDecimal minimumPriceUk;
|
||||||
private String asinFr;
|
private String asinFr;
|
||||||
|
private BigDecimal minimumPriceFr;
|
||||||
private String asinIt;
|
private String asinIt;
|
||||||
|
private BigDecimal minimumPriceIt;
|
||||||
private String asinEs;
|
private String asinEs;
|
||||||
|
private BigDecimal minimumPriceEs;
|
||||||
private LocalDateTime createdAt;
|
private LocalDateTime createdAt;
|
||||||
private LocalDateTime updatedAt;
|
private LocalDateTime updatedAt;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,6 +12,9 @@ import lombok.RequiredArgsConstructor;
|
|||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.transaction.annotation.Transactional;
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.math.RoundingMode;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.LinkedHashSet;
|
import java.util.LinkedHashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -79,6 +82,7 @@ public class SkipPriceAsinService {
|
|||||||
String shopName = normalizeRequired(request.getShopName(), "店铺名不能为空");
|
String shopName = normalizeRequired(request.getShopName(), "店铺名不能为空");
|
||||||
Set<String> countries = normalizeCountries(request.getCountries());
|
Set<String> countries = normalizeCountries(request.getCountries());
|
||||||
Map<String, String> countryAsinMap = normalizeCountryAsinMap(countries, request);
|
Map<String, String> countryAsinMap = normalizeCountryAsinMap(countries, request);
|
||||||
|
Map<String, BigDecimal> countryMinimumPriceMap = normalizeCountryMinimumPriceMap(countries, request);
|
||||||
|
|
||||||
SkipPriceAsinEntity entity = skipPriceAsinMapper.selectOne(new LambdaQueryWrapper<SkipPriceAsinEntity>()
|
SkipPriceAsinEntity entity = skipPriceAsinMapper.selectOne(new LambdaQueryWrapper<SkipPriceAsinEntity>()
|
||||||
.eq(SkipPriceAsinEntity::getGroupId, group.getId())
|
.eq(SkipPriceAsinEntity::getGroupId, group.getId())
|
||||||
@@ -91,7 +95,8 @@ public class SkipPriceAsinService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for (Map.Entry<String, String> entry : countryAsinMap.entrySet()) {
|
for (Map.Entry<String, String> entry : countryAsinMap.entrySet()) {
|
||||||
setCountryAsin(entity, entry.getKey(), entry.getValue());
|
String country = entry.getKey();
|
||||||
|
setCountryData(entity, country, entry.getValue(), countryMinimumPriceMap.get(country));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (entity.getId() == null) {
|
if (entity.getId() == null) {
|
||||||
@@ -108,7 +113,7 @@ public class SkipPriceAsinService {
|
|||||||
SkipPriceAsinEntity entity = getById(id);
|
SkipPriceAsinEntity entity = getById(id);
|
||||||
validateGroupAccess(entity, operatorId, superAdmin);
|
validateGroupAccess(entity, operatorId, superAdmin);
|
||||||
String normalizedCountry = normalizeCountry(country);
|
String normalizedCountry = normalizeCountry(country);
|
||||||
setCountryAsin(entity, normalizedCountry, "");
|
setCountryData(entity, normalizedCountry, "", null);
|
||||||
if (isEmptyRow(entity)) {
|
if (isEmptyRow(entity)) {
|
||||||
skipPriceAsinMapper.deleteById(entity.getId());
|
skipPriceAsinMapper.deleteById(entity.getId());
|
||||||
return;
|
return;
|
||||||
@@ -117,12 +122,13 @@ public class SkipPriceAsinService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Transactional
|
@Transactional
|
||||||
public SkipPriceAsinItemVo updateCountry(Long id, String country, String asin, Long operatorId, boolean superAdmin) {
|
public SkipPriceAsinItemVo updateCountry(Long id, String country, String asin, BigDecimal minimumPrice,
|
||||||
|
Long operatorId, boolean superAdmin) {
|
||||||
SkipPriceAsinEntity entity = getById(id);
|
SkipPriceAsinEntity entity = getById(id);
|
||||||
validateGroupAccess(entity, operatorId, superAdmin);
|
validateGroupAccess(entity, operatorId, superAdmin);
|
||||||
String normalizedCountry = normalizeCountry(country);
|
String normalizedCountry = normalizeCountry(country);
|
||||||
String normalizedAsin = normalizeAsin(asin);
|
String normalizedAsin = normalizeAsin(asin);
|
||||||
setCountryAsin(entity, normalizedCountry, normalizedAsin);
|
setCountryData(entity, normalizedCountry, normalizedAsin, normalizeMinimumPrice(minimumPrice));
|
||||||
skipPriceAsinMapper.updateById(entity);
|
skipPriceAsinMapper.updateById(entity);
|
||||||
SkipPriceAsinEntity saved = getById(entity.getId());
|
SkipPriceAsinEntity saved = getById(entity.getId());
|
||||||
String groupName = "";
|
String groupName = "";
|
||||||
@@ -154,10 +160,15 @@ public class SkipPriceAsinService {
|
|||||||
vo.setGroupName(groupName == null ? "" : groupName);
|
vo.setGroupName(groupName == null ? "" : groupName);
|
||||||
vo.setShopName(entity.getShopName());
|
vo.setShopName(entity.getShopName());
|
||||||
vo.setAsinDe(blankToEmpty(entity.getAsinDe()));
|
vo.setAsinDe(blankToEmpty(entity.getAsinDe()));
|
||||||
|
vo.setMinimumPriceDe(entity.getMinimumPriceDe());
|
||||||
vo.setAsinUk(blankToEmpty(entity.getAsinUk()));
|
vo.setAsinUk(blankToEmpty(entity.getAsinUk()));
|
||||||
|
vo.setMinimumPriceUk(entity.getMinimumPriceUk());
|
||||||
vo.setAsinFr(blankToEmpty(entity.getAsinFr()));
|
vo.setAsinFr(blankToEmpty(entity.getAsinFr()));
|
||||||
|
vo.setMinimumPriceFr(entity.getMinimumPriceFr());
|
||||||
vo.setAsinIt(blankToEmpty(entity.getAsinIt()));
|
vo.setAsinIt(blankToEmpty(entity.getAsinIt()));
|
||||||
|
vo.setMinimumPriceIt(entity.getMinimumPriceIt());
|
||||||
vo.setAsinEs(blankToEmpty(entity.getAsinEs()));
|
vo.setAsinEs(blankToEmpty(entity.getAsinEs()));
|
||||||
|
vo.setMinimumPriceEs(entity.getMinimumPriceEs());
|
||||||
vo.setCreatedAt(entity.getCreatedAt());
|
vo.setCreatedAt(entity.getCreatedAt());
|
||||||
vo.setUpdatedAt(entity.getUpdatedAt());
|
vo.setUpdatedAt(entity.getUpdatedAt());
|
||||||
return vo;
|
return vo;
|
||||||
@@ -213,6 +224,27 @@ public class SkipPriceAsinService {
|
|||||||
return normalized;
|
return normalized;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private Map<String, BigDecimal> normalizeCountryMinimumPriceMap(Set<String> countries, SkipPriceAsinCreateRequest request) {
|
||||||
|
LinkedHashMap<String, BigDecimal> normalized = new LinkedHashMap<>();
|
||||||
|
Map<String, BigDecimal> requestMappings = request.getMinimumPriceMappings();
|
||||||
|
if (requestMappings != null && !requestMappings.isEmpty()) {
|
||||||
|
for (String country : countries) {
|
||||||
|
BigDecimal minimumPrice = requestMappings.get(country);
|
||||||
|
if (minimumPrice == null) {
|
||||||
|
minimumPrice = requestMappings.get(country.toLowerCase(Locale.ROOT));
|
||||||
|
}
|
||||||
|
normalized.put(country, normalizeMinimumPrice(minimumPrice));
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
|
||||||
|
BigDecimal minimumPrice = normalizeMinimumPrice(request.getMinimumPrice());
|
||||||
|
for (String country : countries) {
|
||||||
|
normalized.put(country, minimumPrice);
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
|
||||||
private String normalizeCountry(String country) {
|
private String normalizeCountry(String country) {
|
||||||
String normalized = normalizeBlank(country).toUpperCase(Locale.ROOT);
|
String normalized = normalizeBlank(country).toUpperCase(Locale.ROOT);
|
||||||
if (!SUPPORTED_COUNTRIES.contains(normalized)) {
|
if (!SUPPORTED_COUNTRIES.contains(normalized)) {
|
||||||
@@ -229,6 +261,16 @@ public class SkipPriceAsinService {
|
|||||||
return normalized;
|
return normalized;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private BigDecimal normalizeMinimumPrice(BigDecimal minimumPrice) {
|
||||||
|
if (minimumPrice == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (minimumPrice.compareTo(BigDecimal.ZERO) < 0) {
|
||||||
|
throw new BusinessException("最低价不能小于 0");
|
||||||
|
}
|
||||||
|
return minimumPrice.setScale(2, RoundingMode.HALF_UP);
|
||||||
|
}
|
||||||
|
|
||||||
private String normalizeRequired(String value, String message) {
|
private String normalizeRequired(String value, String message) {
|
||||||
String normalized = normalizeBlank(value);
|
String normalized = normalizeBlank(value);
|
||||||
if (normalized.isEmpty()) {
|
if (normalized.isEmpty()) {
|
||||||
@@ -245,14 +287,29 @@ public class SkipPriceAsinService {
|
|||||||
return value == null ? "" : value;
|
return value == null ? "" : value;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void setCountryAsin(SkipPriceAsinEntity entity, String country, String asin) {
|
private void setCountryData(SkipPriceAsinEntity entity, String country, String asin, BigDecimal minimumPrice) {
|
||||||
String value = normalizeBlank(asin);
|
String value = normalizeBlank(asin);
|
||||||
switch (country) {
|
switch (country) {
|
||||||
case "DE" -> entity.setAsinDe(value);
|
case "DE" -> {
|
||||||
case "UK" -> entity.setAsinUk(value);
|
entity.setAsinDe(value);
|
||||||
case "FR" -> entity.setAsinFr(value);
|
entity.setMinimumPriceDe(minimumPrice);
|
||||||
case "IT" -> entity.setAsinIt(value);
|
}
|
||||||
case "ES" -> entity.setAsinEs(value);
|
case "UK" -> {
|
||||||
|
entity.setAsinUk(value);
|
||||||
|
entity.setMinimumPriceUk(minimumPrice);
|
||||||
|
}
|
||||||
|
case "FR" -> {
|
||||||
|
entity.setAsinFr(value);
|
||||||
|
entity.setMinimumPriceFr(minimumPrice);
|
||||||
|
}
|
||||||
|
case "IT" -> {
|
||||||
|
entity.setAsinIt(value);
|
||||||
|
entity.setMinimumPriceIt(minimumPrice);
|
||||||
|
}
|
||||||
|
case "ES" -> {
|
||||||
|
entity.setAsinEs(value);
|
||||||
|
entity.setMinimumPriceEs(minimumPrice);
|
||||||
|
}
|
||||||
default -> throw new BusinessException("国家参数不支持: " + country);
|
default -> throw new BusinessException("国家参数不支持: " + country);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -262,7 +319,12 @@ public class SkipPriceAsinService {
|
|||||||
&& normalizeBlank(entity.getAsinUk()).isEmpty()
|
&& normalizeBlank(entity.getAsinUk()).isEmpty()
|
||||||
&& normalizeBlank(entity.getAsinFr()).isEmpty()
|
&& normalizeBlank(entity.getAsinFr()).isEmpty()
|
||||||
&& normalizeBlank(entity.getAsinIt()).isEmpty()
|
&& normalizeBlank(entity.getAsinIt()).isEmpty()
|
||||||
&& normalizeBlank(entity.getAsinEs()).isEmpty();
|
&& normalizeBlank(entity.getAsinEs()).isEmpty()
|
||||||
|
&& entity.getMinimumPriceDe() == null
|
||||||
|
&& entity.getMinimumPriceUk() == null
|
||||||
|
&& entity.getMinimumPriceFr() == null
|
||||||
|
&& entity.getMinimumPriceIt() == null
|
||||||
|
&& entity.getMinimumPriceEs() == null;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Map<String, Map<String, String>> listSkipAsinsByShops(List<String> shopNames) {
|
public Map<String, Map<String, String>> listSkipAsinsByShops(List<String> shopNames) {
|
||||||
@@ -312,10 +374,78 @@ public class SkipPriceAsinService {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Map<String, List<Map<String, String>>> listAllSkipAsinDetailsByCountry() {
|
||||||
|
List<SkipPriceAsinEntity> rows = skipPriceAsinMapper.selectList(
|
||||||
|
new LambdaQueryWrapper<SkipPriceAsinEntity>()
|
||||||
|
.orderByDesc(SkipPriceAsinEntity::getId));
|
||||||
|
Map<String, List<Map<String, String>>> grouped = new LinkedHashMap<>();
|
||||||
|
grouped.put("DE", new java.util.ArrayList<>());
|
||||||
|
grouped.put("UK", new java.util.ArrayList<>());
|
||||||
|
grouped.put("FR", new java.util.ArrayList<>());
|
||||||
|
grouped.put("IT", new java.util.ArrayList<>());
|
||||||
|
grouped.put("ES", new java.util.ArrayList<>());
|
||||||
|
|
||||||
|
for (SkipPriceAsinEntity row : rows) {
|
||||||
|
addCountryAsinDetail(grouped.get("DE"), row.getAsinDe(), row.getMinimumPriceDe());
|
||||||
|
addCountryAsinDetail(grouped.get("UK"), row.getAsinUk(), row.getMinimumPriceUk());
|
||||||
|
addCountryAsinDetail(grouped.get("FR"), row.getAsinFr(), row.getMinimumPriceFr());
|
||||||
|
addCountryAsinDetail(grouped.get("IT"), row.getAsinIt(), row.getMinimumPriceIt());
|
||||||
|
addCountryAsinDetail(grouped.get("ES"), row.getAsinEs(), row.getMinimumPriceEs());
|
||||||
|
}
|
||||||
|
return grouped;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public boolean removeByShopCountryAndAsin(String shopName, String country, String asin) {
|
||||||
|
String normalizedShopName = normalizeBlank(shopName);
|
||||||
|
String normalizedCountry = normalizeCountry(country);
|
||||||
|
String normalizedAsin = normalizeAsin(asin);
|
||||||
|
if (normalizedShopName.isEmpty() || normalizedAsin.isEmpty()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
List<SkipPriceAsinEntity> rows = skipPriceAsinMapper.selectList(
|
||||||
|
new LambdaQueryWrapper<SkipPriceAsinEntity>()
|
||||||
|
.eq(SkipPriceAsinEntity::getShopName, normalizedShopName));
|
||||||
|
boolean changed = false;
|
||||||
|
for (SkipPriceAsinEntity row : rows) {
|
||||||
|
String existingAsin = switch (normalizedCountry) {
|
||||||
|
case "DE" -> normalizeBlank(row.getAsinDe());
|
||||||
|
case "UK" -> normalizeBlank(row.getAsinUk());
|
||||||
|
case "FR" -> normalizeBlank(row.getAsinFr());
|
||||||
|
case "IT" -> normalizeBlank(row.getAsinIt());
|
||||||
|
case "ES" -> normalizeBlank(row.getAsinEs());
|
||||||
|
default -> "";
|
||||||
|
};
|
||||||
|
if (!normalizedAsin.equals(existingAsin)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
setCountryData(row, normalizedCountry, "", null);
|
||||||
|
row.setUpdatedAt(LocalDateTime.now());
|
||||||
|
if (isEmptyRow(row)) {
|
||||||
|
skipPriceAsinMapper.deleteById(row.getId());
|
||||||
|
} else {
|
||||||
|
skipPriceAsinMapper.updateById(row);
|
||||||
|
}
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
return changed;
|
||||||
|
}
|
||||||
|
|
||||||
private void addCountryAsin(Set<String> bucket, String asin) {
|
private void addCountryAsin(Set<String> bucket, String asin) {
|
||||||
String normalized = normalizeBlank(asin);
|
String normalized = normalizeBlank(asin);
|
||||||
if (!normalized.isEmpty()) {
|
if (!normalized.isEmpty()) {
|
||||||
bucket.add(normalized);
|
bucket.add(normalized);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void addCountryAsinDetail(List<Map<String, String>> bucket, String asin, BigDecimal minimumPrice) {
|
||||||
|
String normalizedAsin = normalizeBlank(asin);
|
||||||
|
if (normalizedAsin.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Map<String, String> detail = new LinkedHashMap<>();
|
||||||
|
detail.put("asin", normalizedAsin);
|
||||||
|
detail.put("minimumPrice", minimumPrice == null ? "" : minimumPrice.toPlainString());
|
||||||
|
bucket.add(detail);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import com.nanri.aiimage.modules.shopmatch.model.dto.ShopMatchRowDto;
|
|||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.poi.ss.usermodel.Row;
|
import org.apache.poi.ss.usermodel.Row;
|
||||||
import org.apache.poi.ss.usermodel.Sheet;
|
import org.apache.poi.ss.usermodel.Sheet;
|
||||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
@@ -23,10 +23,12 @@ public class ShopMatchExcelAssemblyService {
|
|||||||
|
|
||||||
public void writeWorkbook(File outputXlsx, Map<String, List<ShopMatchRowDto>> countries) {
|
public void writeWorkbook(File outputXlsx, Map<String, List<ShopMatchRowDto>> countries) {
|
||||||
Map<String, List<ShopMatchRowDto>> safe = countries == null ? Map.of() : countries;
|
Map<String, List<ShopMatchRowDto>> safe = countries == null ? Map.of() : countries;
|
||||||
try (XSSFWorkbook wb = new XSSFWorkbook(); FileOutputStream fos = new FileOutputStream(outputXlsx)) {
|
SXSSFWorkbook workbook = new SXSSFWorkbook(200);
|
||||||
|
workbook.setCompressTempFiles(true);
|
||||||
|
try (FileOutputStream fos = new FileOutputStream(outputXlsx)) {
|
||||||
for (ProductRiskCountryCode code : ProductRiskCountryCode.values()) {
|
for (ProductRiskCountryCode code : ProductRiskCountryCode.values()) {
|
||||||
String sheetName = code.getSheetName();
|
String sheetName = code.getSheetName();
|
||||||
Sheet sheet = wb.createSheet(sheetName);
|
Sheet sheet = workbook.createSheet(sheetName);
|
||||||
Row header = sheet.createRow(0);
|
Row header = sheet.createRow(0);
|
||||||
header.createCell(0).setCellValue(HEADER[0]);
|
header.createCell(0).setCellValue(HEADER[0]);
|
||||||
header.createCell(1).setCellValue(HEADER[1]);
|
header.createCell(1).setCellValue(HEADER[1]);
|
||||||
@@ -40,13 +42,19 @@ public class ShopMatchExcelAssemblyService {
|
|||||||
row.createCell(0).setCellValue(item.getAsin() == null ? "" : item.getAsin());
|
row.createCell(0).setCellValue(item.getAsin() == null ? "" : item.getAsin());
|
||||||
row.createCell(1).setCellValue(item.getStatus() == null ? "" : item.getStatus());
|
row.createCell(1).setCellValue(item.getStatus() == null ? "" : item.getStatus());
|
||||||
}
|
}
|
||||||
sheet.autoSizeColumn(0);
|
sheet.setColumnWidth(0, 20 * 256);
|
||||||
sheet.autoSizeColumn(1);
|
sheet.setColumnWidth(1, 18 * 256);
|
||||||
}
|
}
|
||||||
wb.write(fos);
|
workbook.write(fos);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
log.warn("[shop-match] write workbook failed: {}", ex.getMessage());
|
log.warn("[shop-match] write workbook failed: {}", ex.getMessage());
|
||||||
throw new BusinessException("生成 Excel 失败: " + ex.getMessage());
|
throw new BusinessException("generate shop-match excel failed: " + ex.getMessage());
|
||||||
|
} finally {
|
||||||
|
try {
|
||||||
|
workbook.close();
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
}
|
||||||
|
workbook.dispose();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,24 +1,20 @@
|
|||||||
package com.nanri.aiimage.modules.shopmatch.service;
|
package com.nanri.aiimage.modules.shopmatch.service;
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.nanri.aiimage.common.exception.BusinessException;
|
|
||||||
import com.nanri.aiimage.config.TaskPressureProperties;
|
|
||||||
import com.nanri.aiimage.modules.shopmatch.model.dto.ShopMatchShopPayloadDto;
|
import com.nanri.aiimage.modules.shopmatch.model.dto.ShopMatchShopPayloadDto;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
|
||||||
|
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.nio.charset.StandardCharsets;
|
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import java.nio.file.StandardOpenOption;
|
import java.nio.file.StandardOpenOption;
|
||||||
import java.security.MessageDigest;
|
|
||||||
import java.time.Duration;
|
import java.time.Duration;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Comparator;
|
|
||||||
import java.util.HexFormat;
|
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
@@ -28,104 +24,34 @@ import java.util.concurrent.ConcurrentHashMap;
|
|||||||
@Slf4j
|
@Slf4j
|
||||||
public class ShopMatchTaskCacheService {
|
public class ShopMatchTaskCacheService {
|
||||||
|
|
||||||
|
private static final String MODULE_TYPE = "SHOP_MATCH";
|
||||||
private static final long PAYLOAD_TTL_HOURS = 24;
|
private static final long PAYLOAD_TTL_HOURS = 24;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
private final TaskPressureProperties taskPressureProperties;
|
private final TaskPressureProperties taskPressureProperties;
|
||||||
|
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
|
||||||
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
|
private final ConcurrentHashMap<Long, LocalTaskEntityCacheEntry> taskEntityLocalCache = new ConcurrentHashMap<>();
|
||||||
|
|
||||||
public ShopMatchShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) {
|
public ShopMatchShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) {
|
||||||
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
|
return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, ShopMatchShopPayloadDto.class);
|
||||||
return null;
|
|
||||||
}
|
|
||||||
Path file = buildShopPayloadFile(taskId, shopKey);
|
|
||||||
if (!Files.isRegularFile(file)) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
PayloadFile payloadFile = objectMapper.readValue(Files.readString(file), PayloadFile.class);
|
|
||||||
if (payloadFile == null || payloadFile.payload == null) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
if (payloadFile.shopKey == null || !shopKey.equals(payloadFile.shopKey)) {
|
|
||||||
log.warn("[shop-match] payload file shop key mismatch taskId={} shopKey={} storedKey={}",
|
|
||||||
taskId, shopKey, payloadFile.shopKey);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return payloadFile.payload;
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("璇诲彇鍖归厤搴楅摵缂撳瓨澶辫触");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void saveShopMergedPayload(Long taskId, String shopKey, ShopMatchShopPayloadDto payload) {
|
public void saveShopMergedPayload(Long taskId, String shopKey, ShopMatchShopPayloadDto payload) {
|
||||||
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank() || payload == null) {
|
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank() || payload == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
try {
|
taskScopePayloadStorageService.saveScopePayload(taskId, MODULE_TYPE, shopKey, payload);
|
||||||
Files.createDirectories(buildTaskDir(taskId));
|
|
||||||
Files.writeString(
|
|
||||||
buildShopPayloadFile(taskId, shopKey),
|
|
||||||
objectMapper.writeValueAsString(new PayloadFile(shopKey, payload)),
|
|
||||||
StandardOpenOption.CREATE,
|
|
||||||
StandardOpenOption.TRUNCATE_EXISTING,
|
|
||||||
StandardOpenOption.WRITE
|
|
||||||
);
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("鏆傚瓨鍖归厤搴楅摵缁撴灉澶辫触");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void removeShopMergedPayload(Long taskId, String shopKey) {
|
public void removeShopMergedPayload(Long taskId, String shopKey) {
|
||||||
if (taskId == null || taskId <= 0 || shopKey == null || shopKey.isBlank()) {
|
taskScopePayloadStorageService.removeScopePayload(taskId, MODULE_TYPE, shopKey);
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
Files.deleteIfExists(buildShopPayloadFile(taskId, shopKey));
|
|
||||||
} catch (IOException ex) {
|
|
||||||
log.warn("[shop-match] delete shop payload failed taskId={} shopKey={} msg={}", taskId, shopKey, ex.getMessage());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public Map<String, ShopMatchShopPayloadDto> getAllShopMergedPayload(Long taskId) {
|
public Map<String, ShopMatchShopPayloadDto> getAllShopMergedPayload(Long taskId) {
|
||||||
try {
|
return taskScopePayloadStorageService.getAllScopePayload(taskId, MODULE_TYPE, ShopMatchShopPayloadDto.class);
|
||||||
Path taskDir = buildTaskDir(taskId);
|
|
||||||
if (!Files.isDirectory(taskDir)) {
|
|
||||||
return Map.of();
|
|
||||||
}
|
|
||||||
LinkedHashMap<String, ShopMatchShopPayloadDto> out = new LinkedHashMap<>();
|
|
||||||
try (var stream = Files.list(taskDir)) {
|
|
||||||
for (Path path : stream
|
|
||||||
.filter(file -> file.getFileName().toString().endsWith(".json"))
|
|
||||||
.sorted(Comparator.comparing(file -> file.getFileName().toString()))
|
|
||||||
.toList()) {
|
|
||||||
PayloadFile payloadFile = objectMapper.readValue(Files.readString(path), PayloadFile.class);
|
|
||||||
if (payloadFile == null || payloadFile.shopKey == null || payloadFile.shopKey.isBlank() || payloadFile.payload == null) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
out.put(payloadFile.shopKey, payloadFile.payload);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("璇诲彇鍖归厤搴楅摵缂撳瓨澶辫触");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean hasAnyShopMergedPayload(Long taskId) {
|
public boolean hasAnyShopMergedPayload(Long taskId) {
|
||||||
if (taskId == null || taskId <= 0) {
|
return taskScopePayloadStorageService.hasAnyScopePayload(taskId, MODULE_TYPE);
|
||||||
return false;
|
|
||||||
}
|
|
||||||
Path taskDir = buildTaskDir(taskId);
|
|
||||||
if (!Files.isDirectory(taskDir)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
try (var stream = Files.list(taskDir)) {
|
|
||||||
return stream.anyMatch(file -> Files.isRegularFile(file)
|
|
||||||
&& file.getFileName().toString().endsWith(".json"));
|
|
||||||
} catch (IOException ex) {
|
|
||||||
log.warn("[shop-match] scan task cache failed taskId={} msg={}", taskId, ex.getMessage());
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void deleteTaskCache(Long taskId) {
|
public void deleteTaskCache(Long taskId) {
|
||||||
@@ -133,6 +59,7 @@ public class ShopMatchTaskCacheService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
taskEntityLocalCache.remove(taskId);
|
taskEntityLocalCache.remove(taskId);
|
||||||
|
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
|
||||||
try {
|
try {
|
||||||
Path taskDir = buildTaskDir(taskId);
|
Path taskDir = buildTaskDir(taskId);
|
||||||
if (!Files.exists(taskDir)) {
|
if (!Files.exists(taskDir)) {
|
||||||
@@ -140,7 +67,7 @@ public class ShopMatchTaskCacheService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
try (var walk = Files.walk(taskDir)) {
|
try (var walk = Files.walk(taskDir)) {
|
||||||
walk.sorted(Comparator.reverseOrder()).forEach(path -> {
|
walk.sorted(java.util.Comparator.reverseOrder()).forEach(path -> {
|
||||||
try {
|
try {
|
||||||
Files.deleteIfExists(path);
|
Files.deleteIfExists(path);
|
||||||
} catch (IOException ex) {
|
} catch (IOException ex) {
|
||||||
@@ -225,37 +152,10 @@ public class ShopMatchTaskCacheService {
|
|||||||
return Path.of(System.getProperty("java.io.tmpdir"), "shop-match-cache", String.valueOf(taskId));
|
return Path.of(System.getProperty("java.io.tmpdir"), "shop-match-cache", String.valueOf(taskId));
|
||||||
}
|
}
|
||||||
|
|
||||||
private Path buildShopPayloadFile(Long taskId, String shopKey) {
|
|
||||||
return buildTaskDir(taskId).resolve(hashShopKey(shopKey) + ".json");
|
|
||||||
}
|
|
||||||
|
|
||||||
private Path buildTaskEntityFile(Long taskId) {
|
private Path buildTaskEntityFile(Long taskId) {
|
||||||
return buildTaskDir(taskId).resolve("_task-entity.json");
|
return buildTaskDir(taskId).resolve("_task-entity.json");
|
||||||
}
|
}
|
||||||
|
|
||||||
private String hashShopKey(String shopKey) {
|
|
||||||
try {
|
|
||||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
|
||||||
byte[] bytes = digest.digest(shopKey.trim().getBytes(StandardCharsets.UTF_8));
|
|
||||||
return HexFormat.of().formatHex(bytes);
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new IllegalStateException("failed to hash shop key", ex);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static class PayloadFile {
|
|
||||||
public String shopKey;
|
|
||||||
public ShopMatchShopPayloadDto payload;
|
|
||||||
|
|
||||||
public PayloadFile() {
|
|
||||||
}
|
|
||||||
|
|
||||||
public PayloadFile(String shopKey, ShopMatchShopPayloadDto payload) {
|
|
||||||
this.shopKey = shopKey;
|
|
||||||
this.payload = payload;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean isLocalCacheFresh(LocalTaskEntityCacheEntry cached, long now) {
|
private boolean isLocalCacheFresh(LocalTaskEntityCacheEntry cached, long now) {
|
||||||
return cached != null
|
return cached != null
|
||||||
&& now - cached.cachedAtMillis() <= Math.max(0L, taskPressureProperties.getLocalTaskEntityCacheMillis());
|
&& now - cached.cachedAtMillis() <= Math.max(0L, taskPressureProperties.getLocalTaskEntityCacheMillis());
|
||||||
|
|||||||
@@ -44,8 +44,8 @@ public class SplitRunService {
|
|||||||
private static final String ZIP_CONTENT_TYPE = "application/zip";
|
private static final String ZIP_CONTENT_TYPE = "application/zip";
|
||||||
private static final String SPLIT_MODE_ROWS_PER_FILE = "rows_per_file";
|
private static final String SPLIT_MODE_ROWS_PER_FILE = "rows_per_file";
|
||||||
private static final String SPLIT_MODE_PARTS = "parts";
|
private static final String SPLIT_MODE_PARTS = "parts";
|
||||||
private static final String HEADER_SUFFIX_MARKER = "idASIN\u56fd\u5bb6\u72b6\u6001\u4ef7\u683c\u53d8\u4f53\u6570\u91cf";
|
private static final String HEADER_SUFFIX_MARKER = "idASIN国家状态价格变体数量";
|
||||||
private static final String HEADER_STOP_COLUMN = "\u7f29\u7565\u56fe\u5730\u57408";
|
private static final String HEADER_STOP_COLUMN = "缩略图地址8";
|
||||||
private static final DateTimeFormatter ZIP_NAME_DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd");
|
private static final DateTimeFormatter ZIP_NAME_DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd");
|
||||||
|
|
||||||
private final FileTaskMapper fileTaskMapper;
|
private final FileTaskMapper fileTaskMapper;
|
||||||
|
|||||||
@@ -0,0 +1,9 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.mapper;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||||
|
import org.apache.ibatis.annotations.Mapper;
|
||||||
|
|
||||||
|
@Mapper
|
||||||
|
public interface TaskChunkMapper extends BaseMapper<TaskChunkEntity> {
|
||||||
|
}
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.mapper;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||||
|
import org.apache.ibatis.annotations.Mapper;
|
||||||
|
|
||||||
|
@Mapper
|
||||||
|
public interface TaskScopeStateMapper extends BaseMapper<TaskScopeStateEntity> {
|
||||||
|
}
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.model.entity;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.annotation.IdType;
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableId;
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableName;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@TableName("biz_task_chunk")
|
||||||
|
public class TaskChunkEntity {
|
||||||
|
|
||||||
|
@TableId(type = IdType.AUTO)
|
||||||
|
private Long id;
|
||||||
|
private Long taskId;
|
||||||
|
private String moduleType;
|
||||||
|
private String scopeKey;
|
||||||
|
private String scopeHash;
|
||||||
|
private Integer chunkIndex;
|
||||||
|
private Integer chunkTotal;
|
||||||
|
private String payloadJson;
|
||||||
|
private String payloadHash;
|
||||||
|
private LocalDateTime createdAt;
|
||||||
|
private LocalDateTime updatedAt;
|
||||||
|
}
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.model.entity;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.annotation.IdType;
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableId;
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableName;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@TableName("biz_task_scope_state")
|
||||||
|
public class TaskScopeStateEntity {
|
||||||
|
|
||||||
|
@TableId(type = IdType.AUTO)
|
||||||
|
private Long id;
|
||||||
|
private Long taskId;
|
||||||
|
private String moduleType;
|
||||||
|
private String scopeKey;
|
||||||
|
private String scopeHash;
|
||||||
|
private String parsedPayloadJson;
|
||||||
|
private String stateJson;
|
||||||
|
private Integer chunkTotal;
|
||||||
|
private Integer receivedChunkCount;
|
||||||
|
private Integer completed;
|
||||||
|
private LocalDateTime lastChunkAt;
|
||||||
|
private String lastError;
|
||||||
|
private LocalDateTime createdAt;
|
||||||
|
private LocalDateTime updatedAt;
|
||||||
|
}
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.service;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||||
|
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||||
|
import org.springframework.context.event.EventListener;
|
||||||
|
import org.springframework.scheduling.annotation.Scheduled;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.nio.file.attribute.FileTime;
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.Instant;
|
||||||
|
import java.util.Comparator;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
public class TaskScopePayloadBufferMaintenanceService {
|
||||||
|
|
||||||
|
private static final Duration RECOVERY_LOCK_TTL = Duration.ofMinutes(10);
|
||||||
|
private static final Duration CLEANUP_LOCK_TTL = Duration.ofMinutes(10);
|
||||||
|
|
||||||
|
private final TaskScopePayloadStorageService taskScopePayloadStorageService;
|
||||||
|
private final TaskPressureProperties taskPressureProperties;
|
||||||
|
private final DistributedJobLockService distributedJobLockService;
|
||||||
|
|
||||||
|
@EventListener(ApplicationReadyEvent.class)
|
||||||
|
public void recoverBufferedPayloadsOnStartup() {
|
||||||
|
DistributedJobLockService.LockHandle lockHandle =
|
||||||
|
distributedJobLockService.tryLock("task-scope-buffer:recover", RECOVERY_LOCK_TTL);
|
||||||
|
if (lockHandle == null) {
|
||||||
|
log.info("[task-scope-buffer] skip startup recovery because another instance holds the lock");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try (lockHandle) {
|
||||||
|
Path root = taskScopePayloadStorageService.getBufferedPayloadRoot();
|
||||||
|
if (!Files.isDirectory(root)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
int limit = Math.max(0, taskPressureProperties.getScopePayloadRecoveryMaxFiles());
|
||||||
|
if (limit == 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
int recovered = 0;
|
||||||
|
int scanned = 0;
|
||||||
|
for (Path file : listBufferedPayloadFiles(root, limit)) {
|
||||||
|
scanned++;
|
||||||
|
try {
|
||||||
|
BufferFileRef ref = parseBufferFile(root, file);
|
||||||
|
if (ref == null) {
|
||||||
|
Files.deleteIfExists(file);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (taskScopePayloadStorageService.recoverBufferedScopePayload(ref.taskId(), ref.moduleType(), ref.scopeHash())) {
|
||||||
|
recovered++;
|
||||||
|
}
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[task-scope-buffer] startup recovery failed path={} msg={}", file, ex.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (scanned > 0) {
|
||||||
|
log.info("[task-scope-buffer] startup recovery finished scanned={} recovered={} limit={}", scanned, recovered, limit);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Scheduled(cron = "${aiimage.task-pressure.scope-payload-cleanup-cron:15 */30 * * * *}")
|
||||||
|
public void cleanupExpiredBufferedPayloads() {
|
||||||
|
DistributedJobLockService.LockHandle lockHandle =
|
||||||
|
distributedJobLockService.tryLock("task-scope-buffer:cleanup", CLEANUP_LOCK_TTL);
|
||||||
|
if (lockHandle == null) {
|
||||||
|
log.info("[task-scope-buffer] skip cleanup because another instance holds the lock");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try (lockHandle) {
|
||||||
|
Path root = taskScopePayloadStorageService.getBufferedPayloadRoot();
|
||||||
|
if (!Files.isDirectory(root)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Instant cutoff = Instant.now().minus(Duration.ofHours(Math.max(1L, taskPressureProperties.getScopePayloadBufferRetentionHours())));
|
||||||
|
int deleted = 0;
|
||||||
|
for (Path file : listBufferedPayloadFiles(root, Integer.MAX_VALUE)) {
|
||||||
|
try {
|
||||||
|
FileTime lastModified = Files.getLastModifiedTime(file);
|
||||||
|
if (lastModified.toInstant().isAfter(cutoff)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Files.deleteIfExists(file);
|
||||||
|
deleted++;
|
||||||
|
deleteEmptyParents(file.getParent(), root);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[task-scope-buffer] cleanup failed path={} msg={}", file, ex.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (deleted > 0) {
|
||||||
|
log.info("[task-scope-buffer] cleanup finished deleted={} root={}", deleted, root);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Path> listBufferedPayloadFiles(Path root, int limit) {
|
||||||
|
try (var walk = Files.walk(root, 4)) {
|
||||||
|
return walk
|
||||||
|
.filter(Files::isRegularFile)
|
||||||
|
.filter(path -> path.getFileName().toString().endsWith(".json"))
|
||||||
|
.sorted(Comparator.comparing(this::safeLastModified))
|
||||||
|
.limit(limit)
|
||||||
|
.toList();
|
||||||
|
} catch (IOException ex) {
|
||||||
|
log.warn("[task-scope-buffer] scan failed root={} msg={}", root, ex.getMessage());
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Instant safeLastModified(Path path) {
|
||||||
|
try {
|
||||||
|
return Files.getLastModifiedTime(path).toInstant();
|
||||||
|
} catch (IOException ex) {
|
||||||
|
return Instant.EPOCH;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private BufferFileRef parseBufferFile(Path root, Path file) {
|
||||||
|
Path relative = root.relativize(file);
|
||||||
|
if (relative.getNameCount() != 3) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String moduleType = relative.getName(0).toString().toUpperCase();
|
||||||
|
String taskIdRaw = relative.getName(1).toString();
|
||||||
|
String fileName = relative.getName(2).toString();
|
||||||
|
if (!fileName.endsWith(".json")) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
long taskId = Long.parseLong(taskIdRaw);
|
||||||
|
String scopeHash = fileName.substring(0, fileName.length() - 5);
|
||||||
|
if (scopeHash.isBlank()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return new BufferFileRef(moduleType, taskId, scopeHash);
|
||||||
|
} catch (NumberFormatException ex) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void deleteEmptyParents(Path start, Path root) {
|
||||||
|
Path current = start;
|
||||||
|
while (current != null && !current.equals(root)) {
|
||||||
|
try (var list = Files.list(current)) {
|
||||||
|
if (list.findAny().isPresent()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
} catch (IOException ex) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
Files.deleteIfExists(current);
|
||||||
|
} catch (IOException ex) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
current = current.getParent();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private record BufferFileRef(String moduleType, Long taskId, String scopeHash) {}
|
||||||
|
}
|
||||||
@@ -0,0 +1,542 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.service;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||||
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import org.springframework.dao.DuplicateKeyException;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.nio.file.StandardOpenOption;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.security.MessageDigest;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class TaskScopePayloadStorageService {
|
||||||
|
|
||||||
|
private static final Set<String> OSS_BACKED_STATE_MODULES = new HashSet<>(Set.of(
|
||||||
|
"SHOP_MATCH",
|
||||||
|
"PRODUCT_RISK_RESOLVE",
|
||||||
|
"PRICE_TRACK",
|
||||||
|
"PATROL_DELETE"
|
||||||
|
));
|
||||||
|
private static final String OSS_POINTER_PREFIX = "oss:";
|
||||||
|
private static final String LOCAL_BUFFER_DIR = "task-scope-buffer";
|
||||||
|
|
||||||
|
private final TaskScopeStateMapper taskScopeStateMapper;
|
||||||
|
private final ObjectMapper objectMapper;
|
||||||
|
private final OssStorageService ossStorageService;
|
||||||
|
private final TaskPressureProperties taskPressureProperties;
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public <T> void saveScopePayload(Long taskId, String moduleType, String scopeKey, T payload) {
|
||||||
|
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey) || payload == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String normalizedScopeKey = normalize(scopeKey);
|
||||||
|
String scopeHash = hash(normalizedScopeKey);
|
||||||
|
String payloadJson = writeJson(payload, "写入任务范围载荷失败");
|
||||||
|
LocalDateTime now = LocalDateTime.now();
|
||||||
|
TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash);
|
||||||
|
if (shouldStoreStatePayloadInOss(moduleType)) {
|
||||||
|
saveOssBackedScopePayload(taskId, moduleType, normalizedScopeKey, scopeHash, payloadJson, now, state);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
|
||||||
|
if (state == null) {
|
||||||
|
TaskScopeStateEntity entity = new TaskScopeStateEntity();
|
||||||
|
entity.setTaskId(taskId);
|
||||||
|
entity.setModuleType(moduleType);
|
||||||
|
entity.setScopeKey(normalizedScopeKey);
|
||||||
|
entity.setScopeHash(scopeHash);
|
||||||
|
entity.setStateJson(stateJson);
|
||||||
|
entity.setChunkTotal(0);
|
||||||
|
entity.setReceivedChunkCount(0);
|
||||||
|
entity.setCompleted(0);
|
||||||
|
entity.setLastChunkAt(now);
|
||||||
|
entity.setLastError(null);
|
||||||
|
entity.setCreatedAt(now);
|
||||||
|
entity.setUpdatedAt(now);
|
||||||
|
try {
|
||||||
|
taskScopeStateMapper.insert(entity);
|
||||||
|
return;
|
||||||
|
} catch (DuplicateKeyException ignored) {
|
||||||
|
state = getScopeState(taskId, moduleType, scopeHash);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (state == null) {
|
||||||
|
throw new BusinessException("写入任务范围载荷失败");
|
||||||
|
}
|
||||||
|
deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson);
|
||||||
|
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getId, state.getId())
|
||||||
|
.set(TaskScopeStateEntity::getScopeKey, normalizedScopeKey)
|
||||||
|
.set(TaskScopeStateEntity::getStateJson, stateJson)
|
||||||
|
.set(TaskScopeStateEntity::getLastChunkAt, now)
|
||||||
|
.set(TaskScopeStateEntity::getUpdatedAt, now));
|
||||||
|
}
|
||||||
|
|
||||||
|
public <T> T getScopePayload(Long taskId, String moduleType, String scopeKey, Class<T> clazz) {
|
||||||
|
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
TaskScopeStateEntity state = getScopeState(taskId, moduleType, hash(normalize(scopeKey)));
|
||||||
|
String payloadJson = resolveStatePayload(state);
|
||||||
|
if (payloadJson == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return objectMapper.readValue(payloadJson, clazz);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException("读取任务范围载荷失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public <T> Map<String, T> getAllScopePayload(Long taskId, String moduleType, Class<T> clazz) {
|
||||||
|
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
|
||||||
|
return Map.of();
|
||||||
|
}
|
||||||
|
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.select(TaskScopeStateEntity::getScopeKey, TaskScopeStateEntity::getScopeHash, TaskScopeStateEntity::getStateJson)
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, moduleType)
|
||||||
|
.isNotNull(TaskScopeStateEntity::getStateJson));
|
||||||
|
if (states == null || states.isEmpty()) {
|
||||||
|
return Map.of();
|
||||||
|
}
|
||||||
|
Map<String, T> result = new LinkedHashMap<>();
|
||||||
|
for (TaskScopeStateEntity state : states) {
|
||||||
|
String payloadJson = resolveStatePayload(state);
|
||||||
|
if (payloadJson == null || isBlank(state.getScopeKey())) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
result.put(state.getScopeKey(), objectMapper.readValue(payloadJson, clazz));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException("读取任务范围载荷失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public void removeScopePayload(Long taskId, String moduleType, String scopeKey) {
|
||||||
|
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
TaskScopeStateEntity state = getScopeState(taskId, moduleType, hash(normalize(scopeKey)));
|
||||||
|
if (state == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
deleteBufferedPayload(taskId, moduleType, state.getScopeHash());
|
||||||
|
deleteStatePayloadIfNeeded(state.getStateJson());
|
||||||
|
boolean hasParsedPayload = !isBlank(state.getParsedPayloadJson());
|
||||||
|
if (hasParsedPayload) {
|
||||||
|
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getId, state.getId())
|
||||||
|
.set(TaskScopeStateEntity::getStateJson, null)
|
||||||
|
.set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now()));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
taskScopeStateMapper.deleteById(state.getId());
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean hasAnyScopePayload(Long taskId, String moduleType) {
|
||||||
|
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, moduleType)
|
||||||
|
.isNotNull(TaskScopeStateEntity::getStateJson));
|
||||||
|
return count != null && count > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
public long countScopePayload(Long taskId, String moduleType) {
|
||||||
|
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
|
||||||
|
return 0L;
|
||||||
|
}
|
||||||
|
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, moduleType)
|
||||||
|
.isNotNull(TaskScopeStateEntity::getStateJson));
|
||||||
|
return count == null ? 0L : count;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public void deleteTaskScopePayloads(Long taskId, String moduleType) {
|
||||||
|
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, moduleType));
|
||||||
|
if (states == null || states.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (TaskScopeStateEntity state : states) {
|
||||||
|
if (state == null || state.getId() == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
deleteBufferedPayload(taskId, moduleType, state.getScopeHash());
|
||||||
|
deleteStatePayloadIfNeeded(state.getStateJson());
|
||||||
|
if (!isBlank(state.getParsedPayloadJson())) {
|
||||||
|
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getId, state.getId())
|
||||||
|
.set(TaskScopeStateEntity::getStateJson, null)
|
||||||
|
.set(TaskScopeStateEntity::getUpdatedAt, LocalDateTime.now()));
|
||||||
|
} else {
|
||||||
|
taskScopeStateMapper.deleteById(state.getId());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public <T> void saveParsedPayloadMap(Long taskId, String moduleType, Map<String, T> payloadByScope) {
|
||||||
|
if (taskId == null || taskId <= 0 || isBlank(moduleType) || payloadByScope == null || payloadByScope.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
LocalDateTime now = LocalDateTime.now();
|
||||||
|
for (Map.Entry<String, T> entry : payloadByScope.entrySet()) {
|
||||||
|
if (isBlank(entry.getKey()) || entry.getValue() == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
String scopeKey = normalize(entry.getKey());
|
||||||
|
String scopeHash = hash(scopeKey);
|
||||||
|
String parsedPayloadJson = writeJson(entry.getValue(), "写入任务解析载荷失败");
|
||||||
|
TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash);
|
||||||
|
if (state == null) {
|
||||||
|
TaskScopeStateEntity entity = new TaskScopeStateEntity();
|
||||||
|
entity.setTaskId(taskId);
|
||||||
|
entity.setModuleType(moduleType);
|
||||||
|
entity.setScopeKey(scopeKey);
|
||||||
|
entity.setScopeHash(scopeHash);
|
||||||
|
entity.setParsedPayloadJson(parsedPayloadJson);
|
||||||
|
entity.setChunkTotal(0);
|
||||||
|
entity.setReceivedChunkCount(0);
|
||||||
|
entity.setCompleted(0);
|
||||||
|
entity.setCreatedAt(now);
|
||||||
|
entity.setUpdatedAt(now);
|
||||||
|
try {
|
||||||
|
taskScopeStateMapper.insert(entity);
|
||||||
|
continue;
|
||||||
|
} catch (DuplicateKeyException ignored) {
|
||||||
|
state = getScopeState(taskId, moduleType, scopeHash);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (state == null) {
|
||||||
|
throw new BusinessException("写入任务解析载荷失败");
|
||||||
|
}
|
||||||
|
deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), parsedPayloadJson);
|
||||||
|
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getId, state.getId())
|
||||||
|
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
|
||||||
|
.set(TaskScopeStateEntity::getParsedPayloadJson, parsedPayloadJson)
|
||||||
|
.set(TaskScopeStateEntity::getUpdatedAt, now));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public <T> Map<String, T> getParsedPayloadMap(Long taskId, String moduleType, Class<T> clazz) {
|
||||||
|
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
|
||||||
|
return Map.of();
|
||||||
|
}
|
||||||
|
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.select(TaskScopeStateEntity::getScopeKey, TaskScopeStateEntity::getParsedPayloadJson)
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, moduleType)
|
||||||
|
.isNotNull(TaskScopeStateEntity::getParsedPayloadJson));
|
||||||
|
if (states == null || states.isEmpty()) {
|
||||||
|
return Map.of();
|
||||||
|
}
|
||||||
|
Map<String, T> result = new LinkedHashMap<>();
|
||||||
|
for (TaskScopeStateEntity state : states) {
|
||||||
|
if (isBlank(state.getParsedPayloadJson()) || isBlank(state.getScopeKey())) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
result.put(state.getScopeKey(), objectMapper.readValue(resolveParsedPayload(state.getParsedPayloadJson()), clazz));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException("读取任务原始载荷失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public boolean recoverBufferedScopePayload(Long taskId, String moduleType, String scopeHash) {
|
||||||
|
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeHash)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
String payloadJson = readBufferedPayload(taskId, moduleType, scopeHash);
|
||||||
|
if (payloadJson == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash);
|
||||||
|
if (state == null) {
|
||||||
|
deleteBufferedPayload(taskId, moduleType, scopeHash);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (!shouldStoreStatePayloadInOss(moduleType)) {
|
||||||
|
deleteBufferedPayload(taskId, moduleType, scopeHash);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
LocalDateTime now = LocalDateTime.now();
|
||||||
|
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
|
||||||
|
deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson);
|
||||||
|
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getId, state.getId())
|
||||||
|
.set(TaskScopeStateEntity::getStateJson, stateJson)
|
||||||
|
.set(TaskScopeStateEntity::getLastChunkAt, now)
|
||||||
|
.set(TaskScopeStateEntity::getUpdatedAt, now));
|
||||||
|
deleteBufferedPayload(taskId, moduleType, scopeHash);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Path getBufferedPayloadRoot() {
|
||||||
|
return Path.of(System.getProperty("java.io.tmpdir"), LOCAL_BUFFER_DIR);
|
||||||
|
}
|
||||||
|
|
||||||
|
private TaskScopeStateEntity getScopeState(Long taskId, String moduleType, String scopeHash) {
|
||||||
|
return taskScopeStateMapper.selectOne(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, moduleType)
|
||||||
|
.eq(TaskScopeStateEntity::getScopeHash, scopeHash)
|
||||||
|
.last("limit 1"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private String storeStatePayload(String moduleType, Long taskId, String scopeHash, String payloadJson) {
|
||||||
|
if (!shouldStoreStatePayloadInOss(moduleType)) {
|
||||||
|
return payloadJson;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
String pointer = OSS_POINTER_PREFIX + ossStorageService.uploadTaskScopePayload(moduleType, taskId, scopeHash, payloadJson);
|
||||||
|
return objectMapper.writeValueAsString(pointer);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException("写入任务范围载荷失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveStatePayload(TaskScopeStateEntity state) {
|
||||||
|
if (state == null || isBlank(state.getStateJson())) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String bufferedPayload = readBufferedPayload(state.getTaskId(), state.getModuleType(), state.getScopeHash());
|
||||||
|
if (bufferedPayload != null) {
|
||||||
|
return bufferedPayload;
|
||||||
|
}
|
||||||
|
String ossPointer = extractOssPointer(state.getStateJson());
|
||||||
|
if (ossPointer == null) {
|
||||||
|
return state.getStateJson();
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return ossStorageService.readObjectAsString(stripOssPointerPrefix(ossPointer));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException("读取任务范围载荷失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void deleteStatePayloadIfNeeded(String value) {
|
||||||
|
String ossPointer = extractOssPointer(value);
|
||||||
|
if (ossPointer == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
ossStorageService.deleteObject(stripOssPointerPrefix(ossPointer));
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveParsedPayload(String value) {
|
||||||
|
if (isBlank(value) || !isOssPointer(value)) {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return ossStorageService.readObjectAsString(stripOssPointerPrefix(value));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException("读取任务解析载荷失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void deleteParsedPayloadIfNeeded(String oldValue, String newValue) {
|
||||||
|
if (isBlank(oldValue) || oldValue.equals(newValue) || !isOssPointer(oldValue)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
ossStorageService.deleteObject(stripOssPointerPrefix(oldValue));
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void deleteReplacedStatePayloadIfNeeded(String oldValue, String newValue) {
|
||||||
|
if (isBlank(oldValue) || oldValue.equals(newValue)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
deleteStatePayloadIfNeeded(oldValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void saveOssBackedScopePayload(Long taskId,
|
||||||
|
String moduleType,
|
||||||
|
String normalizedScopeKey,
|
||||||
|
String scopeHash,
|
||||||
|
String payloadJson,
|
||||||
|
LocalDateTime now,
|
||||||
|
TaskScopeStateEntity state) {
|
||||||
|
if (state == null) {
|
||||||
|
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
|
||||||
|
TaskScopeStateEntity entity = new TaskScopeStateEntity();
|
||||||
|
entity.setTaskId(taskId);
|
||||||
|
entity.setModuleType(moduleType);
|
||||||
|
entity.setScopeKey(normalizedScopeKey);
|
||||||
|
entity.setScopeHash(scopeHash);
|
||||||
|
entity.setStateJson(stateJson);
|
||||||
|
entity.setChunkTotal(0);
|
||||||
|
entity.setReceivedChunkCount(0);
|
||||||
|
entity.setCompleted(0);
|
||||||
|
entity.setLastChunkAt(now);
|
||||||
|
entity.setLastError(null);
|
||||||
|
entity.setCreatedAt(now);
|
||||||
|
entity.setUpdatedAt(now);
|
||||||
|
try {
|
||||||
|
taskScopeStateMapper.insert(entity);
|
||||||
|
deleteBufferedPayload(taskId, moduleType, scopeHash);
|
||||||
|
return;
|
||||||
|
} catch (DuplicateKeyException ignored) {
|
||||||
|
state = getScopeState(taskId, moduleType, scopeHash);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (state == null) {
|
||||||
|
throw new BusinessException("刷新任务范围缓冲载荷失败");
|
||||||
|
}
|
||||||
|
|
||||||
|
writeBufferedPayload(taskId, moduleType, scopeHash, payloadJson);
|
||||||
|
if (!shouldFlushBufferedPayload(state, now)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
|
||||||
|
deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson);
|
||||||
|
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getId, state.getId())
|
||||||
|
.set(TaskScopeStateEntity::getScopeKey, normalizedScopeKey)
|
||||||
|
.set(TaskScopeStateEntity::getStateJson, stateJson)
|
||||||
|
.set(TaskScopeStateEntity::getLastChunkAt, now)
|
||||||
|
.set(TaskScopeStateEntity::getUpdatedAt, now));
|
||||||
|
deleteBufferedPayload(taskId, moduleType, scopeHash);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String writeJson(Object value, String message) {
|
||||||
|
try {
|
||||||
|
return objectMapper.writeValueAsString(value);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException(message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalize(String value) {
|
||||||
|
return value == null ? "" : value.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isBlank(String value) {
|
||||||
|
return value == null || value.isBlank();
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean shouldStoreStatePayloadInOss(String moduleType) {
|
||||||
|
return OSS_BACKED_STATE_MODULES.contains(normalize(moduleType).toUpperCase());
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isOssPointer(String value) {
|
||||||
|
return !isBlank(value) && value.startsWith(OSS_POINTER_PREFIX);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String extractOssPointer(String value) {
|
||||||
|
if (isBlank(value)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (isOssPointer(value)) {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
String decoded = objectMapper.readValue(value, String.class);
|
||||||
|
return isOssPointer(decoded) ? decoded : null;
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String stripOssPointerPrefix(String value) {
|
||||||
|
return value.substring(OSS_POINTER_PREFIX.length());
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean shouldFlushBufferedPayload(TaskScopeStateEntity state, LocalDateTime now) {
|
||||||
|
long flushIntervalMillis = Math.max(0L, taskPressureProperties.getScopePayloadFlushIntervalMillis());
|
||||||
|
if (flushIntervalMillis <= 0L || state.getUpdatedAt() == null) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return java.time.Duration.between(state.getUpdatedAt(), now).toMillis() >= flushIntervalMillis;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void writeBufferedPayload(Long taskId, String moduleType, String scopeHash, String payloadJson) {
|
||||||
|
try {
|
||||||
|
Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash);
|
||||||
|
Files.createDirectories(path.getParent());
|
||||||
|
Files.writeString(path, payloadJson, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE);
|
||||||
|
} catch (IOException ex) {
|
||||||
|
throw new BusinessException("写入任务范围缓冲载荷失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String readBufferedPayload(Long taskId, String moduleType, String scopeHash) {
|
||||||
|
try {
|
||||||
|
Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash);
|
||||||
|
if (!Files.isRegularFile(path)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return Files.readString(path);
|
||||||
|
} catch (IOException ex) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void deleteBufferedPayload(Long taskId, String moduleType, String scopeHash) {
|
||||||
|
try {
|
||||||
|
Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash);
|
||||||
|
Files.deleteIfExists(path);
|
||||||
|
} catch (IOException ignored) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Path buildBufferedPayloadPath(Long taskId, String moduleType, String scopeHash) {
|
||||||
|
String normalizedModuleType = normalize(moduleType).toLowerCase();
|
||||||
|
return Path.of(System.getProperty("java.io.tmpdir"), LOCAL_BUFFER_DIR, normalizedModuleType, String.valueOf(taskId), scopeHash + ".json");
|
||||||
|
}
|
||||||
|
|
||||||
|
private String hash(String value) {
|
||||||
|
try {
|
||||||
|
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||||
|
byte[] bytes = digest.digest(normalize(value).getBytes(StandardCharsets.UTF_8));
|
||||||
|
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
||||||
|
for (byte b : bytes) {
|
||||||
|
sb.append(String.format("%02x", b));
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new IllegalStateException("failed to hash scope key", ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -50,7 +50,7 @@ public class ZiniaoShopSwitchService {
|
|||||||
if (value == null) {
|
if (value == null) {
|
||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
return value.replace("\u3000", " ").trim();
|
return value.replace(" ", " ").trim();
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isSkippableMatchError(BusinessException ex) {
|
public boolean isSkippableMatchError(BusinessException ex) {
|
||||||
@@ -59,9 +59,9 @@ public class ZiniaoShopSwitchService {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return (message.contains("code=40004")
|
return (message.contains("code=40004")
|
||||||
|| message.contains("userId\u5b58\u5728\u65e0\u6548\u7684\u53c2\u6570\u503c")
|
|| message.contains("userId存在无效的参数值")
|
||||||
|| message.contains("\u65e0\u6743\u9650")
|
|| message.contains("无权限")
|
||||||
|| message.contains("\u6ca1\u6709\u6743\u9650"))
|
|| message.contains("没有权限"))
|
||||||
&& !message.contains("\u767d\u540d\u5355");
|
&& !message.contains("白名单");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -99,6 +99,10 @@ aiimage:
|
|||||||
task-pressure:
|
task-pressure:
|
||||||
local-task-entity-cache-millis: ${AIIMAGE_TASK_LOCAL_ENTITY_CACHE_MILLIS:3000}
|
local-task-entity-cache-millis: ${AIIMAGE_TASK_LOCAL_ENTITY_CACHE_MILLIS:3000}
|
||||||
db-select-batch-size: ${AIIMAGE_TASK_DB_SELECT_BATCH_SIZE:200}
|
db-select-batch-size: ${AIIMAGE_TASK_DB_SELECT_BATCH_SIZE:200}
|
||||||
|
scope-payload-flush-interval-millis: ${AIIMAGE_TASK_SCOPE_PAYLOAD_FLUSH_INTERVAL_MILLIS:15000}
|
||||||
|
scope-payload-buffer-retention-hours: ${AIIMAGE_TASK_SCOPE_PAYLOAD_BUFFER_RETENTION_HOURS:24}
|
||||||
|
scope-payload-recovery-max-files: ${AIIMAGE_TASK_SCOPE_PAYLOAD_RECOVERY_MAX_FILES:200}
|
||||||
|
scope-payload-cleanup-cron: ${AIIMAGE_TASK_SCOPE_PAYLOAD_CLEANUP_CRON:15 */30 * * * *}
|
||||||
security:
|
security:
|
||||||
shop-credential-key: ${AIIMAGE_SHOP_CREDENTIAL_KEY:change-me-shop-credential-key}
|
shop-credential-key: ${AIIMAGE_SHOP_CREDENTIAL_KEY:change-me-shop-credential-key}
|
||||||
internal-token: ${AIIMAGE_INTERNAL_TOKEN:}
|
internal-token: ${AIIMAGE_INTERNAL_TOKEN:}
|
||||||
|
|||||||
@@ -0,0 +1,76 @@
|
|||||||
|
SET @schema_name = DATABASE();
|
||||||
|
|
||||||
|
SET @sql = IF(
|
||||||
|
EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM information_schema.COLUMNS
|
||||||
|
WHERE TABLE_SCHEMA = @schema_name
|
||||||
|
AND TABLE_NAME = 'biz_skip_price_asin'
|
||||||
|
AND COLUMN_NAME = 'minimum_price_de'
|
||||||
|
),
|
||||||
|
'SELECT 1',
|
||||||
|
'ALTER TABLE biz_skip_price_asin ADD COLUMN minimum_price_de DECIMAL(10,2) NULL DEFAULT NULL COMMENT ''DE minimum price'' AFTER asin_de'
|
||||||
|
);
|
||||||
|
PREPARE stmt FROM @sql;
|
||||||
|
EXECUTE stmt;
|
||||||
|
DEALLOCATE PREPARE stmt;
|
||||||
|
|
||||||
|
SET @sql = IF(
|
||||||
|
EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM information_schema.COLUMNS
|
||||||
|
WHERE TABLE_SCHEMA = @schema_name
|
||||||
|
AND TABLE_NAME = 'biz_skip_price_asin'
|
||||||
|
AND COLUMN_NAME = 'minimum_price_uk'
|
||||||
|
),
|
||||||
|
'SELECT 1',
|
||||||
|
'ALTER TABLE biz_skip_price_asin ADD COLUMN minimum_price_uk DECIMAL(10,2) NULL DEFAULT NULL COMMENT ''UK minimum price'' AFTER asin_uk'
|
||||||
|
);
|
||||||
|
PREPARE stmt FROM @sql;
|
||||||
|
EXECUTE stmt;
|
||||||
|
DEALLOCATE PREPARE stmt;
|
||||||
|
|
||||||
|
SET @sql = IF(
|
||||||
|
EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM information_schema.COLUMNS
|
||||||
|
WHERE TABLE_SCHEMA = @schema_name
|
||||||
|
AND TABLE_NAME = 'biz_skip_price_asin'
|
||||||
|
AND COLUMN_NAME = 'minimum_price_fr'
|
||||||
|
),
|
||||||
|
'SELECT 1',
|
||||||
|
'ALTER TABLE biz_skip_price_asin ADD COLUMN minimum_price_fr DECIMAL(10,2) NULL DEFAULT NULL COMMENT ''FR minimum price'' AFTER asin_fr'
|
||||||
|
);
|
||||||
|
PREPARE stmt FROM @sql;
|
||||||
|
EXECUTE stmt;
|
||||||
|
DEALLOCATE PREPARE stmt;
|
||||||
|
|
||||||
|
SET @sql = IF(
|
||||||
|
EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM information_schema.COLUMNS
|
||||||
|
WHERE TABLE_SCHEMA = @schema_name
|
||||||
|
AND TABLE_NAME = 'biz_skip_price_asin'
|
||||||
|
AND COLUMN_NAME = 'minimum_price_it'
|
||||||
|
),
|
||||||
|
'SELECT 1',
|
||||||
|
'ALTER TABLE biz_skip_price_asin ADD COLUMN minimum_price_it DECIMAL(10,2) NULL DEFAULT NULL COMMENT ''IT minimum price'' AFTER asin_it'
|
||||||
|
);
|
||||||
|
PREPARE stmt FROM @sql;
|
||||||
|
EXECUTE stmt;
|
||||||
|
DEALLOCATE PREPARE stmt;
|
||||||
|
|
||||||
|
SET @sql = IF(
|
||||||
|
EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM information_schema.COLUMNS
|
||||||
|
WHERE TABLE_SCHEMA = @schema_name
|
||||||
|
AND TABLE_NAME = 'biz_skip_price_asin'
|
||||||
|
AND COLUMN_NAME = 'minimum_price_es'
|
||||||
|
),
|
||||||
|
'SELECT 1',
|
||||||
|
'ALTER TABLE biz_skip_price_asin ADD COLUMN minimum_price_es DECIMAL(10,2) NULL DEFAULT NULL COMMENT ''ES minimum price'' AFTER asin_es'
|
||||||
|
);
|
||||||
|
PREPARE stmt FROM @sql;
|
||||||
|
EXECUTE stmt;
|
||||||
|
DEALLOCATE PREPARE stmt;
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
CREATE TABLE IF NOT EXISTS biz_task_scope_state (
|
||||||
|
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key',
|
||||||
|
task_id BIGINT NOT NULL COMMENT 'task id',
|
||||||
|
module_type VARCHAR(32) NOT NULL COMMENT 'module type',
|
||||||
|
scope_key VARCHAR(1000) NOT NULL COMMENT 'scope identifier',
|
||||||
|
scope_hash CHAR(64) NOT NULL COMMENT 'hash of scope key',
|
||||||
|
parsed_payload_json JSON NULL COMMENT 'parsed source payload',
|
||||||
|
state_json JSON NULL COMMENT 'merged intermediate state',
|
||||||
|
chunk_total INT NOT NULL DEFAULT 0 COMMENT 'expected chunk count',
|
||||||
|
received_chunk_count INT NOT NULL DEFAULT 0 COMMENT 'received chunk count',
|
||||||
|
completed TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'whether scope is complete',
|
||||||
|
last_chunk_at DATETIME NULL COMMENT 'last chunk received time',
|
||||||
|
last_error VARCHAR(1000) NULL COMMENT 'last error message',
|
||||||
|
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
|
||||||
|
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
|
||||||
|
UNIQUE KEY uk_task_scope (task_id, module_type, scope_hash),
|
||||||
|
KEY idx_task_completed (task_id, module_type, completed),
|
||||||
|
KEY idx_scope_hash (scope_hash)
|
||||||
|
) COMMENT='task scope state';
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS biz_task_chunk (
|
||||||
|
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key',
|
||||||
|
task_id BIGINT NOT NULL COMMENT 'task id',
|
||||||
|
module_type VARCHAR(32) NOT NULL COMMENT 'module type',
|
||||||
|
scope_key VARCHAR(1000) NOT NULL COMMENT 'scope identifier',
|
||||||
|
scope_hash CHAR(64) NOT NULL COMMENT 'hash of scope key',
|
||||||
|
chunk_index INT NOT NULL COMMENT 'chunk index starting from 1',
|
||||||
|
chunk_total INT NOT NULL DEFAULT 0 COMMENT 'total chunk count',
|
||||||
|
payload_json JSON NOT NULL COMMENT 'chunk payload',
|
||||||
|
payload_hash CHAR(64) NOT NULL COMMENT 'hash of payload json',
|
||||||
|
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
|
||||||
|
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
|
||||||
|
UNIQUE KEY uk_task_scope_chunk (task_id, module_type, scope_hash, chunk_index),
|
||||||
|
KEY idx_task_scope (task_id, module_type, scope_hash),
|
||||||
|
KEY idx_task_module (task_id, module_type)
|
||||||
|
) COMMENT='task chunk detail';
|
||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -12,7 +12,7 @@ import pymysql
|
|||||||
from werkzeug.security import generate_password_hash
|
from werkzeug.security import generate_password_hash
|
||||||
|
|
||||||
from utils.db import get_db
|
from utils.db import get_db
|
||||||
from utils.auth import admin_required, get_current_admin_role
|
from utils.auth import admin_required, login_required, get_current_admin_role
|
||||||
|
|
||||||
|
|
||||||
ALLOWED_DEDUPE_TOTAL_DATA_ADMIN_USERNAMES = {'刘丽泓'}
|
ALLOWED_DEDUPE_TOTAL_DATA_ADMIN_USERNAMES = {'刘丽泓'}
|
||||||
@@ -353,8 +353,82 @@ def _load_current_admin_menu_items():
|
|||||||
return role, current_row, items, None
|
return role, current_row, items, None
|
||||||
|
|
||||||
|
|
||||||
|
def _load_current_backend_menu_items():
|
||||||
|
role, current_row = get_current_admin_role()
|
||||||
|
if not role or not current_row:
|
||||||
|
return None, None, None, (jsonify({'success': False, 'error': '需要登录'}), 403)
|
||||||
|
user_id = _get_current_admin_id(current_row)
|
||||||
|
if not user_id:
|
||||||
|
return role, current_row, None, (jsonify({'success': False, 'error': '当前登录用户缺少有效ID'}), 400)
|
||||||
|
result, error_response, status = _proxy_backend_java(
|
||||||
|
'GET',
|
||||||
|
f'/api/admin/permission-users/{user_id}/column-permissions',
|
||||||
|
params={'menuType': 'admin'},
|
||||||
|
)
|
||||||
|
if error_response is not None:
|
||||||
|
return role, current_row, None, (error_response, status)
|
||||||
|
items = [_format_permission_item(item) for item in (result.get('data') or [])]
|
||||||
|
return role, current_row, items, None
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_backend_menu_access(*menu_names):
|
||||||
|
role, current_row = get_current_admin_role()
|
||||||
|
if role == 'super_admin':
|
||||||
|
return role, current_row, None
|
||||||
|
if not role or not current_row:
|
||||||
|
return role, current_row, (jsonify({'success': False, 'error': '需要登录'}), 403)
|
||||||
|
key_set, route_set = _get_current_admin_permission_sets()
|
||||||
|
for menu_name in menu_names:
|
||||||
|
config = ADMIN_MENU_ACCESS_CONFIG.get(menu_name) or {}
|
||||||
|
column_key = (config.get('column_key') or '').strip()
|
||||||
|
route_path = (config.get('route_path') or '').strip()
|
||||||
|
if (column_key and column_key in key_set) or (route_path and route_path in route_set):
|
||||||
|
return role, current_row, None
|
||||||
|
fallback_menu = menu_names[0] if menu_names else ''
|
||||||
|
fallback_config = ADMIN_MENU_ACCESS_CONFIG.get(fallback_menu) or {}
|
||||||
|
error_message = fallback_config.get('error') or '无权访问当前模块'
|
||||||
|
return role, current_row, (jsonify({'success': False, 'error': error_message}), 403)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_column_ids_by_route_paths(route_paths, menu_type='admin'):
|
||||||
|
normalized_paths = [(path or '').strip() for path in (route_paths or []) if (path or '').strip()]
|
||||||
|
if not normalized_paths:
|
||||||
|
return []
|
||||||
|
conn = get_db()
|
||||||
|
try:
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
placeholders = ','.join(['%s'] * len(normalized_paths))
|
||||||
|
cur.execute(
|
||||||
|
f"SELECT id FROM columns WHERE menu_type = %s AND route_path IN ({placeholders}) ORDER BY id ASC",
|
||||||
|
[menu_type] + normalized_paths,
|
||||||
|
)
|
||||||
|
rows = cur.fetchall()
|
||||||
|
return [int(row['id']) for row in rows if row.get('id') is not None]
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _grant_backend_menu_permissions(user_ids, route_paths):
|
||||||
|
normalized_user_ids = [int(uid) for uid in (user_ids or []) if str(uid).isdigit() and int(uid) > 0]
|
||||||
|
column_ids = _get_column_ids_by_route_paths(route_paths, 'admin')
|
||||||
|
if not normalized_user_ids or not column_ids:
|
||||||
|
return
|
||||||
|
conn = get_db()
|
||||||
|
try:
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
for user_id in normalized_user_ids:
|
||||||
|
for column_id in column_ids:
|
||||||
|
cur.execute(
|
||||||
|
"INSERT IGNORE INTO user_column_permission (user_id, column_id) VALUES (%s, %s)",
|
||||||
|
(user_id, column_id),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
@admin_api.route('/current-user')
|
@admin_api.route('/current-user')
|
||||||
@admin_required
|
@login_required
|
||||||
def get_admin_current_user():
|
def get_admin_current_user():
|
||||||
role, current_row = get_current_admin_role()
|
role, current_row = get_current_admin_role()
|
||||||
if not role or not current_row:
|
if not role or not current_row:
|
||||||
@@ -370,16 +444,16 @@ def get_admin_current_user():
|
|||||||
|
|
||||||
|
|
||||||
@admin_api.route('/current-user/menus')
|
@admin_api.route('/current-user/menus')
|
||||||
@admin_required
|
@login_required
|
||||||
def get_admin_current_user_menus():
|
def get_admin_current_user_menus():
|
||||||
_, _, items, denied = _load_current_admin_menu_items()
|
_, _, items, denied = _load_current_backend_menu_items()
|
||||||
if denied:
|
if denied:
|
||||||
return denied
|
return denied
|
||||||
return jsonify({'success': True, 'items': items})
|
return jsonify({'success': True, 'items': items})
|
||||||
|
|
||||||
|
|
||||||
@admin_api.route('/logout', methods=['POST'])
|
@admin_api.route('/logout', methods=['POST'])
|
||||||
@admin_required
|
@login_required
|
||||||
def admin_logout():
|
def admin_logout():
|
||||||
session.clear()
|
session.clear()
|
||||||
return jsonify({'success': True, 'msg': '退出成功', 'redirect': '/login'})
|
return jsonify({'success': True, 'msg': '退出成功', 'redirect': '/login'})
|
||||||
@@ -388,17 +462,14 @@ def admin_logout():
|
|||||||
# ---------- 用户管理 ----------
|
# ---------- 用户管理 ----------
|
||||||
|
|
||||||
@admin_api.route('/users')
|
@admin_api.route('/users')
|
||||||
@admin_required
|
@login_required
|
||||||
def list_users():
|
def list_users():
|
||||||
"""分页获取用户列表,支持用户名模糊搜索和按管理员归属筛选普通用户"""
|
"""分页获取用户列表,支持用户名模糊搜索和按管理员归属筛选普通用户。"""
|
||||||
role, current_row = get_current_admin_role()
|
role, current_row = get_current_admin_role()
|
||||||
if not role:
|
if not role:
|
||||||
return jsonify({'success': False, 'error': '需要管理员权限'}), 403
|
return jsonify({'success': False, 'error': '需要登录'}), 403
|
||||||
page = max(1, int(request.args.get('page', 1)))
|
page = max(1, int(request.args.get('page', 1)))
|
||||||
_, _, denied = _ensure_admin_menu_access('users', 'history', 'shop-manage', 'skip-price-asin')
|
_, _, denied = _ensure_backend_menu_access('users', 'history', 'shop-manage', 'skip-price-asin')
|
||||||
if denied:
|
|
||||||
return denied
|
|
||||||
_, _, denied = _ensure_admin_menu_access('users', 'history', 'shop-manage', 'skip-price-asin')
|
|
||||||
if denied:
|
if denied:
|
||||||
return denied
|
return denied
|
||||||
page_size = min(999, max(5, int(request.args.get('page_size', 15))))
|
page_size = min(999, max(5, int(request.args.get('page_size', 15))))
|
||||||
@@ -407,6 +478,7 @@ def list_users():
|
|||||||
created_by_id_arg = request.args.get('created_by_id') or request.args.get('admin_id')
|
created_by_id_arg = request.args.get('created_by_id') or request.args.get('admin_id')
|
||||||
created_by_id = int(created_by_id_arg) if created_by_id_arg and str(created_by_id_arg).isdigit() else None
|
created_by_id = int(created_by_id_arg) if created_by_id_arg and str(created_by_id_arg).isdigit() else None
|
||||||
can_view_all_users = role == 'super_admin' or (current_row and (current_row.get('username') or '').strip().lower() == 'admin')
|
can_view_all_users = role == 'super_admin' or (current_row and (current_row.get('username') or '').strip().lower() == 'admin')
|
||||||
|
can_view_child_users = role == 'admin' and not can_view_all_users
|
||||||
if not can_view_all_users:
|
if not can_view_all_users:
|
||||||
created_by_id = None
|
created_by_id = None
|
||||||
try:
|
try:
|
||||||
@@ -435,7 +507,7 @@ def list_users():
|
|||||||
total = cur.fetchone()['total']
|
total = cur.fetchone()['total']
|
||||||
cur.execute("SELECT id, username FROM users WHERE role = 'admin' ORDER BY id")
|
cur.execute("SELECT id, username FROM users WHERE role = 'admin' ORDER BY id")
|
||||||
admins = [{'id': r['id'], 'username': r['username']} for r in cur.fetchall()]
|
admins = [{'id': r['id'], 'username': r['username']} for r in cur.fetchall()]
|
||||||
else:
|
elif can_view_child_users:
|
||||||
admin_id = current_row['id']
|
admin_id = current_row['id']
|
||||||
where_parts = ["(u.id = %s OR (u.role = 'normal' AND u.created_by_id = %s))"]
|
where_parts = ["(u.id = %s OR (u.role = 'normal' AND u.created_by_id = %s))"]
|
||||||
params = [admin_id, admin_id]
|
params = [admin_id, admin_id]
|
||||||
@@ -458,6 +530,28 @@ def list_users():
|
|||||||
)
|
)
|
||||||
total = cur.fetchone()['total']
|
total = cur.fetchone()['total']
|
||||||
admins = []
|
admins = []
|
||||||
|
else:
|
||||||
|
where_parts = ["u.id = %s"]
|
||||||
|
params = [current_row['id']]
|
||||||
|
if search_username:
|
||||||
|
where_parts.append("u.username LIKE %s")
|
||||||
|
params.append("%" + search_username + "%")
|
||||||
|
where_sql = " AND ".join(where_parts)
|
||||||
|
cur.execute(
|
||||||
|
"""SELECT u.id, u.username, u.is_admin, u.role, u.created_at, u.created_by_id,
|
||||||
|
creator.username AS creator_username
|
||||||
|
FROM users u
|
||||||
|
LEFT JOIN users creator ON creator.id = u.created_by_id
|
||||||
|
WHERE """ + where_sql + """ ORDER BY u.id LIMIT %s OFFSET %s""",
|
||||||
|
tuple(params) + (page_size, offset),
|
||||||
|
)
|
||||||
|
rows = cur.fetchall()
|
||||||
|
cur.execute(
|
||||||
|
"SELECT COUNT(*) as total FROM users u WHERE " + where_sql,
|
||||||
|
tuple(params),
|
||||||
|
)
|
||||||
|
total = cur.fetchone()['total']
|
||||||
|
admins = []
|
||||||
items = [
|
items = [
|
||||||
{
|
{
|
||||||
'id': r['id'],
|
'id': r['id'],
|
||||||
@@ -485,7 +579,6 @@ def list_users():
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
return jsonify({'success': False, 'error': str(e)})
|
return jsonify({'success': False, 'error': str(e)})
|
||||||
|
|
||||||
|
|
||||||
@admin_api.route('/user', methods=['POST'])
|
@admin_api.route('/user', methods=['POST'])
|
||||||
@admin_required
|
@admin_required
|
||||||
def create_user():
|
def create_user():
|
||||||
@@ -1341,9 +1434,9 @@ def delete_dedupe_total_data(item_id):
|
|||||||
# ---------- 店铺管理 ----------
|
# ---------- 店铺管理 ----------
|
||||||
|
|
||||||
@admin_api.route('/shop-manages')
|
@admin_api.route('/shop-manages')
|
||||||
@admin_required
|
@login_required
|
||||||
def list_shop_manages():
|
def list_shop_manages():
|
||||||
role, current_row, denied = _ensure_admin_menu_access('shop-manage')
|
role, current_row, denied = _ensure_backend_menu_access('shop-manage')
|
||||||
if denied:
|
if denied:
|
||||||
return denied
|
return denied
|
||||||
page = max(1, int(request.args.get('page', 1)))
|
page = max(1, int(request.args.get('page', 1)))
|
||||||
@@ -1397,9 +1490,9 @@ def list_shop_manages():
|
|||||||
|
|
||||||
|
|
||||||
@admin_api.route('/shop-manage', methods=['POST'])
|
@admin_api.route('/shop-manage', methods=['POST'])
|
||||||
@admin_required
|
@login_required
|
||||||
def create_shop_manage():
|
def create_shop_manage():
|
||||||
role, current_row, denied = _ensure_admin_menu_access('shop-manage')
|
role, current_row, denied = _ensure_backend_menu_access('shop-manage')
|
||||||
if denied:
|
if denied:
|
||||||
return denied
|
return denied
|
||||||
data = request.get_json() or {}
|
data = request.get_json() or {}
|
||||||
@@ -1441,9 +1534,9 @@ def create_shop_manage():
|
|||||||
|
|
||||||
|
|
||||||
@admin_api.route('/shop-manage/<int:item_id>', methods=['PUT'])
|
@admin_api.route('/shop-manage/<int:item_id>', methods=['PUT'])
|
||||||
@admin_required
|
@login_required
|
||||||
def update_shop_manage(item_id):
|
def update_shop_manage(item_id):
|
||||||
role, current_row, denied = _ensure_admin_menu_access('shop-manage')
|
role, current_row, denied = _ensure_backend_menu_access('shop-manage')
|
||||||
if denied:
|
if denied:
|
||||||
return denied
|
return denied
|
||||||
data = request.get_json() or {}
|
data = request.get_json() or {}
|
||||||
@@ -1483,9 +1576,9 @@ def update_shop_manage(item_id):
|
|||||||
|
|
||||||
|
|
||||||
@admin_api.route('/shop-manage/<int:item_id>', methods=['DELETE'])
|
@admin_api.route('/shop-manage/<int:item_id>', methods=['DELETE'])
|
||||||
@admin_required
|
@login_required
|
||||||
def delete_shop_manage(item_id):
|
def delete_shop_manage(item_id):
|
||||||
role, current_row, denied = _ensure_admin_menu_access('shop-manage')
|
role, current_row, denied = _ensure_backend_menu_access('shop-manage')
|
||||||
if denied:
|
if denied:
|
||||||
return denied
|
return denied
|
||||||
result, error_response, status = _proxy_backend_java(
|
result, error_response, status = _proxy_backend_java(
|
||||||
@@ -1505,9 +1598,9 @@ def delete_shop_manage(item_id):
|
|||||||
|
|
||||||
|
|
||||||
@admin_api.route('/shop-manage-groups')
|
@admin_api.route('/shop-manage-groups')
|
||||||
@admin_required
|
@login_required
|
||||||
def list_shop_manage_groups():
|
def list_shop_manage_groups():
|
||||||
role, current_row, denied = _ensure_admin_menu_access('shop-manage', 'skip-price-asin')
|
role, current_row, denied = _ensure_backend_menu_access('shop-manage', 'skip-price-asin')
|
||||||
if denied:
|
if denied:
|
||||||
return denied
|
return denied
|
||||||
params = {}
|
params = {}
|
||||||
@@ -1543,12 +1636,13 @@ def list_shop_manage_groups():
|
|||||||
|
|
||||||
|
|
||||||
@admin_api.route('/shop-manage-group', methods=['POST'])
|
@admin_api.route('/shop-manage-group', methods=['POST'])
|
||||||
@admin_required
|
@login_required
|
||||||
def create_shop_manage_group():
|
def create_shop_manage_group():
|
||||||
role, current_row, denied = _ensure_admin_menu_access('shop-manage', 'skip-price-asin')
|
role, current_row, denied = _ensure_backend_menu_access('shop-manage', 'skip-price-asin')
|
||||||
if denied:
|
if denied:
|
||||||
return denied
|
return denied
|
||||||
data = request.get_json() or {}
|
data = request.get_json() or {}
|
||||||
|
grant_menu_routes = data.get('grant_menu_routes') or []
|
||||||
payload = {
|
payload = {
|
||||||
'groupName': (data.get('group_name') or '').strip(),
|
'groupName': (data.get('group_name') or '').strip(),
|
||||||
'memberUserIds': data.get('member_user_ids') or [],
|
'memberUserIds': data.get('member_user_ids') or [],
|
||||||
@@ -1565,6 +1659,7 @@ def create_shop_manage_group():
|
|||||||
)
|
)
|
||||||
if error_response is not None:
|
if error_response is not None:
|
||||||
return error_response, status
|
return error_response, status
|
||||||
|
_grant_backend_menu_permissions(data.get('member_user_ids') or [], grant_menu_routes)
|
||||||
item = result.get('data') or {}
|
item = result.get('data') or {}
|
||||||
return jsonify({
|
return jsonify({
|
||||||
'success': True,
|
'success': True,
|
||||||
@@ -1586,12 +1681,13 @@ def create_shop_manage_group():
|
|||||||
|
|
||||||
|
|
||||||
@admin_api.route('/shop-manage-group/<int:item_id>', methods=['PUT'])
|
@admin_api.route('/shop-manage-group/<int:item_id>', methods=['PUT'])
|
||||||
@admin_required
|
@login_required
|
||||||
def update_shop_manage_group(item_id):
|
def update_shop_manage_group(item_id):
|
||||||
role, current_row, denied = _ensure_admin_menu_access('shop-manage', 'skip-price-asin')
|
role, current_row, denied = _ensure_backend_menu_access('shop-manage', 'skip-price-asin')
|
||||||
if denied:
|
if denied:
|
||||||
return denied
|
return denied
|
||||||
data = request.get_json() or {}
|
data = request.get_json() or {}
|
||||||
|
grant_menu_routes = data.get('grant_menu_routes') or []
|
||||||
payload = {
|
payload = {
|
||||||
'groupName': (data.get('group_name') or '').strip(),
|
'groupName': (data.get('group_name') or '').strip(),
|
||||||
'memberUserIds': data.get('member_user_ids') or [],
|
'memberUserIds': data.get('member_user_ids') or [],
|
||||||
@@ -1607,6 +1703,7 @@ def update_shop_manage_group(item_id):
|
|||||||
)
|
)
|
||||||
if error_response is not None:
|
if error_response is not None:
|
||||||
return error_response, status
|
return error_response, status
|
||||||
|
_grant_backend_menu_permissions(data.get('member_user_ids') or [], grant_menu_routes)
|
||||||
item = result.get('data') or {}
|
item = result.get('data') or {}
|
||||||
return jsonify({
|
return jsonify({
|
||||||
'success': True,
|
'success': True,
|
||||||
@@ -1628,9 +1725,9 @@ def update_shop_manage_group(item_id):
|
|||||||
|
|
||||||
|
|
||||||
@admin_api.route('/skip-price-asins')
|
@admin_api.route('/skip-price-asins')
|
||||||
@admin_required
|
@login_required
|
||||||
def list_skip_price_asins():
|
def list_skip_price_asins():
|
||||||
role, current_row, denied = _ensure_admin_menu_access('skip-price-asin')
|
role, current_row, denied = _ensure_backend_menu_access('skip-price-asin')
|
||||||
if denied:
|
if denied:
|
||||||
return denied
|
return denied
|
||||||
page = max(1, int(request.args.get('page', 1)))
|
page = max(1, int(request.args.get('page', 1)))
|
||||||
@@ -1668,10 +1765,15 @@ def list_skip_price_asins():
|
|||||||
'group_name': item.get('groupName') or '',
|
'group_name': item.get('groupName') or '',
|
||||||
'shop_name': item.get('shopName') or '',
|
'shop_name': item.get('shopName') or '',
|
||||||
'asin_de': item.get('asinDe') or '',
|
'asin_de': item.get('asinDe') or '',
|
||||||
|
'minimum_price_de': item.get('minimumPriceDe'),
|
||||||
'asin_uk': item.get('asinUk') or '',
|
'asin_uk': item.get('asinUk') or '',
|
||||||
|
'minimum_price_uk': item.get('minimumPriceUk'),
|
||||||
'asin_fr': item.get('asinFr') or '',
|
'asin_fr': item.get('asinFr') or '',
|
||||||
|
'minimum_price_fr': item.get('minimumPriceFr'),
|
||||||
'asin_it': item.get('asinIt') or '',
|
'asin_it': item.get('asinIt') or '',
|
||||||
|
'minimum_price_it': item.get('minimumPriceIt'),
|
||||||
'asin_es': item.get('asinEs') or '',
|
'asin_es': item.get('asinEs') or '',
|
||||||
|
'minimum_price_es': item.get('minimumPriceEs'),
|
||||||
'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16],
|
'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16],
|
||||||
'updated_at': (item.get('updatedAt') or '').replace('T', ' ')[:16],
|
'updated_at': (item.get('updatedAt') or '').replace('T', ' ')[:16],
|
||||||
}
|
}
|
||||||
@@ -1687,25 +1789,34 @@ def list_skip_price_asins():
|
|||||||
|
|
||||||
|
|
||||||
@admin_api.route('/skip-price-asin', methods=['POST'])
|
@admin_api.route('/skip-price-asin', methods=['POST'])
|
||||||
@admin_required
|
@login_required
|
||||||
def create_skip_price_asin():
|
def create_skip_price_asin():
|
||||||
role, current_row, denied = _ensure_admin_menu_access('skip-price-asin')
|
role, current_row, denied = _ensure_backend_menu_access('skip-price-asin')
|
||||||
if denied:
|
if denied:
|
||||||
return denied
|
return denied
|
||||||
data = request.get_json() or {}
|
data = request.get_json() or {}
|
||||||
asin_mappings = data.get('asin_mappings') or {}
|
asin_mappings = data.get('asin_mappings') or {}
|
||||||
|
minimum_price_mappings = data.get('minimum_price_mappings') or {}
|
||||||
fallback_asin = (data.get('asin') or '').strip()
|
fallback_asin = (data.get('asin') or '').strip()
|
||||||
if not fallback_asin and isinstance(asin_mappings, dict):
|
if not fallback_asin and isinstance(asin_mappings, dict):
|
||||||
for value in asin_mappings.values():
|
for value in asin_mappings.values():
|
||||||
fallback_asin = (value or '').strip()
|
fallback_asin = (value or '').strip()
|
||||||
if fallback_asin:
|
if fallback_asin:
|
||||||
break
|
break
|
||||||
|
fallback_minimum_price = data.get('minimum_price')
|
||||||
|
if fallback_minimum_price in ('', None) and isinstance(minimum_price_mappings, dict):
|
||||||
|
for value in minimum_price_mappings.values():
|
||||||
|
if value not in ('', None):
|
||||||
|
fallback_minimum_price = value
|
||||||
|
break
|
||||||
payload = {
|
payload = {
|
||||||
'groupId': data.get('group_id'),
|
'groupId': data.get('group_id'),
|
||||||
'shopName': (data.get('shop_name') or '').strip(),
|
'shopName': (data.get('shop_name') or '').strip(),
|
||||||
'countries': data.get('countries') or [],
|
'countries': data.get('countries') or [],
|
||||||
'asin': fallback_asin,
|
'asin': fallback_asin,
|
||||||
'asinMappings': asin_mappings,
|
'asinMappings': asin_mappings,
|
||||||
|
'minimumPrice': fallback_minimum_price,
|
||||||
|
'minimumPriceMappings': minimum_price_mappings,
|
||||||
}
|
}
|
||||||
result, error_response, status = _proxy_backend_java(
|
result, error_response, status = _proxy_backend_java(
|
||||||
'POST',
|
'POST',
|
||||||
@@ -1728,10 +1839,15 @@ def create_skip_price_asin():
|
|||||||
'group_name': item.get('groupName') or '',
|
'group_name': item.get('groupName') or '',
|
||||||
'shop_name': item.get('shopName') or '',
|
'shop_name': item.get('shopName') or '',
|
||||||
'asin_de': item.get('asinDe') or '',
|
'asin_de': item.get('asinDe') or '',
|
||||||
|
'minimum_price_de': item.get('minimumPriceDe'),
|
||||||
'asin_uk': item.get('asinUk') or '',
|
'asin_uk': item.get('asinUk') or '',
|
||||||
|
'minimum_price_uk': item.get('minimumPriceUk'),
|
||||||
'asin_fr': item.get('asinFr') or '',
|
'asin_fr': item.get('asinFr') or '',
|
||||||
|
'minimum_price_fr': item.get('minimumPriceFr'),
|
||||||
'asin_it': item.get('asinIt') or '',
|
'asin_it': item.get('asinIt') or '',
|
||||||
|
'minimum_price_it': item.get('minimumPriceIt'),
|
||||||
'asin_es': item.get('asinEs') or '',
|
'asin_es': item.get('asinEs') or '',
|
||||||
|
'minimum_price_es': item.get('minimumPriceEs'),
|
||||||
'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16],
|
'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16],
|
||||||
'updated_at': (item.get('updatedAt') or '').replace('T', ' ')[:16],
|
'updated_at': (item.get('updatedAt') or '').replace('T', ' ')[:16],
|
||||||
},
|
},
|
||||||
@@ -1739,9 +1855,9 @@ def create_skip_price_asin():
|
|||||||
|
|
||||||
|
|
||||||
@admin_api.route('/skip-price-asin/<int:item_id>/country/<country>', methods=['DELETE'])
|
@admin_api.route('/skip-price-asin/<int:item_id>/country/<country>', methods=['DELETE'])
|
||||||
@admin_required
|
@login_required
|
||||||
def delete_skip_price_asin_country(item_id, country):
|
def delete_skip_price_asin_country(item_id, country):
|
||||||
role, current_row, denied = _ensure_admin_menu_access('skip-price-asin')
|
role, current_row, denied = _ensure_backend_menu_access('skip-price-asin')
|
||||||
if denied:
|
if denied:
|
||||||
return denied
|
return denied
|
||||||
result, error_response, status = _proxy_backend_java(
|
result, error_response, status = _proxy_backend_java(
|
||||||
@@ -1758,9 +1874,9 @@ def delete_skip_price_asin_country(item_id, country):
|
|||||||
|
|
||||||
|
|
||||||
@admin_api.route('/shop-manage-group/<int:item_id>', methods=['DELETE'])
|
@admin_api.route('/shop-manage-group/<int:item_id>', methods=['DELETE'])
|
||||||
@admin_required
|
@login_required
|
||||||
def delete_shop_manage_group(item_id):
|
def delete_shop_manage_group(item_id):
|
||||||
role, current_row, denied = _ensure_admin_menu_access('shop-manage', 'skip-price-asin')
|
role, current_row, denied = _ensure_backend_menu_access('shop-manage', 'skip-price-asin')
|
||||||
if denied:
|
if denied:
|
||||||
return denied
|
return denied
|
||||||
result, error_response, status = _proxy_backend_java(
|
result, error_response, status = _proxy_backend_java(
|
||||||
@@ -1777,13 +1893,16 @@ def delete_shop_manage_group(item_id):
|
|||||||
|
|
||||||
|
|
||||||
@admin_api.route('/skip-price-asin/<int:item_id>/country/<country>', methods=['PUT'])
|
@admin_api.route('/skip-price-asin/<int:item_id>/country/<country>', methods=['PUT'])
|
||||||
@admin_required
|
@login_required
|
||||||
def update_skip_price_asin_country(item_id, country):
|
def update_skip_price_asin_country(item_id, country):
|
||||||
role, current_row, denied = _ensure_admin_menu_access('skip-price-asin')
|
role, current_row, denied = _ensure_backend_menu_access('skip-price-asin')
|
||||||
if denied:
|
if denied:
|
||||||
return denied
|
return denied
|
||||||
data = request.get_json() or {}
|
data = request.get_json() or {}
|
||||||
payload = {'asin': (data.get('asin') or '').strip()}
|
payload = {
|
||||||
|
'asin': (data.get('asin') or '').strip(),
|
||||||
|
'minimumPrice': data.get('minimum_price'),
|
||||||
|
}
|
||||||
result, error_response, status = _proxy_backend_java(
|
result, error_response, status = _proxy_backend_java(
|
||||||
'PUT',
|
'PUT',
|
||||||
f'/api/admin/skip-price-asins/{item_id}/countries/{country}',
|
f'/api/admin/skip-price-asins/{item_id}/countries/{country}',
|
||||||
@@ -1805,10 +1924,15 @@ def update_skip_price_asin_country(item_id, country):
|
|||||||
'group_name': item.get('groupName') or '',
|
'group_name': item.get('groupName') or '',
|
||||||
'shop_name': item.get('shopName') or '',
|
'shop_name': item.get('shopName') or '',
|
||||||
'asin_de': item.get('asinDe') or '',
|
'asin_de': item.get('asinDe') or '',
|
||||||
|
'minimum_price_de': item.get('minimumPriceDe'),
|
||||||
'asin_uk': item.get('asinUk') or '',
|
'asin_uk': item.get('asinUk') or '',
|
||||||
|
'minimum_price_uk': item.get('minimumPriceUk'),
|
||||||
'asin_fr': item.get('asinFr') or '',
|
'asin_fr': item.get('asinFr') or '',
|
||||||
|
'minimum_price_fr': item.get('minimumPriceFr'),
|
||||||
'asin_it': item.get('asinIt') or '',
|
'asin_it': item.get('asinIt') or '',
|
||||||
|
'minimum_price_it': item.get('minimumPriceIt'),
|
||||||
'asin_es': item.get('asinEs') or '',
|
'asin_es': item.get('asinEs') or '',
|
||||||
|
'minimum_price_es': item.get('minimumPriceEs'),
|
||||||
'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16],
|
'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16],
|
||||||
'updated_at': (item.get('updatedAt') or '').replace('T', ' ')[:16],
|
'updated_at': (item.get('updatedAt') or '').replace('T', ' ')[:16],
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ from flask import Blueprint, request, redirect, url_for, session, jsonify
|
|||||||
from werkzeug.security import check_password_hash
|
from werkzeug.security import check_password_hash
|
||||||
|
|
||||||
from utils.db import get_db
|
from utils.db import get_db
|
||||||
from utils.auth import login_required, is_session_user_valid, is_current_user_admin
|
from utils.auth import login_required, is_session_user_valid
|
||||||
from utils.render import render_html
|
from utils.render import render_html
|
||||||
|
|
||||||
auth = Blueprint('auth', __name__, url_prefix='')
|
auth = Blueprint('auth', __name__, url_prefix='')
|
||||||
@@ -13,7 +13,7 @@ auth = Blueprint('auth', __name__, url_prefix='')
|
|||||||
|
|
||||||
@auth.route('/login', methods=['GET', 'POST'])
|
@auth.route('/login', methods=['GET', 'POST'])
|
||||||
def login():
|
def login():
|
||||||
if session.get('user_id') and is_session_user_valid() and is_current_user_admin():
|
if session.get('user_id') and is_session_user_valid():
|
||||||
return redirect(url_for('main.admin_page'))
|
return redirect(url_for('main.admin_page'))
|
||||||
if request.method == 'POST':
|
if request.method == 'POST':
|
||||||
data = request.get_json() if request.is_json else request.form
|
data = request.get_json() if request.is_json else request.form
|
||||||
@@ -31,22 +31,17 @@ def login():
|
|||||||
(username,)
|
(username,)
|
||||||
)
|
)
|
||||||
row = cur.fetchone()
|
row = cur.fetchone()
|
||||||
|
conn.close()
|
||||||
if row and check_password_hash(row['password_hash'], password):
|
if row and check_password_hash(row['password_hash'], password):
|
||||||
session.permanent = True
|
session.permanent = True
|
||||||
session['user_id'] = row['id']
|
session['user_id'] = row['id']
|
||||||
session['username'] = username
|
session['username'] = username
|
||||||
if not row.get('is_admin'):
|
|
||||||
session.clear()
|
|
||||||
if request.is_json:
|
|
||||||
return jsonify({'success': False, 'error': '需要管理员权限'}), 403
|
|
||||||
return render_html('login.html', error='需要管理员权限')
|
|
||||||
if request.is_json:
|
if request.is_json:
|
||||||
return jsonify({'success': True, 'redirect': url_for('main.admin_page')})
|
return jsonify({'success': True, 'redirect': url_for('main.admin_page')})
|
||||||
return redirect(url_for('main.admin_page'))
|
return redirect(url_for('main.admin_page'))
|
||||||
conn.close()
|
except Exception as exc:
|
||||||
except Exception as e:
|
|
||||||
if request.is_json:
|
if request.is_json:
|
||||||
return jsonify({'success': False, 'error': str(e)})
|
return jsonify({'success': False, 'error': str(exc)})
|
||||||
return render_html('login.html', error='登录失败,请稍后重试')
|
return render_html('login.html', error='登录失败,请稍后重试')
|
||||||
if request.is_json:
|
if request.is_json:
|
||||||
return jsonify({'success': False, 'error': '用户名或密码错误'})
|
return jsonify({'success': False, 'error': '用户名或密码错误'})
|
||||||
@@ -57,7 +52,7 @@ def login():
|
|||||||
@auth.route('/api/auth/check')
|
@auth.route('/api/auth/check')
|
||||||
@login_required
|
@login_required
|
||||||
def api_auth_check():
|
def api_auth_check():
|
||||||
"""校验登录状态,用于页面加载时判断是否已登录"""
|
"""校验登录状态,用于页面加载时判断是否已登录。"""
|
||||||
if not session.get('user_id'):
|
if not session.get('user_id'):
|
||||||
return jsonify({'logged_in': False})
|
return jsonify({'logged_in': False})
|
||||||
try:
|
try:
|
||||||
@@ -66,7 +61,7 @@ def api_auth_check():
|
|||||||
cur.execute("SELECT machine, is_admin FROM users WHERE id = %s", (session['user_id'],))
|
cur.execute("SELECT machine, is_admin FROM users WHERE id = %s", (session['user_id'],))
|
||||||
row = cur.fetchone()
|
row = cur.fetchone()
|
||||||
conn.close()
|
conn.close()
|
||||||
if not row or not row.get('is_admin'):
|
if not row:
|
||||||
return jsonify({'logged_in': False})
|
return jsonify({'logged_in': False})
|
||||||
except Exception:
|
except Exception:
|
||||||
return jsonify({'logged_in': False})
|
return jsonify({'logged_in': False})
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
import os
|
import os
|
||||||
from flask import Blueprint, redirect, url_for, send_file, session
|
from flask import Blueprint, redirect, url_for, send_file, session
|
||||||
|
|
||||||
from utils.auth import login_required, admin_required, is_session_user_valid, is_current_user_admin
|
from utils.auth import login_required, is_session_user_valid
|
||||||
from utils.render import render_html
|
from utils.render import render_html
|
||||||
|
|
||||||
main = Blueprint('main', __name__, url_prefix='')
|
main = Blueprint('main', __name__, url_prefix='')
|
||||||
@@ -15,14 +15,13 @@ STATIC_DIR = os.path.join(BASE_DIR, 'static')
|
|||||||
|
|
||||||
@main.route('/')
|
@main.route('/')
|
||||||
def index():
|
def index():
|
||||||
if session.get('user_id') and is_session_user_valid() and is_current_user_admin():
|
if session.get('user_id') and is_session_user_valid():
|
||||||
return redirect(url_for('main.admin_page'))
|
return redirect(url_for('main.admin_page'))
|
||||||
return redirect(url_for('auth.login'))
|
return redirect(url_for('auth.login'))
|
||||||
|
|
||||||
|
|
||||||
@main.route('/admin')
|
@main.route('/admin')
|
||||||
@login_required
|
@login_required
|
||||||
@admin_required
|
|
||||||
def admin_page():
|
def admin_page():
|
||||||
return render_html('admin.html')
|
return render_html('admin.html')
|
||||||
|
|
||||||
|
|||||||
@@ -23,8 +23,8 @@ bucket_path = "nanri-image/"
|
|||||||
file_url_pre = f"https://{bucket}.oss-cn-hangzhou.aliyuncs.com/"
|
file_url_pre = f"https://{bucket}.oss-cn-hangzhou.aliyuncs.com/"
|
||||||
|
|
||||||
import os
|
import os
|
||||||
# backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://127.0.0.1:18080').rstrip('/')
|
backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://127.0.0.1:18080').rstrip('/')
|
||||||
backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://8.136.19.173:18080').rstrip('/')
|
# backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://8.136.19.173:18080').rstrip('/')
|
||||||
os.environ['OSS_ACCESS_KEY_ID'] = accessKeyId
|
os.environ['OSS_ACCESS_KEY_ID'] = accessKeyId
|
||||||
os.environ['OSS_ACCESS_KEY_SECRET'] = accessKeySecret
|
os.environ['OSS_ACCESS_KEY_SECRET'] = accessKeySecret
|
||||||
os.environ['SECRET_KEY'] = "ddffc7c1d02121d9554d7b080b2511b6"
|
os.environ['SECRET_KEY'] = "ddffc7c1d02121d9554d7b080b2511b6"
|
||||||
|
|||||||
98
backend/run.sh
Normal file
98
backend/run.sh
Normal file
@@ -0,0 +1,98 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
PORT="${PORT:-15124}"
|
||||||
|
HOST="${HOST:-0.0.0.0}"
|
||||||
|
APP_MODULE="${APP_MODULE:-app.py}"
|
||||||
|
PID_FILE="${PID_FILE:-backend.pid}"
|
||||||
|
LOG_FILE="${LOG_FILE:-backend.log}"
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
cd "$SCRIPT_DIR"
|
||||||
|
|
||||||
|
echo "检查端口 ${PORT} ..."
|
||||||
|
if command -v ss >/dev/null 2>&1; then
|
||||||
|
if ss -ltn | awk '{print $4}' | grep -Eq "(^|:)${PORT}$"; then
|
||||||
|
echo "错误: 端口 ${PORT} 已被占用,请先释放后再启动。"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
elif command -v netstat >/dev/null 2>&1; then
|
||||||
|
if netstat -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "(^|:)${PORT}$"; then
|
||||||
|
echo "错误: 端口 ${PORT} 已被占用,请先释放后再启动。"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo "警告: 未找到 ss 或 netstat,跳过端口占用检查。"
|
||||||
|
fi
|
||||||
|
echo "端口 ${PORT} 未被占用。"
|
||||||
|
echo "工作目录已切换至: ${SCRIPT_DIR}"
|
||||||
|
|
||||||
|
PYTHON_BIN=""
|
||||||
|
if [[ -x "${SCRIPT_DIR}/venv/bin/python" ]]; then
|
||||||
|
PYTHON_BIN="${SCRIPT_DIR}/venv/bin/python"
|
||||||
|
elif [[ -x "${SCRIPT_DIR}/.venv/bin/python" ]]; then
|
||||||
|
PYTHON_BIN="${SCRIPT_DIR}/.venv/bin/python"
|
||||||
|
elif command -v python3 >/dev/null 2>&1; then
|
||||||
|
PYTHON_BIN="$(command -v python3)"
|
||||||
|
elif command -v python >/dev/null 2>&1; then
|
||||||
|
PYTHON_BIN="$(command -v python)"
|
||||||
|
else
|
||||||
|
echo "错误: 未找到可用的 Python,请先安装 Python 3.9+。"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "使用 Python: ${PYTHON_BIN}"
|
||||||
|
"${PYTHON_BIN}" - <<'PY'
|
||||||
|
import sys
|
||||||
|
major, minor = sys.version_info[:2]
|
||||||
|
if (major, minor) < (3, 9):
|
||||||
|
print(f"错误: 当前 Python 版本为 {major}.{minor},项目至少需要 Python 3.9。")
|
||||||
|
print("原因: requirement.txt 中的 Flask 3.x、pandas 2.x 等依赖不支持 Python 3.6。")
|
||||||
|
raise SystemExit(1)
|
||||||
|
print(f"Python 版本检查通过: {major}.{minor}")
|
||||||
|
PY
|
||||||
|
|
||||||
|
if [[ ! -f "requirement.txt" ]]; then
|
||||||
|
echo "错误: 未找到 requirement.txt。"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "检查关键依赖 ..."
|
||||||
|
if ! "${PYTHON_BIN}" - <<'PY'
|
||||||
|
import importlib.util
|
||||||
|
modules = ("flask", "flask_cors", "pymysql", "werkzeug")
|
||||||
|
missing = [name for name in modules if importlib.util.find_spec(name) is None]
|
||||||
|
if missing:
|
||||||
|
print("缺少依赖: " + ", ".join(missing))
|
||||||
|
raise SystemExit(1)
|
||||||
|
print("关键依赖已安装。")
|
||||||
|
PY
|
||||||
|
then
|
||||||
|
echo "开始安装 requirement.txt ..."
|
||||||
|
"${PYTHON_BIN}" -m pip install --upgrade pip
|
||||||
|
"${PYTHON_BIN}" -m pip install -r requirement.txt
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -f "${PID_FILE}" ]]; then
|
||||||
|
OLD_PID="$(cat "${PID_FILE}" 2>/dev/null || true)"
|
||||||
|
if [[ -n "${OLD_PID}" ]] && kill -0 "${OLD_PID}" 2>/dev/null; then
|
||||||
|
echo "错误: 检测到服务已在运行,PID=${OLD_PID}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
rm -f "${PID_FILE}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "启动 Flask 服务 ${HOST}:${PORT} ..."
|
||||||
|
nohup "${PYTHON_BIN}" "${APP_MODULE}" >> "${LOG_FILE}" 2>&1 &
|
||||||
|
NEW_PID=$!
|
||||||
|
echo "${NEW_PID}" > "${PID_FILE}"
|
||||||
|
|
||||||
|
sleep 2
|
||||||
|
if kill -0 "${NEW_PID}" 2>/dev/null; then
|
||||||
|
echo "启动成功,PID=${NEW_PID}"
|
||||||
|
echo "日志文件: ${SCRIPT_DIR}/${LOG_FILE}"
|
||||||
|
else
|
||||||
|
echo "错误: 服务启动失败,请检查日志 ${SCRIPT_DIR}/${LOG_FILE}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
Binary file not shown.
@@ -42,9 +42,13 @@ def get_current_admin_role():
|
|||||||
)
|
)
|
||||||
row = cur.fetchone()
|
row = cur.fetchone()
|
||||||
conn.close()
|
conn.close()
|
||||||
if not row or not row.get('is_admin'):
|
if not row:
|
||||||
return None, row
|
return None, None
|
||||||
role = row.get('role') or ('super_admin' if row.get('created_by_id') is None else 'admin')
|
role = (row.get('role') or '').strip().lower()
|
||||||
|
if not role:
|
||||||
|
role = 'super_admin' if row.get('is_admin') and row.get('created_by_id') is None else (
|
||||||
|
'admin' if row.get('is_admin') else 'normal'
|
||||||
|
)
|
||||||
return role, row
|
return role, row
|
||||||
except Exception:
|
except Exception:
|
||||||
return None, None
|
return None, None
|
||||||
@@ -56,7 +60,14 @@ def is_current_user_admin():
|
|||||||
|
|
||||||
|
|
||||||
def _is_ajax_request():
|
def _is_ajax_request():
|
||||||
return request.headers.get('X-Requested-With') == 'XMLHttpRequest'
|
if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
|
||||||
|
return True
|
||||||
|
if request.path.startswith('/api/'):
|
||||||
|
return True
|
||||||
|
accept = (request.headers.get('Accept') or '').lower()
|
||||||
|
if 'application/json' in accept:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def login_required(f):
|
def login_required(f):
|
||||||
|
|||||||
@@ -769,10 +769,10 @@
|
|||||||
<option value="ES">西班牙</option>
|
<option value="ES">西班牙</option>
|
||||||
</select>
|
</select>
|
||||||
</div>
|
</div>
|
||||||
<div class="form-group" style="min-width:260px;flex:1;">
|
<div class="form-group" style="min-width:320px;flex:1;">
|
||||||
<label>ASIN</label>
|
<label>ASIN / 最低价</label>
|
||||||
<div id="skipPriceAsinInputs" style="display:flex;flex-direction:column;gap:8px;">
|
<div id="skipPriceAsinInputs" style="display:flex;flex-direction:column;gap:8px;">
|
||||||
<div style="color:#999;font-size:13px;">请选择国家后输入对应 ASIN</div>
|
<div style="color:#999;font-size:13px;">请选择国家后输入对应 ASIN 和最低价</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<button class="btn" id="btnCreateSkipPriceAsin">新增 ASIN</button>
|
<button class="btn" id="btnCreateSkipPriceAsin">新增 ASIN</button>
|
||||||
@@ -966,7 +966,7 @@
|
|||||||
<label>组员</label>
|
<label>组员</label>
|
||||||
<select id="shopManageGroupMemberSelect" multiple size="6" style="min-height:140px;">
|
<select id="shopManageGroupMemberSelect" multiple size="6" style="min-height:140px;">
|
||||||
</select>
|
</select>
|
||||||
<div style="color:#888;font-size:12px;margin-top:6px;">可添加当前组长创建的普通员工账号,按住 Ctrl 或 Command 可多选。</div>
|
<div id="shopManageGroupMemberHelp" style="color:#888;font-size:12px;margin-top:6px;">可添加当前组长创建的普通员工账号,按住 Ctrl 或 Command 可多选。</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div style="display:flex;gap:8px;justify-content:flex-end;margin-bottom:12px;">
|
<div style="display:flex;gap:8px;justify-content:flex-end;margin-bottom:12px;">
|
||||||
@@ -1034,6 +1034,35 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div class="modal-mask" id="editSkipPriceAsinModal">
|
||||||
|
<div class="modal">
|
||||||
|
<h3>编辑跳过跟价 ASIN</h3>
|
||||||
|
<input type="hidden" id="editSkipPriceAsinId">
|
||||||
|
<input type="hidden" id="editSkipPriceAsinCountry">
|
||||||
|
<div class="form-group">
|
||||||
|
<label>店铺名</label>
|
||||||
|
<input type="text" id="editSkipPriceAsinShopName" readonly style="background:#f5f5f5;">
|
||||||
|
</div>
|
||||||
|
<div class="form-group">
|
||||||
|
<label>国家</label>
|
||||||
|
<input type="text" id="editSkipPriceAsinCountryLabel" readonly style="background:#f5f5f5;">
|
||||||
|
</div>
|
||||||
|
<div class="form-group">
|
||||||
|
<label>ASIN</label>
|
||||||
|
<input type="text" id="editSkipPriceAsinValue" placeholder="请输入 ASIN">
|
||||||
|
</div>
|
||||||
|
<div class="form-group">
|
||||||
|
<label>最低价</label>
|
||||||
|
<input type="number" id="editSkipPriceMinimumPrice" min="0" step="0.01" placeholder="请输入最低价,可留空">
|
||||||
|
</div>
|
||||||
|
<p class="msg" id="msgEditSkipPriceAsin"></p>
|
||||||
|
<div style="margin-top:16px;display:flex;gap:8px;">
|
||||||
|
<button class="btn" id="btnSaveSkipPriceAsin">保存</button>
|
||||||
|
<button class="btn btn-secondary" id="btnCloseEditSkipPriceAsin">取消</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div class="modal-mask" id="editShopManageModal">
|
<div class="modal-mask" id="editShopManageModal">
|
||||||
<div class="modal">
|
<div class="modal">
|
||||||
<h3>编辑店铺</h3>
|
<h3>编辑店铺</h3>
|
||||||
@@ -1655,22 +1684,6 @@
|
|||||||
document.getElementById('historyListBody').innerHTML = '<tr><td colspan="5" class="empty-tip">请求失败</td></tr>';
|
document.getElementById('historyListBody').innerHTML = '<tr><td colspan="5" class="empty-tip">请求失败</td></tr>';
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
function loadUserOptions() {
|
|
||||||
fetch('/api/admin/users?page=1&page_size=999')
|
|
||||||
.then(function (r) { return r.json(); })
|
|
||||||
.then(function (res) {
|
|
||||||
var sel = document.getElementById('filterUser');
|
|
||||||
var cur = sel.value;
|
|
||||||
sel.innerHTML = '<option value="">全部用户</option>';
|
|
||||||
(res.items || []).forEach(function (u) {
|
|
||||||
var opt = document.createElement('option');
|
|
||||||
opt.value = u.id;
|
|
||||||
opt.textContent = u.username + ' (' + roleLabel(u.role || 'normal') + ')';
|
|
||||||
sel.appendChild(opt);
|
|
||||||
});
|
|
||||||
sel.value = cur || '';
|
|
||||||
});
|
|
||||||
}
|
|
||||||
var shopManageAllUsers = [];
|
var shopManageAllUsers = [];
|
||||||
function getEligibleShopManageGroupUsers(leaderUserId) {
|
function getEligibleShopManageGroupUsers(leaderUserId) {
|
||||||
var canViewAllMembers = currentUserRole === 'super_admin';
|
var canViewAllMembers = currentUserRole === 'super_admin';
|
||||||
@@ -1683,15 +1696,29 @@
|
|||||||
}
|
}
|
||||||
function refreshShopManageGroupMemberSelect(leaderUserId, selectedUserIds) {
|
function refreshShopManageGroupMemberSelect(leaderUserId, selectedUserIds) {
|
||||||
var sel = document.getElementById('shopManageGroupMemberSelect');
|
var sel = document.getElementById('shopManageGroupMemberSelect');
|
||||||
|
var helpEl = document.getElementById('shopManageGroupMemberHelp');
|
||||||
if (!sel) return;
|
if (!sel) return;
|
||||||
var selectedMap = {};
|
var selectedMap = {};
|
||||||
(selectedUserIds || []).forEach(function (id) {
|
(selectedUserIds || []).forEach(function (id) {
|
||||||
selectedMap[String(id)] = true;
|
selectedMap[String(id)] = true;
|
||||||
});
|
});
|
||||||
var options = getEligibleShopManageGroupUsers(leaderUserId).map(function (u) {
|
var eligibleUsers = getEligibleShopManageGroupUsers(leaderUserId);
|
||||||
|
var options = eligibleUsers.map(function (u) {
|
||||||
return '<option value="' + u.id + '"' + (selectedMap[String(u.id)] ? ' selected' : '') + '>' + (u.username || '') + '</option>';
|
return '<option value="' + u.id + '"' + (selectedMap[String(u.id)] ? ' selected' : '') + '>' + (u.username || '') + '</option>';
|
||||||
});
|
});
|
||||||
|
if (options.length) {
|
||||||
sel.innerHTML = options.join('');
|
sel.innerHTML = options.join('');
|
||||||
|
if (helpEl) {
|
||||||
|
helpEl.textContent = '可添加当前组长创建的普通员工账号,按住 Ctrl 或 Command 可多选。';
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
sel.innerHTML = '<option value="" disabled>当前组长暂无可添加组员</option>';
|
||||||
|
if (helpEl) {
|
||||||
|
helpEl.textContent = currentUserRole === 'normal'
|
||||||
|
? '普通账号没有下属普通员工时,这里会为空;当前账号只能作为组长使用。'
|
||||||
|
: '当前组长名下暂无可添加的普通员工账号。';
|
||||||
|
}
|
||||||
}
|
}
|
||||||
function setShopManageGroupLeader(leaderUserId, leaderUsername, selectedUserIds) {
|
function setShopManageGroupLeader(leaderUserId, leaderUsername, selectedUserIds) {
|
||||||
var leaderIdEl = document.getElementById('shopManageGroupLeaderUserId');
|
var leaderIdEl = document.getElementById('shopManageGroupLeaderUserId');
|
||||||
@@ -1712,6 +1739,9 @@
|
|||||||
return fetch('/api/admin/users?page=1&page_size=999')
|
return fetch('/api/admin/users?page=1&page_size=999')
|
||||||
.then(function (r) { return r.json(); })
|
.then(function (r) { return r.json(); })
|
||||||
.then(function (res) {
|
.then(function (res) {
|
||||||
|
if (!res.success) {
|
||||||
|
throw new Error(res.error || '加载用户失败');
|
||||||
|
}
|
||||||
var items = res.items || [];
|
var items = res.items || [];
|
||||||
shopManageAllUsers = items.map(function (u) {
|
shopManageAllUsers = items.map(function (u) {
|
||||||
return {
|
return {
|
||||||
@@ -1736,6 +1766,17 @@
|
|||||||
sel.appendChild(opt);
|
sel.appendChild(opt);
|
||||||
});
|
});
|
||||||
sel.value = cur || '';
|
sel.value = cur || '';
|
||||||
|
})
|
||||||
|
.catch(function () {
|
||||||
|
shopManageAllUsers = [];
|
||||||
|
var sel = document.getElementById('filterUser');
|
||||||
|
if (sel) {
|
||||||
|
sel.innerHTML = '<option value="">全部用户</option>';
|
||||||
|
}
|
||||||
|
refreshShopManageGroupMemberSelect(
|
||||||
|
document.getElementById('shopManageGroupLeaderUserId') ? document.getElementById('shopManageGroupLeaderUserId').value : '',
|
||||||
|
[]
|
||||||
|
);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
document.getElementById('btnFilterHistory').onclick = function () { loadHistory(1); };
|
document.getElementById('btnFilterHistory').onclick = function () { loadHistory(1); };
|
||||||
@@ -2184,6 +2225,7 @@
|
|||||||
// ========== 店铺管理 ==========
|
// ========== 店铺管理 ==========
|
||||||
var shopManagePage = 1, shopManagePageSize = 15;
|
var shopManagePage = 1, shopManagePageSize = 15;
|
||||||
var shopManageGroups = [];
|
var shopManageGroups = [];
|
||||||
|
var currentShopManageGroupGrantRoutes = [];
|
||||||
|
|
||||||
function buildShopManageQuery(page) {
|
function buildShopManageQuery(page) {
|
||||||
var query = 'page=' + (page || 1) + '&page_size=' + shopManagePageSize;
|
var query = 'page=' + (page || 1) + '&page_size=' + shopManagePageSize;
|
||||||
@@ -2313,6 +2355,18 @@
|
|||||||
refreshShopManageGroupMemberSelect(currentUserId, []);
|
refreshShopManageGroupMemberSelect(currentUserId, []);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function setShopManageGroupGrantRoutes(routes) {
|
||||||
|
currentShopManageGroupGrantRoutes = Array.isArray(routes) ? routes.slice() : [];
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateShopManageGroupButtonsAccess() {
|
||||||
|
var canManageGroups = !!currentUserId;
|
||||||
|
['btnManageShopGroups', 'btnManageShopGroupsFromEdit', 'btnManageSkipPriceAsinGroups'].forEach(function (id) {
|
||||||
|
var btn = document.getElementById(id);
|
||||||
|
if (btn) btn.style.display = canManageGroups ? '' : 'none';
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
function openShopManageGroupModal() {
|
function openShopManageGroupModal() {
|
||||||
resetShopManageGroupForm();
|
resetShopManageGroupForm();
|
||||||
loadUserOptions()
|
loadUserOptions()
|
||||||
@@ -2331,10 +2385,13 @@
|
|||||||
}
|
}
|
||||||
tbody.innerHTML = shopManageGroups.map(function (item, index) {
|
tbody.innerHTML = shopManageGroups.map(function (item, index) {
|
||||||
var memberNames = Array.isArray(item.member_usernames) ? item.member_usernames.join('、') : '';
|
var memberNames = Array.isArray(item.member_usernames) ? item.member_usernames.join('、') : '';
|
||||||
|
var canEdit = currentUserRole === 'super_admin' || String(item.leader_user_id || '') === String(currentUserId || '');
|
||||||
|
var actionHtml = canEdit
|
||||||
|
? ('<button class="btn btn-sm" data-shop-group-edit="' + item.id + '">编辑</button> ' +
|
||||||
|
'<button class="btn btn-sm btn-danger" data-shop-group-delete="' + item.id + '" data-shop-group-name="' + (item.group_name || '').replace(/"/g, '"') + '">删除</button>')
|
||||||
|
: '<span style="color:#999;">-</span>';
|
||||||
return '<tr><td>' + (index + 1) + '</td><td>' + (item.group_name || '') + '</td><td>' + (item.leader_username || '') + '</td><td>' + (item.member_count || 0) + '</td><td>' + (memberNames || '-') + '</td><td>' + (item.created_at || '') + '</td><td>' + (item.updated_at || '') + '</td><td>' +
|
return '<tr><td>' + (index + 1) + '</td><td>' + (item.group_name || '') + '</td><td>' + (item.leader_username || '') + '</td><td>' + (item.member_count || 0) + '</td><td>' + (memberNames || '-') + '</td><td>' + (item.created_at || '') + '</td><td>' + (item.updated_at || '') + '</td><td>' +
|
||||||
'<button class="btn btn-sm" data-shop-group-edit="' + item.id + '">编辑</button> ' +
|
actionHtml + '</td></tr>';
|
||||||
'<button class="btn btn-sm btn-danger" data-shop-group-delete="' + item.id + '" data-shop-group-name="' + (item.group_name || '').replace(/"/g, '"') + '">删除</button>' +
|
|
||||||
'</td></tr>';
|
|
||||||
}).join('');
|
}).join('');
|
||||||
|
|
||||||
document.querySelectorAll('[data-shop-group-edit]').forEach(function (btn) {
|
document.querySelectorAll('[data-shop-group-edit]').forEach(function (btn) {
|
||||||
@@ -2378,9 +2435,18 @@
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
document.getElementById('btnManageShopGroups').onclick = openShopManageGroupModal;
|
document.getElementById('btnManageShopGroups').onclick = function () {
|
||||||
document.getElementById('btnManageShopGroupsFromEdit').onclick = openShopManageGroupModal;
|
setShopManageGroupGrantRoutes(['shop-manage']);
|
||||||
document.getElementById('btnManageSkipPriceAsinGroups').onclick = openShopManageGroupModal;
|
openShopManageGroupModal();
|
||||||
|
};
|
||||||
|
document.getElementById('btnManageShopGroupsFromEdit').onclick = function () {
|
||||||
|
setShopManageGroupGrantRoutes(['shop-manage']);
|
||||||
|
openShopManageGroupModal();
|
||||||
|
};
|
||||||
|
document.getElementById('btnManageSkipPriceAsinGroups').onclick = function () {
|
||||||
|
setShopManageGroupGrantRoutes(['skip-price-asin']);
|
||||||
|
openShopManageGroupModal();
|
||||||
|
};
|
||||||
document.getElementById('btnSearchShopManage').onclick = function () {
|
document.getElementById('btnSearchShopManage').onclick = function () {
|
||||||
loadShopManage(1);
|
loadShopManage(1);
|
||||||
};
|
};
|
||||||
@@ -2411,7 +2477,11 @@
|
|||||||
fetch(url, {
|
fetch(url, {
|
||||||
method: method,
|
method: method,
|
||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
body: JSON.stringify({ group_name: groupName, member_user_ids: memberUserIds })
|
body: JSON.stringify({
|
||||||
|
group_name: groupName,
|
||||||
|
member_user_ids: memberUserIds,
|
||||||
|
grant_menu_routes: currentShopManageGroupGrantRoutes
|
||||||
|
})
|
||||||
})
|
})
|
||||||
.then(function (r) { return r.json(); })
|
.then(function (r) { return r.json(); })
|
||||||
.then(function (res) {
|
.then(function (res) {
|
||||||
@@ -2520,11 +2590,11 @@
|
|||||||
var skipPriceAsinPage = 1, skipPriceAsinPageSize = 15;
|
var skipPriceAsinPage = 1, skipPriceAsinPageSize = 15;
|
||||||
var chooseSkipPriceAsinShopPage = 1, chooseSkipPriceAsinShopPageSize = 10;
|
var chooseSkipPriceAsinShopPage = 1, chooseSkipPriceAsinShopPageSize = 10;
|
||||||
var skipPriceCountryColumns = [
|
var skipPriceCountryColumns = [
|
||||||
{ code: 'DE', field: 'asin_de', label: '德国' },
|
{ code: 'DE', field: 'asin_de', minimumPriceField: 'minimum_price_de', label: '德国' },
|
||||||
{ code: 'UK', field: 'asin_uk', label: '英国' },
|
{ code: 'UK', field: 'asin_uk', minimumPriceField: 'minimum_price_uk', label: '英国' },
|
||||||
{ code: 'FR', field: 'asin_fr', label: '法国' },
|
{ code: 'FR', field: 'asin_fr', minimumPriceField: 'minimum_price_fr', label: '法国' },
|
||||||
{ code: 'IT', field: 'asin_it', label: '意大利' },
|
{ code: 'IT', field: 'asin_it', minimumPriceField: 'minimum_price_it', label: '意大利' },
|
||||||
{ code: 'ES', field: 'asin_es', label: '西班牙' }
|
{ code: 'ES', field: 'asin_es', minimumPriceField: 'minimum_price_es', label: '西班牙' }
|
||||||
];
|
];
|
||||||
function buildChooseSkipPriceAsinShopQuery(page) {
|
function buildChooseSkipPriceAsinShopQuery(page) {
|
||||||
var query = 'page=' + (page || 1) + '&page_size=' + chooseSkipPriceAsinShopPageSize;
|
var query = 'page=' + (page || 1) + '&page_size=' + chooseSkipPriceAsinShopPageSize;
|
||||||
@@ -2607,41 +2677,68 @@
|
|||||||
return option.value;
|
return option.value;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
function formatSkipPriceMinimumPrice(value) {
|
||||||
|
if (value === null || value === undefined || value === '') return '';
|
||||||
|
var num = Number(value);
|
||||||
|
if (!isFinite(num)) return String(value);
|
||||||
|
return num.toFixed(2);
|
||||||
|
}
|
||||||
|
function getSkipPriceCountryLabel(countryCode) {
|
||||||
|
var country = skipPriceCountryColumns.find(function (item) { return item.code === countryCode; });
|
||||||
|
return country ? country.label : countryCode;
|
||||||
|
}
|
||||||
function renderSkipPriceAsinInputs() {
|
function renderSkipPriceAsinInputs() {
|
||||||
var container = document.getElementById('skipPriceAsinInputs');
|
var container = document.getElementById('skipPriceAsinInputs');
|
||||||
var selectedCountries = getSelectedSkipPriceCountries();
|
var selectedCountries = getSelectedSkipPriceCountries();
|
||||||
var existingValues = {};
|
var existingValues = {};
|
||||||
|
var existingMinimumPrices = {};
|
||||||
container.querySelectorAll('[data-skip-price-country-input]').forEach(function (input) {
|
container.querySelectorAll('[data-skip-price-country-input]').forEach(function (input) {
|
||||||
existingValues[input.getAttribute('data-skip-price-country-input')] = input.value;
|
existingValues[input.getAttribute('data-skip-price-country-input')] = input.value;
|
||||||
});
|
});
|
||||||
|
container.querySelectorAll('[data-skip-price-country-minimum-price-input]').forEach(function (input) {
|
||||||
|
existingMinimumPrices[input.getAttribute('data-skip-price-country-minimum-price-input')] = input.value;
|
||||||
|
});
|
||||||
if (!selectedCountries.length) {
|
if (!selectedCountries.length) {
|
||||||
container.innerHTML = '<div style="color:#999;font-size:13px;">请选择国家后输入 ASIN</div>';
|
container.innerHTML = '<div style="color:#999;font-size:13px;">请选择国家后输入 ASIN 和最低价</div>';
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
container.innerHTML = selectedCountries.map(function (countryCode) {
|
container.innerHTML = selectedCountries.map(function (countryCode) {
|
||||||
var country = skipPriceCountryColumns.find(function (item) { return item.code === countryCode; });
|
var label = getSkipPriceCountryLabel(countryCode);
|
||||||
var label = country ? country.label : countryCode;
|
|
||||||
var value = existingValues[countryCode] || '';
|
var value = existingValues[countryCode] || '';
|
||||||
|
var minimumPrice = existingMinimumPrices[countryCode] || '';
|
||||||
return '<div style="display:flex;align-items:center;gap:8px;flex-wrap:wrap;">' +
|
return '<div style="display:flex;align-items:center;gap:8px;flex-wrap:wrap;">' +
|
||||||
'<span style="min-width:56px;color:#555;">' + label + '</span>' +
|
'<span style="min-width:56px;color:#555;">' + label + '</span>' +
|
||||||
'<input type="text" data-skip-price-country-input="' + countryCode + '" value="' + value.replace(/"/g, '"') + '" placeholder="请输入' + label + ' ASIN" style="flex:1;min-width:180px;">' +
|
'<input type="text" data-skip-price-country-input="' + countryCode + '" value="' + value.replace(/"/g, '"') + '" placeholder="请输入' + label + ' ASIN" style="flex:1;min-width:180px;">' +
|
||||||
|
'<input type="number" data-skip-price-country-minimum-price-input="' + countryCode + '" value="' + minimumPrice.replace(/"/g, '"') + '" min="0" step="0.01" placeholder="最低价" style="width:140px;">' +
|
||||||
'</div>';
|
'</div>';
|
||||||
}).join('');
|
}).join('');
|
||||||
}
|
}
|
||||||
function collectSkipPriceAsinMappings(countries) {
|
function collectSkipPriceAsinMappings(countries) {
|
||||||
var mappings = {};
|
var asinMappings = {};
|
||||||
|
var minimumPriceMappings = {};
|
||||||
for (var i = 0; i < countries.length; i++) {
|
for (var i = 0; i < countries.length; i++) {
|
||||||
var countryCode = countries[i];
|
var countryCode = countries[i];
|
||||||
var input = document.querySelector('[data-skip-price-country-input="' + countryCode + '"]');
|
var input = document.querySelector('[data-skip-price-country-input="' + countryCode + '"]');
|
||||||
|
var minimumPriceInput = document.querySelector('[data-skip-price-country-minimum-price-input="' + countryCode + '"]');
|
||||||
var asin = input ? (input.value || '').trim().toUpperCase() : '';
|
var asin = input ? (input.value || '').trim().toUpperCase() : '';
|
||||||
if (!asin) {
|
if (!asin) {
|
||||||
var country = skipPriceCountryColumns.find(function (item) { return item.code === countryCode; });
|
var label = getSkipPriceCountryLabel(countryCode);
|
||||||
var label = country ? country.label : countryCode;
|
|
||||||
throw new Error(label + ' ASIN 不能为空');
|
throw new Error(label + ' ASIN 不能为空');
|
||||||
}
|
}
|
||||||
mappings[countryCode] = asin;
|
var minimumPrice = minimumPriceInput ? (minimumPriceInput.value || '').trim() : '';
|
||||||
|
if (minimumPrice) {
|
||||||
|
var minimumPriceNumber = Number(minimumPrice);
|
||||||
|
if (!isFinite(minimumPriceNumber) || minimumPriceNumber < 0) {
|
||||||
|
throw new Error(getSkipPriceCountryLabel(countryCode) + ' 最低价格式不正确');
|
||||||
}
|
}
|
||||||
return mappings;
|
minimumPriceMappings[countryCode] = minimumPrice;
|
||||||
|
}
|
||||||
|
asinMappings[countryCode] = asin;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
asinMappings: asinMappings,
|
||||||
|
minimumPriceMappings: minimumPriceMappings
|
||||||
|
};
|
||||||
}
|
}
|
||||||
function buildSkipPriceAsinQuery(page) {
|
function buildSkipPriceAsinQuery(page) {
|
||||||
var query = 'page=' + (page || 1) + '&page_size=' + skipPriceAsinPageSize;
|
var query = 'page=' + (page || 1) + '&page_size=' + skipPriceAsinPageSize;
|
||||||
@@ -2654,37 +2751,49 @@
|
|||||||
return query;
|
return query;
|
||||||
}
|
}
|
||||||
function renderSkipPriceAsinCell(item, country) {
|
function renderSkipPriceAsinCell(item, country) {
|
||||||
var value = item[country.field] || '';
|
var asinValue = item[country.field] || '';
|
||||||
if (!value) return '<span style="color:#999;">-</span>';
|
var minimumPriceValue = formatSkipPriceMinimumPrice(item[country.minimumPriceField]);
|
||||||
|
var hasValue = !!asinValue || !!minimumPriceValue;
|
||||||
|
var infoHtml = hasValue
|
||||||
|
? ('<div style="display:flex;flex-direction:column;gap:4px;min-width:0;">' +
|
||||||
|
'<span>' + (asinValue || '<span style="color:#999;">-</span>') + '</span>' +
|
||||||
|
'<span style="color:#666;font-size:12px;">最低价:' + (minimumPriceValue || '-') + '</span>' +
|
||||||
|
'</div>')
|
||||||
|
: '<span style="color:#999;">-</span>';
|
||||||
|
var deleteHtml = hasValue
|
||||||
|
? ('<button class="btn btn-sm btn-danger" data-skip-price-asin-delete="' + item.id + '" data-country="' + country.code + '" data-shop-name="' + (item.shop_name || '').replace(/"/g, '"') + '">删除</button>')
|
||||||
|
: '';
|
||||||
return '<div style="display:flex;align-items:center;gap:8px;flex-wrap:wrap;">' +
|
return '<div style="display:flex;align-items:center;gap:8px;flex-wrap:wrap;">' +
|
||||||
'<span>' + value + '</span>' +
|
infoHtml +
|
||||||
'<button class="btn btn-sm" data-skip-price-asin-edit="' + item.id + '" data-country="' + country.code + '" data-asin="' + value.replace(/"/g, '"') + '" data-shop-name="' + (item.shop_name || '').replace(/"/g, '"') + '">编辑</button>' +
|
'<button class="btn btn-sm" data-skip-price-asin-edit="' + item.id + '" data-country="' + country.code + '" data-asin="' + asinValue.replace(/"/g, '"') + '" data-minimum-price="' + minimumPriceValue.replace(/"/g, '"') + '" data-shop-name="' + (item.shop_name || '').replace(/"/g, '"') + '">编辑</button>' +
|
||||||
'<button class="btn btn-sm btn-danger" data-skip-price-asin-delete="' + item.id + '" data-country="' + country.code + '" data-shop-name="' + (item.shop_name || '').replace(/"/g, '"') + '">删除</button>' +
|
deleteHtml +
|
||||||
'</div>';
|
'</div>';
|
||||||
}
|
}
|
||||||
|
function openEditSkipPriceAsinModal(itemId, countryCode, shopName, asinValue, minimumPriceValue) {
|
||||||
|
document.getElementById('editSkipPriceAsinId').value = itemId || '';
|
||||||
|
document.getElementById('editSkipPriceAsinCountry').value = countryCode || '';
|
||||||
|
document.getElementById('editSkipPriceAsinShopName').value = shopName || '';
|
||||||
|
document.getElementById('editSkipPriceAsinCountryLabel').value = getSkipPriceCountryLabel(countryCode || '');
|
||||||
|
document.getElementById('editSkipPriceAsinValue').value = asinValue || '';
|
||||||
|
document.getElementById('editSkipPriceMinimumPrice').value = minimumPriceValue || '';
|
||||||
|
document.getElementById('msgEditSkipPriceAsin').textContent = '';
|
||||||
|
document.getElementById('msgEditSkipPriceAsin').className = 'msg';
|
||||||
|
document.getElementById('editSkipPriceAsinModal').classList.add('show');
|
||||||
|
}
|
||||||
function bindSkipPriceAsinActions() {
|
function bindSkipPriceAsinActions() {
|
||||||
document.querySelectorAll('[data-skip-price-asin-edit]').forEach(function (btn) {
|
document.querySelectorAll('[data-skip-price-asin-edit]').forEach(function (btn) {
|
||||||
btn.onclick = function () {
|
btn.onclick = function () {
|
||||||
var shopName = (btn.dataset.shopName || '').replace(/"/g, '"');
|
var shopName = (btn.dataset.shopName || '').replace(/"/g, '"');
|
||||||
var countryCode = btn.dataset.country || '';
|
var countryCode = btn.dataset.country || '';
|
||||||
var currentAsin = (btn.dataset.asin || '').replace(/"/g, '"');
|
var currentAsin = (btn.dataset.asin || '').replace(/"/g, '"');
|
||||||
var nextAsin = prompt('请输入店铺“' + shopName + '”在 ' + countryCode + ' 的 ASIN', currentAsin);
|
var currentMinimumPrice = (btn.dataset.minimumPrice || '').replace(/"/g, '"');
|
||||||
if (nextAsin === null) return;
|
openEditSkipPriceAsinModal(
|
||||||
nextAsin = (nextAsin || '').trim();
|
btn.dataset.skipPriceAsinEdit,
|
||||||
if (!nextAsin) {
|
countryCode,
|
||||||
alert('ASIN 不能为空');
|
shopName,
|
||||||
return;
|
currentAsin,
|
||||||
}
|
currentMinimumPrice
|
||||||
fetch('/api/admin/skip-price-asin/' + btn.dataset.skipPriceAsinEdit + '/country/' + countryCode, {
|
);
|
||||||
method: 'PUT',
|
|
||||||
headers: { 'Content-Type': 'application/json' },
|
|
||||||
body: JSON.stringify({ asin: nextAsin })
|
|
||||||
})
|
|
||||||
.then(function (r) { return r.json(); })
|
|
||||||
.then(function (res) {
|
|
||||||
if (res.success) loadSkipPriceAsin(skipPriceAsinPage);
|
|
||||||
else alert(res.error || '保存失败');
|
|
||||||
});
|
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
document.querySelectorAll('[data-skip-price-asin-delete]').forEach(function (btn) {
|
document.querySelectorAll('[data-skip-price-asin-delete]').forEach(function (btn) {
|
||||||
@@ -2733,7 +2842,6 @@
|
|||||||
document.getElementById('skipPriceAsinListBody').innerHTML = '<tr><td colspan="8" class="empty-tip">请求失败</td></tr>';
|
document.getElementById('skipPriceAsinListBody').innerHTML = '<tr><td colspan="8" class="empty-tip">请求失败</td></tr>';
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
document.getElementById('btnManageSkipPriceAsinGroups').onclick = openShopManageGroupModal;
|
|
||||||
document.getElementById('btnChooseSkipPriceAsinShop').onclick = openChooseSkipPriceAsinShopModal;
|
document.getElementById('btnChooseSkipPriceAsinShop').onclick = openChooseSkipPriceAsinShopModal;
|
||||||
document.getElementById('btnSearchChooseSkipPriceAsinShop').onclick = function () {
|
document.getElementById('btnSearchChooseSkipPriceAsinShop').onclick = function () {
|
||||||
loadChooseSkipPriceAsinShops(1);
|
loadChooseSkipPriceAsinShops(1);
|
||||||
@@ -2750,6 +2858,58 @@
|
|||||||
document.getElementById('btnCloseChooseSkipPriceAsinShopModal').onclick = function () {
|
document.getElementById('btnCloseChooseSkipPriceAsinShopModal').onclick = function () {
|
||||||
document.getElementById('chooseSkipPriceAsinShopModal').classList.remove('show');
|
document.getElementById('chooseSkipPriceAsinShopModal').classList.remove('show');
|
||||||
};
|
};
|
||||||
|
document.getElementById('btnCloseEditSkipPriceAsin').onclick = function () {
|
||||||
|
document.getElementById('editSkipPriceAsinModal').classList.remove('show');
|
||||||
|
};
|
||||||
|
document.getElementById('btnSaveSkipPriceAsin').onclick = function () {
|
||||||
|
var itemId = (document.getElementById('editSkipPriceAsinId').value || '').trim();
|
||||||
|
var countryCode = (document.getElementById('editSkipPriceAsinCountry').value || '').trim();
|
||||||
|
var asin = (document.getElementById('editSkipPriceAsinValue').value || '').trim().toUpperCase();
|
||||||
|
var minimumPrice = (document.getElementById('editSkipPriceMinimumPrice').value || '').trim();
|
||||||
|
var msgEl = document.getElementById('msgEditSkipPriceAsin');
|
||||||
|
msgEl.textContent = '';
|
||||||
|
msgEl.className = 'msg';
|
||||||
|
if (!itemId || !countryCode) {
|
||||||
|
msgEl.textContent = '缺少编辑记录';
|
||||||
|
msgEl.className = 'msg err';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!asin) {
|
||||||
|
msgEl.textContent = 'ASIN 不能为空';
|
||||||
|
msgEl.className = 'msg err';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (minimumPrice) {
|
||||||
|
var minimumPriceNumber = Number(minimumPrice);
|
||||||
|
if (!isFinite(minimumPriceNumber) || minimumPriceNumber < 0) {
|
||||||
|
msgEl.textContent = '最低价格式不正确';
|
||||||
|
msgEl.className = 'msg err';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fetch('/api/admin/skip-price-asin/' + itemId + '/country/' + countryCode, {
|
||||||
|
method: 'PUT',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({
|
||||||
|
asin: asin,
|
||||||
|
minimum_price: minimumPrice || null
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.then(function (r) { return r.json(); })
|
||||||
|
.then(function (res) {
|
||||||
|
if (!res.success) {
|
||||||
|
msgEl.textContent = res.error || '保存失败';
|
||||||
|
msgEl.className = 'msg err';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
document.getElementById('editSkipPriceAsinModal').classList.remove('show');
|
||||||
|
loadSkipPriceAsin(skipPriceAsinPage);
|
||||||
|
})
|
||||||
|
.catch(function () {
|
||||||
|
msgEl.textContent = '请求失败';
|
||||||
|
msgEl.className = 'msg err';
|
||||||
|
});
|
||||||
|
};
|
||||||
document.getElementById('skipPriceAsinCountries').addEventListener('change', renderSkipPriceAsinInputs);
|
document.getElementById('skipPriceAsinCountries').addEventListener('change', renderSkipPriceAsinInputs);
|
||||||
document.getElementById('btnSearchSkipPriceAsin').onclick = function () {
|
document.getElementById('btnSearchSkipPriceAsin').onclick = function () {
|
||||||
loadSkipPriceAsin(1);
|
loadSkipPriceAsin(1);
|
||||||
@@ -2779,8 +2939,11 @@
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
var asinMappings = {};
|
var asinMappings = {};
|
||||||
|
var minimumPriceMappings = {};
|
||||||
try {
|
try {
|
||||||
asinMappings = collectSkipPriceAsinMappings(countries);
|
var mappings = collectSkipPriceAsinMappings(countries);
|
||||||
|
asinMappings = mappings.asinMappings || {};
|
||||||
|
minimumPriceMappings = mappings.minimumPriceMappings || {};
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
msgEl.textContent = err.message || '请输入 ASIN';
|
msgEl.textContent = err.message || '请输入 ASIN';
|
||||||
msgEl.className = 'msg err';
|
msgEl.className = 'msg err';
|
||||||
@@ -2799,7 +2962,8 @@
|
|||||||
shop_name: shopName,
|
shop_name: shopName,
|
||||||
countries: countries,
|
countries: countries,
|
||||||
asin: fallbackAsin,
|
asin: fallbackAsin,
|
||||||
asin_mappings: asinMappings
|
asin_mappings: asinMappings,
|
||||||
|
minimum_price_mappings: minimumPriceMappings
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
.then(function (r) { return r.json(); })
|
.then(function (r) { return r.json(); })
|
||||||
@@ -3245,13 +3409,19 @@
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
var item = res.item || {};
|
var item = res.item || {};
|
||||||
|
currentUserId = item.id || currentUserId;
|
||||||
|
currentUserRole = item.role || currentUserRole;
|
||||||
|
currentUserUsername = item.username || currentUserUsername;
|
||||||
nameEl.textContent = item.username || '管理员';
|
nameEl.textContent = item.username || '管理员';
|
||||||
if (item.role) {
|
if (item.role) {
|
||||||
roleEl.textContent = item.role === 'super_admin' ? '超级管理员' : '管理员';
|
roleEl.textContent = item.role === 'super_admin'
|
||||||
|
? '超级管理员'
|
||||||
|
: (item.role === 'admin' ? '管理员' : '普通账号');
|
||||||
roleEl.style.display = 'inline-block';
|
roleEl.style.display = 'inline-block';
|
||||||
} else {
|
} else {
|
||||||
roleEl.style.display = 'none';
|
roleEl.style.display = 'none';
|
||||||
}
|
}
|
||||||
|
updateShopManageGroupButtonsAccess();
|
||||||
})
|
})
|
||||||
.catch(function () {
|
.catch(function () {
|
||||||
fetch('/api/auth/check')
|
fetch('/api/auth/check')
|
||||||
@@ -3259,10 +3429,12 @@
|
|||||||
.then(function (res) {
|
.then(function (res) {
|
||||||
document.getElementById('adminCurrentUsername').textContent = res && res.logged_in ? '当前用户' : '未登录';
|
document.getElementById('adminCurrentUsername').textContent = res && res.logged_in ? '当前用户' : '未登录';
|
||||||
document.getElementById('adminCurrentUserRole').style.display = 'none';
|
document.getElementById('adminCurrentUserRole').style.display = 'none';
|
||||||
|
updateShopManageGroupButtonsAccess();
|
||||||
})
|
})
|
||||||
.catch(function () {
|
.catch(function () {
|
||||||
document.getElementById('adminCurrentUsername').textContent = '当前用户';
|
document.getElementById('adminCurrentUsername').textContent = '当前用户';
|
||||||
document.getElementById('adminCurrentUserRole').style.display = 'none';
|
document.getElementById('adminCurrentUserRole').style.display = 'none';
|
||||||
|
updateShopManageGroupButtonsAccess();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -358,6 +358,7 @@ const pendingQueue = ref<PatrolDeleteShopQueueItem[]>([]);
|
|||||||
const activeTaskId = ref<number | null>(null);
|
const activeTaskId = ref<number | null>(null);
|
||||||
const activeQueueItem = ref<PatrolDeleteShopQueueItem | null>(null);
|
const activeQueueItem = ref<PatrolDeleteShopQueueItem | null>(null);
|
||||||
const queueWorkerRunning = ref(false);
|
const queueWorkerRunning = ref(false);
|
||||||
|
const autoQueueEnabled = ref(false);
|
||||||
let historyPollTimer: number | null = null;
|
let historyPollTimer: number | null = null;
|
||||||
|
|
||||||
const matchedRunnableItems = computed(() =>
|
const matchedRunnableItems = computed(() =>
|
||||||
@@ -774,6 +775,11 @@ async function runMatch() {
|
|||||||
const data = await matchPatrolDeleteShops(names);
|
const data = await matchPatrolDeleteShops(names);
|
||||||
matchedItems.value = mergeMatchedItems(matchedItems.value, data.items || []);
|
matchedItems.value = mergeMatchedItems(matchedItems.value, data.items || []);
|
||||||
saveMatchedItems();
|
saveMatchedItems();
|
||||||
|
if (autoQueueEnabled.value) {
|
||||||
|
pendingQueue.value = mergeQueueItems(pendingQueue.value, matchedRunnableItems.value);
|
||||||
|
saveQueueState();
|
||||||
|
void processQueue();
|
||||||
|
}
|
||||||
ElMessage.success(`本次返回 ${data.items?.length || 0} 条匹配结果`);
|
ElMessage.success(`本次返回 ${data.items?.length || 0} 条匹配结果`);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
ElMessage.error(error instanceof Error ? error.message : "匹配失败");
|
ElMessage.error(error instanceof Error ? error.message : "匹配失败");
|
||||||
@@ -963,6 +969,7 @@ async function processQueue() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function pushToPythonQueue() {
|
async function pushToPythonQueue() {
|
||||||
|
autoQueueEnabled.value = true;
|
||||||
const runnable = matchedRunnableItems.value;
|
const runnable = matchedRunnableItems.value;
|
||||||
if (!runnable.length) {
|
if (!runnable.length) {
|
||||||
ElMessage.warning("请先匹配可用店铺");
|
ElMessage.warning("请先匹配可用店铺");
|
||||||
@@ -1016,6 +1023,7 @@ onMounted(async () => {
|
|||||||
await Promise.all([loadCandidates(), loadDashboard(), loadHistory()]);
|
await Promise.all([loadCandidates(), loadDashboard(), loadHistory()]);
|
||||||
|
|
||||||
if (activeTaskId.value || pendingQueue.value.length) {
|
if (activeTaskId.value || pendingQueue.value.length) {
|
||||||
|
autoQueueEnabled.value = true;
|
||||||
queuePushResult.value =
|
queuePushResult.value =
|
||||||
activeTaskId.value != null
|
activeTaskId.value != null
|
||||||
? `检测到未完成队列,继续等待任务 ${activeTaskId.value} 完成并自动接续后续店铺`
|
? `检测到未完成队列,继续等待任务 ${activeTaskId.value} 完成并自动接续后续店铺`
|
||||||
|
|||||||
@@ -312,6 +312,8 @@ const matchedItems = ref<PriceTrackShopQueueItem[]>([])
|
|||||||
const adding = ref(false)
|
const adding = ref(false)
|
||||||
const matching = ref(false)
|
const matching = ref(false)
|
||||||
const pushing = ref(false)
|
const pushing = ref(false)
|
||||||
|
const queueWorkerRunning = ref(false)
|
||||||
|
const autoQueueEnabled = ref(false)
|
||||||
const queuePushResult = ref('')
|
const queuePushResult = ref('')
|
||||||
const queuePayloadText = ref('')
|
const queuePayloadText = ref('')
|
||||||
const matchAsinRowsByCountry = ref<Record<string, PriceTrackAsinParsedRow[]>>({})
|
const matchAsinRowsByCountry = ref<Record<string, PriceTrackAsinParsedRow[]>>({})
|
||||||
@@ -787,6 +789,9 @@ async function runMatch() {
|
|||||||
}
|
}
|
||||||
matchedItems.value = [...nextByShop.values()]
|
matchedItems.value = [...nextByShop.values()]
|
||||||
saveMatchedItemsToStorage()
|
saveMatchedItemsToStorage()
|
||||||
|
if (autoQueueEnabled.value) {
|
||||||
|
void processMatchedQueue()
|
||||||
|
}
|
||||||
ElMessage.success(`匹配 ${batch.length} 个店铺`)
|
ElMessage.success(`匹配 ${batch.length} 个店铺`)
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
ElMessage.error(e instanceof Error ? e.message : '匹配失败')
|
ElMessage.error(e instanceof Error ? e.message : '匹配失败')
|
||||||
@@ -813,6 +818,19 @@ function rowKeyForMatch(row: { shopName?: string; shopId?: number | string | nul
|
|||||||
return `${(row.shopName || '').trim()}\u0001${row.shopId ?? ''}`
|
return `${(row.shopName || '').trim()}\u0001${row.shopId ?? ''}`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function buildSkipAsinDeletePolicy() {
|
||||||
|
const enabled = statusModeEnabled.value && !asinModeEnabled.value
|
||||||
|
return {
|
||||||
|
enabled,
|
||||||
|
mode: enabled ? 'DELETE_WHEN_PRICE_BELOW_MINIMUM' : 'NONE',
|
||||||
|
deleteWhenPriceBelowMinimum: enabled,
|
||||||
|
compareField: 'price',
|
||||||
|
thresholdField: 'minimumPrice',
|
||||||
|
target: 'skip_asin',
|
||||||
|
source: 'frontend-price-track',
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function removeMatchedRowsLocally(rows: Array<{ shopName?: string; shopId?: number | string | null }>) {
|
function removeMatchedRowsLocally(rows: Array<{ shopName?: string; shopId?: number | string | null }>) {
|
||||||
if (!rows.length) return
|
if (!rows.length) return
|
||||||
const keys = new Set(rows.map((row) => rowKeyForMatch(row)))
|
const keys = new Set(rows.map((row) => rowKeyForMatch(row)))
|
||||||
@@ -903,6 +921,9 @@ async function pushToPythonQueueLegacy() {
|
|||||||
// 店铺简要信息(用于展示)
|
// 店铺简要信息(用于展示)
|
||||||
shop_names: matchedRows.map((i) => i.shopName).filter(Boolean),
|
shop_names: matchedRows.map((i) => i.shopName).filter(Boolean),
|
||||||
country_codes: [...orderedCountryCodes.value],
|
country_codes: [...orderedCountryCodes.value],
|
||||||
|
skip_asin_delete_policy: buildSkipAsinDeletePolicy(),
|
||||||
|
delete_skip_asin_when_price_below_minimum: statusModeEnabled.value && !asinModeEnabled.value,
|
||||||
|
deleteSkipAsinWhenPriceBelowMinimum: statusModeEnabled.value && !asinModeEnabled.value,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
queuePayloadText.value = JSON.stringify(queuePayload, null, 2)
|
queuePayloadText.value = JSON.stringify(queuePayload, null, 2)
|
||||||
@@ -930,6 +951,9 @@ function buildQueuePayload(taskVo: PriceTrackCreateTaskVo, row: PriceTrackShopQu
|
|||||||
const taskItem = taskVo.items?.[0]
|
const taskItem = taskVo.items?.[0]
|
||||||
const shopName = (row.shopName || '').trim()
|
const shopName = (row.shopName || '').trim()
|
||||||
const skipAsinsByCountry = row.skipAsins || taskVo.skipAsinsByCountry || {}
|
const skipAsinsByCountry = row.skipAsins || taskVo.skipAsinsByCountry || {}
|
||||||
|
const skipAsinDetailsByCountry = taskVo.skipAsinDetailsByCountry || {}
|
||||||
|
const minimumPriceByCountryAndAsin = taskVo.minimumPriceByCountryAndAsin || {}
|
||||||
|
const skipAsinDeletePolicy = buildSkipAsinDeletePolicy()
|
||||||
const resultId = taskItem?.resultId ?? null
|
const resultId = taskItem?.resultId ?? null
|
||||||
const loopRunId = taskItem?.loopRunId ?? null
|
const loopRunId = taskItem?.loopRunId ?? null
|
||||||
const roundIndex = taskItem?.roundIndex ?? null
|
const roundIndex = taskItem?.roundIndex ?? null
|
||||||
@@ -963,6 +987,11 @@ function buildQueuePayload(taskVo: PriceTrackCreateTaskVo, row: PriceTrackShopQu
|
|||||||
mode: statusModeEnabled.value ? 'status' : 'asin',
|
mode: statusModeEnabled.value ? 'status' : 'asin',
|
||||||
skip_asins: skipAsinsByCountry,
|
skip_asins: skipAsinsByCountry,
|
||||||
skip_asins_by_country: skipAsinsByCountry,
|
skip_asins_by_country: skipAsinsByCountry,
|
||||||
|
skip_asin_details_by_country: skipAsinDetailsByCountry,
|
||||||
|
minimum_price_by_country_and_asin: minimumPriceByCountryAndAsin,
|
||||||
|
skip_asin_delete_policy: skipAsinDeletePolicy,
|
||||||
|
delete_skip_asin_when_price_below_minimum: skipAsinDeletePolicy.deleteWhenPriceBelowMinimum,
|
||||||
|
deleteSkipAsinWhenPriceBelowMinimum: skipAsinDeletePolicy.deleteWhenPriceBelowMinimum,
|
||||||
asin_rows_by_country: Object.keys(matchAsinRowsByCountry.value).length
|
asin_rows_by_country: Object.keys(matchAsinRowsByCountry.value).length
|
||||||
? matchAsinRowsByCountry.value
|
? matchAsinRowsByCountry.value
|
||||||
: (taskVo.asinRowsByCountry || {}),
|
: (taskVo.asinRowsByCountry || {}),
|
||||||
@@ -1039,6 +1068,18 @@ async function waitForTaskTerminal(taskId: number) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function pushToPythonQueue() {
|
async function pushToPythonQueue() {
|
||||||
|
autoQueueEnabled.value = true
|
||||||
|
await processMatchedQueue()
|
||||||
|
}
|
||||||
|
|
||||||
|
function nextMatchedQueueItem() {
|
||||||
|
return matchedItems.value.find((item) => item.matched)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function processMatchedQueue() {
|
||||||
|
if (queueWorkerRunning.value) {
|
||||||
|
return
|
||||||
|
}
|
||||||
if (!hasValidMode.value) {
|
if (!hasValidMode.value) {
|
||||||
ElMessage.warning('请至少选择一个跟价模式')
|
ElMessage.warning('请至少选择一个跟价模式')
|
||||||
return
|
return
|
||||||
@@ -1057,13 +1098,19 @@ async function pushToPythonQueue() {
|
|||||||
ElMessage.warning('无可用匹配店铺')
|
ElMessage.warning('无可用匹配店铺')
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
queueWorkerRunning.value = true
|
||||||
pushing.value = true
|
pushing.value = true
|
||||||
queuePayloadText.value = ''
|
queuePayloadText.value = ''
|
||||||
let successCount = 0
|
let successCount = 0
|
||||||
let failedCount = 0
|
let failedCount = 0
|
||||||
try {
|
try {
|
||||||
for (let index = 0; index < matchedRows.length; index += 1) {
|
let index = 0
|
||||||
const row = matchedRows[index]
|
while (autoQueueEnabled.value) {
|
||||||
|
const row = nextMatchedQueueItem()
|
||||||
|
if (!row) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
index += 1
|
||||||
try {
|
try {
|
||||||
let attempt = 0
|
let attempt = 0
|
||||||
const taskVo = await (async () => {
|
const taskVo = await (async () => {
|
||||||
@@ -1107,37 +1154,38 @@ async function pushToPythonQueue() {
|
|||||||
failedCount += 1
|
failedCount += 1
|
||||||
queuePushResult.value = '任务 ' + taskVo.taskId + ' 推送失败,已自动继续下一条:' + (pushResult?.error || '未知错误')
|
queuePushResult.value = '任务 ' + taskVo.taskId + ' 推送失败,已自动继续下一条:' + (pushResult?.error || '未知错误')
|
||||||
ElMessage.error(queuePushResult.value)
|
ElMessage.error(queuePushResult.value)
|
||||||
|
removeMatchedRowsLocally([row])
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
addPollingTask(taskVo.taskId)
|
addPollingTask(taskVo.taskId)
|
||||||
scheduleNextPoll(true)
|
scheduleNextPoll(true)
|
||||||
removeMatchedRowsLocally([row])
|
removeMatchedRowsLocally([row])
|
||||||
queuePushResult.value = matchedRows.length > 1
|
queuePushResult.value = '任务 ' + taskVo.taskId + ' 已入队,等待执行完成'
|
||||||
? '任务 ' + taskVo.taskId + ' 已入队,等待完成后继续下一条(' + (index + 1) + '/' + matchedRows.length + ')'
|
|
||||||
: '任务 ' + taskVo.taskId + ' 已入队,等待执行完成'
|
|
||||||
const finalStatus = await waitForTaskTerminal(taskVo.taskId)
|
const finalStatus = await waitForTaskTerminal(taskVo.taskId)
|
||||||
if (finalStatus !== 'SUCCESS') {
|
if (finalStatus !== 'SUCCESS') {
|
||||||
failedCount += 1
|
failedCount += 1
|
||||||
queuePushResult.value = '任务 ' + taskVo.taskId + ' 执行失败,已自动继续下一条(' + (index + 1) + '/' + matchedRows.length + ')'
|
queuePushResult.value = '任务 ' + taskVo.taskId + ' 执行失败,已自动继续下一条'
|
||||||
ElMessage.error(queuePushResult.value)
|
ElMessage.error(queuePushResult.value)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
successCount += 1
|
successCount += 1
|
||||||
queuePushResult.value = index + 1 < matchedRows.length
|
queuePushResult.value = '任务 ' + taskVo.taskId + ' 已完成'
|
||||||
? '任务 ' + taskVo.taskId + ' 已完成,继续推送下一条(' + (index + 1) + '/' + matchedRows.length + ')'
|
|
||||||
: '任务 ' + taskVo.taskId + ' 已完成'
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
failedCount += 1
|
failedCount += 1
|
||||||
queuePushResult.value = '第 ' + (index + 1) + '/' + matchedRows.length + ' 条任务处理失败,已自动继续下一条:' + (error instanceof Error ? error.message : '未知错误')
|
queuePushResult.value = '第 ' + index + ' 条任务处理失败,已自动继续下一条:' + (error instanceof Error ? error.message : '未知错误')
|
||||||
ElMessage.error(queuePushResult.value)
|
ElMessage.error(queuePushResult.value)
|
||||||
|
removeMatchedRowsLocally([row])
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (successCount > 0 || failedCount > 0) {
|
||||||
ElMessage.success('店铺任务推送已完成:成功 ' + successCount + ' 条,失败 ' + failedCount + ' 条')
|
ElMessage.success('店铺任务推送已完成:成功 ' + successCount + ' 条,失败 ' + failedCount + ' 条')
|
||||||
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
queuePushResult.value = e instanceof Error ? e.message : '推送异常'
|
queuePushResult.value = e instanceof Error ? e.message : '推送异常'
|
||||||
ElMessage.error(queuePushResult.value)
|
ElMessage.error(queuePushResult.value)
|
||||||
} finally {
|
} finally {
|
||||||
|
queueWorkerRunning.value = false
|
||||||
pushing.value = false
|
pushing.value = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -229,6 +229,8 @@ const matchedItems = ref<ProductRiskShopQueueItem[]>([])
|
|||||||
const adding = ref(false)
|
const adding = ref(false)
|
||||||
const matching = ref(false)
|
const matching = ref(false)
|
||||||
const pushing = ref(false)
|
const pushing = ref(false)
|
||||||
|
const queueWorkerRunning = ref(false)
|
||||||
|
const autoQueueEnabled = ref(false)
|
||||||
const queuePushResult = ref('')
|
const queuePushResult = ref('')
|
||||||
const queuePayloadText = ref('')
|
const queuePayloadText = ref('')
|
||||||
|
|
||||||
@@ -897,6 +899,9 @@ async function runMatch() {
|
|||||||
const batch = res.items || []
|
const batch = res.items || []
|
||||||
matchedItems.value = mergeMatchedItems(matchedItems.value, batch)
|
matchedItems.value = mergeMatchedItems(matchedItems.value, batch)
|
||||||
saveMatchedItemsToStorage()
|
saveMatchedItemsToStorage()
|
||||||
|
if (autoQueueEnabled.value) {
|
||||||
|
void processMatchedQueue()
|
||||||
|
}
|
||||||
if (!batch.length) {
|
if (!batch.length) {
|
||||||
ElMessage.warning('未返回匹配结果')
|
ElMessage.warning('未返回匹配结果')
|
||||||
} else {
|
} else {
|
||||||
@@ -982,6 +987,18 @@ function formatMatchRemark(row: ProductRiskShopQueueItem) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function pushToPythonQueue() {
|
async function pushToPythonQueue() {
|
||||||
|
autoQueueEnabled.value = true
|
||||||
|
await processMatchedQueue()
|
||||||
|
}
|
||||||
|
|
||||||
|
function nextMatchedQueueItem() {
|
||||||
|
return matchedItems.value.find((item) => item.matched)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function processMatchedQueue() {
|
||||||
|
if (queueWorkerRunning.value) {
|
||||||
|
return
|
||||||
|
}
|
||||||
if (!matchedItems.value.length) {
|
if (!matchedItems.value.length) {
|
||||||
ElMessage.warning('请先匹配店铺')
|
ElMessage.warning('请先匹配店铺')
|
||||||
return
|
return
|
||||||
@@ -996,13 +1013,19 @@ async function pushToPythonQueue() {
|
|||||||
ElMessage.warning('没有已匹配的店铺可推送')
|
ElMessage.warning('没有已匹配的店铺可推送')
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
queueWorkerRunning.value = true
|
||||||
pushing.value = true
|
pushing.value = true
|
||||||
queuePayloadText.value = ''
|
queuePayloadText.value = ''
|
||||||
let successCount = 0
|
let successCount = 0
|
||||||
let failedCount = 0
|
let failedCount = 0
|
||||||
try {
|
try {
|
||||||
for (let i = 0; i < toPush.length; i++) {
|
let index = 0
|
||||||
const item = toPush[i]
|
while (autoQueueEnabled.value) {
|
||||||
|
const item = nextMatchedQueueItem()
|
||||||
|
if (!item) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
index += 1
|
||||||
try {
|
try {
|
||||||
const created = await createProductRiskTaskWithRetry([item])
|
const created = await createProductRiskTaskWithRetry([item])
|
||||||
const taskId = created.taskId
|
const taskId = created.taskId
|
||||||
@@ -1031,39 +1054,40 @@ async function pushToPythonQueue() {
|
|||||||
if (!pushResult?.success) {
|
if (!pushResult?.success) {
|
||||||
removePollingTask(taskId)
|
removePollingTask(taskId)
|
||||||
failedCount += 1
|
failedCount += 1
|
||||||
queuePushResult.value = '第 ' + (i + 1) + '/' + toPush.length + ' 条推送失败:' + (pushResult?.error || '未知错误') + ',已自动继续下一条'
|
queuePushResult.value = '第 ' + index + ' 条推送失败:' + (pushResult?.error || '未知错误') + ',已自动继续下一条'
|
||||||
ElMessage.error(queuePushResult.value)
|
ElMessage.error(queuePushResult.value)
|
||||||
|
removeMatchedRowsLocally([item])
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
removeMatchedRowsLocally([item])
|
removeMatchedRowsLocally([item])
|
||||||
queuePushResult.value = toPush.length > 1
|
queuePushResult.value = '任务 ' + taskId + ' 已入队,等待执行完成...'
|
||||||
? '任务 ' + taskId + ':第 ' + (i + 1) + '/' + toPush.length + ' 条店铺已入队,等待执行完成...'
|
|
||||||
: '任务 ' + taskId + ' 已入队,等待执行完成...'
|
|
||||||
const finalStatus = await waitForTaskTerminal(taskId)
|
const finalStatus = await waitForTaskTerminal(taskId)
|
||||||
if (finalStatus !== 'SUCCESS') {
|
if (finalStatus !== 'SUCCESS') {
|
||||||
failedCount += 1
|
failedCount += 1
|
||||||
queuePushResult.value = '任务 ' + taskId + ' 执行失败,已自动继续下一条(' + (i + 1) + '/' + toPush.length + ')'
|
queuePushResult.value = '任务 ' + taskId + ' 执行失败,已自动继续下一条'
|
||||||
ElMessage.error(queuePushResult.value)
|
ElMessage.error(queuePushResult.value)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
successCount += 1
|
successCount += 1
|
||||||
queuePushResult.value = i + 1 < toPush.length
|
queuePushResult.value = '任务 ' + taskId + ' 已完成'
|
||||||
? '任务 ' + taskId + ' 已完成,继续推送下一条(' + (i + 1) + '/' + toPush.length + ')'
|
|
||||||
: '任务 ' + taskId + ' 已完成'
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
failedCount += 1
|
failedCount += 1
|
||||||
queuePushResult.value = '第 ' + (i + 1) + '/' + toPush.length + ' 条任务处理失败,已自动继续下一条:' + (error instanceof Error ? error.message : '未知错误')
|
queuePushResult.value = '第 ' + index + ' 条任务处理失败,已自动继续下一条:' + (error instanceof Error ? error.message : '未知错误')
|
||||||
ElMessage.error(queuePushResult.value)
|
ElMessage.error(queuePushResult.value)
|
||||||
|
removeMatchedRowsLocally([item])
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (successCount > 0 || failedCount > 0) {
|
||||||
ElMessage.success('店铺推送已完成:成功 ' + successCount + ' 条,失败 ' + failedCount + ' 条')
|
ElMessage.success('店铺推送已完成:成功 ' + successCount + ' 条,失败 ' + failedCount + ' 条')
|
||||||
|
}
|
||||||
await loadHistory()
|
await loadHistory()
|
||||||
ensurePolling(true)
|
ensurePolling(true)
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
queuePushResult.value = e instanceof Error ? e.message : '推送异常'
|
queuePushResult.value = e instanceof Error ? e.message : '推送异常'
|
||||||
ElMessage.error(queuePushResult.value)
|
ElMessage.error(queuePushResult.value)
|
||||||
} finally {
|
} finally {
|
||||||
|
queueWorkerRunning.value = false
|
||||||
pushing.value = false
|
pushing.value = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -121,6 +121,8 @@ const shopMatchListingFilter = ref<ListingFilterValue>('Active')
|
|||||||
const adding = ref(false)
|
const adding = ref(false)
|
||||||
const matching = ref(false)
|
const matching = ref(false)
|
||||||
const pushing = ref(false)
|
const pushing = ref(false)
|
||||||
|
const queueWorkerRunning = ref(false)
|
||||||
|
const autoQueueEnabled = ref(false)
|
||||||
const queuePushResult = ref('')
|
const queuePushResult = ref('')
|
||||||
const queuePayloadText = ref('')
|
const queuePayloadText = ref('')
|
||||||
const orderedCountryCodes = ref<string[]>(['DE', 'UK', 'FR', 'IT', 'ES'])
|
const orderedCountryCodes = ref<string[]>(['DE', 'UK', 'FR', 'IT', 'ES'])
|
||||||
@@ -241,7 +243,7 @@ function addScheduleValue() { const last = schedulePickerValues.value[schedulePi
|
|||||||
function removeScheduleValue(index: number) { if (schedulePickerValues.value.length <= 1) return; schedulePickerValues.value = schedulePickerValues.value.filter((_, idx) => idx !== index) }
|
function removeScheduleValue(index: number) { if (schedulePickerValues.value.length <= 1) return; schedulePickerValues.value = schedulePickerValues.value.filter((_, idx) => idx !== index) }
|
||||||
function updateScheduleValue(index: number, value: string | null) { const list = [...schedulePickerValues.value]; list[index] = value || ''; schedulePickerValues.value = sanitizeSchedulePickerValues(list, false) }
|
function updateScheduleValue(index: number, value: string | null) { const list = [...schedulePickerValues.value]; list[index] = value || ''; schedulePickerValues.value = sanitizeSchedulePickerValues(list, false) }
|
||||||
function mergeMatchedItems(base: ShopMatchShopQueueItem[], incoming: ShopMatchShopQueueItem[]) { const map = new Map<string, ShopMatchShopQueueItem>(); for (const item of base) map.set(rowKeyForMatch(item), item); for (const item of incoming) map.set(rowKeyForMatch(item), item); return Array.from(map.values()) }
|
function mergeMatchedItems(base: ShopMatchShopQueueItem[], incoming: ShopMatchShopQueueItem[]) { const map = new Map<string, ShopMatchShopQueueItem>(); for (const item of base) map.set(rowKeyForMatch(item), item); for (const item of incoming) map.set(rowKeyForMatch(item), item); return Array.from(map.values()) }
|
||||||
async function runMatch() { const names = selectedCandidates.value.map((item) => item.shop_name).filter(Boolean); if (!names.length) { ElMessage.warning('请先选择候选店铺'); return } matching.value = true; try { const data = await matchShopMatchShops(names); matchedItems.value = mergeMatchedItems(matchedItems.value, data.items || []); saveMatchedItemsToStorage(); ElMessage.success(`本次返回 ${data.items?.length || 0} 条匹配结果`) } catch (error) { ElMessage.error(error instanceof Error ? error.message : '匹配失败') } finally { matching.value = false } }
|
async function runMatch() { const names = selectedCandidates.value.map((item) => item.shop_name).filter(Boolean); if (!names.length) { ElMessage.warning('请先选择候选店铺'); return } matching.value = true; try { const data = await matchShopMatchShops(names); matchedItems.value = mergeMatchedItems(matchedItems.value, data.items || []); saveMatchedItemsToStorage(); if (autoQueueEnabled.value) void processMatchedQueue(); ElMessage.success(`本次返回 ${data.items?.length || 0} 条匹配结果`) } catch (error) { ElMessage.error(error instanceof Error ? error.message : '匹配失败') } finally { matching.value = false } }
|
||||||
function removeMatchedRow(row: ShopMatchShopQueueItem) { matchedItems.value = matchedItems.value.filter((item) => rowKeyForMatch(item) !== rowKeyForMatch(row)); saveMatchedItemsToStorage() }
|
function removeMatchedRow(row: ShopMatchShopQueueItem) { matchedItems.value = matchedItems.value.filter((item) => rowKeyForMatch(item) !== rowKeyForMatch(row)); saveMatchedItemsToStorage() }
|
||||||
function removeMatchedRowsLocally(rows: ShopMatchHistoryItem[] | ShopMatchShopQueueItem[]) { const keys = new Set(rows.map((row) => rowKeyForMatch(row as ShopMatchShopQueueItem))); matchedItems.value = matchedItems.value.filter((item) => !keys.has(rowKeyForMatch(item))); saveMatchedItemsToStorage() }
|
function removeMatchedRowsLocally(rows: ShopMatchHistoryItem[] | ShopMatchShopQueueItem[]) { const keys = new Set(rows.map((row) => rowKeyForMatch(row as ShopMatchShopQueueItem))); matchedItems.value = matchedItems.value.filter((item) => !keys.has(rowKeyForMatch(item))); saveMatchedItemsToStorage() }
|
||||||
function parseScheduleValues() {
|
function parseScheduleValues() {
|
||||||
@@ -289,10 +291,9 @@ function sanitizeSchedulePickerValues(values: string[], ensureOne: boolean) {
|
|||||||
function buildQueuePayload(taskId: number, item: Pick<ShopMatchHistoryItem, 'shopName' | 'shopId' | 'platform' | 'companyName' | 'matchStatus' | 'matchMessage'>, countryCodes: string[], stageIndex?: number, finalStage = true) { return { type: 'shop-match-run', ts: Date.now(), data: { taskId, user_id: currentUserId(), items: [{ shopName: item.shopName, shopId: item.shopId, platform: item.platform, companyName: item.companyName, matched: true, matchStatus: item.matchStatus, matchMessage: item.matchMessage }], country_codes: [...countryCodes], risk_listing_filter: shopMatchListingFilter.value, stage_index: stageIndex, final_stage: finalStage } } }
|
function buildQueuePayload(taskId: number, item: Pick<ShopMatchHistoryItem, 'shopName' | 'shopId' | 'platform' | 'companyName' | 'matchStatus' | 'matchMessage'>, countryCodes: string[], stageIndex?: number, finalStage = true) { return { type: 'shop-match-run', ts: Date.now(), data: { taskId, user_id: currentUserId(), items: [{ shopName: item.shopName, shopId: item.shopId, platform: item.platform, companyName: item.companyName, matched: true, matchStatus: item.matchStatus, matchMessage: item.matchMessage }], country_codes: [...countryCodes], risk_listing_filter: shopMatchListingFilter.value, stage_index: stageIndex, final_stage: finalStage } } }
|
||||||
function nextScheduleStage(taskId: number) { const task = taskSnapshots.value[taskId]?.task; const stages = task?.scheduleStages || []; const currentStageIndex = task?.currentStageIndex; if (typeof currentStageIndex !== 'number') return null; const stage = stages.find((item) => item.stageIndex === currentStageIndex); if (!stage?.scheduledAt) return null; return { stageIndex: currentStageIndex, scheduledAt: stage.scheduledAt, finalStage: currentStageIndex === stages.length - 1 } }
|
function nextScheduleStage(taskId: number) { const task = taskSnapshots.value[taskId]?.task; const stages = task?.scheduleStages || []; const currentStageIndex = task?.currentStageIndex; if (typeof currentStageIndex !== 'number') return null; const stage = stages.find((item) => item.stageIndex === currentStageIndex); if (!stage?.scheduledAt) return null; return { stageIndex: currentStageIndex, scheduledAt: stage.scheduledAt, finalStage: currentStageIndex === stages.length - 1 } }
|
||||||
async function dispatchScheduledTask(taskId: number, item: ShopMatchHistoryItem) { const api = getPywebviewApi(); if (!api?.enqueue_json) throw new Error('当前客户端未提供 enqueue_json'); const stage = nextScheduleStage(taskId); const snapshot = taskSnapshots.value[taskId]; const countryCodes = snapshot?.task?.countryCodes || orderedCountryCodes.value; if (!stage) throw new Error(`任务 ${taskId} 缺少待执行阶段`); clearDispatchTimer(taskId); await withTransientRetry(() => activateShopMatchTask(taskId, stage.stageIndex), (attempt, maxAttempts) => { queuePushResult.value = `任务 ${taskId} 等待后端恢复后再激活执行(${attempt}/${maxAttempts})...` }); taskDetails.value = { ...taskDetails.value, [taskId]: 'RUNNING' }; if (snapshot?.task) taskSnapshots.value = { ...taskSnapshots.value, [taskId]: { ...snapshot, task: { ...snapshot.task, status: 'RUNNING', activeStageIndex: stage.stageIndex } } }; saveTaskSnapshotsToStorage(); saveTaskDetailsToStorage(); const payload = buildQueuePayload(taskId, item, countryCodes, stage.stageIndex, stage.finalStage); queuePayloadText.value = JSON.stringify(payload, null, 2); const result = await api.enqueue_json(payload); if (!result?.success) throw new Error(result?.error || `任务 ${taskId} 推送失败`); addPollingTask(taskId); queuePushResult.value = `任务 ${taskId} 已推送第 ${stage.stageIndex + 1} 轮`; ensurePolling(true) }
|
async function dispatchScheduledTask(taskId: number, item: ShopMatchHistoryItem) { const api = getPywebviewApi(); if (!api?.enqueue_json) throw new Error('当前客户端未提供 enqueue_json'); const stage = nextScheduleStage(taskId); const snapshot = taskSnapshots.value[taskId]; const countryCodes = snapshot?.task?.countryCodes || orderedCountryCodes.value; if (!stage) throw new Error(`任务 ${taskId} 缺少待执行阶段`); clearDispatchTimer(taskId); await withTransientRetry(() => activateShopMatchTask(taskId, stage.stageIndex), (attempt, maxAttempts) => { queuePushResult.value = `任务 ${taskId} 等待后端恢复后再激活执行(${attempt}/${maxAttempts})...` }); taskDetails.value = { ...taskDetails.value, [taskId]: 'RUNNING' }; if (snapshot?.task) taskSnapshots.value = { ...taskSnapshots.value, [taskId]: { ...snapshot, task: { ...snapshot.task, status: 'RUNNING', activeStageIndex: stage.stageIndex } } }; saveTaskSnapshotsToStorage(); saveTaskDetailsToStorage(); const payload = buildQueuePayload(taskId, item, countryCodes, stage.stageIndex, stage.finalStage); queuePayloadText.value = JSON.stringify(payload, null, 2); const result = await api.enqueue_json(payload); if (!result?.success) throw new Error(result?.error || `任务 ${taskId} 推送失败`); addPollingTask(taskId); queuePushResult.value = `任务 ${taskId} 已推送第 ${stage.stageIndex + 1} 轮`; ensurePolling(true) }
|
||||||
function hasRunningTask() { return Object.values(taskDetails.value).some((status) => status === 'RUNNING') || Object.values(taskSnapshots.value).some((detail) => detail.task?.status === 'RUNNING') }
|
|
||||||
function scheduledTimestamp(value?: string) { const date = parseScheduleDateTime(value); return date ? date.getTime() : Number.NaN }
|
function scheduledTimestamp(value?: string) { const date = parseScheduleDateTime(value); return date ? date.getTime() : Number.NaN }
|
||||||
function findNextReadyScheduledTask() { const taskIds = new Set<number>(pollingTaskIds.value); for (const key of Object.keys(taskSnapshots.value)) { const id = normalizeTaskId(key); if (id > 0) taskIds.add(id) } for (const item of historyItems.value) { const id = normalizeTaskId(item.taskId); if (id > 0) taskIds.add(id) } const ready = Array.from(taskIds).map((taskId) => { const snapshot = taskSnapshots.value[taskId]; const taskStatus = taskDetails.value[taskId] || snapshot?.task?.status; const item = snapshot?.items?.[0] || historyItems.value.find((row) => normalizeTaskId(row.taskId) === taskId); const stage = nextScheduleStage(taskId); if (taskStatus !== 'SCHEDULED' || !item || !stage?.scheduledAt) return null; const scheduledTime = scheduledTimestamp(stage.scheduledAt); if (!Number.isFinite(scheduledTime) || scheduledTime > Date.now()) return null; return { taskId, item, scheduledTime } }).filter((item): item is { taskId: number; item: ShopMatchHistoryItem; scheduledTime: number } => !!item); ready.sort((a, b) => a.scheduledTime - b.scheduledTime || a.taskId - b.taskId); return ready[0] || null }
|
function findReadyScheduledTasks() { const taskIds = new Set<number>(pollingTaskIds.value); for (const key of Object.keys(taskSnapshots.value)) { const id = normalizeTaskId(key); if (id > 0) taskIds.add(id) } for (const item of historyItems.value) { const id = normalizeTaskId(item.taskId); if (id > 0) taskIds.add(id) } const ready = Array.from(taskIds).map((taskId) => { const snapshot = taskSnapshots.value[taskId]; const taskStatus = taskDetails.value[taskId] || snapshot?.task?.status; const item = snapshot?.items?.[0] || historyItems.value.find((row) => normalizeTaskId(row.taskId) === taskId); const stage = nextScheduleStage(taskId); if (taskStatus !== 'SCHEDULED' || !item || !stage?.scheduledAt) return null; const scheduledTime = scheduledTimestamp(stage.scheduledAt); if (!Number.isFinite(scheduledTime) || scheduledTime > Date.now()) return null; return { taskId, item, scheduledTime } }).filter((item): item is { taskId: number; item: ShopMatchHistoryItem; scheduledTime: number } => !!item); ready.sort((a, b) => a.scheduledTime - b.scheduledTime || a.taskId - b.taskId); return ready }
|
||||||
async function pumpScheduledQueue() { if (scheduledDispatchInFlight.value || hasRunningTask()) return; const readyTask = findNextReadyScheduledTask(); if (!readyTask) return; scheduledDispatchInFlight.value = true; try { await dispatchScheduledTask(readyTask.taskId, readyTask.item) } catch (error) { const message = error instanceof Error ? error.message : `任务 ${readyTask.taskId} 调度失败`; const key = `${readyTask.taskId}:${message}`; const now = Date.now(); queuePushResult.value = message; if (lastScheduledErrorKey !== key || now - lastScheduledErrorAt > 10000) { lastScheduledErrorKey = key; lastScheduledErrorAt = now; ElMessage.error(message) } await Promise.allSettled([loadHistory(), refreshTaskBatch()]) } finally { scheduledDispatchInFlight.value = false } }
|
async function pumpScheduledQueue() { if (scheduledDispatchInFlight.value) return; const readyTasks = findReadyScheduledTasks(); if (!readyTasks.length) return; scheduledDispatchInFlight.value = true; try { for (const readyTask of readyTasks) { try { await dispatchScheduledTask(readyTask.taskId, readyTask.item) } catch (error) { const message = error instanceof Error ? error.message : `任务 ${readyTask.taskId} 调度失败`; const key = `${readyTask.taskId}:${message}`; const now = Date.now(); queuePushResult.value = message; if (lastScheduledErrorKey !== key || now - lastScheduledErrorAt > 10000) { lastScheduledErrorKey = key; lastScheduledErrorAt = now; ElMessage.error(message) } await Promise.allSettled([loadHistory(), refreshTaskBatch()]) } } } finally { scheduledDispatchInFlight.value = false } }
|
||||||
function scheduleDispatch(taskId: number, scheduledAt?: string) { if (!scheduledAt) return; clearDispatchTimer(taskId); const time = scheduledTimestamp(scheduledAt); if (!Number.isFinite(time)) return; const delay = Math.max(0, time - Date.now()); const timer = window.setTimeout(() => { dispatchTimers.delete(taskId); void pumpScheduledQueue() }, delay); dispatchTimers.set(taskId, timer) }
|
function scheduleDispatch(taskId: number, scheduledAt?: string) { if (!scheduledAt) return; clearDispatchTimer(taskId); const time = scheduledTimestamp(scheduledAt); if (!Number.isFinite(time)) return; const delay = Math.max(0, time - Date.now()); const timer = window.setTimeout(() => { dispatchTimers.delete(taskId); void pumpScheduledQueue() }, delay); dispatchTimers.set(taskId, timer) }
|
||||||
function restoreScheduledDispatches() { const taskIds = new Set<number>(pollingTaskIds.value); for (const key of Object.keys(taskSnapshots.value)) { const id = normalizeTaskId(key); if (id > 0) taskIds.add(id) } for (const item of historyItems.value) { const id = normalizeTaskId(item.taskId); if (id > 0) taskIds.add(id) } for (const taskId of taskIds) { const snapshot = taskSnapshots.value[taskId]; const taskStatus = snapshot?.task?.status || taskDetails.value[taskId]; const stage = nextScheduleStage(taskId); if (taskStatus === 'SCHEDULED' && stage?.scheduledAt) scheduleDispatch(taskId, stage.scheduledAt); else clearDispatchTimer(taskId) } void pumpScheduledQueue() }
|
function restoreScheduledDispatches() { const taskIds = new Set<number>(pollingTaskIds.value); for (const key of Object.keys(taskSnapshots.value)) { const id = normalizeTaskId(key); if (id > 0) taskIds.add(id) } for (const item of historyItems.value) { const id = normalizeTaskId(item.taskId); if (id > 0) taskIds.add(id) } for (const taskId of taskIds) { const snapshot = taskSnapshots.value[taskId]; const taskStatus = snapshot?.task?.status || taskDetails.value[taskId]; const stage = nextScheduleStage(taskId); if (taskStatus === 'SCHEDULED' && stage?.scheduledAt) scheduleDispatch(taskId, stage.scheduledAt); else clearDispatchTimer(taskId) } void pumpScheduledQueue() }
|
||||||
function startScheduleHeartbeat() { if (scheduleHeartbeatTimer) return; scheduleHeartbeatTimer = window.setInterval(() => { void pumpScheduledQueue() }, 1000) }
|
function startScheduleHeartbeat() { if (scheduleHeartbeatTimer) return; scheduleHeartbeatTimer = window.setInterval(() => { void pumpScheduledQueue() }, 1000) }
|
||||||
@@ -344,7 +345,9 @@ async function deleteTaskRecord(item: ShopMatchHistoryItem) {
|
|||||||
}
|
}
|
||||||
function formatMatchStatus(status?: string) { const value = (status || '').trim(); return { MATCHED: '已匹配', PENDING: '待匹配', CONFLICT: '需人工确认', INDEX_STALE: '索引过期' }[value] || value || '—' }
|
function formatMatchStatus(status?: string) { const value = (status || '').trim(); return { MATCHED: '已匹配', PENDING: '待匹配', CONFLICT: '需人工确认', INDEX_STALE: '索引过期' }[value] || value || '—' }
|
||||||
function formatMatchRemark(row: ShopMatchShopQueueItem) { const message = (row.matchMessage || '').trim(); if (message) return message; if (row.matched && row.matchStatus === 'MATCHED') return '紫鸟索引已命中,可推送队列'; if (row.matched) return '已关联索引,请结合状态列查看是否可推送'; return '未命中或未就绪,请检查店铺名与索引刷新' }
|
function formatMatchRemark(row: ShopMatchShopQueueItem) { const message = (row.matchMessage || '').trim(); if (message) return message; if (row.matched && row.matchStatus === 'MATCHED') return '紫鸟索引已命中,可推送队列'; if (row.matched) return '已关联索引,请结合状态列查看是否可推送'; return '未命中或未就绪,请检查店铺名与索引刷新' }
|
||||||
async function pushToPythonQueue() { const api = getPywebviewApi(); if (!api?.enqueue_json) { ElMessage.error('当前客户端未提供 enqueue_json'); return } const matched = matchedItems.value.filter((item) => item.matched); if (!matched.length) { ElMessage.warning('请先匹配可用店铺'); return } if (!orderedCountryCodes.value.length) { ElMessage.warning('请至少勾选一个国家'); return } let scheduleValues: string[] | undefined; try { scheduleValues = parseScheduleValues() } catch (error) { ElMessage.error(error instanceof Error ? error.message : '定时配置无效'); return } pushing.value = true; queuePayloadText.value = ''; let successCount = 0; let failedCount = 0; try { for (let index = 0; index < matched.length; index += 1) { const item = matched[index]; try { const created = await withTransientRetry(() => createShopMatchTask([item], orderedCountryCodes.value, scheduleValues), (attempt, maxAttempts) => { queuePushResult.value = `后端服务暂时不可用,正在重试创建任务(${attempt}/${maxAttempts})...` }); const taskId = created.taskId; const initialStatus = scheduleValues?.length ? 'SCHEDULED' : 'RUNNING'; const snapshot: ShopMatchTaskDetailVo = { task: { id: taskId, status: initialStatus, scheduledAt: scheduleValues?.[0], countryCodes: [...orderedCountryCodes.value], currentStageIndex: scheduleValues?.length ? 0 : undefined, activeStageIndex: undefined, scheduleStages: (scheduleValues || []).map((value, stageIndex) => ({ stageIndex, scheduledAt: value, status: stageIndex === 0 ? 'SCHEDULED' : 'PENDING' })) }, items: created.items }; taskSnapshots.value = { ...taskSnapshots.value, [taskId]: snapshot }; taskDetails.value = { ...taskDetails.value, [taskId]: initialStatus }; saveTaskSnapshotsToStorage(); saveTaskDetailsToStorage(); const createdItem = created.items?.[0]; if (!createdItem) continue; if (scheduleValues?.length) { successCount += 1; restoreScheduledDispatches(); queuePushResult.value = `任务 ${taskId} 已创建,共 ${scheduleValues.length} 个执行时间点`; removeMatchedRowsLocally([item]); continue } const payload = buildQueuePayload(taskId, item, orderedCountryCodes.value); queuePayloadText.value = JSON.stringify(payload, null, 2); const result = await api.enqueue_json(payload); if (!result?.success) { failedCount += 1; queuePushResult.value = `任务 ${taskId} 推送失败,已自动继续下一条:${result?.error || '未知错误'}`; ElMessage.error(queuePushResult.value); continue } addPollingTask(taskId); ensurePolling(true); removeMatchedRowsLocally([item]); queuePushResult.value = matched.length > 1 ? `任务 ${taskId} 已入队,等待完成后继续推送下一条(${index + 1}/${matched.length})` : `任务 ${taskId} 已入队,等待执行完成`; const finalStatus = await waitForTaskTerminal(taskId); if (finalStatus !== 'SUCCESS') { failedCount += 1; queuePushResult.value = `任务 ${taskId} 执行失败,已自动继续下一条(${index + 1}/${matched.length})`; ElMessage.error(queuePushResult.value); continue } successCount += 1; queuePushResult.value = index + 1 < matched.length ? `任务 ${taskId} 已完成,继续推送下一条(${index + 1}/${matched.length})` : `任务 ${taskId} 已完成` } catch (error) { failedCount += 1; queuePushResult.value = `第 ${index + 1}/${matched.length} 条任务处理失败,已自动继续下一条:${error instanceof Error ? error.message : '未知错误'}`; ElMessage.error(queuePushResult.value); continue } } restoreScheduledDispatches(); if (!scheduleValues?.length) ensurePolling(true); ElMessage.success(scheduleValues?.length ? `定时任务已创建:成功 ${successCount} 条,失败 ${failedCount} 条` : `店铺推送已完成:成功 ${successCount} 条,失败 ${failedCount} 条`) } catch (error) { const message = error instanceof Error ? error.message : '推送失败'; queuePushResult.value = message; ElMessage.error(message) } finally { pushing.value = false } }
|
async function pushToPythonQueue() { autoQueueEnabled.value = true; await processMatchedQueue() }
|
||||||
|
function nextMatchedQueueItem() { return matchedItems.value.find((item) => item.matched) }
|
||||||
|
async function processMatchedQueue() { if (queueWorkerRunning.value) return; const api = getPywebviewApi(); if (!api?.enqueue_json) { ElMessage.error('当前客户端未提供 enqueue_json'); return } const matched = matchedItems.value.filter((item) => item.matched); if (!matched.length) { ElMessage.warning('请先匹配可用店铺'); return } if (!orderedCountryCodes.value.length) { ElMessage.warning('请至少勾选一个国家'); return } let scheduleValues: string[] | undefined; try { scheduleValues = parseScheduleValues() } catch (error) { ElMessage.error(error instanceof Error ? error.message : '定时配置无效'); return } queueWorkerRunning.value = true; pushing.value = true; queuePayloadText.value = ''; let successCount = 0; let failedCount = 0; let index = 0; try { while (autoQueueEnabled.value) { const item = nextMatchedQueueItem(); if (!item) break; index += 1; try { const created = await withTransientRetry(() => createShopMatchTask([item], orderedCountryCodes.value, scheduleValues), (attempt, maxAttempts) => { queuePushResult.value = `后端服务暂时不可用,正在重试创建任务(${attempt}/${maxAttempts})...` }); const taskId = created.taskId; const initialStatus = scheduleValues?.length ? 'SCHEDULED' : 'RUNNING'; const snapshot: ShopMatchTaskDetailVo = { task: { id: taskId, status: initialStatus, scheduledAt: scheduleValues?.[0], countryCodes: [...orderedCountryCodes.value], currentStageIndex: scheduleValues?.length ? 0 : undefined, activeStageIndex: undefined, scheduleStages: (scheduleValues || []).map((value, stageIndex) => ({ stageIndex, scheduledAt: value, status: stageIndex === 0 ? 'SCHEDULED' : 'PENDING' })) }, items: created.items }; taskSnapshots.value = { ...taskSnapshots.value, [taskId]: snapshot }; taskDetails.value = { ...taskDetails.value, [taskId]: initialStatus }; saveTaskSnapshotsToStorage(); saveTaskDetailsToStorage(); const createdItem = created.items?.[0]; if (!createdItem) { removeMatchedRowsLocally([item]); continue } if (scheduleValues?.length) { successCount += 1; restoreScheduledDispatches(); queuePushResult.value = `任务 ${taskId} 已创建,共 ${scheduleValues.length} 个执行时间点`; removeMatchedRowsLocally([item]); continue } const payload = buildQueuePayload(taskId, item, orderedCountryCodes.value); queuePayloadText.value = JSON.stringify(payload, null, 2); const result = await api.enqueue_json(payload); if (!result?.success) { failedCount += 1; queuePushResult.value = `任务 ${taskId} 推送失败,已自动继续下一条:${result?.error || '未知错误'}`; ElMessage.error(queuePushResult.value); removeMatchedRowsLocally([item]); continue } addPollingTask(taskId); ensurePolling(true); removeMatchedRowsLocally([item]); queuePushResult.value = `任务 ${taskId} 已入队,等待执行完成`; const finalStatus = await waitForTaskTerminal(taskId); if (finalStatus !== 'SUCCESS') { failedCount += 1; queuePushResult.value = `任务 ${taskId} 执行失败,已自动继续下一条`; ElMessage.error(queuePushResult.value); continue } successCount += 1; queuePushResult.value = `任务 ${taskId} 已完成` } catch (error) { failedCount += 1; queuePushResult.value = `第 ${index} 条任务处理失败,已自动继续下一条:${error instanceof Error ? error.message : '未知错误'}`; ElMessage.error(queuePushResult.value); removeMatchedRowsLocally([item]); continue } } restoreScheduledDispatches(); if (!scheduleValues?.length) ensurePolling(true); if (successCount > 0 || failedCount > 0) ElMessage.success(scheduleValues?.length ? `定时任务已创建:成功 ${successCount} 条,失败 ${failedCount} 条` : `店铺推送已完成:成功 ${successCount} 条,失败 ${failedCount} 条`) } catch (error) { const message = error instanceof Error ? error.message : '推送失败'; queuePushResult.value = message; ElMessage.error(message) } finally { queueWorkerRunning.value = false; pushing.value = false } }
|
||||||
onMounted(async () => { loadPollingIdsFromStorage(); loadTaskDetailsFromStorage(); loadTaskSnapshotsFromStorage(); loadMatchedItemsFromStorage(); schedulePickerValues.value = sanitizeSchedulePickerValues(schedulePickerValues.value, scheduleEnabled.value); syncPollingIdsWithTaskState(); await Promise.allSettled([loadCandidates(), loadCountryPreference(), loadHistory(), loadDashboard()]); startScheduleHeartbeat(); window.addEventListener('focus', handleScheduleRecovery); document.addEventListener('visibilitychange', handleScheduleRecovery); restoreScheduledDispatches(); if (pollingTaskIds.value.length) ensurePolling(true) })
|
onMounted(async () => { loadPollingIdsFromStorage(); loadTaskDetailsFromStorage(); loadTaskSnapshotsFromStorage(); loadMatchedItemsFromStorage(); schedulePickerValues.value = sanitizeSchedulePickerValues(schedulePickerValues.value, scheduleEnabled.value); syncPollingIdsWithTaskState(); await Promise.allSettled([loadCandidates(), loadCountryPreference(), loadHistory(), loadDashboard()]); startScheduleHeartbeat(); window.addEventListener('focus', handleScheduleRecovery); document.addEventListener('visibilitychange', handleScheduleRecovery); restoreScheduledDispatches(); if (pollingTaskIds.value.length) ensurePolling(true) })
|
||||||
onUnmounted(() => { stopPolling(); stopScheduleHeartbeat(); window.removeEventListener('focus', handleScheduleRecovery); document.removeEventListener('visibilitychange', handleScheduleRecovery); if (countryPrefSaveTimer) clearTimeout(countryPrefSaveTimer); for (const timer of dispatchTimers.values()) window.clearTimeout(timer); dispatchTimers.clear() })
|
onUnmounted(() => { stopPolling(); stopScheduleHeartbeat(); window.removeEventListener('focus', handleScheduleRecovery); document.removeEventListener('visibilitychange', handleScheduleRecovery); if (countryPrefSaveTimer) clearTimeout(countryPrefSaveTimer); for (const timer of dispatchTimers.values()) window.clearTimeout(timer); dispatchTimers.clear() })
|
||||||
</script>
|
</script>
|
||||||
|
|||||||
@@ -1211,7 +1211,9 @@ export interface PriceTrackShopQueueItem {
|
|||||||
export interface PriceTrackMatchShopsVo {
|
export interface PriceTrackMatchShopsVo {
|
||||||
items: PriceTrackShopQueueItem[];
|
items: PriceTrackShopQueueItem[];
|
||||||
skipAsinsByCountry?: Record<string, string[]>;
|
skipAsinsByCountry?: Record<string, string[]>;
|
||||||
|
skipAsinDetailsByCountry?: Record<string, Array<{ asin?: string; minimumPrice?: string }>>;
|
||||||
asinRowsByCountry?: Record<string, PriceTrackAsinParsedRow[]>;
|
asinRowsByCountry?: Record<string, PriceTrackAsinParsedRow[]>;
|
||||||
|
minimumPriceByCountryAndAsin?: Record<string, Record<string, string>>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface PriceTrackCountryPreferenceVo {
|
export interface PriceTrackCountryPreferenceVo {
|
||||||
@@ -1268,7 +1270,9 @@ export interface PriceTrackCreateTaskVo {
|
|||||||
taskId: number;
|
taskId: number;
|
||||||
items: PriceTrackHistoryItem[];
|
items: PriceTrackHistoryItem[];
|
||||||
skipAsinsByCountry?: Record<string, string[]>;
|
skipAsinsByCountry?: Record<string, string[]>;
|
||||||
|
skipAsinDetailsByCountry?: Record<string, Array<{ asin?: string; minimumPrice?: string }>>;
|
||||||
asinRowsByCountry?: Record<string, PriceTrackAsinParsedRow[]>;
|
asinRowsByCountry?: Record<string, PriceTrackAsinParsedRow[]>;
|
||||||
|
minimumPriceByCountryAndAsin?: Record<string, Record<string, string>>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface PriceTrackTaskSummary {
|
export interface PriceTrackTaskSummary {
|
||||||
|
|||||||
17
new_web_source/patrol-delete.html
Normal file
17
new_web_source/patrol-delete.html
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html lang="zh-CN">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<title>巡店删除 - 数富AI</title>
|
||||||
|
<script type="module" crossorigin src="/assets/patrol-delete.js"></script>
|
||||||
|
<link rel="modulepreload" crossorigin href="/assets/pywebview-DuyK2jB1.js">
|
||||||
|
<link rel="modulepreload" crossorigin href="/assets/zh-cn-Z9PBnl-n.js">
|
||||||
|
<link rel="stylesheet" crossorigin href="/assets/pywebview-D74jYk6P.css">
|
||||||
|
<link rel="stylesheet" crossorigin href="/assets/patrol-delete-DIvFZUTh.css">
|
||||||
|
<link rel="stylesheet" crossorigin href="/assets/el-input-CezJelDw.css">
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div id="app"></div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@@ -5,9 +5,10 @@
|
|||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
<title>跟价 - 数富AI</title>
|
<title>跟价 - 数富AI</title>
|
||||||
<script type="module" crossorigin src="/assets/price-track.js"></script>
|
<script type="module" crossorigin src="/assets/price-track.js"></script>
|
||||||
<link rel="modulepreload" crossorigin href="/assets/pywebview-DiP0HdY6.js">
|
<link rel="modulepreload" crossorigin href="/assets/pywebview-DuyK2jB1.js">
|
||||||
<link rel="stylesheet" crossorigin href="/assets/pywebview-D-mMH8F6.css">
|
<link rel="modulepreload" crossorigin href="/assets/brand-CKgrwtni.js">
|
||||||
<link rel="stylesheet" crossorigin href="/assets/price-track-BO5PtL0W.css">
|
<link rel="stylesheet" crossorigin href="/assets/pywebview-D74jYk6P.css">
|
||||||
|
<link rel="stylesheet" crossorigin href="/assets/price-track-BccJT4Qr.css">
|
||||||
<link rel="stylesheet" crossorigin href="/assets/el-input-CezJelDw.css">
|
<link rel="stylesheet" crossorigin href="/assets/el-input-CezJelDw.css">
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
|
|||||||
@@ -5,11 +5,12 @@
|
|||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
<title>匹配店铺 - 数富AI</title>
|
<title>匹配店铺 - 数富AI</title>
|
||||||
<script type="module" crossorigin src="/assets/shop-match.js"></script>
|
<script type="module" crossorigin src="/assets/shop-match.js"></script>
|
||||||
<link rel="modulepreload" crossorigin href="/assets/pywebview-DiP0HdY6.js">
|
<link rel="modulepreload" crossorigin href="/assets/pywebview-DuyK2jB1.js">
|
||||||
|
<link rel="modulepreload" crossorigin href="/assets/zh-cn-Z9PBnl-n.js">
|
||||||
<link rel="modulepreload" crossorigin href="/assets/listingFilters-BpGOU_pJ.js">
|
<link rel="modulepreload" crossorigin href="/assets/listingFilters-BpGOU_pJ.js">
|
||||||
<link rel="stylesheet" crossorigin href="/assets/pywebview-D-mMH8F6.css">
|
<link rel="stylesheet" crossorigin href="/assets/pywebview-D74jYk6P.css">
|
||||||
<link rel="stylesheet" crossorigin href="/assets/listingFilters-CK58rX4v.css">
|
<link rel="stylesheet" crossorigin href="/assets/listingFilters-CK58rX4v.css">
|
||||||
<link rel="stylesheet" crossorigin href="/assets/shop-match-MsxXMG_G.css">
|
<link rel="stylesheet" crossorigin href="/assets/shop-match-DNFA8ceR.css">
|
||||||
<link rel="stylesheet" crossorigin href="/assets/el-input-CezJelDw.css">
|
<link rel="stylesheet" crossorigin href="/assets/el-input-CezJelDw.css">
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
|
|||||||
Reference in New Issue
Block a user