super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

更新新增三个模块

Browse Source
This commit is contained in:
super
2026-03-22 11:20:09 +08:00
parent 74a5fb9ce1
commit 8e66926b9e
112 changed files with 5411 additions and 606 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
app/__pycache__/app.cpython-312.pyc
View File

Binary file not shown.

BIN
app/__pycache__/config.cpython-312.pyc
View File

Binary file not shown.

1
app/assets/convert-B191qXN9.css Normal file
View File

File diff suppressed because one or more lines are too long

1
app/assets/convert-D4Yv2oRy.css
View File

File diff suppressed because one or more lines are too long

2
app/assets/convert.js
View File

File diff suppressed because one or more lines are too long

1
app/assets/dedupe-BHE_BCAJ.css
View File

File diff suppressed because one or more lines are too long

1
app/assets/dedupe-BKyfmsjj.css Normal file
View File

File diff suppressed because one or more lines are too long

2
app/assets/dedupe.js
View File

File diff suppressed because one or more lines are too long

4
app/assets/pywebview-Rm5Hhqpf.js → app/assets/pywebview-DVgP2-kW.js
View File

File diff suppressed because one or more lines are too long

2
app/assets/pywebview-Chz98E9c.css → app/assets/pywebview-IqgMfeBe.css
View File

File diff suppressed because one or more lines are too long

1
app/assets/split-CYvQPVKL.css Normal file
View File

File diff suppressed because one or more lines are too long

1
app/assets/split-_O6NHqDj.css
View File

File diff suppressed because one or more lines are too long

2
app/assets/split.js
View File

File diff suppressed because one or more lines are too long

BIN
app/blueprints/__pycache__/brand.cpython-312.pyc
View File

Binary file not shown.

25
app/blueprints/brand.py
View File

@@ -57,14 +57,23 @@ def _push_task_event(task_id, event):
def _expand_folder_xlsx(folder_path):
"""返回文件夹下所有 .xlsx 文件的绝对路径列表"""
"""返回文件夹下所有 .xlsx 文件的绝对路径和相对路径列表（递归）"""
if not folder_path or not os.path.isdir(folder_path):
return []
paths = []
for name in os.listdir(folder_path):
if name.endswith('.xlsx') or name.endswith('.XLSX'):
paths.append(os.path.normpath(os.path.join(folder_path, name)))
return paths
items = []
root = os.path.normpath(folder_path)
for current_root, _, filenames in os.walk(root):
for name in filenames:
if not (name.endswith('.xlsx') or name.endswith('.XLSX')):
continue
absolute_path = os.path.normpath(os.path.join(current_root, name))
relative_path = os.path.relpath(absolute_path, root).replace('\\', '/')
items.append({
'absolutePath': absolute_path,
'relativePath': relative_path,
})
items.sort(key=lambda item: item['relativePath'])
return items
def _upload_local_xlsx_to_oss(local_path, user_id):
@@ -478,8 +487,8 @@ def api_brand_expand_folder():
folder = (data.get('folder') or '').strip()
if not folder:
return jsonify({'success': False, 'error': '请提供 folder 路径'}), 400
paths = _expand_folder_xlsx(folder)
return jsonify({'success': True, 'paths': paths})
items = _expand_folder_xlsx(folder)
return jsonify({'success': True, 'items': items})
except Exception as e:
return jsonify({'success': False, 'error': str(e)}), 500

3
app/main.py
View File

@@ -145,7 +145,7 @@ class WindowAPI:
)
return result[0] if result else ''
def upload_file_to_java(self, file_path):
def upload_file_to_java(self, file_path, relative_path=None):
"""按本地路径读取文件并上传到 Java 后端临时目录。"""
if not file_path or not str(file_path).strip():
return {'success': False, 'error': '文件路径为空'}
@@ -157,6 +157,7 @@ class WindowAPI:
response = requests.post(
f"{JAVA_API_BASE}/api/files/upload",
files={'file': (os.path.basename(normalized_path), file_obj)},
data={'relativePath': str(relative_path).strip()} if relative_path else None,
timeout=120,
)
response.raise_for_status()

6
app/new_web_source/convert.html
View File

@@ -5,9 +5,9 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>格式转换 - 数富AI</title>
<script type="module" crossorigin src="/assets/convert.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-Rm5Hhqpf.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-Chz98E9c.css">
<link rel="stylesheet" crossorigin href="/assets/convert-D4Yv2oRy.css">
<link rel="modulepreload" crossorigin href="/assets/pywebview-DVgP2-kW.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-IqgMfeBe.css">
<link rel="stylesheet" crossorigin href="/assets/convert-B191qXN9.css">
</head>
<body>
<div id="app"></div>

6
app/new_web_source/dedupe.html
View File

@@ -5,9 +5,9 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>数据去重 - 数富AI</title>
<script type="module" crossorigin src="/assets/dedupe.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-Rm5Hhqpf.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-Chz98E9c.css">
<link rel="stylesheet" crossorigin href="/assets/dedupe-BHE_BCAJ.css">
<link rel="modulepreload" crossorigin href="/assets/pywebview-DVgP2-kW.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-IqgMfeBe.css">
<link rel="stylesheet" crossorigin href="/assets/dedupe-BKyfmsjj.css">
</head>
<body>
<div id="app"></div>

6
app/new_web_source/split.html
View File

@@ -5,9 +5,9 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>数据拆分 - 数富AI</title>
<script type="module" crossorigin src="/assets/split.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-Rm5Hhqpf.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-Chz98E9c.css">
<link rel="stylesheet" crossorigin href="/assets/split-_O6NHqDj.css">
<link rel="modulepreload" crossorigin href="/assets/pywebview-DVgP2-kW.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-IqgMfeBe.css">
<link rel="stylesheet" crossorigin href="/assets/split-CYvQPVKL.css">
</head>
<body>
<div id="app"></div>

3
backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/dto/ConvertRunRequest.java
View File

@@ -20,4 +20,7 @@ public class ConvertRunRequest {
@NotBlank(message = "请选择模板")
@Schema(description = "模板编码，例如 uk_offer")
private String templateId;
@Schema(description = "文件夹上传场景下的总压缩包名称（不含扩展名）")
private String archiveName;
}

3
backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/dto/UploadedSourceFileDto.java
View File

@@ -14,4 +14,7 @@ public class UploadedSourceFileDto {
@Schema(description = "原始文件名")
private String originalFilename;
@Schema(description = "相对上传目录路径")
private String relativePath;
}

253
backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertRunService.java
View File

@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.convert.service;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.json.JSONUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
@@ -22,24 +23,38 @@ import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@Service
@RequiredArgsConstructor
public class ConvertRunService {
private static final String MODULE_TYPE = "CONVERT";
private static final String TEMPLATE_CODE_FIVE_COUNTRIES = "uk_offer";
private static final List<String> FIVE_COUNTRY_OUTPUT_FILES = List.of(
"\u82f1\u56fd.txt",
"\u6cd5\u56fd.txt",
"\u5fb7\u56fd.txt",
"\u897f\u73ed\u7259.txt",
"\u610f\u5927\u5229.txt"
);
private final FileTaskMapper fileTaskMapper;
private final FileResultMapper fileResultMapper;
private final StorageProperties storageProperties;
@@ -51,8 +66,8 @@ public class ConvertRunService {
ConvertTemplateEntity template = convertTemplateService.getByCode(request.getTemplateId());
FileTaskEntity task = new FileTaskEntity();
task.setTaskNo("CONVERT-" + IdUtil.getSnowflakeNextIdStr());
task.setModuleType("CONVERT");
task.setTaskNo(MODULE_TYPE + "-" + IdUtil.getSnowflakeNextIdStr());
task.setModuleType(MODULE_TYPE);
task.setTaskMode("IMMEDIATE");
task.setStatus("SUCCESS");
task.setSourceFileCount(request.getFiles().size());
@@ -63,63 +78,107 @@ public class ConvertRunService {
task.setFinishedAt(LocalDateTime.now());
fileTaskMapper.insert(task);
boolean folderMode = request.getArchiveName() != null && !request.getArchiveName().isBlank();
List<ConvertResultItemVo> items = new ArrayList<>();
int successCount = 0;
int failedCount = 0;
List<ConvertArchiveEntry> archiveEntries = new ArrayList<>();
for (UploadedSourceFileDto sourceFile : request.getFiles()) {
ConvertResultItemVo item = new ConvertResultItemVo();
item.setSourceFilename(sourceFile.getOriginalFilename());
try {
File inputFile = findLocalSourceFile(sourceFile.getFileKey());
if (inputFile == null || !inputFile.exists()) {
throw new BusinessException("上传文件不存在，请重新上传");
throw new BusinessException("Uploaded source file does not exist, please upload again.");
}
String inputName = sourceFile.getOriginalFilename() == null ? inputFile.getName() : sourceFile.getOriginalFilename();
String outputFilename = template.getOutputFilename() == null || template.getOutputFilename().isBlank()
? "结果.txt"
: template.getOutputFilename();
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "convert-result"));
File outputFile = buildNamedOutputFile(outputDir, outputFilename);
String inputName = sourceFile.getOriginalFilename() == null
? inputFile.getName()
: sourceFile.getOriginalFilename();
List<GeneratedConvertFile> generatedFiles = generateOutputFiles(inputFile, template);
if (folderMode) {
for (GeneratedConvertFile generatedFile : generatedFiles) {
archiveEntries.add(new ConvertArchiveEntry(sourceFile.getRelativePath(), inputName, generatedFile));
}
successCount++;
continue;
}
writeTxtByLegacyRules(inputFile, outputFile, template);
boolean useZipPackage = request.getFiles().size() == 1 && generatedFiles.size() > 1;
File packagedResultFile = useZipPackage
? packageGeneratedFilesAsZip(inputName, generatedFiles)
: generatedFiles.getFirst().file();
String packagedFilename = useZipPackage
? FileUtil.mainName(inputName) + ".zip"
: generatedFiles.getFirst().filename();
String contentType = useZipPackage ? "application/zip" : "text/plain";
String ossObjectKey = ossStorageService.uploadResultFile(outputFile, "CONVERT");
String ossObjectKey = ossStorageService.uploadResultFile(packagedResultFile, MODULE_TYPE);
String downloadUrl = ossStorageService.generateDownloadUrl(ossObjectKey);
ConvertResultItemVo item = new ConvertResultItemVo();
item.setSourceFilename(inputName);
item.setSuccess(true);
item.setOutputFilename(outputFilename);
item.setOutputFilename(packagedFilename);
item.setDownloadUrl(downloadUrl);
successCount++;
FileResultEntity resultEntity = new FileResultEntity();
resultEntity.setTaskId(task.getId());
resultEntity.setModuleType("CONVERT");
resultEntity.setModuleType(MODULE_TYPE);
resultEntity.setSourceFilename(inputName);
resultEntity.setResultFilename(outputFilename);
resultEntity.setResultFilename(packagedFilename);
resultEntity.setResultFileUrl(downloadUrl);
resultEntity.setResultFileSize(outputFile.length());
resultEntity.setResultContentType("text/plain");
resultEntity.setResultFileSize(packagedResultFile.length());
resultEntity.setResultContentType(contentType);
resultEntity.setSuccess(1);
resultEntity.setCreatedAt(LocalDateTime.now());
fileResultMapper.insert(resultEntity);
item.setResultId(resultEntity.getId());
items.add(item);
successCount++;
} catch (Exception ex) {
ConvertResultItemVo item = new ConvertResultItemVo();
item.setSourceFilename(sourceFile.getOriginalFilename());
item.setSuccess(false);
item.setError(ex.getMessage());
items.add(item);
failedCount++;
FileResultEntity resultEntity = new FileResultEntity();
resultEntity.setTaskId(task.getId());
resultEntity.setModuleType("CONVERT");
resultEntity.setModuleType(MODULE_TYPE);
resultEntity.setSourceFilename(sourceFile.getOriginalFilename());
resultEntity.setSuccess(0);
resultEntity.setErrorMessage(ex.getMessage());
resultEntity.setCreatedAt(LocalDateTime.now());
fileResultMapper.insert(resultEntity);
}
items.add(item);
}
if (folderMode && !archiveEntries.isEmpty()) {
File zipFile = packageFolderConvertResultsAsZip(request.getArchiveName(), archiveEntries);
String ossObjectKey = ossStorageService.uploadResultFile(zipFile, MODULE_TYPE);
String downloadUrl = ossStorageService.generateDownloadUrl(ossObjectKey);
ConvertResultItemVo item = new ConvertResultItemVo();
item.setSourceFilename(request.getArchiveName());
item.setOutputFilename(zipFile.getName());
item.setSuccess(true);
item.setDownloadUrl(downloadUrl);
FileResultEntity resultEntity = new FileResultEntity();
resultEntity.setTaskId(task.getId());
resultEntity.setModuleType(MODULE_TYPE);
resultEntity.setSourceFilename(request.getArchiveName());
resultEntity.setResultFilename(zipFile.getName());
resultEntity.setResultFileUrl(downloadUrl);
resultEntity.setResultFileSize(zipFile.length());
resultEntity.setResultContentType("application/zip");
resultEntity.setSuccess(1);
resultEntity.setCreatedAt(LocalDateTime.now());
fileResultMapper.insert(resultEntity);
item.setResultId(resultEntity.getId());
items.add(0, item);
}
task.setSuccessFileCount(successCount);
@@ -138,18 +197,18 @@ public class ConvertRunService {
public List<ConvertResultItemVo> listHistory() {
return fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getModuleType, "CONVERT")
.eq(FileResultEntity::getSuccess, 1)
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.orderByDesc(FileResultEntity::getCreatedAt)
.last("limit 50"))
.last("limit 200"))
.stream()
.map(entity -> {
ConvertResultItemVo vo = new ConvertResultItemVo();
vo.setResultId(entity.getId());
vo.setSourceFilename(entity.getCreatedAt() == null ? "" : entity.getCreatedAt().toLocalDate().toString().replace("-", ""));
vo.setSourceFilename(entity.getSourceFilename());
vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(entity.getResultFileUrl());
vo.setSuccess(true);
vo.setDownloadUrl(entity.getSuccess() != null && entity.getSuccess() == 1 ? entity.getResultFileUrl() : null);
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
vo.setError(entity.getErrorMessage());
return vo;
})
.toList();
@@ -157,41 +216,71 @@ public class ConvertRunService {
public void deleteHistory(Long resultId) {
FileResultEntity entity = fileResultMapper.selectById(resultId);
if (entity == null || !"CONVERT".equals(entity.getModuleType())) {
throw new BusinessException("记录不存在");
if (entity == null || !MODULE_TYPE.equals(entity.getModuleType())) {
throw new BusinessException("Convert result record does not exist.");
}
fileResultMapper.deleteById(resultId);
}
public File getResultFile(Long resultId) {
FileResultEntity entity = fileResultMapper.selectById(resultId);
if (entity == null || entity.getResultFileUrl() == null || !"CONVERT".equals(entity.getModuleType())) {
throw new BusinessException("结果文件不存在");
if (entity == null || entity.getResultFileUrl() == null || !MODULE_TYPE.equals(entity.getModuleType())) {
throw new BusinessException("Convert result file does not exist.");
}
File file = new File(entity.getResultFileUrl());
if (!file.exists()) {
throw new BusinessException("结果文件不存在");
throw new BusinessException("Convert result file does not exist.");
}
return file;
}
private void writeTxtByLegacyRules(File inputFile, File outputFile, ConvertTemplateEntity templateEntity) throws IOException {
private List<GeneratedConvertFile> generateOutputFiles(File inputFile, ConvertTemplateEntity templateEntity) throws IOException {
List<String> outputFilenames = resolveOutputFilenames(templateEntity);
List<String> rows = buildTxtRows(inputFile, templateEntity);
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "convert-result"));
List<GeneratedConvertFile> generatedFiles = new ArrayList<>();
for (String outputFilename : outputFilenames) {
File outputFile = buildNamedOutputFile(outputDir, outputFilename);
FileUtil.writeLines(rows, outputFile, StandardCharsets.UTF_8);
generatedFiles.add(new GeneratedConvertFile(outputFilename, outputFile));
}
return generatedFiles;
}
private List<String> resolveOutputFilenames(ConvertTemplateEntity templateEntity) {
if (TEMPLATE_CODE_FIVE_COUNTRIES.equals(templateEntity.getTemplateCode())) {
return FIVE_COUNTRY_OUTPUT_FILES;
}
String outputFilename = templateEntity.getOutputFilename();
if (outputFilename == null || outputFilename.isBlank()) {
return List.of("result.txt");
}
return List.of(outputFilename);
}
private List<String> buildTxtRows(File inputFile, ConvertTemplateEntity templateEntity) throws IOException {
List<String> requiredColumns = readStringList(templateEntity.getRequiredSourceColumnsJson());
List<String> headerColumns = readStringList(templateEntity.getHeaderColumnsJson());
Map<String, String> fieldMapping = readStringMap(templateEntity.getFieldMappingJson());
Map<String, String> defaults = readStringMap(templateEntity.getDefaultsJson());
List<String> preambleLines = readStringList(templateEntity.getPreambleLinesJson());
int blankHeaderRowsAfterSchema = templateEntity.getBlankHeaderRowsAfterSchema() == null ? 0 : templateEntity.getBlankHeaderRowsAfterSchema();
int blankHeaderRowsAfterSchema = templateEntity.getBlankHeaderRowsAfterSchema() == null
? 0
: templateEntity.getBlankHeaderRowsAfterSchema();
DataFormatter formatter = new DataFormatter();
try (FileInputStream fis = new FileInputStream(inputFile); Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis)) {
try (FileInputStream fis = new FileInputStream(inputFile);
Workbook workbook = WorkbookFactory.create(fis)) {
Sheet sheet = workbook.getSheetAt(0);
Row headerRow = sheet.getRow(0);
if (headerRow == null) {
throw new BusinessException("Excel 表头为空");
throw new BusinessException("Excel header row is empty.");
}
Map<String, Integer> headerMap = new HashMap<>();
Map<String, Integer> headerMap = new LinkedHashMap<>();
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
String value = normalizeCellText(formatter.formatCellValue(headerRow.getCell(i)));
if (!value.isBlank() && !headerMap.containsKey(value)) {
@@ -199,9 +288,11 @@ public class ConvertRunService {
}
}
List<String> missing = requiredColumns.stream().filter(column -> !headerMap.containsKey(column)).toList();
List<String> missing = requiredColumns.stream()
.filter(column -> !headerMap.containsKey(column))
.toList();
if (!missing.isEmpty()) {
throw new BusinessException("缺少列：" + String.join("、", missing));
throw new BusinessException("Missing required columns: " + String.join(", ", missing));
}
List<String> rows = new ArrayList<>();
@@ -219,6 +310,7 @@ public class ConvertRunService {
if (row == null) {
continue;
}
String asin = asinIndex >= 0 ? normalizeCellText(formatter.formatCellValue(row.getCell(asinIndex))) : "";
if (asin.isBlank()) {
continue;
@@ -233,7 +325,9 @@ public class ConvertRunService {
if (fieldMapping.containsKey(column)) {
String sourceColumn = fieldMapping.get(column);
Integer sourceIndex = headerMap.get(sourceColumn);
String value = sourceIndex == null ? "" : normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex)));
String value = sourceIndex == null
? ""
: normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex)));
lineValues.add(value);
continue;
}
@@ -243,10 +337,11 @@ public class ConvertRunService {
}
lineValues.add("");
}
rows.add(String.join("\t", lineValues));
rowIndex++;
}
FileUtil.writeLines(rows, outputFile, StandardCharsets.UTF_8);
return rows;
}
}
@@ -255,20 +350,70 @@ public class ConvertRunService {
if (json == null || json.isBlank()) {
return new ArrayList<>();
}
return objectMapper.readValue(json, new TypeReference<List<String>>() {});
return objectMapper.readValue(json, new TypeReference<List<String>>() {
});
} catch (Exception ex) {
throw new BusinessException("模板配置解析失败");
throw new BusinessException("Template config parsing failed.");
}
}
private File packageGeneratedFilesAsZip(String inputName, List<GeneratedConvertFile> generatedFiles) {
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "convert-result"));
File zipFile = buildNamedOutputFile(outputDir, FileUtil.mainName(inputName) + ".zip");
try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zipFile))) {
for (GeneratedConvertFile generatedFile : generatedFiles) {
ZipEntry entry = new ZipEntry(generatedFile.filename());
zos.putNextEntry(entry);
Files.copy(generatedFile.file().toPath(), zos);
zos.closeEntry();
}
} catch (Exception ex) {
throw new BusinessException("Convert result packaging failed: " + ex.getMessage());
}
return zipFile;
}
private File packageFolderConvertResultsAsZip(String archiveName, List<ConvertArchiveEntry> archiveEntries) {
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "convert-result"));
File zipFile = buildNamedOutputFile(outputDir, archiveName + ".zip");
try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zipFile))) {
for (ConvertArchiveEntry entry : archiveEntries) {
String zipEntryName = buildFolderZipEntry(entry.relativePath(), entry.inputName(), entry.generatedFile().filename(), true);
zos.putNextEntry(new ZipEntry(zipEntryName));
Files.copy(entry.generatedFile().file().toPath(), zos);
zos.closeEntry();
}
} catch (Exception ex) {
throw new BusinessException("Convert folder result packaging failed: " + ex.getMessage());
}
return zipFile;
}
private String buildFolderZipEntry(String relativePath, String inputName, String outputFilename, boolean includeSourceStemFolder) {
String normalizedRelativePath = relativePath == null ? "" : relativePath.replace('\\', '/');
String relativeDir = normalizedRelativePath.contains("/")
? normalizedRelativePath.substring(0, normalizedRelativePath.lastIndexOf('/'))
: "";
List<String> parts = new ArrayList<>();
if (!relativeDir.isBlank()) {
parts.add(relativeDir);
}
if (includeSourceStemFolder) {
parts.add(FileUtil.mainName(inputName));
}
parts.add(outputFilename);
return String.join("/", parts);
}
private Map<String, String> readStringMap(String json) {
try {
if (json == null || json.isBlank()) {
return new HashMap<>();
}
return objectMapper.readValue(json, new TypeReference<Map<String, String>>() {});
return objectMapper.readValue(json, new TypeReference<Map<String, String>>() {
});
} catch (Exception ex) {
throw new BusinessException("模板配置解析失败");
throw new BusinessException("Template config parsing failed.");
}
}
@@ -277,8 +422,11 @@ public class ConvertRunService {
if (!baseDir.exists()) {
return null;
}
List<File> matchedFiles = FileUtil.loopFiles(baseDir, pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey));
return matchedFiles.isEmpty() ? null : matchedFiles.getFirst();
List<File> matchedFiles = FileUtil.loopFiles(
baseDir,
pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey)
);
return matchedFiles.isEmpty() ? null : matchedFiles.get(0);
}
private File buildNamedOutputFile(File outputDir, String filename) {
@@ -286,6 +434,7 @@ public class ConvertRunService {
if (!candidate.exists()) {
return candidate;
}
String mainName = FileUtil.mainName(filename);
String extName = FileUtil.extName(filename);
int index = 2;
@@ -314,4 +463,10 @@ public class ConvertRunService {
.trim()
.replaceAll("\\s+", " ");
}
private record GeneratedConvertFile(String filename, File file) {
}
private record ConvertArchiveEntry(String relativePath, String inputName, GeneratedConvertFile generatedFile) {
}
}

1
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeRunController.java
View File

@@ -45,6 +45,7 @@ public class DedupeRunController {
- 仅保留 selectedColumns 中指定的列；
- keepIntegerIds=true 时保留纯数字 ID；
- keepUnderscoreIds=true 时保留类似 1_1 的 ID；
- keepIntegerMainIdsWhenNoSubIds=true 且文件中不存在 1_1 这类子 ID 时，会自动保留纯数字主 ID；
- 输出文件命名遵循 原文件名_cleaned.xlsx，若重名自动追加序号。
""")
@ApiResponses({

93
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeTotalDataController.java Normal file
View File

@@ -0,0 +1,93 @@
package com.nanri.aiimage.modules.dedupe.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataCreateRequest;
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataUpdateRequest;
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataImportVo;
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataItemVo;
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataPageVo;
import com.nanri.aiimage.modules.dedupe.service.DedupeTotalDataService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/admin/dedupe-total-data")
@Tag(name = "数据去重总数据", description = "维护数据去重模块的总数据列表，支持增删改查。")
public class DedupeTotalDataController {
private final DedupeTotalDataService dedupeTotalDataService;
@GetMapping
@Operation(summary = "分页查询总数据", description = "分页查询数据去重总数据，支持按值模糊搜索。")
@ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功", content = @Content(schema = @Schema(implementation = DedupeTotalDataPageVo.class)))
})
public ApiResponse<DedupeTotalDataPageVo> page(
@Parameter(description = "页码") @RequestParam(defaultValue = "1") Long page,
@Parameter(description = "每页数量") @RequestParam(defaultValue = "15") Long pageSize,
@Parameter(description = "模糊搜索关键字") @RequestParam(required = false) String keyword) {
return ApiResponse.success(dedupeTotalDataService.page(page, pageSize, keyword));
}
@PostMapping
@Operation(summary = "新增总数据", description = "新增一条数据去重总数据。")
@ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "创建成功", content = @Content(schema = @Schema(implementation = DedupeTotalDataItemVo.class))),
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "参数不合法或数据重复")
})
public ApiResponse<DedupeTotalDataItemVo> create(@Valid @RequestBody DedupeTotalDataCreateRequest request) {
return ApiResponse.success("创建成功", dedupeTotalDataService.create(request));
}
@PostMapping("/import")
@Operation(summary = "导入总数据", description = "上传 xlsx 文件，读取 ASIN 列并批量导入数据去重总数据。")
@ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "导入成功", content = @Content(schema = @Schema(implementation = DedupeTotalDataImportVo.class))),
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "文件不合法或缺少 ASIN 列")
})
public ApiResponse<DedupeTotalDataImportVo> importExcel(
@Parameter(description = "xlsx 文件", required = true) @RequestParam("file") MultipartFile file) {
return ApiResponse.success("导入成功", dedupeTotalDataService.importFromExcel(file));
}
@PutMapping("/{id}")
@Operation(summary = "更新总数据", description = "按 ID 更新一条数据去重总数据。")
@ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "更新成功", content = @Content(schema = @Schema(implementation = DedupeTotalDataItemVo.class))),
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "参数不合法或数据重复"),
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "数据不存在")
})
public ApiResponse<DedupeTotalDataItemVo> update(
@Parameter(description = "主键ID", required = true) @PathVariable Long id,
@Valid @RequestBody DedupeTotalDataUpdateRequest request) {
return ApiResponse.success("更新成功", dedupeTotalDataService.update(id, request));
}
@DeleteMapping("/{id}")
@Operation(summary = "删除总数据", description = "按 ID 删除一条数据去重总数据。")
@ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "删除成功"),
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "数据不存在")
})
public ApiResponse<Void> delete(@Parameter(description = "主键ID", required = true) @PathVariable Long id) {
dedupeTotalDataService.delete(id);
return ApiResponse.success("删除成功", null);
}
}

9
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/mapper/DedupeTotalDataMapper.java Normal file
View File

@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.dedupe.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.dedupe.model.entity.DedupeTotalDataEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface DedupeTotalDataMapper extends BaseMapper<DedupeTotalDataEntity> {
}

8
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeRunRequest.java
View File

@@ -21,8 +21,14 @@ public class DedupeRunRequest {
private List<String> selectedColumns;
@Schema(description = "是否保留纯数字 ID")
private boolean keepIntegerIds = true;
private boolean keepIntegerIds = false;
@Schema(description = "是否保留带下划线 ID")
private boolean keepUnderscoreIds = true;
@Schema(description = "当文件中不存在下划线子 ID 时，是否自动保留纯数字主 ID")
private boolean keepIntegerMainIdsWhenNoSubIds = true;
@Schema(description = "文件夹上传场景下的总压缩包名称（不含扩展名）")
private String archiveName;
}

3
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeSourceFileDto.java
View File

@@ -14,4 +14,7 @@ public class DedupeSourceFileDto {
@Schema(description = "原始文件名")
private String originalFilename;
@Schema(description = "相对上传目录路径")
private String relativePath;
}

16
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeTotalDataCreateRequest.java Normal file
View File

@@ -0,0 +1,16 @@
package com.nanri.aiimage.modules.dedupe.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.Size;
import lombok.Data;
@Data
@Schema(description = "数据去重总数据新增请求")
public class DedupeTotalDataCreateRequest {
@NotBlank(message = "数据不能为空")
@Size(max = 128, message = "数据长度不能超过128个字符")
@Schema(description = "总数据值", requiredMode = Schema.RequiredMode.REQUIRED)
private String dataValue;
}

16
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeTotalDataUpdateRequest.java Normal file
View File

@@ -0,0 +1,16 @@
package com.nanri.aiimage.modules.dedupe.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.Size;
import lombok.Data;
@Data
@Schema(description = "数据去重总数据更新请求")
public class DedupeTotalDataUpdateRequest {
@NotBlank(message = "数据不能为空")
@Size(max = 128, message = "数据长度不能超过128个字符")
@Schema(description = "总数据值", requiredMode = Schema.RequiredMode.REQUIRED)
private String dataValue;
}

19
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/entity/DedupeTotalDataEntity.java Normal file
View File

@@ -0,0 +1,19 @@
package com.nanri.aiimage.modules.dedupe.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("biz_dedupe_total_data")
public class DedupeTotalDataEntity {
@TableId(type = IdType.AUTO)
private Long id;
private String dataValue;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}

21
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeTotalDataImportVo.java Normal file
View File

@@ -0,0 +1,21 @@
package com.nanri.aiimage.modules.dedupe.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "数据去重总数据导入结果")
public class DedupeTotalDataImportVo {
@Schema(description = "读取总行数")
private Integer totalRows;
@Schema(description = "读取到的 ASIN 数量")
private Integer asinCount;
@Schema(description = "实际新增数量")
private Integer insertedCount;
@Schema(description = "跳过数量")
private Integer skippedCount;
}

20
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeTotalDataItemVo.java Normal file
View File

@@ -0,0 +1,20 @@
package com.nanri.aiimage.modules.dedupe.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@Schema(description = "数据去重总数据项")
public class DedupeTotalDataItemVo {
@Schema(description = "主键ID")
private Long id;
@Schema(description = "总数据值")
private String dataValue;
@Schema(description = "创建时间")
private LocalDateTime createdAt;
}

23
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeTotalDataPageVo.java Normal file
View File

@@ -0,0 +1,23 @@
package com.nanri.aiimage.modules.dedupe.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "数据去重总数据分页结果")
public class DedupeTotalDataPageVo {
@Schema(description = "列表")
private List<DedupeTotalDataItemVo> items;
@Schema(description = "总数")
private Long total;
@Schema(description = "页码")
private Long page;
@Schema(description = "每页数量")
private Long pageSize;
}

148
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java
View File

@@ -28,11 +28,14 @@ import org.springframework.stereotype.Service;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.nio.file.Files;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
@@ -46,7 +49,7 @@ public class DedupeRunService {
private final OssStorageService ossStorageService;
public DedupeRunVo run(DedupeRunRequest request) {
if (!request.isKeepIntegerIds() && !request.isKeepUnderscoreIds()) {
if (!request.isKeepIntegerIds() && !request.isKeepUnderscoreIds() && !request.isKeepIntegerMainIdsWhenNoSubIds()) {
throw new BusinessException("请至少选择一种 ID 保留规则");
}
@@ -63,9 +66,11 @@ public class DedupeRunService {
task.setFinishedAt(LocalDateTime.now());
fileTaskMapper.insert(task);
boolean folderMode = request.getArchiveName() != null && !request.getArchiveName().isBlank();
List<DedupeResultItemVo> items = new ArrayList<>();
int successCount = 0;
int failedCount = 0;
List<DedupeArchiveEntry> archiveEntries = new ArrayList<>();
for (DedupeSourceFileDto sourceFile : request.getFiles()) {
DedupeResultItemVo item = new DedupeResultItemVo();
@@ -81,13 +86,28 @@ public class DedupeRunService {
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "dedupe-result"));
File outputFile = buildNamedOutputFile(outputDir, outputFilename);
cleanExcelByLegacyRules(inputFile, outputFile, request.getSelectedColumns(), request.isKeepIntegerIds(), request.isKeepUnderscoreIds());
cleanExcelByLegacyRules(
inputFile,
outputFile,
request.getSelectedColumns(),
request.isKeepIntegerIds(),
request.isKeepUnderscoreIds(),
request.isKeepIntegerMainIdsWhenNoSubIds()
);
if (folderMode) {
archiveEntries.add(new DedupeArchiveEntry(sourceFile.getRelativePath(), inputName, outputFile));
successCount++;
continue;
}
String ossObjectKey = ossStorageService.uploadResultFile(outputFile, "DEDUPE");
String downloadUrl = ossStorageService.generateDownloadUrl(ossObjectKey);
String downloadFilename = request.getFiles().size() == 1 ? inputName : outputFile.getName();
item.setSuccess(true);
item.setOutputFilename(outputFile.getName());
item.setOutputFilename(downloadFilename);
item.setDownloadUrl(downloadUrl);
successCount++;
@@ -95,7 +115,7 @@ public class DedupeRunService {
resultEntity.setTaskId(task.getId());
resultEntity.setModuleType("DEDUPE");
resultEntity.setSourceFilename(inputName);
resultEntity.setResultFilename(outputFile.getName());
resultEntity.setResultFilename(downloadFilename);
resultEntity.setResultFileUrl(downloadUrl);
resultEntity.setResultFileSize(outputFile.length());
resultEntity.setResultContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
@@ -119,6 +139,32 @@ public class DedupeRunService {
items.add(item);
}
if (folderMode && !archiveEntries.isEmpty()) {
File zipFile = packageFolderDedupeResultsAsZip(request.getArchiveName(), archiveEntries);
String ossObjectKey = ossStorageService.uploadResultFile(zipFile, "DEDUPE");
String downloadUrl = ossStorageService.generateDownloadUrl(ossObjectKey);
DedupeResultItemVo item = new DedupeResultItemVo();
item.setSourceFilename(request.getArchiveName());
item.setOutputFilename(zipFile.getName());
item.setSuccess(true);
item.setDownloadUrl(downloadUrl);
FileResultEntity resultEntity = new FileResultEntity();
resultEntity.setTaskId(task.getId());
resultEntity.setModuleType("DEDUPE");
resultEntity.setSourceFilename(request.getArchiveName());
resultEntity.setResultFilename(zipFile.getName());
resultEntity.setResultFileUrl(downloadUrl);
resultEntity.setResultFileSize(zipFile.length());
resultEntity.setResultContentType("application/zip");
resultEntity.setSuccess(1);
resultEntity.setCreatedAt(LocalDateTime.now());
fileResultMapper.insert(resultEntity);
item.setResultId(resultEntity.getId());
items.add(0, item);
}
task.setSuccessFileCount(successCount);
task.setFailedFileCount(failedCount);
task.setResultJson(JSONUtil.toJsonStr(items));
@@ -136,17 +182,17 @@ public class DedupeRunService {
public List<DedupeResultItemVo> listHistory() {
return fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getModuleType, "DEDUPE")
.eq(FileResultEntity::getSuccess, 1)
.orderByDesc(FileResultEntity::getCreatedAt)
.last("limit 50"))
.stream()
.map(entity -> {
DedupeResultItemVo vo = new DedupeResultItemVo();
vo.setResultId(entity.getId());
vo.setSourceFilename(entity.getCreatedAt() == null ? "" : entity.getCreatedAt().toLocalDate().toString().replace("-", ""));
vo.setSourceFilename(entity.getSourceFilename());
vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(entity.getResultFileUrl());
vo.setSuccess(true);
vo.setDownloadUrl(entity.getSuccess() != null && entity.getSuccess() == 1 ? entity.getResultFileUrl() : null);
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
vo.setError(entity.getErrorMessage());
return vo;
})
.toList();
@@ -173,7 +219,8 @@ public class DedupeRunService {
}
private void cleanExcelByLegacyRules(File inputFile, File outputFile, List<String> selectedColumns,
boolean keepIntegerIds, boolean keepUnderscoreIds) throws Exception {
boolean keepIntegerIds, boolean keepUnderscoreIds,
boolean keepIntegerMainIdsWhenNoSubIds) throws Exception {
DataFormatter formatter = new DataFormatter();
try (FileInputStream fis = new FileInputStream(inputFile);
Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis);
@@ -215,6 +262,26 @@ public class DedupeRunService {
}
Integer idColumnIndex = headerMap.get("id");
Map<String, Boolean> mainIdHasSubIdMap = new HashMap<>();
if (idColumnIndex != null) {
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
continue;
}
String idValue = normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex)));
String mainId = extractMainId(idValue);
if (mainId.isEmpty()) {
continue;
}
if (!mainIdHasSubIdMap.containsKey(mainId)) {
mainIdHasSubIdMap.put(mainId, false);
}
if (idValue.matches("\\d+_\\d+")) {
mainIdHasSubIdMap.put(mainId, true);
}
}
}
int outputRowIndex = 1;
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
@@ -223,7 +290,7 @@ public class DedupeRunService {
}
if (idColumnIndex != null) {
String idValue = normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex)));
if (!shouldKeepId(idValue, keepIntegerIds, keepUnderscoreIds)) {
if (!shouldKeepId(idValue, keepIntegerIds, keepUnderscoreIds, keepIntegerMainIdsWhenNoSubIds, mainIdHasSubIdMap)) {
continue;
}
}
@@ -242,14 +309,68 @@ public class DedupeRunService {
}
}
private boolean shouldKeepId(String text, boolean keepIntegerIds, boolean keepUnderscoreIds) {
private File packageFolderDedupeResultsAsZip(String archiveName, List<DedupeArchiveEntry> archiveEntries) {
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "dedupe-result"));
File zipFile = buildNamedOutputFile(outputDir, archiveName + ".zip");
try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zipFile))) {
for (DedupeArchiveEntry entry : archiveEntries) {
String zipEntryName = buildFolderZipEntry(entry.relativePath(), entry.inputName(), entry.inputName(), false);
zos.putNextEntry(new ZipEntry(zipEntryName));
Files.copy(entry.outputFile().toPath(), zos);
zos.closeEntry();
}
} catch (Exception ex) {
throw new BusinessException("去重结果打包失败：" + ex.getMessage());
}
return zipFile;
}
private String buildFolderZipEntry(String relativePath, String inputName, String outputFilename, boolean includeSourceStemFolder) {
String normalizedRelativePath = relativePath == null ? "" : relativePath.replace('\\', '/');
String relativeDir = normalizedRelativePath.contains("/")
? normalizedRelativePath.substring(0, normalizedRelativePath.lastIndexOf('/'))
: "";
List<String> parts = new ArrayList<>();
if (!relativeDir.isBlank()) {
parts.add(relativeDir);
}
if (includeSourceStemFolder) {
parts.add(FileUtil.mainName(inputName));
}
parts.add(outputFilename);
return String.join("/", parts);
}
private boolean shouldKeepId(String text, boolean keepIntegerIds, boolean keepUnderscoreIds,
boolean keepIntegerMainIdsWhenNoSubIds, Map<String, Boolean> mainIdHasSubIdMap) {
if (text == null || text.isBlank()) {
return false;
}
if (keepUnderscoreIds && text.matches("\\d+_\\d+")) {
return true;
}
if (keepIntegerIds && text.matches("\\d+")) {
return true;
}
return keepUnderscoreIds && text.matches("\\d+_\\d+");
if (keepIntegerMainIdsWhenNoSubIds && text.matches("\\d+")) {
String mainId = extractMainId(text);
return !mainId.isEmpty() && !Boolean.TRUE.equals(mainIdHasSubIdMap.get(mainId));
}
return false;
}
private String extractMainId(String text) {
if (text == null || text.isBlank()) {
return "";
}
if (text.matches("\\d+")) {
return text;
}
if (text.matches("\\d+_\\d+")) {
int idx = text.indexOf('_');
return idx > 0 ? text.substring(0, idx) : "";
}
return "";
}
private File findLocalSourceFile(String fileKey) {
@@ -300,4 +421,7 @@ public class DedupeRunService {
.trim()
.replaceAll("\\s+", " ");
}
private record DedupeArchiveEntry(String relativePath, String inputName, File outputFile) {
}
}

215
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java Normal file
View File

@@ -0,0 +1,215 @@
package com.nanri.aiimage.modules.dedupe.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper;
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataCreateRequest;
import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataUpdateRequest;
import com.nanri.aiimage.modules.dedupe.model.entity.DedupeTotalDataEntity;
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataImportVo;
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataItemVo;
import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataPageVo;
import lombok.RequiredArgsConstructor;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
import java.io.InputStream;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@Service
@RequiredArgsConstructor
public class DedupeTotalDataService {
private final DedupeTotalDataMapper dedupeTotalDataMapper;
public DedupeTotalDataPageVo page(long page, long pageSize, String keyword) {
long safePage = Math.max(page, 1);
long safePageSize = Math.min(Math.max(pageSize, 1), 100);
String safeKeyword = keyword == null ? "" : keyword.trim();
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
.like(!safeKeyword.isEmpty(), DedupeTotalDataEntity::getDataValue, safeKeyword)
.orderByDesc(DedupeTotalDataEntity::getId);
Long total = dedupeTotalDataMapper.selectCount(query);
List<DedupeTotalDataItemVo> items = dedupeTotalDataMapper.selectList(query.last("LIMIT " + ((safePage - 1) * safePageSize) + ", " + safePageSize))
.stream()
.map(this::toItemVo)
.toList();
DedupeTotalDataPageVo vo = new DedupeTotalDataPageVo();
vo.setItems(items);
vo.setTotal(total);
vo.setPage(safePage);
vo.setPageSize(safePageSize);
return vo;
}
@Transactional
public DedupeTotalDataItemVo create(DedupeTotalDataCreateRequest request) {
String dataValue = normalizeDataValue(request.getDataValue());
ensureUnique(dataValue, null);
DedupeTotalDataEntity entity = new DedupeTotalDataEntity();
entity.setDataValue(dataValue);
dedupeTotalDataMapper.insert(entity);
return toItemVo(getById(entity.getId()));
}
@Transactional
public DedupeTotalDataImportVo importFromExcel(MultipartFile file) {
if (file == null || file.isEmpty()) {
throw new BusinessException("请上传 xlsx 文件");
}
String filename = file.getOriginalFilename() == null ? "" : file.getOriginalFilename().toLowerCase();
if (!(filename.endsWith(".xlsx") || filename.endsWith(".xls"))) {
throw new BusinessException("仅支持 .xlsx 或 .xls 文件");
}
DataFormatter formatter = new DataFormatter();
try (InputStream inputStream = file.getInputStream();
Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(inputStream)) {
Sheet sheet = workbook.getSheetAt(0);
Row headerRow = sheet.getRow(0);
if (headerRow == null) {
throw new BusinessException("Excel 表头为空");
}
Map<String, Integer> headerMap = new HashMap<>();
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
Cell cell = headerRow.getCell(i);
String value = normalizeExcelText(cell == null ? null : formatter.formatCellValue(cell));
if (!value.isBlank() && !headerMap.containsKey(value)) {
headerMap.put(value, i);
}
}
Integer asinIndex = headerMap.get("ASIN");
if (asinIndex == null) {
throw new BusinessException("缺少 ASIN 列");
}
Set<String> seenInFile = new HashSet<>();
int totalRows = Math.max(sheet.getLastRowNum(), 0);
int asinCount = 0;
int insertedCount = 0;
int skippedCount = 0;
for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
skippedCount++;
continue;
}
String asin = normalizeExcelText(formatter.formatCellValue(row.getCell(asinIndex)));
if (asin.isBlank()) {
skippedCount++;
continue;
}
asinCount++;
String dataValue = normalizeDataValue(asin);
if (!seenInFile.add(dataValue)) {
skippedCount++;
continue;
}
if (existsDataValue(dataValue)) {
skippedCount++;
continue;
}
DedupeTotalDataEntity entity = new DedupeTotalDataEntity();
entity.setDataValue(dataValue);
dedupeTotalDataMapper.insert(entity);
insertedCount++;
}
DedupeTotalDataImportVo vo = new DedupeTotalDataImportVo();
vo.setTotalRows(totalRows);
vo.setAsinCount(asinCount);
vo.setInsertedCount(insertedCount);
vo.setSkippedCount(skippedCount);
return vo;
} catch (BusinessException e) {
throw e;
} catch (Exception e) {
throw new BusinessException("导入 Excel 失败");
}
}
@Transactional
public DedupeTotalDataItemVo update(Long id, DedupeTotalDataUpdateRequest request) {
DedupeTotalDataEntity entity = getById(id);
String dataValue = normalizeDataValue(request.getDataValue());
ensureUnique(dataValue, id);
entity.setDataValue(dataValue);
dedupeTotalDataMapper.updateById(entity);
return toItemVo(getById(id));
}
@Transactional
public void delete(Long id) {
DedupeTotalDataEntity entity = getById(id);
dedupeTotalDataMapper.deleteById(entity.getId());
}
private DedupeTotalDataEntity getById(Long id) {
DedupeTotalDataEntity entity = dedupeTotalDataMapper.selectById(id);
if (entity == null) {
throw new BusinessException("数据不存在");
}
return entity;
}
private String normalizeDataValue(String value) {
String dataValue = value == null ? "" : value.trim();
if (dataValue.isEmpty()) {
throw new BusinessException("数据不能为空");
}
return dataValue;
}
private void ensureUnique(String dataValue, Long excludeId) {
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
.eq(DedupeTotalDataEntity::getDataValue, dataValue)
.last("LIMIT 1");
DedupeTotalDataEntity exists = dedupeTotalDataMapper.selectOne(query);
if (exists != null && (excludeId == null || !exists.getId().equals(excludeId))) {
throw new BusinessException("数据已存在");
}
}
private boolean existsDataValue(String dataValue) {
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
.eq(DedupeTotalDataEntity::getDataValue, dataValue)
.last("LIMIT 1");
return dedupeTotalDataMapper.selectOne(query) != null;
}
private String normalizeExcelText(String value) {
if (value == null) {
return "";
}
return value.replace("\ufeff", "")
.replace("\u3000", " ")
.replace("\r\n", " ")
.replace("\r", " ")
.replace("\n", " ")
.replace("\t", " ")
.trim()
.replaceAll("\\s+", " ");
}
private DedupeTotalDataItemVo toItemVo(DedupeTotalDataEntity entity) {
DedupeTotalDataItemVo vo = new DedupeTotalDataItemVo();
vo.setId(entity.getId());
vo.setDataValue(entity.getDataValue());
vo.setCreatedAt(entity.getCreatedAt());
return vo;
}
}

5
backend-java/src/main/java/com/nanri/aiimage/modules/file/controller/FileUploadController.java
View File

@@ -46,8 +46,9 @@ public class FileUploadController {
})
public ApiResponse<UploadFileVo> upload(
@Parameter(name = "file", description = "待上传的 Excel 或业务源文件", required = true, in = ParameterIn.QUERY)
MultipartFile file) throws Exception {
return ApiResponse.success(localFileStorageService.saveTempFile(file));
MultipartFile file,
@RequestParam(required = false) String relativePath) throws Exception {
return ApiResponse.success(localFileStorageService.saveTempFile(file, relativePath));
}
@GetMapping("/excel-info")

3
backend-java/src/main/java/com/nanri/aiimage/modules/file/model/vo/UploadFileVo.java
View File

@@ -18,4 +18,7 @@ public class UploadFileVo {
@Schema(description = "文件大小")
private Long size;
@Schema(description = "相对上传目录路径")
private String relativePath;
}

3
backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalFileStorageService.java
View File

@@ -29,7 +29,7 @@ public class LocalFileStorageService {
private final StorageProperties storageProperties;
public UploadFileVo saveTempFile(MultipartFile file) throws IOException {
public UploadFileVo saveTempFile(MultipartFile file, String relativePath) throws IOException {
File tempDir = FileUtil.file(storageProperties.getLocalTempDir());
File parentDir = tempDir.getParentFile();
if (parentDir != null) {
@@ -46,6 +46,7 @@ public class LocalFileStorageService {
vo.setOriginalFilename(file.getOriginalFilename());
vo.setLocalPath(target.getAbsolutePath());
vo.setSize(file.getSize());
vo.setRelativePath(relativePath);
return vo;
}

3
backend-java/src/main/java/com/nanri/aiimage/modules/split/controller/SplitRunController.java
View File

@@ -46,7 +46,8 @@ public class SplitRunController {
- splitMode=rows_per_file 时按固定条数切分；
- splitMode=parts 时按总条数尽量均分；
- 输出文件命名遵循 原文件名_split_序号.xlsx；
- 返回结果中 rowCount 表示该结果文件包含的数据行数。
- 本次拆分任务产生的所有 xlsx 会统一打成一个 zip 上传；
- 返回结果中 rowCount 表示该压缩包包含的数据总行数，entries 表示压缩包内文件摘要。
""")
@ApiResponses({
@io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "执行成功，返回拆分结果统计", content = @Content(schema = @Schema(implementation = SplitRunVo.class))),

3
backend-java/src/main/java/com/nanri/aiimage/modules/split/model/dto/SplitRunRequest.java
View File

@@ -30,4 +30,7 @@ public class SplitRunRequest {
@Schema(description = "按份数拆分时的份数")
private Integer parts;
@Schema(description = "文件夹上传场景下的总压缩包名称（不含扩展名）")
private String archiveName;
}

3
backend-java/src/main/java/com/nanri/aiimage/modules/split/model/dto/SplitSourceFileDto.java
View File

@@ -14,4 +14,7 @@ public class SplitSourceFileDto {
@Schema(description = "原始文件名")
private String originalFilename;
@Schema(description = "相对上传目录路径")
private String relativePath;
}

15
backend-java/src/main/java/com/nanri/aiimage/modules/split/model/vo/SplitArchiveEntryVo.java Normal file
View File

@@ -0,0 +1,15 @@
package com.nanri.aiimage.modules.split.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "拆分压缩包内文件项")
public class SplitArchiveEntryVo {
@Schema(description = "文件名")
private String filename;
@Schema(description = "该文件包含的数据行数")
private Integer rowCount;
}

10
backend-java/src/main/java/com/nanri/aiimage/modules/split/model/vo/SplitResultItemVo.java
View File

@@ -3,6 +3,8 @@ package com.nanri.aiimage.modules.split.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "数据拆分结果项")
public class SplitResultItemVo {
@@ -22,9 +24,15 @@ public class SplitResultItemVo {
@Schema(description = "错误信息")
private String error;
@Schema(description = "结果行数")
@Schema(description = "结果总行数")
private Integer rowCount;
@Schema(description = "下载地址")
private String downloadUrl;
@Schema(description = "压缩包内文件数量")
private Integer entryCount;
@Schema(description = "压缩包内文件列表")
private List<SplitArchiveEntryVo> entries;
}

307
backend-java/src/main/java/com/nanri/aiimage/modules/split/service/SplitRunService.java
View File

@@ -3,13 +3,15 @@ package com.nanri.aiimage.modules.split.service;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.json.JSONUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.split.model.dto.SplitRunRequest;
import com.nanri.aiimage.modules.split.model.dto.SplitSourceFileDto;
import com.nanri.aiimage.modules.split.model.vo.SplitArchiveEntryVo;
import com.nanri.aiimage.modules.split.model.vo.SplitResultItemVo;
import com.nanri.aiimage.modules.split.model.vo.SplitRunVo;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
@@ -20,6 +22,7 @@ import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
@@ -28,18 +31,28 @@ import org.springframework.stereotype.Service;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.nio.file.Files;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@Service
@RequiredArgsConstructor
public class SplitRunService {
private static final String MODULE_TYPE = "SPLIT";
private static final String ZIP_CONTENT_TYPE = "application/zip";
private static final String SPLIT_MODE_ROWS_PER_FILE = "rows_per_file";
private static final String SPLIT_MODE_PARTS = "parts";
private static final String HEADER_SUFFIX_MARKER = "idASIN\u56fd\u5bb6\u72b6\u6001\u4ef7\u683c\u53d8\u4f53\u6570\u91cf";
private static final String HEADER_STOP_COLUMN = "\u7f29\u7565\u56fe\u5730\u57408";
private static final DateTimeFormatter ZIP_NAME_DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd");
private final FileTaskMapper fileTaskMapper;
private final FileResultMapper fileResultMapper;
private final StorageProperties storageProperties;
@@ -49,8 +62,8 @@ public class SplitRunService {
validateRequest(request);
FileTaskEntity task = new FileTaskEntity();
task.setTaskNo("SPLIT-" + IdUtil.getSnowflakeNextIdStr());
task.setModuleType("SPLIT");
task.setTaskNo(MODULE_TYPE + "-" + IdUtil.getSnowflakeNextIdStr());
task.setModuleType(MODULE_TYPE);
task.setTaskMode("IMMEDIATE");
task.setStatus("SUCCESS");
task.setSourceFileCount(request.getFiles().size());
@@ -61,56 +74,42 @@ public class SplitRunService {
task.setFinishedAt(LocalDateTime.now());
fileTaskMapper.insert(task);
boolean folderMode = request.getArchiveName() != null && !request.getArchiveName().isBlank();
List<SplitResultItemVo> items = new ArrayList<>();
int successCount = 0;
int failedCount = 0;
List<TaskSplitChunkResult> allChunkResults = new ArrayList<>();
List<String> successSourceNames = new ArrayList<>();
for (SplitSourceFileDto sourceFile : request.getFiles()) {
try {
File inputFile = findLocalSourceFile(sourceFile.getFileKey());
if (inputFile == null || !inputFile.exists()) {
throw new BusinessException("上传文件不存在，请重新上传");
throw new BusinessException("Uploaded source file does not exist, please upload again.");
}
String inputName = sourceFile.getOriginalFilename() == null ? inputFile.getName() : sourceFile.getOriginalFilename();
String inputName = sourceFile.getOriginalFilename() == null
? inputFile.getName()
: sourceFile.getOriginalFilename();
List<SplitChunkResult> chunkResults = splitExcelByLegacyRules(inputFile, inputName, request);
for (SplitChunkResult chunkResult : chunkResults) {
SplitResultItemVo item = new SplitResultItemVo();
item.setSourceFilename(inputName);
item.setOutputFilename(chunkResult.outputFile().getName());
item.setSuccess(true);
item.setRowCount(chunkResult.rowCount());
String ossObjectKey = ossStorageService.uploadResultFile(chunkResult.outputFile(), "SPLIT");
String downloadUrl = ossStorageService.generateDownloadUrl(ossObjectKey);
FileResultEntity resultEntity = new FileResultEntity();
resultEntity.setTaskId(task.getId());
resultEntity.setModuleType("SPLIT");
resultEntity.setSourceFilename(inputName);
resultEntity.setResultFilename(chunkResult.outputFile().getName());
resultEntity.setResultFileUrl(downloadUrl);
resultEntity.setResultFileSize(chunkResult.outputFile().length());
resultEntity.setResultContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
resultEntity.setRowCount(chunkResult.rowCount());
resultEntity.setSuccess(1);
resultEntity.setCreatedAt(LocalDateTime.now());
fileResultMapper.insert(resultEntity);
item.setDownloadUrl(downloadUrl);
item.setResultId(resultEntity.getId());
items.add(item);
allChunkResults.add(new TaskSplitChunkResult(sourceFile.getRelativePath(), inputName, chunkResult.outputFile(), chunkResult.rowCount()));
}
successSourceNames.add(inputName);
successCount++;
} catch (Exception ex) {
SplitResultItemVo item = new SplitResultItemVo();
item.setSourceFilename(sourceFile.getOriginalFilename());
item.setSuccess(false);
item.setError(ex.getMessage());
item.setEntries(new ArrayList<>());
item.setEntryCount(0);
items.add(item);
failedCount++;
FileResultEntity resultEntity = new FileResultEntity();
resultEntity.setTaskId(task.getId());
resultEntity.setModuleType("SPLIT");
resultEntity.setModuleType(MODULE_TYPE);
resultEntity.setSourceFilename(sourceFile.getOriginalFilename());
resultEntity.setSuccess(0);
resultEntity.setErrorMessage(ex.getMessage());
@@ -119,6 +118,38 @@ public class SplitRunService {
}
}
if (!allChunkResults.isEmpty()) {
File zipFile = packageTaskSplitResultsAsZip(request.getArchiveName(), successSourceNames, allChunkResults);
String ossObjectKey = ossStorageService.uploadResultFile(zipFile, MODULE_TYPE);
String downloadUrl = ossStorageService.generateDownloadUrl(ossObjectKey);
SplitResultItemVo item = new SplitResultItemVo();
item.setSourceFilename(folderMode && request.getArchiveName() != null && !request.getArchiveName().isBlank()
? request.getArchiveName()
: successSourceNames.size() == 1 ? successSourceNames.get(0) : "Current split task");
item.setOutputFilename(zipFile.getName());
item.setSuccess(true);
item.setRowCount(allChunkResults.stream().mapToInt(TaskSplitChunkResult::rowCount).sum());
item.setDownloadUrl(downloadUrl);
item.setEntryCount(allChunkResults.size());
item.setEntries(allChunkResults.stream().map(this::toArchiveEntry).toList());
FileResultEntity resultEntity = new FileResultEntity();
resultEntity.setTaskId(task.getId());
resultEntity.setModuleType(MODULE_TYPE);
resultEntity.setSourceFilename(item.getSourceFilename());
resultEntity.setResultFilename(zipFile.getName());
resultEntity.setResultFileUrl(downloadUrl);
resultEntity.setResultFileSize(zipFile.length());
resultEntity.setResultContentType(ZIP_CONTENT_TYPE);
resultEntity.setRowCount(item.getRowCount());
resultEntity.setSuccess(1);
resultEntity.setCreatedAt(LocalDateTime.now());
fileResultMapper.insert(resultEntity);
item.setResultId(resultEntity.getId());
items.add(0, item);
}
task.setSuccessFileCount(successCount);
task.setFailedFileCount(failedCount);
task.setResultJson(JSONUtil.toJsonStr(items));
@@ -135,19 +166,21 @@ public class SplitRunService {
public List<SplitResultItemVo> listHistory() {
return fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getModuleType, "SPLIT")
.eq(FileResultEntity::getSuccess, 1)
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.orderByDesc(FileResultEntity::getCreatedAt)
.last("limit 100"))
.stream()
.map(entity -> {
SplitResultItemVo vo = new SplitResultItemVo();
vo.setResultId(entity.getId());
vo.setSourceFilename(entity.getCreatedAt() == null ? "" : entity.getCreatedAt().toLocalDate().toString().replace("-", ""));
vo.setSourceFilename(entity.getSourceFilename());
vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(entity.getResultFileUrl());
vo.setDownloadUrl(entity.getSuccess() != null && entity.getSuccess() == 1 ? entity.getResultFileUrl() : null);
vo.setRowCount(entity.getRowCount());
vo.setSuccess(true);
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
vo.setError(entity.getErrorMessage());
vo.setEntryCount(0);
vo.setEntries(new ArrayList<>());
return vo;
})
.toList();
@@ -155,55 +188,45 @@ public class SplitRunService {
public void deleteHistory(Long resultId) {
FileResultEntity entity = fileResultMapper.selectById(resultId);
if (entity == null || !"SPLIT".equals(entity.getModuleType())) {
throw new BusinessException("记录不存在");
if (entity == null || !MODULE_TYPE.equals(entity.getModuleType())) {
throw new BusinessException("Split result record does not exist.");
}
fileResultMapper.deleteById(resultId);
}
public Resource getResultFile(Long resultId) {
FileResultEntity entity = fileResultMapper.selectById(resultId);
if (entity == null || entity.getResultFileUrl() == null || !"SPLIT".equals(entity.getModuleType())) {
throw new BusinessException("结果文件不存在");
if (entity == null || entity.getResultFileUrl() == null || !MODULE_TYPE.equals(entity.getModuleType())) {
throw new BusinessException("Split result file does not exist.");
}
File file = new File(entity.getResultFileUrl());
if (!file.exists()) {
throw new BusinessException("结果文件不存在");
throw new BusinessException("Split result file does not exist.");
}
return new FileSystemResource(file);
}
private List<SplitChunkResult> splitExcelByLegacyRules(File inputFile, String originalFilename, SplitRunRequest request) throws Exception {
private List<SplitChunkResult> splitExcelByLegacyRules(
File inputFile,
String originalFilename,
SplitRunRequest request
) throws Exception {
DataFormatter formatter = new DataFormatter();
try (FileInputStream fis = new FileInputStream(inputFile); Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis)) {
try (FileInputStream fis = new FileInputStream(inputFile);
Workbook workbook = WorkbookFactory.create(fis)) {
Sheet sheet = workbook.getSheetAt(0);
Row headerRow = sheet.getRow(0);
if (headerRow == null) {
throw new BusinessException("Excel 表头为空");
throw new BusinessException("Excel header row is empty.");
}
Map<String, Integer> headerMap = new HashMap<>();
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
Cell cell = headerRow.getCell(i);
String value = normalizeCellText(cell == null ? null : formatter.formatCellValue(cell));
if (value.isBlank()) {
continue;
}
value = value.split("idASIN国家状态价格变体数量", 2)[0].trim().isEmpty()
? value
: value.split("idASIN国家状态价格变体数量", 2)[0].trim();
if (headerMap.containsKey(value)) {
continue;
}
headerMap.put(value, i);
if ("缩略图地址8".equals(value)) {
break;
}
}
List<String> missing = request.getSelectedColumns().stream().filter(column -> !headerMap.containsKey(column)).toList();
Map<String, Integer> headerMap = buildHeaderMap(headerRow, formatter);
List<String> missing = request.getSelectedColumns().stream()
.filter(column -> !headerMap.containsKey(column))
.toList();
if (!missing.isEmpty()) {
throw new BusinessException("缺少列：" + String.join("、", missing));
throw new BusinessException("Missing required columns: " + String.join(", ", missing));
}
List<List<String>> rowsData = new ArrayList<>();
@@ -215,39 +238,23 @@ public class SplitRunService {
List<String> projected = new ArrayList<>();
for (String column : request.getSelectedColumns()) {
Integer sourceIndex = headerMap.get(column);
String value = sourceIndex == null ? "" : normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex)));
String value = sourceIndex == null
? ""
: normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex)));
projected.add(value);
}
rowsData.add(projected);
}
int totalRows = rowsData.size();
if (totalRows == 0) {
throw new BusinessException("没有可拆分的数据行");
}
List<List<List<String>>> chunks = new ArrayList<>();
if ("rows_per_file".equals(request.getSplitMode())) {
int chunkSize = request.getRowsPerFile();
for (int i = 0; i < totalRows; i += chunkSize) {
chunks.add(rowsData.subList(i, Math.min(i + chunkSize, totalRows)));
}
} else {
int actualParts = Math.min(request.getParts(), totalRows);
int base = totalRows / actualParts;
int remainder = totalRows % actualParts;
int start = 0;
for (int i = 0; i < actualParts; i++) {
int size = base + (i < remainder ? 1 : 0);
int end = start + size;
chunks.add(rowsData.subList(start, end));
start = end;
}
if (rowsData.isEmpty()) {
throw new BusinessException("No data rows available for splitting.");
}
List<List<List<String>>> chunks = buildChunks(rowsData, request);
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "split-result"));
String sourceStem = FileUtil.mainName(originalFilename);
List<SplitChunkResult> results = new ArrayList<>();
for (int i = 0; i < chunks.size(); i++) {
File outputFile = buildNamedOutputFile(outputDir, sourceStem + "_split_" + (i + 1) + ".xlsx");
try (XSSFWorkbook outputWorkbook = new XSSFWorkbook()) {
@@ -269,19 +276,63 @@ public class SplitRunService {
}
results.add(new SplitChunkResult(outputFile, chunks.get(i).size()));
}
return results;
}
}
private Map<String, Integer> buildHeaderMap(Row headerRow, DataFormatter formatter) {
Map<String, Integer> headerMap = new HashMap<>();
for (int i = 0; i < headerRow.getLastCellNum(); i++) {
Cell cell = headerRow.getCell(i);
String value = normalizeHeaderName(cell == null ? null : formatter.formatCellValue(cell));
if (value.isBlank() || headerMap.containsKey(value)) {
continue;
}
headerMap.put(value, i);
if (HEADER_STOP_COLUMN.equals(value)) {
break;
}
}
return headerMap;
}
private List<List<List<String>>> buildChunks(List<List<String>> rowsData, SplitRunRequest request) {
int totalRows = rowsData.size();
List<List<List<String>>> chunks = new ArrayList<>();
if (SPLIT_MODE_ROWS_PER_FILE.equals(request.getSplitMode())) {
int chunkSize = request.getRowsPerFile();
for (int i = 0; i < totalRows; i += chunkSize) {
chunks.add(rowsData.subList(i, Math.min(i + chunkSize, totalRows)));
}
return chunks;
}
int actualParts = Math.min(request.getParts(), totalRows);
int base = totalRows / actualParts;
int remainder = totalRows % actualParts;
int start = 0;
for (int i = 0; i < actualParts; i++) {
int size = base + (i < remainder ? 1 : 0);
int end = start + size;
chunks.add(rowsData.subList(start, end));
start = end;
}
return chunks;
}
private void validateRequest(SplitRunRequest request) {
if (!List.of("rows_per_file", "parts").contains(request.getSplitMode())) {
throw new BusinessException("拆分模式不合法");
if (!List.of(SPLIT_MODE_ROWS_PER_FILE, SPLIT_MODE_PARTS).contains(request.getSplitMode())) {
throw new BusinessException("Invalid split mode.");
}
if ("rows_per_file".equals(request.getSplitMode()) && (request.getRowsPerFile() == null || request.getRowsPerFile() <= 0)) {
throw new BusinessException("每份条数不合法");
if (SPLIT_MODE_ROWS_PER_FILE.equals(request.getSplitMode())
&& (request.getRowsPerFile() == null || request.getRowsPerFile() <= 0)) {
throw new BusinessException("Invalid rowsPerFile value.");
}
if ("parts".equals(request.getSplitMode()) && (request.getParts() == null || request.getParts() <= 0)) {
throw new BusinessException("拆分份数不合法");
if (SPLIT_MODE_PARTS.equals(request.getSplitMode())
&& (request.getParts() == null || request.getParts() <= 0)) {
throw new BusinessException("Invalid parts value.");
}
}
@@ -290,8 +341,11 @@ public class SplitRunService {
if (!baseDir.exists()) {
return null;
}
List<File> matchedFiles = FileUtil.loopFiles(baseDir, pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey));
return matchedFiles.isEmpty() ? null : matchedFiles.getFirst();
List<File> matchedFiles = FileUtil.loopFiles(
baseDir,
pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey)
);
return matchedFiles.isEmpty() ? null : matchedFiles.get(0);
}
private File buildNamedOutputFile(File outputDir, String filename) {
@@ -299,6 +353,7 @@ public class SplitRunService {
if (!candidate.exists()) {
return candidate;
}
String mainName = FileUtil.mainName(filename);
String extName = FileUtil.extName(filename);
int index = 2;
@@ -314,6 +369,61 @@ public class SplitRunService {
}
}
private File packageTaskSplitResultsAsZip(String archiveName, List<String> successSourceNames, List<TaskSplitChunkResult> chunkResults) {
File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "split-result"));
String zipFilename = archiveName != null && !archiveName.isBlank()
? archiveName + ".zip"
: successSourceNames.size() == 1
? FileUtil.mainName(successSourceNames.getFirst()) + ".zip"
: LocalDateTime.now().format(ZIP_NAME_DATE_FORMATTER) + ".zip";
File zipFile = buildNamedOutputFile(outputDir, zipFilename);
try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zipFile))) {
for (TaskSplitChunkResult chunkResult : chunkResults) {
String zipEntryName = buildFolderZipEntry(chunkResult.relativePath(), chunkResult.sourceFilename(), chunkResult.outputFile().getName(), true);
ZipEntry entry = new ZipEntry(zipEntryName);
zos.putNextEntry(entry);
Files.copy(chunkResult.outputFile().toPath(), zos);
zos.closeEntry();
}
} catch (Exception ex) {
throw new BusinessException("拆分结果打包失败：" + ex.getMessage());
}
return zipFile;
}
private String buildFolderZipEntry(String relativePath, String inputName, String outputFilename, boolean includeSourceStemFolder) {
String normalizedRelativePath = relativePath == null ? "" : relativePath.replace('\\', '/');
String relativeDir = normalizedRelativePath.contains("/")
? normalizedRelativePath.substring(0, normalizedRelativePath.lastIndexOf('/'))
: "";
List<String> parts = new ArrayList<>();
if (!relativeDir.isBlank()) {
parts.add(relativeDir);
}
if (includeSourceStemFolder) {
parts.add(FileUtil.mainName(inputName));
}
parts.add(outputFilename);
return String.join("/", parts);
}
private SplitArchiveEntryVo toArchiveEntry(TaskSplitChunkResult chunkResult) {
SplitArchiveEntryVo vo = new SplitArchiveEntryVo();
vo.setFilename(chunkResult.outputFile().getName());
vo.setRowCount(chunkResult.rowCount());
return vo;
}
private String normalizeHeaderName(String value) {
String normalized = normalizeCellText(value);
if (normalized.isBlank()) {
return normalized;
}
String prefix = normalized.split(HEADER_SUFFIX_MARKER, 2)[0].trim();
return prefix.isEmpty() ? normalized : prefix;
}
private String normalizeCellText(String value) {
if (value == null) {
return "";
@@ -330,4 +440,7 @@ public class SplitRunService {
private record SplitChunkResult(File outputFile, int rowCount) {
}
private record TaskSplitChunkResult(String relativePath, String sourceFilename, File outputFile, int rowCount) {
}
}

8
backend-java/src/main/resources/db/V3__create_dedupe_total_data.sql Normal file
View File

@@ -0,0 +1,8 @@
CREATE TABLE IF NOT EXISTS biz_dedupe_total_data (
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT '主键',
data_value VARCHAR(128) NOT NULL COMMENT '去重总数据值',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
UNIQUE KEY uk_data_value (data_value),
KEY idx_created_at (created_at)
) COMMENT='数据去重总数据表';

5
backend-java/src/main/resources/db/V4__rename_convert_template_to_five_countries.sql Normal file
View File

@@ -0,0 +1,5 @@
UPDATE biz_convert_template
SET template_name = '五国模板',
output_filename = '英国.txt、法国.txt、德国.txt、西班牙.txt、意大利.txt',
updated_at = CURRENT_TIMESTAMP
WHERE template_code = 'uk_offer';

1
backend/.device_id Normal file
View File

@@ -0,0 +1 @@
8cecee3bc02a178bf372ca1c3d02fc5cae3c0a51c8346f6e6a710988599c08be

BIN
backend/__pycache__/ali_oss.cpython-311.pyc Normal file
View File

Binary file not shown.

BIN
backend/__pycache__/ali_oss.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
backend/__pycache__/ali_oss.cpython-39.pyc Normal file
View File

Binary file not shown.

BIN
backend/__pycache__/app.cpython-311.pyc Normal file
View File

Binary file not shown.

BIN
backend/__pycache__/config.cpython-311.pyc Normal file
View File

Binary file not shown.

BIN
backend/__pycache__/config.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
backend/__pycache__/config.cpython-39.pyc Normal file
View File

Binary file not shown.

BIN
backend/__pycache__/db.cpython-311.pyc Normal file
View File

Binary file not shown.

36
backend/ali_oss.py Normal file
View File

@@ -0,0 +1,36 @@
import argparse
import base64
import re
import time
import alibabacloud_oss_v2 as oss
import requests
from config import region, endpoint, bucket, file_url_pre, bucket_path
def upload_file(file_content: bytes, key: str):
credentials_provider = oss.credentials.EnvironmentVariableCredentialsProvider()
cfg = oss.config.load_default()
cfg.credentials_provider = credentials_provider
cfg.region = region
cfg.endpoint = endpoint
client = oss.Client(cfg)
result = client.put_object(
oss.PutObjectRequest(
bucket=bucket, # 存储空间名称
key=key, # 对象名称
body=file_content # 读取文件内容
)
)
# print(result)
return file_url_pre + key
# 脚本入口，当文件被直接运行时调用main函数
if __name__ == "__main__":
with open("测试图片数据/IMG_2685.JPG", "rb") as f:
file_content = f.read()
upload_file(file_content,key=bucket_path+"test.png")

38
backend/app.py Normal file
View File

@@ -0,0 +1,38 @@
"""
卖相AI - Flask 后端
按功能拆分为蓝图：认证(auth)、主页面(main)、管理员API(admin_api)、版本(version)
"""
import os
import secrets
from datetime import timedelta
from flask import Flask
from flask_cors import CORS
from utils.db import init_db
from blueprints.auth import auth
from blueprints.main import main
from blueprints.admin_api import admin_api
from blueprints.version import version_bp
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
app = Flask(__name__, template_folder=BASE_DIR, static_folder=BASE_DIR)
CORS(app)
app.secret_key = os.environ.get('SECRET_KEY', secrets.token_hex(32))
app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(days=7)
# 注册蓝图
app.register_blueprint(auth)
app.register_blueprint(main)
app.register_blueprint(admin_api)
app.register_blueprint(version_bp)
def run_app(host='0.0.0.0', port=15124):
init_db()
app.run(host=host, port=port, threaded=True, use_reloader=False)
if __name__ == '__main__':
run_app()

1
backend/blueprints/__init__.py Normal file
View File

@@ -0,0 +1 @@
# Blueprints 包

BIN
backend/blueprints/__pycache__/__init__.cpython-311.pyc Normal file
View File

Binary file not shown.

BIN
backend/blueprints/__pycache__/__init__.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
backend/blueprints/__pycache__/__init__.cpython-39.pyc Normal file
View File

Binary file not shown.

BIN
backend/blueprints/__pycache__/admin_api.cpython-311.pyc Normal file
View File

Binary file not shown.

BIN
backend/blueprints/__pycache__/admin_api.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
backend/blueprints/__pycache__/admin_api.cpython-39.pyc Normal file
View File

Binary file not shown.

BIN
backend/blueprints/__pycache__/admin_history.cpython-311.pyc Normal file
View File

Binary file not shown.

BIN
backend/blueprints/__pycache__/admin_users.cpython-311.pyc Normal file
View File

Binary file not shown.

BIN
backend/blueprints/__pycache__/admin_versions.cpython-311.pyc Normal file
View File

Binary file not shown.

BIN
backend/blueprints/__pycache__/auth.cpython-311.pyc Normal file
View File

Binary file not shown.

BIN
backend/blueprints/__pycache__/auth.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
backend/blueprints/__pycache__/auth.cpython-39.pyc Normal file
View File

Binary file not shown.

BIN
backend/blueprints/__pycache__/main.cpython-311.pyc Normal file
View File

Binary file not shown.

BIN
backend/blueprints/__pycache__/main.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
backend/blueprints/__pycache__/main.cpython-39.pyc Normal file
View File

Binary file not shown.

BIN
backend/blueprints/__pycache__/version.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
backend/blueprints/__pycache__/version.cpython-39.pyc Normal file
View File

Binary file not shown.

723
backend/blueprints/admin_api.py Normal file
View File

@@ -0,0 +1,723 @@
"""
管理员 API 蓝图：用户管理、生成历史、版本管理（后台）
"""
import json
import os
import re
import requests
from flask import Blueprint, request, jsonify, session
import pymysql
from werkzeug.security import generate_password_hash
from utils.db import get_db
from utils.auth import admin_required, get_current_admin_role
from ali_oss import upload_file as oss_upload_file
try:
from config import bucket_path, backend_java_base_url
except ImportError:
bucket_path = os.environ.get('BUCKET_PATH', 'nanri-image/')
backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://127.0.0.1:18080').rstrip('/')
admin_api = Blueprint('admin_api', __name__, url_prefix='/api/admin')
def _safe_version_key(version):
"""将版本号转为安全的 OSS 对象名部分"""
s = (version or '').strip()
s = re.sub(r'[^\w.\-]', '_', s)
return s or 'unknown'
def _proxy_backend_java(method, path, *, params=None, json_data=None, files=None, data=None):
url = f"{backend_java_base_url}{path}"
try:
resp = requests.request(method=method, url=url, params=params, json=json_data, files=files, data=data, timeout=10)
except requests.RequestException:
return None, jsonify({'success': False, 'error': 'backend-java 服务不可用'}), 502
try:
data = resp.json()
except ValueError:
data = None
if resp.status_code >= 400:
if isinstance(data, dict):
return data, jsonify({'success': False, 'error': data.get('message') or data.get('error') or 'backend-java 请求失败'}), resp.status_code
return None, jsonify({'success': False, 'error': 'backend-java 请求失败'}), resp.status_code
if not isinstance(data, dict):
return None, jsonify({'success': False, 'error': 'backend-java 返回格式错误'}), 502
if not data.get('success'):
return data, jsonify({'success': False, 'error': data.get('message') or '操作失败'}), 200
return data, None, 200
# ---------- 用户管理 ----------
@admin_api.route('/users')
@admin_required
def list_users():
"""分页获取用户列表；支持用户名模糊搜索、指定管理员所属普通用户筛选"""
role, current_row = get_current_admin_role()
if not role:
return jsonify({'success': False, 'error': '需要管理员权限'}), 403
page = max(1, int(request.args.get('page', 1)))
page_size = min(50, max(5, int(request.args.get('page_size', 15))))
offset = (page - 1) * page_size
search_username = (request.args.get('username') or request.args.get('search') or '').strip()
created_by_id_arg = request.args.get('created_by_id') or request.args.get('admin_id')
created_by_id = int(created_by_id_arg) if created_by_id_arg and str(created_by_id_arg).isdigit() else None
if role != 'super_admin':
created_by_id = None
try:
conn = get_db()
with conn.cursor() as cur:
if role == 'super_admin':
where_parts = ["1=1"]
params = []
if search_username:
where_parts.append("u.username LIKE %s")
params.append("%" + search_username + "%")
if created_by_id is not None:
where_parts.append("u.created_by_id = %s")
params.append(created_by_id)
where_sql = " AND ".join(where_parts)
cur.execute(
"""SELECT u.id, u.username, u.is_admin, u.role, u.created_at, u.created_by_id,
creator.username AS creator_username
FROM users u
LEFT JOIN users creator ON creator.id = u.created_by_id
WHERE """ + where_sql + """ ORDER BY u.id LIMIT %s OFFSET %s""",
tuple(params) + (page_size, offset),
)
rows = cur.fetchall()
cur.execute("SELECT COUNT(*) as total FROM users u WHERE " + where_sql, tuple(params))
total = cur.fetchone()['total']
cur.execute("SELECT id, username FROM users WHERE role = 'admin' ORDER BY id")
admins = [{'id': r['id'], 'username': r['username']} for r in cur.fetchall()]
else:
admin_id = current_row['id']
where_parts = ["(u.id = %s OR (u.role = 'normal' AND u.created_by_id = %s))"]
params = [admin_id, admin_id]
if search_username:
where_parts.append("u.username LIKE %s")
params.append("%" + search_username + "%")
where_sql = " AND ".join(where_parts)
cur.execute(
"""SELECT u.id, u.username, u.is_admin, u.role, u.created_at, u.created_by_id,
creator.username AS creator_username
FROM users u
LEFT JOIN users creator ON creator.id = u.created_by_id
WHERE """ + where_sql + """ ORDER BY u.id LIMIT %s OFFSET %s""",
tuple(params) + (page_size, offset),
)
rows = cur.fetchall()
cur.execute(
"SELECT COUNT(*) as total FROM users u WHERE " + where_sql,
tuple(params),
)
total = cur.fetchone()['total']
admins = []
items = [
{
'id': r['id'],
'username': r['username'],
'is_admin': bool(r.get('is_admin')),
'role': r.get('role') or 'normal',
'created_by_id': r.get('created_by_id'),
'creator_username': r.get('creator_username') or '',
'created_at': r['created_at'].strftime('%Y-%m-%d %H:%M') if r.get('created_at') else '',
}
for r in rows
]
conn.close()
return jsonify({
'success': True,
'items': items,
'total': total,
'page': page,
'page_size': page_size,
'current_user_role': role,
'admins': admins,
})
except Exception as e:
return jsonify({'success': False, 'error': str(e)})
@admin_api.route('/user', methods=['POST'])
@admin_required
def create_user():
data = request.get_json() or {}
username = (data.get('username') or '').strip()
password = data.get('password') or ''
role, current_row = get_current_admin_role()
if not role:
return jsonify({'success': False, 'error': '需要管理员权限'}), 403
want_role = (data.get('role') or 'normal').strip() or 'normal'
if want_role not in ('admin', 'normal'):
want_role = 'normal'
if role == 'admin' and want_role == 'admin':
return jsonify({'success': False, 'error': '仅超级管理员可创建管理员'})
if want_role == 'admin':
want_created_by = current_row['id']
elif role == 'super_admin':
want_created_by = data.get('created_by_id')
else:
want_created_by = current_row['id']
if not username or not password:
return jsonify({'success': False, 'error': '用户名和密码不能为空'})
if len(username) < 2:
return jsonify({'success': False, 'error': '用户名至少2个字符'})
if len(password) < 6:
return jsonify({'success': False, 'error': '密码至少6个字符'})
if want_role == 'normal' and role == 'super_admin' and want_created_by is None:
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute("SELECT id FROM users WHERE role = 'admin' ORDER BY id LIMIT 1")
r = cur.fetchone()
conn.close()
want_created_by = r['id'] if r else current_row['id']
except Exception:
want_created_by = current_row['id']
if want_role == 'normal' and want_created_by is None:
want_created_by = current_row['id']
is_admin = 1 if want_role in ('super_admin', 'admin') else 0
pwd_hash = generate_password_hash(password, method='pbkdf2:sha256')
column_ids = data.get('column_ids')
if column_ids is None:
column_ids = []
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute(
"INSERT INTO users (username, password_hash, is_admin, role, created_by_id) VALUES (%s, %s, %s, %s, %s)",
(username, pwd_hash, is_admin, want_role, want_created_by),
)
new_uid = cur.lastrowid
_set_user_column_permissions(cur, new_uid, column_ids)
conn.commit()
conn.close()
return jsonify({'success': True, 'msg': '用户创建成功'})
except pymysql.IntegrityError:
return jsonify({'success': False, 'error': '用户名已存在'})
except Exception as e:
return jsonify({'success': False, 'error': str(e)})
@admin_api.route('/user/<int:uid>', methods=['PUT'])
@admin_required
def update_user(uid):
"""更新用户（密码、角色）"""
data = request.get_json() or {}
password = data.get('password')
want_role = (data.get('role') or '').strip() or data.get('role')
role, current_row = get_current_admin_role()
if not role:
return jsonify({'success': False, 'error': '需要管理员权限'}), 403
if want_role is None and not password:
return jsonify({'success': False, 'error': '请提供要修改的内容'})
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute("SELECT id, role, created_by_id FROM users WHERE id = %s", (uid,))
target = cur.fetchone()
if not target:
conn.close()
return jsonify({'success': False, 'error': '用户不存在'})
if role == 'admin':
if target['role'] != 'normal' or target.get('created_by_id') != current_row['id']:
conn.close()
return jsonify({'success': False, 'error': '只能编辑自己创建的普通用户'}), 403
want_role = None
else:
if target.get('role') == 'super_admin':
conn.close()
return jsonify({'success': False, 'error': '不能修改超级管理员'})
if want_role == 'super_admin':
return jsonify({'success': False, 'error': '不能将用户设为超级管理员'})
if want_role not in ('admin', 'normal', None, ''):
want_role = None
if password:
if len(password) < 6:
conn.close()
return jsonify({'success': False, 'error': '密码至少6个字符'})
pwd_hash = generate_password_hash(password, method='pbkdf2:sha256')
cur.execute("UPDATE users SET password_hash = %s WHERE id = %s", (pwd_hash, uid))
if want_role is not None and want_role != '':
is_admin = 1 if want_role == 'admin' else 0
cur.execute(
"UPDATE users SET is_admin = %s, role = %s WHERE id = %s",
(is_admin, want_role, uid),
)
column_ids = data.get('column_ids')
if column_ids is not None:
_set_user_column_permissions(cur, uid, column_ids)
conn.commit()
conn.close()
return jsonify({'success': True, 'msg': '更新成功'})
except Exception as e:
return jsonify({'success': False, 'error': str(e)})
@admin_api.route('/user/<int:uid>', methods=['DELETE'])
@admin_required
def delete_user(uid):
"""删除用户"""
if session.get('user_id') == uid:
return jsonify({'success': False, 'error': '不能删除当前登录账号'})
role, current_row = get_current_admin_role()
if not role:
return jsonify({'success': False, 'error': '需要管理员权限'}), 403
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute("SELECT id, role, created_by_id FROM users WHERE id = %s", (uid,))
target = cur.fetchone()
if not target:
conn.close()
return jsonify({'success': False, 'error': '用户不存在'})
if target.get('role') == 'super_admin':
conn.close()
return jsonify({'success': False, 'error': '不能删除超级管理员'})
if role == 'admin':
if target.get('role') != 'normal' or target.get('created_by_id') != current_row['id']:
conn.close()
return jsonify({'success': False, 'error': '只能删除自己创建的普通用户'}), 403
cur.execute("DELETE FROM users WHERE id = %s", (uid,))
affected = cur.rowcount
conn.commit()
conn.close()
if affected == 0:
return jsonify({'success': False, 'error': '用户不存在'})
return jsonify({'success': True, 'msg': '删除成功'})
except Exception as e:
return jsonify({'success': False, 'error': str(e)})
# ---------- 生成历史 ----------
@admin_api.route('/history')
@admin_required
def history():
"""管理员分页获取所有生成记录"""
page = max(1, int(request.args.get('page', 1)))
page_size = min(50, max(10, int(request.args.get('page_size', 15))))
offset = (page - 1) * page_size
user_id = request.args.get('user_id', type=int)
time_start = (request.args.get('time_start') or '').strip()
time_end = (request.args.get('time_end') or '').strip()
conditions, params = [], []
if user_id:
conditions.append("h.user_id = %s")
params.append(user_id)
if time_start:
conditions.append("h.created_at >= %s")
params.append(time_start)
if time_end:
conditions.append("h.created_at <= %s")
params.append(time_end + ' 23:59:59' if len(time_end) <= 10 else time_end)
where_clause = " AND ".join(conditions) if conditions else "1=1"
params_count = params[:]
params.extend([page_size, offset])
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute(
"""SELECT h.id, h.user_id, h.created_at, h.panel_type, h.original_urls, h.params, h.result_urls,
h.long_image_url, u.username
FROM image_history h
LEFT JOIN users u ON h.user_id = u.id
WHERE """ + where_clause + """ ORDER BY h.created_at DESC LIMIT %s OFFSET %s""",
params,
)
rows = cur.fetchall()
cur.execute("SELECT COUNT(*) as total FROM image_history h WHERE " + where_clause, params_count)
total = cur.fetchone()['total']
conn.close()
def _parse_json(val, default=None):
if val is None:
return default if default is not None else []
if isinstance(val, (list, dict)):
return val
try:
return json.loads(val)
except Exception:
return default if default is not None else []
items = []
for r in rows:
items.append({
'id': r['id'],
'user_id': r['user_id'],
'username': r.get('username') or '-',
'created_at': r['created_at'].strftime('%Y-%m-%d %H:%M') if r['created_at'] else '',
'panel_type': r['panel_type'] or '',
'original_urls': _parse_json(r['original_urls'], []),
'params': _parse_json(r['params'], {}),
'result_urls': _parse_json(r['result_urls'], []),
'long_image_url': (r.get('long_image_url') or '').strip() or None,
})
return jsonify({'success': True, 'items': items, 'total': total, 'page': page, 'page_size': page_size})
except Exception as e:
return jsonify({'success': False, 'error': str(e)})
# ---------- 栏目权限配置 ----------
@admin_api.route('/columns')
@admin_required
def list_columns():
"""获取栏目列表（用于栏目配置与用户权限选择）"""
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute(
"SELECT id, name, column_key, created_at FROM columns ORDER BY id"
)
rows = cur.fetchall()
conn.close()
items = [
{
'id': r['id'],
'name': r['name'] or '',
'column_key': r['column_key'] or '',
'created_at': r['created_at'].strftime('%Y-%m-%d %H:%M') if r.get('created_at') else '',
}
for r in rows
]
return jsonify({'success': True, 'items': items})
except Exception as e:
return jsonify({'success': False, 'error': str(e)})
@admin_api.route('/column', methods=['POST'])
@admin_required
def create_column():
"""新增栏目"""
data = request.get_json() or {}
name = (data.get('name') or '').strip()
column_key = (data.get('column_key') or '').strip()
if not name:
return jsonify({'success': False, 'error': '栏目名不能为空'})
if not column_key:
return jsonify({'success': False, 'error': '栏目标识不能为空'})
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute(
"INSERT INTO columns (name, column_key) VALUES (%s, %s)",
(name, column_key),
)
cid = cur.lastrowid
conn.commit()
conn.close()
return jsonify({'success': True, 'msg': '创建成功', 'id': cid})
except pymysql.IntegrityError:
return jsonify({'success': False, 'error': '栏目标识已存在'})
except Exception as e:
return jsonify({'success': False, 'error': str(e)})
@admin_api.route('/column/<int:cid>', methods=['PUT'])
@admin_required
def update_column(cid):
"""更新栏目"""
data = request.get_json() or {}
name = (data.get('name') or '').strip()
column_key = (data.get('column_key') or '').strip()
if not name:
return jsonify({'success': False, 'error': '栏目名不能为空'})
if not column_key:
return jsonify({'success': False, 'error': '栏目标识不能为空'})
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute(
"UPDATE columns SET name = %s, column_key = %s WHERE id = %s",
(name, column_key, cid),
)
if cur.rowcount == 0:
conn.close()
return jsonify({'success': False, 'error': '栏目不存在'})
conn.commit()
conn.close()
return jsonify({'success': True, 'msg': '更新成功'})
except pymysql.IntegrityError:
return jsonify({'success': False, 'error': '栏目标识已存在'})
except Exception as e:
return jsonify({'success': False, 'error': str(e)})
@admin_api.route('/column/<int:cid>', methods=['DELETE'])
@admin_required
def delete_column(cid):
"""删除栏目（会同步删除用户栏目权限关联）"""
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute("DELETE FROM user_column_permission WHERE column_id = %s", (cid,))
cur.execute("DELETE FROM columns WHERE id = %s", (cid,))
if cur.rowcount == 0:
conn.close()
return jsonify({'success': False, 'error': '栏目不存在'})
conn.commit()
conn.close()
return jsonify({'success': True, 'msg': '删除成功'})
except Exception as e:
return jsonify({'success': False, 'error': str(e)})
@admin_api.route('/user/<int:uid>/columns')
@admin_required
def get_user_columns(uid):
"""获取某用户的栏目权限 ID 列表（编辑用户时回显）"""
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute(
"SELECT column_id FROM user_column_permission WHERE user_id = %s",
(uid,),
)
rows = cur.fetchall()
conn.close()
column_ids = [r['column_id'] for r in rows]
return jsonify({'success': True, 'column_ids': column_ids})
except Exception as e:
return jsonify({'success': False, 'error': str(e)})
@admin_api.route('/user/<int:uid>/column-permissions')
@admin_required
def get_user_column_permissions(uid):
"""根据用户获取其拥有的栏目权限详细信息"""
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute(
"""
SELECT c.id, c.name, c.column_key, c.created_at
FROM user_column_permission ucp
JOIN columns c ON c.id = ucp.column_id
WHERE ucp.user_id = %s
ORDER BY c.id
""",
(uid,),
)
rows = cur.fetchall()
conn.close()
items = [
{
'id': r['id'],
'name': r['name'] or '',
'column_key': r['column_key'] or '',
'created_at': r['created_at'].strftime('%Y-%m-%d %H:%M') if r.get('created_at') else '',
}
for r in rows
]
return jsonify({'success': True, 'items': items})
except Exception as e:
return jsonify({'success': False, 'error': str(e)})
def _set_user_column_permissions(cur, user_id, column_ids):
"""设置用户栏目权限：先删后插。cur 为已打开的游标。"""
cur.execute("DELETE FROM user_column_permission WHERE user_id = %s", (user_id,))
if column_ids:
column_ids = [int(x) for x in column_ids if x]
for cid in column_ids:
cur.execute(
"INSERT INTO user_column_permission (user_id, column_id) VALUES (%s, %s)",
(user_id, cid),
)
# ---------- 版本管理（web_config） ----------
@admin_api.route('/versions')
@admin_required
def list_versions():
"""获取版本列表"""
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute(
"SELECT id, version, file_url, created_at FROM web_config ORDER BY created_at DESC"
)
rows = cur.fetchall()
conn.close()
items = [
{
'id': r['id'],
'version': r['version'] or '',
'file_url': r['file_url'] or '',
'created_at': r['created_at'].strftime('%Y-%m-%d %H:%M') if r.get('created_at') else '',
}
for r in rows
]
return jsonify({'success': True, 'items': items})
except Exception as e:
return jsonify({'success': False, 'error': str(e)})
@admin_api.route('/version', methods=['POST'])
@admin_required
def upload_version():
"""接收版本号 + zip 文件，上传到 OSS，写入 web_config"""
version = (request.form.get('version') or '').strip()
if not version:
return jsonify({'success': False, 'error': '请填写版本号'})
file_storage = request.files.get('file')
if not file_storage or file_storage.filename == '':
return jsonify({'success': False, 'error': '请选择要上传的 zip 压缩包'})
if not (file_storage.filename or '').lower().endswith('.zip'):
return jsonify({'success': False, 'error': '仅支持 .zip 格式'})
try:
file_content = file_storage.read()
if not file_content:
return jsonify({'success': False, 'error': '文件为空'})
safe_key = _safe_version_key(version)
key = f"{bucket_path}versions/{safe_key}.zip"
file_url = oss_upload_file(file_content, key)
conn = get_db()
with conn.cursor() as cur:
cur.execute(
"INSERT INTO web_config (version, file_url) VALUES (%s, %s)",
(version, file_url)
)
conn.commit()
conn.close()
return jsonify({
'success': True,
'version': version,
'file_url': file_url,
'msg': '上传成功',
})
except Exception as e:
return jsonify({'success': False, 'error': str(e)})
# ---------- 数据去重总数据 ----------
@admin_api.route('/dedupe-total-data')
@admin_required
def list_dedupe_total_data():
page = max(1, int(request.args.get('page', 1)))
page_size = min(100, max(1, int(request.args.get('page_size', 15))))
keyword = (request.args.get('keyword') or '').strip()
data, error_response, status = _proxy_backend_java(
'GET',
'/api/admin/dedupe-total-data',
params={'page': page, 'pageSize': page_size, 'keyword': keyword},
)
if error_response is not None:
return error_response, status
payload = data.get('data') or {}
items = [
{
'id': item.get('id'),
'data_value': item.get('dataValue') or '',
'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16],
}
for item in (payload.get('items') or [])
]
return jsonify({
'success': True,
'items': items,
'total': payload.get('total') or 0,
'page': payload.get('page') or page,
'page_size': payload.get('pageSize') or page_size,
})
@admin_api.route('/dedupe-total-data/import', methods=['POST'])
@admin_required
def import_dedupe_total_data():
file_storage = request.files.get('file')
if not file_storage or file_storage.filename == '':
return jsonify({'success': False, 'error': '请选择 Excel 文件'})
files = {
'file': (file_storage.filename, file_storage.stream, file_storage.mimetype or 'application/octet-stream')
}
result, error_response, status = _proxy_backend_java(
'POST',
'/api/admin/dedupe-total-data/import',
files=files,
)
if error_response is not None:
return error_response, status
summary = result.get('data') or {}
return jsonify({
'success': True,
'msg': result.get('message') or '导入成功',
'summary': {
'total_rows': summary.get('totalRows') or 0,
'asin_count': summary.get('asinCount') or 0,
'inserted_count': summary.get('insertedCount') or 0,
'skipped_count': summary.get('skippedCount') or 0,
},
})
@admin_api.route('/dedupe-total-data', methods=['POST'])
@admin_required
def create_dedupe_total_data():
data = request.get_json() or {}
payload = {'dataValue': (data.get('data_value') or '').strip()}
result, error_response, status = _proxy_backend_java(
'POST',
'/api/admin/dedupe-total-data',
json_data=payload,
)
if error_response is not None:
return error_response, status
item = result.get('data') or {}
return jsonify({
'success': True,
'msg': result.get('message') or '创建成功',
'item': {
'id': item.get('id'),
'data_value': item.get('dataValue') or '',
'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16],
},
})
@admin_api.route('/dedupe-total-data/<int:item_id>', methods=['PUT'])
@admin_required
def update_dedupe_total_data(item_id):
data = request.get_json() or {}
payload = {'dataValue': (data.get('data_value') or '').strip()}
result, error_response, status = _proxy_backend_java(
'PUT',
f'/api/admin/dedupe-total-data/{item_id}',
json_data=payload,
)
if error_response is not None:
return error_response, status
item = result.get('data') or {}
return jsonify({
'success': True,
'msg': result.get('message') or '更新成功',
'item': {
'id': item.get('id'),
'data_value': item.get('dataValue') or '',
'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16],
},
})
@admin_api.route('/dedupe-total-data/<int:item_id>', methods=['DELETE'])
@admin_required
def delete_dedupe_total_data(item_id):
result, error_response, status = _proxy_backend_java(
'DELETE',
f'/api/admin/dedupe-total-data/{item_id}',
)
if error_response is not None:
return error_response, status
return jsonify({
'success': True,
'msg': result.get('message') or '删除成功',
})

74
backend/blueprints/auth.py Normal file
View File

@@ -0,0 +1,74 @@
"""
认证蓝图：登录、登出、登录状态校验
"""
from flask import Blueprint, request, redirect, url_for, session, jsonify
from werkzeug.security import check_password_hash
from utils.db import get_db
from utils.auth import login_required, is_session_user_valid
from utils.render import render_html
auth = Blueprint('auth', __name__, url_prefix='')
@auth.route('/login', methods=['GET', 'POST'])
def login():
if session.get('user_id') and is_session_user_valid():
return redirect(url_for('main.admin_page'))
if request.method == 'POST':
data = request.get_json() if request.is_json else request.form
username = (data.get('username') or '').strip()
password = data.get('password') or ''
if not username or not password:
if request.is_json:
return jsonify({'success': False, 'error': '请输入用户名和密码'})
return render_html('login.html', error='请输入用户名和密码')
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute(
"SELECT id, password_hash, machine, is_admin FROM users WHERE username = %s",
(username,)
)
row = cur.fetchone()
if row and check_password_hash(row['password_hash'], password):
session.permanent = True
session['user_id'] = row['id']
session['username'] = username
if request.is_json:
return jsonify({'success': True, 'redirect': url_for('main.admin_page')})
return redirect(url_for('main.admin_page'))
conn.close()
except Exception as e:
if request.is_json:
return jsonify({'success': False, 'error': str(e)})
return render_html('login.html', error='登录失败，请稍后重试')
if request.is_json:
return jsonify({'success': False, 'error': '用户名或密码错误'})
return render_html('login.html', error='用户名或密码错误')
return render_html('login.html')
@auth.route('/api/auth/check')
@login_required
def api_auth_check():
"""校验登录状态，用于页面加载时判断是否已登录"""
if not session.get('user_id'):
return jsonify({'logged_in': False})
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute("SELECT machine, is_admin FROM users WHERE id = %s", (session['user_id'],))
row = cur.fetchone()
conn.close()
if not row:
return jsonify({'logged_in': False})
except Exception:
return jsonify({'logged_in': False})
return jsonify({'logged_in': True, 'redirect': url_for('main.admin_page')})
@auth.route('/logout')
def logout():
session.clear()
return redirect(url_for('auth.login'))

38
backend/blueprints/main.py Normal file
View File

@@ -0,0 +1,38 @@
"""
主页面蓝图：首页、管理后台页、静态文件
"""
import os
from flask import Blueprint, redirect, url_for, send_file, session
from utils.auth import login_required, admin_required, is_session_user_valid
from utils.render import render_html
main = Blueprint('main', __name__, url_prefix='')
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
STATIC_DIR = os.path.join(BASE_DIR, 'static')
@main.route('/')
def index():
if session.get('user_id') and is_session_user_valid():
return redirect(url_for('main.admin_page'))
return redirect(url_for('auth.login'))
@main.route('/admin')
@login_required
@admin_required
def admin_page():
return render_html('admin.html')
@main.route('/static/<path:filename>')
def serve_static(filename):
"""提供 static 目录及子目录下的静态文件访问"""
filepath = os.path.normpath(os.path.join(STATIC_DIR, filename))
static_abs = os.path.abspath(STATIC_DIR)
file_abs = os.path.abspath(filepath)
if not file_abs.startswith(static_abs) or not os.path.isfile(file_abs):
return '', 404
return send_file(file_abs, as_attachment=False)

40
backend/blueprints/version.py Normal file
View File

@@ -0,0 +1,40 @@
"""
版本公开 API 蓝图：当前版本、最新版本下载链接
"""
import os
from flask import Blueprint, jsonify
from utils.db import get_db
version_bp = Blueprint('version', __name__, url_prefix='/api')
@version_bp.route('/version')
def api_version():
"""检测更新：返回当前版本及可选的最新版本信息"""
return jsonify({
'version': os.environ.get('APP_VERSION', '1.0.0'),
'desc': '',
'url': os.environ.get('APP_UPDATE_URL', ''),
})
@version_bp.route('/version/latest')
def api_version_latest():
"""GET 获取最新版本的下载链接（按创建时间取最新一条）"""
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute(
"SELECT version, file_url FROM web_config ORDER BY created_at DESC LIMIT 1"
)
row = cur.fetchone()
conn.close()
if not row:
return jsonify({'version': None, 'file_url': None})
return jsonify({
'version': row['version'] or '',
'file_url': row['file_url'] or '',
})
except Exception as e:
return jsonify({'error': str(e)}), 500

38
backend/config.py Normal file
View File

@@ -0,0 +1,38 @@
base_url = "https://api.coze.cn/v1"
coze_token = "sat_12nW40INoJxArrDXbY4lSCoudbqkOYTcphC99BP2efWyzxmsk4q81WDX3ezWgqZ5"
workflow_id = "7608812635877900322"
STITCH_WORKFLOW_ID = "7608813873483300907"
# MySQL 配置
mysql_host = "8.136.19.173"
mysql_user = "aiimage"
mysql_password = "WTFrb5y6hNLz6hNy" # 请修改为您的数据库密码
mysql_database = "aiimage"
cache_path = "./user_data"
region = "cn-hangzhou"
endpoint = "oss-cn-hangzhou.aliyuncs.com"
bucket = "nanri-ai-images"
accessKeyId = "LTAI5tNpyvzMNz9f2dHarsm8"
accessKeySecret = "bQSZnFH455i8tzyOgeahJmUzwmhynz"
bucket_path = "nanri-image/"
file_url_pre = f"https://{bucket}.oss-cn-hangzhou.aliyuncs.com/"
import os
backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://127.0.0.1:18080').rstrip('/')
os.environ['OSS_ACCESS_KEY_ID'] = accessKeyId
os.environ['OSS_ACCESS_KEY_SECRET'] = accessKeySecret
os.environ['SECRET_KEY'] = "ddffc7c1d02121d9554d7b080b2511b6"
debug = True
version = "1.0.0"
APP_UPDATE_URL = ""
os.environ['APP_VERSION'] = version
os.environ['APP_UPDATE_URL'] = version

61
backend/requirement.txt Normal file
View File

@@ -0,0 +1,61 @@
alibabacloud-oss-v2==1.2.4
annotated-types==0.7.0
anyio==4.12.1
Authlib==1.6.8
blinker==1.9.0
bottle==0.13.4
certifi==2026.1.4
cffi==2.0.0
charset-normalizer==3.4.4
click==8.1.8
clr_loader==0.2.10
colorama==0.4.6
cozepy==0.20.0
crcmod-plus==2.3.1
cryptography==41.0.0
distro==1.9.0
et_xmlfile==2.0.0
exceptiongroup==1.3.1
Flask==3.1.3
flask-cors==6.0.2
h11==0.16.0
httpcore==1.0.9
httpx==0.28.1
idna==3.11
importlib_metadata==8.7.1
itsdangerous==2.2.0
Jinja2==3.1.6
MarkupSafe==3.0.3
Nuitka==2.8.6
numpy==1.25.2
openpyxl==3.1.5
ordered-set==4.1.0
pandas==2.3.3
pillow==11.3.0
pip==26.0.1
proxy_tools==0.1.0
psutil==7.2.2
pycparser==2.23
pycryptodome==3.23.0
pydantic==2.12.5
pydantic_core==2.41.5
PyMySQL==1.1.2
PyQt5==5.15.11
PyQt5-Qt5==5.15.2
PyQt5_sip==12.17.1
python-dateutil==2.9.0.post0
pythonnet==3.0.5
pytz==2026.1.post1
pywebview==6.1
requests==2.32.5
setuptools==80.9.0
six==1.17.0
typing_extensions==4.15.0
typing-inspection==0.4.2
tzdata==2025.3
urllib3==2.6.3
websockets==14.2
Werkzeug==3.1.6
wheel==0.45.1
zipp==3.23.0
zstandard==0.25.0

BIN
backend/static/bg.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 160 KiB

BIN
backend/tool/__pycache__/devices.cpython-39.pyc Normal file
View File

Binary file not shown.

249
backend/tool/devices.py Normal file
View File

@@ -0,0 +1,249 @@
import hashlib
import platform
import subprocess
import uuid
import os
from typing import Optional
class DeviceIDGenerator:
"""
Windows设备唯一ID生成器
通过收集多个硬件特征来生成稳定的设备唯一标识符
"""
def __init__(self, use_cache: bool = True, cache_file: str = ".device_id"):
"""
初始化设备ID生成器
Args:
use_cache: 是否使用本地缓存
cache_file: 缓存文件名
"""
self.use_cache = use_cache
self.cache_file = cache_file
def _run_wmic_command(self, command: str) -> Optional[str]:
"""
执行WMIC命令并返回结果
Args:
command: WMIC命令
Returns:
命令执行结果，失败则返回None
"""
try:
result = subprocess.run(
command,
shell=True,
capture_output=True,
text=True,
timeout=10
)
if result.returncode == 0 and result.stdout.strip():
return result.stdout.strip()
except (subprocess.TimeoutExpired, Exception):
pass
return None
def _get_motherboard_serial(self) -> Optional[str]:
"""获取主板序列号"""
return self._run_wmic_command("wmic baseboard get serialnumber /value")
def _get_cpu_id(self) -> Optional[str]:
"""获取CPU ID"""
return self._run_wmic_command("wmic cpu get processorid /value")
def _get_bios_serial(self) -> Optional[str]:
"""获取BIOS序列号"""
return self._run_wmic_command("wmic bios get serialnumber /value")
def _get_disk_serial(self) -> Optional[str]:
"""获取系统盘序列号"""
return self._run_wmic_command("wmic diskdrive get serialnumber /value")
def _get_machine_guid(self) -> Optional[str]:
"""获取Windows机器GUID"""
try:
result = subprocess.run(
'reg query "HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Cryptography" /v MachineGuid',
shell=True,
capture_output=True,
text=True,
timeout=10
)
if result.returncode == 0:
for line in result.stdout.split('\n'):
if 'MachineGuid' in line:
return line.split()[-1]
except Exception:
pass
return None
def _extract_value(self, wmic_output: str) -> str:
"""从WMIC输出中提取实际值"""
if not wmic_output:
return ""
lines = wmic_output.split('\n')
for line in lines:
if '=' in line and not line.strip().endswith('='):
return line.split('=', 1)[1].strip()
return ""
def _collect_hardware_info(self) -> dict:
"""
收集硬件信息
Returns:
包含各种硬件信息的字典
"""
hardware_info = {}
# 主板序列号
motherboard = self._get_motherboard_serial()
hardware_info['motherboard'] = self._extract_value(motherboard) if motherboard else ""
# CPU ID
cpu_id = self._get_cpu_id()
hardware_info['cpu'] = self._extract_value(cpu_id) if cpu_id else ""
# BIOS序列号
bios = self._get_bios_serial()
hardware_info['bios'] = self._extract_value(bios) if bios else ""
# 硬盘序列号
disk = self._get_disk_serial()
hardware_info['disk'] = self._extract_value(disk) if disk else ""
# Windows机器GUID
machine_guid = self._get_machine_guid()
hardware_info['machine_guid'] = machine_guid if machine_guid else ""
# 计算机名称
hardware_info['computer_name'] = platform.node()
# MAC地址（作为备用）
hardware_info['mac_address'] = ':'.join(['{:02x}'.format((uuid.getnode() >> elements) & 0xff)
for elements in range(0, 2*6, 2)][::-1])
return hardware_info
def _generate_device_id(self, hardware_info: dict) -> str:
"""
基于硬件信息生成设备ID
Args:
hardware_info: 硬件信息字典
Returns:
32位十六进制设备ID
"""
# 过滤掉空值，并按键排序确保一致性
filtered_info = {k: v for k, v in hardware_info.items() if v and v.strip()}
# 如果没有任何硬件信息，使用MAC地址作为后备方案
if not filtered_info:
filtered_info = {'mac_address': hardware_info.get('mac_address', str(uuid.getnode()))}
# 将所有信息连接成字符串
info_string = '|'.join(f"{k}:{v}" for k, v in sorted(filtered_info.items()))
# 使用SHA256生成哈希值
hash_object = hashlib.sha256(info_string.encode('utf-8'))
device_id = hash_object.hexdigest()
return device_id
def _load_cached_device_id(self) -> Optional[str]:
"""从缓存文件加载设备ID"""
try:
if os.path.exists(self.cache_file):
with open(self.cache_file, 'r', encoding='utf-8') as f:
cached_id = f.read().strip()
if len(cached_id) == 64: # SHA256哈希长度
return cached_id
except Exception:
pass
return None
def _save_device_id_to_cache(self, device_id: str) -> None:
"""将设备ID保存到缓存文件"""
try:
with open(self.cache_file, 'w', encoding='utf-8') as f:
f.write(device_id)
except Exception:
pass
def get_device_id(self) -> str:
"""
获取设备唯一ID
Returns:
64字符的十六进制设备ID
"""
# 如果启用缓存，先尝试从缓存加载
if self.use_cache:
cached_id = self._load_cached_device_id()
if cached_id:
return cached_id
# 收集硬件信息
hardware_info = self._collect_hardware_info()
# 生成设备ID
device_id = self._generate_device_id(hardware_info)
# 保存到缓存
if self.use_cache:
self._save_device_id_to_cache(device_id)
return device_id
def get_device_id_short(self, length: int = 16) -> str:
"""
获取短版本的设备ID
Args:
length: 返回ID的长度
Returns:
指定长度的设备ID
"""
full_id = self.get_device_id()
return full_id[:length]
def get_hardware_info(self) -> dict:
"""
获取硬件信息（用于调试）
Returns:
硬件信息字典
"""
return self._collect_hardware_info()
# 使用示例
def main():
"""使用示例"""
# 创建设备ID生成器实例
device_generator = DeviceIDGenerator()
# 获取完整设备ID（64字符）
device_id = device_generator.get_device_id()
print(f"完整设备ID: {device_id}")
# 获取短版本设备ID（16字符）
short_id = device_generator.get_device_id_short(16)
print(f"短设备ID: {short_id}")
# 查看硬件信息（调试用）
hardware_info = device_generator.get_hardware_info()
print("\n硬件信息:")
for key, value in hardware_info.items():
print(f" {key}: {value}")
if __name__ == "__main__":
main()

1
backend/utils/__init__.py Normal file
View File

@@ -0,0 +1 @@
# Utils 包：数据库、认证装饰器、模板渲染等

BIN
backend/utils/__pycache__/__init__.cpython-311.pyc Normal file
View File

Binary file not shown.

BIN
backend/utils/__pycache__/__init__.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
backend/utils/__pycache__/__init__.cpython-39.pyc Normal file
View File

Binary file not shown.

BIN
backend/utils/__pycache__/auth.cpython-311.pyc Normal file
View File

Binary file not shown.

BIN
backend/utils/__pycache__/auth.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
backend/utils/__pycache__/auth.cpython-39.pyc Normal file
View File

Binary file not shown.

BIN
backend/utils/__pycache__/db.cpython-311.pyc Normal file
View File

Binary file not shown.

BIN
backend/utils/__pycache__/db.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
backend/utils/__pycache__/db.cpython-39.pyc Normal file
View File

Binary file not shown.

BIN
backend/utils/__pycache__/helpers.cpython-311.pyc Normal file
View File

Binary file not shown.

BIN
backend/utils/__pycache__/render.cpython-311.pyc Normal file
View File

Binary file not shown.

BIN
backend/utils/__pycache__/render.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
backend/utils/__pycache__/render.cpython-39.pyc Normal file
View File

Binary file not shown.

81
backend/utils/auth.py Normal file
View File

@@ -0,0 +1,81 @@
"""
认证装饰器与 session 校验
"""
from functools import wraps
from flask import request, redirect, url_for, session, jsonify
from utils.db import get_db
def is_session_user_valid():
"""校验 session 中的 user_id 是否在数据库中仍存在；不存在则清除 session 并返回 False"""
uid = session.get('user_id')
if not uid:
return False
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute("SELECT id FROM users WHERE id = %s", (uid,))
row = cur.fetchone()
conn.close()
if not row:
session.clear()
return False
return True
except Exception:
session.clear()
return False
def get_current_admin_role():
"""获取当前登录用户的管理角色：super_admin / admin / None（非管理员）"""
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute(
"SELECT id, username, is_admin, role, created_by_id FROM users WHERE id = %s",
(session['user_id'],)
)
row = cur.fetchone()
conn.close()
if not row or not row.get('is_admin'):
return None, None
return row.get('role') or ('super_admin' if row.get('created_by_id') is None else 'admin'), row
except Exception:
return None, None
def login_required(f):
@wraps(f)
def decorated(*args, **kwargs):
if not session.get('user_id') or not is_session_user_valid():
if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
return jsonify({'success': False, 'error': '未登录'}), 401
return redirect(url_for('auth.login'))
return f(*args, **kwargs)
return decorated
def admin_required(f):
@wraps(f)
def decorated(*args, **kwargs):
if not session.get('user_id'):
if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
return jsonify({'success': False, 'error': '未登录'}), 401
return redirect(url_for('auth.login'))
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute("SELECT is_admin, role FROM users WHERE id = %s", (session['user_id'],))
row = cur.fetchone()
conn.close()
if not row or not row.get('is_admin'):
if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
return jsonify({'success': False, 'error': '需要管理员权限'}), 403
return redirect(url_for('main.admin_page'))
except Exception as e:
if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
return jsonify({'success': False, 'error': str(e)}), 500
return redirect(url_for('main.admin_page'))
return f(*args, **kwargs)
return decorated

140
backend/utils/db.py Normal file
View File

@@ -0,0 +1,140 @@
"""
数据库连接与初始化
"""
import os
import pymysql
from werkzeug.security import generate_password_hash
try:
from config import mysql_host, mysql_user, mysql_password, mysql_database
except ImportError:
mysql_host = os.environ.get('MYSQL_HOST', 'localhost')
mysql_user = os.environ.get('MYSQL_USER', 'root')
mysql_password = os.environ.get('MYSQL_PASSWORD', '')
mysql_database = os.environ.get('MYSQL_DATABASE', 'maixiang_ai')
def get_db():
return pymysql.connect(
host=mysql_host,
user=mysql_user,
password=mysql_password,
database=mysql_database,
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor
)
def _create_initial_admin():
"""若没有任何管理员，则创建默认超级管理员（首次启动时，仅一个）"""
try:
conn = get_db()
with conn.cursor() as cur:
cur.execute("SELECT id FROM users WHERE role = 'super_admin' LIMIT 1")
if cur.fetchone():
conn.close()
return
admin_user = os.environ.get('ADMIN_USER', 'admin')
admin_pwd = os.environ.get('ADMIN_PASSWORD', 'admin123')
pwd_hash = generate_password_hash(admin_pwd, method='pbkdf2:sha256')
cur.execute(
"INSERT INTO users (username, password_hash, is_admin, role) VALUES (%s, %s, 1, 'super_admin')",
(admin_user, pwd_hash)
)
conn.commit()
conn.close()
except Exception:
pass
def init_db():
"""初始化数据库表，若不存在则创建"""
conn = pymysql.connect(
host=mysql_host,
user=mysql_user,
password=mysql_password,
charset='utf8mb4'
)
try:
with conn.cursor() as cur:
cur.execute(f"CREATE DATABASE IF NOT EXISTS `{mysql_database}` DEFAULT CHARSET utf8mb4")
cur.execute(f"USE `{mysql_database}`")
cur.execute("""
CREATE TABLE IF NOT EXISTS users (
id INT AUTO_INCREMENT PRIMARY KEY,
username VARCHAR(64) NOT NULL UNIQUE,
password_hash VARCHAR(256) NOT NULL,
is_admin TINYINT(1) DEFAULT 0,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
)
""")
cur.execute("""
CREATE TABLE IF NOT EXISTS image_history (
id INT AUTO_INCREMENT PRIMARY KEY,
user_id INT NOT NULL,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
panel_type VARCHAR(64) DEFAULT '',
original_urls JSON,
params JSON,
result_urls JSON,
long_image_url VARCHAR(1024) DEFAULT NULL,
INDEX idx_user_created (user_id, created_at DESC)
)
""")
try:
cur.execute("ALTER TABLE image_history ADD COLUMN long_image_url VARCHAR(1024) DEFAULT NULL")
except Exception:
pass
try:
cur.execute("ALTER TABLE users ADD COLUMN machine VARCHAR(64) DEFAULT NULL")
except Exception:
pass
try:
cur.execute("ALTER TABLE users ADD COLUMN role VARCHAR(20) DEFAULT 'normal'")
except Exception:
pass
try:
cur.execute("ALTER TABLE users ADD COLUMN created_by_id INT NULL")
except Exception:
pass
cur.execute("""
CREATE TABLE IF NOT EXISTS web_config (
id INT AUTO_INCREMENT PRIMARY KEY,
version VARCHAR(64) NOT NULL,
file_url VARCHAR(1024) NOT NULL,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
)
""")
cur.execute("""
CREATE TABLE IF NOT EXISTS columns (
id INT AUTO_INCREMENT PRIMARY KEY,
name VARCHAR(128) NOT NULL COMMENT '栏目名',
column_key VARCHAR(64) NOT NULL COMMENT '栏目标识',
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
UNIQUE KEY uk_column_key (column_key)
)
""")
cur.execute("""
CREATE TABLE IF NOT EXISTS user_column_permission (
user_id INT NOT NULL,
column_id INT NOT NULL,
PRIMARY KEY (user_id, column_id),
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE,
FOREIGN KEY (column_id) REFERENCES columns(id) ON DELETE CASCADE
)
""")
try:
cur.execute("UPDATE users SET role = 'normal' WHERE (role IS NULL OR role = '') AND (is_admin = 0 OR is_admin IS NULL)")
cur.execute("SELECT MIN(id) AS mid FROM users WHERE is_admin = 1")
row = cur.fetchone()
if row and row.get('mid'):
mid = row['mid']
cur.execute("UPDATE users SET role = 'super_admin' WHERE id = %s", (mid,))
cur.execute("UPDATE users SET role = 'admin' WHERE is_admin = 1 AND id != %s", (mid,))
cur.execute("UPDATE users SET created_by_id = %s WHERE role = 'admin' AND (created_by_id IS NULL)", (mid,))
except Exception:
pass
conn.commit()
finally:
conn.close()
_create_initial_admin()

Some files were not shown because too many files have changed in this diff Show More