diff --git a/1.xlsx b/1.xlsx deleted file mode 100644 index a6e124d..0000000 Binary files a/1.xlsx and /dev/null differ diff --git a/app/__pycache__/app.cpython-312.pyc b/app/__pycache__/app.cpython-312.pyc index 234a9f4..c0ed153 100644 Binary files a/app/__pycache__/app.cpython-312.pyc and b/app/__pycache__/app.cpython-312.pyc differ diff --git a/app/__pycache__/config.cpython-312.pyc b/app/__pycache__/config.cpython-312.pyc index d650bd0..6b0f230 100644 Binary files a/app/__pycache__/config.cpython-312.pyc and b/app/__pycache__/config.cpython-312.pyc differ diff --git a/app/blueprints/__pycache__/brand.cpython-312.pyc b/app/blueprints/__pycache__/brand.cpython-312.pyc index 122490f..05adae3 100644 Binary files a/app/blueprints/__pycache__/brand.cpython-312.pyc and b/app/blueprints/__pycache__/brand.cpython-312.pyc differ diff --git a/app/blueprints/brand.py b/app/blueprints/brand.py index 01924dc..1bf2a65 100644 --- a/app/blueprints/brand.py +++ b/app/blueprints/brand.py @@ -57,14 +57,23 @@ def _push_task_event(task_id, event): def _expand_folder_xlsx(folder_path): - """返回文件夹下所有 .xlsx 文件的绝对路径列表""" + """返回文件夹下所有 .xlsx 文件的绝对路径和相对路径列表（递归）""" if not folder_path or not os.path.isdir(folder_path): return [] - paths = [] - for name in os.listdir(folder_path): - if name.endswith('.xlsx') or name.endswith('.XLSX'): - paths.append(os.path.normpath(os.path.join(folder_path, name))) - return paths + items = [] + root = os.path.normpath(folder_path) + for current_root, _, filenames in os.walk(root): + for name in filenames: + if not (name.endswith('.xlsx') or name.endswith('.XLSX')): + continue + absolute_path = os.path.normpath(os.path.join(current_root, name)) + relative_path = os.path.relpath(absolute_path, root).replace('\\', '/') + items.append({ + 'absolutePath': absolute_path, + 'relativePath': relative_path, + }) + items.sort(key=lambda item: item['relativePath']) + return items def _upload_local_xlsx_to_oss(local_path, user_id): @@ -478,8 +487,8 @@ def api_brand_expand_folder(): folder = (data.get('folder') or '').strip() if not folder: return jsonify({'success': False, 'error': '请提供 folder 路径'}), 400 - paths = _expand_folder_xlsx(folder) - return jsonify({'success': True, 'paths': paths}) + items = _expand_folder_xlsx(folder) + return jsonify({'success': True, 'items': items}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 diff --git a/app/main.py b/app/main.py index 517c4ac..96bac03 100644 --- a/app/main.py +++ b/app/main.py @@ -145,7 +145,7 @@ class WindowAPI: ) return result[0] if result else '' - def upload_file_to_java(self, file_path): + def upload_file_to_java(self, file_path, relative_path=None): """按本地路径读取文件并上传到 Java 后端临时目录。""" if not file_path or not str(file_path).strip(): return {'success': False, 'error': '文件路径为空'} @@ -157,6 +157,7 @@ class WindowAPI: response = requests.post( f"{JAVA_API_BASE}/api/files/upload", files={'file': (os.path.basename(normalized_path), file_obj)}, + data={'relativePath': str(relative_path).strip()} if relative_path else None, timeout=120, ) response.raise_for_status() diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/dto/ConvertRunRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/dto/ConvertRunRequest.java index 5aecfa2..bad0cbc 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/dto/ConvertRunRequest.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/dto/ConvertRunRequest.java @@ -20,4 +20,7 @@ public class ConvertRunRequest { @NotBlank(message = "请选择模板") @Schema(description = "模板编码，例如 uk_offer") private String templateId; + + @Schema(description = "文件夹上传场景下的总压缩包名称（不含扩展名）") + private String archiveName; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/dto/UploadedSourceFileDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/dto/UploadedSourceFileDto.java index f4205c6..de0a6a1 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/dto/UploadedSourceFileDto.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/model/dto/UploadedSourceFileDto.java @@ -14,4 +14,7 @@ public class UploadedSourceFileDto { @Schema(description = "原始文件名") private String originalFilename; + + @Schema(description = "相对上传目录路径") + private String relativePath; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertRunService.java index 64e3d97..50445f8 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertRunService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertRunService.java @@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.convert.service; import cn.hutool.core.io.FileUtil; import cn.hutool.core.util.IdUtil; import cn.hutool.json.JSONUtil; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.nanri.aiimage.common.exception.BusinessException; @@ -22,24 +23,38 @@ import org.apache.poi.ss.usermodel.DataFormatter; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.ss.usermodel.WorkbookFactory; import org.springframework.stereotype.Service; import java.io.File; import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; - -import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; @Service @RequiredArgsConstructor public class ConvertRunService { + private static final String MODULE_TYPE = "CONVERT"; + private static final String TEMPLATE_CODE_FIVE_COUNTRIES = "uk_offer"; + private static final List FIVE_COUNTRY_OUTPUT_FILES = List.of( + "\u82f1\u56fd.txt", + "\u6cd5\u56fd.txt", + "\u5fb7\u56fd.txt", + "\u897f\u73ed\u7259.txt", + "\u610f\u5927\u5229.txt" + ); + private final FileTaskMapper fileTaskMapper; private final FileResultMapper fileResultMapper; private final StorageProperties storageProperties; @@ -51,8 +66,8 @@ public class ConvertRunService { ConvertTemplateEntity template = convertTemplateService.getByCode(request.getTemplateId()); FileTaskEntity task = new FileTaskEntity(); - task.setTaskNo("CONVERT-" + IdUtil.getSnowflakeNextIdStr()); - task.setModuleType("CONVERT"); + task.setTaskNo(MODULE_TYPE + "-" + IdUtil.getSnowflakeNextIdStr()); + task.setModuleType(MODULE_TYPE); task.setTaskMode("IMMEDIATE"); task.setStatus("SUCCESS"); task.setSourceFileCount(request.getFiles().size()); @@ -63,63 +78,107 @@ public class ConvertRunService { task.setFinishedAt(LocalDateTime.now()); fileTaskMapper.insert(task); + boolean folderMode = request.getArchiveName() != null && !request.getArchiveName().isBlank(); List items = new ArrayList<>(); int successCount = 0; int failedCount = 0; + List archiveEntries = new ArrayList<>(); for (UploadedSourceFileDto sourceFile : request.getFiles()) { - ConvertResultItemVo item = new ConvertResultItemVo(); - item.setSourceFilename(sourceFile.getOriginalFilename()); try { File inputFile = findLocalSourceFile(sourceFile.getFileKey()); if (inputFile == null || !inputFile.exists()) { - throw new BusinessException("上传文件不存在，请重新上传"); + throw new BusinessException("Uploaded source file does not exist, please upload again."); } - String inputName = sourceFile.getOriginalFilename() == null ? inputFile.getName() : sourceFile.getOriginalFilename(); - String outputFilename = template.getOutputFilename() == null || template.getOutputFilename().isBlank() - ? "结果.txt" - : template.getOutputFilename(); - File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "convert-result")); - File outputFile = buildNamedOutputFile(outputDir, outputFilename); + String inputName = sourceFile.getOriginalFilename() == null + ? inputFile.getName() + : sourceFile.getOriginalFilename(); + List generatedFiles = generateOutputFiles(inputFile, template); + if (folderMode) { + for (GeneratedConvertFile generatedFile : generatedFiles) { + archiveEntries.add(new ConvertArchiveEntry(sourceFile.getRelativePath(), inputName, generatedFile)); + } + successCount++; + continue; + } - writeTxtByLegacyRules(inputFile, outputFile, template); + boolean useZipPackage = request.getFiles().size() == 1 && generatedFiles.size() > 1; + File packagedResultFile = useZipPackage + ? packageGeneratedFilesAsZip(inputName, generatedFiles) + : generatedFiles.getFirst().file(); + String packagedFilename = useZipPackage + ? FileUtil.mainName(inputName) + ".zip" + : generatedFiles.getFirst().filename(); + String contentType = useZipPackage ? "application/zip" : "text/plain"; - String ossObjectKey = ossStorageService.uploadResultFile(outputFile, "CONVERT"); + String ossObjectKey = ossStorageService.uploadResultFile(packagedResultFile, MODULE_TYPE); String downloadUrl = ossStorageService.generateDownloadUrl(ossObjectKey); + ConvertResultItemVo item = new ConvertResultItemVo(); + item.setSourceFilename(inputName); item.setSuccess(true); - item.setOutputFilename(outputFilename); + item.setOutputFilename(packagedFilename); item.setDownloadUrl(downloadUrl); - successCount++; FileResultEntity resultEntity = new FileResultEntity(); resultEntity.setTaskId(task.getId()); - resultEntity.setModuleType("CONVERT"); + resultEntity.setModuleType(MODULE_TYPE); resultEntity.setSourceFilename(inputName); - resultEntity.setResultFilename(outputFilename); + resultEntity.setResultFilename(packagedFilename); resultEntity.setResultFileUrl(downloadUrl); - resultEntity.setResultFileSize(outputFile.length()); - resultEntity.setResultContentType("text/plain"); + resultEntity.setResultFileSize(packagedResultFile.length()); + resultEntity.setResultContentType(contentType); resultEntity.setSuccess(1); resultEntity.setCreatedAt(LocalDateTime.now()); fileResultMapper.insert(resultEntity); item.setResultId(resultEntity.getId()); + + items.add(item); + successCount++; } catch (Exception ex) { + ConvertResultItemVo item = new ConvertResultItemVo(); + item.setSourceFilename(sourceFile.getOriginalFilename()); item.setSuccess(false); item.setError(ex.getMessage()); + items.add(item); failedCount++; FileResultEntity resultEntity = new FileResultEntity(); resultEntity.setTaskId(task.getId()); - resultEntity.setModuleType("CONVERT"); + resultEntity.setModuleType(MODULE_TYPE); resultEntity.setSourceFilename(sourceFile.getOriginalFilename()); resultEntity.setSuccess(0); resultEntity.setErrorMessage(ex.getMessage()); resultEntity.setCreatedAt(LocalDateTime.now()); fileResultMapper.insert(resultEntity); } - items.add(item); + } + + if (folderMode && !archiveEntries.isEmpty()) { + File zipFile = packageFolderConvertResultsAsZip(request.getArchiveName(), archiveEntries); + String ossObjectKey = ossStorageService.uploadResultFile(zipFile, MODULE_TYPE); + String downloadUrl = ossStorageService.generateDownloadUrl(ossObjectKey); + + ConvertResultItemVo item = new ConvertResultItemVo(); + item.setSourceFilename(request.getArchiveName()); + item.setOutputFilename(zipFile.getName()); + item.setSuccess(true); + item.setDownloadUrl(downloadUrl); + + FileResultEntity resultEntity = new FileResultEntity(); + resultEntity.setTaskId(task.getId()); + resultEntity.setModuleType(MODULE_TYPE); + resultEntity.setSourceFilename(request.getArchiveName()); + resultEntity.setResultFilename(zipFile.getName()); + resultEntity.setResultFileUrl(downloadUrl); + resultEntity.setResultFileSize(zipFile.length()); + resultEntity.setResultContentType("application/zip"); + resultEntity.setSuccess(1); + resultEntity.setCreatedAt(LocalDateTime.now()); + fileResultMapper.insert(resultEntity); + item.setResultId(resultEntity.getId()); + items.add(0, item); } task.setSuccessFileCount(successCount); @@ -138,18 +197,18 @@ public class ConvertRunService { public List listHistory() { return fileResultMapper.selectList(new LambdaQueryWrapper() - .eq(FileResultEntity::getModuleType, "CONVERT") - .eq(FileResultEntity::getSuccess, 1) + .eq(FileResultEntity::getModuleType, MODULE_TYPE) .orderByDesc(FileResultEntity::getCreatedAt) - .last("limit 50")) + .last("limit 200")) .stream() .map(entity -> { ConvertResultItemVo vo = new ConvertResultItemVo(); vo.setResultId(entity.getId()); - vo.setSourceFilename(entity.getCreatedAt() == null ? "" : entity.getCreatedAt().toLocalDate().toString().replace("-", "")); + vo.setSourceFilename(entity.getSourceFilename()); vo.setOutputFilename(entity.getResultFilename()); - vo.setDownloadUrl(entity.getResultFileUrl()); - vo.setSuccess(true); + vo.setDownloadUrl(entity.getSuccess() != null && entity.getSuccess() == 1 ? entity.getResultFileUrl() : null); + vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1); + vo.setError(entity.getErrorMessage()); return vo; }) .toList(); @@ -157,41 +216,71 @@ public class ConvertRunService { public void deleteHistory(Long resultId) { FileResultEntity entity = fileResultMapper.selectById(resultId); - if (entity == null || !"CONVERT".equals(entity.getModuleType())) { - throw new BusinessException("记录不存在"); + if (entity == null || !MODULE_TYPE.equals(entity.getModuleType())) { + throw new BusinessException("Convert result record does not exist."); } fileResultMapper.deleteById(resultId); } public File getResultFile(Long resultId) { FileResultEntity entity = fileResultMapper.selectById(resultId); - if (entity == null || entity.getResultFileUrl() == null || !"CONVERT".equals(entity.getModuleType())) { - throw new BusinessException("结果文件不存在"); + if (entity == null || entity.getResultFileUrl() == null || !MODULE_TYPE.equals(entity.getModuleType())) { + throw new BusinessException("Convert result file does not exist."); } + File file = new File(entity.getResultFileUrl()); if (!file.exists()) { - throw new BusinessException("结果文件不存在"); + throw new BusinessException("Convert result file does not exist."); } return file; } - private void writeTxtByLegacyRules(File inputFile, File outputFile, ConvertTemplateEntity templateEntity) throws IOException { + private List generateOutputFiles(File inputFile, ConvertTemplateEntity templateEntity) throws IOException { + List outputFilenames = resolveOutputFilenames(templateEntity); + List rows = buildTxtRows(inputFile, templateEntity); + File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "convert-result")); + + List generatedFiles = new ArrayList<>(); + for (String outputFilename : outputFilenames) { + File outputFile = buildNamedOutputFile(outputDir, outputFilename); + FileUtil.writeLines(rows, outputFile, StandardCharsets.UTF_8); + generatedFiles.add(new GeneratedConvertFile(outputFilename, outputFile)); + } + return generatedFiles; + } + + private List resolveOutputFilenames(ConvertTemplateEntity templateEntity) { + if (TEMPLATE_CODE_FIVE_COUNTRIES.equals(templateEntity.getTemplateCode())) { + return FIVE_COUNTRY_OUTPUT_FILES; + } + + String outputFilename = templateEntity.getOutputFilename(); + if (outputFilename == null || outputFilename.isBlank()) { + return List.of("result.txt"); + } + return List.of(outputFilename); + } + + private List buildTxtRows(File inputFile, ConvertTemplateEntity templateEntity) throws IOException { List requiredColumns = readStringList(templateEntity.getRequiredSourceColumnsJson()); List headerColumns = readStringList(templateEntity.getHeaderColumnsJson()); Map fieldMapping = readStringMap(templateEntity.getFieldMappingJson()); Map defaults = readStringMap(templateEntity.getDefaultsJson()); List preambleLines = readStringList(templateEntity.getPreambleLinesJson()); - int blankHeaderRowsAfterSchema = templateEntity.getBlankHeaderRowsAfterSchema() == null ? 0 : templateEntity.getBlankHeaderRowsAfterSchema(); + int blankHeaderRowsAfterSchema = templateEntity.getBlankHeaderRowsAfterSchema() == null + ? 0 + : templateEntity.getBlankHeaderRowsAfterSchema(); DataFormatter formatter = new DataFormatter(); - try (FileInputStream fis = new FileInputStream(inputFile); Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis)) { + try (FileInputStream fis = new FileInputStream(inputFile); + Workbook workbook = WorkbookFactory.create(fis)) { Sheet sheet = workbook.getSheetAt(0); Row headerRow = sheet.getRow(0); if (headerRow == null) { - throw new BusinessException("Excel 表头为空"); + throw new BusinessException("Excel header row is empty."); } - Map headerMap = new HashMap<>(); + Map headerMap = new LinkedHashMap<>(); for (int i = 0; i < headerRow.getLastCellNum(); i++) { String value = normalizeCellText(formatter.formatCellValue(headerRow.getCell(i))); if (!value.isBlank() && !headerMap.containsKey(value)) { @@ -199,9 +288,11 @@ public class ConvertRunService { } } - List missing = requiredColumns.stream().filter(column -> !headerMap.containsKey(column)).toList(); + List missing = requiredColumns.stream() + .filter(column -> !headerMap.containsKey(column)) + .toList(); if (!missing.isEmpty()) { - throw new BusinessException("缺少列：" + String.join("、", missing)); + throw new BusinessException("Missing required columns: " + String.join(", ", missing)); } List rows = new ArrayList<>(); @@ -219,6 +310,7 @@ public class ConvertRunService { if (row == null) { continue; } + String asin = asinIndex >= 0 ? normalizeCellText(formatter.formatCellValue(row.getCell(asinIndex))) : ""; if (asin.isBlank()) { continue; @@ -233,7 +325,9 @@ public class ConvertRunService { if (fieldMapping.containsKey(column)) { String sourceColumn = fieldMapping.get(column); Integer sourceIndex = headerMap.get(sourceColumn); - String value = sourceIndex == null ? "" : normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex))); + String value = sourceIndex == null + ? "" + : normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex))); lineValues.add(value); continue; } @@ -243,10 +337,11 @@ public class ConvertRunService { } lineValues.add(""); } + rows.add(String.join("\t", lineValues)); rowIndex++; } - FileUtil.writeLines(rows, outputFile, StandardCharsets.UTF_8); + return rows; } } @@ -255,20 +350,70 @@ public class ConvertRunService { if (json == null || json.isBlank()) { return new ArrayList<>(); } - return objectMapper.readValue(json, new TypeReference>() {}); + return objectMapper.readValue(json, new TypeReference>() { + }); } catch (Exception ex) { - throw new BusinessException("模板配置解析失败"); + throw new BusinessException("Template config parsing failed."); } } + private File packageGeneratedFilesAsZip(String inputName, List generatedFiles) { + File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "convert-result")); + File zipFile = buildNamedOutputFile(outputDir, FileUtil.mainName(inputName) + ".zip"); + try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zipFile))) { + for (GeneratedConvertFile generatedFile : generatedFiles) { + ZipEntry entry = new ZipEntry(generatedFile.filename()); + zos.putNextEntry(entry); + Files.copy(generatedFile.file().toPath(), zos); + zos.closeEntry(); + } + } catch (Exception ex) { + throw new BusinessException("Convert result packaging failed: " + ex.getMessage()); + } + return zipFile; + } + + private File packageFolderConvertResultsAsZip(String archiveName, List archiveEntries) { + File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "convert-result")); + File zipFile = buildNamedOutputFile(outputDir, archiveName + ".zip"); + try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zipFile))) { + for (ConvertArchiveEntry entry : archiveEntries) { + String zipEntryName = buildFolderZipEntry(entry.relativePath(), entry.inputName(), entry.generatedFile().filename(), true); + zos.putNextEntry(new ZipEntry(zipEntryName)); + Files.copy(entry.generatedFile().file().toPath(), zos); + zos.closeEntry(); + } + } catch (Exception ex) { + throw new BusinessException("Convert folder result packaging failed: " + ex.getMessage()); + } + return zipFile; + } + + private String buildFolderZipEntry(String relativePath, String inputName, String outputFilename, boolean includeSourceStemFolder) { + String normalizedRelativePath = relativePath == null ? "" : relativePath.replace('\\', '/'); + String relativeDir = normalizedRelativePath.contains("/") + ? normalizedRelativePath.substring(0, normalizedRelativePath.lastIndexOf('/')) + : ""; + List parts = new ArrayList<>(); + if (!relativeDir.isBlank()) { + parts.add(relativeDir); + } + if (includeSourceStemFolder) { + parts.add(FileUtil.mainName(inputName)); + } + parts.add(outputFilename); + return String.join("/", parts); + } + private Map readStringMap(String json) { try { if (json == null || json.isBlank()) { return new HashMap<>(); } - return objectMapper.readValue(json, new TypeReference>() {}); + return objectMapper.readValue(json, new TypeReference>() { + }); } catch (Exception ex) { - throw new BusinessException("模板配置解析失败"); + throw new BusinessException("Template config parsing failed."); } } @@ -277,8 +422,11 @@ public class ConvertRunService { if (!baseDir.exists()) { return null; } - List matchedFiles = FileUtil.loopFiles(baseDir, pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey)); - return matchedFiles.isEmpty() ? null : matchedFiles.getFirst(); + List matchedFiles = FileUtil.loopFiles( + baseDir, + pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey) + ); + return matchedFiles.isEmpty() ? null : matchedFiles.get(0); } private File buildNamedOutputFile(File outputDir, String filename) { @@ -286,6 +434,7 @@ public class ConvertRunService { if (!candidate.exists()) { return candidate; } + String mainName = FileUtil.mainName(filename); String extName = FileUtil.extName(filename); int index = 2; @@ -314,4 +463,10 @@ public class ConvertRunService { .trim() .replaceAll("\\s+", " "); } + + private record GeneratedConvertFile(String filename, File file) { + } + + private record ConvertArchiveEntry(String relativePath, String inputName, GeneratedConvertFile generatedFile) { + } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeRunController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeRunController.java index 4e9c247..850266a 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeRunController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeRunController.java @@ -45,6 +45,7 @@ public class DedupeRunController { - 仅保留 selectedColumns 中指定的列； - keepIntegerIds=true 时保留纯数字 ID； - keepUnderscoreIds=true 时保留类似 1_1 的 ID； + - keepIntegerMainIdsWhenNoSubIds=true 且文件中不存在 1_1 这类子 ID 时，会自动保留纯数字主 ID； - 输出文件命名遵循 原文件名_cleaned.xlsx，若重名自动追加序号。 """) @ApiResponses({ diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeTotalDataController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeTotalDataController.java new file mode 100644 index 0000000..21df710 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeTotalDataController.java @@ -0,0 +1,93 @@ +package com.nanri.aiimage.modules.dedupe.controller; + +import com.nanri.aiimage.common.api.ApiResponse; +import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataCreateRequest; +import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataUpdateRequest; +import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataImportVo; +import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataItemVo; +import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataPageVo; +import com.nanri.aiimage.modules.dedupe.service.DedupeTotalDataService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.validation.Valid; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +@RestController +@RequiredArgsConstructor +@RequestMapping("/api/admin/dedupe-total-data") +@Tag(name = "数据去重总数据", description = "维护数据去重模块的总数据列表，支持增删改查。") +public class DedupeTotalDataController { + + private final DedupeTotalDataService dedupeTotalDataService; + + @GetMapping + @Operation(summary = "分页查询总数据", description = "分页查询数据去重总数据，支持按值模糊搜索。") + @ApiResponses({ + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "查询成功", content = @Content(schema = @Schema(implementation = DedupeTotalDataPageVo.class))) + }) + public ApiResponse page( + @Parameter(description = "页码") @RequestParam(defaultValue = "1") Long page, + @Parameter(description = "每页数量") @RequestParam(defaultValue = "15") Long pageSize, + @Parameter(description = "模糊搜索关键字") @RequestParam(required = false) String keyword) { + return ApiResponse.success(dedupeTotalDataService.page(page, pageSize, keyword)); + } + + @PostMapping + @Operation(summary = "新增总数据", description = "新增一条数据去重总数据。") + @ApiResponses({ + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "创建成功", content = @Content(schema = @Schema(implementation = DedupeTotalDataItemVo.class))), + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "参数不合法或数据重复") + }) + public ApiResponse create(@Valid @RequestBody DedupeTotalDataCreateRequest request) { + return ApiResponse.success("创建成功", dedupeTotalDataService.create(request)); + } + + @PostMapping("/import") + @Operation(summary = "导入总数据", description = "上传 xlsx 文件，读取 ASIN 列并批量导入数据去重总数据。") + @ApiResponses({ + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "导入成功", content = @Content(schema = @Schema(implementation = DedupeTotalDataImportVo.class))), + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "文件不合法或缺少 ASIN 列") + }) + public ApiResponse importExcel( + @Parameter(description = "xlsx 文件", required = true) @RequestParam("file") MultipartFile file) { + return ApiResponse.success("导入成功", dedupeTotalDataService.importFromExcel(file)); + } + + @PutMapping("/{id}") + @Operation(summary = "更新总数据", description = "按 ID 更新一条数据去重总数据。") + @ApiResponses({ + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "更新成功", content = @Content(schema = @Schema(implementation = DedupeTotalDataItemVo.class))), + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "400", description = "参数不合法或数据重复"), + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "数据不存在") + }) + public ApiResponse update( + @Parameter(description = "主键ID", required = true) @PathVariable Long id, + @Valid @RequestBody DedupeTotalDataUpdateRequest request) { + return ApiResponse.success("更新成功", dedupeTotalDataService.update(id, request)); + } + + @DeleteMapping("/{id}") + @Operation(summary = "删除总数据", description = "按 ID 删除一条数据去重总数据。") + @ApiResponses({ + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "删除成功"), + @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "数据不存在") + }) + public ApiResponse delete(@Parameter(description = "主键ID", required = true) @PathVariable Long id) { + dedupeTotalDataService.delete(id); + return ApiResponse.success("删除成功", null); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/mapper/DedupeTotalDataMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/mapper/DedupeTotalDataMapper.java new file mode 100644 index 0000000..bc6b89d --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/mapper/DedupeTotalDataMapper.java @@ -0,0 +1,9 @@ +package com.nanri.aiimage.modules.dedupe.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.nanri.aiimage.modules.dedupe.model.entity.DedupeTotalDataEntity; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface DedupeTotalDataMapper extends BaseMapper { +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeRunRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeRunRequest.java index 63baff1..c141275 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeRunRequest.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeRunRequest.java @@ -21,8 +21,14 @@ public class DedupeRunRequest { private List selectedColumns; @Schema(description = "是否保留纯数字 ID") - private boolean keepIntegerIds = true; + private boolean keepIntegerIds = false; @Schema(description = "是否保留带下划线 ID") private boolean keepUnderscoreIds = true; + + @Schema(description = "当文件中不存在下划线子 ID 时，是否自动保留纯数字主 ID") + private boolean keepIntegerMainIdsWhenNoSubIds = true; + + @Schema(description = "文件夹上传场景下的总压缩包名称（不含扩展名）") + private String archiveName; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeSourceFileDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeSourceFileDto.java index 48d1720..73df87e 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeSourceFileDto.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeSourceFileDto.java @@ -14,4 +14,7 @@ public class DedupeSourceFileDto { @Schema(description = "原始文件名") private String originalFilename; + + @Schema(description = "相对上传目录路径") + private String relativePath; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeTotalDataCreateRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeTotalDataCreateRequest.java new file mode 100644 index 0000000..b4eda85 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeTotalDataCreateRequest.java @@ -0,0 +1,16 @@ +package com.nanri.aiimage.modules.dedupe.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.Size; +import lombok.Data; + +@Data +@Schema(description = "数据去重总数据新增请求") +public class DedupeTotalDataCreateRequest { + + @NotBlank(message = "数据不能为空") + @Size(max = 128, message = "数据长度不能超过128个字符") + @Schema(description = "总数据值", requiredMode = Schema.RequiredMode.REQUIRED) + private String dataValue; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeTotalDataUpdateRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeTotalDataUpdateRequest.java new file mode 100644 index 0000000..380ba75 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/dto/DedupeTotalDataUpdateRequest.java @@ -0,0 +1,16 @@ +package com.nanri.aiimage.modules.dedupe.model.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.Size; +import lombok.Data; + +@Data +@Schema(description = "数据去重总数据更新请求") +public class DedupeTotalDataUpdateRequest { + + @NotBlank(message = "数据不能为空") + @Size(max = 128, message = "数据长度不能超过128个字符") + @Schema(description = "总数据值", requiredMode = Schema.RequiredMode.REQUIRED) + private String dataValue; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/entity/DedupeTotalDataEntity.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/entity/DedupeTotalDataEntity.java new file mode 100644 index 0000000..9928220 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/entity/DedupeTotalDataEntity.java @@ -0,0 +1,19 @@ +package com.nanri.aiimage.modules.dedupe.model.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +@TableName("biz_dedupe_total_data") +public class DedupeTotalDataEntity { + + @TableId(type = IdType.AUTO) + private Long id; + private String dataValue; + private LocalDateTime createdAt; + private LocalDateTime updatedAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeTotalDataImportVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeTotalDataImportVo.java new file mode 100644 index 0000000..022b78d --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeTotalDataImportVo.java @@ -0,0 +1,21 @@ +package com.nanri.aiimage.modules.dedupe.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "数据去重总数据导入结果") +public class DedupeTotalDataImportVo { + + @Schema(description = "读取总行数") + private Integer totalRows; + + @Schema(description = "读取到的 ASIN 数量") + private Integer asinCount; + + @Schema(description = "实际新增数量") + private Integer insertedCount; + + @Schema(description = "跳过数量") + private Integer skippedCount; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeTotalDataItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeTotalDataItemVo.java new file mode 100644 index 0000000..1b3fd6a --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeTotalDataItemVo.java @@ -0,0 +1,20 @@ +package com.nanri.aiimage.modules.dedupe.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +@Schema(description = "数据去重总数据项") +public class DedupeTotalDataItemVo { + + @Schema(description = "主键ID") + private Long id; + + @Schema(description = "总数据值") + private String dataValue; + + @Schema(description = "创建时间") + private LocalDateTime createdAt; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeTotalDataPageVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeTotalDataPageVo.java new file mode 100644 index 0000000..6bc2ca7 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeTotalDataPageVo.java @@ -0,0 +1,23 @@ +package com.nanri.aiimage.modules.dedupe.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.List; + +@Data +@Schema(description = "数据去重总数据分页结果") +public class DedupeTotalDataPageVo { + + @Schema(description = "列表") + private List items; + + @Schema(description = "总数") + private Long total; + + @Schema(description = "页码") + private Long page; + + @Schema(description = "每页数量") + private Long pageSize; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java index f2d8755..8319c2c 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java @@ -28,11 +28,14 @@ import org.springframework.stereotype.Service; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.nio.file.Files; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; @@ -46,7 +49,7 @@ public class DedupeRunService { private final OssStorageService ossStorageService; public DedupeRunVo run(DedupeRunRequest request) { - if (!request.isKeepIntegerIds() && !request.isKeepUnderscoreIds()) { + if (!request.isKeepIntegerIds() && !request.isKeepUnderscoreIds() && !request.isKeepIntegerMainIdsWhenNoSubIds()) { throw new BusinessException("请至少选择一种 ID 保留规则"); } @@ -63,9 +66,11 @@ public class DedupeRunService { task.setFinishedAt(LocalDateTime.now()); fileTaskMapper.insert(task); + boolean folderMode = request.getArchiveName() != null && !request.getArchiveName().isBlank(); List items = new ArrayList<>(); int successCount = 0; int failedCount = 0; + List archiveEntries = new ArrayList<>(); for (DedupeSourceFileDto sourceFile : request.getFiles()) { DedupeResultItemVo item = new DedupeResultItemVo(); @@ -81,13 +86,28 @@ public class DedupeRunService { File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "dedupe-result")); File outputFile = buildNamedOutputFile(outputDir, outputFilename); - cleanExcelByLegacyRules(inputFile, outputFile, request.getSelectedColumns(), request.isKeepIntegerIds(), request.isKeepUnderscoreIds()); + cleanExcelByLegacyRules( + inputFile, + outputFile, + request.getSelectedColumns(), + request.isKeepIntegerIds(), + request.isKeepUnderscoreIds(), + request.isKeepIntegerMainIdsWhenNoSubIds() + ); + + if (folderMode) { + archiveEntries.add(new DedupeArchiveEntry(sourceFile.getRelativePath(), inputName, outputFile)); + successCount++; + continue; + } String ossObjectKey = ossStorageService.uploadResultFile(outputFile, "DEDUPE"); String downloadUrl = ossStorageService.generateDownloadUrl(ossObjectKey); + String downloadFilename = request.getFiles().size() == 1 ? inputName : outputFile.getName(); + item.setSuccess(true); - item.setOutputFilename(outputFile.getName()); + item.setOutputFilename(downloadFilename); item.setDownloadUrl(downloadUrl); successCount++; @@ -95,7 +115,7 @@ public class DedupeRunService { resultEntity.setTaskId(task.getId()); resultEntity.setModuleType("DEDUPE"); resultEntity.setSourceFilename(inputName); - resultEntity.setResultFilename(outputFile.getName()); + resultEntity.setResultFilename(downloadFilename); resultEntity.setResultFileUrl(downloadUrl); resultEntity.setResultFileSize(outputFile.length()); resultEntity.setResultContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); @@ -119,6 +139,32 @@ public class DedupeRunService { items.add(item); } + if (folderMode && !archiveEntries.isEmpty()) { + File zipFile = packageFolderDedupeResultsAsZip(request.getArchiveName(), archiveEntries); + String ossObjectKey = ossStorageService.uploadResultFile(zipFile, "DEDUPE"); + String downloadUrl = ossStorageService.generateDownloadUrl(ossObjectKey); + + DedupeResultItemVo item = new DedupeResultItemVo(); + item.setSourceFilename(request.getArchiveName()); + item.setOutputFilename(zipFile.getName()); + item.setSuccess(true); + item.setDownloadUrl(downloadUrl); + + FileResultEntity resultEntity = new FileResultEntity(); + resultEntity.setTaskId(task.getId()); + resultEntity.setModuleType("DEDUPE"); + resultEntity.setSourceFilename(request.getArchiveName()); + resultEntity.setResultFilename(zipFile.getName()); + resultEntity.setResultFileUrl(downloadUrl); + resultEntity.setResultFileSize(zipFile.length()); + resultEntity.setResultContentType("application/zip"); + resultEntity.setSuccess(1); + resultEntity.setCreatedAt(LocalDateTime.now()); + fileResultMapper.insert(resultEntity); + item.setResultId(resultEntity.getId()); + items.add(0, item); + } + task.setSuccessFileCount(successCount); task.setFailedFileCount(failedCount); task.setResultJson(JSONUtil.toJsonStr(items)); @@ -136,17 +182,17 @@ public class DedupeRunService { public List listHistory() { return fileResultMapper.selectList(new LambdaQueryWrapper() .eq(FileResultEntity::getModuleType, "DEDUPE") - .eq(FileResultEntity::getSuccess, 1) .orderByDesc(FileResultEntity::getCreatedAt) .last("limit 50")) .stream() .map(entity -> { DedupeResultItemVo vo = new DedupeResultItemVo(); vo.setResultId(entity.getId()); - vo.setSourceFilename(entity.getCreatedAt() == null ? "" : entity.getCreatedAt().toLocalDate().toString().replace("-", "")); + vo.setSourceFilename(entity.getSourceFilename()); vo.setOutputFilename(entity.getResultFilename()); - vo.setDownloadUrl(entity.getResultFileUrl()); - vo.setSuccess(true); + vo.setDownloadUrl(entity.getSuccess() != null && entity.getSuccess() == 1 ? entity.getResultFileUrl() : null); + vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1); + vo.setError(entity.getErrorMessage()); return vo; }) .toList(); @@ -173,7 +219,8 @@ public class DedupeRunService { } private void cleanExcelByLegacyRules(File inputFile, File outputFile, List selectedColumns, - boolean keepIntegerIds, boolean keepUnderscoreIds) throws Exception { + boolean keepIntegerIds, boolean keepUnderscoreIds, + boolean keepIntegerMainIdsWhenNoSubIds) throws Exception { DataFormatter formatter = new DataFormatter(); try (FileInputStream fis = new FileInputStream(inputFile); Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis); @@ -215,6 +262,26 @@ public class DedupeRunService { } Integer idColumnIndex = headerMap.get("id"); + Map mainIdHasSubIdMap = new HashMap<>(); + if (idColumnIndex != null) { + for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) { + Row row = sheet.getRow(rowNum); + if (row == null) { + continue; + } + String idValue = normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex))); + String mainId = extractMainId(idValue); + if (mainId.isEmpty()) { + continue; + } + if (!mainIdHasSubIdMap.containsKey(mainId)) { + mainIdHasSubIdMap.put(mainId, false); + } + if (idValue.matches("\\d+_\\d+")) { + mainIdHasSubIdMap.put(mainId, true); + } + } + } int outputRowIndex = 1; for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) { Row row = sheet.getRow(rowNum); @@ -223,7 +290,7 @@ public class DedupeRunService { } if (idColumnIndex != null) { String idValue = normalizeCellText(formatter.formatCellValue(row.getCell(idColumnIndex))); - if (!shouldKeepId(idValue, keepIntegerIds, keepUnderscoreIds)) { + if (!shouldKeepId(idValue, keepIntegerIds, keepUnderscoreIds, keepIntegerMainIdsWhenNoSubIds, mainIdHasSubIdMap)) { continue; } } @@ -242,14 +309,68 @@ public class DedupeRunService { } } - private boolean shouldKeepId(String text, boolean keepIntegerIds, boolean keepUnderscoreIds) { + private File packageFolderDedupeResultsAsZip(String archiveName, List archiveEntries) { + File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "dedupe-result")); + File zipFile = buildNamedOutputFile(outputDir, archiveName + ".zip"); + try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zipFile))) { + for (DedupeArchiveEntry entry : archiveEntries) { + String zipEntryName = buildFolderZipEntry(entry.relativePath(), entry.inputName(), entry.inputName(), false); + zos.putNextEntry(new ZipEntry(zipEntryName)); + Files.copy(entry.outputFile().toPath(), zos); + zos.closeEntry(); + } + } catch (Exception ex) { + throw new BusinessException("去重结果打包失败：" + ex.getMessage()); + } + return zipFile; + } + + private String buildFolderZipEntry(String relativePath, String inputName, String outputFilename, boolean includeSourceStemFolder) { + String normalizedRelativePath = relativePath == null ? "" : relativePath.replace('\\', '/'); + String relativeDir = normalizedRelativePath.contains("/") + ? normalizedRelativePath.substring(0, normalizedRelativePath.lastIndexOf('/')) + : ""; + List parts = new ArrayList<>(); + if (!relativeDir.isBlank()) { + parts.add(relativeDir); + } + if (includeSourceStemFolder) { + parts.add(FileUtil.mainName(inputName)); + } + parts.add(outputFilename); + return String.join("/", parts); + } + + private boolean shouldKeepId(String text, boolean keepIntegerIds, boolean keepUnderscoreIds, + boolean keepIntegerMainIdsWhenNoSubIds, Map mainIdHasSubIdMap) { if (text == null || text.isBlank()) { return false; } + if (keepUnderscoreIds && text.matches("\\d+_\\d+")) { + return true; + } if (keepIntegerIds && text.matches("\\d+")) { return true; } - return keepUnderscoreIds && text.matches("\\d+_\\d+"); + if (keepIntegerMainIdsWhenNoSubIds && text.matches("\\d+")) { + String mainId = extractMainId(text); + return !mainId.isEmpty() && !Boolean.TRUE.equals(mainIdHasSubIdMap.get(mainId)); + } + return false; + } + + private String extractMainId(String text) { + if (text == null || text.isBlank()) { + return ""; + } + if (text.matches("\\d+")) { + return text; + } + if (text.matches("\\d+_\\d+")) { + int idx = text.indexOf('_'); + return idx > 0 ? text.substring(0, idx) : ""; + } + return ""; } private File findLocalSourceFile(String fileKey) { @@ -300,4 +421,7 @@ public class DedupeRunService { .trim() .replaceAll("\\s+", " "); } + + private record DedupeArchiveEntry(String relativePath, String inputName, File outputFile) { + } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java new file mode 100644 index 0000000..3629af9 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java @@ -0,0 +1,215 @@ +package com.nanri.aiimage.modules.dedupe.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.modules.dedupe.mapper.DedupeTotalDataMapper; +import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataCreateRequest; +import com.nanri.aiimage.modules.dedupe.model.dto.DedupeTotalDataUpdateRequest; +import com.nanri.aiimage.modules.dedupe.model.entity.DedupeTotalDataEntity; +import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataImportVo; +import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataItemVo; +import com.nanri.aiimage.modules.dedupe.model.vo.DedupeTotalDataPageVo; +import lombok.RequiredArgsConstructor; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.DataFormatter; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.multipart.MultipartFile; + +import java.io.InputStream; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@Service +@RequiredArgsConstructor +public class DedupeTotalDataService { + + private final DedupeTotalDataMapper dedupeTotalDataMapper; + + public DedupeTotalDataPageVo page(long page, long pageSize, String keyword) { + long safePage = Math.max(page, 1); + long safePageSize = Math.min(Math.max(pageSize, 1), 100); + String safeKeyword = keyword == null ? "" : keyword.trim(); + LambdaQueryWrapper query = new LambdaQueryWrapper() + .like(!safeKeyword.isEmpty(), DedupeTotalDataEntity::getDataValue, safeKeyword) + .orderByDesc(DedupeTotalDataEntity::getId); + Long total = dedupeTotalDataMapper.selectCount(query); + List items = dedupeTotalDataMapper.selectList(query.last("LIMIT " + ((safePage - 1) * safePageSize) + ", " + safePageSize)) + .stream() + .map(this::toItemVo) + .toList(); + DedupeTotalDataPageVo vo = new DedupeTotalDataPageVo(); + vo.setItems(items); + vo.setTotal(total); + vo.setPage(safePage); + vo.setPageSize(safePageSize); + return vo; + } + + @Transactional + public DedupeTotalDataItemVo create(DedupeTotalDataCreateRequest request) { + String dataValue = normalizeDataValue(request.getDataValue()); + ensureUnique(dataValue, null); + DedupeTotalDataEntity entity = new DedupeTotalDataEntity(); + entity.setDataValue(dataValue); + dedupeTotalDataMapper.insert(entity); + return toItemVo(getById(entity.getId())); + } + + @Transactional + public DedupeTotalDataImportVo importFromExcel(MultipartFile file) { + if (file == null || file.isEmpty()) { + throw new BusinessException("请上传 xlsx 文件"); + } + String filename = file.getOriginalFilename() == null ? "" : file.getOriginalFilename().toLowerCase(); + if (!(filename.endsWith(".xlsx") || filename.endsWith(".xls"))) { + throw new BusinessException("仅支持 .xlsx 或 .xls 文件"); + } + + DataFormatter formatter = new DataFormatter(); + try (InputStream inputStream = file.getInputStream(); + Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(inputStream)) { + Sheet sheet = workbook.getSheetAt(0); + Row headerRow = sheet.getRow(0); + if (headerRow == null) { + throw new BusinessException("Excel 表头为空"); + } + + Map headerMap = new HashMap<>(); + for (int i = 0; i < headerRow.getLastCellNum(); i++) { + Cell cell = headerRow.getCell(i); + String value = normalizeExcelText(cell == null ? null : formatter.formatCellValue(cell)); + if (!value.isBlank() && !headerMap.containsKey(value)) { + headerMap.put(value, i); + } + } + + Integer asinIndex = headerMap.get("ASIN"); + if (asinIndex == null) { + throw new BusinessException("缺少 ASIN 列"); + } + + Set seenInFile = new HashSet<>(); + int totalRows = Math.max(sheet.getLastRowNum(), 0); + int asinCount = 0; + int insertedCount = 0; + int skippedCount = 0; + + for (int rowNum = 1; rowNum <= sheet.getLastRowNum(); rowNum++) { + Row row = sheet.getRow(rowNum); + if (row == null) { + skippedCount++; + continue; + } + String asin = normalizeExcelText(formatter.formatCellValue(row.getCell(asinIndex))); + if (asin.isBlank()) { + skippedCount++; + continue; + } + asinCount++; + String dataValue = normalizeDataValue(asin); + if (!seenInFile.add(dataValue)) { + skippedCount++; + continue; + } + if (existsDataValue(dataValue)) { + skippedCount++; + continue; + } + DedupeTotalDataEntity entity = new DedupeTotalDataEntity(); + entity.setDataValue(dataValue); + dedupeTotalDataMapper.insert(entity); + insertedCount++; + } + + DedupeTotalDataImportVo vo = new DedupeTotalDataImportVo(); + vo.setTotalRows(totalRows); + vo.setAsinCount(asinCount); + vo.setInsertedCount(insertedCount); + vo.setSkippedCount(skippedCount); + return vo; + } catch (BusinessException e) { + throw e; + } catch (Exception e) { + throw new BusinessException("导入 Excel 失败"); + } + } + + @Transactional + public DedupeTotalDataItemVo update(Long id, DedupeTotalDataUpdateRequest request) { + DedupeTotalDataEntity entity = getById(id); + String dataValue = normalizeDataValue(request.getDataValue()); + ensureUnique(dataValue, id); + entity.setDataValue(dataValue); + dedupeTotalDataMapper.updateById(entity); + return toItemVo(getById(id)); + } + + @Transactional + public void delete(Long id) { + DedupeTotalDataEntity entity = getById(id); + dedupeTotalDataMapper.deleteById(entity.getId()); + } + + private DedupeTotalDataEntity getById(Long id) { + DedupeTotalDataEntity entity = dedupeTotalDataMapper.selectById(id); + if (entity == null) { + throw new BusinessException("数据不存在"); + } + return entity; + } + + private String normalizeDataValue(String value) { + String dataValue = value == null ? "" : value.trim(); + if (dataValue.isEmpty()) { + throw new BusinessException("数据不能为空"); + } + return dataValue; + } + + private void ensureUnique(String dataValue, Long excludeId) { + LambdaQueryWrapper query = new LambdaQueryWrapper() + .eq(DedupeTotalDataEntity::getDataValue, dataValue) + .last("LIMIT 1"); + DedupeTotalDataEntity exists = dedupeTotalDataMapper.selectOne(query); + if (exists != null && (excludeId == null || !exists.getId().equals(excludeId))) { + throw new BusinessException("数据已存在"); + } + } + + private boolean existsDataValue(String dataValue) { + LambdaQueryWrapper query = new LambdaQueryWrapper() + .eq(DedupeTotalDataEntity::getDataValue, dataValue) + .last("LIMIT 1"); + return dedupeTotalDataMapper.selectOne(query) != null; + } + + private String normalizeExcelText(String value) { + if (value == null) { + return ""; + } + return value.replace("\ufeff", "") + .replace("\u3000", " ") + .replace("\r\n", " ") + .replace("\r", " ") + .replace("\n", " ") + .replace("\t", " ") + .trim() + .replaceAll("\\s+", " "); + } + + private DedupeTotalDataItemVo toItemVo(DedupeTotalDataEntity entity) { + DedupeTotalDataItemVo vo = new DedupeTotalDataItemVo(); + vo.setId(entity.getId()); + vo.setDataValue(entity.getDataValue()); + vo.setCreatedAt(entity.getCreatedAt()); + return vo; + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/file/controller/FileUploadController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/file/controller/FileUploadController.java index 210e3f2..12cffdf 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/file/controller/FileUploadController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/file/controller/FileUploadController.java @@ -46,8 +46,9 @@ public class FileUploadController { }) public ApiResponse upload( @Parameter(name = "file", description = "待上传的 Excel 或业务源文件", required = true, in = ParameterIn.QUERY) - MultipartFile file) throws Exception { - return ApiResponse.success(localFileStorageService.saveTempFile(file)); + MultipartFile file, + @RequestParam(required = false) String relativePath) throws Exception { + return ApiResponse.success(localFileStorageService.saveTempFile(file, relativePath)); } @GetMapping("/excel-info") diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/file/model/vo/UploadFileVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/file/model/vo/UploadFileVo.java index de83fe0..c500ff9 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/file/model/vo/UploadFileVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/file/model/vo/UploadFileVo.java @@ -18,4 +18,7 @@ public class UploadFileVo { @Schema(description = "文件大小") private Long size; + + @Schema(description = "相对上传目录路径") + private String relativePath; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalFileStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalFileStorageService.java index ca18468..865bf52 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalFileStorageService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalFileStorageService.java @@ -29,7 +29,7 @@ public class LocalFileStorageService { private final StorageProperties storageProperties; - public UploadFileVo saveTempFile(MultipartFile file) throws IOException { + public UploadFileVo saveTempFile(MultipartFile file, String relativePath) throws IOException { File tempDir = FileUtil.file(storageProperties.getLocalTempDir()); File parentDir = tempDir.getParentFile(); if (parentDir != null) { @@ -46,6 +46,7 @@ public class LocalFileStorageService { vo.setOriginalFilename(file.getOriginalFilename()); vo.setLocalPath(target.getAbsolutePath()); vo.setSize(file.getSize()); + vo.setRelativePath(relativePath); return vo; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/split/controller/SplitRunController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/split/controller/SplitRunController.java index d46a0c2..879dd69 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/split/controller/SplitRunController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/split/controller/SplitRunController.java @@ -46,7 +46,8 @@ public class SplitRunController { - splitMode=rows_per_file 时按固定条数切分； - splitMode=parts 时按总条数尽量均分； - 输出文件命名遵循 原文件名_split_序号.xlsx； - - 返回结果中 rowCount 表示该结果文件包含的数据行数。 + - 本次拆分任务产生的所有 xlsx 会统一打成一个 zip 上传； + - 返回结果中 rowCount 表示该压缩包包含的数据总行数，entries 表示压缩包内文件摘要。 """) @ApiResponses({ @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "执行成功，返回拆分结果统计", content = @Content(schema = @Schema(implementation = SplitRunVo.class))), diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/dto/SplitRunRequest.java b/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/dto/SplitRunRequest.java index de8fefb..97dbcb9 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/dto/SplitRunRequest.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/dto/SplitRunRequest.java @@ -30,4 +30,7 @@ public class SplitRunRequest { @Schema(description = "按份数拆分时的份数") private Integer parts; + + @Schema(description = "文件夹上传场景下的总压缩包名称（不含扩展名）") + private String archiveName; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/dto/SplitSourceFileDto.java b/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/dto/SplitSourceFileDto.java index 2316ba4..f5a5b25 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/dto/SplitSourceFileDto.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/dto/SplitSourceFileDto.java @@ -14,4 +14,7 @@ public class SplitSourceFileDto { @Schema(description = "原始文件名") private String originalFilename; + + @Schema(description = "相对上传目录路径") + private String relativePath; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/vo/SplitArchiveEntryVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/vo/SplitArchiveEntryVo.java new file mode 100644 index 0000000..c2977de --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/vo/SplitArchiveEntryVo.java @@ -0,0 +1,15 @@ +package com.nanri.aiimage.modules.split.model.vo; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +@Schema(description = "拆分压缩包内文件项") +public class SplitArchiveEntryVo { + + @Schema(description = "文件名") + private String filename; + + @Schema(description = "该文件包含的数据行数") + private Integer rowCount; +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/vo/SplitResultItemVo.java b/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/vo/SplitResultItemVo.java index 95e1c3d..80dc455 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/vo/SplitResultItemVo.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/split/model/vo/SplitResultItemVo.java @@ -3,6 +3,8 @@ package com.nanri.aiimage.modules.split.model.vo; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; +import java.util.List; + @Data @Schema(description = "数据拆分结果项") public class SplitResultItemVo { @@ -22,9 +24,15 @@ public class SplitResultItemVo { @Schema(description = "错误信息") private String error; - @Schema(description = "结果行数") + @Schema(description = "结果总行数") private Integer rowCount; @Schema(description = "下载地址") private String downloadUrl; + + @Schema(description = "压缩包内文件数量") + private Integer entryCount; + + @Schema(description = "压缩包内文件列表") + private List entries; } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/split/service/SplitRunService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/split/service/SplitRunService.java index 555b546..53015a0 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/split/service/SplitRunService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/split/service/SplitRunService.java @@ -3,13 +3,15 @@ package com.nanri.aiimage.modules.split.service; import cn.hutool.core.io.FileUtil; import cn.hutool.core.util.IdUtil; import cn.hutool.json.JSONUtil; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.config.StorageProperties; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; import com.nanri.aiimage.modules.split.model.dto.SplitRunRequest; import com.nanri.aiimage.modules.split.model.dto.SplitSourceFileDto; +import com.nanri.aiimage.modules.split.model.vo.SplitArchiveEntryVo; import com.nanri.aiimage.modules.split.model.vo.SplitResultItemVo; import com.nanri.aiimage.modules.split.model.vo.SplitRunVo; -import com.nanri.aiimage.modules.file.service.oss.OssStorageService; import com.nanri.aiimage.modules.task.mapper.FileResultMapper; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; @@ -20,6 +22,7 @@ import org.apache.poi.ss.usermodel.DataFormatter; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.ss.usermodel.WorkbookFactory; import org.apache.poi.xssf.usermodel.XSSFWorkbook; import org.springframework.core.io.FileSystemResource; import org.springframework.core.io.Resource; @@ -28,18 +31,28 @@ import org.springframework.stereotype.Service; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.nio.file.Files; import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; - -import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; @Service @RequiredArgsConstructor public class SplitRunService { + private static final String MODULE_TYPE = "SPLIT"; + private static final String ZIP_CONTENT_TYPE = "application/zip"; + private static final String SPLIT_MODE_ROWS_PER_FILE = "rows_per_file"; + private static final String SPLIT_MODE_PARTS = "parts"; + private static final String HEADER_SUFFIX_MARKER = "idASIN\u56fd\u5bb6\u72b6\u6001\u4ef7\u683c\u53d8\u4f53\u6570\u91cf"; + private static final String HEADER_STOP_COLUMN = "\u7f29\u7565\u56fe\u5730\u57408"; + private static final DateTimeFormatter ZIP_NAME_DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd"); + private final FileTaskMapper fileTaskMapper; private final FileResultMapper fileResultMapper; private final StorageProperties storageProperties; @@ -49,8 +62,8 @@ public class SplitRunService { validateRequest(request); FileTaskEntity task = new FileTaskEntity(); - task.setTaskNo("SPLIT-" + IdUtil.getSnowflakeNextIdStr()); - task.setModuleType("SPLIT"); + task.setTaskNo(MODULE_TYPE + "-" + IdUtil.getSnowflakeNextIdStr()); + task.setModuleType(MODULE_TYPE); task.setTaskMode("IMMEDIATE"); task.setStatus("SUCCESS"); task.setSourceFileCount(request.getFiles().size()); @@ -61,56 +74,42 @@ public class SplitRunService { task.setFinishedAt(LocalDateTime.now()); fileTaskMapper.insert(task); + boolean folderMode = request.getArchiveName() != null && !request.getArchiveName().isBlank(); List items = new ArrayList<>(); int successCount = 0; int failedCount = 0; + List allChunkResults = new ArrayList<>(); + List successSourceNames = new ArrayList<>(); for (SplitSourceFileDto sourceFile : request.getFiles()) { try { File inputFile = findLocalSourceFile(sourceFile.getFileKey()); if (inputFile == null || !inputFile.exists()) { - throw new BusinessException("上传文件不存在，请重新上传"); + throw new BusinessException("Uploaded source file does not exist, please upload again."); } - String inputName = sourceFile.getOriginalFilename() == null ? inputFile.getName() : sourceFile.getOriginalFilename(); + + String inputName = sourceFile.getOriginalFilename() == null + ? inputFile.getName() + : sourceFile.getOriginalFilename(); List chunkResults = splitExcelByLegacyRules(inputFile, inputName, request); for (SplitChunkResult chunkResult : chunkResults) { - SplitResultItemVo item = new SplitResultItemVo(); - item.setSourceFilename(inputName); - item.setOutputFilename(chunkResult.outputFile().getName()); - item.setSuccess(true); - item.setRowCount(chunkResult.rowCount()); - - String ossObjectKey = ossStorageService.uploadResultFile(chunkResult.outputFile(), "SPLIT"); - String downloadUrl = ossStorageService.generateDownloadUrl(ossObjectKey); - - FileResultEntity resultEntity = new FileResultEntity(); - resultEntity.setTaskId(task.getId()); - resultEntity.setModuleType("SPLIT"); - resultEntity.setSourceFilename(inputName); - resultEntity.setResultFilename(chunkResult.outputFile().getName()); - resultEntity.setResultFileUrl(downloadUrl); - resultEntity.setResultFileSize(chunkResult.outputFile().length()); - resultEntity.setResultContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); - resultEntity.setRowCount(chunkResult.rowCount()); - resultEntity.setSuccess(1); - resultEntity.setCreatedAt(LocalDateTime.now()); - fileResultMapper.insert(resultEntity); - item.setDownloadUrl(downloadUrl); - item.setResultId(resultEntity.getId()); - items.add(item); + allChunkResults.add(new TaskSplitChunkResult(sourceFile.getRelativePath(), inputName, chunkResult.outputFile(), chunkResult.rowCount())); } + successSourceNames.add(inputName); successCount++; } catch (Exception ex) { SplitResultItemVo item = new SplitResultItemVo(); item.setSourceFilename(sourceFile.getOriginalFilename()); item.setSuccess(false); item.setError(ex.getMessage()); + item.setEntries(new ArrayList<>()); + item.setEntryCount(0); items.add(item); failedCount++; FileResultEntity resultEntity = new FileResultEntity(); resultEntity.setTaskId(task.getId()); - resultEntity.setModuleType("SPLIT"); + resultEntity.setModuleType(MODULE_TYPE); resultEntity.setSourceFilename(sourceFile.getOriginalFilename()); resultEntity.setSuccess(0); resultEntity.setErrorMessage(ex.getMessage()); @@ -119,6 +118,38 @@ public class SplitRunService { } } + if (!allChunkResults.isEmpty()) { + File zipFile = packageTaskSplitResultsAsZip(request.getArchiveName(), successSourceNames, allChunkResults); + String ossObjectKey = ossStorageService.uploadResultFile(zipFile, MODULE_TYPE); + String downloadUrl = ossStorageService.generateDownloadUrl(ossObjectKey); + + SplitResultItemVo item = new SplitResultItemVo(); + item.setSourceFilename(folderMode && request.getArchiveName() != null && !request.getArchiveName().isBlank() + ? request.getArchiveName() + : successSourceNames.size() == 1 ? successSourceNames.get(0) : "Current split task"); + item.setOutputFilename(zipFile.getName()); + item.setSuccess(true); + item.setRowCount(allChunkResults.stream().mapToInt(TaskSplitChunkResult::rowCount).sum()); + item.setDownloadUrl(downloadUrl); + item.setEntryCount(allChunkResults.size()); + item.setEntries(allChunkResults.stream().map(this::toArchiveEntry).toList()); + + FileResultEntity resultEntity = new FileResultEntity(); + resultEntity.setTaskId(task.getId()); + resultEntity.setModuleType(MODULE_TYPE); + resultEntity.setSourceFilename(item.getSourceFilename()); + resultEntity.setResultFilename(zipFile.getName()); + resultEntity.setResultFileUrl(downloadUrl); + resultEntity.setResultFileSize(zipFile.length()); + resultEntity.setResultContentType(ZIP_CONTENT_TYPE); + resultEntity.setRowCount(item.getRowCount()); + resultEntity.setSuccess(1); + resultEntity.setCreatedAt(LocalDateTime.now()); + fileResultMapper.insert(resultEntity); + item.setResultId(resultEntity.getId()); + items.add(0, item); + } + task.setSuccessFileCount(successCount); task.setFailedFileCount(failedCount); task.setResultJson(JSONUtil.toJsonStr(items)); @@ -135,19 +166,21 @@ public class SplitRunService { public List listHistory() { return fileResultMapper.selectList(new LambdaQueryWrapper() - .eq(FileResultEntity::getModuleType, "SPLIT") - .eq(FileResultEntity::getSuccess, 1) + .eq(FileResultEntity::getModuleType, MODULE_TYPE) .orderByDesc(FileResultEntity::getCreatedAt) .last("limit 100")) .stream() .map(entity -> { SplitResultItemVo vo = new SplitResultItemVo(); vo.setResultId(entity.getId()); - vo.setSourceFilename(entity.getCreatedAt() == null ? "" : entity.getCreatedAt().toLocalDate().toString().replace("-", "")); + vo.setSourceFilename(entity.getSourceFilename()); vo.setOutputFilename(entity.getResultFilename()); - vo.setDownloadUrl(entity.getResultFileUrl()); + vo.setDownloadUrl(entity.getSuccess() != null && entity.getSuccess() == 1 ? entity.getResultFileUrl() : null); vo.setRowCount(entity.getRowCount()); - vo.setSuccess(true); + vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1); + vo.setError(entity.getErrorMessage()); + vo.setEntryCount(0); + vo.setEntries(new ArrayList<>()); return vo; }) .toList(); @@ -155,55 +188,45 @@ public class SplitRunService { public void deleteHistory(Long resultId) { FileResultEntity entity = fileResultMapper.selectById(resultId); - if (entity == null || !"SPLIT".equals(entity.getModuleType())) { - throw new BusinessException("记录不存在"); + if (entity == null || !MODULE_TYPE.equals(entity.getModuleType())) { + throw new BusinessException("Split result record does not exist."); } fileResultMapper.deleteById(resultId); } public Resource getResultFile(Long resultId) { FileResultEntity entity = fileResultMapper.selectById(resultId); - if (entity == null || entity.getResultFileUrl() == null || !"SPLIT".equals(entity.getModuleType())) { - throw new BusinessException("结果文件不存在"); + if (entity == null || entity.getResultFileUrl() == null || !MODULE_TYPE.equals(entity.getModuleType())) { + throw new BusinessException("Split result file does not exist."); } + File file = new File(entity.getResultFileUrl()); if (!file.exists()) { - throw new BusinessException("结果文件不存在"); + throw new BusinessException("Split result file does not exist."); } return new FileSystemResource(file); } - private List splitExcelByLegacyRules(File inputFile, String originalFilename, SplitRunRequest request) throws Exception { + private List splitExcelByLegacyRules( + File inputFile, + String originalFilename, + SplitRunRequest request + ) throws Exception { DataFormatter formatter = new DataFormatter(); - try (FileInputStream fis = new FileInputStream(inputFile); Workbook workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(fis)) { + try (FileInputStream fis = new FileInputStream(inputFile); + Workbook workbook = WorkbookFactory.create(fis)) { Sheet sheet = workbook.getSheetAt(0); Row headerRow = sheet.getRow(0); if (headerRow == null) { - throw new BusinessException("Excel 表头为空"); + throw new BusinessException("Excel header row is empty."); } - Map headerMap = new HashMap<>(); - for (int i = 0; i < headerRow.getLastCellNum(); i++) { - Cell cell = headerRow.getCell(i); - String value = normalizeCellText(cell == null ? null : formatter.formatCellValue(cell)); - if (value.isBlank()) { - continue; - } - value = value.split("idASIN国家状态价格变体数量", 2)[0].trim().isEmpty() - ? value - : value.split("idASIN国家状态价格变体数量", 2)[0].trim(); - if (headerMap.containsKey(value)) { - continue; - } - headerMap.put(value, i); - if ("缩略图地址8".equals(value)) { - break; - } - } - - List missing = request.getSelectedColumns().stream().filter(column -> !headerMap.containsKey(column)).toList(); + Map headerMap = buildHeaderMap(headerRow, formatter); + List missing = request.getSelectedColumns().stream() + .filter(column -> !headerMap.containsKey(column)) + .toList(); if (!missing.isEmpty()) { - throw new BusinessException("缺少列：" + String.join("、", missing)); + throw new BusinessException("Missing required columns: " + String.join(", ", missing)); } List> rowsData = new ArrayList<>(); @@ -215,39 +238,23 @@ public class SplitRunService { List projected = new ArrayList<>(); for (String column : request.getSelectedColumns()) { Integer sourceIndex = headerMap.get(column); - String value = sourceIndex == null ? "" : normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex))); + String value = sourceIndex == null + ? "" + : normalizeCellText(formatter.formatCellValue(row.getCell(sourceIndex))); projected.add(value); } rowsData.add(projected); } - int totalRows = rowsData.size(); - if (totalRows == 0) { - throw new BusinessException("没有可拆分的数据行"); - } - - List>> chunks = new ArrayList<>(); - if ("rows_per_file".equals(request.getSplitMode())) { - int chunkSize = request.getRowsPerFile(); - for (int i = 0; i < totalRows; i += chunkSize) { - chunks.add(rowsData.subList(i, Math.min(i + chunkSize, totalRows))); - } - } else { - int actualParts = Math.min(request.getParts(), totalRows); - int base = totalRows / actualParts; - int remainder = totalRows % actualParts; - int start = 0; - for (int i = 0; i < actualParts; i++) { - int size = base + (i < remainder ? 1 : 0); - int end = start + size; - chunks.add(rowsData.subList(start, end)); - start = end; - } + if (rowsData.isEmpty()) { + throw new BusinessException("No data rows available for splitting."); } + List>> chunks = buildChunks(rowsData, request); File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "split-result")); String sourceStem = FileUtil.mainName(originalFilename); List results = new ArrayList<>(); + for (int i = 0; i < chunks.size(); i++) { File outputFile = buildNamedOutputFile(outputDir, sourceStem + "_split_" + (i + 1) + ".xlsx"); try (XSSFWorkbook outputWorkbook = new XSSFWorkbook()) { @@ -269,19 +276,63 @@ public class SplitRunService { } results.add(new SplitChunkResult(outputFile, chunks.get(i).size())); } + return results; } } + private Map buildHeaderMap(Row headerRow, DataFormatter formatter) { + Map headerMap = new HashMap<>(); + for (int i = 0; i < headerRow.getLastCellNum(); i++) { + Cell cell = headerRow.getCell(i); + String value = normalizeHeaderName(cell == null ? null : formatter.formatCellValue(cell)); + if (value.isBlank() || headerMap.containsKey(value)) { + continue; + } + headerMap.put(value, i); + if (HEADER_STOP_COLUMN.equals(value)) { + break; + } + } + return headerMap; + } + + private List>> buildChunks(List> rowsData, SplitRunRequest request) { + int totalRows = rowsData.size(); + List>> chunks = new ArrayList<>(); + + if (SPLIT_MODE_ROWS_PER_FILE.equals(request.getSplitMode())) { + int chunkSize = request.getRowsPerFile(); + for (int i = 0; i < totalRows; i += chunkSize) { + chunks.add(rowsData.subList(i, Math.min(i + chunkSize, totalRows))); + } + return chunks; + } + + int actualParts = Math.min(request.getParts(), totalRows); + int base = totalRows / actualParts; + int remainder = totalRows % actualParts; + int start = 0; + for (int i = 0; i < actualParts; i++) { + int size = base + (i < remainder ? 1 : 0); + int end = start + size; + chunks.add(rowsData.subList(start, end)); + start = end; + } + return chunks; + } + private void validateRequest(SplitRunRequest request) { - if (!List.of("rows_per_file", "parts").contains(request.getSplitMode())) { - throw new BusinessException("拆分模式不合法"); + if (!List.of(SPLIT_MODE_ROWS_PER_FILE, SPLIT_MODE_PARTS).contains(request.getSplitMode())) { + throw new BusinessException("Invalid split mode."); } - if ("rows_per_file".equals(request.getSplitMode()) && (request.getRowsPerFile() == null || request.getRowsPerFile() <= 0)) { - throw new BusinessException("每份条数不合法"); + if (SPLIT_MODE_ROWS_PER_FILE.equals(request.getSplitMode()) + && (request.getRowsPerFile() == null || request.getRowsPerFile() <= 0)) { + throw new BusinessException("Invalid rowsPerFile value."); } - if ("parts".equals(request.getSplitMode()) && (request.getParts() == null || request.getParts() <= 0)) { - throw new BusinessException("拆分份数不合法"); + if (SPLIT_MODE_PARTS.equals(request.getSplitMode()) + && (request.getParts() == null || request.getParts() <= 0)) { + throw new BusinessException("Invalid parts value."); } } @@ -290,8 +341,11 @@ public class SplitRunService { if (!baseDir.exists()) { return null; } - List matchedFiles = FileUtil.loopFiles(baseDir, pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey)); - return matchedFiles.isEmpty() ? null : matchedFiles.getFirst(); + List matchedFiles = FileUtil.loopFiles( + baseDir, + pathname -> pathname.isFile() && pathname.getName().startsWith(fileKey) + ); + return matchedFiles.isEmpty() ? null : matchedFiles.get(0); } private File buildNamedOutputFile(File outputDir, String filename) { @@ -299,6 +353,7 @@ public class SplitRunService { if (!candidate.exists()) { return candidate; } + String mainName = FileUtil.mainName(filename); String extName = FileUtil.extName(filename); int index = 2; @@ -314,6 +369,61 @@ public class SplitRunService { } } + private File packageTaskSplitResultsAsZip(String archiveName, List successSourceNames, List chunkResults) { + File outputDir = FileUtil.mkdir(FileUtil.file(storageProperties.getLocalTempDir(), "split-result")); + String zipFilename = archiveName != null && !archiveName.isBlank() + ? archiveName + ".zip" + : successSourceNames.size() == 1 + ? FileUtil.mainName(successSourceNames.getFirst()) + ".zip" + : LocalDateTime.now().format(ZIP_NAME_DATE_FORMATTER) + ".zip"; + File zipFile = buildNamedOutputFile(outputDir, zipFilename); + try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zipFile))) { + for (TaskSplitChunkResult chunkResult : chunkResults) { + String zipEntryName = buildFolderZipEntry(chunkResult.relativePath(), chunkResult.sourceFilename(), chunkResult.outputFile().getName(), true); + ZipEntry entry = new ZipEntry(zipEntryName); + zos.putNextEntry(entry); + Files.copy(chunkResult.outputFile().toPath(), zos); + zos.closeEntry(); + } + } catch (Exception ex) { + throw new BusinessException("拆分结果打包失败：" + ex.getMessage()); + } + return zipFile; + } + + private String buildFolderZipEntry(String relativePath, String inputName, String outputFilename, boolean includeSourceStemFolder) { + String normalizedRelativePath = relativePath == null ? "" : relativePath.replace('\\', '/'); + String relativeDir = normalizedRelativePath.contains("/") + ? normalizedRelativePath.substring(0, normalizedRelativePath.lastIndexOf('/')) + : ""; + List parts = new ArrayList<>(); + if (!relativeDir.isBlank()) { + parts.add(relativeDir); + } + if (includeSourceStemFolder) { + parts.add(FileUtil.mainName(inputName)); + } + parts.add(outputFilename); + return String.join("/", parts); + } + + private SplitArchiveEntryVo toArchiveEntry(TaskSplitChunkResult chunkResult) { + SplitArchiveEntryVo vo = new SplitArchiveEntryVo(); + vo.setFilename(chunkResult.outputFile().getName()); + vo.setRowCount(chunkResult.rowCount()); + return vo; + } + + private String normalizeHeaderName(String value) { + String normalized = normalizeCellText(value); + if (normalized.isBlank()) { + return normalized; + } + + String prefix = normalized.split(HEADER_SUFFIX_MARKER, 2)[0].trim(); + return prefix.isEmpty() ? normalized : prefix; + } + private String normalizeCellText(String value) { if (value == null) { return ""; @@ -330,4 +440,7 @@ public class SplitRunService { private record SplitChunkResult(File outputFile, int rowCount) { } + + private record TaskSplitChunkResult(String relativePath, String sourceFilename, File outputFile, int rowCount) { + } } diff --git a/backend-java/src/main/resources/db/V3__create_dedupe_total_data.sql b/backend-java/src/main/resources/db/V3__create_dedupe_total_data.sql new file mode 100644 index 0000000..0f9ca2d --- /dev/null +++ b/backend-java/src/main/resources/db/V3__create_dedupe_total_data.sql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS biz_dedupe_total_data ( + id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT '主键', + data_value VARCHAR(128) NOT NULL COMMENT '去重总数据值', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + UNIQUE KEY uk_data_value (data_value), + KEY idx_created_at (created_at) +) COMMENT='数据去重总数据表'; diff --git a/backend-java/src/main/resources/db/V4__rename_convert_template_to_five_countries.sql b/backend-java/src/main/resources/db/V4__rename_convert_template_to_five_countries.sql new file mode 100644 index 0000000..8b923aa --- /dev/null +++ b/backend-java/src/main/resources/db/V4__rename_convert_template_to_five_countries.sql @@ -0,0 +1,5 @@ +UPDATE biz_convert_template +SET template_name = '五国模板', + output_filename = '英国.txt、法国.txt、德国.txt、西班牙.txt、意大利.txt', + updated_at = CURRENT_TIMESTAMP +WHERE template_code = 'uk_offer'; diff --git a/backend/.device_id b/backend/.device_id new file mode 100644 index 0000000..6e07d8f --- /dev/null +++ b/backend/.device_id @@ -0,0 +1 @@ +8cecee3bc02a178bf372ca1c3d02fc5cae3c0a51c8346f6e6a710988599c08be \ No newline at end of file diff --git a/backend/__pycache__/ali_oss.cpython-311.pyc b/backend/__pycache__/ali_oss.cpython-311.pyc new file mode 100644 index 0000000..26cc96a Binary files /dev/null and b/backend/__pycache__/ali_oss.cpython-311.pyc differ diff --git a/backend/__pycache__/ali_oss.cpython-312.pyc b/backend/__pycache__/ali_oss.cpython-312.pyc new file mode 100644 index 0000000..1053f3e Binary files /dev/null and b/backend/__pycache__/ali_oss.cpython-312.pyc differ diff --git a/backend/__pycache__/ali_oss.cpython-39.pyc b/backend/__pycache__/ali_oss.cpython-39.pyc new file mode 100644 index 0000000..573a87f Binary files /dev/null and b/backend/__pycache__/ali_oss.cpython-39.pyc differ diff --git a/backend/__pycache__/app.cpython-311.pyc b/backend/__pycache__/app.cpython-311.pyc new file mode 100644 index 0000000..d267e73 Binary files /dev/null and b/backend/__pycache__/app.cpython-311.pyc differ diff --git a/backend/__pycache__/config.cpython-311.pyc b/backend/__pycache__/config.cpython-311.pyc new file mode 100644 index 0000000..e65d9f5 Binary files /dev/null and b/backend/__pycache__/config.cpython-311.pyc differ diff --git a/backend/__pycache__/config.cpython-312.pyc b/backend/__pycache__/config.cpython-312.pyc new file mode 100644 index 0000000..4fefbc2 Binary files /dev/null and b/backend/__pycache__/config.cpython-312.pyc differ diff --git a/backend/__pycache__/config.cpython-39.pyc b/backend/__pycache__/config.cpython-39.pyc new file mode 100644 index 0000000..81cae7a Binary files /dev/null and b/backend/__pycache__/config.cpython-39.pyc differ diff --git a/backend/__pycache__/db.cpython-311.pyc b/backend/__pycache__/db.cpython-311.pyc new file mode 100644 index 0000000..9216b0d Binary files /dev/null and b/backend/__pycache__/db.cpython-311.pyc differ diff --git a/backend/ali_oss.py b/backend/ali_oss.py new file mode 100644 index 0000000..b31ee99 --- /dev/null +++ b/backend/ali_oss.py @@ -0,0 +1,36 @@ +import argparse +import base64 +import re +import time +import alibabacloud_oss_v2 as oss +import requests + +from config import region, endpoint, bucket, file_url_pre, bucket_path + + +def upload_file(file_content: bytes, key: str): + credentials_provider = oss.credentials.EnvironmentVariableCredentialsProvider() + cfg = oss.config.load_default() + cfg.credentials_provider = credentials_provider + cfg.region = region + cfg.endpoint = endpoint + client = oss.Client(cfg) + + result = client.put_object( + oss.PutObjectRequest( + bucket=bucket, # 存储空间名称 + key=key, # 对象名称 + body=file_content # 读取文件内容 + ) + ) + # print(result) + return file_url_pre + key + + + + +# 脚本入口，当文件被直接运行时调用main函数 +if __name__ == "__main__": + with open("测试图片数据/IMG_2685.JPG", "rb") as f: + file_content = f.read() + upload_file(file_content,key=bucket_path+"test.png") \ No newline at end of file diff --git a/backend/app.py b/backend/app.py new file mode 100644 index 0000000..201ace9 --- /dev/null +++ b/backend/app.py @@ -0,0 +1,38 @@ +""" +卖相AI - Flask 后端 +按功能拆分为蓝图：认证(auth)、主页面(main)、管理员API(admin_api)、版本(version) +""" +import os +import secrets +from datetime import timedelta + +from flask import Flask +from flask_cors import CORS + +from utils.db import init_db +from blueprints.auth import auth +from blueprints.main import main +from blueprints.admin_api import admin_api +from blueprints.version import version_bp + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +app = Flask(__name__, template_folder=BASE_DIR, static_folder=BASE_DIR) + +CORS(app) +app.secret_key = os.environ.get('SECRET_KEY', secrets.token_hex(32)) +app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(days=7) + +# 注册蓝图 +app.register_blueprint(auth) +app.register_blueprint(main) +app.register_blueprint(admin_api) +app.register_blueprint(version_bp) + + +def run_app(host='0.0.0.0', port=15124): + init_db() + app.run(host=host, port=port, threaded=True, use_reloader=False) + + +if __name__ == '__main__': + run_app() diff --git a/backend/blueprints/__init__.py b/backend/blueprints/__init__.py new file mode 100644 index 0000000..836aa1e --- /dev/null +++ b/backend/blueprints/__init__.py @@ -0,0 +1 @@ +# Blueprints 包 diff --git a/backend/blueprints/__pycache__/__init__.cpython-311.pyc b/backend/blueprints/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000..936e378 Binary files /dev/null and b/backend/blueprints/__pycache__/__init__.cpython-311.pyc differ diff --git a/backend/blueprints/__pycache__/__init__.cpython-312.pyc b/backend/blueprints/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..0bb197f Binary files /dev/null and b/backend/blueprints/__pycache__/__init__.cpython-312.pyc differ diff --git a/backend/blueprints/__pycache__/__init__.cpython-39.pyc b/backend/blueprints/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..b7806be Binary files /dev/null and b/backend/blueprints/__pycache__/__init__.cpython-39.pyc differ diff --git a/backend/blueprints/__pycache__/admin_api.cpython-311.pyc b/backend/blueprints/__pycache__/admin_api.cpython-311.pyc new file mode 100644 index 0000000..454cf8d Binary files /dev/null and b/backend/blueprints/__pycache__/admin_api.cpython-311.pyc differ diff --git a/backend/blueprints/__pycache__/admin_api.cpython-312.pyc b/backend/blueprints/__pycache__/admin_api.cpython-312.pyc new file mode 100644 index 0000000..1a75368 Binary files /dev/null and b/backend/blueprints/__pycache__/admin_api.cpython-312.pyc differ diff --git a/backend/blueprints/__pycache__/admin_api.cpython-39.pyc b/backend/blueprints/__pycache__/admin_api.cpython-39.pyc new file mode 100644 index 0000000..4a2a7bb Binary files /dev/null and b/backend/blueprints/__pycache__/admin_api.cpython-39.pyc differ diff --git a/backend/blueprints/__pycache__/admin_history.cpython-311.pyc b/backend/blueprints/__pycache__/admin_history.cpython-311.pyc new file mode 100644 index 0000000..8fdc8bd Binary files /dev/null and b/backend/blueprints/__pycache__/admin_history.cpython-311.pyc differ diff --git a/backend/blueprints/__pycache__/admin_users.cpython-311.pyc b/backend/blueprints/__pycache__/admin_users.cpython-311.pyc new file mode 100644 index 0000000..5c88c41 Binary files /dev/null and b/backend/blueprints/__pycache__/admin_users.cpython-311.pyc differ diff --git a/backend/blueprints/__pycache__/admin_versions.cpython-311.pyc b/backend/blueprints/__pycache__/admin_versions.cpython-311.pyc new file mode 100644 index 0000000..309a20f Binary files /dev/null and b/backend/blueprints/__pycache__/admin_versions.cpython-311.pyc differ diff --git a/backend/blueprints/__pycache__/auth.cpython-311.pyc b/backend/blueprints/__pycache__/auth.cpython-311.pyc new file mode 100644 index 0000000..f5f8202 Binary files /dev/null and b/backend/blueprints/__pycache__/auth.cpython-311.pyc differ diff --git a/backend/blueprints/__pycache__/auth.cpython-312.pyc b/backend/blueprints/__pycache__/auth.cpython-312.pyc new file mode 100644 index 0000000..cd1faab Binary files /dev/null and b/backend/blueprints/__pycache__/auth.cpython-312.pyc differ diff --git a/backend/blueprints/__pycache__/auth.cpython-39.pyc b/backend/blueprints/__pycache__/auth.cpython-39.pyc new file mode 100644 index 0000000..3f8fdc1 Binary files /dev/null and b/backend/blueprints/__pycache__/auth.cpython-39.pyc differ diff --git a/backend/blueprints/__pycache__/main.cpython-311.pyc b/backend/blueprints/__pycache__/main.cpython-311.pyc new file mode 100644 index 0000000..bd52ed9 Binary files /dev/null and b/backend/blueprints/__pycache__/main.cpython-311.pyc differ diff --git a/backend/blueprints/__pycache__/main.cpython-312.pyc b/backend/blueprints/__pycache__/main.cpython-312.pyc new file mode 100644 index 0000000..bc4d2ca Binary files /dev/null and b/backend/blueprints/__pycache__/main.cpython-312.pyc differ diff --git a/backend/blueprints/__pycache__/main.cpython-39.pyc b/backend/blueprints/__pycache__/main.cpython-39.pyc new file mode 100644 index 0000000..a407705 Binary files /dev/null and b/backend/blueprints/__pycache__/main.cpython-39.pyc differ diff --git a/backend/blueprints/__pycache__/version.cpython-312.pyc b/backend/blueprints/__pycache__/version.cpython-312.pyc new file mode 100644 index 0000000..e7f2ea8 Binary files /dev/null and b/backend/blueprints/__pycache__/version.cpython-312.pyc differ diff --git a/backend/blueprints/__pycache__/version.cpython-39.pyc b/backend/blueprints/__pycache__/version.cpython-39.pyc new file mode 100644 index 0000000..6da875c Binary files /dev/null and b/backend/blueprints/__pycache__/version.cpython-39.pyc differ diff --git a/backend/blueprints/admin_api.py b/backend/blueprints/admin_api.py new file mode 100644 index 0000000..51db218 --- /dev/null +++ b/backend/blueprints/admin_api.py @@ -0,0 +1,723 @@ +""" +管理员 API 蓝图：用户管理、生成历史、版本管理（后台） +""" +import json +import os +import re + +import requests +from flask import Blueprint, request, jsonify, session + +import pymysql +from werkzeug.security import generate_password_hash + +from utils.db import get_db +from utils.auth import admin_required, get_current_admin_role +from ali_oss import upload_file as oss_upload_file + +try: + from config import bucket_path, backend_java_base_url +except ImportError: + bucket_path = os.environ.get('BUCKET_PATH', 'nanri-image/') + backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://127.0.0.1:18080').rstrip('/') + +admin_api = Blueprint('admin_api', __name__, url_prefix='/api/admin') + + +def _safe_version_key(version): + """将版本号转为安全的 OSS 对象名部分""" + s = (version or '').strip() + s = re.sub(r'[^\w.\-]', '_', s) + return s or 'unknown' + + +def _proxy_backend_java(method, path, *, params=None, json_data=None, files=None, data=None): + url = f"{backend_java_base_url}{path}" + try: + resp = requests.request(method=method, url=url, params=params, json=json_data, files=files, data=data, timeout=10) + except requests.RequestException: + return None, jsonify({'success': False, 'error': 'backend-java 服务不可用'}), 502 + try: + data = resp.json() + except ValueError: + data = None + if resp.status_code >= 400: + if isinstance(data, dict): + return data, jsonify({'success': False, 'error': data.get('message') or data.get('error') or 'backend-java 请求失败'}), resp.status_code + return None, jsonify({'success': False, 'error': 'backend-java 请求失败'}), resp.status_code + if not isinstance(data, dict): + return None, jsonify({'success': False, 'error': 'backend-java 返回格式错误'}), 502 + if not data.get('success'): + return data, jsonify({'success': False, 'error': data.get('message') or '操作失败'}), 200 + return data, None, 200 + + +# ---------- 用户管理 ---------- + +@admin_api.route('/users') +@admin_required +def list_users(): + """分页获取用户列表；支持用户名模糊搜索、指定管理员所属普通用户筛选""" + role, current_row = get_current_admin_role() + if not role: + return jsonify({'success': False, 'error': '需要管理员权限'}), 403 + page = max(1, int(request.args.get('page', 1))) + page_size = min(50, max(5, int(request.args.get('page_size', 15)))) + offset = (page - 1) * page_size + search_username = (request.args.get('username') or request.args.get('search') or '').strip() + created_by_id_arg = request.args.get('created_by_id') or request.args.get('admin_id') + created_by_id = int(created_by_id_arg) if created_by_id_arg and str(created_by_id_arg).isdigit() else None + if role != 'super_admin': + created_by_id = None + try: + conn = get_db() + with conn.cursor() as cur: + if role == 'super_admin': + where_parts = ["1=1"] + params = [] + if search_username: + where_parts.append("u.username LIKE %s") + params.append("%" + search_username + "%") + if created_by_id is not None: + where_parts.append("u.created_by_id = %s") + params.append(created_by_id) + where_sql = " AND ".join(where_parts) + cur.execute( + """SELECT u.id, u.username, u.is_admin, u.role, u.created_at, u.created_by_id, + creator.username AS creator_username + FROM users u + LEFT JOIN users creator ON creator.id = u.created_by_id + WHERE """ + where_sql + """ ORDER BY u.id LIMIT %s OFFSET %s""", + tuple(params) + (page_size, offset), + ) + rows = cur.fetchall() + cur.execute("SELECT COUNT(*) as total FROM users u WHERE " + where_sql, tuple(params)) + total = cur.fetchone()['total'] + cur.execute("SELECT id, username FROM users WHERE role = 'admin' ORDER BY id") + admins = [{'id': r['id'], 'username': r['username']} for r in cur.fetchall()] + else: + admin_id = current_row['id'] + where_parts = ["(u.id = %s OR (u.role = 'normal' AND u.created_by_id = %s))"] + params = [admin_id, admin_id] + if search_username: + where_parts.append("u.username LIKE %s") + params.append("%" + search_username + "%") + where_sql = " AND ".join(where_parts) + cur.execute( + """SELECT u.id, u.username, u.is_admin, u.role, u.created_at, u.created_by_id, + creator.username AS creator_username + FROM users u + LEFT JOIN users creator ON creator.id = u.created_by_id + WHERE """ + where_sql + """ ORDER BY u.id LIMIT %s OFFSET %s""", + tuple(params) + (page_size, offset), + ) + rows = cur.fetchall() + cur.execute( + "SELECT COUNT(*) as total FROM users u WHERE " + where_sql, + tuple(params), + ) + total = cur.fetchone()['total'] + admins = [] + items = [ + { + 'id': r['id'], + 'username': r['username'], + 'is_admin': bool(r.get('is_admin')), + 'role': r.get('role') or 'normal', + 'created_by_id': r.get('created_by_id'), + 'creator_username': r.get('creator_username') or '', + 'created_at': r['created_at'].strftime('%Y-%m-%d %H:%M') if r.get('created_at') else '', + } + for r in rows + ] + conn.close() + return jsonify({ + 'success': True, + 'items': items, + 'total': total, + 'page': page, + 'page_size': page_size, + 'current_user_role': role, + 'admins': admins, + }) + except Exception as e: + return jsonify({'success': False, 'error': str(e)}) + + +@admin_api.route('/user', methods=['POST']) +@admin_required +def create_user(): + data = request.get_json() or {} + username = (data.get('username') or '').strip() + password = data.get('password') or '' + role, current_row = get_current_admin_role() + if not role: + return jsonify({'success': False, 'error': '需要管理员权限'}), 403 + want_role = (data.get('role') or 'normal').strip() or 'normal' + if want_role not in ('admin', 'normal'): + want_role = 'normal' + if role == 'admin' and want_role == 'admin': + return jsonify({'success': False, 'error': '仅超级管理员可创建管理员'}) + if want_role == 'admin': + want_created_by = current_row['id'] + elif role == 'super_admin': + want_created_by = data.get('created_by_id') + else: + want_created_by = current_row['id'] + if not username or not password: + return jsonify({'success': False, 'error': '用户名和密码不能为空'}) + if len(username) < 2: + return jsonify({'success': False, 'error': '用户名至少2个字符'}) + if len(password) < 6: + return jsonify({'success': False, 'error': '密码至少6个字符'}) + if want_role == 'normal' and role == 'super_admin' and want_created_by is None: + try: + conn = get_db() + with conn.cursor() as cur: + cur.execute("SELECT id FROM users WHERE role = 'admin' ORDER BY id LIMIT 1") + r = cur.fetchone() + conn.close() + want_created_by = r['id'] if r else current_row['id'] + except Exception: + want_created_by = current_row['id'] + if want_role == 'normal' and want_created_by is None: + want_created_by = current_row['id'] + is_admin = 1 if want_role in ('super_admin', 'admin') else 0 + pwd_hash = generate_password_hash(password, method='pbkdf2:sha256') + column_ids = data.get('column_ids') + if column_ids is None: + column_ids = [] + try: + conn = get_db() + with conn.cursor() as cur: + cur.execute( + "INSERT INTO users (username, password_hash, is_admin, role, created_by_id) VALUES (%s, %s, %s, %s, %s)", + (username, pwd_hash, is_admin, want_role, want_created_by), + ) + new_uid = cur.lastrowid + _set_user_column_permissions(cur, new_uid, column_ids) + conn.commit() + conn.close() + return jsonify({'success': True, 'msg': '用户创建成功'}) + except pymysql.IntegrityError: + return jsonify({'success': False, 'error': '用户名已存在'}) + except Exception as e: + return jsonify({'success': False, 'error': str(e)}) + + +@admin_api.route('/user/', methods=['PUT']) +@admin_required +def update_user(uid): + """更新用户（密码、角色）""" + data = request.get_json() or {} + password = data.get('password') + want_role = (data.get('role') or '').strip() or data.get('role') + role, current_row = get_current_admin_role() + if not role: + return jsonify({'success': False, 'error': '需要管理员权限'}), 403 + if want_role is None and not password: + return jsonify({'success': False, 'error': '请提供要修改的内容'}) + try: + conn = get_db() + with conn.cursor() as cur: + cur.execute("SELECT id, role, created_by_id FROM users WHERE id = %s", (uid,)) + target = cur.fetchone() + if not target: + conn.close() + return jsonify({'success': False, 'error': '用户不存在'}) + if role == 'admin': + if target['role'] != 'normal' or target.get('created_by_id') != current_row['id']: + conn.close() + return jsonify({'success': False, 'error': '只能编辑自己创建的普通用户'}), 403 + want_role = None + else: + if target.get('role') == 'super_admin': + conn.close() + return jsonify({'success': False, 'error': '不能修改超级管理员'}) + if want_role == 'super_admin': + return jsonify({'success': False, 'error': '不能将用户设为超级管理员'}) + if want_role not in ('admin', 'normal', None, ''): + want_role = None + if password: + if len(password) < 6: + conn.close() + return jsonify({'success': False, 'error': '密码至少6个字符'}) + pwd_hash = generate_password_hash(password, method='pbkdf2:sha256') + cur.execute("UPDATE users SET password_hash = %s WHERE id = %s", (pwd_hash, uid)) + if want_role is not None and want_role != '': + is_admin = 1 if want_role == 'admin' else 0 + cur.execute( + "UPDATE users SET is_admin = %s, role = %s WHERE id = %s", + (is_admin, want_role, uid), + ) + column_ids = data.get('column_ids') + if column_ids is not None: + _set_user_column_permissions(cur, uid, column_ids) + conn.commit() + conn.close() + return jsonify({'success': True, 'msg': '更新成功'}) + except Exception as e: + return jsonify({'success': False, 'error': str(e)}) + + +@admin_api.route('/user/', methods=['DELETE']) +@admin_required +def delete_user(uid): + """删除用户""" + if session.get('user_id') == uid: + return jsonify({'success': False, 'error': '不能删除当前登录账号'}) + role, current_row = get_current_admin_role() + if not role: + return jsonify({'success': False, 'error': '需要管理员权限'}), 403 + try: + conn = get_db() + with conn.cursor() as cur: + cur.execute("SELECT id, role, created_by_id FROM users WHERE id = %s", (uid,)) + target = cur.fetchone() + if not target: + conn.close() + return jsonify({'success': False, 'error': '用户不存在'}) + if target.get('role') == 'super_admin': + conn.close() + return jsonify({'success': False, 'error': '不能删除超级管理员'}) + if role == 'admin': + if target.get('role') != 'normal' or target.get('created_by_id') != current_row['id']: + conn.close() + return jsonify({'success': False, 'error': '只能删除自己创建的普通用户'}), 403 + cur.execute("DELETE FROM users WHERE id = %s", (uid,)) + affected = cur.rowcount + conn.commit() + conn.close() + if affected == 0: + return jsonify({'success': False, 'error': '用户不存在'}) + return jsonify({'success': True, 'msg': '删除成功'}) + except Exception as e: + return jsonify({'success': False, 'error': str(e)}) + + +# ---------- 生成历史 ---------- + +@admin_api.route('/history') +@admin_required +def history(): + """管理员分页获取所有生成记录""" + page = max(1, int(request.args.get('page', 1))) + page_size = min(50, max(10, int(request.args.get('page_size', 15)))) + offset = (page - 1) * page_size + user_id = request.args.get('user_id', type=int) + time_start = (request.args.get('time_start') or '').strip() + time_end = (request.args.get('time_end') or '').strip() + conditions, params = [], [] + if user_id: + conditions.append("h.user_id = %s") + params.append(user_id) + if time_start: + conditions.append("h.created_at >= %s") + params.append(time_start) + if time_end: + conditions.append("h.created_at <= %s") + params.append(time_end + ' 23:59:59' if len(time_end) <= 10 else time_end) + where_clause = " AND ".join(conditions) if conditions else "1=1" + params_count = params[:] + params.extend([page_size, offset]) + try: + conn = get_db() + with conn.cursor() as cur: + cur.execute( + """SELECT h.id, h.user_id, h.created_at, h.panel_type, h.original_urls, h.params, h.result_urls, + h.long_image_url, u.username + FROM image_history h + LEFT JOIN users u ON h.user_id = u.id + WHERE """ + where_clause + """ ORDER BY h.created_at DESC LIMIT %s OFFSET %s""", + params, + ) + rows = cur.fetchall() + cur.execute("SELECT COUNT(*) as total FROM image_history h WHERE " + where_clause, params_count) + total = cur.fetchone()['total'] + conn.close() + + def _parse_json(val, default=None): + if val is None: + return default if default is not None else [] + if isinstance(val, (list, dict)): + return val + try: + return json.loads(val) + except Exception: + return default if default is not None else [] + + items = [] + for r in rows: + items.append({ + 'id': r['id'], + 'user_id': r['user_id'], + 'username': r.get('username') or '-', + 'created_at': r['created_at'].strftime('%Y-%m-%d %H:%M') if r['created_at'] else '', + 'panel_type': r['panel_type'] or '', + 'original_urls': _parse_json(r['original_urls'], []), + 'params': _parse_json(r['params'], {}), + 'result_urls': _parse_json(r['result_urls'], []), + 'long_image_url': (r.get('long_image_url') or '').strip() or None, + }) + return jsonify({'success': True, 'items': items, 'total': total, 'page': page, 'page_size': page_size}) + except Exception as e: + return jsonify({'success': False, 'error': str(e)}) + + +# ---------- 栏目权限配置 ---------- + +@admin_api.route('/columns') +@admin_required +def list_columns(): + """获取栏目列表（用于栏目配置与用户权限选择）""" + try: + conn = get_db() + with conn.cursor() as cur: + cur.execute( + "SELECT id, name, column_key, created_at FROM columns ORDER BY id" + ) + rows = cur.fetchall() + conn.close() + items = [ + { + 'id': r['id'], + 'name': r['name'] or '', + 'column_key': r['column_key'] or '', + 'created_at': r['created_at'].strftime('%Y-%m-%d %H:%M') if r.get('created_at') else '', + } + for r in rows + ] + return jsonify({'success': True, 'items': items}) + except Exception as e: + return jsonify({'success': False, 'error': str(e)}) + + +@admin_api.route('/column', methods=['POST']) +@admin_required +def create_column(): + """新增栏目""" + data = request.get_json() or {} + name = (data.get('name') or '').strip() + column_key = (data.get('column_key') or '').strip() + if not name: + return jsonify({'success': False, 'error': '栏目名不能为空'}) + if not column_key: + return jsonify({'success': False, 'error': '栏目标识不能为空'}) + try: + conn = get_db() + with conn.cursor() as cur: + cur.execute( + "INSERT INTO columns (name, column_key) VALUES (%s, %s)", + (name, column_key), + ) + cid = cur.lastrowid + conn.commit() + conn.close() + return jsonify({'success': True, 'msg': '创建成功', 'id': cid}) + except pymysql.IntegrityError: + return jsonify({'success': False, 'error': '栏目标识已存在'}) + except Exception as e: + return jsonify({'success': False, 'error': str(e)}) + + +@admin_api.route('/column/', methods=['PUT']) +@admin_required +def update_column(cid): + """更新栏目""" + data = request.get_json() or {} + name = (data.get('name') or '').strip() + column_key = (data.get('column_key') or '').strip() + if not name: + return jsonify({'success': False, 'error': '栏目名不能为空'}) + if not column_key: + return jsonify({'success': False, 'error': '栏目标识不能为空'}) + try: + conn = get_db() + with conn.cursor() as cur: + cur.execute( + "UPDATE columns SET name = %s, column_key = %s WHERE id = %s", + (name, column_key, cid), + ) + if cur.rowcount == 0: + conn.close() + return jsonify({'success': False, 'error': '栏目不存在'}) + conn.commit() + conn.close() + return jsonify({'success': True, 'msg': '更新成功'}) + except pymysql.IntegrityError: + return jsonify({'success': False, 'error': '栏目标识已存在'}) + except Exception as e: + return jsonify({'success': False, 'error': str(e)}) + + +@admin_api.route('/column/', methods=['DELETE']) +@admin_required +def delete_column(cid): + """删除栏目（会同步删除用户栏目权限关联）""" + try: + conn = get_db() + with conn.cursor() as cur: + cur.execute("DELETE FROM user_column_permission WHERE column_id = %s", (cid,)) + cur.execute("DELETE FROM columns WHERE id = %s", (cid,)) + if cur.rowcount == 0: + conn.close() + return jsonify({'success': False, 'error': '栏目不存在'}) + conn.commit() + conn.close() + return jsonify({'success': True, 'msg': '删除成功'}) + except Exception as e: + return jsonify({'success': False, 'error': str(e)}) + + +@admin_api.route('/user//columns') +@admin_required +def get_user_columns(uid): + """获取某用户的栏目权限 ID 列表（编辑用户时回显）""" + try: + conn = get_db() + with conn.cursor() as cur: + cur.execute( + "SELECT column_id FROM user_column_permission WHERE user_id = %s", + (uid,), + ) + rows = cur.fetchall() + conn.close() + column_ids = [r['column_id'] for r in rows] + return jsonify({'success': True, 'column_ids': column_ids}) + except Exception as e: + return jsonify({'success': False, 'error': str(e)}) + + +@admin_api.route('/user//column-permissions') +@admin_required +def get_user_column_permissions(uid): + """根据用户获取其拥有的栏目权限详细信息""" + try: + conn = get_db() + with conn.cursor() as cur: + cur.execute( + """ + SELECT c.id, c.name, c.column_key, c.created_at + FROM user_column_permission ucp + JOIN columns c ON c.id = ucp.column_id + WHERE ucp.user_id = %s + ORDER BY c.id + """, + (uid,), + ) + rows = cur.fetchall() + conn.close() + items = [ + { + 'id': r['id'], + 'name': r['name'] or '', + 'column_key': r['column_key'] or '', + 'created_at': r['created_at'].strftime('%Y-%m-%d %H:%M') if r.get('created_at') else '', + } + for r in rows + ] + return jsonify({'success': True, 'items': items}) + except Exception as e: + return jsonify({'success': False, 'error': str(e)}) + + +def _set_user_column_permissions(cur, user_id, column_ids): + """设置用户栏目权限：先删后插。cur 为已打开的游标。""" + cur.execute("DELETE FROM user_column_permission WHERE user_id = %s", (user_id,)) + if column_ids: + column_ids = [int(x) for x in column_ids if x] + for cid in column_ids: + cur.execute( + "INSERT INTO user_column_permission (user_id, column_id) VALUES (%s, %s)", + (user_id, cid), + ) + + +# ---------- 版本管理（web_config） ---------- + +@admin_api.route('/versions') +@admin_required +def list_versions(): + """获取版本列表""" + try: + conn = get_db() + with conn.cursor() as cur: + cur.execute( + "SELECT id, version, file_url, created_at FROM web_config ORDER BY created_at DESC" + ) + rows = cur.fetchall() + conn.close() + items = [ + { + 'id': r['id'], + 'version': r['version'] or '', + 'file_url': r['file_url'] or '', + 'created_at': r['created_at'].strftime('%Y-%m-%d %H:%M') if r.get('created_at') else '', + } + for r in rows + ] + return jsonify({'success': True, 'items': items}) + except Exception as e: + return jsonify({'success': False, 'error': str(e)}) + + +@admin_api.route('/version', methods=['POST']) +@admin_required +def upload_version(): + """接收版本号 + zip 文件，上传到 OSS，写入 web_config""" + version = (request.form.get('version') or '').strip() + if not version: + return jsonify({'success': False, 'error': '请填写版本号'}) + file_storage = request.files.get('file') + if not file_storage or file_storage.filename == '': + return jsonify({'success': False, 'error': '请选择要上传的 zip 压缩包'}) + if not (file_storage.filename or '').lower().endswith('.zip'): + return jsonify({'success': False, 'error': '仅支持 .zip 格式'}) + try: + file_content = file_storage.read() + if not file_content: + return jsonify({'success': False, 'error': '文件为空'}) + safe_key = _safe_version_key(version) + key = f"{bucket_path}versions/{safe_key}.zip" + file_url = oss_upload_file(file_content, key) + conn = get_db() + with conn.cursor() as cur: + cur.execute( + "INSERT INTO web_config (version, file_url) VALUES (%s, %s)", + (version, file_url) + ) + conn.commit() + conn.close() + return jsonify({ + 'success': True, + 'version': version, + 'file_url': file_url, + 'msg': '上传成功', + }) + except Exception as e: + return jsonify({'success': False, 'error': str(e)}) + + +# ---------- 数据去重总数据 ---------- + +@admin_api.route('/dedupe-total-data') +@admin_required +def list_dedupe_total_data(): + page = max(1, int(request.args.get('page', 1))) + page_size = min(100, max(1, int(request.args.get('page_size', 15)))) + keyword = (request.args.get('keyword') or '').strip() + data, error_response, status = _proxy_backend_java( + 'GET', + '/api/admin/dedupe-total-data', + params={'page': page, 'pageSize': page_size, 'keyword': keyword}, + ) + if error_response is not None: + return error_response, status + payload = data.get('data') or {} + items = [ + { + 'id': item.get('id'), + 'data_value': item.get('dataValue') or '', + 'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16], + } + for item in (payload.get('items') or []) + ] + return jsonify({ + 'success': True, + 'items': items, + 'total': payload.get('total') or 0, + 'page': payload.get('page') or page, + 'page_size': payload.get('pageSize') or page_size, + }) + + +@admin_api.route('/dedupe-total-data/import', methods=['POST']) +@admin_required +def import_dedupe_total_data(): + file_storage = request.files.get('file') + if not file_storage or file_storage.filename == '': + return jsonify({'success': False, 'error': '请选择 Excel 文件'}) + files = { + 'file': (file_storage.filename, file_storage.stream, file_storage.mimetype or 'application/octet-stream') + } + result, error_response, status = _proxy_backend_java( + 'POST', + '/api/admin/dedupe-total-data/import', + files=files, + ) + if error_response is not None: + return error_response, status + summary = result.get('data') or {} + return jsonify({ + 'success': True, + 'msg': result.get('message') or '导入成功', + 'summary': { + 'total_rows': summary.get('totalRows') or 0, + 'asin_count': summary.get('asinCount') or 0, + 'inserted_count': summary.get('insertedCount') or 0, + 'skipped_count': summary.get('skippedCount') or 0, + }, + }) + + +@admin_api.route('/dedupe-total-data', methods=['POST']) +@admin_required +def create_dedupe_total_data(): + data = request.get_json() or {} + payload = {'dataValue': (data.get('data_value') or '').strip()} + result, error_response, status = _proxy_backend_java( + 'POST', + '/api/admin/dedupe-total-data', + json_data=payload, + ) + if error_response is not None: + return error_response, status + item = result.get('data') or {} + return jsonify({ + 'success': True, + 'msg': result.get('message') or '创建成功', + 'item': { + 'id': item.get('id'), + 'data_value': item.get('dataValue') or '', + 'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16], + }, + }) + + +@admin_api.route('/dedupe-total-data/', methods=['PUT']) +@admin_required +def update_dedupe_total_data(item_id): + data = request.get_json() or {} + payload = {'dataValue': (data.get('data_value') or '').strip()} + result, error_response, status = _proxy_backend_java( + 'PUT', + f'/api/admin/dedupe-total-data/{item_id}', + json_data=payload, + ) + if error_response is not None: + return error_response, status + item = result.get('data') or {} + return jsonify({ + 'success': True, + 'msg': result.get('message') or '更新成功', + 'item': { + 'id': item.get('id'), + 'data_value': item.get('dataValue') or '', + 'created_at': (item.get('createdAt') or '').replace('T', ' ')[:16], + }, + }) + + +@admin_api.route('/dedupe-total-data/', methods=['DELETE']) +@admin_required +def delete_dedupe_total_data(item_id): + result, error_response, status = _proxy_backend_java( + 'DELETE', + f'/api/admin/dedupe-total-data/{item_id}', + ) + if error_response is not None: + return error_response, status + return jsonify({ + 'success': True, + 'msg': result.get('message') or '删除成功', + }) diff --git a/backend/blueprints/auth.py b/backend/blueprints/auth.py new file mode 100644 index 0000000..5a459c1 --- /dev/null +++ b/backend/blueprints/auth.py @@ -0,0 +1,74 @@ +""" +认证蓝图：登录、登出、登录状态校验 +""" +from flask import Blueprint, request, redirect, url_for, session, jsonify +from werkzeug.security import check_password_hash + +from utils.db import get_db +from utils.auth import login_required, is_session_user_valid +from utils.render import render_html + +auth = Blueprint('auth', __name__, url_prefix='') + + +@auth.route('/login', methods=['GET', 'POST']) +def login(): + if session.get('user_id') and is_session_user_valid(): + return redirect(url_for('main.admin_page')) + if request.method == 'POST': + data = request.get_json() if request.is_json else request.form + username = (data.get('username') or '').strip() + password = data.get('password') or '' + if not username or not password: + if request.is_json: + return jsonify({'success': False, 'error': '请输入用户名和密码'}) + return render_html('login.html', error='请输入用户名和密码') + try: + conn = get_db() + with conn.cursor() as cur: + cur.execute( + "SELECT id, password_hash, machine, is_admin FROM users WHERE username = %s", + (username,) + ) + row = cur.fetchone() + if row and check_password_hash(row['password_hash'], password): + session.permanent = True + session['user_id'] = row['id'] + session['username'] = username + if request.is_json: + return jsonify({'success': True, 'redirect': url_for('main.admin_page')}) + return redirect(url_for('main.admin_page')) + conn.close() + except Exception as e: + if request.is_json: + return jsonify({'success': False, 'error': str(e)}) + return render_html('login.html', error='登录失败，请稍后重试') + if request.is_json: + return jsonify({'success': False, 'error': '用户名或密码错误'}) + return render_html('login.html', error='用户名或密码错误') + return render_html('login.html') + + +@auth.route('/api/auth/check') +@login_required +def api_auth_check(): + """校验登录状态，用于页面加载时判断是否已登录""" + if not session.get('user_id'): + return jsonify({'logged_in': False}) + try: + conn = get_db() + with conn.cursor() as cur: + cur.execute("SELECT machine, is_admin FROM users WHERE id = %s", (session['user_id'],)) + row = cur.fetchone() + conn.close() + if not row: + return jsonify({'logged_in': False}) + except Exception: + return jsonify({'logged_in': False}) + return jsonify({'logged_in': True, 'redirect': url_for('main.admin_page')}) + + +@auth.route('/logout') +def logout(): + session.clear() + return redirect(url_for('auth.login')) diff --git a/backend/blueprints/main.py b/backend/blueprints/main.py new file mode 100644 index 0000000..7bbcdfc --- /dev/null +++ b/backend/blueprints/main.py @@ -0,0 +1,38 @@ +""" +主页面蓝图：首页、管理后台页、静态文件 +""" +import os +from flask import Blueprint, redirect, url_for, send_file, session + +from utils.auth import login_required, admin_required, is_session_user_valid +from utils.render import render_html + +main = Blueprint('main', __name__, url_prefix='') + +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +STATIC_DIR = os.path.join(BASE_DIR, 'static') + + +@main.route('/') +def index(): + if session.get('user_id') and is_session_user_valid(): + return redirect(url_for('main.admin_page')) + return redirect(url_for('auth.login')) + + +@main.route('/admin') +@login_required +@admin_required +def admin_page(): + return render_html('admin.html') + + +@main.route('/static/') +def serve_static(filename): + """提供 static 目录及子目录下的静态文件访问""" + filepath = os.path.normpath(os.path.join(STATIC_DIR, filename)) + static_abs = os.path.abspath(STATIC_DIR) + file_abs = os.path.abspath(filepath) + if not file_abs.startswith(static_abs) or not os.path.isfile(file_abs): + return '', 404 + return send_file(file_abs, as_attachment=False) diff --git a/backend/blueprints/version.py b/backend/blueprints/version.py new file mode 100644 index 0000000..300d3cc --- /dev/null +++ b/backend/blueprints/version.py @@ -0,0 +1,40 @@ +""" +版本公开 API 蓝图：当前版本、最新版本下载链接 +""" +import os +from flask import Blueprint, jsonify + +from utils.db import get_db + +version_bp = Blueprint('version', __name__, url_prefix='/api') + + +@version_bp.route('/version') +def api_version(): + """检测更新：返回当前版本及可选的最新版本信息""" + return jsonify({ + 'version': os.environ.get('APP_VERSION', '1.0.0'), + 'desc': '', + 'url': os.environ.get('APP_UPDATE_URL', ''), + }) + + +@version_bp.route('/version/latest') +def api_version_latest(): + """GET 获取最新版本的下载链接（按创建时间取最新一条）""" + try: + conn = get_db() + with conn.cursor() as cur: + cur.execute( + "SELECT version, file_url FROM web_config ORDER BY created_at DESC LIMIT 1" + ) + row = cur.fetchone() + conn.close() + if not row: + return jsonify({'version': None, 'file_url': None}) + return jsonify({ + 'version': row['version'] or '', + 'file_url': row['file_url'] or '', + }) + except Exception as e: + return jsonify({'error': str(e)}), 500 diff --git a/backend/config.py b/backend/config.py new file mode 100644 index 0000000..f32d0ac --- /dev/null +++ b/backend/config.py @@ -0,0 +1,38 @@ +base_url = "https://api.coze.cn/v1" +coze_token = "sat_12nW40INoJxArrDXbY4lSCoudbqkOYTcphC99BP2efWyzxmsk4q81WDX3ezWgqZ5" +workflow_id = "7608812635877900322" +STITCH_WORKFLOW_ID = "7608813873483300907" + + +# MySQL 配置 +mysql_host = "8.136.19.173" +mysql_user = "aiimage" +mysql_password = "WTFrb5y6hNLz6hNy" # 请修改为您的数据库密码 +mysql_database = "aiimage" + + +cache_path = "./user_data" + +region = "cn-hangzhou" +endpoint = "oss-cn-hangzhou.aliyuncs.com" +bucket = "nanri-ai-images" +accessKeyId = "LTAI5tNpyvzMNz9f2dHarsm8" +accessKeySecret = "bQSZnFH455i8tzyOgeahJmUzwmhynz" +bucket_path = "nanri-image/" + +file_url_pre = f"https://{bucket}.oss-cn-hangzhou.aliyuncs.com/" + +import os +backend_java_base_url = os.environ.get('BACKEND_JAVA_BASE_URL', 'http://127.0.0.1:18080').rstrip('/') +os.environ['OSS_ACCESS_KEY_ID'] = accessKeyId +os.environ['OSS_ACCESS_KEY_SECRET'] = accessKeySecret +os.environ['SECRET_KEY'] = "ddffc7c1d02121d9554d7b080b2511b6" + + +debug = True +version = "1.0.0" +APP_UPDATE_URL = "" +os.environ['APP_VERSION'] = version +os.environ['APP_UPDATE_URL'] = version + + diff --git a/backend/requirement.txt b/backend/requirement.txt new file mode 100644 index 0000000..1eb81b2 --- /dev/null +++ b/backend/requirement.txt @@ -0,0 +1,61 @@ +alibabacloud-oss-v2==1.2.4 +annotated-types==0.7.0 +anyio==4.12.1 +Authlib==1.6.8 +blinker==1.9.0 +bottle==0.13.4 +certifi==2026.1.4 +cffi==2.0.0 +charset-normalizer==3.4.4 +click==8.1.8 +clr_loader==0.2.10 +colorama==0.4.6 +cozepy==0.20.0 +crcmod-plus==2.3.1 +cryptography==41.0.0 +distro==1.9.0 +et_xmlfile==2.0.0 +exceptiongroup==1.3.1 +Flask==3.1.3 +flask-cors==6.0.2 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +idna==3.11 +importlib_metadata==8.7.1 +itsdangerous==2.2.0 +Jinja2==3.1.6 +MarkupSafe==3.0.3 +Nuitka==2.8.6 +numpy==1.25.2 +openpyxl==3.1.5 +ordered-set==4.1.0 +pandas==2.3.3 +pillow==11.3.0 +pip==26.0.1 +proxy_tools==0.1.0 +psutil==7.2.2 +pycparser==2.23 +pycryptodome==3.23.0 +pydantic==2.12.5 +pydantic_core==2.41.5 +PyMySQL==1.1.2 +PyQt5==5.15.11 +PyQt5-Qt5==5.15.2 +PyQt5_sip==12.17.1 +python-dateutil==2.9.0.post0 +pythonnet==3.0.5 +pytz==2026.1.post1 +pywebview==6.1 +requests==2.32.5 +setuptools==80.9.0 +six==1.17.0 +typing_extensions==4.15.0 +typing-inspection==0.4.2 +tzdata==2025.3 +urllib3==2.6.3 +websockets==14.2 +Werkzeug==3.1.6 +wheel==0.45.1 +zipp==3.23.0 +zstandard==0.25.0 diff --git a/backend/static/bg.jpg b/backend/static/bg.jpg new file mode 100644 index 0000000..de6e38d Binary files /dev/null and b/backend/static/bg.jpg differ diff --git a/backend/tool/__pycache__/devices.cpython-39.pyc b/backend/tool/__pycache__/devices.cpython-39.pyc new file mode 100644 index 0000000..efea90f Binary files /dev/null and b/backend/tool/__pycache__/devices.cpython-39.pyc differ diff --git a/backend/tool/devices.py b/backend/tool/devices.py new file mode 100644 index 0000000..a82b5d5 --- /dev/null +++ b/backend/tool/devices.py @@ -0,0 +1,249 @@ +import hashlib +import platform +import subprocess +import uuid +import os +from typing import Optional + + +class DeviceIDGenerator: + """ + Windows设备唯一ID生成器 + 通过收集多个硬件特征来生成稳定的设备唯一标识符 + """ + + def __init__(self, use_cache: bool = True, cache_file: str = ".device_id"): + """ + 初始化设备ID生成器 + + Args: + use_cache: 是否使用本地缓存 + cache_file: 缓存文件名 + """ + self.use_cache = use_cache + self.cache_file = cache_file + + def _run_wmic_command(self, command: str) -> Optional[str]: + """ + 执行WMIC命令并返回结果 + + Args: + command: WMIC命令 + + Returns: + 命令执行结果，失败则返回None + """ + try: + result = subprocess.run( + command, + shell=True, + capture_output=True, + text=True, + timeout=10 + ) + if result.returncode == 0 and result.stdout.strip(): + return result.stdout.strip() + except (subprocess.TimeoutExpired, Exception): + pass + return None + + def _get_motherboard_serial(self) -> Optional[str]: + """获取主板序列号""" + return self._run_wmic_command("wmic baseboard get serialnumber /value") + + def _get_cpu_id(self) -> Optional[str]: + """获取CPU ID""" + return self._run_wmic_command("wmic cpu get processorid /value") + + def _get_bios_serial(self) -> Optional[str]: + """获取BIOS序列号""" + return self._run_wmic_command("wmic bios get serialnumber /value") + + def _get_disk_serial(self) -> Optional[str]: + """获取系统盘序列号""" + return self._run_wmic_command("wmic diskdrive get serialnumber /value") + + def _get_machine_guid(self) -> Optional[str]: + """获取Windows机器GUID""" + try: + result = subprocess.run( + 'reg query "HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Cryptography" /v MachineGuid', + shell=True, + capture_output=True, + text=True, + timeout=10 + ) + if result.returncode == 0: + for line in result.stdout.split('\n'): + if 'MachineGuid' in line: + return line.split()[-1] + except Exception: + pass + return None + + def _extract_value(self, wmic_output: str) -> str: + """从WMIC输出中提取实际值""" + if not wmic_output: + return "" + + lines = wmic_output.split('\n') + for line in lines: + if '=' in line and not line.strip().endswith('='): + return line.split('=', 1)[1].strip() + return "" + + def _collect_hardware_info(self) -> dict: + """ + 收集硬件信息 + + Returns: + 包含各种硬件信息的字典 + """ + hardware_info = {} + + # 主板序列号 + motherboard = self._get_motherboard_serial() + hardware_info['motherboard'] = self._extract_value(motherboard) if motherboard else "" + + # CPU ID + cpu_id = self._get_cpu_id() + hardware_info['cpu'] = self._extract_value(cpu_id) if cpu_id else "" + + # BIOS序列号 + bios = self._get_bios_serial() + hardware_info['bios'] = self._extract_value(bios) if bios else "" + + # 硬盘序列号 + disk = self._get_disk_serial() + hardware_info['disk'] = self._extract_value(disk) if disk else "" + + # Windows机器GUID + machine_guid = self._get_machine_guid() + hardware_info['machine_guid'] = machine_guid if machine_guid else "" + + # 计算机名称 + hardware_info['computer_name'] = platform.node() + + # MAC地址（作为备用） + hardware_info['mac_address'] = ':'.join(['{:02x}'.format((uuid.getnode() >> elements) & 0xff) + for elements in range(0, 2*6, 2)][::-1]) + + return hardware_info + + def _generate_device_id(self, hardware_info: dict) -> str: + """ + 基于硬件信息生成设备ID + + Args: + hardware_info: 硬件信息字典 + + Returns: + 32位十六进制设备ID + """ + # 过滤掉空值，并按键排序确保一致性 + filtered_info = {k: v for k, v in hardware_info.items() if v and v.strip()} + + # 如果没有任何硬件信息，使用MAC地址作为后备方案 + if not filtered_info: + filtered_info = {'mac_address': hardware_info.get('mac_address', str(uuid.getnode()))} + + # 将所有信息连接成字符串 + info_string = '|'.join(f"{k}:{v}" for k, v in sorted(filtered_info.items())) + + # 使用SHA256生成哈希值 + hash_object = hashlib.sha256(info_string.encode('utf-8')) + device_id = hash_object.hexdigest() + + return device_id + + def _load_cached_device_id(self) -> Optional[str]: + """从缓存文件加载设备ID""" + try: + if os.path.exists(self.cache_file): + with open(self.cache_file, 'r', encoding='utf-8') as f: + cached_id = f.read().strip() + if len(cached_id) == 64: # SHA256哈希长度 + return cached_id + except Exception: + pass + return None + + def _save_device_id_to_cache(self, device_id: str) -> None: + """将设备ID保存到缓存文件""" + try: + with open(self.cache_file, 'w', encoding='utf-8') as f: + f.write(device_id) + except Exception: + pass + + def get_device_id(self) -> str: + """ + 获取设备唯一ID + + Returns: + 64字符的十六进制设备ID + """ + # 如果启用缓存，先尝试从缓存加载 + if self.use_cache: + cached_id = self._load_cached_device_id() + if cached_id: + return cached_id + + # 收集硬件信息 + hardware_info = self._collect_hardware_info() + + # 生成设备ID + device_id = self._generate_device_id(hardware_info) + + # 保存到缓存 + if self.use_cache: + self._save_device_id_to_cache(device_id) + + return device_id + + def get_device_id_short(self, length: int = 16) -> str: + """ + 获取短版本的设备ID + + Args: + length: 返回ID的长度 + + Returns: + 指定长度的设备ID + """ + full_id = self.get_device_id() + return full_id[:length] + + def get_hardware_info(self) -> dict: + """ + 获取硬件信息（用于调试） + + Returns: + 硬件信息字典 + """ + return self._collect_hardware_info() + + +# 使用示例 +def main(): + """使用示例""" + # 创建设备ID生成器实例 + device_generator = DeviceIDGenerator() + + # 获取完整设备ID（64字符） + device_id = device_generator.get_device_id() + print(f"完整设备ID: {device_id}") + + # 获取短版本设备ID（16字符） + short_id = device_generator.get_device_id_short(16) + print(f"短设备ID: {short_id}") + + # 查看硬件信息（调试用） + hardware_info = device_generator.get_hardware_info() + print("\n硬件信息:") + for key, value in hardware_info.items(): + print(f" {key}: {value}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/backend/utils/__init__.py b/backend/utils/__init__.py new file mode 100644 index 0000000..ea41342 --- /dev/null +++ b/backend/utils/__init__.py @@ -0,0 +1 @@ +# Utils 包：数据库、认证装饰器、模板渲染等 diff --git a/backend/utils/__pycache__/__init__.cpython-311.pyc b/backend/utils/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000..bebca67 Binary files /dev/null and b/backend/utils/__pycache__/__init__.cpython-311.pyc differ diff --git a/backend/utils/__pycache__/__init__.cpython-312.pyc b/backend/utils/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..ac66fbc Binary files /dev/null and b/backend/utils/__pycache__/__init__.cpython-312.pyc differ diff --git a/backend/utils/__pycache__/__init__.cpython-39.pyc b/backend/utils/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..5e4624c Binary files /dev/null and b/backend/utils/__pycache__/__init__.cpython-39.pyc differ diff --git a/backend/utils/__pycache__/auth.cpython-311.pyc b/backend/utils/__pycache__/auth.cpython-311.pyc new file mode 100644 index 0000000..3b0f649 Binary files /dev/null and b/backend/utils/__pycache__/auth.cpython-311.pyc differ diff --git a/backend/utils/__pycache__/auth.cpython-312.pyc b/backend/utils/__pycache__/auth.cpython-312.pyc new file mode 100644 index 0000000..680ff01 Binary files /dev/null and b/backend/utils/__pycache__/auth.cpython-312.pyc differ diff --git a/backend/utils/__pycache__/auth.cpython-39.pyc b/backend/utils/__pycache__/auth.cpython-39.pyc new file mode 100644 index 0000000..f652ac1 Binary files /dev/null and b/backend/utils/__pycache__/auth.cpython-39.pyc differ diff --git a/backend/utils/__pycache__/db.cpython-311.pyc b/backend/utils/__pycache__/db.cpython-311.pyc new file mode 100644 index 0000000..2115e39 Binary files /dev/null and b/backend/utils/__pycache__/db.cpython-311.pyc differ diff --git a/backend/utils/__pycache__/db.cpython-312.pyc b/backend/utils/__pycache__/db.cpython-312.pyc new file mode 100644 index 0000000..fef6ce0 Binary files /dev/null and b/backend/utils/__pycache__/db.cpython-312.pyc differ diff --git a/backend/utils/__pycache__/db.cpython-39.pyc b/backend/utils/__pycache__/db.cpython-39.pyc new file mode 100644 index 0000000..d80fda9 Binary files /dev/null and b/backend/utils/__pycache__/db.cpython-39.pyc differ diff --git a/backend/utils/__pycache__/helpers.cpython-311.pyc b/backend/utils/__pycache__/helpers.cpython-311.pyc new file mode 100644 index 0000000..a0229d1 Binary files /dev/null and b/backend/utils/__pycache__/helpers.cpython-311.pyc differ diff --git a/backend/utils/__pycache__/render.cpython-311.pyc b/backend/utils/__pycache__/render.cpython-311.pyc new file mode 100644 index 0000000..3640106 Binary files /dev/null and b/backend/utils/__pycache__/render.cpython-311.pyc differ diff --git a/backend/utils/__pycache__/render.cpython-312.pyc b/backend/utils/__pycache__/render.cpython-312.pyc new file mode 100644 index 0000000..71c8de0 Binary files /dev/null and b/backend/utils/__pycache__/render.cpython-312.pyc differ diff --git a/backend/utils/__pycache__/render.cpython-39.pyc b/backend/utils/__pycache__/render.cpython-39.pyc new file mode 100644 index 0000000..4c1dcd1 Binary files /dev/null and b/backend/utils/__pycache__/render.cpython-39.pyc differ diff --git a/backend/utils/auth.py b/backend/utils/auth.py new file mode 100644 index 0000000..efe8eb3 --- /dev/null +++ b/backend/utils/auth.py @@ -0,0 +1,81 @@ +""" +认证装饰器与 session 校验 +""" +from functools import wraps +from flask import request, redirect, url_for, session, jsonify + +from utils.db import get_db + + +def is_session_user_valid(): + """校验 session 中的 user_id 是否在数据库中仍存在；不存在则清除 session 并返回 False""" + uid = session.get('user_id') + if not uid: + return False + try: + conn = get_db() + with conn.cursor() as cur: + cur.execute("SELECT id FROM users WHERE id = %s", (uid,)) + row = cur.fetchone() + conn.close() + if not row: + session.clear() + return False + return True + except Exception: + session.clear() + return False + + +def get_current_admin_role(): + """获取当前登录用户的管理角色：super_admin / admin / None（非管理员）""" + try: + conn = get_db() + with conn.cursor() as cur: + cur.execute( + "SELECT id, username, is_admin, role, created_by_id FROM users WHERE id = %s", + (session['user_id'],) + ) + row = cur.fetchone() + conn.close() + if not row or not row.get('is_admin'): + return None, None + return row.get('role') or ('super_admin' if row.get('created_by_id') is None else 'admin'), row + except Exception: + return None, None + + +def login_required(f): + @wraps(f) + def decorated(*args, **kwargs): + if not session.get('user_id') or not is_session_user_valid(): + if request.headers.get('X-Requested-With') == 'XMLHttpRequest': + return jsonify({'success': False, 'error': '未登录'}), 401 + return redirect(url_for('auth.login')) + return f(*args, **kwargs) + return decorated + + +def admin_required(f): + @wraps(f) + def decorated(*args, **kwargs): + if not session.get('user_id'): + if request.headers.get('X-Requested-With') == 'XMLHttpRequest': + return jsonify({'success': False, 'error': '未登录'}), 401 + return redirect(url_for('auth.login')) + try: + conn = get_db() + with conn.cursor() as cur: + cur.execute("SELECT is_admin, role FROM users WHERE id = %s", (session['user_id'],)) + row = cur.fetchone() + conn.close() + if not row or not row.get('is_admin'): + if request.headers.get('X-Requested-With') == 'XMLHttpRequest': + return jsonify({'success': False, 'error': '需要管理员权限'}), 403 + return redirect(url_for('main.admin_page')) + except Exception as e: + if request.headers.get('X-Requested-With') == 'XMLHttpRequest': + return jsonify({'success': False, 'error': str(e)}), 500 + return redirect(url_for('main.admin_page')) + return f(*args, **kwargs) + return decorated diff --git a/backend/utils/db.py b/backend/utils/db.py new file mode 100644 index 0000000..bb156fd --- /dev/null +++ b/backend/utils/db.py @@ -0,0 +1,140 @@ +""" +数据库连接与初始化 +""" +import os +import pymysql +from werkzeug.security import generate_password_hash + +try: + from config import mysql_host, mysql_user, mysql_password, mysql_database +except ImportError: + mysql_host = os.environ.get('MYSQL_HOST', 'localhost') + mysql_user = os.environ.get('MYSQL_USER', 'root') + mysql_password = os.environ.get('MYSQL_PASSWORD', '') + mysql_database = os.environ.get('MYSQL_DATABASE', 'maixiang_ai') + + +def get_db(): + return pymysql.connect( + host=mysql_host, + user=mysql_user, + password=mysql_password, + database=mysql_database, + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor + ) + + +def _create_initial_admin(): + """若没有任何管理员，则创建默认超级管理员（首次启动时，仅一个）""" + try: + conn = get_db() + with conn.cursor() as cur: + cur.execute("SELECT id FROM users WHERE role = 'super_admin' LIMIT 1") + if cur.fetchone(): + conn.close() + return + admin_user = os.environ.get('ADMIN_USER', 'admin') + admin_pwd = os.environ.get('ADMIN_PASSWORD', 'admin123') + pwd_hash = generate_password_hash(admin_pwd, method='pbkdf2:sha256') + cur.execute( + "INSERT INTO users (username, password_hash, is_admin, role) VALUES (%s, %s, 1, 'super_admin')", + (admin_user, pwd_hash) + ) + conn.commit() + conn.close() + except Exception: + pass + + +def init_db(): + """初始化数据库表，若不存在则创建""" + conn = pymysql.connect( + host=mysql_host, + user=mysql_user, + password=mysql_password, + charset='utf8mb4' + ) + try: + with conn.cursor() as cur: + cur.execute(f"CREATE DATABASE IF NOT EXISTS `{mysql_database}` DEFAULT CHARSET utf8mb4") + cur.execute(f"USE `{mysql_database}`") + cur.execute(""" + CREATE TABLE IF NOT EXISTS users ( + id INT AUTO_INCREMENT PRIMARY KEY, + username VARCHAR(64) NOT NULL UNIQUE, + password_hash VARCHAR(256) NOT NULL, + is_admin TINYINT(1) DEFAULT 0, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP + ) + """) + cur.execute(""" + CREATE TABLE IF NOT EXISTS image_history ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + panel_type VARCHAR(64) DEFAULT '', + original_urls JSON, + params JSON, + result_urls JSON, + long_image_url VARCHAR(1024) DEFAULT NULL, + INDEX idx_user_created (user_id, created_at DESC) + ) + """) + try: + cur.execute("ALTER TABLE image_history ADD COLUMN long_image_url VARCHAR(1024) DEFAULT NULL") + except Exception: + pass + try: + cur.execute("ALTER TABLE users ADD COLUMN machine VARCHAR(64) DEFAULT NULL") + except Exception: + pass + try: + cur.execute("ALTER TABLE users ADD COLUMN role VARCHAR(20) DEFAULT 'normal'") + except Exception: + pass + try: + cur.execute("ALTER TABLE users ADD COLUMN created_by_id INT NULL") + except Exception: + pass + cur.execute(""" + CREATE TABLE IF NOT EXISTS web_config ( + id INT AUTO_INCREMENT PRIMARY KEY, + version VARCHAR(64) NOT NULL, + file_url VARCHAR(1024) NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP + ) + """) + cur.execute(""" + CREATE TABLE IF NOT EXISTS columns ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(128) NOT NULL COMMENT '栏目名', + column_key VARCHAR(64) NOT NULL COMMENT '栏目标识', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + UNIQUE KEY uk_column_key (column_key) + ) + """) + cur.execute(""" + CREATE TABLE IF NOT EXISTS user_column_permission ( + user_id INT NOT NULL, + column_id INT NOT NULL, + PRIMARY KEY (user_id, column_id), + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + FOREIGN KEY (column_id) REFERENCES columns(id) ON DELETE CASCADE + ) + """) + try: + cur.execute("UPDATE users SET role = 'normal' WHERE (role IS NULL OR role = '') AND (is_admin = 0 OR is_admin IS NULL)") + cur.execute("SELECT MIN(id) AS mid FROM users WHERE is_admin = 1") + row = cur.fetchone() + if row and row.get('mid'): + mid = row['mid'] + cur.execute("UPDATE users SET role = 'super_admin' WHERE id = %s", (mid,)) + cur.execute("UPDATE users SET role = 'admin' WHERE is_admin = 1 AND id != %s", (mid,)) + cur.execute("UPDATE users SET created_by_id = %s WHERE role = 'admin' AND (created_by_id IS NULL)", (mid,)) + except Exception: + pass + conn.commit() + finally: + conn.close() + _create_initial_admin() diff --git a/backend/utils/render.py b/backend/utils/render.py new file mode 100644 index 0000000..745e027 --- /dev/null +++ b/backend/utils/render.py @@ -0,0 +1,22 @@ +""" +模板渲染：支持加密 HTML 解密后渲染 +""" +import os +from flask import render_template, render_template_string + +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def render_html(template_name: str, **context): + """读取 HTML 模板：若为加密文件则先解密，再渲染。未加密或解密失败时按明文渲染。""" + path = os.path.join(BASE_DIR, "web_source", template_name) + if not os.path.isfile(path): + return render_template(template_name, **context) + with open(path, "rb") as f: + raw = f.read() + try: + from html_crypto import decrypt + content = decrypt(raw).decode("utf-8") + except Exception: + content = raw.decode("utf-8", errors="replace") + return render_template_string(content, **context) diff --git a/backend/web_source/admin.html b/backend/web_source/admin.html new file mode 100644 index 0000000..2f6a304 --- /dev/null +++ b/backend/web_source/admin.html @@ -0,0 +1,1121 @@ + + + + + + 管理后台 - 南日AI + + + + +

管理后台

+ +
+
用户管理
+
栏目权限配置
+
数据去重总数据
+
查看生成记录
+
版本管理
+
+ + +
+
+

创建用户

+

无注册入口，仅管理员可在此创建用户。层级：超级管理员 → 管理员 → 普通号。

+
+ + +
+
+ + +
+
+ + +
+ +
+ +
+ +
+ +

+
+ +
+

用户列表

+
+
+ + +
+ + +
+ + + + + + + + + + + + +
ID用户名角色所属管理员创建时间操作
+ +
+
+ + +
+
+

新增栏目

+
+
+ + +
+
+ + +
+ +
+

+
+
+

栏目列表

+ + + + + + + + + + + +
ID栏目名栏目标识创建时间操作
+
+
+ + +
+
+

发布新版本

+
+ + +
+
+ + +
+ +

+
+
+

版本列表

+ + + + + + + + + + +
版本号下载链接发布时间操作
+
+
+ + +
+
+

新增总数据

+
+
+ + +
+ +
+

+
+
+

总数据列表

+
+
+ + +
+ +
+ + + + + + + + + + +
ID总数据值创建时间操作
+ +
+
+ + +
+
+

筛选条件

+
+
+ + +
+
+ + +
+
+ + +
+ +
+
+
+

生成记录

+ + + + + + + + + + + +
ID用户类型创建时间结果预览
+ +
+
+ + + + + + + + + + + + + diff --git a/backend/web_source/login.html b/backend/web_source/login.html new file mode 100644 index 0000000..6e2b784 --- /dev/null +++ b/backend/web_source/login.html @@ -0,0 +1,164 @@ + + + + + + 登录 - 南日AI + + + +
+ 南日AI +
+ + + + + diff --git a/frontend-vue/src/pages/brand/components/BrandConvertTab.vue b/frontend-vue/src/pages/brand/components/BrandConvertTab.vue index 85d2977..3fd8a4c 100644 --- a/frontend-vue/src/pages/brand/components/BrandConvertTab.vue +++ b/frontend-vue/src/pages/brand/components/BrandConvertTab.vue @@ -4,142 +4,167 @@
+ +
+
转换结果
+ +
+
+
+ 已处理文件 + {{ convertSummary.total }} +
+
+ 成功结果 + {{ convertSummary.successCount }} +
+
+ 失败文件 + {{ convertSummary.failedCount }}
- -
-
转换说明
-
- +
+
+ 生成的结果压缩包列表 +
- -
+
+ 暂无转换结果，完成格式转换后会在这里展示结果压缩包 +
-
- - - {{ convertRunning ? '正在生成并上传结果，请稍候…' : '选择文件后即可执行格式转换，结果会显示在右侧供下载' }} - -
- +
    +
  • +
    + {{ item.sourceFilename || '-' }} +
    压缩包文件：{{ item.outputFilename }}
    +
    错误信息：{{ item.error }}
    +
    -
    -
    转换结果
    - -
    -
    -
    - 已处理文件 - {{ convertSummary.total }} -
    -
    - 成功结果 - {{ convertSummary.successCount }} -
    -
    - 失败文件 - {{ convertSummary.failedCount }} +
    + + {{ item.success ? '已完成' : '失败' }} + + + +
    +
    • +
- -
-
- 生成的 txt 列表 -
- -
- 暂无转换结果，完成格式转换后会在这里展示输出文件 -
- -
    -
  • -
    - {{ item.sourceFilename || '-' }} -
    输出文件：{{ item.outputFilename }}
    -
    错误信息：{{ item.error }}
    -
    - -
    - - {{ item.success ? '已完成' : '失败' }} - - - -
    -
    • -
-
- - + @@ -148,53 +173,89 @@ import { computed, onMounted, ref } from 'vue' import { ElMessage } from 'element-plus' import BrandTopBar from './BrandTopBar.vue' -import { expandBrandFolder } from '@/shared/api/brand' -import { deleteConvertHistory, getConvertHistory, getConvertTemplates, runConvert, type ConvertResultItem, type ConvertRunVo, type ConvertTemplateVo } from '@/shared/api/java-modules' +import { expandBrandFolder, getBrandTemplateXlsxUrl, getBrandTemplateZipUrl } from '@/shared/api/brand' +import type { BrandExpandFolderItem } from '@/shared/api/brand' +import { + deleteConvertHistory, + deleteConvertTemplate, + getConvertHistory, + getConvertTemplates, + importConvertTemplate, + runConvert, + setDefaultConvertTemplate, + type ConvertResultItem, + type ConvertRunVo, + type ConvertTemplateVo, +} from '@/shared/api/java-modules' import { getPywebviewApi, type UploadedJavaFile } from '@/shared/bridges/pywebview' const convertSelectedPaths = ref([]) +const convertArchiveName = ref('') const convertUploadedFiles = ref([]) const convertRunning = ref(false) const convertResultItems = ref([]) const convertSummary = ref({ total: 0, successCount: 0, failedCount: 0, items: [] }) const convertTemplates = ref([]) const convertTemplateId = ref('') +const templateUploading = ref(false) +const templateSavingDefault = ref(false) +const templateDeleting = ref(false) +const templateInputRef = ref() + const convertDisplayPaths = computed(() => convertSelectedPaths.value.slice(0, 8)) const currentConvertTemplate = computed( () => convertTemplates.value.find((item) => item.id === convertTemplateId.value) || null, ) +const canDeleteCurrentTemplate = computed( + () => Boolean(currentConvertTemplate.value && !currentConvertTemplate.value.builtIn), +) -async function uploadPathsToJava(paths: string[]) { +async function uploadPathsToJava(paths: Array) { const api = getPywebviewApi() if (!api?.upload_file_to_java) { throw new Error('当前桌面端未提供文件上传桥接能力') } + const uploaded: UploadedJavaFile[] = [] - for (const path of paths) { - const result = await api.upload_file_to_java(path) + for (const item of paths) { + const filePath = typeof item === 'string' ? item : item.absolutePath + const relativePath = typeof item === 'string' ? undefined : item.relativePath + const result = await api.upload_file_to_java(filePath, relativePath) if (!result?.success || !result.data) { - throw new Error(result?.error || result?.message || `上传失败：${path}`) + throw new Error(result?.error || result?.message || `上传失败：${filePath}`) } uploaded.push(result.data) } return uploaded } -async function loadConvertTemplates() { +async function loadConvertTemplates(preferredTemplateId?: string) { const templates = await getConvertTemplates() convertTemplates.value = templates || [] + + if (preferredTemplateId && convertTemplates.value.some((item) => item.id === preferredTemplateId)) { + convertTemplateId.value = preferredTemplateId + return + } + + const existing = convertTemplates.value.find((item) => item.id === convertTemplateId.value) + if (existing) { + convertTemplateId.value = existing.id + return + } + const defaultTemplate = convertTemplates.value.find((item) => item.isDefault) if (defaultTemplate) { convertTemplateId.value = defaultTemplate.id return } - if (convertTemplates.value.length && !convertTemplates.value.some((item) => item.id === convertTemplateId.value)) { - convertTemplateId.value = convertTemplates.value[0].id - } + + convertTemplateId.value = convertTemplates.value[0]?.id || '' } async function handleSelectedPaths(paths: string[], successMessage: string) { convertSelectedPaths.value = paths + convertArchiveName.value = '' convertUploadedFiles.value = await uploadPathsToJava(paths) ElMessage.success(successMessage) } @@ -225,10 +286,13 @@ async function selectConvertFolder() { try { const folder = await api.select_brand_folder() if (!folder) return + convertArchiveName.value = folder.split(/[/\\]/).filter(Boolean).pop() || '' const result = await expandBrandFolder(folder) - if (result.success && result.paths?.length) { - await handleSelectedPaths(result.paths, `已选择文件夹内 ${result.paths.length} 个 xlsx 文件`) + if (result.success && result.items?.length) { + convertSelectedPaths.value = result.items.map((item) => item.relativePath) + convertUploadedFiles.value = await uploadPathsToJava(result.items) + ElMessage.success(`已选择文件夹内 ${result.items.length} 个 xlsx 文件`) return } @@ -238,6 +302,116 @@ async function selectConvertFolder() { } } +function openTemplatePicker() { + templateInputRef.value?.click() +} + +async function handleTemplateFileChange(event: Event) { + const input = event.target as HTMLInputElement + const file = input.files?.[0] + input.value = '' + + if (!file) { + return + } + + try { + templateUploading.value = true + const templateContent = await file.text() + const templateName = file.name.replace(/\.[^.]+$/, '') || file.name + + const template = await importConvertTemplate({ + templateName, + templateContent, + }) + + await loadConvertTemplates(template.id) + ElMessage.success(`模板已上传：${template.templateName}`) + } catch (error) { + ElMessage.error(error instanceof Error ? error.message : '模板上传失败') + } finally { + templateUploading.value = false + } +} + +async function saveCurrentTemplateAsDefault() { + if (!currentConvertTemplate.value) { + ElMessage.warning('请先选择模板') + return + } + + try { + templateSavingDefault.value = true + await setDefaultConvertTemplate(currentConvertTemplate.value.templateCode || currentConvertTemplate.value.id) + await loadConvertTemplates(currentConvertTemplate.value.id) + ElMessage.success('默认模板已更新') + } catch (error) { + ElMessage.error(error instanceof Error ? error.message : '设置默认模板失败') + } finally { + templateSavingDefault.value = false + } +} + +async function removeCurrentTemplate() { + if (!currentConvertTemplate.value) { + ElMessage.warning('请先选择模板') + return + } + if (currentConvertTemplate.value.builtIn) { + ElMessage.warning('内置模板不允许删除') + return + } + + try { + templateDeleting.value = true + await deleteConvertTemplate(currentConvertTemplate.value.templateCode || currentConvertTemplate.value.id) + const deletedTemplateId = currentConvertTemplate.value.id + await loadConvertTemplates() + if (convertTemplateId.value === deletedTemplateId) { + convertTemplateId.value = convertTemplates.value[0]?.id || '' + } + ElMessage.success('模板已删除') + } catch (error) { + ElMessage.error(error instanceof Error ? error.message : '删除模板失败') + } finally { + templateDeleting.value = false + } +} + +async function saveTemplateFromUrl(url: string, fallbackFilename: string, bridgeMethod?: 'save_template_xlsx' | 'save_template_zip') { + const api = getPywebviewApi() + + if (bridgeMethod && api?.[bridgeMethod]) { + const result = await api[bridgeMethod]!() + if (result.success) { + ElMessage.success(`已保存：${result.path || fallbackFilename}`) + } else if (result.error && result.error !== '用户取消') { + ElMessage.error(result.error) + } + return + } + + if (api?.save_file_from_url) { + const result = await api.save_file_from_url(url, fallbackFilename) + if (result.success) { + ElMessage.success(`已保存：${result.path || fallbackFilename}`) + } else if (result.error && result.error !== '用户取消') { + ElMessage.error(result.error) + } + return + } + + window.open(url, '_blank') +} + +async function downloadTemplateXlsx() { + await saveTemplateFromUrl(getBrandTemplateXlsxUrl(), '品牌文档格式_模板.xlsx', 'save_template_xlsx') +} + +async function downloadTemplateZip() { + await saveTemplateFromUrl(getBrandTemplateZipUrl(), '模板2-以文件夹方式上传.zip', 'save_template_zip') +} + async function submitConvertRun() { if (!convertUploadedFiles.value.length) { ElMessage.warning('请先选择待转换 Excel 文件或文件夹') @@ -251,8 +425,16 @@ async function submitConvertRun() { try { convertRunning.value = true const result = await runConvert({ - files: convertUploadedFiles.value.map((item) => ({ fileKey: item.fileKey, originalFilename: item.originalFilename })), + files: convertUploadedFiles.value.map((item) => ({ + fileKey: item.fileKey, + originalFilename: item.originalFilename, + relativePath: item.relativePath, + })), templateId: convertTemplateId.value, + archiveName: + convertUploadedFiles.value.some((item) => !!item.relativePath) && convertArchiveName.value + ? convertArchiveName.value + : undefined, }) convertSummary.value = result convertResultItems.value = result.items || [] @@ -318,58 +500,490 @@ onMounted(() => { diff --git a/frontend-vue/src/pages/brand/components/BrandDedupeTab.vue b/frontend-vue/src/pages/brand/components/BrandDedupeTab.vue index 6789bcc..bbafb93 100644 --- a/frontend-vue/src/pages/brand/components/BrandDedupeTab.vue +++ b/frontend-vue/src/pages/brand/components/BrandDedupeTab.vue @@ -4,143 +4,141 @@
-
读取首个 Excel 文件的第一行作为表头，用于动态选择需要保留的列。
-
+
+
去重结果
-
ID 保留规则
-
-
-
+ @@ -149,16 +147,18 @@ import { computed, onMounted, ref } from 'vue' import { ElMessage } from 'element-plus' import BrandTopBar from './BrandTopBar.vue' -import { expandBrandFolder } from '@/shared/api/brand' +import { expandBrandFolder, type BrandExpandFolderItem } from '@/shared/api/brand' import { deleteDedupeHistory, getDedupeHistory, getExcelInfo, runDedupe, type DedupeResultItem, type DedupeRunVo } from '@/shared/api/java-modules' import { getPywebviewApi, type UploadedJavaFile } from '@/shared/bridges/pywebview' const cleanAvailableColumns = ref([]) const cleanSelectedColumns = ref([]) const cleanSelectedPaths = ref([]) +const cleanArchiveName = ref('') const cleanUploadedFiles = ref([]) -const cleanKeepIntegerIds = ref(true) +const cleanKeepIntegerIds = ref(false) const cleanKeepUnderscoreIds = ref(true) +const cleanKeepIntegerMainIdsWhenNoSubIds = ref(true) const cleanRunning = ref(false) const cleanResultItems = ref([]) const cleanSummary = ref({ total: 0, successCount: 0, failedCount: 0, items: [] }) @@ -172,16 +172,18 @@ function clearAllCleanColumns() { cleanSelectedColumns.value = [] } -async function uploadPathsToJava(paths: string[]) { +async function uploadPathsToJava(paths: Array) { const api = getPywebviewApi() if (!api?.upload_file_to_java) { throw new Error('当前桌面端未提供文件上传桥接能力') } const uploaded: UploadedJavaFile[] = [] - for (const path of paths) { - const result = await api.upload_file_to_java(path) + for (const item of paths) { + const filePath = typeof item === 'string' ? item : item.absolutePath + const relativePath = typeof item === 'string' ? undefined : item.relativePath + const result = await api.upload_file_to_java(filePath, relativePath) if (!result?.success || !result.data) { - throw new Error(result?.error || result?.message || `上传失败：${path}`) + throw new Error(result?.error || result?.message || `上传失败：${filePath}`) } uploaded.push(result.data) } @@ -209,6 +211,7 @@ async function loadCleanHeaders(fileKey: string) { async function handleSelectedPaths(paths: string[], successMessage: string) { cleanSelectedPaths.value = paths + cleanArchiveName.value = '' cleanUploadedFiles.value = await uploadPathsToJava(paths) if (cleanUploadedFiles.value.length > 0) { await loadCleanHeaders(cleanUploadedFiles.value[0].fileKey) @@ -242,10 +245,16 @@ async function selectCleanFolder() { try { const folder = await api.select_brand_folder() if (!folder) return + cleanArchiveName.value = folder.split(/[/\\]/).filter(Boolean).pop() || '' const result = await expandBrandFolder(folder) - if (result.success && result.paths?.length) { - await handleSelectedPaths(result.paths, `已选择文件夹内 ${result.paths.length} 个 xlsx 文件`) + if (result.success && result.items?.length) { + cleanSelectedPaths.value = result.items.map((item) => item.relativePath) + cleanUploadedFiles.value = await uploadPathsToJava(result.items) + if (cleanUploadedFiles.value.length > 0) { + await loadCleanHeaders(cleanUploadedFiles.value[0].fileKey) + } + ElMessage.success(`已选择文件夹内 ${result.items.length} 个 xlsx 文件`) return } @@ -264,7 +273,7 @@ async function submitCleanRun() { ElMessage.warning('请至少选择一列保留列') return } - if (!cleanKeepIntegerIds.value && !cleanKeepUnderscoreIds.value) { + if (!cleanKeepIntegerIds.value && !cleanKeepUnderscoreIds.value && !cleanKeepIntegerMainIdsWhenNoSubIds.value) { ElMessage.warning('请至少选择一种 ID 保留规则') return } @@ -272,10 +281,15 @@ async function submitCleanRun() { try { cleanRunning.value = true const result = await runDedupe({ - files: cleanUploadedFiles.value.map((item) => ({ fileKey: item.fileKey, originalFilename: item.originalFilename })), + files: cleanUploadedFiles.value.map((item) => ({ fileKey: item.fileKey, originalFilename: item.originalFilename, relativePath: item.relativePath })), selectedColumns: cleanSelectedColumns.value, keepIntegerIds: cleanKeepIntegerIds.value, keepUnderscoreIds: cleanKeepUnderscoreIds.value, + keepIntegerMainIdsWhenNoSubIds: cleanKeepIntegerMainIdsWhenNoSubIds.value, + archiveName: + cleanUploadedFiles.value.some((item) => !!item.relativePath) && cleanArchiveName.value + ? cleanArchiveName.value + : undefined, }) cleanSummary.value = result cleanResultItems.value = result.items || [] @@ -340,65 +354,440 @@ onMounted(() => { diff --git a/frontend-vue/src/pages/brand/components/BrandSplitTab.vue b/frontend-vue/src/pages/brand/components/BrandSplitTab.vue index 48d5696..903fa2b 100644 --- a/frontend-vue/src/pages/brand/components/BrandSplitTab.vue +++ b/frontend-vue/src/pages/brand/components/BrandSplitTab.vue @@ -12,7 +12,7 @@ -
+