Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin

app/.env

@@ -13,7 +13,13 @@ client_name=ShuFuAI
 
 
 # java_api_base=http://47.111.163.154:18080
+<<<<<<< HEAD
 # java_api_base=http://127.0.0.1:18080
 java_api_base=http://8.136.19.173:18080
+=======
+java_api_base=http://127.0.0.1:18080
+# java_api_base=http://8.136.19.173:18080
+# java_api_base=http://121.196.149.225:18080
+>>>>>>> e4bf104ae20efcb34816c5d3da9c3372ecd92d93
 
 

app/amazon/detail_spider.py

@@ -441,26 +441,39 @@ class SpiderTask:
                     runing_task[task_id]["status"] = "failed"
                     runing_task[task_id]["error"] = str(e)
 
-    def post_result(self, task_id: int, chunkIndex:int,chunkTotal: int, asin: str, error:str="",
-                    item_data:dict={},
-                    is_done: bool = False):
+    def post_result(self, task_id: int, chunkIndex:int,chunkTotal: int, asin: str, error:str="",
+                    item_data:dict={},
+                    is_done: bool = False):
         """回传处理结果到API
         """
 
         url = f"{DELETE_BRAND_API_BASE}/api/appearance-patent/tasks/{task_id}/result"
 
+<<<<<<< HEAD
         payload ={
           "submissionId": f"{int(time.time())}",
           "chunkIndex": chunkIndex,
           "chunkTotal": chunkTotal,
           "error": error,
           "groups": item_data,
+=======
+        submission_id = f"appearance-patent:{task_id}"
+        payload ={
+          "submissionId": submission_id,
+          "chunkIndex": chunkIndex,
+          "chunkTotal": chunkTotal,
+          "error": error,
+          "items": [
+            item_data
+          ],
+>>>>>>> e4bf104ae20efcb34816c5d3da9c3372ecd92d93
           "done": is_done
         }
 
         max_retries = 3
-        for retry in range(max_retries):
-            try:
+        for retry in range(max_retries):
+            try:
+                request_timeout = 300 if is_done else 30
                 print("================【详情采集】=====================")
                 self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
                 self.log(f"回传URL: {url}")
@@ -469,7 +482,7 @@ class SpiderTask:
                     url,
                     json=payload,
                     headers={"Content-Type": "application/json"},
-                    timeout=30,
+                    timeout=request_timeout,
                     verify=False
                 )
                 self.log(f"回传结果: {response.text}")
@@ -506,6 +519,7 @@ if __name__ == '__main__':
         "url": "",
         "title": "低"
     },
+<<<<<<< HEAD
     {
         "sourceFileKey": "",
         "sourceFilename": "",
@@ -8374,4 +8388,9 @@ if __name__ == '__main__':
 ]
     # spide.process_task(task_data)
     res = SpiderTask.group_by_id_prefix(task_data)
-    print(res)
+    print(res)
+=======
+    "code": None
+}
+    spide.process_task(task_data)
+>>>>>>> e4bf104ae20efcb34816c5d3da9c3372ecd92d93

app/web_source/brand.html

@@ -785,7 +785,24 @@
         else if (status === 'cancelled') showToast('已取消');
     }
 
-    function pollRunTask(taskId) {
+    function verifyTerminalTaskStatus(taskId, expectedStatus) {
+        return fetch('/api/brand/tasks/' + taskId, {
+            credentials: 'include',
+            headers: withUidHeaders({ 'X-Requested-With': 'XMLHttpRequest' })
+        })
+            .then(function(r) { return r.json(); })
+            .then(function(res) {
+                if (!res.success || !res.task) return null;
+                var actualStatus = (res.task.status || '').toLowerCase();
+                if (actualStatus === 'success' || actualStatus === 'failed' || actualStatus === 'cancelled') {
+                    return actualStatus;
+                }
+                return null;
+            })
+            .catch(function() { return null; });
+    }
+
+    function pollRunTask(taskId) {
         pollTaskId = taskId;
         var runProgressFill = document.getElementById('runProgressFill');
         var runProgressText = document.getElementById('runProgressText');
@@ -848,15 +865,18 @@
                                 break;
                             }
                         } catch (err) {}
-                    }
-
-                    if (terminalStatus) {
-                        try { runTaskEventAbortController.abort(); } catch (e2) {}
-                        runTaskEventAbortController = null;
-                        onRunTaskFinished(terminalStatus);
-                        return;
-                    }
-                }
+                    }
+
+                    if (terminalStatus) {
+                        var verifiedStatus = await verifyTerminalTaskStatus(taskId, terminalStatus);
+                        if (verifiedStatus) {
+                            try { runTaskEventAbortController.abort(); } catch (e2) {}
+                            runTaskEventAbortController = null;
+                            onRunTaskFinished(verifiedStatus);
+                            return;
+                        }
+                    }
+                }
             } catch (err) {}
         })();
 
@@ -918,7 +938,7 @@
                 });
         }
         tick();
-        pollTimer = setInterval(tick, 10000);
+        pollTimer = setInterval(tick, 3000);
     }
 
     document.getElementById('btnRun').onclick = function() {
@@ -1128,7 +1148,11 @@
     });
 
     loadTasks();
-    // setInterval(loadTasks, 10000);
+    setInterval(function() {
+        if (cachedTasks.some(function(t) { return (t.status || '').toLowerCase() === 'running'; })) {
+            loadTasks();
+        }
+    }, 3000);
 
     if (window.addEventListener) {
         window.addEventListener('pywebviewready', function() {