Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin

app/amazon/detail_spider.py

@@ -441,26 +441,39 @@ class SpiderTask:
                     runing_task[task_id]["status"] = "failed"
                     runing_task[task_id]["error"] = str(e)
 
-    def post_result(self, task_id: int, chunkIndex:int,chunkTotal: int, asin: str, error:str="",
-                    item_data:dict={},
-                    is_done: bool = False):
+    def post_result(self, task_id: int, chunkIndex:int,chunkTotal: int, asin: str, error:str="",
+                    item_data:dict={},
+                    is_done: bool = False):
         """回传处理结果到API
         """
 
         url = f"{DELETE_BRAND_API_BASE}/api/appearance-patent/tasks/{task_id}/result"
 
+<<<<<<< HEAD
         payload ={
           "submissionId": f"{int(time.time())}",
           "chunkIndex": chunkIndex,
           "chunkTotal": chunkTotal,
           "error": error,
           "groups": item_data,
+=======
+        submission_id = f"appearance-patent:{task_id}"
+        payload ={
+          "submissionId": submission_id,
+          "chunkIndex": chunkIndex,
+          "chunkTotal": chunkTotal,
+          "error": error,
+          "items": [
+            item_data
+          ],
+>>>>>>> e4bf104ae20efcb34816c5d3da9c3372ecd92d93
           "done": is_done
         }
 
         max_retries = 3
-        for retry in range(max_retries):
-            try:
+        for retry in range(max_retries):
+            try:
+                request_timeout = 300 if is_done else 30
                 print("================【详情采集】=====================")
                 self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
                 self.log(f"回传URL: {url}")
@@ -469,7 +482,7 @@ class SpiderTask:
                     url,
                     json=payload,
                     headers={"Content-Type": "application/json"},
-                    timeout=30,
+                    timeout=request_timeout,
                     verify=False
                 )
                 self.log(f"回传结果: {response.text}")
@@ -506,6 +519,7 @@ if __name__ == '__main__':
         "url": "",
         "title": "低"
     },
+<<<<<<< HEAD
     {
         "sourceFileKey": "",
         "sourceFilename": "",
@@ -8374,4 +8388,9 @@ if __name__ == '__main__':
 ]
     # spide.process_task(task_data)
     res = SpiderTask.group_by_id_prefix(task_data)
-    print(res)
+    print(res)
+=======
+    "code": None
+}
+    spide.process_task(task_data)
+>>>>>>> e4bf104ae20efcb34816c5d3da9c3372ecd92d93