fix:fix alist download bug(opennaslab#456)

evell1992 · Feb 20, 2024 · d9220ed · d9220ed
1 parent 1958503
commit d9220ed
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 7 deletions.
diff --git a/kubespider/api/values.py b/kubespider/api/values.py
@@ -4,6 +4,7 @@
 from enum import Enum
 
 from api import types
+from utils.helper import get_unique_hash
 
 FILE_TYPE_TO_PATH = {
     types.FILE_TYPE_COMMON: "Common",
@@ -88,7 +89,7 @@ def __init__(self, url: str, path: str,
         self.path = path
         self.link_type = link_type
         self.file_type = file_type
-        self.uid = uid if uid is not None else url
+        self.uid = uid if uid else get_unique_hash(url)
 
 
 class Task(Extra):

diff --git a/kubespider/core/period_server.py b/kubespider/core/period_server.py
@@ -56,7 +56,7 @@ def run_single_provider(self, provider: SourceProvider) -> TypeError:
 
         err = None
         for source in links:
-            if helper.get_unique_hash(source.url) in state:
+            if source.uid in state:
                 continue
             if source.link_type is None:
                 source.link_type = link_type
@@ -75,7 +75,7 @@ def run_single_provider(self, provider: SourceProvider) -> TypeError:
                 )
                 break
             #  add resource to state
-            state.append(helper.get_unique_hash(source.url))
+            state.append(source.uid)
 
             notification_server.kubespider_notification_server.send_message(
                 title=f"[{provider_name}] start download", url=source.url, path=source.path,

diff --git a/kubespider/source_provider/alist_source_provider/provider.py b/kubespider/source_provider/alist_source_provider/provider.py
@@ -6,7 +6,7 @@
 from api import types
 from api.values import Event, Resource
 from utils.config_reader import AbsConfigReader
-from utils.helper import get_request_controller, retry
+from utils.helper import get_request_controller, retry, get_unique_hash
 
 
 class AlistSourceProvider(provider.SourceProvider):
@@ -90,7 +90,7 @@ def fs_list(self, page, per_page, path) -> tuple:
         content = resp.get("data").get("content")
         alist_provider = resp.get("data").get("provider")
         total = resp.get("data").get("total")
-        if code:
+        if code == 200:
             return content, alist_provider, total
         raise ValueError(f"response error: {resp}")
 
@@ -116,13 +116,17 @@ def get_all_files(self, path) -> list[Resource]:
                 files += self.get_all_files(new_path)
             else:
                 item["file_type"] = types.FILE_TYPE_COMMON
-                uri = f'{os.path.join("/d", os.path.join(item.get("path"), item.get("name")))}'
-                item["link"] = urljoin(self.host, quote(uri) + f'?modified={item.get("modified")}')
+                uri = os.path.join("/d", os.path.join(item.get("path", "").strip('/'), item.get("name", "")))
+                md5 = (item.get("hash_info") or {}).get("md5", get_unique_hash(uri))
+                sign = item.get("sign", "")
+                modified = item.get("modified", "")
+                item["link"] = urljoin(self.host, quote(uri) + f'?modified={modified}&sign={sign}')
                 files.append(Resource(
                     url=item.pop("link"),
                     path=item.pop("path"),
                     link_type=self.get_link_type(),
                     file_type=item.pop("file_type"),
+                    uid=md5,
                     **item
                 ))
         return files