Skip to content

Commit

Permalink
perf: 完善静态类型检查
Browse files Browse the repository at this point in the history
  • Loading branch information
@Johnserf-Seed
Johnserf-Seed committed Jul 9, 2024
1 parent abe63e9 commit ecb20a7
Show file tree
Hide file tree
Showing 5 changed files with 75 additions and 73 deletions.
5 changes: 3 additions & 2 deletions f2/apps/douyin/crawler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import gzip
import traceback

from typing import Dict
from google.protobuf import json_format

from f2.log.logger import logger
Expand Down Expand Up @@ -57,7 +58,7 @@
class DouyinCrawler(BaseCrawler):
def __init__(
self,
kwargs: dict = ...,
kwargs: Dict = ...,
):
# 需要与cli同步
proxies = kwargs.get("proxies", {"http://": None, "https://": None})
Expand Down Expand Up @@ -307,7 +308,7 @@ async def __aexit__(self, exc_type, exc_val, exc_tb):


class DouyinWebSocketCrawler(WebSocketCrawler):
def __init__(self, kwargs: dict = ..., callbacks: dict = None):
def __init__(self, kwargs: Dict = ..., callbacks: Dict = None):
# 需要与cli同步
self.headers = kwargs.get("headers", {}) | {
"Cookie": f"ttwid={TokenManager.gen_ttwid()};"
Expand Down
40 changes: 20 additions & 20 deletions f2/apps/douyin/dl.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# path: f2/apps/douyin/dl.py

from typing import Any, Dict, Union
from typing import Any, Dict, List, Union

from f2.i18n.translator import _
from f2.log.logger import logger
Expand All @@ -11,7 +11,7 @@


class DouyinDownloader(BaseDownloader):
def __init__(self, kwargs: dict = {}):
def __init__(self, kwargs: Dict = {}):
if kwargs["cookie"] is None:
raise ValueError(
_(
Expand All @@ -34,14 +34,14 @@ async def save_last_aweme_id(self, sec_user_id: str, aweme_id: int) -> None:
await db.update_user_info(sec_user_id=sec_user_id, last_aweme_id=aweme_id)

async def create_download_tasks(
self, kwargs: dict, aweme_datas: Union[list, dict], user_path: Any
self, kwargs: Dict, aweme_datas: Union[List, Dict], user_path: Any
) -> None:
"""
创建下载任务
Args:
kwargs (dict): 命令行参数
aweme_datas (list, dict): 作品数据列表或字典
kwargs (Dict): 命令行参数
aweme_datas (List, Dict): 作品数据列表或字典
user_path (str): 用户目录路径
"""

Expand Down Expand Up @@ -78,14 +78,14 @@ async def create_download_tasks(
await self.execute_tasks()

async def handler_download(
self, kwargs: dict, aweme_data_dict: dict, user_path: Any
self, kwargs: Dict, aweme_data_dict: Dict, user_path: Any
) -> None:
"""
处理下载任务
Args:
kwargs (dict): 命令行参数
aweme_data_dict (dict): 作品数据字典
kwargs (Dict): 命令行参数
aweme_data_dict (Dict): 作品数据字典
user_path (Any): 用户目录路径
"""
self.base_path = (
Expand Down Expand Up @@ -204,14 +204,14 @@ async def download_images(self):
)

async def create_music_download_tasks(
self, kwargs: dict, music_datas: Union[list, dict], user_path: Any
self, kwargs: Dict, music_datas: Union[List, Dict], user_path: Any
) -> None:
"""
创建音乐下载任务
Args:
kwargs (dict): 命令行参数
music_datas (list, dict): 音乐数据列表或字典
kwargs (Dict): 命令行参数
music_datas (List, Dict): 音乐数据列表或字典
user_path (Any): 用户目录路径
"""

Expand All @@ -233,14 +233,14 @@ async def create_music_download_tasks(
await self.execute_tasks()

async def handler_music_download(
self, kwargs: dict, music_data_dict: dict, user_path: Any
self, kwargs: Dict, music_data_dict: Dict, user_path: Any
) -> None:
"""
处理音乐下载任务
Args:
kwargs (dict): 命令行参数
music_data_dict (dict): 音乐数据字典
kwargs (Dict): 命令行参数
music_data_dict (Dict): 音乐数据字典
user_path (Any): 用户目录路径
"""

Expand Down Expand Up @@ -277,14 +277,14 @@ async def handler_music_download(
)

async def create_stream_tasks(
self, kwargs: dict, webcast_datas: Union[list, dict], user_path: Any
self, kwargs: Dict, webcast_datas: Union[List, Dict], user_path: Any
) -> None:
"""
创建视频流下载任务
Args:
kwargs (dict): 命令行参数
aweme_datas (list, dict): 作品数据列表或字典
kwargs (Dict): 命令行参数
aweme_datas (List, Dict): 作品数据列表或字典
user_path (Any): 用户目录路径
"""

Expand All @@ -306,14 +306,14 @@ async def create_stream_tasks(
await self.execute_tasks()

async def handler_stream(
self, kwargs: dict, webcast_data_dict: dict, user_path: Any
self, kwargs: Dict, webcast_data_dict: Dict, user_path: Any
) -> None:
"""
处理视频流下载任务
Args:
kwargs (dict): 命令行参数
aweme_data_dict (dict): 直播数据字典
kwargs (Dict): 命令行参数
aweme_data_dict (Dict): 直播数据字典
user_path (Any): 用户目录路径
"""
custom_fields = {
Expand Down
57 changes: 29 additions & 28 deletions f2/apps/douyin/filter.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# path: f2/apps/douyin/filter.py

from typing import List, Dict, Any
from f2.utils.json_filter import JSONModel
from f2.utils.utils import _get_first_item_from_list, timestamp_2_str, replaceT

Expand Down Expand Up @@ -119,10 +120,10 @@ def unique_id(self):
def user_age(self):
return self._get_attr_value("$.user.user_age")

def _to_raw(self) -> dict:
def _to_raw(self) -> Dict:
return self._data

def _to_dict(self) -> dict:
def _to_dict(self) -> Dict:
return {
prop_name: getattr(self, prop_name)
for prop_name in dir(self)
Expand Down Expand Up @@ -311,10 +312,10 @@ def max_cursor(self):
def min_cursor(self):
return self._get_attr_value("$.min_cursor")

def _to_raw(self) -> dict:
def _to_raw(self) -> Dict:
return self._data

def _to_dict(self) -> dict:
def _to_dict(self) -> Dict:
return {
prop_name: getattr(self, prop_name)
for prop_name in dir(self)
Expand Down Expand Up @@ -479,10 +480,10 @@ def nickname_raw(self):
def uid(self):
return self._get_list_attr_value("$.collects_list[*].user_info.uid")

def _to_raw(self) -> dict:
def _to_raw(self) -> Dict:
return self._data

def _to_dict(self) -> dict:
def _to_dict(self) -> Dict:
return {
prop_name: getattr(self, prop_name)
for prop_name in dir(self)
Expand Down Expand Up @@ -605,10 +606,10 @@ def owner_id(self):
def sec_uid(self):
return self._get_list_attr_value("$.mc_list[*].sec_uid")

def _to_raw(self) -> dict:
def _to_raw(self) -> Dict:
return self._data

def _to_dict(self) -> dict:
def _to_dict(self) -> Dict:
return {
prop_name: getattr(self, prop_name)
for prop_name in dir(self)
Expand Down Expand Up @@ -835,10 +836,10 @@ def uid(self):
def unique_id(self):
return self._get_list_attr_value("$.followings[*].unique_id")

def _to_raw(self) -> dict:
def _to_raw(self) -> Dict:
return self._data

def _to_dict(self) -> dict:
def _to_dict(self) -> Dict:
return {
prop_name: getattr(self, prop_name)
for prop_name in dir(self)
Expand Down Expand Up @@ -1414,10 +1415,10 @@ def video_play_addr(self):
def images(self):
return self._get_list_attr_value("$.aweme_detail.images[*].url_list[0]")

def _to_raw(self) -> dict:
def _to_raw(self) -> Dict:
return self._data

def _to_dict(self) -> dict:
def _to_dict(self) -> Dict:
return {
prop_name: getattr(self, prop_name)
for prop_name in dir(self)
Expand Down Expand Up @@ -1536,10 +1537,10 @@ def DiggAuth(self):
def ShareAuth(self):
return self._get_attr_value("$.data.data[0].room_auth.Share")

def _to_raw(self) -> dict:
def _to_raw(self) -> Dict:
return self._data

def _to_dict(self) -> dict:
def _to_dict(self) -> Dict:
return {
prop_name: getattr(self, prop_name)
for prop_name in dir(self)
Expand Down Expand Up @@ -1654,10 +1655,10 @@ def follower_count(self):
def sec_uid(self):
return self._get_attr_value("$.data.room.owner.sec_uid")

def _to_raw(self) -> dict:
def _to_raw(self) -> Dict:
return self._data

def _to_dict(self) -> dict:
def _to_dict(self) -> Dict:
return {
prop_name: getattr(self, prop_name)
for prop_name in dir(self)
Expand Down Expand Up @@ -1900,10 +1901,10 @@ def music_title(self):
def music_title_raw(self):
return self._get_list_attr_value("$.data[*].aweme.music.title")

def _to_raw(self) -> dict:
def _to_raw(self) -> Dict:
return self._data

def _to_dict(self) -> dict:
def _to_dict(self) -> Dict:
return {
prop_name: getattr(self, prop_name)
for prop_name in dir(self)
Expand Down Expand Up @@ -2005,10 +2006,10 @@ def error_code(self):
def message(self):
return self._get_attr_value("$.message")

def _to_raw(self) -> dict:
def _to_raw(self) -> Dict:
return self._data

def _to_dict(self) -> dict:
def _to_dict(self) -> Dict:
return {
prop_name: getattr(self, prop_name)
for prop_name in dir(self)
Expand Down Expand Up @@ -2045,10 +2046,10 @@ def message(self):
def verify_ticket(self):
return self._get_attr_value("$.verify_ticket")

def _to_raw(self) -> dict:
def _to_raw(self) -> Dict:
return self._data

def _to_dict(self) -> dict:
def _to_dict(self) -> Dict:
return {
prop_name: getattr(self, prop_name)
for prop_name in dir(self)
Expand Down Expand Up @@ -2085,10 +2086,10 @@ def cursor(self):
def now(self):
return timestamp_2_str(str(self._get_attr_value("$.extra.now")))

def _to_raw(self) -> dict:
def _to_raw(self) -> Dict:
return self._data

def _to_dict(self) -> dict:
def _to_dict(self) -> Dict:
return {
prop_name: getattr(self, prop_name)
for prop_name in dir(self)
Expand Down Expand Up @@ -2137,10 +2138,10 @@ def user_uid(self):
def user_uid_type(self):
return self._get_attr_value("$.user_uid_type")

def _to_raw(self) -> dict:
def _to_raw(self) -> Dict:
return self._data

def _to_dict(self) -> dict:
def _to_dict(self) -> Dict:
return {
prop_name: getattr(self, prop_name)
for prop_name in dir(self)
Expand Down Expand Up @@ -2263,10 +2264,10 @@ def stream_orientation(self):
"$.data.data.[*].room.stream_url.stream_orientation"
)

def _to_raw(self) -> dict:
def _to_raw(self) -> Dict:
return self._data

def _to_dict(self) -> dict:
def _to_dict(self) -> Dict:
return {
prop_name: getattr(self, prop_name)
for prop_name in dir(self)
Expand Down
Loading

0 comments on commit ecb20a7

Please sign in to comment.