Revert "V0.30 dev" Rollback to 0.29.4 because 0.30 is not stable yet

.codeclimate.yml

@@ -16,4 +16,3 @@ exclude_paths:
 - "external_pkgs/"
 - "more_configs/"
 - "zmirror/external_pkgs/"
-- "zmirror/tests/"

.coveragerc

@@ -4,7 +4,6 @@ omit =
     tests/*
     more_configs/*
     zmirror/external_pkgs/*
-    zmirror/tests/*
     test.py
     wsgi.py
     config.py

.gitignore

@@ -329,7 +329,7 @@ config_zjumail.py
 custom_func_.py
 custom_func_t.py
 img/
-/zmirror/tests/sample/
+/tests/sample/
 /config.py._unittest_raw
 /custom_func.py._unittest_raw
 /error_dump/

README.md

@@ -1,5 +1,5 @@
 # zmirror
-[![zmirror version](https://img.shields.io/badge/version-0.30.0-blue.svg)](https://github.com/aploium/zmirror)
+[![zmirror version](https://img.shields.io/badge/version-0.29.4-blue.svg)](https://github.com/aploium/zmirror)
 [![zmirror Build Status](https://travis-ci.org/aploium/zmirror.svg?branch=master)](https://travis-ci.org/aploium/zmirror)
 [![zmirror unittest coverage](https://codecov.io/gh/aploium/zmirror/branch/master/graph/badge.svg)](https://codecov.io/gh/aploium/zmirror)
 [![zmirror Dependency Status](https://www.versioneye.com/user/projects/57addd5358ae9200345e108c/badge.svg?style=flat-square)](https://www.versioneye.com/user/projects/57addd5358ae9200345e108c)  

codecov.yml

@@ -12,7 +12,6 @@ coverage:
   range: "0...100"
   ignore:
   - zmirror/external_pkgs/.*
-  - zmirror/tests/.*
   - external_pkgs/.*
   - more_configs/.*
   - tests/.*

custom_func.sample.py

@@ -146,7 +146,7 @@ def demo__custom_identity_verify(identity_dict):
     """
     import hashlib
     import requests
-    from zmirror import cfg
+    import config
 
     if 'cc98_username' not in identity_dict or 'cc98_password' not in identity_dict:
         return False
@@ -155,8 +155,8 @@ def demo__custom_identity_verify(identity_dict):
         pass_md5 = hashlib.md5()
         pass_md5.update(identity_dict['cc98_password'].encode())
         pass_md5 = pass_md5.hexdigest()
-        if cfg.is_use_proxy:
-            proxy = cfg.requests_proxies
+        if config.is_use_proxy:
+            proxy = config.requests_proxies
         else:
             proxy = None
         r = requests.post('http://www.cc98.org/sign.asp', data={
@@ -175,14 +175,14 @@ def demo__custom_identity_verify(identity_dict):
 
 # Demo for Twitter
 def demo__handle_expand_url(mobj):
+    import config
     from zmirror.zmirror import add_ssrf_allowed_domain, get_group
-    from zmirror import cfg
 
     domain = get_group('domain', mobj)
     if not domain:
         return mobj.group()
     add_ssrf_allowed_domain(domain)
-    if 'https' in get_group('scheme', mobj) or cfg.force_https_domains == 'ALL':
+    if 'https' in get_group('scheme', mobj) or config.force_https_domains == 'ALL':
         scheme_prefix = 'https-'
     else:
         scheme_prefix = ''

more_configs/custom_func_twitter.py

@@ -7,8 +7,8 @@
 """
 import re
 from zmirror.zmirror import add_ssrf_allowed_domain, get_group, \
-    encode_mirror_url, decode_mirror_url
-from zmirror import cfg
+    force_https_domains, my_host_scheme, my_host_name, encode_mirror_url, \
+    decode_mirror_url
 
 regex_twitter_data_expanded = re.compile(
     r'''data-expanded-url\s*=\s*'''
@@ -21,7 +21,7 @@ def handle_expand_url(mobj):
     if not domain:
         return mobj.group()
     add_ssrf_allowed_domain(domain)
-    if 'https' in get_group('scheme', mobj) or cfg.force_https_domains == 'ALL':
+    if 'https' in get_group('scheme', mobj) or force_https_domains == 'ALL':
         scheme_prefix = 'https-'
     else:
         scheme_prefix = ''
@@ -34,7 +34,7 @@ def custom_response_text_rewriter(raw_text, content_mime, remote_url):
     regex_twitter_data_expanded.sub(handle_expand_url, raw_text)
 
     # For twitter t.co redirect
-    raw_text = raw_text.replace('https://t.co/', cfg.my_host_scheme + cfg.my_host_name + '/extdomains/https-t.co/')
+    raw_text = raw_text.replace('https://t.co/', my_host_scheme + my_host_name + '/extdomains/https-t.co/')
 
     # For twitter video
     if decode_mirror_url()["domain"] == 'video.twimg.com':

more_configs/custom_func_youtube.py

@@ -10,7 +10,6 @@
 import traceback
 from urllib.parse import urlsplit
 from zmirror.zmirror import *
-from zmirror import cfg
 
 # 如果你想视频服务器和网页服务器分开, 通过多个视频服务器来进行负载均衡,
 #   请设置在主网页服务器上设置 is_master = True ,在视频服务器上部署后设置 is_master = False
@@ -21,7 +20,7 @@
 is_master = False
 
 # 注: 如果使用了非标准端口, 请在下面所有需要填写域名的地方后面加上端口号, 如 'videocdn1.mycdn.com:20822'
-videocdn_this_site_name = cfg.my_host_name
+videocdn_this_site_name = my_host_name
 # 如果你想视频服务器和网页服务器分开, 请注释掉上面一行, 去掉下面一行的注释, 并把域名改成这一台视频服务器的域名
 # videocdn_this_site_name = 'videocdn1.mycdn.com'  # 使用标准端口
 # videocdn_this_site_name = 'videocdn1.mycdn.com:20822'  # 使用非标准端口
@@ -40,7 +39,7 @@
 
 # get videocdn domain's root domain
 try:
-    if videocdn_this_site_name != cfg.my_host_name:
+    if videocdn_this_site_name != my_host_name:
         temp0 = videocdn_domain_list[0]
     else:
         temp0 = videocdn_this_site_name
@@ -65,30 +64,30 @@
 REGEX_OF_URL = r'(https?|ftp):\/\/[^\s/$.?#].[^\s]*'
 
 regex_youtube_video_videoplayback_resolve = re.compile(
-    ('https(%|%25)3A(%|%25)2F(%|%25)2F' if cfg.my_host_scheme == 'http://' else '') +
+    ('https(%|%25)3A(%|%25)2F(%|%25)2F' if my_host_scheme == 'http://' else '') +
     r'''(?P<prefix>r\d+---sn-[a-z0-9]{8})\.googlevideo\.com(?P<percent>%|%25)2Fvideoplayback(%|%25)3F''',
     flags=re.IGNORECASE)
 
 regex_youtube_video_url_resolve = re.compile(
-    (r'https:(?P<escape_slash>\\?)/\\?/' if cfg.my_host_scheme == 'http://' else '') +
+    (r'https:(?P<escape_slash>\\?)/\\?/' if my_host_scheme == 'http://' else '') +
     r'''(?P<prefix>r\d+---sn-[a-z0-9]{8})\.googlevideo\.com''')
 
 regex_youtube_video_c_videoplayback_resolve = re.compile(
-    ('https://' if cfg.my_host_scheme == 'http://' else '') +
+    ('https://' if my_host_scheme == 'http://' else '') +
     r'''(?P<prefix>r\d+---sn-[a-z0-9]{8})\.c\.youtube\.com/videoplayback\?''',
     flags=re.IGNORECASE)
 
 
 def custom_response_text_rewriter(raw_text, content_mime, remote_url):
     # if 'html' in content_mime or 'x-www-form-urlencoded' in content_mime:
     raw_text = regex_youtube_video_videoplayback_resolve.sub(
-        ('http\g<percent>3A\g<percent>2F\g<percent>2F' if cfg.my_host_scheme == 'http://' else '') +
+        ('http\g<percent>3A\g<percent>2F\g<percent>2F' if my_host_scheme == 'http://' else '') +
         video_cdn_domain + '\g<percent>2Fvideoplayback\g<percent>3Fewmytbserver\g<percent>3D\g<prefix>\g<percent>26', raw_text)
     raw_text = regex_youtube_video_url_resolve.sub(
-        ('http:\g<escape_slash>/\g<escape_slash>/' if cfg.my_host_scheme == 'http://' else '') + video_cdn_domain, raw_text)
+        ('http:\g<escape_slash>/\g<escape_slash>/' if my_host_scheme == 'http://' else '') + video_cdn_domain, raw_text)
 
     raw_text = regex_youtube_video_c_videoplayback_resolve.sub(
-        ('http://' if cfg.my_host_scheme == 'http://' else '') +
+        ('http://' if my_host_scheme == 'http://' else '') +
         video_cdn_domain + '/videoplayback?ewmytbserver=\g<prefix>&', raw_text)
 
     if 'javascript' in content_mime:
@@ -97,7 +96,7 @@ def custom_response_text_rewriter(raw_text, content_mime, remote_url):
                                     + videocdn_video_root_domain.replace('.', r'\\.')
                                     + '$')
 
-        _buff = re.escape(videocdn_video_root_domain) + '|' + re.escape(cfg.my_host_name_root)
+        _buff = re.escape(videocdn_video_root_domain) + '|' + re.escape(my_host_name_root)
         raw_text = raw_text.replace(r'-nocookie)?\.com\/|(m\.)?[a-z0-9\-]',
                                     r'-nocookie)?\.com\/|' + _buff + r'|(m\.)?[a-z0-9\-]')  # xp
 

test.py

@@ -1,9 +1,8 @@
 # coding=utf-8
 import os
 import unittest
-from zmirror.tests import *
-
-os.environ['ZMIRROR_UNITTEST'] = "True"
+from tests import *
 
 if __name__ == '__main__':
+    os.environ['ZMIRROR_UNITTEST'] = "True"
     unittest.main()

zmirror/tests/base_class.py → tests/base_class.py

@@ -56,18 +56,6 @@ def tearDownClass(cls):
     def reload_zmirror(self, configs_dict=None):
         self.del_temp_var()
 
-        for m in [x for x in sys.modules if x.startswith("zmirror")]:
-            # print("ref_counter {} {}".format(m, sys.getrefcount(sys.modules[m])))
-            del sys.modules[m]
-
-        # importlib.reload(importlib.import_module("config_default"))
-        # importlib.reload(importlib.import_module("config"))
-        # importlib.reload(importlib.import_module("zmirror.cfg"))
-        # importlib.reload(importlib.import_module("zmirror.utils_complex"))
-        # importlib.reload(importlib.import_module("zmirror.cache_system"))
-        # importlib.reload(importlib.import_module("zmirror.zmirror"))
-        # importlib.reload(importlib.import_module("zmirror"))
-        # importlib.reload(importlib.import_module("zmirror.cfg"))
         import config
         importlib.reload(config)
 
@@ -80,10 +68,6 @@ def reload_zmirror(self, configs_dict=None):
             for config_name, config_value in configs_dict.items():
                 setattr(config, config_name, config_value)
 
-        import zmirror.cfg as cfg
-
-        # print(self.C.target_domain, cfg.target_domain, cfg.allowed_domains_set)
-
         import zmirror.cache_system as cache_system
         import zmirror.zmirror as zmirror
         importlib.reload(cache_system)

zmirror/tests/test_cdn.py → tests/test_cdn.py

@@ -53,7 +53,7 @@ def test_img_cdn(self):
             )  # type: Response
 
             self.assertEqual(  # 此时应出现重定向
-                self.zmirror.cfg.cdn_redirect_code_if_cannot_hard_rewrite
+                self.zmirror.cdn_redirect_code_if_cannot_hard_rewrite
                 , self.rv2.status_code, msg=self.dump()
             )
             self.assertEqual(
@@ -82,7 +82,7 @@ def test_img_cdn(self):
             )  # type: Response
 
             self.assertEqual(  # 此时应出现重定向
-                self.zmirror.cfg.cdn_redirect_code_if_cannot_hard_rewrite
+                self.zmirror.cdn_redirect_code_if_cannot_hard_rewrite
                 , self.rv2.status_code, msg=self.dump())
             self.assertEqual(
                 # 查询参数会被编码进url
@@ -119,7 +119,7 @@ def test_long_query_str_encode_compress(self):
             )  # type: Response
 
             self.assertEqual(  # 此时应出现重定向
-                self.zmirror.cfg.cdn_redirect_code_if_cannot_hard_rewrite
+                self.zmirror.cdn_redirect_code_if_cannot_hard_rewrite
                 , self.rv2.status_code, msg=self.dump())
             self.assertIn(
                 "/image/jpeg_{salt}z_.".format(salt=self.zmirror.cdn_url_query_encode_salt),
@@ -130,7 +130,7 @@ def test_long_query_str_encode_compress(self):
             self.rv2.location,
             environ_base=env(),
             headers=headers(
-                user_agent=DEFAULT_USER_AGENT + " " + self.zmirror.cfg.spider_ua_white_list[0]
+                user_agent=DEFAULT_USER_AGENT + " " + self.zmirror.spider_ua_white_list[0]
             )
         )  # type: Response
         self.assertEqual("image/jpeg", self.rv3.content_type, msg=self.dump())
@@ -159,7 +159,7 @@ def test_cdn_excluded_ua(self):
         )  # type: Response
 
         self.assertEqual(  # 此时应出现重定向
-            self.zmirror.cfg.cdn_redirect_code_if_cannot_hard_rewrite
+            self.zmirror.cdn_redirect_code_if_cannot_hard_rewrite
             , self.rv2.status_code, msg=self.dump())
         self.assertEqual(
             # 查询参数会被编码进url
@@ -174,7 +174,7 @@ def test_cdn_excluded_ua(self):
             self.url("/image/jpeg?love=luciaz"),
             environ_base=env(),
             headers=headers(
-                user_agent=DEFAULT_USER_AGENT + " " + self.zmirror.cfg.spider_ua_white_list[0]
+                user_agent=DEFAULT_USER_AGENT + " " + self.zmirror.spider_ua_white_list[0]
             )
         )  # type: Response
 
@@ -189,7 +189,7 @@ def test_cdn_excluded_ua(self):
                      ),
             environ_base=env(),
             headers=headers(
-                user_agent=DEFAULT_USER_AGENT + " " + self.zmirror.cfg.spider_ua_white_list[0]
+                user_agent=DEFAULT_USER_AGENT + " " + self.zmirror.spider_ua_white_list[0]
             )
         )  # type: Response
 

zmirror/tests/test_connection_pool.py → tests/test_connection_pool.py

@@ -43,8 +43,8 @@ def test_keep_alive(self):
         )  # type: Response
         time_non_alive = self.zmirror.parse.remote_response.elapsed
 
-        max_fail_count = 5
-        for _ in range(15):
+        max_fail_count = 3
+        for _ in range(10):
             self.rv2 = self.client.head(
                 self.url("/"),
                 environ_base=env(),

zmirror/tests/test_functions.py → tests/test_functions.py

@@ -171,12 +171,12 @@ def test__is_target_domain_use_https(self):
 
     def test__add_ssrf_allowed_domain(self):
         self.zmirror.add_ssrf_allowed_domain("www.example.com")
-        self.assertIn("www.example.com", self.zmirror.cfg.allowed_domains_set)
+        self.assertIn("www.example.com", self.zmirror.allowed_domains_set)
 
     def test__check_global_ua_pass(self):
         self.assertFalse(self.zmirror.check_global_ua_pass(None))
         self.assertTrue(self.zmirror.check_global_ua_pass(
-            self.zmirror.cfg.global_ua_white_name)
+            self.zmirror.global_ua_white_name)
         )
 
     def test__is_content_type_using_cdn(self):
@@ -235,27 +235,22 @@ def test__embed_real_url_to_embedded_url(self):
         )
 
     def test__encoding_detect(self):
-        self.assertEqual(
-            "UTF-8",
-            self.zmirror.encoding_detect("测试中文".encode(encoding="utf-8"))
-        )
-
-        self.zmirror.cfg.force_decode_remote_using_encode = "utf-8"
+        self.zmirror.force_decode_remote_using_encode = "utf-8"
         self.assertEqual(
             "utf-8",
             self.zmirror.encoding_detect("测试中文".encode(encoding="gbk"))
         )
 
-        self.zmirror.cfg.force_decode_remote_using_encode = None
-        self.zmirror.cfg.possible_charsets = ["gbk", "utf-8"]
+        self.zmirror.force_decode_remote_using_encode = None
+        self.zmirror.possible_charsets = ["gbk", "utf-8"]
         self.assertEqual(
             "utf-8",
             self.zmirror.encoding_detect("测试中文".encode(encoding="utf-8"))
         )
-        self.assertEqual(
-            "gbk",
-            self.zmirror.encoding_detect("测试中文".encode(encoding="gbk"))
-        )
+
+        self.zmirror.possible_charsets = None
+        self.zmirror.cchardet_available = False
+        self.assertIsNone(self.zmirror.encoding_detect("测试中文".encode(encoding="utf-8")))
 
     def test__get_group(self):
         import re