Merge pull request #36 from genxnetwork/fix_validate_files

Fixed validate files for cases with concatenating public keys

validate_files.py

@@ -14,7 +14,7 @@ def wrapper(*args, **kwargs):
                 try:
                     result = func(*args, **kwargs)
                     return result
-                except Exception:
+                except Exception as e:
                     pass
         return wrapper
     return dec
@@ -41,15 +41,25 @@ def get_http_filesize(url):
         url = urlunparse(url_parsed)
 
     file = urlopen(Request(url, method='HEAD'))
-    return file.headers.get('Content-Length')
+    return int(file.headers.get('Content-Length'))
 
 
-def get_filesize(data):
-    url = data['url']
+def get_url(url, access_keys):
+    if url.startswith('https://dataset1000genomes.blob.core.windows.net'):
+        return url + access_keys['1000g_public_key']
+    if url.startswith('https://bioinformatics.file.core.windows.net'):
+        return url + access_keys['azure_public_key']
+    return url
+
+
+def get_filesize(data, access_keys):
     if 'expand_rule' in data:
+        url = data['url']
         key = data['expand_rule']['key']
         values = data['expand_rule']['values']
-        return [get_filesize({'url': url.replace(key, str(val))}) for val in values]
+        return [get_filesize({'url': url.replace(key, str(val))}, access_keys) for val in values]
+    else:
+        url = get_url(data['url'], access_keys)
 
     if urlparse(url).scheme == 'ftp':
         return get_ftp_filesize(url)
@@ -59,12 +69,12 @@ def get_filesize(data):
 def main(yaml_url):
     with open(yaml_url, 'r') as file:
         content = yaml.safe_load(file)
-
+    access_keys = {k: content.get(k) for k in ('azure_public_key', '1000g_public_key')}
     errors = {}
     for k, v in content['reference'].items():
         if 'url' not in v or 'filesize' not in v:
             continue
-        filesize = get_filesize(v)
+        filesize = get_filesize(v, access_keys)
         if str(filesize) != str(v['filesize']):
             errors[k] = f'Expected {v["filesize"]}, got {filesize}'