Add support for python 3 (#536)

.appveyor.yml

@@ -1,20 +1,22 @@
 build: false
 
 environment:
-    PYTHON: "C:\\Python27"
-    TOXENV: "py27"
+  matrix:
+    - PYTHON: "C:\\Python27"
+      PYTHON_VERSION: "2.7"
+      TOXENV: "py27"
+    - PYTHON: "C:\\Python36"
+      PYTHON_VERSION: "3.6"
+      TOXENV: "py36"
 
 init:
-    - ECHO %PYTHON%
-    - ECHO %TOXENV%
-    - SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%
+  - ECHO %PYTHON%
+  - ECHO %TOXENV%
+  - SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%
 
 install:
-    - python -m pip install --upgrade pip
-    # Latest tox 2.3.x is currently busted:
-    # https://bitbucket.org/hpk42/tox/issues/314/tox-command-busted-on-windows
-    - pip install tox==2.2.0
-
+  - python -m pip install --upgrade pip
+  - pip install tox==3.7.0
 
 test_script:
-    - tox
+  - tox

.gitignore

@@ -1,6 +1,8 @@
+.idea/
 .cache/
 .tox/
 *.egg-info
 *.pyc
+.coverage
 build
 dist

.travis.yml

@@ -8,29 +8,35 @@ language: python
 
 matrix:
     include:
-        - python: 2.7
+        - name: "Run test suite on Linux with Python 2.7"
           os: linux
+          python: 2.7
           env: TOXENV=py27
-        - python: 2.7
+        - name: "Run test suite on Linux with Python 3.6"
           os: linux
+          python: 3.6
+          env: TOXENV=py36
+        - name: "Check code style with pylama on Linux - Python 3.6"
+          os: linux
+          python: 3.6
           env: TOXENV=pylama
-        - language: generic
-          os: osx
-          # 7.2 is OS X 10.11.x
+          # 10.1 is OS X 10.13.x
           # https://docs.travis-ci.com/user/languages/objective-c/#Supported-OS-X-iOS-SDK-versions
-          osx_image: xcode7.2
+        - name: "Run test suite on OS X with Python 2.7"
+          os: osx
+          osx_image: xcode10.1
+          language: generic
+          python: 2.7
           env: TOXENV=py27
+        - name: "Run test suite on OS X with Python 3.6"
+          os: osx
+          osx_image: xcode9.4
+          language: generic
+          env: TOXENV=py36
 
-install:
-    - if [ "$TRAVIS_OS_NAME" = "osx" ]; then
-        curl -O -s https://bootstrap.pypa.io/get-pip.py;
-        python get-pip.py --user;
-        export PATH=$PATH:~/Library/Python/2.7/bin;
-        pip install --user tox virtualenv;
-      else
-        pip install tox virtualenv;
-      fi
 
+install:
+    - ./.travis/install.sh
 
 script:
     - tox

.travis/install.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -e
+set -x
+
+ci_requirements="tox virtualenv"
+
+if [ "$TRAVIS_OS_NAME" == "osx" ]; then
+    curl -O https://bootstrap.pypa.io/get-pip.py
+    python get-pip.py --user
+fi
+
+pip install $ci_requirements

mozdownload/parser.py

@@ -7,10 +7,10 @@
 from __future__ import absolute_import, unicode_literals
 
 import re
-import urllib
-from HTMLParser import HTMLParser
 
 import requests
+from six.moves.html_parser import HTMLParser
+from six.moves.urllib.parse import unquote
 
 
 class DirectoryParser(HTMLParser):
@@ -68,7 +68,7 @@ def handle_starttag(self, tag, attrs):
                 # Links look like: /pub/firefox/nightly/2015/
                 # We have to trim the fragment down to the last item. Also to ensure we
                 # always get it, we remove a possible final slash first
-                url = urllib.unquote(attr[1])
+                url = unquote(attr[1])
                 self.active_url = url.rstrip('/').split('/')[-1]
 
                 return

mozdownload/scraper.py

@@ -10,22 +10,20 @@
 import os
 import re
 import sys
-import urllib
 from datetime import datetime
-from urlparse import urlparse
 
 import mozinfo
 import progressbar as pb
 import redo
 import requests
+from six.moves.urllib.parse import quote, urlparse
 
 from mozdownload import errors
+from mozdownload import treeherder
 from mozdownload.parser import DirectoryParser
 from mozdownload.timezones import PacificTimezone
-from mozdownload import treeherder
 from mozdownload.utils import urljoin
 
-
 APPLICATIONS = ('devedition', 'firefox', 'fennec', 'thunderbird')
 
 # Some applications contain all locales in a single build
@@ -164,6 +162,7 @@ def _create_directory_parser(self, url):
     @property
     def binary(self):
         """Return the name of the build."""
+
         def _get_binary():
             # Retrieve all entries from the remote virtual folder
             parser = self._create_directory_parser(self.path)
@@ -195,8 +194,8 @@ def binary_regex(self):
     @property
     def url(self):
         """Return the URL of the build."""
-        return urllib.quote(urljoin(self.path, self.binary),
-                            safe='%/:=&?~#+!$,;\'@()*[]|')
+        return quote(urljoin(self.path, self.binary),
+                     safe='%/:=&?~#+!$,;\'@()*[]|')
 
     @property
     def path(self):
@@ -248,6 +247,7 @@ def detect_platform(self):
 
     def download(self):
         """Download the specified file."""
+
         def total_seconds(td):
             # Keep backward compatibility with Python 2.6 which doesn't have
             # this method
@@ -417,7 +417,7 @@ def get_latest_build_date(self):
         parser.entries = parser.filter(r'.*%s\.txt' % self.platform_regex)
         if not parser.entries:
             message = 'Status file for %s build cannot be found' % \
-                self.platform_regex
+                      self.platform_regex
             raise errors.NotFoundError(message, url)
 
         # Read status file for the platform, retrieve build id,
@@ -457,7 +457,7 @@ def is_build_dir(self, folder_name):
     def get_build_info_for_date(self, date, build_index=None):
         """Return the build information for a given date."""
         url = urljoin(self.base_url, self.monthly_build_list_regex)
-        has_time = date and date.time()
+        has_time = date and date.time() and date.strftime('%H-%M-%S') != '00-00-00'
 
         self.logger.info('Retrieving list of builds from %s' % url)
         parser = self._create_directory_parser(url)
@@ -467,7 +467,7 @@ def get_build_info_for_date(self, date, build_index=None):
             # ensure to select the correct subfolder for localized builds
             'L10N': '(-l10n)?' if self.locale_build else '',
             'PLATFORM': '' if self.application not in (
-                        'fennec') else '-' + self.platform
+                'fennec') else '-' + self.platform
         }
 
         parser.entries = parser.filter(regex)
@@ -482,7 +482,7 @@ def get_build_info_for_date(self, date, build_index=None):
         if not parser.entries:
             date_format = '%Y-%m-%d-%H-%M-%S' if has_time else '%Y-%m-%d'
             message = 'Folder for builds on %s has not been found' % \
-                self.date.strftime(date_format)
+                      self.date.strftime(date_format)
             raise errors.NotFoundError(message, url)
 
         # If no index has been given, set it to the last build of the day.
@@ -663,8 +663,8 @@ def query_versions(self, version=None):
         parser = self._create_directory_parser(url)
         if version:
             versions = parser.filter(RELEASE_AND_CANDIDATE_LATEST_VERSIONS[version])
-            from distutils.version import LooseVersion
-            versions.sort(key=LooseVersion)
+            from packaging.version import LegacyVersion
+            versions.sort(key=LegacyVersion)
             return [versions[-1]]
         else:
             return parser.entries
@@ -690,7 +690,7 @@ def get_build_info(self):
         parser = self._create_directory_parser(url)
         if not parser.entries:
             message = 'Folder for specific candidate builds at %s has not' \
-                'been found' % url
+                      'been found' % url
             raise errors.NotFoundError(message, url)
 
         self.show_matching_builds(parser.entries)
@@ -909,11 +909,11 @@ def get_build_info_for_index(self, build_index=None):
             # If a timestamp is given, retrieve the folder with the timestamp
             # as name
             parser.entries = self.timestamp in parser.entries and \
-                [self.timestamp]
+                             [self.timestamp]
 
         elif self.date:
             # If date is given, retrieve the subset of builds on that date
-            parser.entries = filter(self.date_matches, parser.entries)
+            parser.entries = list(filter(self.date_matches, parser.entries))
 
         if not parser.entries:
             message = 'No builds have been found'

mozdownload/treeherder.py

@@ -7,12 +7,12 @@
 from __future__ import absolute_import, unicode_literals
 
 import logging
+import six
 
 from thclient import TreeherderClient
 
 from mozdownload.errors import NotSupportedError
 
-
 PLATFORM_MAP = {
     'android-api-9': {'build_platform': 'android-2-3-armv7-api9'},
     'android-api-11': {'build_platform': 'android-4-0-armv7-api11'},
@@ -69,11 +69,11 @@ def query_builds_by_revision(self, revision, job_type_name='Build', debug_build=
 
         try:
             self.logger.info('Querying {url} for list of builds for revision: {revision}'.format(
-                             url=self.client.server_url, revision=revision))
+                url=self.client.server_url, revision=revision))
 
             # Retrieve the option hash to filter for type of build (opt, and debug for now)
             option_hash = None
-            for key, values in self.client.get_option_collection_hash().iteritems():
+            for key, values in six.iteritems(self.client.get_option_collection_hash()):
                 for value in values:
                     if value['name'] == ('debug' if debug_build else 'opt'):
                         option_hash = key

requirements.txt

@@ -1,5 +1,6 @@
-mozinfo >= 0.9
+mozinfo >= 1.0.0
+packaging >= 19.0
 progressbar2 >= 3.34.3
 redo==2.0.3
-requests >= 2.9.1, <3.0.0
-treeherder-client==5.0.0
+requests >= 2.21.0, <3.0.0
+treeherder-client >= 5.0.0, <6.0.0

tests/cli/test_cli_arguments.py

@@ -13,4 +13,4 @@ def test_unrecognized_argument():
                                          stderr=subprocess.STDOUT)
     except subprocess.CalledProcessError as e:
         output = e.output
-    assert re.search(r'mozdownload: error: unrecognized arguments: --abc', output) is not None
+    assert re.search(r'mozdownload: error: unrecognized arguments: --abc'.encode('utf-8'), output) is not None

tests/cli/test_cli_print_url.py

@@ -17,4 +17,4 @@ def test_print_url_argument():
         output = subprocess.check_output(args, stderr=subprocess.STDOUT)
     except subprocess.CalledProcessError as e:
         output = e.output
-    assert re.search(url, output) is not None
+    assert re.search(url.encode('utf-8'), output) is not None

tests/cli/test_correct_scraper.py

@@ -37,6 +37,8 @@
         'fname': '8fcac92cfcad-firefox-38.0a1.en-US.win32.installer.exe',
     },
 }
+
+
 @pytest.mark.parametrize("data", tests.values())
 def test_correct_cli_scraper(httpd, tmpdir, data, mocker):
     query_builds_by_revision = mocker.patch('mozdownload.treeherder.Treeherder.query_builds_by_revision')

tests/cli/test_output.py

@@ -1,3 +1,9 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
 import subprocess
 
 from mozdownload import __version__, cli
@@ -6,4 +12,4 @@
 def test_cli_executes():
     """Test that cli will start and print usage message"""
     output = subprocess.check_output(['mozdownload', '--help'])
-    assert cli.__doc__.format(__version__) in output
+    assert cli.__doc__.format(__version__) in output.decode("utf-8")

tests/daily_scraper/test_daily_scraper.py

@@ -5,8 +5,9 @@
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
-import urllib
+
 import pytest
+from six.moves.urllib.parse import unquote
 
 from mozdownload import DailyScraper
 from mozdownload.utils import urljoin
@@ -48,15 +49,15 @@
     ({'platform': 'win32', 'branch': 'mozilla-central', 'date': '2013-07-02', 'build_number': 1},
      '2013-07-02-03-12-13-mozilla-central-firefox-27.0a1.en-US.win32.installer.exe',
      'firefox/nightly/2013/07/2013-07-02-03-12-13-mozilla-central/firefox-27.0a1.en-US.win32.installer.exe'),
-     # Old stub format
+    # Old stub format
     ({'platform': 'win32', 'branch': 'mozilla-central', 'date': '2013-09-30', 'is_stub_installer': True},
      '2013-09-30-03-02-04-mozilla-central-firefox-27.0a1.en-US.win32.installer-stub.exe',
      'firefox/nightly/2013/09/2013-09-30-03-02-04-mozilla-central/firefox-27.0a1.en-US.win32.installer-stub.exe'),
-     # Old file name format
+    # Old file name format
     ({'platform': 'win64', 'branch': 'mozilla-central', 'date': '2013-09-30'},
      '2013-09-30-03-02-04-mozilla-central-firefox-27.0a1.en-US.win64-x86_64.installer.exe',
      'firefox/nightly/2013/09/2013-09-30-03-02-04-mozilla-central/firefox-27.0a1.en-US.win64-x86_64.installer.exe'),
-     # New stub format
+    # New stub format
     ({'platform': 'win32', 'branch': 'mozilla-central', 'is_stub_installer': True},
      '2013-10-01-03-02-04-mozilla-central-Firefox Installer.en-US.exe',
      'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/Firefox Installer.en-US.exe'),
@@ -126,6 +127,7 @@
      'mobile/nightly/2016/02/2016-02-02-00-40-08-mozilla-aurora-android-api-15/fennec-46.0a2.multi.android-arm.apk'),
 ]
 
+
 @pytest.mark.parametrize("args,filename,url", firefox_tests + thunderbird_tests + fennec_tests)
 def test_scraper(httpd, tmpdir, args, filename, url):
     """Testing various download scenarios for DailyScraper"""
@@ -134,4 +136,4 @@ def test_scraper(httpd, tmpdir, args, filename, url):
     expected_target = os.path.join(str(tmpdir), filename)
     assert scraper.filename == expected_target
 
-    assert urllib.unquote(scraper.url) == urljoin(httpd.get_url(), url)
+    assert unquote(scraper.url) == urljoin(httpd.get_url(), url)

tests/directory_parser/test_directory_parser.py

@@ -6,6 +6,8 @@
 
 import os
 
+import six
+
 from mozdownload.parser import DirectoryParser
 from mozdownload.utils import urljoin
 
@@ -44,7 +46,10 @@ def test_filter(httpd):
     parser.entries = parser.filter(r'^\d+$')
 
     # Get only the subdirectories of the folder
-    dirs = os.walk(folder_path).next()[1]
+    if six.PY2:
+        dirs = os.walk(folder_path).next()[1]
+    elif six.PY3:
+        dirs = os.walk(folder_path).__next__()[1]
     dirs.sort()
     assert parser.entries == dirs