diff --git a/.gitignore b/.gitignore index 155e5513..f4c984de 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ *.egg-info *.pyc .coverage -build -dist .pytest_cache/ .vscode/ +build +dist diff --git a/mozdownload/scraper.py b/mozdownload/scraper.py index 0904c1ea..0e930702 100755 --- a/mozdownload/scraper.py +++ b/mozdownload/scraper.py @@ -31,9 +31,25 @@ # Used if the application is named differently than the subfolder on the server APPLICATIONS_TO_FTP_DIRECTORY = {'fennec': 'mobile'} -# Used if the application is named differently then the binary on the server +# Used if the application is named differently than the binary on the server APPLICATIONS_TO_BINARY_NAME = {'devedition': 'firefox'} +APPLICATION_BUILD_FILENAME = { + app: ( + '%(TIMESTAMP)s-%(BRANCH)s-%(NAME)s' if app != 'fenix' + else '%(TIMESTAMP)s-%(NAME)s' + ) + for app in APPLICATIONS +} + +APPLICATION_REGEX = { + app: ( + r'%(DATE)s-(\d+-)+%(BRANCH)s%(L10N)s%(PLATFORM)s$' if app != 'fenix' + else r'%(DATE)s-(\d+-){3}fenix-(\d+\.\d.\d)%(PLATFORM)s$' + ) + for app in APPLICATIONS +} + # Base URL for the path to all builds BASE_URL = 'https://archive.mozilla.org/pub/' @@ -59,7 +75,10 @@ 'android-api-11': r'android-arm', 'android-api-15': r'android-arm', 'android-api-16': r'android-arm', + 'android-arm64-v8a': r'android-arm64-v8a', + 'android-armeabi-v7a': r'android-armeabi-v7a', 'android-x86': r'android-i386', + 'android-x86_64': r'android-x86_64', 'linux': r'linux-i686', 'linux64': r'linux-x86_64', 'mac': r'mac', @@ -213,8 +232,6 @@ def path_regex(self): @property def platform_regex(self): """Return the platform fragment of the URL.""" - if self.application == "fenix": - return self.platform return PLATFORM_FRAGMENTS[self.platform] @property @@ -473,19 +490,14 @@ def get_build_info_for_date(self, date, build_index=None): self.logger.info('Retrieving list of builds from %s' % url) parser = self._create_directory_parser(url) - regex = r'%(DATE)s-(\d+-)+%(BRANCH)s%(L10N)s%(PLATFORM)s$' % { + regex = APPLICATION_REGEX[self.application] % { 'DATE': date.strftime('%Y-%m-%d'), 'BRANCH': self.branch, # ensure to select the correct subfolder for localized builds 'L10N': '(-l10n)?' if self.locale_build else '', 'PLATFORM': '' if self.application not in ( - 'fennec') else '-' + self.platform + 'fenix', 'fennec') else '-' + self.platform } - if self.application == 'fenix': - regex = r'%(DATE)s-(\d+-){3}fenix-(\d+\.\d.\d)-%(PLATFORM)s$' % { - 'DATE': date.strftime('%Y-%m-%d'), - 'PLATFORM': self.platform - } parser.entries = parser.filter(regex) parser.entries = parser.filter(self.is_build_dir) @@ -525,7 +537,7 @@ def binary_regex(self): 'android-api-15': r'\.%(EXT)s$', 'android-api-16': r'\.%(EXT)s$', 'android-arm64-v8a': r'\.%(EXT)s$', - 'armeabi-v7a': r'\.%(EXT)s$', + 'android-armeabi-v7a': r'\.%(EXT)s$', 'android-x86': r'\.%(EXT)s$', 'android-x86_64': r'\.%(EXT)s$', 'linux': r'\.%(EXT)s$', @@ -554,10 +566,7 @@ def build_filename(self, binary): # If it's not available use the build's date timestamp = self.date.strftime('%Y-%m-%d') - if self.application == 'fenix': - return binary - - return '%(TIMESTAMP)s-%(BRANCH)s-%(NAME)s' % { + return APPLICATION_BUILD_FILENAME[self.application] % { 'TIMESTAMP': timestamp, 'BRANCH': self.branch, 'NAME': binary} diff --git a/tests/daily_scraper/test_daily_scraper.py b/tests/daily_scraper/test_daily_scraper.py index 54cc6875..3a2c9ed4 100644 --- a/tests/daily_scraper/test_daily_scraper.py +++ b/tests/daily_scraper/test_daily_scraper.py @@ -129,16 +129,16 @@ fenix_tests = [ ({'application': 'fenix', 'platform': 'android-arm64-v8a', 'date': '2022-11-14'}, - 'fenix-108.0b1.multi.android-arm64-v8a.apk', + '2022-11-14-17-01-36-fenix-108.0b1.multi.android-arm64-v8a.apk', 'fenix/nightly/2022/11/2022-11-14-17-01-36-fenix-108.0b1-android-arm64-v8a/fenix-108.0b1.multi.android-arm64-v8a.apk'), ({'application': 'fenix', 'platform': 'android-x86', 'date': '2022-11-14'}, - 'fenix-108.0b1.multi.android-x86.apk', + '2022-11-14-17-01-36-fenix-108.0b1.multi.android-x86.apk', 'fenix/nightly/2022/11/2022-11-14-17-01-36-fenix-108.0b1-android-x86/fenix-108.0b1.multi.android-x86.apk'), - ({'application': 'fenix', 'platform': 'android-arm64-v8a', 'date': '2022-11-14'}, - 'fenix-108.0b1.multi.android-arm64-v8a.apk', - 'fenix/nightly/2022/11/2022-11-14-17-01-36-fenix-108.0b1-android-arm64-v8a/fenix-108.0b1.multi.android-arm64-v8a.apk'), + ({'application': 'fenix', 'platform': 'android-armeabi-v7a', 'date': '2022-11-14'}, + '2022-11-14-17-01-36-fenix-108.0b1.multi.android-armeabi-v7a.apk', + 'fenix/nightly/2022/11/2022-11-14-17-01-36-fenix-108.0b1-android-armeabi-v7a/fenix-108.0b1.multi.android-armeabi-v7a.apk'), ({'application': 'fenix', 'platform': 'android-x86_64', 'date': '2022-11-14'}, - 'fenix-108.0b1.multi.android-x86_64.apk', + '2022-11-14-17-01-36-fenix-108.0b1.multi.android-x86_64.apk', 'fenix/nightly/2022/11/2022-11-14-17-01-36-fenix-108.0b1-android-x86_64/fenix-108.0b1.multi.android-x86_64.apk'), ]