From 597c90d0fdce3f0903cf3f60697ab7ccd939d081 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Sat, 6 Jan 2024 19:32:22 +0000 Subject: [PATCH 1/6] fix entrypoint and add dockerhub back in --- .github/workflows/code.yml | 165 ++++---- docs/reference/api.rst | 4 +- src/gphotos_sync/Main.py | 523 ------------------------- src/gphotos_sync/__main__.py | 527 +++++++++++++++++++++++++- tests/test_credentials/.gphotos.token | 2 +- tests/test_setup.py | 8 +- 6 files changed, 599 insertions(+), 630 deletions(-) delete mode 100644 src/gphotos_sync/Main.py diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index e6470a1b..00223e05 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -105,96 +105,8 @@ jobs: install_options: dist/*.whl - name: Test module --version works using the installed wheel - # If more than one module in src/ replace with module name to test - run: python -m $(ls src | head -1) --version - - container: - needs: [lint, dist, test] - runs-on: ubuntu-latest - - permissions: - contents: read - packages: write - - env: - TEST_TAG: "testing" - - steps: - - name: Checkout - uses: actions/checkout@v4 - - # image names must be all lower case - - name: Generate image repo name - run: echo IMAGE_REPOSITORY=ghcr.io/$(tr '[:upper:]' '[:lower:]' <<< "${{ github.repository }}") >> $GITHUB_ENV - - - name: Download wheel and lockfiles - uses: actions/download-artifact@v3 - with: - path: artifacts/ - - - name: Log in to GitHub Docker Registry - if: github.event_name != 'pull_request' - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - - - name: Build and export to Docker local cache - uses: docker/build-push-action@v5 - with: - # Note build-args, context, file, and target must all match between this - # step and the later build-push-action, otherwise the second build-push-action - # will attempt to build the image again - build-args: | - PIP_OPTIONS=-r lockfiles/requirements.txt dist/*.whl - context: artifacts/ - file: ./Dockerfile - target: runtime - load: true - tags: ${{ env.TEST_TAG }} - # If you have a long docker build (2+ minutes), uncomment the - # following to turn on caching. For short build times this - # makes it a little slower - #cache-from: type=gha - #cache-to: type=gha,mode=max - - - name: Test cli works in cached runtime image - run: docker run docker.io/library/${{ env.TEST_TAG }} --version - - - name: Create tags for publishing image - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.IMAGE_REPOSITORY }} - tags: | - type=ref,event=tag - type=raw,value=latest, enable=${{ github.ref_type == 'tag' }} - # type=edge,branch=main - # Add line above to generate image for every commit to given branch, - # and uncomment the end of if clause in next step - - - name: Push cached image to container registry - if: github.ref_type == 'tag' # || github.ref_name == 'main' - uses: docker/build-push-action@v5 - # This does not build the image again, it will find the image in the - # Docker cache and publish it - with: - # Note build-args, context, file, and target must all match between this - # step and the previous build-push-action, otherwise this step will - # attempt to build the image again - build-args: | - PIP_OPTIONS=-r lockfiles/requirements.txt dist/*.whl - context: artifacts/ - file: ./Dockerfile - target: runtime - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + # Check that the command line entry point works + run: gphotos-sync --version release: # upload to PyPI and make a release on every tag @@ -229,3 +141,76 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.PYPI_TOKEN }} + + make-container: + needs: [lint, dist, test] + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: checkout + uses: actions/checkout@v2 + + - uses: actions/download-artifact@v2 + with: + name: dist + path: dist + + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Log in to GitHub Docker Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ghcr.io/${{ github.repository }} + # github repo and dockerhub tag must match for this to work + ${{ github.repository }} + # all pull requests share a single tag 'pr' + tags: | + type=ref,event=branch + type=ref,event=tag + type=raw,value=latest,enable=${{ github.event_name != 'pull_request' }} + type=raw,value=pr + + # required for multi-arch build + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + + - name: Build runtime image + uses: docker/build-push-action@v3 + with: + file: .devcontainer/Dockerfile + context: . + platforms: linux/amd64,linux/arm/v7,linux/arm64/v8 + push: true + build-args: BASE=python:3.10-slim + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/docs/reference/api.rst b/docs/reference/api.rst index c1e95363..7392b413 100644 --- a/docs/reference/api.rst +++ b/docs/reference/api.rst @@ -13,10 +13,10 @@ This is the internal code reference for gphotos_sync Version number as calculated by setuptools_scm -.. automodule:: gphotos_sync.Main +.. automodule:: gphotos_sync.__main__ :members: - ``gphotos_sync.Main`` + ``gphotos_sync.__main__`` ----------------------------------------- .. automodule:: gphotos_sync.BaseMedia :members: diff --git a/src/gphotos_sync/Main.py b/src/gphotos_sync/Main.py deleted file mode 100644 index 170cb528..00000000 --- a/src/gphotos_sync/Main.py +++ /dev/null @@ -1,523 +0,0 @@ -# coding: utf8 -import logging -import os -import sys -from argparse import ArgumentParser, Namespace -from datetime import datetime -from pathlib import Path -from typing import Optional -from xmlrpc.client import DateTime - -from appdirs import AppDirs - -from gphotos_sync import Utils, __version__ -from gphotos_sync.authorize import Authorize -from gphotos_sync.Checks import do_check, get_check -from gphotos_sync.GoogleAlbumsSync import GoogleAlbumsSync -from gphotos_sync.GooglePhotosDownload import GooglePhotosDownload # type: ignore -from gphotos_sync.GooglePhotosIndex import GooglePhotosIndex -from gphotos_sync.LocalData import LocalData -from gphotos_sync.LocalFilesScan import LocalFilesScan -from gphotos_sync.Logging import setup_logging -from gphotos_sync.restclient import RestClient -from gphotos_sync.Settings import Settings - -if os.name == "nt": - import subprocess - - orig_Popen = subprocess.Popen - - class Popen_patch(subprocess.Popen): - def __init__(self, *args, **kargs): - startupinfo = subprocess.STARTUPINFO() - startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW - kargs["startupinfo"] = startupinfo - super().__init__(*args, **kargs) - - subprocess.Popen = Popen_patch # type: ignore -else: - import fcntl - -APP_NAME = "gphotos-sync" -log = logging.getLogger(__name__) - - -class GooglePhotosSyncMain: - def __init__(self): - self.data_store: LocalData - self.google_photos_client: RestClient - self.google_photos_idx: GooglePhotosIndex - self.google_photos_down: GooglePhotosDownload - self.google_albums_sync: GoogleAlbumsSync - self.local_files_scan: LocalFilesScan - self._start_date: Optional[DateTime] - self._end_date = Optional[DateTime] - - self.auth: Authorize - - try: - version_string = "version: {}, database schema version {}".format( - __version__, LocalData.VERSION - ) - except TypeError: - version_string = "(version not available)" - - parser = ArgumentParser( - epilog=version_string, description="Google Photos download tool" - ) - parser.add_argument( - "--version", - action="store_true", - help="report version and exit", - ) - parser.add_argument( - "root_folder", - help="root of the local folders to download into", - nargs="?", - ) - album_group = parser.add_mutually_exclusive_group() - album_group.add_argument( - "--album", - action="store", - help="only synchronize the contents of a single album. " - 'use quotes e.g. "album name" for album names with spaces', - ) - album_group.add_argument( - "--album-regex", - action="store", - metavar="REGEX", - help="""only synchronize albums that match regular expression. - regex is case insensitive and unanchored. e.g. to select two albums: - "^(a full album name|another full name)$" """, - ) - parser.add_argument( - "--log-level", - help="Set log level. Options: critical, error, warning, info, debug, trace. " - "trace logs all Google API calls to a file with suffix .trace", - default="warning", - ) - parser.add_argument( - "--logfile", - action="store", - help="full path to debug level logfile, default: /gphotos.log. " - "If a directory is specified then a unique filename will be " - "generated.", - ) - parser.add_argument( - "--compare-folder", - action="store", - help="DEPRECATED: root of the local folders to compare to the Photos Library", - ) - parser.add_argument( - "--favourites-only", - action="store_true", - help="only download media marked as favourite (star)", - ) - parser.add_argument( - "--flush-index", - action="store_true", - help="delete the index db, re-scan everything", - ) - parser.add_argument( - "--rescan", - action="store_true", - help="rescan entire library, ignoring last scan date. Use this if you " - "have added photos to the library that " - "predate the last sync, or you have deleted some of the local " - "files", - ) - parser.add_argument( - "--retry-download", - action="store_true", - help="check for the existence of files marked as already downloaded " - "and re-download any missing ones. Use " - "this if you have deleted some local files", - ) - parser.add_argument( - "--skip-video", action="store_true", help="skip video types in sync" - ) - parser.add_argument( - "--skip-shared-albums", - action="store_true", - help="skip albums that only appear in 'Sharing'", - ) - parser.add_argument( - "--album-date-by-first-photo", - action="store_true", - help="Make the album date the same as its earliest " - "photo. The default is its last photo", - ) - parser.add_argument( - "--start-date", - help="Set the earliest date of files to sync" "format YYYY-MM-DD", - default=None, - ) - parser.add_argument( - "--end-date", - help="Set the latest date of files to sync" "format YYYY-MM-DD", - default=None, - ) - parser.add_argument( - "--db-path", - help="Specify a pre-existing folder for the index database. " - "Defaults to the root of the local download folders", - default=None, - ) - parser.add_argument( - "--albums-path", - help="Specify a folder for the albums " - "Defaults to the 'albums' in the local download folders", - default="albums", - ) - parser.add_argument( - "--photos-path", - help="Specify a folder for the photo files. " - "Defaults to the 'photos' in the local download folders", - default="photos", - ) - parser.add_argument( - "--use-flat-path", - action="store_true", - help="Mandate use of a flat directory structure ('YYYY-MMM') and not " - "a nested one ('YYYY/MM') . ", - ) - parser.add_argument( - "--omit-album-date", - action="store_true", - help="Don't include year and month in album folder names.", - ) - parser.add_argument( - "--album-invert", - action="store_true", - help="Inverts the sorting direction of files within an album. " - "Default sorting is descending from newest to olders. " - "This causes it to be the other way around.", - ) - parser.add_argument("--new-token", action="store_true", help="Request new token") - parser.add_argument( - "--index-only", - action="store_true", - help="Only build the index of files in .gphotos.db - no downloads", - ) - parser.add_argument( - "--skip-index", - action="store_true", - help="Use index from previous run and start download immediately", - ) - parser.add_argument( - "--do-delete", - action="store_true", - help="""Remove local copies of files that were deleted. - Must be used with --flush-index since the deleted items must be removed - from the index""", - ) - parser.add_argument( - "--skip-files", - action="store_true", - help="Dont download files, just refresh the album links (for testing)", - ) - parser.add_argument( - "--skip-albums", action="store_true", help="Dont download albums (for testing)" - ) - parser.add_argument( - "--use-hardlinks", - action="store_true", - help="Use hardlinks instead of symbolic links in albums and comparison" - " folders", - ) - parser.add_argument( - "--no-album-index", - action="store_true", - help="only index the photos library - skip indexing of folder contents " - "(for testing)", - ) - parser.add_argument( - "--case-insensitive-fs", - action="store_true", - help="add this flag if your filesystem is case insensitive", - ) - parser.add_argument( - "--max-retries", - help="Set the number of retries on network timeout / failures", - type=int, - default=20, - ) - parser.add_argument( - "--max-threads", - help="Set the number of concurrent threads to use for parallel " - "download of media - reduce this number if network load is " - "excessive", - type=int, - default=20, - ) - parser.add_argument( - "--secret", - help="Path to client secret file (by default this is in the " - "application config directory)", - ) - parser.add_argument( - "--archived", - action="store_true", - help="Download media items that have been marked as archived", - ) - parser.add_argument( - "--progress", - action="store_true", - help="show progress of indexing and downloading in warning log", - ) - parser.add_argument( - "--max-filename", - help="Set the maxiumum filename length for target filesystem." - "This overrides the automatic detection.", - default=0, - ) - parser.add_argument( - "--ntfs", - action="store_true", - help="Declare that the target filesystem is ntfs (or ntfs like)." - "This overrides the automatic detection.", - ) - parser.add_argument( - "--month-format", - action="store", - metavar="FMT", - help="Configure the month/day formatting for the album folder/file " - "path (default: %%m%%d).", - default="%m%d", - ) - parser.add_argument( - "--path-format", - action="store", - metavar="FMT", - help="Configure the formatting for the album folder/file path. The " - "formatting can include up to 2 positionals arguments; `month` and " - "`album_name`. The default value is `{0} {1}`." - "When used with --use-flat-path option, it can include up to 3 " - "positionals arguments; `year`, `month` and `album_name`. In this case " - "the default value is `{0}-{1} {2}`", - default=None, - ) - parser.add_argument( - "--port", - help="Set the port for login flow redirect", - type=int, - default=8080, - ) - parser.add_argument( - "--image-timeout", - help="Set the time in seconds to wait for an image to download", - type=int, - default=60, - ) - parser.add_argument( - "--video-timeout", - help="Set the time in seconds to wait for a video to download", - type=int, - default=2000, - ) - parser.add_help = True - - def setup(self, args: Namespace, db_path: Path): - root_folder = Path(args.root_folder).absolute() - - compare_folder = None - if args.compare_folder: - compare_folder = Path(args.compare_folder).absolute() - app_dirs = AppDirs(APP_NAME) - - self.data_store = LocalData(db_path, args.flush_index) - - credentials_file = db_path / ".gphotos.token" - if args.secret: - secret_file = Path(args.secret) - else: - secret_file = Path(app_dirs.user_config_dir) / "client_secret.json" - if args.new_token and credentials_file.exists(): - credentials_file.unlink() - - scope = [ - "https://www.googleapis.com/auth/photoslibrary.readonly", - "https://www.googleapis.com/auth/photoslibrary.sharing", - ] - photos_api_url = ( - "https://photoslibrary.googleapis.com/$discovery" "/rest?version=v1" - ) - - self.auth = Authorize( - scope, - credentials_file, - secret_file, - int(args.max_retries), - port=args.port, - ) - self.auth.authorize() - - settings = Settings( - start_date=Utils.string_to_date(args.start_date), # type: ignore - end_date=Utils.string_to_date(args.end_date), # type: ignore - shared_albums=not args.skip_shared_albums, - album_index=not args.no_album_index, - use_start_date=args.album_date_by_first_photo, - album=args.album, - album_regex=args.album_regex, - favourites_only=args.favourites_only, - retry_download=args.retry_download, - case_insensitive_fs=args.case_insensitive_fs, - include_video=not args.skip_video, - rescan=args.rescan, - archived=args.archived, - photos_path=Path(args.photos_path), - albums_path=Path(args.albums_path), - use_flat_path=args.use_flat_path, - max_retries=int(args.max_retries), - max_threads=int(args.max_threads), - omit_album_date=args.omit_album_date, - album_invert=args.album_invert, - use_hardlinks=args.use_hardlinks, - progress=args.progress, - ntfs_override=args.ntfs, - month_format=args.month_format, - path_format=args.path_format, - image_timeout=args.image_timeout, - video_timeout=args.video_timeout, - ) - - self.google_photos_client = RestClient( - photos_api_url, self.auth.session # type: ignore - ) - self.google_photos_idx = GooglePhotosIndex( - self.google_photos_client, root_folder, self.data_store, settings - ) - self.google_photos_down = GooglePhotosDownload( - self.google_photos_client, root_folder, self.data_store, settings - ) - self.google_albums_sync = GoogleAlbumsSync( - self.google_photos_client, - root_folder, - self.data_store, - args.flush_index or args.retry_download or args.rescan, - settings, - ) - if args.compare_folder: - self.local_files_scan = LocalFilesScan( - root_folder, compare_folder, self.data_store # type: ignore - ) - - def do_sync(self, args: Namespace): - files_downloaded = 0 - with self.data_store: - if not args.skip_index: - if not args.skip_files and not args.album and not args.album_regex: - self.google_photos_idx.index_photos_media() - - if not args.index_only: - if not args.skip_files: - files_downloaded = self.google_photos_down.download_photo_media() - - if ( - not args.skip_albums - and not args.skip_index - and (files_downloaded > 0 or args.skip_files or args.rescan) - ) or (args.album is not None or args.album_regex is not None): - self.google_albums_sync.index_album_media() - # run download again to pick up files indexed in albums only - if not args.index_only: - if not args.skip_files: - files_downloaded = ( - self.google_photos_down.download_photo_media() - ) - - if not args.index_only: - if ( - not args.skip_albums - and (files_downloaded > 0 or args.skip_files or args.rescan) - or (args.album is not None or args.album_regex is not None) - ): - self.google_albums_sync.create_album_content_links() - if args.do_delete: - self.google_photos_idx.check_for_removed() - - if args.compare_folder: - if not args.skip_index: - self.local_files_scan.scan_local_files() - self.google_photos_idx.get_extra_meta() - self.local_files_scan.find_missing_gphotos() - - def start(self, args: Namespace): - self.do_sync(args) - - @staticmethod - def fs_checks(root_folder: Path, args): - Utils.minimum_date(root_folder) - # store the root folder filesystem checks globally for all to inspect - do_check(root_folder, int(args.max_filename), bool(args.ntfs)) - - # check if symlinks are supported - # NTFS supports symlinks, but is_symlink() fails - if not args.ntfs: - if not get_check().is_symlink: # type: ignore - args.skip_albums = True - - # check if file system is case sensitive - if not args.case_insensitive_fs: - if not get_check().is_case_sensitive: # type: ignore - args.case_insensitive_fs = True - - return args - - def main(self, test_args: Optional[dict] = None): - start_time = datetime.now() - args = self.parser.parse_args(test_args) # type: ignore - - if args.version: - print(__version__) - exit(0) - else: - if args.root_folder is None: - self.parser.print_help() - print("\nERROR: Please supply root_folder in which to save photos") - exit(1) - - root_folder = Path(args.root_folder).absolute() - db_path = Path(args.db_path) if args.db_path else root_folder - if not root_folder.exists(): - root_folder.mkdir(parents=True, mode=0o700) - - setup_logging(args.log_level, args.logfile, root_folder) - log.warning(f"gphotos-sync {__version__} {start_time}") - - args = self.fs_checks(root_folder, args) - - lock_file = db_path / "gphotos.lock" - fp = lock_file.open("w") - with fp: - try: - if os.name != "nt": - fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB) - except IOError: - log.warning("EXITING: database is locked") - sys.exit(0) - - log.info(self.version_string) - - # configure and launch - # noinspection PyBroadException - try: - self.setup(args, db_path) - self.start(args) - except KeyboardInterrupt: - log.error("User cancelled download") - log.debug("Traceback", exc_info=True) - exit(1) - except BaseException: - log.error("\nProcess failed.", exc_info=True) - exit(1) - finally: - log.warning("Done.") - - elapsed_time = datetime.now() - start_time - log.info("Elapsed time = %s", elapsed_time) - - -def main(): - GooglePhotosSyncMain().main() diff --git a/src/gphotos_sync/__main__.py b/src/gphotos_sync/__main__.py index 235986b2..170cb528 100644 --- a/src/gphotos_sync/__main__.py +++ b/src/gphotos_sync/__main__.py @@ -1,16 +1,523 @@ -from argparse import ArgumentParser +# coding: utf8 +import logging +import os +import sys +from argparse import ArgumentParser, Namespace +from datetime import datetime +from pathlib import Path +from typing import Optional +from xmlrpc.client import DateTime -from . import __version__ +from appdirs import AppDirs -__all__ = ["main"] +from gphotos_sync import Utils, __version__ +from gphotos_sync.authorize import Authorize +from gphotos_sync.Checks import do_check, get_check +from gphotos_sync.GoogleAlbumsSync import GoogleAlbumsSync +from gphotos_sync.GooglePhotosDownload import GooglePhotosDownload # type: ignore +from gphotos_sync.GooglePhotosIndex import GooglePhotosIndex +from gphotos_sync.LocalData import LocalData +from gphotos_sync.LocalFilesScan import LocalFilesScan +from gphotos_sync.Logging import setup_logging +from gphotos_sync.restclient import RestClient +from gphotos_sync.Settings import Settings +if os.name == "nt": + import subprocess -def main(args=None): - parser = ArgumentParser() - parser.add_argument("--version", action="version", version=__version__) - args = parser.parse_args(args) + orig_Popen = subprocess.Popen + class Popen_patch(subprocess.Popen): + def __init__(self, *args, **kargs): + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + kargs["startupinfo"] = startupinfo + super().__init__(*args, **kargs) -# test with: python -m gphotos_sync -if __name__ == "__main__": - main() + subprocess.Popen = Popen_patch # type: ignore +else: + import fcntl + +APP_NAME = "gphotos-sync" +log = logging.getLogger(__name__) + + +class GooglePhotosSyncMain: + def __init__(self): + self.data_store: LocalData + self.google_photos_client: RestClient + self.google_photos_idx: GooglePhotosIndex + self.google_photos_down: GooglePhotosDownload + self.google_albums_sync: GoogleAlbumsSync + self.local_files_scan: LocalFilesScan + self._start_date: Optional[DateTime] + self._end_date = Optional[DateTime] + + self.auth: Authorize + + try: + version_string = "version: {}, database schema version {}".format( + __version__, LocalData.VERSION + ) + except TypeError: + version_string = "(version not available)" + + parser = ArgumentParser( + epilog=version_string, description="Google Photos download tool" + ) + parser.add_argument( + "--version", + action="store_true", + help="report version and exit", + ) + parser.add_argument( + "root_folder", + help="root of the local folders to download into", + nargs="?", + ) + album_group = parser.add_mutually_exclusive_group() + album_group.add_argument( + "--album", + action="store", + help="only synchronize the contents of a single album. " + 'use quotes e.g. "album name" for album names with spaces', + ) + album_group.add_argument( + "--album-regex", + action="store", + metavar="REGEX", + help="""only synchronize albums that match regular expression. + regex is case insensitive and unanchored. e.g. to select two albums: + "^(a full album name|another full name)$" """, + ) + parser.add_argument( + "--log-level", + help="Set log level. Options: critical, error, warning, info, debug, trace. " + "trace logs all Google API calls to a file with suffix .trace", + default="warning", + ) + parser.add_argument( + "--logfile", + action="store", + help="full path to debug level logfile, default: /gphotos.log. " + "If a directory is specified then a unique filename will be " + "generated.", + ) + parser.add_argument( + "--compare-folder", + action="store", + help="DEPRECATED: root of the local folders to compare to the Photos Library", + ) + parser.add_argument( + "--favourites-only", + action="store_true", + help="only download media marked as favourite (star)", + ) + parser.add_argument( + "--flush-index", + action="store_true", + help="delete the index db, re-scan everything", + ) + parser.add_argument( + "--rescan", + action="store_true", + help="rescan entire library, ignoring last scan date. Use this if you " + "have added photos to the library that " + "predate the last sync, or you have deleted some of the local " + "files", + ) + parser.add_argument( + "--retry-download", + action="store_true", + help="check for the existence of files marked as already downloaded " + "and re-download any missing ones. Use " + "this if you have deleted some local files", + ) + parser.add_argument( + "--skip-video", action="store_true", help="skip video types in sync" + ) + parser.add_argument( + "--skip-shared-albums", + action="store_true", + help="skip albums that only appear in 'Sharing'", + ) + parser.add_argument( + "--album-date-by-first-photo", + action="store_true", + help="Make the album date the same as its earliest " + "photo. The default is its last photo", + ) + parser.add_argument( + "--start-date", + help="Set the earliest date of files to sync" "format YYYY-MM-DD", + default=None, + ) + parser.add_argument( + "--end-date", + help="Set the latest date of files to sync" "format YYYY-MM-DD", + default=None, + ) + parser.add_argument( + "--db-path", + help="Specify a pre-existing folder for the index database. " + "Defaults to the root of the local download folders", + default=None, + ) + parser.add_argument( + "--albums-path", + help="Specify a folder for the albums " + "Defaults to the 'albums' in the local download folders", + default="albums", + ) + parser.add_argument( + "--photos-path", + help="Specify a folder for the photo files. " + "Defaults to the 'photos' in the local download folders", + default="photos", + ) + parser.add_argument( + "--use-flat-path", + action="store_true", + help="Mandate use of a flat directory structure ('YYYY-MMM') and not " + "a nested one ('YYYY/MM') . ", + ) + parser.add_argument( + "--omit-album-date", + action="store_true", + help="Don't include year and month in album folder names.", + ) + parser.add_argument( + "--album-invert", + action="store_true", + help="Inverts the sorting direction of files within an album. " + "Default sorting is descending from newest to olders. " + "This causes it to be the other way around.", + ) + parser.add_argument("--new-token", action="store_true", help="Request new token") + parser.add_argument( + "--index-only", + action="store_true", + help="Only build the index of files in .gphotos.db - no downloads", + ) + parser.add_argument( + "--skip-index", + action="store_true", + help="Use index from previous run and start download immediately", + ) + parser.add_argument( + "--do-delete", + action="store_true", + help="""Remove local copies of files that were deleted. + Must be used with --flush-index since the deleted items must be removed + from the index""", + ) + parser.add_argument( + "--skip-files", + action="store_true", + help="Dont download files, just refresh the album links (for testing)", + ) + parser.add_argument( + "--skip-albums", action="store_true", help="Dont download albums (for testing)" + ) + parser.add_argument( + "--use-hardlinks", + action="store_true", + help="Use hardlinks instead of symbolic links in albums and comparison" + " folders", + ) + parser.add_argument( + "--no-album-index", + action="store_true", + help="only index the photos library - skip indexing of folder contents " + "(for testing)", + ) + parser.add_argument( + "--case-insensitive-fs", + action="store_true", + help="add this flag if your filesystem is case insensitive", + ) + parser.add_argument( + "--max-retries", + help="Set the number of retries on network timeout / failures", + type=int, + default=20, + ) + parser.add_argument( + "--max-threads", + help="Set the number of concurrent threads to use for parallel " + "download of media - reduce this number if network load is " + "excessive", + type=int, + default=20, + ) + parser.add_argument( + "--secret", + help="Path to client secret file (by default this is in the " + "application config directory)", + ) + parser.add_argument( + "--archived", + action="store_true", + help="Download media items that have been marked as archived", + ) + parser.add_argument( + "--progress", + action="store_true", + help="show progress of indexing and downloading in warning log", + ) + parser.add_argument( + "--max-filename", + help="Set the maxiumum filename length for target filesystem." + "This overrides the automatic detection.", + default=0, + ) + parser.add_argument( + "--ntfs", + action="store_true", + help="Declare that the target filesystem is ntfs (or ntfs like)." + "This overrides the automatic detection.", + ) + parser.add_argument( + "--month-format", + action="store", + metavar="FMT", + help="Configure the month/day formatting for the album folder/file " + "path (default: %%m%%d).", + default="%m%d", + ) + parser.add_argument( + "--path-format", + action="store", + metavar="FMT", + help="Configure the formatting for the album folder/file path. The " + "formatting can include up to 2 positionals arguments; `month` and " + "`album_name`. The default value is `{0} {1}`." + "When used with --use-flat-path option, it can include up to 3 " + "positionals arguments; `year`, `month` and `album_name`. In this case " + "the default value is `{0}-{1} {2}`", + default=None, + ) + parser.add_argument( + "--port", + help="Set the port for login flow redirect", + type=int, + default=8080, + ) + parser.add_argument( + "--image-timeout", + help="Set the time in seconds to wait for an image to download", + type=int, + default=60, + ) + parser.add_argument( + "--video-timeout", + help="Set the time in seconds to wait for a video to download", + type=int, + default=2000, + ) + parser.add_help = True + + def setup(self, args: Namespace, db_path: Path): + root_folder = Path(args.root_folder).absolute() + + compare_folder = None + if args.compare_folder: + compare_folder = Path(args.compare_folder).absolute() + app_dirs = AppDirs(APP_NAME) + + self.data_store = LocalData(db_path, args.flush_index) + + credentials_file = db_path / ".gphotos.token" + if args.secret: + secret_file = Path(args.secret) + else: + secret_file = Path(app_dirs.user_config_dir) / "client_secret.json" + if args.new_token and credentials_file.exists(): + credentials_file.unlink() + + scope = [ + "https://www.googleapis.com/auth/photoslibrary.readonly", + "https://www.googleapis.com/auth/photoslibrary.sharing", + ] + photos_api_url = ( + "https://photoslibrary.googleapis.com/$discovery" "/rest?version=v1" + ) + + self.auth = Authorize( + scope, + credentials_file, + secret_file, + int(args.max_retries), + port=args.port, + ) + self.auth.authorize() + + settings = Settings( + start_date=Utils.string_to_date(args.start_date), # type: ignore + end_date=Utils.string_to_date(args.end_date), # type: ignore + shared_albums=not args.skip_shared_albums, + album_index=not args.no_album_index, + use_start_date=args.album_date_by_first_photo, + album=args.album, + album_regex=args.album_regex, + favourites_only=args.favourites_only, + retry_download=args.retry_download, + case_insensitive_fs=args.case_insensitive_fs, + include_video=not args.skip_video, + rescan=args.rescan, + archived=args.archived, + photos_path=Path(args.photos_path), + albums_path=Path(args.albums_path), + use_flat_path=args.use_flat_path, + max_retries=int(args.max_retries), + max_threads=int(args.max_threads), + omit_album_date=args.omit_album_date, + album_invert=args.album_invert, + use_hardlinks=args.use_hardlinks, + progress=args.progress, + ntfs_override=args.ntfs, + month_format=args.month_format, + path_format=args.path_format, + image_timeout=args.image_timeout, + video_timeout=args.video_timeout, + ) + + self.google_photos_client = RestClient( + photos_api_url, self.auth.session # type: ignore + ) + self.google_photos_idx = GooglePhotosIndex( + self.google_photos_client, root_folder, self.data_store, settings + ) + self.google_photos_down = GooglePhotosDownload( + self.google_photos_client, root_folder, self.data_store, settings + ) + self.google_albums_sync = GoogleAlbumsSync( + self.google_photos_client, + root_folder, + self.data_store, + args.flush_index or args.retry_download or args.rescan, + settings, + ) + if args.compare_folder: + self.local_files_scan = LocalFilesScan( + root_folder, compare_folder, self.data_store # type: ignore + ) + + def do_sync(self, args: Namespace): + files_downloaded = 0 + with self.data_store: + if not args.skip_index: + if not args.skip_files and not args.album and not args.album_regex: + self.google_photos_idx.index_photos_media() + + if not args.index_only: + if not args.skip_files: + files_downloaded = self.google_photos_down.download_photo_media() + + if ( + not args.skip_albums + and not args.skip_index + and (files_downloaded > 0 or args.skip_files or args.rescan) + ) or (args.album is not None or args.album_regex is not None): + self.google_albums_sync.index_album_media() + # run download again to pick up files indexed in albums only + if not args.index_only: + if not args.skip_files: + files_downloaded = ( + self.google_photos_down.download_photo_media() + ) + + if not args.index_only: + if ( + not args.skip_albums + and (files_downloaded > 0 or args.skip_files or args.rescan) + or (args.album is not None or args.album_regex is not None) + ): + self.google_albums_sync.create_album_content_links() + if args.do_delete: + self.google_photos_idx.check_for_removed() + + if args.compare_folder: + if not args.skip_index: + self.local_files_scan.scan_local_files() + self.google_photos_idx.get_extra_meta() + self.local_files_scan.find_missing_gphotos() + + def start(self, args: Namespace): + self.do_sync(args) + + @staticmethod + def fs_checks(root_folder: Path, args): + Utils.minimum_date(root_folder) + # store the root folder filesystem checks globally for all to inspect + do_check(root_folder, int(args.max_filename), bool(args.ntfs)) + + # check if symlinks are supported + # NTFS supports symlinks, but is_symlink() fails + if not args.ntfs: + if not get_check().is_symlink: # type: ignore + args.skip_albums = True + + # check if file system is case sensitive + if not args.case_insensitive_fs: + if not get_check().is_case_sensitive: # type: ignore + args.case_insensitive_fs = True + + return args + + def main(self, test_args: Optional[dict] = None): + start_time = datetime.now() + args = self.parser.parse_args(test_args) # type: ignore + + if args.version: + print(__version__) + exit(0) + else: + if args.root_folder is None: + self.parser.print_help() + print("\nERROR: Please supply root_folder in which to save photos") + exit(1) + + root_folder = Path(args.root_folder).absolute() + db_path = Path(args.db_path) if args.db_path else root_folder + if not root_folder.exists(): + root_folder.mkdir(parents=True, mode=0o700) + + setup_logging(args.log_level, args.logfile, root_folder) + log.warning(f"gphotos-sync {__version__} {start_time}") + + args = self.fs_checks(root_folder, args) + + lock_file = db_path / "gphotos.lock" + fp = lock_file.open("w") + with fp: + try: + if os.name != "nt": + fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB) + except IOError: + log.warning("EXITING: database is locked") + sys.exit(0) + + log.info(self.version_string) + + # configure and launch + # noinspection PyBroadException + try: + self.setup(args, db_path) + self.start(args) + except KeyboardInterrupt: + log.error("User cancelled download") + log.debug("Traceback", exc_info=True) + exit(1) + except BaseException: + log.error("\nProcess failed.", exc_info=True) + exit(1) + finally: + log.warning("Done.") + + elapsed_time = datetime.now() - start_time + log.info("Elapsed time = %s", elapsed_time) + + +def main(): + GooglePhotosSyncMain().main() diff --git a/tests/test_credentials/.gphotos.token b/tests/test_credentials/.gphotos.token index 7cf84e76..35c0ad9e 100644 --- a/tests/test_credentials/.gphotos.token +++ b/tests/test_credentials/.gphotos.token @@ -1 +1 @@ -{"access_token": "ya29.a0AfB_byAAnSEyVYmlxFIXG7A7lH8jkrSrd4RSnunI4rYyHfzdG-ZjSQiFxo5-OHV5rBgEnHoFMKK1Rj5PKxLCXVCEr-0yMiJs61wCCfzg36-UWQegZTV3kBotQH_Qk7HTkZcphp11fW5lbc3POClog-aOCfJLIbOe2UfA63MaCgYKAZsSARMSFQHGX2Mi3joogOa2oHnOerg_ojwUQg0174", "expires_in": 3599, "scope": ["https://www.googleapis.com/auth/photoslibrary.sharing", "https://www.googleapis.com/auth/photoslibrary.readonly"], "token_type": "Bearer", "expires_at": 1703971275.0828457, "refresh_token": "1//03CEqAzsnP-8PCgYIARAAGAMSNwF-L9Irz4_ilhRw0HIwVImT4gTCUPlV8YaCTYQiIjD4juWOI5eQh_-Rzh9nTmBND0jliOnabq4"} \ No newline at end of file +{"access_token": "ya29.a0AfB_byAreGK4x3LmWMoEdyxynFhxmVdluIku4_wrxknJNMivmNheLMhz1UT2bya7Oq_sKzgljfBXgEvVl5ODSq6XRu53doa0zOzeKJiEbl9KbdqqTlbVy2r5HV_FJ2weHCzihyhDkUtcFduPSsLQNFffLjZubPLvTGDtdN0aCgYKAXISARMSFQHGX2MiqhOMZleHPcJE2ubZkU7jvw0174", "expires_in": 3599, "scope": ["https://www.googleapis.com/auth/photoslibrary.sharing", "https://www.googleapis.com/auth/photoslibrary.readonly"], "token_type": "Bearer", "expires_at": 1704573379.582039, "refresh_token": "1//03CEqAzsnP-8PCgYIARAAGAMSNwF-L9Irz4_ilhRw0HIwVImT4gTCUPlV8YaCTYQiIjD4juWOI5eQh_-Rzh9nTmBND0jliOnabq4"} \ No newline at end of file diff --git a/tests/test_setup.py b/tests/test_setup.py index 085b1101..9c00b30e 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -4,9 +4,9 @@ from appdirs import AppDirs -from gphotos_sync import Main +from gphotos_sync import __main__ +from gphotos_sync.__main__ import GooglePhotosSyncMain from gphotos_sync.Checks import do_check -from gphotos_sync.Main import GooglePhotosSyncMain # if we are debugging requests library is too noisy logging.getLogger("requests").setLevel(logging.WARNING) @@ -23,8 +23,8 @@ class SetupDbAndCredentials: def __init__(self): # set up the test account credentials - Main.APP_NAME = "gphotos-sync-test" - app_dirs = AppDirs(Main.APP_NAME) + __main__.APP_NAME = "gphotos-sync-test" + app_dirs = AppDirs(__main__.APP_NAME) self.test_folder = Path(__file__).absolute().parent / "test_credentials" user_data = Path(app_dirs.user_data_dir) if not user_data.exists(): From 1c7f19328fa47119c51fd0cdcc3ab2b85532fcf4 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Sat, 6 Jan 2024 20:29:23 +0000 Subject: [PATCH 2/6] update version discovery --- src/gphotos_sync/__init__.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/gphotos_sync/__init__.py b/src/gphotos_sync/__init__.py index 5875d991..647efe3d 100644 --- a/src/gphotos_sync/__init__.py +++ b/src/gphotos_sync/__init__.py @@ -1,14 +1,6 @@ -try: - # Use live version from git - from setuptools_scm import get_version +from importlib.metadata import version # noqa - # Warning: If the install is nested to the same depth, this will always succeed - tmp_version = get_version(root="../../", relative_to=__file__) - del get_version -except (ImportError, LookupError): - # Use installed version - from ._version import version as __version__ -else: - __version__ = tmp_version +__version__ = version("gphotos-sync") +del version __all__ = ["__version__"] From ac3b55d33bd9ad8f155df30e982a911049cea548 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Sat, 6 Jan 2024 20:30:38 +0000 Subject: [PATCH 3/6] make container python 3.12 --- .github/workflows/code.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 00223e05..e0dac4b9 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -5,7 +5,7 @@ on: pull_request: env: # The target python version, which must match the Dockerfile version - CONTAINER_PYTHON: "3.11" + CONTAINER_PYTHON: "3.12" jobs: lint: From 3aad19ba8095948d3cdd2c10122310ae1f49faa1 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Sat, 6 Jan 2024 20:55:23 +0000 Subject: [PATCH 4/6] fix version when loaded as a module --- src/gphotos_sync/__main__.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/gphotos_sync/__main__.py b/src/gphotos_sync/__main__.py index 170cb528..bd8cd577 100644 --- a/src/gphotos_sync/__main__.py +++ b/src/gphotos_sync/__main__.py @@ -383,7 +383,8 @@ def setup(self, args: Namespace, db_path: Path): ) self.google_photos_client = RestClient( - photos_api_url, self.auth.session # type: ignore + photos_api_url, + self.auth.session, # type: ignore ) self.google_photos_idx = GooglePhotosIndex( self.google_photos_client, root_folder, self.data_store, settings @@ -400,7 +401,9 @@ def setup(self, args: Namespace, db_path: Path): ) if args.compare_folder: self.local_files_scan = LocalFilesScan( - root_folder, compare_folder, self.data_store # type: ignore + root_folder, + compare_folder, + self.data_store, # type: ignore ) def do_sync(self, args: Namespace): @@ -521,3 +524,7 @@ def main(self, test_args: Optional[dict] = None): def main(): GooglePhotosSyncMain().main() + + +if __name__ == "__main__": + main() From fc467adac36eb827c8e059f3aa0e2cf443b6fdad Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Sat, 6 Jan 2024 21:03:33 +0000 Subject: [PATCH 5/6] linting --- src/gphotos_sync/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gphotos_sync/__main__.py b/src/gphotos_sync/__main__.py index bd8cd577..3d5ffe34 100644 --- a/src/gphotos_sync/__main__.py +++ b/src/gphotos_sync/__main__.py @@ -402,7 +402,7 @@ def setup(self, args: Namespace, db_path: Path): if args.compare_folder: self.local_files_scan = LocalFilesScan( root_folder, - compare_folder, + compare_folder, # type: ignore self.data_store, # type: ignore ) From 7bd16d02d4b6ef6acf33f99c480c4f2e94d9607f Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Sat, 6 Jan 2024 21:22:23 +0000 Subject: [PATCH 6/6] REALLY make the container python 3.12 --- .devcontainer/Dockerfile | 2 +- .devcontainer/local_build.sh | 2 +- .github/workflows/code.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 7c17f271..66c54fce 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -4,7 +4,7 @@ # container. The devcontainer should be rootful and use podman or docker # with user namespaces. -ARG BASE="mcr.microsoft.com/vscode/devcontainers/python:0-3.10-bullseye" +ARG BASE="mcr.microsoft.com/devcontainers/python:dev-3.12-bullseye" FROM ${BASE} as base # use root to pin where the packages will install diff --git a/.devcontainer/local_build.sh b/.devcontainer/local_build.sh index 106fec34..58482fca 100755 --- a/.devcontainer/local_build.sh +++ b/.devcontainer/local_build.sh @@ -18,5 +18,5 @@ echo building $container_name ... # run the build with required build-args for a runtime build cd ${THIS_DIR} ln -s ../dist . -docker build --build-arg BASE=python:3.10-slim -t $container_name .. --file ./Dockerfile +docker build --build-arg BASE=python:3.12-slim -t $container_name .. --file ./Dockerfile unlink dist diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index e0dac4b9..5b25fd94 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -209,7 +209,7 @@ jobs: context: . platforms: linux/amd64,linux/arm/v7,linux/arm64/v8 push: true - build-args: BASE=python:3.10-slim + build-args: BASE=python:3.12-slim tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=local,src=/tmp/.buildx-cache