Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add --ignore to ignore user #546

Merged
merged 7 commits into from
Nov 24, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,9 @@ The following options are common between both the `archive` and `download` comma
- Can be specified multiple times
- Disables certain modules from being used
- See [Disabling Modules](#disabling-modules) for more information and a list of module names
- `--ignore-user`
- This will add a user to ignore
- Can be specified multiple times
- `--include-id-file`
- This will add any submission with the IDs in the files provided
- Can be specified multiple times
Expand Down
1 change: 1 addition & 0 deletions bdfr/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
click.option('--authenticate', is_flag=True, default=None),
click.option('--config', type=str, default=None),
click.option('--disable-module', multiple=True, default=None, type=str),
click.option('--ignore-user', type=str, multiple=True, default=None),
click.option('--include-id-file', multiple=True, default=None),
click.option('--log', type=str, default=None),
click.option('--saved', is_flag=True, default=None),
Expand Down
5 changes: 5 additions & 0 deletions bdfr/archiver.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@ def __init__(self, args: Configuration):
def download(self):
for generator in self.reddit_lists:
for submission in generator:
if submission.author.name in self.args.ignore_user:
logger.debug(
f'Submission {submission.id} in {submission.subreddit.display_name} skipped'
f' due to {submission.author.name} being an ignored user')
continue
logger.debug(f'Attempting to archive submission {submission.id}')
self.write_entry(submission)

Expand Down
1 change: 1 addition & 0 deletions bdfr/configuration.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ def __init__(self):
self.exclude_id_file = []
self.file_scheme: str = '{REDDITOR}_{TITLE}_{POSTID}'
self.folder_scheme: str = '{SUBREDDIT}'
self.ignore_user = []
self.include_id_file = []
self.limit: Optional[int] = None
self.link: list[str] = []
Expand Down
5 changes: 5 additions & 0 deletions bdfr/downloader.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,11 @@ def _download_submission(self, submission: praw.models.Submission):
elif submission.subreddit.display_name.lower() in self.args.skip_subreddit:
logger.debug(f'Submission {submission.id} in {submission.subreddit.display_name} in skip list')
return
elif submission.author.name in self.args.ignore_user:
logger.debug(
f'Submission {submission.id} in {submission.subreddit.display_name} skipped'
f' due to {submission.author.name} being an ignored user')
return
elif not isinstance(submission, praw.models.Submission):
logger.warning(f'{submission.id} is not a submission')
return
Expand Down
6 changes: 3 additions & 3 deletions bdfr/site_downloaders/download_factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from bdfr.site_downloaders.base_downloader import BaseDownloader
from bdfr.site_downloaders.direct import Direct
from bdfr.site_downloaders.erome import Erome
from bdfr.site_downloaders.fallback_downloaders.youtubedl_fallback import YoutubeDlFallback
from bdfr.site_downloaders.fallback_downloaders.ytdlp_fallback import YtdlpFallback
from bdfr.site_downloaders.gallery import Gallery
from bdfr.site_downloaders.gfycat import Gfycat
from bdfr.site_downloaders.imgur import Imgur
Expand Down Expand Up @@ -49,8 +49,8 @@ def pull_lever(url: str) -> Type[BaseDownloader]:
return PornHub
elif re.match(r'vidble\.com', sanitised_url):
return Vidble
elif YoutubeDlFallback.can_handle_link(sanitised_url):
return YoutubeDlFallback
elif YtdlpFallback.can_handle_link(sanitised_url):
return YtdlpFallback
else:
raise NotADownloadableLinkError(f'No downloader module exists for url {url}')

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@
logger = logging.getLogger(__name__)


class YoutubeDlFallback(BaseFallbackDownloader, Youtube):
class YtdlpFallback(BaseFallbackDownloader, Youtube):
def __init__(self, post: Submission):
super(YoutubeDlFallback, self).__init__(post)
super(YtdlpFallback, self).__init__(post)

def find_resources(self, authenticator: Optional[SiteAuthenticator] = None) -> list[Resource]:
out = Resource(
Expand All @@ -31,7 +31,7 @@ def find_resources(self, authenticator: Optional[SiteAuthenticator] = None) -> l
@staticmethod
def can_handle_link(url: str) -> bool:
try:
attributes = YoutubeDlFallback.get_video_attributes(url)
attributes = YtdlpFallback.get_video_attributes(url)
except NotADownloadableLinkError:
return False
if attributes:
Expand Down
15 changes: 15 additions & 0 deletions tests/integration_tests/test_archive_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -106,3 +106,18 @@ def test_cli_archive_long(test_args: list[str], tmp_path: Path):
result = runner.invoke(cli, test_args)
assert result.exit_code == 0
assert re.search(r'Writing entry .*? to file in .*? format', result.output)


@pytest.mark.online
@pytest.mark.reddit
@pytest.mark.skipif(not does_test_config_exist, reason='A test config file is required for integration tests')
@pytest.mark.parametrize('test_args', (
['--ignore-user', 'ArjanEgges', '-l', 'm3hxzd'],
))
def test_cli_archive_ignore_user(test_args: list[str], tmp_path: Path):
runner = CliRunner()
test_args = create_basic_args_for_archive_runner(test_args, tmp_path)
result = runner.invoke(cli, test_args)
assert result.exit_code == 0
assert 'being an ignored user' in result.output
assert 'Attempting to archive submission' not in result.output
15 changes: 15 additions & 0 deletions tests/integration_tests/test_download_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -337,3 +337,18 @@ def test_cli_download_include_id_file(tmp_path: Path):
result = runner.invoke(cli, test_args)
assert result.exit_code == 0
assert 'Downloaded submission' in result.output


@pytest.mark.online
@pytest.mark.reddit
@pytest.mark.skipif(not does_test_config_exist, reason='A test config file is required for integration tests')
@pytest.mark.parametrize('test_args', (
['--ignore-user', 'ArjanEgges', '-l', 'm3hxzd'],
))
def test_cli_download_ignore_user(test_args: list[str], tmp_path: Path):
runner = CliRunner()
test_args = create_basic_args_for_download_runner(test_args, tmp_path)
result = runner.invoke(cli, test_args)
assert result.exit_code == 0
assert 'Downloaded submission' not in result.output
assert 'being an ignored user' in result.output
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

from bdfr.exceptions import NotADownloadableLinkError
from bdfr.resource import Resource
from bdfr.site_downloaders.fallback_downloaders.youtubedl_fallback import YoutubeDlFallback
from bdfr.site_downloaders.fallback_downloaders.ytdlp_fallback import YtdlpFallback


@pytest.mark.online
Expand All @@ -17,7 +17,7 @@
('https://milesmatrix.bandcamp.com/album/la-boum/', False),
))
def test_can_handle_link(test_url: str, expected: bool):
result = YoutubeDlFallback.can_handle_link(test_url)
result = YtdlpFallback.can_handle_link(test_url)
assert result == expected


Expand All @@ -27,7 +27,7 @@ def test_can_handle_link(test_url: str, expected: bool):
))
def test_info_extraction_bad(test_url: str):
with pytest.raises(NotADownloadableLinkError):
YoutubeDlFallback.get_video_attributes(test_url)
YtdlpFallback.get_video_attributes(test_url)


@pytest.mark.online
Expand All @@ -41,7 +41,7 @@ def test_info_extraction_bad(test_url: str):
def test_find_resources(test_url: str, expected_hash: str):
test_submission = MagicMock()
test_submission.url = test_url
downloader = YoutubeDlFallback(test_submission)
downloader = YtdlpFallback(test_submission)
resources = downloader.find_resources()
assert len(resources) == 1
assert isinstance(resources[0], Resource)
Expand Down
10 changes: 5 additions & 5 deletions tests/site_downloaders/test_download_factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from bdfr.site_downloaders.direct import Direct
from bdfr.site_downloaders.download_factory import DownloadFactory
from bdfr.site_downloaders.erome import Erome
from bdfr.site_downloaders.fallback_downloaders.youtubedl_fallback import YoutubeDlFallback
from bdfr.site_downloaders.fallback_downloaders.ytdlp_fallback import YtdlpFallback
from bdfr.site_downloaders.gallery import Gallery
from bdfr.site_downloaders.gfycat import Gfycat
from bdfr.site_downloaders.imgur import Imgur
Expand Down Expand Up @@ -42,10 +42,10 @@
('https://i.imgur.com/3SKrQfK.jpg?1', Direct),
('https://dynasty-scans.com/system/images_images/000/017/819/original/80215103_p0.png?1612232781', Direct),
('https://m.imgur.com/a/py3RW0j', Imgur),
('https://v.redd.it/9z1dnk3xr5k61', YoutubeDlFallback),
('https://streamable.com/dt46y', YoutubeDlFallback),
('https://vimeo.com/channels/31259/53576664', YoutubeDlFallback),
('http://video.pbs.org/viralplayer/2365173446/', YoutubeDlFallback),
('https://v.redd.it/9z1dnk3xr5k61', YtdlpFallback),
('https://streamable.com/dt46y', YtdlpFallback),
('https://vimeo.com/channels/31259/53576664', YtdlpFallback),
('http://video.pbs.org/viralplayer/2365173446/', YtdlpFallback),
('https://www.pornhub.com/view_video.php?viewkey=ph5a2ee0461a8d0', PornHub),
))
def test_factory_lever_good(test_submission_url: str, expected_class: BaseDownloader, reddit_instance: praw.Reddit):
Expand Down
8 changes: 5 additions & 3 deletions tests/test_file_name_formatter.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
# coding=utf-8

import platform
import sys
import unittest.mock
from datetime import datetime
from pathlib import Path
Expand All @@ -14,7 +15,7 @@
from bdfr.file_name_formatter import FileNameFormatter
from bdfr.resource import Resource
from bdfr.site_downloaders.base_downloader import BaseDownloader
from bdfr.site_downloaders.fallback_downloaders.youtubedl_fallback import YoutubeDlFallback
from bdfr.site_downloaders.fallback_downloaders.ytdlp_fallback import YtdlpFallback


@pytest.fixture()
Expand Down Expand Up @@ -213,6 +214,7 @@ def test_preserve_id_append_when_shortening(test_filename: str, test_ending: str
assert len(str(result)) <= FileNameFormatter.find_max_path_length()


@pytest.mark.skipif(sys.platform == 'win32', reason='Test broken on windows github')
def test_shorten_filename_real(submission: MagicMock, tmp_path: Path):
submission.title = 'A' * 500
submission.author.name = 'test'
Expand Down Expand Up @@ -402,8 +404,8 @@ def test_windows_max_path(tmp_path: Path):
@pytest.mark.online
@pytest.mark.reddit
@pytest.mark.parametrize(('test_reddit_id', 'test_downloader', 'expected_names'), (
('gphmnr', YoutubeDlFallback, {'He has a lot to say today.mp4'}),
('d0oir2', YoutubeDlFallback, {"Crunk's finest moment. Welcome to the new subreddit!.mp4"}),
('gphmnr', YtdlpFallback, {'He has a lot to say today.mp4'}),
('d0oir2', YtdlpFallback, {"Crunk's finest moment. Welcome to the new subreddit!.mp4"}),
))
def test_name_submission(
test_reddit_id: str,
Expand Down