From ef9c792f54cd59b50a6e874c2e643a80bcaf8ee6 Mon Sep 17 00:00:00 2001 From: Daniel Widerin Date: Sat, 9 Jan 2016 12:41:30 +0100 Subject: [PATCH 01/37] Add GitLab repo sync and webhook support --- media/css/core.css | 6 + readthedocs/oauth/services/__init__.py | 6 +- readthedocs/oauth/services/gitlab.py | 222 +++++++++++++++++++++++++ readthedocs/settings/base.py | 1 + 4 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 readthedocs/oauth/services/gitlab.py diff --git a/media/css/core.css b/media/css/core.css index 6e2060514e9..a2aa0501ff0 100644 --- a/media/css/core.css +++ b/media/css/core.css @@ -672,6 +672,12 @@ a.socialaccount-provider.github:before { content: "\f09b"; } +div.project-import-remote form.import-connect-gitlab button:before, +a.socialaccount-provider.gitlab:before { + font-family: FontAwesome; + content: "\f1d3"; +} + div.project-import-remote form.import-connect-bitbucket button:before, a.socialaccount-provider.bitbucket:before, a.socialaccount-provider.bitbucket_oauth2:before { diff --git a/readthedocs/oauth/services/__init__.py b/readthedocs/oauth/services/__init__.py index 37ff9ca7851..5acfb6126b8 100644 --- a/readthedocs/oauth/services/__init__.py +++ b/readthedocs/oauth/services/__init__.py @@ -10,4 +10,8 @@ getattr(settings, 'OAUTH_BITBUCKET_SERVICE', 'readthedocs.oauth.services.bitbucket.BitbucketService')) -registry = [GitHubService, BitbucketService] +GitLabService = import_by_path( + getattr(settings, 'OAUTH_GITLAB_SERVICE', + 'readthedocs.oauth.services.gitlab.GitLabService')) + +registry = [GitHubService, BitbucketService, GitLabService] diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py new file mode 100644 index 00000000000..0e66b409ff2 --- /dev/null +++ b/readthedocs/oauth/services/gitlab.py @@ -0,0 +1,222 @@ +"""OAuth utility functions""" + +import logging +import json +import re + +from django.conf import settings +from requests.exceptions import RequestException +from allauth.socialaccount.models import SocialToken +from allauth.socialaccount.providers.gitlab.views import GitLabOAuth2Adapter + +from readthedocs.restapi.client import api + +from ..models import RemoteOrganization, RemoteRepository +from .base import Service, DEFAULT_PRIVACY_LEVEL + + +log = logging.getLogger(__name__) + + +class GitLabService(Service): + """Provider service for GitLab""" + + adapter = GitLabOAuth2Adapter + url_pattern = re.compile(re.escape(adapter.provider_base_url)) + + def paginate(self, url, **kwargs): + """Combines return from GitLab pagination. GitLab uses + LinkHeaders, see: http://www.w3.org/wiki/LinkHeader + + :param url: start url to get the data from. + :param kwargs: optional parameters passed to .get() method + + See http://doc.gitlab.com/ce/api/README.html#pagination + """ + resp = self.get_session().get(url, data=kwargs) + result = resp.json() + next_url = resp.links.get('next', {}).get('url') + if next_url: + result.extend(self.paginate(next_url, **kwargs)) + return result + + def sync(self): + """Sync repositories from GitLab API""" + org = None + repos = self.paginate( + '{url}/api/v3/projects'.format(url=self.adapter.provider_base_url), + per_page=100, + order_by='path', + sort='asc' + ) + for repo in repos: + # Skip archived repositories + if repo.get('archived', False): + continue + if not org or org.slug != repo['namespace']['id']: + org = self.create_organization(repo['namespace']) + + self.create_repository(repo, organization=org) + + def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL, + organization=None): + """Update or create a repository from GitLab API response + + :param fields: dictionary of response data from API + :param privacy: privacy level to support + :param organization: remote organization to associate with + :type organization: RemoteOrganization + :rtype: RemoteRepository + """ + # See: http://doc.gitlab.com/ce/api/projects.html#projects + repo_is_public = fields['visibility_level'] == 20 + + def is_owned_by(owner_id): + return self.account.extra_data['id'] == owner_id + + if privacy == 'private' or (repo_is_public and privacy == 'public'): + try: + repo = RemoteRepository.objects.get( + full_name=fields['name'], + users=self.user, + account=self.account, + ) + except RemoteRepository.DoesNotExist: + repo = RemoteRepository.objects.create( + full_name=fields['name'], + account=self.account, + ) + repo.users.add(self.user) + + if repo.organization and repo.organization != organization: + log.debug('Not importing %s because mismatched orgs' % + fields['name']) + return None + else: + repo.organization = organization + repo.name = fields['name'] + repo.full_name = fields['name_with_namespace'] + repo.description = fields['description'] + repo.ssh_url = fields['ssh_url_to_repo'] + repo.html_url = fields['web_url'] + repo.private = not fields['public'] + repo.clone_url = fields['http_url_to_repo'] + repo.admin = not repo_is_public + if not repo.admin and organization: + repo.admin = is_owned_by(fields['owner']['id']) + repo.vcs = 'git' + repo.account = self.account + repo.avatar_url = fields.get('avatar_url') + repo.json = json.dumps(fields) + repo.save() + return repo + else: + log.info( + 'Not importing {0} because mismatched type: public={1}'.format( + fields['name_with_namespace'], + fields['public'], + ) + ) + + def create_organization(self, fields): + """Update or create remote organization from GitLab API response + + :param fields: dictionary response of data from API + :rtype: RemoteOrganization + """ + try: + organization = RemoteOrganization.objects.get( + slug=fields.get('path'), + users=self.user, + account=self.account, + ) + except RemoteOrganization.DoesNotExist: + organization = RemoteOrganization.objects.create( + slug=fields.get('path'), + account=self.account, + ) + organization.users.add(self.user) + + organization.name = fields.get('name') + organization.account = self.account + organization.url = '{url}/{path}'.format( + url=self.adapter.provider_base_url, path=fields.get('path') + ) + if fields.get('avatar'): + organization.avatar_url = '{url}/{avatar}'.format( + url=self.adapter.provider_base_url, + avatar=fields['avatar']['url'], + ) + organization.json = json.dumps(fields) + organization.save() + return organization + + def setup_webhook(self, project): + """Set up GitLab project webhook for project + + :param project: project to set up webhook for + :type project: Project + :returns: boolean based on webhook set up success + :rtype: bool + """ + session = self.get_session() + + # See: http://doc.gitlab.com/ce/api/projects.html#add-project-hook + data = json.dumps({ + 'id': 'readthedocs', + 'push_events': True, + 'issues_events': False, + 'merge_requests_events': False, + 'note_events': False, + 'tag_push_events': True, + 'url': 'https://{0}/gitlab'.format(settings.PRODUCTION_DOMAIN), + }) + resp = None + try: + repositories = RemoteRepository.objects.filter( + clone_url=project.vcs_repo().repo_url + ) + assert repositories + repo_id = repositories[0].get_serialized()['id'] + resp = session.post( + '{url}/api/v3/projects/{repo_id}/hooks'.format( + url=self.adapter.provider_base_url, + repo_id=repo_id, + ), + data=data, + headers={'content-type': 'application/json'} + ) + if resp.status_code == 201: + log.info('GitLab webhook creation successful for project: %s', # noqa + project) + return True + except (AssertionError, RemoteRepository.DoesNotExist) as ex: + log.error('GitLab remote repository not found', exc_info=ex) + except RequestException as ex: + pass + else: + ex = False + + log.error('GitLab webhook creation failed for project: %s', # noqa + project, exc_info=ex) + + @classmethod + def get_token_for_project(cls, project, force_local=False): + """Get access token for project by iterating over project users""" + # TODO why does this only target GitHub? + if not getattr(settings, 'ALLOW_PRIVATE_REPOS', False): + return None + token = None + try: + if getattr(settings, 'DONT_HIT_DB', True) and not force_local: + token = api.project(project.pk).token().get()['token'] + else: + for user in project.users.all(): + tokens = SocialToken.objects.filter( + account__user=user, + app__provider=cls.adapter.provider_id) + if tokens.exists(): + token = tokens[0].token + except Exception: + log.error('Failed to get token for user', exc_info=True) + return token diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 437af36602d..e64bd1daf37 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -108,6 +108,7 @@ def INSTALLED_APPS(self): # noqa 'allauth.account', 'allauth.socialaccount', 'allauth.socialaccount.providers.github', + 'allauth.socialaccount.providers.gitlab', 'allauth.socialaccount.providers.bitbucket', 'allauth.socialaccount.providers.bitbucket_oauth2', ] From cf5c47c45ab322f00c6cadc9efbbfa5a0ce97cb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Knecht?= Date: Thu, 16 Jun 2016 07:57:18 +0200 Subject: [PATCH 02/37] oauth: gitlab: use unicode whenever format is used --- readthedocs/oauth/services/gitlab.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 0e66b409ff2..8566546d800 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -44,7 +44,7 @@ def sync(self): """Sync repositories from GitLab API""" org = None repos = self.paginate( - '{url}/api/v3/projects'.format(url=self.adapter.provider_base_url), + u'{url}/api/v3/projects'.format(url=self.adapter.provider_base_url), per_page=100, order_by='path', sort='asc' @@ -112,7 +112,7 @@ def is_owned_by(owner_id): return repo else: log.info( - 'Not importing {0} because mismatched type: public={1}'.format( + u'Not importing {0} because mismatched type: public={1}'.format( fields['name_with_namespace'], fields['public'], ) @@ -139,11 +139,11 @@ def create_organization(self, fields): organization.name = fields.get('name') organization.account = self.account - organization.url = '{url}/{path}'.format( + organization.url = u'{url}/{path}'.format( url=self.adapter.provider_base_url, path=fields.get('path') ) if fields.get('avatar'): - organization.avatar_url = '{url}/{avatar}'.format( + organization.avatar_url = u'{url}/{avatar}'.format( url=self.adapter.provider_base_url, avatar=fields['avatar']['url'], ) @@ -169,7 +169,7 @@ def setup_webhook(self, project): 'merge_requests_events': False, 'note_events': False, 'tag_push_events': True, - 'url': 'https://{0}/gitlab'.format(settings.PRODUCTION_DOMAIN), + 'url': u'https://{0}/gitlab'.format(settings.PRODUCTION_DOMAIN), }) resp = None try: @@ -179,7 +179,7 @@ def setup_webhook(self, project): assert repositories repo_id = repositories[0].get_serialized()['id'] resp = session.post( - '{url}/api/v3/projects/{repo_id}/hooks'.format( + u'{url}/api/v3/projects/{repo_id}/hooks'.format( url=self.adapter.provider_base_url, repo_id=repo_id, ), From 2356d845c3dda3755618f91852419e4dccbf5c73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Knecht?= Date: Mon, 20 Jun 2016 06:47:48 +0200 Subject: [PATCH 03/37] oauth: gitlab: prevent duplicate entries after sync --- readthedocs/oauth/services/gitlab.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 8566546d800..5c21b737090 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -77,13 +77,13 @@ def is_owned_by(owner_id): if privacy == 'private' or (repo_is_public and privacy == 'public'): try: repo = RemoteRepository.objects.get( - full_name=fields['name'], + full_name=fields['name_with_namespace'], users=self.user, account=self.account, ) except RemoteRepository.DoesNotExist: repo = RemoteRepository.objects.create( - full_name=fields['name'], + full_name=fields['name_with_namespace'], account=self.account, ) repo.users.add(self.user) From 7113e94b568ef28aaff9e7b1c7e18304142c76e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Knecht?= Date: Mon, 20 Jun 2016 06:50:29 +0200 Subject: [PATCH 04/37] oauth: gitlab: avoid KeyError when owner is missing --- readthedocs/oauth/services/gitlab.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 5c21b737090..3475021971d 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -102,7 +102,7 @@ def is_owned_by(owner_id): repo.private = not fields['public'] repo.clone_url = fields['http_url_to_repo'] repo.admin = not repo_is_public - if not repo.admin and organization: + if not repo.admin and 'owner' in fields: repo.admin = is_owned_by(fields['owner']['id']) repo.vcs = 'git' repo.account = self.account From 5d7d6c1649864b9eff9580a7336f81a8288a4164 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Knecht?= Date: Mon, 20 Jun 2016 06:53:36 +0200 Subject: [PATCH 05/37] oauth: gitlab: setup_webhook must return a tuple --- readthedocs/oauth/services/gitlab.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 3475021971d..33dcc1efc4b 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -189,7 +189,7 @@ def setup_webhook(self, project): if resp.status_code == 201: log.info('GitLab webhook creation successful for project: %s', # noqa project) - return True + return (True, resp) except (AssertionError, RemoteRepository.DoesNotExist) as ex: log.error('GitLab remote repository not found', exc_info=ex) except RequestException as ex: @@ -199,6 +199,7 @@ def setup_webhook(self, project): log.error('GitLab webhook creation failed for project: %s', # noqa project, exc_info=ex) + return (False, resp) @classmethod def get_token_for_project(cls, project, force_local=False): From f9df35e3cba0a71961057d18091ce70ed4fd98a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Knecht?= Date: Mon, 20 Jun 2016 07:08:23 +0200 Subject: [PATCH 06/37] oauth: gitlab: set a default avatar if none is returned --- media/images/fa-bookmark.svg | 1 + media/images/fa-users.svg | 1 + readthedocs/oauth/services/gitlab.py | 17 ++++++++++++++--- 3 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 media/images/fa-bookmark.svg create mode 100644 media/images/fa-users.svg diff --git a/media/images/fa-bookmark.svg b/media/images/fa-bookmark.svg new file mode 100644 index 00000000000..0daca66b42b --- /dev/null +++ b/media/images/fa-bookmark.svg @@ -0,0 +1 @@ + diff --git a/media/images/fa-users.svg b/media/images/fa-users.svg new file mode 100644 index 00000000000..7da2bd570b4 --- /dev/null +++ b/media/images/fa-users.svg @@ -0,0 +1 @@ + diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 33dcc1efc4b..e772c17fa5e 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -8,6 +8,7 @@ from requests.exceptions import RequestException from allauth.socialaccount.models import SocialToken from allauth.socialaccount.providers.gitlab.views import GitLabOAuth2Adapter +from urlparse import urljoin from readthedocs.restapi.client import api @@ -23,6 +24,10 @@ class GitLabService(Service): adapter = GitLabOAuth2Adapter url_pattern = re.compile(re.escape(adapter.provider_base_url)) + default_avatar = { + 'repo': urljoin(settings.MEDIA_URL, 'images/fa-bookmark.svg'), + 'org': urljoin(settings.MEDIA_URL, 'images/fa-users.svg'), + } def paginate(self, url, **kwargs): """Combines return from GitLab pagination. GitLab uses @@ -106,7 +111,10 @@ def is_owned_by(owner_id): repo.admin = is_owned_by(fields['owner']['id']) repo.vcs = 'git' repo.account = self.account - repo.avatar_url = fields.get('avatar_url') + owner = fields.get('owner') or {} + repo.avatar_url = (fields.get('avatar_url') or + owner.get('avatar_url') or + self.default_avatar['repo']) repo.json = json.dumps(fields) repo.save() return repo @@ -142,11 +150,14 @@ def create_organization(self, fields): organization.url = u'{url}/{path}'.format( url=self.adapter.provider_base_url, path=fields.get('path') ) - if fields.get('avatar'): + avatar = fields.get('avatar') or {} + if avatar.get('url'): organization.avatar_url = u'{url}/{avatar}'.format( url=self.adapter.provider_base_url, - avatar=fields['avatar']['url'], + avatar=avatar.get('url'), ) + else: + organization.avatar_url = self.default_avatar['org'] organization.json = json.dumps(fields) organization.save() return organization From e4adb98e6c18a9de1636ea3e5bdbe05444d25eac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Knecht?= Date: Mon, 20 Jun 2016 08:32:46 +0200 Subject: [PATCH 07/37] oauth: gitlab: use SSH url if repo is private --- readthedocs/oauth/services/gitlab.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index e772c17fa5e..0fe4b997342 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -105,8 +105,9 @@ def is_owned_by(owner_id): repo.ssh_url = fields['ssh_url_to_repo'] repo.html_url = fields['web_url'] repo.private = not fields['public'] - repo.clone_url = fields['http_url_to_repo'] repo.admin = not repo_is_public + repo.clone_url = (repo.admin and repo.ssh_url or + fields['http_url_to_repo']) if not repo.admin and 'owner' in fields: repo.admin = is_owned_by(fields['owner']['id']) repo.vcs = 'git' From da7471f553fe821c2c02761bf82ecad9f9341102 Mon Sep 17 00:00:00 2001 From: Timo Steidle Date: Fri, 5 Aug 2016 11:33:32 +0200 Subject: [PATCH 08/37] Added tests for the GitLabService --- readthedocs/rtd_tests/tests/test_oauth.py | 135 +++++++++++++++++++++- 1 file changed, 133 insertions(+), 2 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_oauth.py b/readthedocs/rtd_tests/tests/test_oauth.py index 9b879ca760f..16433942d64 100644 --- a/readthedocs/rtd_tests/tests/test_oauth.py +++ b/readthedocs/rtd_tests/tests/test_oauth.py @@ -1,12 +1,13 @@ from django.test import TestCase from django.contrib.auth.models import User -from allauth.socialaccount.models import SocialToken +from mock import Mock +from readthedocs.projects import constants from readthedocs.projects.models import Project -from readthedocs.oauth.services import GitHubService, BitbucketService from readthedocs.oauth.models import RemoteRepository, RemoteOrganization +from readthedocs.oauth.services import GitHubService, BitbucketService, GitLabService class GitHubOAuthTests(TestCase): @@ -279,3 +280,133 @@ def test_import_with_no_token(self): '''User without a Bitbucket SocialToken does not return a service''' services = BitbucketService.for_user(self.user) self.assertEqual(services, []) + + +class GitLabOAuthTests(TestCase): + + fixtures = ["eric", "test_data"] + + repo_response_data = { + 'forks_count': 12, + 'container_registry_enabled': None, + 'web_url': 'https://gitlab.com/testorga/testrepo', + 'wiki_enabled': True, + 'public_builds': True, + 'id': 2, + 'merge_requests_enabled': True, + 'archived': False, + 'snippets_enabled': False, + 'http_url_to_repo': 'https://gitlab.com/testorga/testrepo.git', + 'namespace': { + 'share_with_group_lock': False, + 'name': 'Test Orga', + 'created_at': '2014-07-11T13:38:53.510Z', + 'description': '', + 'updated_at': '2014-07-11T13:38:53.510Z', + 'avatar': { + 'url': None + }, + 'path': 'testorga', + 'visibility_level': 20, + 'id': 5, + 'owner_id': None + }, + 'star_count': 0, + 'avatar_url': 'http://placekitten.com/50/50', + 'issues_enabled': True, + 'path_with_namespace': 'testorga/testrepo', + 'public': True, + 'description': 'Test Repo', + 'default_branch': 'master', + 'ssh_url_to_repo': 'git@gitlab.com:testorga/testrepo.git', + 'path': 'testrepo', + 'visibility_level': 20, + 'permissions': { + 'group_access': { + 'notification_level': 3, + 'access_level': 40 + }, + 'project_access': None + }, + 'open_issues_count': 2, + 'last_activity_at': '2016-03-01T09:22:34.344Z', + 'name': 'testrepo', + 'name_with_namespace': 'testorga / testrepo', + 'created_at': '2015-11-02T13:52:42.821Z', + 'builds_enabled': True, + 'creator_id': 5, + 'shared_runners_enabled': True, + 'tag_list': [] + } + + def setUp(self): + self.client.login(username='eric', password='test') + self.user = User.objects.get(pk=1) + self.project = Project.objects.get(slug='pip') + self.org = RemoteOrganization.objects.create(slug='testorga', json='') + self.privacy = self.project.version_privacy_level + self.service = GitLabService(user=self.user, account=None) + + def get_private_repo_data(self): + """Manipulate repo response data to get private repo data.""" + data = self.repo_response_data.copy() + data.update({ + 'visibility_level': 10, + 'public': False, + }) + return data + + def test_make_project_pass(self): + repo = self.service.create_repository( + self.repo_response_data, organization=self.org, privacy=self.privacy) + self.assertIsInstance(repo, RemoteRepository) + self.assertEqual(repo.name, 'testrepo') + self.assertEqual(repo.full_name, 'testorga / testrepo') + self.assertEqual(repo.description, 'Test Repo') + self.assertEqual(repo.avatar_url, 'http://placekitten.com/50/50') + self.assertIn(self.user, repo.users.all()) + self.assertEqual(repo.organization, self.org) + self.assertEqual(repo.clone_url, 'https://gitlab.com/testorga/testrepo.git') + self.assertEqual(repo.ssh_url, 'git@gitlab.com:testorga/testrepo.git') + self.assertEqual(repo.html_url, 'https://gitlab.com/testorga/testrepo') + + def test_make_private_project_fail(self): + repo = self.service.create_repository( + self.get_private_repo_data(), organization=self.org, privacy=self.privacy) + self.assertIsNone(repo) + + def test_make_private_project_success(self): + repo = self.service.create_repository( + self.get_private_repo_data(), organization=self.org, privacy=constants.PRIVATE) + self.assertIsInstance(repo, RemoteRepository) + self.assertTrue(repo.private, True) + + def test_make_organization(self): + org = self.service.create_organization(self.repo_response_data['namespace']) + self.assertIsInstance(org, RemoteOrganization) + self.assertEqual(org.slug, 'testorga') + self.assertEqual(org.name, 'Test Orga') + self.assertEqual(org.avatar_url, '/media/images/fa-users.svg') + self.assertEqual(org.url, 'https://gitlab.com/testorga') + + def test_sync_skip_archived_repo(self): + data = self.repo_response_data + data['archived'] = True + create_repo_mock = Mock() + create_orga_mock = Mock() + setattr(self.service, 'paginate', Mock(return_value=[data])) + setattr(self.service, 'create_repository', create_repo_mock) + setattr(self.service, 'create_organization', create_orga_mock) + self.service.sync() + self.assertFalse(create_repo_mock.called) + self.assertFalse(create_orga_mock.called) + + def test_sync_create_repo_and_orga(self): + create_repo_mock = Mock() + create_orga_mock = Mock(return_value=self.org) + setattr(self.service, 'paginate', Mock(return_value=[self.repo_response_data])) + setattr(self.service, 'create_repository', create_repo_mock) + setattr(self.service, 'create_organization', create_orga_mock) + self.service.sync() + create_repo_mock.assert_called_once_with(self.repo_response_data, organization=self.org) + create_orga_mock.assert_called_once_with(self.repo_response_data['namespace']) From aec69413b1d57bdb9773c7b9c434f78c5fbcf7ff Mon Sep 17 00:00:00 2001 From: Timo Steidle Date: Fri, 5 Aug 2016 11:48:40 +0200 Subject: [PATCH 09/37] Added some documentation for the Gitlab integration --- docs/features.rst | 2 +- docs/webhooks.rst | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/features.rst b/docs/features.rst index b4dae131a58..6f1de62f0f4 100644 --- a/docs/features.rst +++ b/docs/features.rst @@ -13,7 +13,7 @@ More information can be found in the :doc:`vcs` page. Auto-updating ------------- -The :doc:`webhooks` page talks about the different ways you can ping RTD to let us know your project has been updated. We have official support for GitHub, and anywhere else we have a generic post-commit hook that allows you to POST to a URL to get your documentation built. +The :doc:`webhooks` page talks about the different ways you can ping RTD to let us know your project has been updated. We have official support for GitHub, Bitbucket and GitLab, and anywhere else we have a generic post-commit hook that allows you to POST to a URL to get your documentation built. Internationalization -------------------- diff --git a/docs/webhooks.rst b/docs/webhooks.rst index e912e15f529..926cc7512c3 100644 --- a/docs/webhooks.rst +++ b/docs/webhooks.rst @@ -9,7 +9,7 @@ worked with push knows, pushing a doc update to your repo and watching it get updated within seconds is an awesome feeling. GitHub ---------- +------ If your project is hosted on GitHub, you can easily add a hook that will rebuild your docs whenever you push updates: @@ -27,7 +27,7 @@ If you ever need to manually set the webhook on GitHub, you can point it at ``https://readthedocs.org/github``. Bitbucket ------------ +--------- If your project is hosted on Bitbucket, you can easily add a hook that will rebuild your docs whenever you push updates: @@ -40,6 +40,17 @@ your docs whenever you push updates: If you ever need to manually set the webhook on Bitbucket, you can point it at ``https://readthedocs.org/bitbucket``. +GitLab +------ + +If your project is hosted on GitLab, you can manually set the webhook on Gitlab and +point it at ``https://readthedocs.org/gitlab``: + +* Click the settings icon for your project +* Select "Webhooks" +* Enter the above URL, select "Push events" and "Enable SSL verification" +* Click "Add Webhook" + Others ------ From 299a2b54258ab589c1c08ec266f2cc93b2e5fd51 Mon Sep 17 00:00:00 2001 From: Timo Steidle Date: Fri, 5 Aug 2016 14:11:42 +0200 Subject: [PATCH 10/37] Improved url_pattern of the GitLabService to support private repositories --- readthedocs/oauth/services/gitlab.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 0fe4b997342..f29f7ec1de5 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -4,11 +4,15 @@ import json import re +try: + from urlparse import urljoin, urlparse +except ImportError: + from urllib.parse import urljoin, urlparse + from django.conf import settings from requests.exceptions import RequestException from allauth.socialaccount.models import SocialToken from allauth.socialaccount.providers.gitlab.views import GitLabOAuth2Adapter -from urlparse import urljoin from readthedocs.restapi.client import api @@ -23,7 +27,9 @@ class GitLabService(Service): """Provider service for GitLab""" adapter = GitLabOAuth2Adapter - url_pattern = re.compile(re.escape(adapter.provider_base_url)) + # Just use the network location to determine if it's a GitLab project + # because private repos have another base url, eg. git@gitlab.example.com + url_pattern = re.compile(re.escape(urlparse(adapter.provider_base_url).netloc)) default_avatar = { 'repo': urljoin(settings.MEDIA_URL, 'images/fa-bookmark.svg'), 'org': urljoin(settings.MEDIA_URL, 'images/fa-users.svg'), From 8a1084844ce28503d9c028b3c7b0f39cf897e16f Mon Sep 17 00:00:00 2001 From: Timo Steidle Date: Fri, 5 Aug 2016 14:35:39 +0200 Subject: [PATCH 11/37] Fixed linting error --- readthedocs/oauth/services/gitlab.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index f29f7ec1de5..44c66d7b549 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -4,10 +4,7 @@ import json import re -try: - from urlparse import urljoin, urlparse -except ImportError: - from urllib.parse import urljoin, urlparse +from urlparse import urljoin, urlparse from django.conf import settings from requests.exceptions import RequestException From 97a4fb15f0144e04aaa797fe2c30382629ccf70f Mon Sep 17 00:00:00 2001 From: Timo Steidle Date: Wed, 10 Aug 2016 11:30:28 +0200 Subject: [PATCH 12/37] Simplified exception handling --- readthedocs/oauth/services/gitlab.py | 30 ++++++++++++---------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 44c66d7b549..69edf5c5313 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -187,12 +187,14 @@ def setup_webhook(self, project): 'url': u'https://{0}/gitlab'.format(settings.PRODUCTION_DOMAIN), }) resp = None + repositories = RemoteRepository.objects.filter(clone_url=project.vcs_repo().repo_url) + + if not repositories.exists(): + log.error('GitLab remote repository not found') + return False, resp + + repo_id = repositories[0].get_serialized()['id'] try: - repositories = RemoteRepository.objects.filter( - clone_url=project.vcs_repo().repo_url - ) - assert repositories - repo_id = repositories[0].get_serialized()['id'] resp = session.post( u'{url}/api/v3/projects/{repo_id}/hooks'.format( url=self.adapter.provider_base_url, @@ -202,19 +204,13 @@ def setup_webhook(self, project): headers={'content-type': 'application/json'} ) if resp.status_code == 201: - log.info('GitLab webhook creation successful for project: %s', # noqa - project) - return (True, resp) - except (AssertionError, RemoteRepository.DoesNotExist) as ex: - log.error('GitLab remote repository not found', exc_info=ex) - except RequestException as ex: - pass + log.info('GitLab webhook creation successful for project: %s', project) + return True, resp + except RequestException: + log.error('GitLab webhook creation failed for project: %s', project, exc_info=True) else: - ex = False - - log.error('GitLab webhook creation failed for project: %s', # noqa - project, exc_info=ex) - return (False, resp) + log.error('GitLab webhook creation failed for project: %s', project) + return False, resp @classmethod def get_token_for_project(cls, project, force_local=False): From dff4d7db45630b1899bc1c2a0d1b8a01cc914c95 Mon Sep 17 00:00:00 2001 From: Timo Steidle Date: Wed, 10 Aug 2016 11:31:25 +0200 Subject: [PATCH 13/37] Use PRODUCTION_DOMAIN as webhook id --- readthedocs/oauth/services/gitlab.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 69edf5c5313..f7418df57b7 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -178,7 +178,7 @@ def setup_webhook(self, project): # See: http://doc.gitlab.com/ce/api/projects.html#add-project-hook data = json.dumps({ - 'id': 'readthedocs', + 'id': settings.PRODUCTION_DOMAIN, 'push_events': True, 'issues_events': False, 'merge_requests_events': False, From 4fbe6bbb142dc814f463829d7525bbd2a87acd35 Mon Sep 17 00:00:00 2001 From: Timo Steidle Date: Wed, 10 Aug 2016 11:36:09 +0200 Subject: [PATCH 14/37] Make imports python3 compatible --- readthedocs/oauth/services/gitlab.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index f7418df57b7..6e668f7c036 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -4,7 +4,10 @@ import json import re -from urlparse import urljoin, urlparse +try: + from urlparse import urljoin, urlparse +except ImportError: + from urllib.parse import urljoin, urlparse # noqa from django.conf import settings from requests.exceptions import RequestException From 50285fe2dd4dc1a77620ccd2b9b62b98f3619054 Mon Sep 17 00:00:00 2001 From: Timo Steidle Date: Wed, 10 Aug 2016 12:01:35 +0200 Subject: [PATCH 15/37] Use the GitLab project ID, not the PRODUCTION_DOMAIN --- readthedocs/oauth/services/gitlab.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 6e668f7c036..bed20faf3fd 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -178,10 +178,17 @@ def setup_webhook(self, project): :rtype: bool """ session = self.get_session() + resp = None + repositories = RemoteRepository.objects.filter(clone_url=project.vcs_repo().repo_url) + if not repositories.exists(): + log.error('GitLab remote repository not found') + return False, resp + + repo_id = repositories[0].get_serialized()['id'] # See: http://doc.gitlab.com/ce/api/projects.html#add-project-hook data = json.dumps({ - 'id': settings.PRODUCTION_DOMAIN, + 'id': repo_id, 'push_events': True, 'issues_events': False, 'merge_requests_events': False, @@ -189,14 +196,7 @@ def setup_webhook(self, project): 'tag_push_events': True, 'url': u'https://{0}/gitlab'.format(settings.PRODUCTION_DOMAIN), }) - resp = None - repositories = RemoteRepository.objects.filter(clone_url=project.vcs_repo().repo_url) - if not repositories.exists(): - log.error('GitLab remote repository not found') - return False, resp - - repo_id = repositories[0].get_serialized()['id'] try: resp = session.post( u'{url}/api/v3/projects/{repo_id}/hooks'.format( From 6c8d1c30c0c9dd326dd46c0de39a77358b1d5510 Mon Sep 17 00:00:00 2001 From: Daniel Widerin Date: Wed, 19 Oct 2016 18:12:32 +0200 Subject: [PATCH 16/37] Update urls as mentioned in comments --- readthedocs/oauth/services/gitlab.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index bed20faf3fd..f1b94d3a587 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -42,7 +42,7 @@ def paginate(self, url, **kwargs): :param url: start url to get the data from. :param kwargs: optional parameters passed to .get() method - See http://doc.gitlab.com/ce/api/README.html#pagination + See https://docs.gitlab.com/ce/api/README.html#pagination """ resp = self.get_session().get(url, data=kwargs) result = resp.json() @@ -79,7 +79,7 @@ def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL, :type organization: RemoteOrganization :rtype: RemoteRepository """ - # See: http://doc.gitlab.com/ce/api/projects.html#projects + # See: https://docs.gitlab.com/ce/api/projects.html#projects repo_is_public = fields['visibility_level'] == 20 def is_owned_by(owner_id): From 0cb1ed5fab920fb5ddac25ca6e256a1537f3f36e Mon Sep 17 00:00:00 2001 From: Daniel Widerin Date: Wed, 19 Oct 2016 18:17:47 +0200 Subject: [PATCH 17/37] Update webhook docs as mentioned in comments --- docs/webhooks.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/webhooks.rst b/docs/webhooks.rst index 926cc7512c3..1220f8b641d 100644 --- a/docs/webhooks.rst +++ b/docs/webhooks.rst @@ -43,8 +43,8 @@ you can point it at ``https://readthedocs.org/bitbucket``. GitLab ------ -If your project is hosted on GitLab, you can manually set the webhook on Gitlab and -point it at ``https://readthedocs.org/gitlab``: +If your project is hosted on GitLab.com, you can manually set the webhook on +Gitlab.com and point it at ``https://readthedocs.org/gitlab``: * Click the settings icon for your project * Select "Webhooks" From c33b6914c18f34a9e63bd2ddb2c1b5f4337e28d2 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 27 Nov 2017 16:11:10 -0500 Subject: [PATCH 18/37] Style changes --- .flake8 | 2 +- .isort.cfg | 2 +- readthedocs/oauth/services/gitlab.py | 95 ++++++++++++++++------------ 3 files changed, 56 insertions(+), 43 deletions(-) diff --git a/.flake8 b/.flake8 index d514d9f38fb..88163430fb6 100644 --- a/.flake8 +++ b/.flake8 @@ -1,3 +1,3 @@ [flake8] -ignore = E125,D100,D101,D102,D105,D200,D211,P101,FI15,FI16,FI12,FI11,FI17,FI50,FI53,FI54 +ignore = E125,D100,D101,D102,D105,D200,D211,P101,FI15,FI16,FI12,FI11,FI17,FI50,FI53,FI54,MQ101,T000 max-line-length = 80 diff --git a/.isort.cfg b/.isort.cfg index 8dcef42b370..ffb33a90030 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -5,6 +5,6 @@ multi_line_output=4 default_section=FIRSTPARTY known_readthedocs=readthedocs known_readthedocsinc=readthedocsinc -known_third_party=celery,stripe,requests,pytz,builtins,django,annoying,mock +known_third_party=celery,stripe,requests,pytz,builtins,django,annoying,mock,allauth sections=FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER add_imports=from __future__ import division, from __future__ import print_function, from __future__ import unicode_literals diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index f1b94d3a587..b15ad19d03a 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -1,48 +1,53 @@ -"""OAuth utility functions""" +# -*- coding: utf-8 -*- +"""OAuth utility functions.""" + +from __future__ import division, print_function, unicode_literals -import logging import json +import logging import re -try: - from urlparse import urljoin, urlparse -except ImportError: - from urllib.parse import urljoin, urlparse # noqa - -from django.conf import settings -from requests.exceptions import RequestException from allauth.socialaccount.models import SocialToken from allauth.socialaccount.providers.gitlab.views import GitLabOAuth2Adapter +from django.conf import settings +from requests.exceptions import RequestException from readthedocs.restapi.client import api from ..models import RemoteOrganization, RemoteRepository -from .base import Service, DEFAULT_PRIVACY_LEVEL +from .base import DEFAULT_PRIVACY_LEVEL, Service +try: + from urlparse import urljoin, urlparse +except ImportError: + from urllib.parse import urljoin, urlparse # noqa log = logging.getLogger(__name__) class GitLabService(Service): - """Provider service for GitLab""" + + """Provider service for GitLab.""" adapter = GitLabOAuth2Adapter # Just use the network location to determine if it's a GitLab project # because private repos have another base url, eg. git@gitlab.example.com - url_pattern = re.compile(re.escape(urlparse(adapter.provider_base_url).netloc)) + url_pattern = re.compile( + re.escape(urlparse(adapter.provider_base_url).netloc)) default_avatar = { 'repo': urljoin(settings.MEDIA_URL, 'images/fa-bookmark.svg'), - 'org': urljoin(settings.MEDIA_URL, 'images/fa-users.svg'), + 'org': urljoin(settings.MEDIA_URL, 'images/fa-users.svg'), } def paginate(self, url, **kwargs): - """Combines return from GitLab pagination. GitLab uses - LinkHeaders, see: http://www.w3.org/wiki/LinkHeader + """ + Combine return from GitLab pagination. GitLab uses LinkHeaders. + + See: http://www.w3.org/wiki/LinkHeader. + See: https://docs.gitlab.com/ce/api/README.html#pagination :param url: start url to get the data from. :param kwargs: optional parameters passed to .get() method - - See https://docs.gitlab.com/ce/api/README.html#pagination """ resp = self.get_session().get(url, data=kwargs) result = resp.json() @@ -52,13 +57,13 @@ def paginate(self, url, **kwargs): return result def sync(self): - """Sync repositories from GitLab API""" + """Sync repositories from GitLab API.""" org = None repos = self.paginate( u'{url}/api/v3/projects'.format(url=self.adapter.provider_base_url), per_page=100, order_by='path', - sort='asc' + sort='asc', ) for repo in repos: # Skip archived repositories @@ -69,9 +74,10 @@ def sync(self): self.create_repository(repo, organization=org) - def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL, - organization=None): - """Update or create a repository from GitLab API response + def create_repository( + self, fields, privacy=DEFAULT_PRIVACY_LEVEL, organization=None): + """ + Update or create a repository from GitLab API response. :param fields: dictionary of response data from API :param privacy: privacy level to support @@ -100,8 +106,10 @@ def is_owned_by(owner_id): repo.users.add(self.user) if repo.organization and repo.organization != organization: - log.debug('Not importing %s because mismatched orgs' % - fields['name']) + log.debug( + 'Not importing %s because mismatched orgs', + fields['name'], + ) return None else: repo.organization = organization @@ -112,16 +120,16 @@ def is_owned_by(owner_id): repo.html_url = fields['web_url'] repo.private = not fields['public'] repo.admin = not repo_is_public - repo.clone_url = (repo.admin and repo.ssh_url or - fields['http_url_to_repo']) + repo.clone_url = ( + repo.admin and repo.ssh_url or fields['http_url_to_repo']) if not repo.admin and 'owner' in fields: repo.admin = is_owned_by(fields['owner']['id']) repo.vcs = 'git' repo.account = self.account owner = fields.get('owner') or {} - repo.avatar_url = (fields.get('avatar_url') or - owner.get('avatar_url') or - self.default_avatar['repo']) + repo.avatar_url = ( + fields.get('avatar_url') or owner.get('avatar_url') or + self.default_avatar['repo']) repo.json = json.dumps(fields) repo.save() return repo @@ -130,11 +138,11 @@ def is_owned_by(owner_id): u'Not importing {0} because mismatched type: public={1}'.format( fields['name_with_namespace'], fields['public'], - ) - ) + )) def create_organization(self, fields): - """Update or create remote organization from GitLab API response + """ + Update or create remote organization from GitLab API response. :param fields: dictionary response of data from API :rtype: RemoteOrganization @@ -155,8 +163,7 @@ def create_organization(self, fields): organization.name = fields.get('name') organization.account = self.account organization.url = u'{url}/{path}'.format( - url=self.adapter.provider_base_url, path=fields.get('path') - ) + url=self.adapter.provider_base_url, path=fields.get('path')) avatar = fields.get('avatar') or {} if avatar.get('url'): organization.avatar_url = u'{url}/{avatar}'.format( @@ -170,7 +177,8 @@ def create_organization(self, fields): return organization def setup_webhook(self, project): - """Set up GitLab project webhook for project + """ + Set up GitLab project webhook for project. :param project: project to set up webhook for :type project: Project @@ -179,7 +187,8 @@ def setup_webhook(self, project): """ session = self.get_session() resp = None - repositories = RemoteRepository.objects.filter(clone_url=project.vcs_repo().repo_url) + repositories = RemoteRepository.objects.filter( + clone_url=project.vcs_repo().repo_url) if not repositories.exists(): log.error('GitLab remote repository not found') @@ -204,21 +213,25 @@ def setup_webhook(self, project): repo_id=repo_id, ), data=data, - headers={'content-type': 'application/json'} + headers={'content-type': 'application/json'}, ) if resp.status_code == 201: - log.info('GitLab webhook creation successful for project: %s', project) + log.info( + 'GitLab webhook creation successful for project: %s', + project) return True, resp except RequestException: - log.error('GitLab webhook creation failed for project: %s', project, exc_info=True) + log.error( + 'GitLab webhook creation failed for project: %s', project, + exc_info=True) else: log.error('GitLab webhook creation failed for project: %s', project) return False, resp @classmethod def get_token_for_project(cls, project, force_local=False): - """Get access token for project by iterating over project users""" - # TODO why does this only target GitHub? + """Get access token for project by iterating over project users.""" + # TODO: why does this only target GitHub? if not getattr(settings, 'ALLOW_PRIVATE_REPOS', False): return None token = None From be5ccd08f48eb9a942221a450d4a748c7e55d14b Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 27 Nov 2017 16:12:34 -0500 Subject: [PATCH 19/37] New interface for paginated results --- readthedocs/oauth/services/gitlab.py | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index b15ad19d03a..cae54665e69 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -39,22 +39,11 @@ class GitLabService(Service): 'org': urljoin(settings.MEDIA_URL, 'images/fa-users.svg'), } - def paginate(self, url, **kwargs): - """ - Combine return from GitLab pagination. GitLab uses LinkHeaders. - - See: http://www.w3.org/wiki/LinkHeader. - See: https://docs.gitlab.com/ce/api/README.html#pagination + def get_next_url_to_paginate(self, response): + return response.links.get('next', {}).get('url') - :param url: start url to get the data from. - :param kwargs: optional parameters passed to .get() method - """ - resp = self.get_session().get(url, data=kwargs) - result = resp.json() - next_url = resp.links.get('next', {}).get('url') - if next_url: - result.extend(self.paginate(next_url, **kwargs)) - return result + def get_paginated_results(self, response): + return response.json() def sync(self): """Sync repositories from GitLab API.""" From 7e38e701bdb0dc332a0dafd2e9dcb4b9e7b72973 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 27 Nov 2017 18:36:51 -0500 Subject: [PATCH 20/37] Improve gitlab integration's code --- .flake8 | 2 +- .isort.cfg | 4 +- readthedocs/builds/utils.py | 23 ++- readthedocs/oauth/services/base.py | 71 +++++---- readthedocs/oauth/services/gitlab.py | 213 +++++++++++++++++---------- readthedocs/settings/base.py | 9 -- 6 files changed, 202 insertions(+), 120 deletions(-) diff --git a/.flake8 b/.flake8 index 88163430fb6..000e4b8f94b 100644 --- a/.flake8 +++ b/.flake8 @@ -1,3 +1,3 @@ [flake8] -ignore = E125,D100,D101,D102,D105,D200,D211,P101,FI15,FI16,FI12,FI11,FI17,FI50,FI53,FI54,MQ101,T000 +ignore = E125,D100,D101,D102,D103,D105,D107,D200,D202,D211,P101,FI15,FI16,FI12,FI11,FI17,FI50,FI53,FI54,MQ101,T000 max-line-length = 80 diff --git a/.isort.cfg b/.isort.cfg index ffb33a90030..5ae1b71c441 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -3,8 +3,6 @@ line_length=80 indent=' ' multi_line_output=4 default_section=FIRSTPARTY -known_readthedocs=readthedocs -known_readthedocsinc=readthedocsinc -known_third_party=celery,stripe,requests,pytz,builtins,django,annoying,mock,allauth +known_third_party=celery,stripe,requests,pytz,builtins,django,annoying,mock,allauth,oauthlib,requests_oauthlib sections=FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER add_imports=from __future__ import division, from __future__ import print_function, from __future__ import unicode_literals diff --git a/readthedocs/builds/utils.py b/readthedocs/builds/utils.py index 604b83c1977..c727b62dd30 100644 --- a/readthedocs/builds/utils.py +++ b/readthedocs/builds/utils.py @@ -1,8 +1,10 @@ +# -*- coding: utf-8 -*- """Utilities for the builds app.""" -from __future__ import absolute_import -import re +from __future__ import ( + absolute_import, division, print_function, unicode_literals) +import re GH_REGEXS = [ re.compile('github.com/(.+)/(.+)(?:\.git){1}'), @@ -16,6 +18,14 @@ re.compile('bitbucket.org:(.+)/(.+)\.git'), ] +# TODO: I think this can be different than `gitlab.com` +# self.adapter.provider_base_url +GL_REGEXS = [ + re.compile('gitlab.com/(.+)/(.+)/'), + re.compile('gitlab.com/(.+)/(.+)'), + re.compile('gitlab.com:(.+)/(.+)\.git'), +] + def get_github_username_repo(url): if 'github' in url: @@ -33,3 +43,12 @@ def get_bitbucket_username_repo(url=None): if match: return match.groups() return (None, None) + + +def get_gitlab_username_repo(url=None): + if 'gitlab' in url: + for regex in GL_REGEXS: + match = regex.search(url) + if match: + return match.groups() + return (None, None) diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index 67896e5b19c..4e812e512af 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -1,16 +1,18 @@ +# -*- coding: utf-8 -*- """OAuth utility functions.""" -from __future__ import absolute_import -from builtins import object +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import logging from datetime import datetime +from allauth.socialaccount.models import SocialAccount +from builtins import object from django.conf import settings -from requests_oauthlib import OAuth2Session -from requests.exceptions import RequestException from oauthlib.oauth2.rfc6749.errors import InvalidClientIdError -from allauth.socialaccount.models import SocialAccount - +from requests.exceptions import RequestException +from requests_oauthlib import OAuth2Session DEFAULT_PRIVACY_LEVEL = getattr(settings, 'DEFAULT_PRIVACY_LEVEL', 'public') @@ -19,7 +21,8 @@ class Service(object): - """Service mapping for local accounts + """ + Service mapping for local accounts. :param user: User to use in token lookup and session creation :param account: :py:class:`SocialAccount` instance for user @@ -35,11 +38,11 @@ def __init__(self, user, account): @classmethod def for_user(cls, user): - """Return a list of instances if user has an account for the provider""" + """Return list of instances if user has an account for the provider.""" try: accounts = SocialAccount.objects.filter( user=user, - provider=cls.adapter.provider_id + provider=cls.adapter.provider_id, ) return [cls(user=user, account=account) for account in accounts] except SocialAccount.DoesNotExist: @@ -58,12 +61,12 @@ def get_session(self): return self.session def create_session(self): - """Create OAuth session for user + """ + Create OAuth session for user. This configures the OAuth session based on the :py:class:`SocialToken` attributes. If there is an ``expires_at``, treat the session as an auto - renewing token. Some providers expire tokens after as little as 2 - hours. + renewing token. Some providers expire tokens after as little as 2 hours. """ token = self.account.socialtoken_set.first() if token is None: @@ -88,13 +91,14 @@ def create_session(self): 'client_secret': token.app.secret, }, auto_refresh_url=self.get_adapter().access_token_url, - token_updater=self.token_updater(token) + token_updater=self.token_updater(token), ) return self.session or None def token_updater(self, token): - """Update token given data from OAuth response + """ + Update token given data from OAuth response. Expect the following response into the closure:: @@ -107,6 +111,7 @@ def token_updater(self, token): u'expires_at': 1449218652.558185 } """ + def _updater(data): token.token = data['access_token'] token.expires_at = datetime.fromtimestamp(data['expires_at']) @@ -115,14 +120,17 @@ def _updater(data): return _updater - def paginate(self, url): - """Recursively combine results from service's pagination. + def paginate(self, url, **kwargs): + """ + Recursively combine results from service's pagination. :param url: start url to get the data from. :type url: unicode + :param kwargs: optional parameters passed to .get() method + :type kwargs: dict """ try: - resp = self.get_session().get(url) + resp = self.get_session().get(url, data=kwargs) next_url = self.get_next_url_to_paginate(resp) results = self.get_paginated_results(resp) if next_url: @@ -134,27 +142,30 @@ def paginate(self, url): raise Exception('You should reconnect your account') # Catch exceptions with request or deserializing JSON except (RequestException, ValueError): - # Response data should always be JSON, still try to log if not though + # Response data should always be JSON, still try to log if not + # though try: debug_data = resp.json() except ValueError: debug_data = resp.content - log.debug('paginate failed at %s with response: %s', url, debug_data) - finally: + log.debug( + 'paginate failed at %s with response: %s', url, debug_data) + else: return [] def sync(self): raise NotImplementedError - def create_repository(self, fields, privacy=DEFAULT_PRIVACY_LEVEL, - organization=None): + def create_repository( + self, fields, privacy=DEFAULT_PRIVACY_LEVEL, organization=None): raise NotImplementedError def create_organization(self, fields): raise NotImplementedError def get_next_url_to_paginate(self, response): - """Return the next url to feed the `paginate` method. + """ + Return the next url to feed the `paginate` method. :param response: response from where to get the `next_url` attribute :type response: requests.Response @@ -162,7 +173,8 @@ def get_next_url_to_paginate(self, response): raise NotImplementedError def get_paginated_results(self, response): - """Return the results for the current response/page. + """ + Return the results for the current response/page. :param response: response from where to get the results. :type response: requests.Response @@ -177,15 +189,16 @@ def update_webhook(self, project, integration): @classmethod def is_project_service(cls, project): - """Determine if this is the service the project is using + """ + Determine if this is the service the project is using. .. note:: + This should be deprecated in favor of attaching the - :py:class:`RemoteRepository` to the project instance. This is a slight - improvement on the legacy check for webhooks + :py:class:`RemoteRepository` to the project instance. This is a + slight improvement on the legacy check for webhooks """ # TODO Replace this check by keying project to remote repos return ( cls.url_pattern is not None and - cls.url_pattern.search(project.repo) is not None - ) + cls.url_pattern.search(project.repo) is not None) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index cae54665e69..d271a68b143 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -7,12 +7,13 @@ import logging import re -from allauth.socialaccount.models import SocialToken from allauth.socialaccount.providers.gitlab.views import GitLabOAuth2Adapter from django.conf import settings +from django.core.urlresolvers import reverse from requests.exceptions import RequestException -from readthedocs.restapi.client import api +from readthedocs.builds import utils as build_utils +from readthedocs.integrations.models import Integration from ..models import RemoteOrganization, RemoteRepository from .base import DEFAULT_PRIVACY_LEVEL, Service @@ -27,7 +28,13 @@ class GitLabService(Service): - """Provider service for GitLab.""" + """ + Provider service for GitLab. + + See: + - https://docs.gitlab.com/ce/integration/oauth_provider.html + - https://docs.gitlab.com/ce/api/oauth2.html + """ adapter = GitLabOAuth2Adapter # Just use the network location to determine if it's a GitLab project @@ -46,22 +53,37 @@ def get_paginated_results(self, response): return response.json() def sync(self): - """Sync repositories from GitLab API.""" - org = None + """ + Sync repositories and organizations from GitLab API. + + See: https://docs.gitlab.com/ce/api/projects.html + """ repos = self.paginate( - u'{url}/api/v3/projects'.format(url=self.adapter.provider_base_url), + '{url}/api/v4/projects'.format(url=self.adapter.provider_base_url), per_page=100, + archived=False, order_by='path', sort='asc', + membership=True, ) - for repo in repos: - # Skip archived repositories - if repo.get('archived', False): - continue - if not org or org.slug != repo['namespace']['id']: - org = self.create_organization(repo['namespace']) - self.create_repository(repo, organization=org) + try: + org = None + for repo in repos: + # Skip archived repositories + # if repo.get('archived', False): + # continue + + # organization data is included in the /projects response + if org is None or org.slug != repo['namespace']['path']: + org = self.create_organization(repo['namespace']) + + self.create_repository(repo, organization=org) + except (TypeError, ValueError): + log.exception('Error syncing GitLab repositories') + raise Exception( + 'Could not sync your GitLab repositories, try reconnecting ' + 'your account') def create_repository( self, fields, privacy=DEFAULT_PRIVACY_LEVEL, organization=None): @@ -74,12 +96,11 @@ def create_repository( :type organization: RemoteOrganization :rtype: RemoteRepository """ - # See: https://docs.gitlab.com/ce/api/projects.html#projects - repo_is_public = fields['visibility_level'] == 20 def is_owned_by(owner_id): return self.account.extra_data['id'] == owner_id + repo_is_public = fields['visibility'] == 'public' if privacy == 'private' or (repo_is_public and privacy == 'public'): try: repo = RemoteRepository.objects.get( @@ -102,32 +123,40 @@ def is_owned_by(owner_id): return None else: repo.organization = organization + repo.name = fields['name'] - repo.full_name = fields['name_with_namespace'] repo.description = fields['description'] repo.ssh_url = fields['ssh_url_to_repo'] repo.html_url = fields['web_url'] - repo.private = not fields['public'] + repo.private = not repo_is_public + if repo.private: + repo.clone_url = repo.ssh_url + else: + repo.clone_url = fields['http_url_to_repo'] + + # TODO: review this repo.admin logic repo.admin = not repo_is_public - repo.clone_url = ( - repo.admin and repo.ssh_url or fields['http_url_to_repo']) if not repo.admin and 'owner' in fields: repo.admin = is_owned_by(fields['owner']['id']) + repo.vcs = 'git' repo.account = self.account + + # TODO: do we want default avatar URL? owner = fields.get('owner') or {} repo.avatar_url = ( fields.get('avatar_url') or owner.get('avatar_url') or self.default_avatar['repo']) + repo.json = json.dumps(fields) repo.save() return repo else: log.info( - u'Not importing {0} because mismatched type: public={1}'.format( - fields['name_with_namespace'], - fields['public'], - )) + 'Not importing %s because mismatched type: visibility=%s', + fields['name_with_namespace'], + fields['visibility'], + ) def create_organization(self, fields): """ @@ -151,20 +180,54 @@ def create_organization(self, fields): organization.name = fields.get('name') organization.account = self.account - organization.url = u'{url}/{path}'.format( - url=self.adapter.provider_base_url, path=fields.get('path')) + organization.url = '{url}/{path}'.format( + url=self.adapter.provider_base_url, + path=fields.get('path'), + ) avatar = fields.get('avatar') or {} if avatar.get('url'): - organization.avatar_url = u'{url}/{avatar}'.format( + organization.avatar_url = '{url}/{avatar}'.format( url=self.adapter.provider_base_url, avatar=avatar.get('url'), ) else: + # TODO: do we want default avatar URL here? organization.avatar_url = self.default_avatar['org'] + organization.json = json.dumps(fields) organization.save() return organization + def get_webhook_data(self, repository, integration, project): + """ + Get webhook JSON data to post to the API. + + See: http://doc.gitlab.com/ce/api/projects.html#add-project-hook + """ + return json.dumps({ + 'id': repository.id, + 'push_events': True, + 'tag_push_events': True, + 'url': 'https://{domain}/{path}'.format( + domain=settings.PRODUCTION_DOMAIN, + path=reverse( + 'api_webhook', + kwargs={ + 'project_slug': project.slug, + 'integration_pk': integration.pk, + }, + ), + ), + + # Optional + 'issues_events': False, + 'merge_requests_events': False, + 'note_events': False, + 'job_events': False, + 'pipeline_events': False, + 'wiki_events': False, + }) + def setup_webhook(self, project): """ Set up GitLab project webhook for project. @@ -175,65 +238,63 @@ def setup_webhook(self, project): :rtype: bool """ session = self.get_session() + owner, repo = build_utils.get_gitlab_username_repo(url=project.repo) + integration, _ = Integration.objects.get_or_create( + project=project, + integration_type=Integration.GITLAB_WEBHOOK, + ) + data = self.get_webhook_data(project.repo) resp = None - repositories = RemoteRepository.objects.filter( - clone_url=project.vcs_repo().repo_url) - - if not repositories.exists(): - log.error('GitLab remote repository not found') - return False, resp - - repo_id = repositories[0].get_serialized()['id'] - # See: http://doc.gitlab.com/ce/api/projects.html#add-project-hook - data = json.dumps({ - 'id': repo_id, - 'push_events': True, - 'issues_events': False, - 'merge_requests_events': False, - 'note_events': False, - 'tag_push_events': True, - 'url': u'https://{0}/gitlab'.format(settings.PRODUCTION_DOMAIN), - }) - try: resp = session.post( - u'{url}/api/v3/projects/{repo_id}/hooks'.format( + '{url}/api/v4/projects/{repo}/hooks'.format( url=self.adapter.provider_base_url, - repo_id=repo_id, + repo=repo, ), data=data, headers={'content-type': 'application/json'}, ) if resp.status_code == 201: + integration.provider_data = resp.json() + integration.save() log.info( 'GitLab webhook creation successful for project: %s', - project) - return True, resp - except RequestException: - log.error( - 'GitLab webhook creation failed for project: %s', project, - exc_info=True) + project, + ) + return (True, resp) + except (RequestException, ValueError): + log.exception( + 'GitLab webhook creation failed for project: %s', + project, + ) else: - log.error('GitLab webhook creation failed for project: %s', project) - return False, resp - - @classmethod - def get_token_for_project(cls, project, force_local=False): - """Get access token for project by iterating over project users.""" - # TODO: why does this only target GitHub? - if not getattr(settings, 'ALLOW_PRIVATE_REPOS', False): - return None - token = None - try: - if getattr(settings, 'DONT_HIT_DB', True) and not force_local: - token = api.project(project.pk).token().get()['token'] - else: - for user in project.users.all(): - tokens = SocialToken.objects.filter( - account__user=user, - app__provider=cls.adapter.provider_id) - if tokens.exists(): - token = tokens[0].token - except Exception: - log.error('Failed to get token for user', exc_info=True) - return token + log.exception( + 'GitLab webhook creation failed for project: %s', + project, + ) + return (False, resp) + + def update_webhook(self, project, integration): + # TODO: to implement + pass + + # @classmethod + # def get_token_for_project(cls, project, force_local=False): + # """Get access token for project by iterating over project users.""" + # # TODO: why does this only target GitHub? + # if not getattr(settings, 'ALLOW_PRIVATE_REPOS', False): + # return None + # token = None + # try: + # if getattr(settings, 'DONT_HIT_DB', True) and not force_local: + # token = api.project(project.pk).token().get()['token'] + # else: + # for user in project.users.all(): + # tokens = SocialToken.objects.filter( + # account__user=user, + # app__provider=cls.adapter.provider_id) + # if tokens.exists(): + # token = tokens[0].token + # except Exception: + # log.error('Failed to get token for user', exc_info=True) + # return token diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index a098eb0047d..7dde8e73591 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -231,15 +231,6 @@ def INSTALLED_APPS(self): # noqa CELERY_CREATE_MISSING_QUEUES = True CELERY_DEFAULT_QUEUE = 'celery' - # Wildcards not supported: https://github.com/celery/celery/issues/150 - CELERY_ROUTES = { - 'readthedocs.oauth.tasks.SyncBitBucketRepositories': { - 'queue': 'web', - }, - 'readthedocs.oauth.tasks.SyncGitHubRepositories': { - 'queue': 'web', - }, - } # Docker DOCKER_ENABLE = False From 90e4ba1a9ee0677cfc539a0fa5e099cf59901ddc Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 27 Nov 2017 19:22:43 -0500 Subject: [PATCH 21/37] Fix sync orgs and webhook create --- readthedocs/builds/utils.py | 2 +- readthedocs/oauth/services/gitlab.py | 60 +++++++++++++++++++++------- 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/readthedocs/builds/utils.py b/readthedocs/builds/utils.py index c727b62dd30..1457a123c67 100644 --- a/readthedocs/builds/utils.py +++ b/readthedocs/builds/utils.py @@ -21,7 +21,7 @@ # TODO: I think this can be different than `gitlab.com` # self.adapter.provider_base_url GL_REGEXS = [ - re.compile('gitlab.com/(.+)/(.+)/'), + re.compile('gitlab.com/(.+)/(.+)(?:\.git){1}'), re.compile('gitlab.com/(.+)/(.+)'), re.compile('gitlab.com:(.+)/(.+)\.git'), ] diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index d271a68b143..89a8be90a6e 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -58,6 +58,10 @@ def sync(self): See: https://docs.gitlab.com/ce/api/projects.html """ + self.sync_repositories() + self.sync_organizations() + + def sync_repositories(self): repos = self.paginate( '{url}/api/v4/projects'.format(url=self.adapter.provider_base_url), per_page=100, @@ -68,23 +72,44 @@ def sync(self): ) try: - org = None for repo in repos: - # Skip archived repositories - # if repo.get('archived', False): - # continue - - # organization data is included in the /projects response - if org is None or org.slug != repo['namespace']['path']: - org = self.create_organization(repo['namespace']) - - self.create_repository(repo, organization=org) + self.create_repository(repo) except (TypeError, ValueError): log.exception('Error syncing GitLab repositories') raise Exception( 'Could not sync your GitLab repositories, try reconnecting ' 'your account') + def sync_organizations(self): + orgs = self.paginate( + '{url}/api/v4/groups'.format(url=self.adapter.provider_base_url), + per_page=100, + all_available=False, + order_by='path', + sort='asc', + ) + + try: + for org in orgs: + org_obj = self.create_organization(org) + org_repos = self.paginate( + '{url}/api/v4/groups/{id}/projects'.format( + url=self.adapter.provider_base_url, + id=org['id'], + ), + per_page=100, + archived=False, + order_by='path', + sort='asc', + ) + for repo in org_repos: + self.create_repository(repo, organization=org_obj) + except (TypeError, ValueError): + log.exception('Error syncing GitLab organizations') + raise Exception( + 'Could not sync your GitLab organization, try reconnecting ' + 'your account') + def create_repository( self, fields, privacy=DEFAULT_PRIVACY_LEVEL, organization=None): """ @@ -198,14 +223,14 @@ def create_organization(self, fields): organization.save() return organization - def get_webhook_data(self, repository, integration, project): + def get_webhook_data(self, repo_id, integration, project): """ Get webhook JSON data to post to the API. See: http://doc.gitlab.com/ce/api/projects.html#add-project-hook """ return json.dumps({ - 'id': repository.id, + 'id': repo_id, 'push_events': True, 'tag_push_events': True, 'url': 'https://{domain}/{path}'.format( @@ -243,13 +268,18 @@ def setup_webhook(self, project): project=project, integration_type=Integration.GITLAB_WEBHOOK, ) - data = self.get_webhook_data(project.repo) + + # The ID or URL-encoded path of the project + # https://docs.gitlab.com/ce/api/README.html#namespaced-path-encoding + repo_id = '{}%2F{}'.format(owner, repo) + + data = self.get_webhook_data(repo_id, integration, project) resp = None try: resp = session.post( - '{url}/api/v4/projects/{repo}/hooks'.format( + '{url}/api/v4/projects/{repo_id}/hooks'.format( url=self.adapter.provider_base_url, - repo=repo, + repo_id=repo_id, ), data=data, headers={'content-type': 'application/json'}, From 1e8a8a8564a247baf278c6e32f447fed0110c01a Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 27 Nov 2017 19:37:36 -0500 Subject: [PATCH 22/37] Fix webhook URL --- readthedocs/oauth/services/gitlab.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 89a8be90a6e..088e2812dd2 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -233,7 +233,7 @@ def get_webhook_data(self, repo_id, integration, project): 'id': repo_id, 'push_events': True, 'tag_push_events': True, - 'url': 'https://{domain}/{path}'.format( + 'url': 'https://{domain}{path}'.format( domain=settings.PRODUCTION_DOMAIN, path=reverse( 'api_webhook', From 2fa1c51c4dafb61808ff383e3d882fea28a3bd5b Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 28 Nov 2017 10:47:09 -0500 Subject: [PATCH 23/37] Update webhook workflow for GitLab --- readthedocs/oauth/services/gitlab.py | 75 +++++++++++++++++++--------- readthedocs/oauth/utils.py | 38 ++++++++------ 2 files changed, 75 insertions(+), 38 deletions(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 088e2812dd2..ed1f5bc979d 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -305,26 +305,55 @@ def setup_webhook(self, project): return (False, resp) def update_webhook(self, project, integration): - # TODO: to implement - pass - - # @classmethod - # def get_token_for_project(cls, project, force_local=False): - # """Get access token for project by iterating over project users.""" - # # TODO: why does this only target GitHub? - # if not getattr(settings, 'ALLOW_PRIVATE_REPOS', False): - # return None - # token = None - # try: - # if getattr(settings, 'DONT_HIT_DB', True) and not force_local: - # token = api.project(project.pk).token().get()['token'] - # else: - # for user in project.users.all(): - # tokens = SocialToken.objects.filter( - # account__user=user, - # app__provider=cls.adapter.provider_id) - # if tokens.exists(): - # token = tokens[0].token - # except Exception: - # log.error('Failed to get token for user', exc_info=True) - # return token + """ + Update webhook integration. + + :param project: project to set up webhook for + :type project: Project + + :param integration: Webhook integration to update + :type integration: Integration + + :returns: boolean based on webhook update success, and requests Response + object + + :rtype: (Bool, Response) + """ + session = self.get_session() + + # The ID or URL-encoded path of the project + # https://docs.gitlab.com/ce/api/README.html#namespaced-path-encoding + repo_id = json.loads(project.remote_repository.json).get('id') + + data = self.get_webhook_data(repo_id, project, integration) + hook_id = integration.provider_data.get('id') + resp = None + try: + resp = session.put( + '{url}/api/v4/projects/{repo_id}/hooks/{hook_id}'.format( + url=self.adapter.provider_base_url, + repo_id=repo_id, + hook_id=hook_id, + ), + data=data, + headers={'content-type': 'application/json'}, + ) + if resp.status_code in 200: + recv_data = resp.json() + integration.provider_data = recv_data + integration.save() + log.info( + 'GitLab webhook update successful for project: %s', project) + return (True, resp) + # Catch exceptions with request or deserializing JSON + except (RequestException, ValueError): + log.exception( + 'GitLab webhook update failed for project: %s', project) + else: + log.error('GitLab webhook update failed for project: %s', project) + try: + debug_data = resp.json() + except ValueError: + debug_data = resp.content + log.debug('GitLab webhook update failure response: %s', debug_data) + return (False, resp) diff --git a/readthedocs/oauth/utils.py b/readthedocs/oauth/utils.py index e3a75ed7c65..b8476da10c9 100644 --- a/readthedocs/oauth/utils.py +++ b/readthedocs/oauth/utils.py @@ -1,23 +1,30 @@ +# -*- coding: utf-8 -*- """Support code for OAuth, including webhook support.""" -from __future__ import absolute_import + +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import logging from django.contrib import messages from django.utils.translation import ugettext_lazy as _ from readthedocs.integrations.models import Integration -from readthedocs.oauth.services import registry, GitHubService, BitbucketService +from readthedocs.oauth.services import ( + BitbucketService, GitHubService, GitLabService, registry) log = logging.getLogger(__name__) SERVICE_MAP = { Integration.GITHUB_WEBHOOK: GitHubService, Integration.BITBUCKET_WEBHOOK: BitbucketService, + Integration.GITLAB_WEBHOOK: GitLabService, } def attach_webhook(project, request=None): - """Add post-commit hook on project import + """ + Add post-commit hook on project import. This is a brute force approach to adding a webhook to a repository. We try all accounts until we set up a webhook. This should remain around for legacy @@ -31,9 +38,9 @@ def attach_webhook(project, request=None): else: messages.error( request, - _('Webhook activation failed. ' - 'There are no connected services for this project.') - ) + _( + 'Webhook activation failed. ' + 'There are no connected services for this project.')) return None user_accounts = service.for_user(request.user) @@ -48,21 +55,22 @@ def attach_webhook(project, request=None): if user_accounts: messages.error( request, - _('Webhook activation failed. Make sure you have permissions to set it.') + _( + 'Webhook activation failed. Make sure you have permissions to ' + 'set it.'), ) else: messages.error( request, - _('No accounts available to set webhook on. ' + _( + 'No accounts available to set webhook on. ' 'Please connect your {network} account.'.format( - network=service.adapter(request).get_provider().name - )) - ) + network=service.adapter(request).get_provider().name))) return False def update_webhook(project, integration, request=None): - """Update a specific project integration instead of brute forcing""" + """Update a specific project integration instead of brute forcing.""" service_cls = SERVICE_MAP.get(integration.integration_type) if service_cls is None: return None @@ -76,9 +84,9 @@ def update_webhook(project, integration, request=None): return True messages.error( request, - _('Webhook activation failed. ' - 'Make sure you have the necessary permissions.') - ) + _( + 'Webhook activation failed. ' + 'Make sure you have the necessary permissions.')) project.has_valid_webhook = False project.save() return False From c2f16d649b46b154f4af45310067fb3a414065c4 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 28 Nov 2017 11:44:47 -0500 Subject: [PATCH 24/37] Better yapf setting to fulfill PEP257 D202 pep257: D202 / No blank lines allowed after function docstring (found 1) Disable this yapf feature beacuse of PEP257 --- .pre-commit-config.yaml | 1 + .style.yapf | 2 +- readthedocs/oauth/services/base.py | 1 - readthedocs/oauth/services/gitlab.py | 1 - 4 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b713b46410c..bc363b1e931 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,6 +35,7 @@ repos: - id: yapf exclude: 'migrations|settings|scripts' additional_dependencies: ['futures'] + args: ['--style=.style.yapf', '--parallel'] - repo: git@github.com:FalconSocial/pre-commit-python-sorter.git sha: b57843b0b874df1d16eb0bef00b868792cb245c2 diff --git a/.style.yapf b/.style.yapf index 3b41722a8b2..14cf5c70108 100644 --- a/.style.yapf +++ b/.style.yapf @@ -26,7 +26,7 @@ ALLOW_MULTILINE_DICTIONARY_KEYS=False # # <------ this blank line # def method(): # ... -BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF=True +BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF=False # Insert a blank line before a class-level docstring. BLANK_LINE_BEFORE_CLASS_DOCSTRING=True diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index 4e812e512af..839007e712e 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -111,7 +111,6 @@ def token_updater(self, token): u'expires_at': 1449218652.558185 } """ - def _updater(data): token.token = data['access_token'] token.expires_at = datetime.fromtimestamp(data['expires_at']) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index ed1f5bc979d..f80fc002fda 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -121,7 +121,6 @@ def create_repository( :type organization: RemoteOrganization :rtype: RemoteRepository """ - def is_owned_by(owner_id): return self.account.extra_data['id'] == owner_id From e77efa44086a84b2a1dcfaa9a31b092fa24e0dc3 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 28 Nov 2017 11:48:01 -0500 Subject: [PATCH 25/37] Lint fix --- readthedocs/oauth/services/gitlab.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index f80fc002fda..e51cc29d6d3 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -337,7 +337,7 @@ def update_webhook(self, project, integration): data=data, headers={'content-type': 'application/json'}, ) - if resp.status_code in 200: + if resp.status_code == 200: recv_data = resp.json() integration.provider_data = recv_data integration.save() From b1ff48c433aa56cbc67acd5eabe54b39f4ffb367 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 28 Nov 2017 12:37:45 -0500 Subject: [PATCH 26/37] Remove default avatars for GitLab --- media/images/fa-bookmark.svg | 1 - media/images/fa-users.svg | 1 - readthedocs/oauth/services/gitlab.py | 17 ++++------------- 3 files changed, 4 insertions(+), 15 deletions(-) delete mode 100644 media/images/fa-bookmark.svg delete mode 100644 media/images/fa-users.svg diff --git a/media/images/fa-bookmark.svg b/media/images/fa-bookmark.svg deleted file mode 100644 index 0daca66b42b..00000000000 --- a/media/images/fa-bookmark.svg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/media/images/fa-users.svg b/media/images/fa-users.svg deleted file mode 100644 index 7da2bd570b4..00000000000 --- a/media/images/fa-users.svg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index e51cc29d6d3..93bdb71a2b0 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -41,10 +41,6 @@ class GitLabService(Service): # because private repos have another base url, eg. git@gitlab.example.com url_pattern = re.compile( re.escape(urlparse(adapter.provider_base_url).netloc)) - default_avatar = { - 'repo': urljoin(settings.MEDIA_URL, 'images/fa-bookmark.svg'), - 'org': urljoin(settings.MEDIA_URL, 'images/fa-users.svg'), - } def get_next_url_to_paginate(self, response): return response.links.get('next', {}).get('url') @@ -166,11 +162,9 @@ def is_owned_by(owner_id): repo.vcs = 'git' repo.account = self.account - # TODO: do we want default avatar URL? owner = fields.get('owner') or {} repo.avatar_url = ( - fields.get('avatar_url') or owner.get('avatar_url') or - self.default_avatar['repo']) + fields.get('avatar_url') or owner.get('avatar_url')) repo.json = json.dumps(fields) repo.save() @@ -208,15 +202,12 @@ def create_organization(self, fields): url=self.adapter.provider_base_url, path=fields.get('path'), ) - avatar = fields.get('avatar') or {} - if avatar.get('url'): + avatar_url = fields.get('avatar', {}).get('url') + if avatar_url: organization.avatar_url = '{url}/{avatar}'.format( url=self.adapter.provider_base_url, - avatar=avatar.get('url'), + avatar=avatar_url, ) - else: - # TODO: do we want default avatar URL here? - organization.avatar_url = self.default_avatar['org'] organization.json = json.dumps(fields) organization.save() From 4db7339aac7a52d7a35b6359eff8032e8aadf6ae Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 28 Nov 2017 12:50:56 -0500 Subject: [PATCH 27/37] Do not use REGEX to get owner and repo --- readthedocs/oauth/services/gitlab.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 93bdb71a2b0..968271b462a 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -12,7 +12,6 @@ from django.core.urlresolvers import reverse from requests.exceptions import RequestException -from readthedocs.builds import utils as build_utils from readthedocs.integrations.models import Integration from ..models import RemoteOrganization, RemoteRepository @@ -253,7 +252,6 @@ def setup_webhook(self, project): :rtype: bool """ session = self.get_session() - owner, repo = build_utils.get_gitlab_username_repo(url=project.repo) integration, _ = Integration.objects.get_or_create( project=project, integration_type=Integration.GITLAB_WEBHOOK, @@ -261,7 +259,7 @@ def setup_webhook(self, project): # The ID or URL-encoded path of the project # https://docs.gitlab.com/ce/api/README.html#namespaced-path-encoding - repo_id = '{}%2F{}'.format(owner, repo) + repo_id = json.loads(project.remote_repository.json).get('id') data = self.get_webhook_data(repo_id, integration, project) resp = None From b8e1b2545a093e84d665e4ef17047015a2eb9c8b Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 28 Nov 2017 14:31:33 -0500 Subject: [PATCH 28/37] Fix GitLab tests --- readthedocs/oauth/services/gitlab.py | 8 +- readthedocs/rtd_tests/tests/test_oauth.py | 139 ++++++++++++---------- 2 files changed, 78 insertions(+), 69 deletions(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 968271b462a..8bf8255dd93 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -201,13 +201,7 @@ def create_organization(self, fields): url=self.adapter.provider_base_url, path=fields.get('path'), ) - avatar_url = fields.get('avatar', {}).get('url') - if avatar_url: - organization.avatar_url = '{url}/{avatar}'.format( - url=self.adapter.provider_base_url, - avatar=avatar_url, - ) - + organization.avatar_url = fields.get('avatar_url') organization.json = json.dumps(fields) organization.save() return organization diff --git a/readthedocs/rtd_tests/tests/test_oauth.py b/readthedocs/rtd_tests/tests/test_oauth.py index 9737a4fda9e..3a431238c2e 100644 --- a/readthedocs/rtd_tests/tests/test_oauth.py +++ b/readthedocs/rtd_tests/tests/test_oauth.py @@ -4,7 +4,6 @@ from django.contrib.auth.models import User from django.test import TestCase -from mock import Mock from readthedocs.oauth.models import RemoteOrganization, RemoteRepository from readthedocs.oauth.services import ( @@ -288,56 +287,92 @@ class GitLabOAuthTests(TestCase): fixtures = ['eric', 'test_data'] repo_response_data = { + 'lfs_enabled': True, + 'request_access_enabled': False, + 'approvals_before_merge': 0, 'forks_count': 12, - 'container_registry_enabled': None, + 'only_allow_merge_if_all_discussions_are_resolved': False, + 'container_registry_enabled': True, 'web_url': 'https://gitlab.com/testorga/testrepo', + 'owner': { + 'username': 'testorga', + 'web_url': 'https://gitlab.com/testorga', + 'name': 'Test Orga', + 'state': 'active', + 'avatar_url': 'https://secure.gravatar.com/avatar/test', + 'id': 42, + }, 'wiki_enabled': True, - 'public_builds': True, - 'id': 2, + 'id': 42, 'merge_requests_enabled': True, 'archived': False, - 'snippets_enabled': False, + 'snippets_enabled': True, 'http_url_to_repo': 'https://gitlab.com/testorga/testrepo.git', 'namespace': { - 'share_with_group_lock': False, + 'kind': 'user', 'name': 'Test Orga', - 'created_at': '2014-07-11T13:38:53.510Z', - 'description': '', - 'updated_at': '2014-07-11T13:38:53.510Z', - 'avatar': { - 'url': None, - }, + 'parent_id': None, + 'plan': 'early_adopter', 'path': 'testorga', - 'visibility_level': 20, - 'id': 5, - 'owner_id': None, + 'id': 42, + 'full_path': 'testorga', + }, + 'star_count': 1, + '_links': { + 'repo_branches': 'http://gitlab.com/api/v4/projects/42/repository/branches', + 'merge_requests': 'http://gitlab.com/api/v4/projects/42/merge_requests', + 'self': 'http://gitlab.com/api/v4/projects/42', + 'labels': 'http://gitlab.com/api/v4/projects/42/labels', + 'members': 'http://gitlab.com/api/v4/projects/42/members', + 'events': 'http://gitlab.com/api/v4/projects/42/events', + 'issues': 'http://gitlab.com/api/v4/projects/42/issues', }, - 'star_count': 0, - 'avatar_url': 'http://placekitten.com/50/50', + 'resolve_outdated_diff_discussions': False, 'issues_enabled': True, 'path_with_namespace': 'testorga/testrepo', - 'public': True, + 'ci_config_path': None, + 'shared_with_groups': [], 'description': 'Test Repo', 'default_branch': 'master', + 'visibility': 'public', 'ssh_url_to_repo': 'git@gitlab.com:testorga/testrepo.git', + 'public_jobs': True, 'path': 'testrepo', - 'visibility_level': 20, + 'import_status': 'none', + 'only_allow_merge_if_pipeline_succeeds': False, + 'open_issues_count': 0, + 'last_activity_at': '2017-11-28T14:21:17.570Z', + 'name': 'testrepo', + 'printing_merge_request_link_enabled': True, + 'name_with_namespace': 'testorga / testrepo', + 'created_at': '2017-11-27T19:19:30.906Z', + 'shared_runners_enabled': True, + 'creator_id': 389803, + 'avatar_url': None, 'permissions': { - 'group_access': { + 'group_access': None, + 'project_access': { 'notification_level': 3, 'access_level': 40, }, - 'project_access': None, }, - 'open_issues_count': 2, - 'last_activity_at': '2016-03-01T09:22:34.344Z', - 'name': 'testrepo', - 'name_with_namespace': 'testorga / testrepo', - 'created_at': '2015-11-02T13:52:42.821Z', - 'builds_enabled': True, - 'creator_id': 5, - 'shared_runners_enabled': True, 'tag_list': [], + 'jobs_enabled': True, + } + + group_response_data = { + 'id': 1, + 'name': 'Test Orga', + 'path': 'testorga', + 'description': 'An interesting group', + 'visibility': 'public', + 'lfs_enabled': True, + 'avatar_url': 'https://secure.gravatar.com/avatar/test', + 'web_url': 'https://gitlab.com/groups/testorga', + 'request_access_enabled': False, + 'full_name': 'Test Orga', + 'full_path': 'testorga', + 'parent_id': None, } def setUp(self): @@ -352,8 +387,7 @@ def get_private_repo_data(self): """Manipulate repo response data to get private repo data.""" data = self.repo_response_data.copy() data.update({ - 'visibility_level': 10, - 'public': False, + 'visibility': 'private', }) return data @@ -365,11 +399,16 @@ def test_make_project_pass(self): self.assertEqual(repo.name, 'testrepo') self.assertEqual(repo.full_name, 'testorga / testrepo') self.assertEqual(repo.description, 'Test Repo') - self.assertEqual(repo.avatar_url, 'http://placekitten.com/50/50') + self.assertEqual( + repo.avatar_url, + 'https://secure.gravatar.com/avatar/test', + ) self.assertIn(self.user, repo.users.all()) self.assertEqual(repo.organization, self.org) self.assertEqual( - repo.clone_url, 'https://gitlab.com/testorga/testrepo.git') + repo.clone_url, + 'https://gitlab.com/testorga/testrepo.git', + ) self.assertEqual(repo.ssh_url, 'git@gitlab.com:testorga/testrepo.git') self.assertEqual(repo.html_url, 'https://gitlab.com/testorga/testrepo') @@ -387,36 +426,12 @@ def test_make_private_project_success(self): self.assertTrue(repo.private, True) def test_make_organization(self): - org = self.service.create_organization( - self.repo_response_data['namespace']) + org = self.service.create_organization(self.group_response_data) self.assertIsInstance(org, RemoteOrganization) self.assertEqual(org.slug, 'testorga') self.assertEqual(org.name, 'Test Orga') - self.assertEqual(org.avatar_url, '/media/images/fa-users.svg') + self.assertEqual( + org.avatar_url, + 'https://secure.gravatar.com/avatar/test', + ) self.assertEqual(org.url, 'https://gitlab.com/testorga') - - def test_sync_skip_archived_repo(self): - data = self.repo_response_data - data['archived'] = True - create_repo_mock = Mock() - create_orga_mock = Mock() - setattr(self.service, 'paginate', Mock(return_value=[data])) - setattr(self.service, 'create_repository', create_repo_mock) - setattr(self.service, 'create_organization', create_orga_mock) - self.service.sync() - self.assertFalse(create_repo_mock.called) - self.assertFalse(create_orga_mock.called) - - def test_sync_create_repo_and_orga(self): - create_repo_mock = Mock() - create_orga_mock = Mock(return_value=self.org) - setattr( - self.service, 'paginate', - Mock(return_value=[self.repo_response_data])) - setattr(self.service, 'create_repository', create_repo_mock) - setattr(self.service, 'create_organization', create_orga_mock) - self.service.sync() - create_repo_mock.assert_called_once_with( - self.repo_response_data, organization=self.org) - create_orga_mock.assert_called_once_with( - self.repo_response_data['namespace']) From ad6ba743a5589f545f298584ca5e8d286c06faee Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 28 Nov 2017 15:50:52 -0500 Subject: [PATCH 29/37] Mock `is_owned_by` properly --- readthedocs/oauth/services/gitlab.py | 8 ++++---- readthedocs/rtd_tests/tests/test_oauth.py | 11 ++++++++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 8bf8255dd93..6dbb158c928 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -105,6 +105,9 @@ def sync_organizations(self): 'Could not sync your GitLab organization, try reconnecting ' 'your account') + def is_owned_by(self, owner_id): + return self.account.extra_data['id'] == owner_id + def create_repository( self, fields, privacy=DEFAULT_PRIVACY_LEVEL, organization=None): """ @@ -116,8 +119,6 @@ def create_repository( :type organization: RemoteOrganization :rtype: RemoteRepository """ - def is_owned_by(owner_id): - return self.account.extra_data['id'] == owner_id repo_is_public = fields['visibility'] == 'public' if privacy == 'private' or (repo_is_public and privacy == 'public'): @@ -153,10 +154,9 @@ def is_owned_by(owner_id): else: repo.clone_url = fields['http_url_to_repo'] - # TODO: review this repo.admin logic repo.admin = not repo_is_public if not repo.admin and 'owner' in fields: - repo.admin = is_owned_by(fields['owner']['id']) + repo.admin = self.is_owned_by(fields['owner']['id']) repo.vcs = 'git' repo.account = self.account diff --git a/readthedocs/rtd_tests/tests/test_oauth.py b/readthedocs/rtd_tests/tests/test_oauth.py index 3a431238c2e..44fc0f36257 100644 --- a/readthedocs/rtd_tests/tests/test_oauth.py +++ b/readthedocs/rtd_tests/tests/test_oauth.py @@ -2,6 +2,7 @@ from __future__ import ( absolute_import, division, print_function, unicode_literals) +import mock from django.contrib.auth.models import User from django.test import TestCase @@ -392,9 +393,11 @@ def get_private_repo_data(self): return data def test_make_project_pass(self): - repo = self.service.create_repository( - self.repo_response_data, organization=self.org, - privacy=self.privacy) + with mock.patch('readthedocs.oauth.services.gitlab.GitLabService.is_owned_by') as m: # yapf: disable + m.return_value = True + repo = self.service.create_repository( + self.repo_response_data, organization=self.org, + privacy=self.privacy) self.assertIsInstance(repo, RemoteRepository) self.assertEqual(repo.name, 'testrepo') self.assertEqual(repo.full_name, 'testorga / testrepo') @@ -411,6 +414,8 @@ def test_make_project_pass(self): ) self.assertEqual(repo.ssh_url, 'git@gitlab.com:testorga/testrepo.git') self.assertEqual(repo.html_url, 'https://gitlab.com/testorga/testrepo') + self.assertTrue(repo.admin) + self.assertFalse(repo.private) def test_make_private_project_fail(self): repo = self.service.create_repository( From 66f8ea20c001316fa8713ebaeb1dc1654a8fbe6f Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 28 Nov 2017 16:23:03 -0500 Subject: [PATCH 30/37] Added button to Import Project dashboard --- readthedocs/templates/projects/project_import.html | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/readthedocs/templates/projects/project_import.html b/readthedocs/templates/projects/project_import.html index f6067ce713f..a7d5124287b 100644 --- a/readthedocs/templates/projects/project_import.html +++ b/readthedocs/templates/projects/project_import.html @@ -105,6 +105,17 @@

{% trans "Import a Repository" %}

+
+ + + +
+
Date: Tue, 28 Nov 2017 16:32:28 -0500 Subject: [PATCH 31/37] Feedback on docs --- docs/webhooks.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/webhooks.rst b/docs/webhooks.rst index 581a8516d1d..dc856726878 100644 --- a/docs/webhooks.rst +++ b/docs/webhooks.rst @@ -9,8 +9,7 @@ receive a webhook notification, we determine if the change is related to an active version for your project, and if it is, a build is triggered for that version. -GitHub ------- +.. _integration-detail: Webhook Integrations -------------------- @@ -52,7 +51,7 @@ GitHub .. note:: The webhook secret is not yet respected Bitbucket ---------- +~~~~~~~~~ * Go to the **Settings** page for your project * Click **Webhooks** and then **Add webhook** @@ -62,7 +61,7 @@ Bitbucket * Finish by clicking **Save** GitLab ------- +~~~~~~ * Go to the **Settings** page for your project * Click **Integrations** From 4e6f4a6ea1638dad175dd125840cadac55dd8326 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 28 Nov 2017 16:47:06 -0500 Subject: [PATCH 32/37] Fix isort settings --- .isort.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index 988aafe7b81..d619e783262 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -2,8 +2,8 @@ line_length=80 indent=' ' multi_line_output=4 -default_section=FIRSTPARTY -known_firstparty=readthedocs,readthedocsinc -known_third_party=celery,stripe,requests,pytz,builtins,django,annoying,mock,allauth,oauthlib,requests_oauthlib,readthedocs_build +default_section=THIRDPARTY +known_first_party=readthedocs,readthedocsinc +known_third_party=mock sections=FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER add_imports=from __future__ import division, from __future__ import print_function, from __future__ import unicode_literals From 0ce170296ba21407b83558a6a5081e3f173b05a8 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 29 Nov 2017 09:32:40 -0500 Subject: [PATCH 33/37] Fix lint --- readthedocs/oauth/services/gitlab.py | 1 - 1 file changed, 1 deletion(-) diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 6dbb158c928..8e29bc1383c 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -119,7 +119,6 @@ def create_repository( :type organization: RemoteOrganization :rtype: RemoteRepository """ - repo_is_public = fields['visibility'] == 'public' if privacy == 'private' or (repo_is_public and privacy == 'public'): try: From 1e668e1635139e0e59fb54e0998bf995551259db Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 4 Dec 2017 11:34:08 -0500 Subject: [PATCH 34/37] Styles / Linting --- .flake8 | 2 +- readthedocs/builds/models.py | 225 ++++++++------ readthedocs/doc_builder/backends/sphinx.py | 151 +++++---- readthedocs/integrations/models.py | 73 +++-- readthedocs/oauth/models.py | 108 ++++--- readthedocs/projects/constants.py | 338 +++++++++++---------- readthedocs/projects/forms.py | 173 ++++++----- readthedocs/projects/views/base.py | 4 +- readthedocs/restapi/views/footer_views.py | 101 +++--- 9 files changed, 661 insertions(+), 514 deletions(-) diff --git a/.flake8 b/.flake8 index 000e4b8f94b..319e43e8d6d 100644 --- a/.flake8 +++ b/.flake8 @@ -1,3 +1,3 @@ [flake8] -ignore = E125,D100,D101,D102,D103,D105,D107,D200,D202,D211,P101,FI15,FI16,FI12,FI11,FI17,FI50,FI53,FI54,MQ101,T000 +ignore = E125,D100,D101,D102,D103,D105,D106,D107,D200,D202,D211,P101,FI15,FI16,FI12,FI11,FI17,FI50,FI53,FI54,MQ101,T000 max-line-length = 80 diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 10d8a9e5aa6..4ccee817446 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -1,36 +1,39 @@ +# -*- coding: utf-8 -*- """Models for the builds app.""" -from __future__ import absolute_import +from __future__ import ( + absolute_import, division, print_function, unicode_literals) import logging import os.path import re +from builtins import object from shutil import rmtree -from builtins import object from django.conf import settings from django.core.urlresolvers import reverse from django.db import models from django.utils.encoding import python_2_unicode_compatible -from django.utils.translation import ugettext_lazy as _, ugettext +from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import ugettext from guardian.shortcuts import assign from taggit.managers import TaggableManager -from .constants import (BUILD_STATE, BUILD_TYPES, VERSION_TYPES, - LATEST, NON_REPOSITORY_VERSIONS, STABLE, - BUILD_STATE_FINISHED, BRANCH, TAG) +from readthedocs.core.utils import broadcast +from readthedocs.projects.constants import ( + BITBUCKET_REGEXS, BITBUCKET_URL, GITHUB_REGEXS, GITHUB_URL, PRIVACY_CHOICES, + PRIVATE) +from readthedocs.projects.models import APIProject, Project + +from .constants import ( + BRANCH, BUILD_STATE, BUILD_STATE_FINISHED, BUILD_TYPES, LATEST, + NON_REPOSITORY_VERSIONS, STABLE, TAG, VERSION_TYPES) from .managers import VersionManager from .querysets import BuildQuerySet, RelatedBuildQuerySet, VersionQuerySet from .version_slug import VersionSlugField -from readthedocs.core.utils import broadcast -from readthedocs.projects.constants import (PRIVACY_CHOICES, GITHUB_URL, - GITHUB_REGEXS, BITBUCKET_URL, - BITBUCKET_REGEXS, PRIVATE) -from readthedocs.projects.models import Project, APIProject - - -DEFAULT_VERSION_PRIVACY_LEVEL = getattr(settings, 'DEFAULT_VERSION_PRIVACY_LEVEL', 'public') +DEFAULT_VERSION_PRIVACY_LEVEL = getattr( + settings, 'DEFAULT_VERSION_PRIVACY_LEVEL', 'public') log = logging.getLogger(__name__) @@ -40,11 +43,16 @@ class Version(models.Model): """Version of a ``Project``.""" - project = models.ForeignKey(Project, verbose_name=_('Project'), - related_name='versions') + project = models.ForeignKey( + Project, + verbose_name=_('Project'), + related_name='versions', + ) type = models.CharField( - _('Type'), max_length=20, - choices=VERSION_TYPES, default='unknown', + _('Type'), + max_length=20, + choices=VERSION_TYPES, + default='unknown', ) # used by the vcs backend @@ -64,16 +72,19 @@ class Version(models.Model): #: in the URL to identify this version in a project. It's also used in the #: filesystem to determine how the paths for this version are called. It #: must not be used for any other identifying purposes. - slug = VersionSlugField(_('Slug'), max_length=255, - populate_from='verbose_name') + slug = VersionSlugField( + _('Slug'), max_length=255, populate_from='verbose_name') supported = models.BooleanField(_('Supported'), default=True) active = models.BooleanField(_('Active'), default=False) built = models.BooleanField(_('Built'), default=False) uploaded = models.BooleanField(_('Uploaded'), default=False) privacy_level = models.CharField( - _('Privacy Level'), max_length=20, choices=PRIVACY_CHOICES, - default=DEFAULT_VERSION_PRIVACY_LEVEL, help_text=_("Level of privacy for this Version.") + _('Privacy Level'), + max_length=20, + choices=PRIVACY_CHOICES, + default=DEFAULT_VERSION_PRIVACY_LEVEL, + help_text=_('Level of privacy for this Version.'), ) tags = TaggableManager(blank=True) machine = models.BooleanField(_('Machine Created'), default=False) @@ -86,15 +97,15 @@ class Meta(object): permissions = ( # Translators: Permission around whether a user can view the # version - ('view_version', _('View Version')), - ) + ('view_version', _('View Version')),) def __str__(self): - return ugettext(u"Version %(version)s of %(project)s (%(pk)s)" % { - 'version': self.verbose_name, - 'project': self.project, - 'pk': self.pk - }) + return ugettext( + 'Version {version} of {project} ({pk})'.format( + version=self.verbose_name, + project=self.project, + pk=self.pk, + )) @property def commit_name(self): @@ -139,12 +150,16 @@ def commit_name(self): def get_absolute_url(self): if not self.built and not self.uploaded: - return reverse('project_version_detail', kwargs={ - 'project_slug': self.project.slug, - 'version_slug': self.slug, - }) + return reverse( + 'project_version_detail', + kwargs={ + 'project_slug': self.project.slug, + 'version_slug': self.slug, + }, + ) private = self.privacy_level == PRIVATE - return self.project.get_docs_url(version_slug=self.slug, private=private) + return self.project.get_docs_url( + version_slug=self.slug, private=private) def save(self, *args, **kwargs): # pylint: disable=arguments-differ """Add permissions to the Version for all owners on save.""" @@ -156,28 +171,32 @@ def save(self, *args, **kwargs): # pylint: disable=arguments-differ self.project.sync_supported_versions() except Exception: log.exception('failed to sync supported versions') - broadcast(type='app', task=tasks.symlink_project, args=[self.project.pk]) + broadcast( + type='app', task=tasks.symlink_project, args=[self.project.pk]) return obj def delete(self, *args, **kwargs): # pylint: disable=arguments-differ from readthedocs.projects import tasks log.info('Removing files for version %s', self.slug) broadcast(type='app', task=tasks.clear_artifacts, args=[self.pk]) - broadcast(type='app', task=tasks.symlink_project, args=[self.project.pk]) + broadcast( + type='app', task=tasks.symlink_project, args=[self.project.pk]) super(Version, self).delete(*args, **kwargs) @property def identifier_friendly(self): - """Return display friendly identifier""" + """Return display friendly identifier.""" if re.match(r'^[0-9a-f]{40}$', self.identifier, re.I): return self.identifier[:8] return self.identifier def get_subdomain_url(self): private = self.privacy_level == PRIVATE - return self.project.get_docs_url(version_slug=self.slug, - lang_slug=self.project.language, - private=private) + return self.project.get_docs_url( + version_slug=self.slug, + lang_slug=self.project.language, + private=private, + ) def get_downloads(self, pretty=False): project = self.project @@ -186,16 +205,20 @@ def get_downloads(self, pretty=False): if project.has_pdf(self.slug): data['PDF'] = project.get_production_media_url('pdf', self.slug) if project.has_htmlzip(self.slug): - data['HTML'] = project.get_production_media_url('htmlzip', self.slug) + data['HTML'] = project.get_production_media_url( + 'htmlzip', self.slug) if project.has_epub(self.slug): - data['Epub'] = project.get_production_media_url('epub', self.slug) + data['Epub'] = project.get_production_media_url( + 'epub', self.slug) else: if project.has_pdf(self.slug): data['pdf'] = project.get_production_media_url('pdf', self.slug) if project.has_htmlzip(self.slug): - data['htmlzip'] = project.get_production_media_url('htmlzip', self.slug) + data['htmlzip'] = project.get_production_media_url( + 'htmlzip', self.slug) if project.has_epub(self.slug): - data['epub'] = project.get_production_media_url('epub', self.slug) + data['epub'] = project.get_production_media_url( + 'epub', self.slug) return data def get_conf_py_path(self): @@ -205,14 +228,15 @@ def get_conf_py_path(self): return conf_py_path def get_build_path(self): - """Return version build path if path exists, otherwise `None`""" + """Return version build path if path exists, otherwise `None`.""" path = self.project.checkout_path(version=self.slug) if os.path.exists(path): return path return None def clean_build_path(self): - """Clean build path for project version + """ + Clean build path for project version. Ensure build path is clean for project version. Used to ensure stale build checkouts for each project version are removed. @@ -220,24 +244,20 @@ def clean_build_path(self): try: path = self.get_build_path() if path is not None: - log.debug('Removing build path {0} for {1}'.format( - path, self)) + log.debug('Removing build path {0} for {1}'.format(path, self)) rmtree(path) except OSError: log.exception('Build path cleanup failed') - def get_github_url(self, docroot, filename, source_suffix='.rst', action='view'): + def get_github_url( + self, docroot, filename, source_suffix='.rst', action='view'): """ Return a GitHub URL for a given filename. - `docroot` - Location of documentation in repository - `filename` - Name of file - `source_suffix` - File suffix of documentation format - `action` - `view` (default) or `edit` + :param docroot: Location of documentation in repository + :param filename: Name of file + :param source_suffix: File suffix of documentation format + :param action: `view` (default) or `edit` """ repo_url = self.project.repo if 'github' not in repo_url: @@ -247,9 +267,9 @@ def get_github_url(self, docroot, filename, source_suffix='.rst', action='view') return '' else: if docroot[0] != '/': - docroot = "/%s" % docroot + docroot = '/{}'.format(docroot) if docroot[-1] != '/': - docroot = "%s/" % docroot + docroot = '{}/'.format(docroot) if action == 'view': action_string = 'blob' @@ -303,7 +323,8 @@ def get_bitbucket_url(self, docroot, filename, source_suffix='.rst'): class APIVersion(Version): - """Version proxy model for API data deserialization + """ + Version proxy model for API data deserialization. This replaces the pattern where API data was deserialized into a mocked :py:cls:`Version` object. This pattern was confusing, as it was not explicit @@ -340,19 +361,20 @@ class VersionAlias(models.Model): """Alias for a ``Version``.""" - project = models.ForeignKey(Project, verbose_name=_('Project'), - related_name='aliases') + project = models.ForeignKey( + Project, verbose_name=_('Project'), related_name='aliases') from_slug = models.CharField(_('From slug'), max_length=255, default='') - to_slug = models.CharField(_('To slug'), max_length=255, default='', - blank=True) + to_slug = models.CharField( + _('To slug'), max_length=255, default='', blank=True) largest = models.BooleanField(_('Largest'), default=False) def __str__(self): - return ugettext(u"Alias for %(project)s: %(from)s -> %(to)s" % { - 'project': self.project, - 'from': self.from_slug, - 'to': self.to_slug, - }) + return ugettext( + 'Alias for {project}: {_from} -> {to}'.format( + project=self.project, + _from=self.from_slug, + to=self.to_slug, + )) @python_2_unicode_compatible @@ -360,14 +382,14 @@ class Build(models.Model): """Build data.""" - project = models.ForeignKey(Project, verbose_name=_('Project'), - related_name='builds') - version = models.ForeignKey(Version, verbose_name=_('Version'), null=True, - related_name='builds') - type = models.CharField(_('Type'), max_length=55, choices=BUILD_TYPES, - default='html') - state = models.CharField(_('State'), max_length=55, choices=BUILD_STATE, - default='finished') + project = models.ForeignKey( + Project, verbose_name=_('Project'), related_name='builds') + version = models.ForeignKey( + Version, verbose_name=_('Version'), null=True, related_name='builds') + type = models.CharField( + _('Type'), max_length=55, choices=BUILD_TYPES, default='html') + state = models.CharField( + _('State'), max_length=55, choices=BUILD_STATE, default='finished') date = models.DateTimeField(_('Date'), auto_now_add=True) success = models.BooleanField(_('Success'), default=True) @@ -376,14 +398,16 @@ class Build(models.Model): output = models.TextField(_('Output'), default='', blank=True) error = models.TextField(_('Error'), default='', blank=True) exit_code = models.IntegerField(_('Exit code'), null=True, blank=True) - commit = models.CharField(_('Commit'), max_length=255, null=True, blank=True) + commit = models.CharField( + _('Commit'), max_length=255, null=True, blank=True) length = models.IntegerField(_('Build Length'), null=True, blank=True) - builder = models.CharField(_('Builder'), max_length=255, null=True, blank=True) + builder = models.CharField( + _('Builder'), max_length=255, null=True, blank=True) - cold_storage = models.NullBooleanField(_('Cold Storage'), - help_text='Build steps stored outside the database.') + cold_storage = models.NullBooleanField( + _('Cold Storage'), help_text='Build steps stored outside the database.') # Manager @@ -392,17 +416,17 @@ class Build(models.Model): class Meta(object): ordering = ['-date'] get_latest_by = 'date' - index_together = [ - ['version', 'state', 'type'] - ] + index_together = [['version', 'state', 'type']] def __str__(self): - return ugettext(u"Build %(project)s for %(usernames)s (%(pk)s)" % { - 'project': self.project, - 'usernames': ' '.join(self.project.users.all() - .values_list('username', flat=True)), - 'pk': self.pk, - }) + return ugettext( + 'Build {project} for {usernames} ({pk})'.format( + project=self.project, + usernames=' '.join( + self.project.users.all().values_list('username', flat=True), + ), + pk=self.pk, + )) @models.permalink def get_absolute_url(self): @@ -410,13 +434,14 @@ def get_absolute_url(self): @property def finished(self): - """Return if build has a finished state""" + """Return if build has a finished state.""" return self.state == BUILD_STATE_FINISHED class BuildCommandResultMixin(object): - """Mixin for common command result methods/properties + """ + Mixin for common command result methods/properties. Shared methods between the database model :py:class:`BuildCommandResult` and non-model respresentations of build command results from the API @@ -424,12 +449,13 @@ class BuildCommandResultMixin(object): @property def successful(self): - """Did the command exit with a successful exit code""" + """Did the command exit with a successful exit code.""" return self.exit_code == 0 @property def failed(self): - """Did the command exit with a failing exit code + """ + Did the command exit with a failing exit code. Helper for inverse of :py:meth:`successful` """ @@ -441,8 +467,8 @@ class BuildCommandResult(BuildCommandResultMixin, models.Model): """Build command for a ``Build``.""" - build = models.ForeignKey(Build, verbose_name=_('Build'), - related_name='commands') + build = models.ForeignKey( + Build, verbose_name=_('Build'), related_name='commands') command = models.TextField(_('Command')) description = models.TextField(_('Description'), blank=True) @@ -459,12 +485,13 @@ class Meta(object): objects = RelatedBuildQuerySet.as_manager() def __str__(self): - return (ugettext(u'Build command {pk} for build {build}') - .format(pk=self.pk, build=self.build)) + return ( + ugettext('Build command {pk} for build {build}') + .format(pk=self.pk, build=self.build)) @property def run_time(self): - """Total command runtime in seconds""" + """Total command runtime in seconds.""" if self.start_time is not None and self.end_time is not None: diff = self.end_time - self.start_time return diff.seconds diff --git a/readthedocs/doc_builder/backends/sphinx.py b/readthedocs/doc_builder/backends/sphinx.py index b1088b94117..17599c8559a 100644 --- a/readthedocs/doc_builder/backends/sphinx.py +++ b/readthedocs/doc_builder/backends/sphinx.py @@ -1,30 +1,33 @@ -"""Sphinx_ backend for building docs. +# -*- coding: utf-8 -*- +""" +Sphinx_ backend for building docs. .. _Sphinx: http://www.sphinx-doc.org/ - """ -from __future__ import absolute_import -import os -import sys +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import codecs -from glob import glob import logging +import os +import sys import zipfile +from glob import glob import six +from django.conf import settings from django.template import loader as template_loader from django.template.loader import render_to_string -from django.conf import settings from readthedocs.builds import utils as version_utils -from readthedocs.projects.utils import safe_write from readthedocs.projects.exceptions import ProjectImportError +from readthedocs.projects.utils import safe_write from readthedocs.restapi.client import api from ..base import BaseBuilder, restoring_chdir +from ..constants import PDF_RE, SPHINX_STATIC_DIR, SPHINX_TEMPLATE_DIR +from ..environments import BuildCommand, DockerBuildCommand from ..exceptions import BuildEnvironmentError -from ..environments import DockerBuildCommand, BuildCommand -from ..constants import SPHINX_TEMPLATE_DIR, SPHINX_STATIC_DIR, PDF_RE from ..signals import finalize_sphinx_context_data log = logging.getLogger(__name__) @@ -38,30 +41,35 @@ def __init__(self, *args, **kwargs): super(BaseSphinx, self).__init__(*args, **kwargs) try: self.old_artifact_path = os.path.join( - self.project.conf_dir(self.version.slug), - self.sphinx_build_dir) + self.project.conf_dir(self.version.slug), self.sphinx_build_dir) except ProjectImportError: docs_dir = self.docs_dir() - self.old_artifact_path = os.path.join(docs_dir, self.sphinx_build_dir) + self.old_artifact_path = os.path.join( + docs_dir, + self.sphinx_build_dir, + ) def _write_config(self, master_doc='index'): """Create ``conf.py`` if it doesn't exist.""" docs_dir = self.docs_dir() - conf_template = render_to_string('sphinx/conf.py.conf', - {'project': self.project, - 'version': self.version, - 'template_dir': SPHINX_TEMPLATE_DIR, - 'master_doc': master_doc, - }) + conf_template = render_to_string( + 'sphinx/conf.py.conf', { + 'project': self.project, + 'version': self.version, + 'template_dir': SPHINX_TEMPLATE_DIR, + 'master_doc': master_doc, + }) conf_file = os.path.join(docs_dir, 'conf.py') safe_write(conf_file, conf_template) def get_config_params(self): """Get configuration parameters to be rendered into the conf file.""" # TODO this should be handled better in the theme - conf_py_path = os.path.join(os.path.sep, - self.version.get_conf_py_path(), - '') + conf_py_path = os.path.join( + os.path.sep, + self.version.get_conf_py_path(), + '', + ) remote_version = self.version.commit_name github_user, github_repo = version_utils.get_github_username_repo( @@ -69,7 +77,7 @@ def get_config_params(self): github_version_is_editable = (self.version.type == 'branch') display_github = github_user is not None - bitbucket_user, bitbucket_repo = version_utils.get_bitbucket_username_repo( + bitbucket_user, bitbucket_repo = version_utils.get_bitbucket_username_repo( # noqa url=self.project.repo) bitbucket_version_is_editable = (self.version.type == 'branch') display_bitbucket = bitbucket_user is not None @@ -89,7 +97,11 @@ def get_config_params(self): 'static_path': SPHINX_STATIC_DIR, 'template_path': SPHINX_TEMPLATE_DIR, 'conf_py_path': conf_py_path, - 'api_host': getattr(settings, 'PUBLIC_API_URL', 'https://readthedocs.org'), + 'api_host': getattr( + settings, + 'PUBLIC_API_URL', + 'https://readthedocs.org', + ), 'commit': self.project.vcs_repo(self.version.slug).commit, 'versions': versions, 'downloads': downloads, @@ -137,14 +149,17 @@ def append_conf(self, **__): rendered = tmpl.render(self.get_config_params()) with outfile: - outfile.write("\n") + outfile.write('\n') outfile.write(rendered) # Print the contents of conf.py in order to make the rendered # configfile visible in the build logs self.run( - 'cat', os.path.relpath(outfile_path, - self.project.checkout_path(self.version.slug)), + 'cat', + os.path.relpath( + outfile_path, + self.project.checkout_path(self.version.slug), + ), cwd=self.project.checkout_path(self.version.slug), ) @@ -154,22 +169,23 @@ def build(self): build_command = [ 'python', self.python_env.venv_bin(filename='sphinx-build'), - '-T' + '-T', ] if self._force: build_command.append('-E') build_command.extend([ - '-b', self.sphinx_builder, - '-d', '_build/doctrees-{format}'.format(format=self.sphinx_builder), - '-D', 'language={lang}'.format(lang=project.language), + '-b', + self.sphinx_builder, + '-d', + '_build/doctrees-{format}'.format(format=self.sphinx_builder), + '-D', + 'language={lang}'.format(lang=project.language), '.', - self.sphinx_build_dir + self.sphinx_build_dir, ]) cmd_ret = self.run( - *build_command, - cwd=project.conf_dir(self.version.slug), - bin_path=self.python_env.venv_bin() - ) + *build_command, cwd=project.conf_dir(self.version.slug), + bin_path=self.python_env.venv_bin()) return cmd_ret.successful @@ -217,8 +233,11 @@ class LocalMediaBuilder(BaseSphinx): @restoring_chdir def move(self, **__): - log.info("Creating zip file from %s", self.old_artifact_path) - target_file = os.path.join(self.target, '%s.zip' % self.project.slug) + log.info('Creating zip file from %s', self.old_artifact_path) + target_file = os.path.join( + self.target, + '{}.zip'.format(self.project.slug), + ) if not os.path.exists(self.target): os.makedirs(self.target) if os.path.exists(target_file): @@ -232,9 +251,9 @@ def move(self, **__): to_write = os.path.join(root, fname) archive.write( filename=to_write, - arcname=os.path.join("%s-%s" % (self.project.slug, - self.version.slug), - to_write) + arcname=os.path.join( + '{}-{}'.format(self.project.slug, self.version.slug), + to_write), ) archive.close() @@ -245,18 +264,21 @@ class EpubBuilder(BaseSphinx): sphinx_build_dir = '_build/epub' def move(self, **__): - from_globs = glob(os.path.join(self.old_artifact_path, "*.epub")) + from_globs = glob(os.path.join(self.old_artifact_path, '*.epub')) if not os.path.exists(self.target): os.makedirs(self.target) if from_globs: from_file = from_globs[0] - to_file = os.path.join(self.target, "%s.epub" % self.project.slug) + to_file = os.path.join( + self.target, + '{}.epub'.format(self.project.slug), + ) self.run('mv', '-f', from_file, to_file) class LatexBuildCommand(BuildCommand): - """Ignore LaTeX exit code if there was file output""" + """Ignore LaTeX exit code if there was file output.""" def run(self): super(LatexBuildCommand, self).run() @@ -268,7 +290,7 @@ def run(self): class DockerLatexBuildCommand(DockerBuildCommand): - """Ignore LaTeX exit code if there was file output""" + """Ignore LaTeX exit code if there was file output.""" def run(self): super(DockerLatexBuildCommand, self).run() @@ -294,13 +316,16 @@ def build(self): self.run( 'python', self.python_env.venv_bin(filename='sphinx-build'), - '-b', 'latex', - '-D', 'language={lang}'.format(lang=self.project.language), - '-d', '_build/doctrees', + '-b', + 'latex', + '-D', + 'language={lang}'.format(lang=self.project.language), + '-d', + '_build/doctrees', '.', '_build/latex', cwd=cwd, - bin_path=self.python_env.venv_bin() + bin_path=self.python_env.venv_bin(), ) latex_cwd = os.path.join(cwd, '_build', 'latex') tex_files = glob(os.path.join(latex_cwd, '*.tex')) @@ -310,17 +335,12 @@ def build(self): # Run LaTeX -> PDF conversions pdflatex_cmds = [ - ['pdflatex', - '-interaction=nonstopmode', - tex_file] - for tex_file in tex_files] + ['pdflatex', '-interaction=nonstopmode', tex_file] + for tex_file in tex_files] # yapf: disable makeindex_cmds = [ - ['makeindex', - '-s', - 'python.ist', - '{0}.idx'.format( - os.path.splitext(os.path.relpath(tex_file, latex_cwd))[0])] - for tex_file in tex_files] + ['makeindex', '-s', 'python.ist', '{0}.idx'.format( + os.path.splitext(os.path.relpath(tex_file, latex_cwd))[0])] + for tex_file in tex_files] # yapf: disable if self.build_env.command_class == DockerBuildCommand: latex_class = DockerLatexBuildCommand @@ -348,10 +368,14 @@ def move(self, **__): if not os.path.exists(self.target): os.makedirs(self.target) - exact = os.path.join(self.old_artifact_path, "%s.pdf" % self.project.slug) + exact = os.path.join( + self.old_artifact_path, + '{}.pdf'.format(self.project.slug), + ) exact_upper = os.path.join( self.old_artifact_path, - "%s.pdf" % self.project.slug.capitalize()) + '{}.pdf'.format(self.project.slug.capitalize()), + ) if self.pdf_file_name and os.path.exists(self.pdf_file_name): from_file = self.pdf_file_name @@ -360,11 +384,12 @@ def move(self, **__): elif os.path.exists(exact_upper): from_file = exact_upper else: - from_globs = glob(os.path.join(self.old_artifact_path, "*.pdf")) + from_globs = glob(os.path.join(self.old_artifact_path, '*.pdf')) if from_globs: from_file = max(from_globs, key=os.path.getmtime) else: from_file = None if from_file: - to_file = os.path.join(self.target, "%s.pdf" % self.project.slug) + to_file = os.path.join( + self.target, '{}.pdf'.format(self.project.slug)) self.run('mv', '-f', from_file, to_file) diff --git a/readthedocs/integrations/models.py b/readthedocs/integrations/models.py index eb9950522ea..40fd6424598 100644 --- a/readthedocs/integrations/models.py +++ b/readthedocs/integrations/models.py @@ -1,34 +1,37 @@ -"""Integration models for external services""" +# -*- coding: utf-8 -*- +"""Integration models for external services.""" + +from __future__ import ( + absolute_import, division, print_function, unicode_literals) -from __future__ import absolute_import -from __future__ import division -from builtins import str -from past.utils import old_div -from builtins import object import json -import uuid import re +import uuid +from builtins import object, str -from django.db import models, transaction -from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation +from django.contrib.contenttypes.fields import ( + GenericForeignKey, GenericRelation) from django.contrib.contenttypes.models import ContentType +from django.db import models, transaction from django.utils.encoding import python_2_unicode_compatible from django.utils.safestring import mark_safe from django.utils.translation import ugettext_lazy as _ -from rest_framework import status from jsonfield import JSONField +from past.utils import old_div from pygments import highlight -from pygments.lexers import JsonLexer from pygments.formatters import HtmlFormatter +from pygments.lexers import JsonLexer +from rest_framework import status from readthedocs.core.fields import default_token from readthedocs.projects.models import Project + from .utils import normalize_request_payload class HttpExchangeManager(models.Manager): - """HTTP exchange manager methods""" + """HTTP exchange manager methods.""" # Filter rules for request headers to remove from the output REQ_FILTER_RULES = [ @@ -38,7 +41,8 @@ class HttpExchangeManager(models.Manager): @transaction.atomic def from_exchange(self, req, resp, related_object, payload=None): - """Create object from Django request and response objects + """ + Create object from Django request and response objects. If an explicit Request ``payload`` is not specified, the payload will be determined directly from the Request object. This makes a good effort to @@ -66,8 +70,9 @@ def from_exchange(self, req, resp, related_object, payload=None): request_headers = dict( (key[5:].title().replace('_', '-'), str(val)) for (key, val) in list(req.META.items()) - if key.startswith('HTTP_') - ) + if key.startswith('HTTP_'), + ) # yapf: disable + request_headers['Content-Type'] = req.content_type # Remove unwanted headers for filter_rule in self.REQ_FILTER_RULES: @@ -103,7 +108,7 @@ def delete_limit(self, related_object, limit=10): app_label=related_object._meta.app_label, # pylint: disable=protected-access model=related_object._meta.model_name, # pylint: disable=protected-access ), - object_id=related_object.pk + object_id=related_object.pk, ) for exchange in queryset[limit:]: exchange.delete() @@ -112,7 +117,7 @@ def delete_limit(self, related_object, limit=10): @python_2_unicode_compatible class HttpExchange(models.Model): - """HTTP request/response exchange""" + """HTTP request/response exchange.""" id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) @@ -129,7 +134,8 @@ class HttpExchange(models.Model): response_body = models.TextField(_('Response body')) status_code = models.IntegerField( - _('Status code'), default=status.HTTP_200_OK + _('Status code'), + default=status.HTTP_200_OK, ) objects = HttpExchangeManager() @@ -146,7 +152,7 @@ def failed(self): return int(old_div(self.status_code, 100)) != 2 def formatted_json(self, field): - """Try to return pretty printed and Pygment highlighted code""" + """Try to return pretty printed and Pygment highlighted code.""" value = getattr(self, field) or '' try: if not isinstance(value, dict): @@ -169,9 +175,11 @@ def formatted_response_body(self): class IntegrationQuerySet(models.QuerySet): - """Return a subclass of Integration, based on the integration type + """ + Return a subclass of Integration, based on the integration type. .. note:: + This doesn't affect queries currently, only fetching of an object """ @@ -180,12 +188,13 @@ def _get_subclass(self, integration_type): class_map = dict( (cls.integration_type_id, cls) for cls in self.model.__subclasses__() - if hasattr(cls, 'integration_type_id') - ) + if hasattr(cls, 'integration_type_id'), + ) # yapf: disable return class_map.get(integration_type) def _get_subclass_replacement(self, original): - """Replace model instance on Integration subclasses + """ + Replace model instance on Integration subclasses. This is based on the ``integration_type`` field, and is used to provide specific functionality to and integration via a proxy subclass of the @@ -207,7 +216,8 @@ def subclass(self, instance): return self._get_subclass_replacement(instance) def create(self, **kwargs): - """Override of create method to use subclass instance instead + """ + Override of create method to use subclass instance instead. Instead of using the underlying Integration model to create this instance, we get the correct subclass to use instead. This allows for @@ -225,7 +235,7 @@ def create(self, **kwargs): @python_2_unicode_compatible class Integration(models.Model): - """Inbound webhook integration for projects""" + """Inbound webhook integration for projects.""" GITHUB_WEBHOOK = 'github_webhook' BITBUCKET_WEBHOOK = 'bitbucket_webhook' @@ -245,12 +255,12 @@ class Integration(models.Model): integration_type = models.CharField( _('Integration type'), max_length=32, - choices=INTEGRATIONS + choices=INTEGRATIONS, ) provider_data = JSONField(_('Provider data')) exchanges = GenericRelation( 'HttpExchange', - related_query_name='integrations' + related_query_name='integrations', ) objects = IntegrationQuerySet.as_manager() @@ -259,8 +269,9 @@ class Integration(models.Model): has_sync = False def __str__(self): - return (_('{0} for {1}') - .format(self.get_integration_type_display(), self.project.name)) + return ( + _('{0} for {1}') + .format(self.get_integration_type_display(), self.project.name)) class GitHubWebhook(Integration): @@ -304,7 +315,7 @@ class Meta(object): proxy = True def save(self, *args, **kwargs): # pylint: disable=arguments-differ - """Ensure model has token data before saving""" + """Ensure model has token data before saving.""" try: token = self.provider_data.get('token') except (AttributeError, TypeError): @@ -317,5 +328,5 @@ def save(self, *args, **kwargs): # pylint: disable=arguments-differ @property def token(self): - """Get or generate a secret token for authentication""" + """Get or generate a secret token for authentication.""" return self.provider_data.get('token') diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index e5ab447c4cc..38d252cb8ed 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -1,24 +1,26 @@ -"""OAuth service models""" +# -*- coding: utf-8 -*- +"""OAuth service models.""" + +from __future__ import ( + absolute_import, division, print_function, unicode_literals) -from __future__ import absolute_import -from builtins import object import json +from builtins import object -from django.db import models -from django.db.models import Q +from allauth.socialaccount.models import SocialAccount from django.conf import settings from django.contrib.auth.models import User +from django.core.urlresolvers import reverse +from django.core.validators import URLValidator +from django.db import models +from django.db.models import Q from django.utils.encoding import python_2_unicode_compatible from django.utils.translation import ugettext_lazy as _ -from django.core.validators import URLValidator -from django.core.urlresolvers import reverse -from allauth.socialaccount.models import SocialAccount from readthedocs.projects.constants import REPO_CHOICES from readthedocs.projects.models import Project -from .querysets import RemoteRepositoryQuerySet, RemoteOrganizationQuerySet - +from .querysets import RemoteOrganizationQuerySet, RemoteRepositoryQuerySet DEFAULT_PRIVACY_LEVEL = getattr(settings, 'DEFAULT_PRIVACY_LEVEL', 'public') @@ -26,7 +28,8 @@ @python_2_unicode_compatible class RemoteOrganization(models.Model): - """Organization from remote service + """ + Organization from remote service. This encapsulates both Github and Bitbucket """ @@ -35,8 +38,8 @@ class RemoteOrganization(models.Model): pub_date = models.DateTimeField(_('Publication date'), auto_now_add=True) modified_date = models.DateTimeField(_('Modified date'), auto_now=True) - users = models.ManyToManyField(User, verbose_name=_('Users'), - related_name='oauth_organizations') + users = models.ManyToManyField( + User, verbose_name=_('Users'), related_name='oauth_organizations') account = models.ForeignKey( SocialAccount, verbose_name=_('Connected account'), related_name='remote_organizations', null=True, blank=True) @@ -46,8 +49,8 @@ class RemoteOrganization(models.Model): name = models.CharField(_('Name'), max_length=255, null=True, blank=True) email = models.EmailField(_('Email'), max_length=255, null=True, blank=True) avatar_url = models.URLField(_('Avatar image URL'), null=True, blank=True) - url = models.URLField(_('URL to organization page'), max_length=200, - null=True, blank=True) + url = models.URLField( + _('URL to organization page'), max_length=200, null=True, blank=True) json = models.TextField(_('Serialized API response')) @@ -69,7 +72,8 @@ def get_serialized(self, key=None, default=None): @python_2_unicode_compatible class RemoteRepository(models.Model): - """Remote importable repositories + """ + Remote importable repositories. This models Github and Bitbucket importable repositories """ @@ -79,8 +83,8 @@ class RemoteRepository(models.Model): modified_date = models.DateTimeField(_('Modified date'), auto_now=True) # This should now be a OneToOne - users = models.ManyToManyField(User, verbose_name=_('Users'), - related_name='oauth_repositories') + users = models.ManyToManyField( + User, verbose_name=_('Users'), related_name='oauth_repositories') account = models.ForeignKey( SocialAccount, verbose_name=_('Connected account'), related_name='remote_repositories', null=True, blank=True) @@ -89,30 +93,51 @@ class RemoteRepository(models.Model): related_name='repositories', null=True, blank=True) active = models.BooleanField(_('Active'), default=False) - project = models.OneToOneField(Project, on_delete=models.SET_NULL, - related_name='remote_repository', null=True, - blank=True) + project = models.OneToOneField( + Project, + on_delete=models.SET_NULL, + related_name='remote_repository', + null=True, + blank=True, + ) name = models.CharField(_('Name'), max_length=255) full_name = models.CharField(_('Full Name'), max_length=255) - description = models.TextField(_('Description'), blank=True, null=True, - help_text=_('Description of the project')) - avatar_url = models.URLField(_('Owner avatar image URL'), null=True, - blank=True) + description = models.TextField( + _('Description'), + blank=True, + null=True, + help_text=_('Description of the project'), + ) + avatar_url = models.URLField( + _('Owner avatar image URL'), + null=True, + blank=True, + ) - ssh_url = models.URLField(_('SSH URL'), max_length=512, blank=True, - validators=[URLValidator(schemes=['ssh'])]) + ssh_url = models.URLField( + _('SSH URL'), + max_length=512, + blank=True, + validators=[URLValidator(schemes=['ssh'])], + ) clone_url = models.URLField( _('Repository clone URL'), max_length=512, blank=True, validators=[ - URLValidator(schemes=['http', 'https', 'ssh', 'git', 'svn'])]) + URLValidator(schemes=['http', 'https', 'ssh', 'git', 'svn']) + ], + ) html_url = models.URLField(_('HTML URL'), null=True, blank=True) private = models.BooleanField(_('Private repository'), default=False) admin = models.BooleanField(_('Has admin privilege'), default=False) - vcs = models.CharField(_('vcs'), max_length=200, blank=True, - choices=REPO_CHOICES) + vcs = models.CharField( + _('vcs'), + max_length=200, + blank=True, + choices=REPO_CHOICES, + ) json = models.TextField(_('Serialized API response')) @@ -123,7 +148,7 @@ class Meta(object): verbose_name_plural = 'remote repositories' def __str__(self): - return "Remote repository: %s" % (self.html_url) + return 'Remote repository: {}'.format(self.html_url) def get_serialized(self, key=None, default=None): try: @@ -136,11 +161,11 @@ def get_serialized(self, key=None, default=None): @property def clone_fuzzy_url(self): - """Try to match against several permutations of project URL""" - pass + """Try to match against several permutations of project URL.""" + return def matches(self, user): - """Projects that exist with repository URL already""" + """Projects that exist with repository URL already.""" # Support Git scheme GitHub url format that may exist in database fuzzy_url = self.clone_url.replace('git://', '').replace('.git', '') projects = (Project @@ -148,8 +173,13 @@ def matches(self, user): .public(user) .filter(Q(repo=self.clone_url) | Q(repo__iendswith=fuzzy_url) | - Q(repo__iendswith=fuzzy_url + '.git'))) - return [{'id': project.slug, - 'url': reverse('projects_detail', - kwargs={'project_slug': project.slug})} - for project in projects] + Q(repo__iendswith=fuzzy_url + '.git'))) # yapf: disable + return [{ + 'id': project.slug, + 'url': reverse( + 'projects_detail', + kwargs={ + 'project_slug': project.slug, + }, + ), + } for project in projects] diff --git a/readthedocs/projects/constants.py b/readthedocs/projects/constants.py index a3ff8d6483a..660284ee4c9 100644 --- a/readthedocs/projects/constants.py +++ b/readthedocs/projects/constants.py @@ -1,10 +1,14 @@ -"""Project constants +# -*- coding: utf-8 -*- +""" +Project constants. Default values and other various configuration for projects, including available theme names and repository types. """ -from __future__ import absolute_import +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import re from django.utils.translation import ugettext_lazy as _ @@ -93,7 +97,7 @@ ) IMPORTANT_VERSION_FILTERS = { - 'slug': 'important' + 'slug': 'important', } # in the future this constant can be replaced with a implementation that @@ -108,173 +112,171 @@ # Languages supported for the lang_slug in the URL # Translations for builtin Sphinx messages only available for a subset of these LANGUAGES = ( - ("aa", "Afar"), - ("ab", "Abkhaz"), - ("af", "Afrikaans"), - ("am", "Amharic"), - ("ar", "Arabic"), - ("as", "Assamese"), - ("ay", "Aymara"), - ("az", "Azerbaijani"), - ("ba", "Bashkir"), - ("be", "Belarusian"), - ("bg", "Bulgarian"), - ("bh", "Bihari"), - ("bi", "Bislama"), - ("bn", "Bengali"), - ("bo", "Tibetan"), - ("br", "Breton"), - ("ca", "Catalan"), - ("co", "Corsican"), - ("cs", "Czech"), - ("cy", "Welsh"), - ("da", "Danish"), - ("de", "German"), - ("dz", "Dzongkha"), - ("el", "Greek"), - ("en", "English"), - ("eo", "Esperanto"), - ("es", "Spanish"), - ("et", "Estonian"), - ("eu", "Basque"), - ("fa", "Iranian"), - ("fi", "Finnish"), - ("fj", "Fijian"), - ("fo", "Faroese"), - ("fr", "French"), - ("fy", "Western Frisian"), - ("ga", "Irish"), - ("gd", "Scottish Gaelic"), - ("gl", "Galician"), - ("gn", "Guarani"), - ("gu", "Gujarati"), - ("ha", "Hausa"), - ("hi", "Hindi"), - ("he", "Hebrew"), - ("hr", "Croatian"), - ("hu", "Hungarian"), - ("hy", "Armenian"), - ("ia", "Interlingua"), - ("id", "Indonesian"), - ("ie", "Interlingue"), - ("ik", "Inupiaq"), - ("is", "Icelandic"), - ("it", "Italian"), - ("iu", "Inuktitut"), - ("ja", "Japanese"), - ("jv", "Javanese"), - ("ka", "Georgian"), - ("kk", "Kazakh"), - ("kl", "Kalaallisut"), - ("km", "Khmer"), - ("kn", "Kannada"), - ("ko", "Korean"), - ("ks", "Kashmiri"), - ("ku", "Kurdish"), - ("ky", "Kyrgyz"), - ("la", "Latin"), - ("ln", "Lingala"), - ("lo", "Lao"), - ("lt", "Lithuanian"), - ("lv", "Latvian"), - ("mg", "Malagasy"), - ("mi", "Maori"), - ("mk", "Macedonian"), - ("ml", "Malayalam"), - ("mn", "Mongolian"), - ("mr", "Marathi"), - ("ms", "Malay"), - ("mt", "Maltese"), - ("my", "Burmese"), - ("na", "Nauru"), - ("ne", "Nepali"), - ("nl", "Dutch"), - ("no", "Norwegian"), - ("oc", "Occitan"), - ("om", "Oromo"), - ("or", "Oriya"), - ("pa", "Panjabi"), - ("pl", "Polish"), - ("ps", "Pashto"), - ("pt", "Portuguese"), - ("qu", "Quechua"), - ("rm", "Romansh"), - ("rn", "Kirundi"), - ("ro", "Romanian"), - ("ru", "Russian"), - ("rw", "Kinyarwanda"), - ("sa", "Sanskrit"), - ("sd", "Sindhi"), - ("sg", "Sango"), - ("si", "Sinhala"), - ("sk", "Slovak"), - ("sl", "Slovenian"), - ("sm", "Samoan"), - ("sn", "Shona"), - ("so", "Somali"), - ("sq", "Albanian"), - ("sr", "Serbian"), - ("ss", "Swati"), - ("st", "Southern Sotho"), - ("su", "Sudanese"), - ("sv", "Swedish"), - ("sw", "Swahili"), - ("ta", "Tamil"), - ("te", "Telugu"), - ("tg", "Tajik"), - ("th", "Thai"), - ("ti", "Tigrinya"), - ("tk", "Turkmen"), - ("tl", "Tagalog"), - ("tn", "Tswana"), - ("to", "Tonga"), - ("tr", "Turkish"), - ("ts", "Tsonga"), - ("tt", "Tatar"), - ("tw", "Twi"), - ("ug", "Uyghur"), - ("uk", "Ukrainian"), - ("ur", "Urdu"), - ("uz", "Uzbek"), - ("vi", "Vietnamese"), - ("vo", "Volapuk"), - ("wo", "Wolof"), - ("xh", "Xhosa"), - ("yi", "Yiddish"), - ("yo", "Yoruba"), - ("za", "Zhuang"), - ("zh", "Chinese"), - ("zu", "Zulu"), + ('aa', 'Afar'), + ('ab', 'Abkhaz'), + ('af', 'Afrikaans'), + ('am', 'Amharic'), + ('ar', 'Arabic'), + ('as', 'Assamese'), + ('ay', 'Aymara'), + ('az', 'Azerbaijani'), + ('ba', 'Bashkir'), + ('be', 'Belarusian'), + ('bg', 'Bulgarian'), + ('bh', 'Bihari'), + ('bi', 'Bislama'), + ('bn', 'Bengali'), + ('bo', 'Tibetan'), + ('br', 'Breton'), + ('ca', 'Catalan'), + ('co', 'Corsican'), + ('cs', 'Czech'), + ('cy', 'Welsh'), + ('da', 'Danish'), + ('de', 'German'), + ('dz', 'Dzongkha'), + ('el', 'Greek'), + ('en', 'English'), + ('eo', 'Esperanto'), + ('es', 'Spanish'), + ('et', 'Estonian'), + ('eu', 'Basque'), + ('fa', 'Iranian'), + ('fi', 'Finnish'), + ('fj', 'Fijian'), + ('fo', 'Faroese'), + ('fr', 'French'), + ('fy', 'Western Frisian'), + ('ga', 'Irish'), + ('gd', 'Scottish Gaelic'), + ('gl', 'Galician'), + ('gn', 'Guarani'), + ('gu', 'Gujarati'), + ('ha', 'Hausa'), + ('hi', 'Hindi'), + ('he', 'Hebrew'), + ('hr', 'Croatian'), + ('hu', 'Hungarian'), + ('hy', 'Armenian'), + ('ia', 'Interlingua'), + ('id', 'Indonesian'), + ('ie', 'Interlingue'), + ('ik', 'Inupiaq'), + ('is', 'Icelandic'), + ('it', 'Italian'), + ('iu', 'Inuktitut'), + ('ja', 'Japanese'), + ('jv', 'Javanese'), + ('ka', 'Georgian'), + ('kk', 'Kazakh'), + ('kl', 'Kalaallisut'), + ('km', 'Khmer'), + ('kn', 'Kannada'), + ('ko', 'Korean'), + ('ks', 'Kashmiri'), + ('ku', 'Kurdish'), + ('ky', 'Kyrgyz'), + ('la', 'Latin'), + ('ln', 'Lingala'), + ('lo', 'Lao'), + ('lt', 'Lithuanian'), + ('lv', 'Latvian'), + ('mg', 'Malagasy'), + ('mi', 'Maori'), + ('mk', 'Macedonian'), + ('ml', 'Malayalam'), + ('mn', 'Mongolian'), + ('mr', 'Marathi'), + ('ms', 'Malay'), + ('mt', 'Maltese'), + ('my', 'Burmese'), + ('na', 'Nauru'), + ('ne', 'Nepali'), + ('nl', 'Dutch'), + ('no', 'Norwegian'), + ('oc', 'Occitan'), + ('om', 'Oromo'), + ('or', 'Oriya'), + ('pa', 'Panjabi'), + ('pl', 'Polish'), + ('ps', 'Pashto'), + ('pt', 'Portuguese'), + ('qu', 'Quechua'), + ('rm', 'Romansh'), + ('rn', 'Kirundi'), + ('ro', 'Romanian'), + ('ru', 'Russian'), + ('rw', 'Kinyarwanda'), + ('sa', 'Sanskrit'), + ('sd', 'Sindhi'), + ('sg', 'Sango'), + ('si', 'Sinhala'), + ('sk', 'Slovak'), + ('sl', 'Slovenian'), + ('sm', 'Samoan'), + ('sn', 'Shona'), + ('so', 'Somali'), + ('sq', 'Albanian'), + ('sr', 'Serbian'), + ('ss', 'Swati'), + ('st', 'Southern Sotho'), + ('su', 'Sudanese'), + ('sv', 'Swedish'), + ('sw', 'Swahili'), + ('ta', 'Tamil'), + ('te', 'Telugu'), + ('tg', 'Tajik'), + ('th', 'Thai'), + ('ti', 'Tigrinya'), + ('tk', 'Turkmen'), + ('tl', 'Tagalog'), + ('tn', 'Tswana'), + ('to', 'Tonga'), + ('tr', 'Turkish'), + ('ts', 'Tsonga'), + ('tt', 'Tatar'), + ('tw', 'Twi'), + ('ug', 'Uyghur'), + ('uk', 'Ukrainian'), + ('ur', 'Urdu'), + ('uz', 'Uzbek'), + ('vi', 'Vietnamese'), + ('vo', 'Volapuk'), + ('wo', 'Wolof'), + ('xh', 'Xhosa'), + ('yi', 'Yiddish'), + ('yo', 'Yoruba'), + ('za', 'Zhuang'), + ('zh', 'Chinese'), + ('zu', 'Zulu'), # Try these to test our non-2 letter language support - ("nb_NO", "Norwegian Bokmal"), - ("pt_BR", "Brazilian Portuguese"), - ("uk_UA", "Ukrainian"), - ("zh_CN", "Simplified Chinese"), - ("zh_TW", "Traditional Chinese"), + ('nb_NO', 'Norwegian Bokmal'), + ('pt_BR', 'Brazilian Portuguese'), + ('uk_UA', 'Ukrainian'), + ('zh_CN', 'Simplified Chinese'), + ('zh_TW', 'Traditional Chinese'), ) -LANGUAGES_REGEX = "|".join( - [re.escape(code[0]) for code in LANGUAGES] -) +LANGUAGES_REGEX = '|'.join([re.escape(code[0]) for code in LANGUAGES]) PROGRAMMING_LANGUAGES = ( - ("words", "Only Words"), - ("py", "Python"), - ("js", "JavaScript"), - ("php", "PHP"), - ("ruby", "Ruby"), - ("perl", "Perl"), - ("java", "Java"), - ("go", "Go"), - ("julia", "Julia"), - ("c", "C"), - ("csharp", "C#"), - ("cpp", "C++"), - ("objc", "Objective-C"), - ("other", "Other"), + ('words', 'Only Words'), + ('py', 'Python'), + ('js', 'JavaScript'), + ('php', 'PHP'), + ('ruby', 'Ruby'), + ('perl', 'Perl'), + ('java', 'Java'), + ('go', 'Go'), + ('julia', 'Julia'), + ('c', 'C'), + ('csharp', 'C#'), + ('cpp', 'C++'), + ('objc', 'Objective-C'), + ('other', 'Other'), ) -LOG_TEMPLATE = u"(Build) [{project}:{version}] {msg}" +LOG_TEMPLATE = '(Build) [{project}:{version}] {msg}' PROJECT_PK_REGEX = '(?:[-\w]+)' PROJECT_SLUG_REGEX = '(?:[-\w]+)' @@ -289,7 +291,9 @@ re.compile('bitbucket.org/(.+)/(.+)/'), re.compile('bitbucket.org/(.+)/(.+)'), ] -GITHUB_URL = (u'https://github.com/{user}/{repo}/' - '{action}/{version}{docroot}{path}{source_suffix}') -BITBUCKET_URL = (u'https://bitbucket.org/{user}/{repo}/' - 'src/{version}{docroot}{path}{source_suffix}') +GITHUB_URL = ( + 'https://github.com/{user}/{repo}/' + '{action}/{version}{docroot}{path}{source_suffix}') +BITBUCKET_URL = ( + 'https://bitbucket.org/{user}/{repo}/' + 'src/{version}{docroot}{path}{source_suffix}') diff --git a/readthedocs/projects/forms.py b/readthedocs/projects/forms.py index 6e83ddcb83c..bd7f7c07786 100644 --- a/readthedocs/projects/forms.py +++ b/readthedocs/projects/forms.py @@ -1,10 +1,12 @@ -"""Project forms""" +# -*- coding: utf-8 -*- +"""Project forms.""" -from __future__ import absolute_import +from __future__ import ( + absolute_import, division, print_function, unicode_literals) +from builtins import object from random import choice -from builtins import object from django import forms from django.conf import settings from django.contrib.auth.models import User @@ -16,24 +18,25 @@ from textclassifier.validators import ClassifierValidator from readthedocs.builds.constants import TAG -from readthedocs.core.utils import trigger_build, slugify +from readthedocs.core.utils import slugify, trigger_build from readthedocs.integrations.models import Integration from readthedocs.oauth.models import RemoteRepository from readthedocs.projects import constants from readthedocs.projects.exceptions import ProjectSpamError from readthedocs.projects.models import ( - Project, ProjectRelationship, EmailHook, WebHook, Domain, Feature) + Domain, EmailHook, Feature, Project, ProjectRelationship, WebHook) from readthedocs.redirects.models import Redirect class ProjectForm(forms.ModelForm): - """Project form + """ + Project form. :param user: If provided, add this user as a project user on save """ - required_css_class = "required" + required_css_class = 'required' def __init__(self, *args, **kwargs): self.user = kwargs.pop('user', None) @@ -49,14 +52,15 @@ def save(self, commit=True): class ProjectTriggerBuildMixin(object): - """Mixin to trigger build on form save + """ + Mixin to trigger build on form save. This should be replaced with signals instead of calling trigger_build explicitly. """ def save(self, commit=True): - """Trigger build on commit save""" + """Trigger build on commit save.""" project = super(ProjectTriggerBuildMixin, self).save(commit) if commit: trigger_build(project=project) @@ -65,14 +69,14 @@ def save(self, commit=True): class ProjectBackendForm(forms.Form): - """Get the import backend""" + """Get the import backend.""" backend = forms.CharField() class ProjectBasicsForm(ProjectForm): - """Form for basic project fields""" + """Form for basic project fields.""" class Meta(object): model = Project @@ -89,13 +93,13 @@ def __init__(self, *args, **kwargs): if show_advanced: self.fields['advanced'] = forms.BooleanField( required=False, - label=_('Edit advanced project options') + label=_('Edit advanced project options'), ) self.fields['repo'].widget.attrs['placeholder'] = self.placehold_repo() self.fields['repo'].widget.attrs['required'] = True def save(self, commit=True): - """Add remote repository relationship to the project instance""" + """Add remote repository relationship to the project instance.""" instance = super(ProjectBasicsForm, self).save(commit) remote_repo = self.cleaned_data.get('remote_repository', None) if remote_repo: @@ -112,18 +116,18 @@ def clean_name(self): potential_slug = slugify(name) if Project.objects.filter(slug=potential_slug).exists(): raise forms.ValidationError( - _('Invalid project name, a project already exists with that name')) + _('Invalid project name, a project already exists with that name')) # yapf: disable # noqa return name def clean_repo(self): repo = self.cleaned_data.get('repo', '').strip() pvt_repos = getattr(settings, 'ALLOW_PRIVATE_REPOS', False) if '&&' in repo or '|' in repo: - raise forms.ValidationError(_(u'Invalid character in repo name')) + raise forms.ValidationError(_('Invalid character in repo name')) elif '@' in repo and not pvt_repos: raise forms.ValidationError( - _(u'It looks like you entered a private repo - please use the ' - u'public (http:// or git://) clone url')) + _('It looks like you entered a private repo - please use the ' + 'public (http:// or git://) clone url')) # yapf: disable return repo def clean_remote_repository(self): @@ -136,7 +140,7 @@ def clean_remote_repository(self): users=self.user, ) except RemoteRepository.DoesNotExist: - raise forms.ValidationError(_(u'Repository invalid')) + raise forms.ValidationError(_('Repository invalid')) def placehold_repo(self): return choice([ @@ -152,7 +156,7 @@ def placehold_repo(self): class ProjectExtraForm(ProjectForm): - """Additional project information form""" + """Additional project information form.""" class Meta(object): model = Project @@ -168,18 +172,21 @@ class Meta(object): description = forms.CharField( validators=[ClassifierValidator(raises=ProjectSpamError)], required=False, - widget=forms.Textarea + widget=forms.Textarea, ) class ProjectAdvancedForm(ProjectTriggerBuildMixin, ProjectForm): - """Advanced project option form""" + """Advanced project option form.""" python_interpreter = forms.ChoiceField( - choices=constants.PYTHON_CHOICES, initial='python', - help_text=_("(Beta) The Python interpreter used to create the virtual " - "environment.")) + choices=constants.PYTHON_CHOICES, + initial='python', + help_text=_( + '(Beta) The Python interpreter used to create the virtual ' + 'environment.'), + ) class Meta(object): model = Project @@ -210,24 +217,26 @@ def clean_conf_py_file(self): if filename and 'conf.py' not in filename: raise forms.ValidationError( _('Your configuration file is invalid, make sure it contains ' - 'conf.py in it.')) + 'conf.py in it.')) # yapf: disable return filename class UpdateProjectForm(ProjectTriggerBuildMixin, ProjectBasicsForm, ProjectExtraForm): - class Meta(object): model = Project fields = ( # Basics - 'name', 'repo', 'repo_type', + 'name', + 'repo', + 'repo_type', # Extra # 'allow_comments', # 'comment_moderation', 'description', 'documentation_type', - 'language', 'programming_language', + 'language', + 'programming_language', 'project_url', 'tags', ) @@ -235,7 +244,7 @@ class Meta(object): class ProjectRelationshipForm(forms.ModelForm): - """Form to add/update project relationships""" + """Form to add/update project relationships.""" parent = forms.CharField(widget=forms.HiddenInput(), required=False) @@ -258,24 +267,27 @@ def clean_parent(self): if self.project.superprojects.exists(): # This validation error is mostly for testing, users shouldn't see # this in normal circumstances - raise forms.ValidationError(_("Subproject nesting is not supported")) + raise forms.ValidationError( + _('Subproject nesting is not supported')) return self.project def get_subproject_queryset(self): - """Return scrubbed subproject choice queryset + """ + Return scrubbed subproject choice queryset. This removes projects that are either already a subproject of another project, or are a superproject, as neither case is supported. """ - queryset = (Project.objects.for_admin_user(self.user) - .exclude(subprojects__isnull=False) - .exclude(superprojects__isnull=False)) + queryset = ( + Project.objects.for_admin_user(self.user) + .exclude(subprojects__isnull=False) + .exclude(superprojects__isnull=False)) return queryset class DualCheckboxWidget(forms.CheckboxInput): - """Checkbox with link to the version's built documentation""" + """Checkbox with link to the version's built documentation.""" def __init__(self, version, attrs=None, check_test=bool): super(DualCheckboxWidget, self).__init__(attrs, check_test) @@ -284,21 +296,21 @@ def __init__(self, version, attrs=None, check_test=bool): def render(self, name, value, attrs=None): checkbox = super(DualCheckboxWidget, self).render(name, value, attrs) icon = self.render_icon() - return mark_safe(u'%s%s' % (checkbox, icon)) + return mark_safe('{}{}'.format(checkbox, icon)) def render_icon(self): context = { 'MEDIA_URL': settings.MEDIA_URL, 'built': self.version.built, 'uploaded': self.version.uploaded, - 'url': self.version.get_absolute_url() + 'url': self.version.get_absolute_url(), } return render_to_string('projects/includes/icon_built.html', context) class BaseVersionsForm(forms.Form): - """Form for versions page""" + """Form for versions page.""" def save(self): versions = self.project.versions.all() @@ -310,14 +322,11 @@ def save(self): self.project.save() def save_version(self, version): - """Save version if there has been a change, trigger a rebuild""" + """Save version if there has been a change, trigger a rebuild.""" new_value = self.cleaned_data.get('version-%s' % version.slug, None) - privacy_level = self.cleaned_data.get('privacy-%s' % version.slug, - None) - if ((new_value is None or - new_value == version.active) and ( - privacy_level is None or - privacy_level == version.privacy_level)): + privacy_level = self.cleaned_data.get('privacy-%s' % version.slug, None) + if ((new_value is None or new_value == version.active) and + (privacy_level is None or privacy_level == version.privacy_level)): return version.active = new_value version.privacy_level = privacy_level @@ -327,7 +336,7 @@ def save_version(self, version): def build_versions_form(project): - """Versions form with a list of versions and version privacy levels""" + """Versions form with a list of versions and version privacy levels.""" attrs = { 'project': project, } @@ -336,15 +345,16 @@ def build_versions_form(project): if active.exists(): choices = [(version.slug, version.verbose_name) for version in active] attrs['default-version'] = forms.ChoiceField( - label=_("Default Version"), + label=_('Default Version'), choices=choices, initial=project.get_default_version(), ) for version in versions_qs: - field_name = 'version-%s' % version.slug - privacy_name = 'privacy-%s' % version.slug + field_name = 'version-{}'.format(version.slug) + privacy_name = 'privacy-{}'.format(version.slug) if version.type == TAG: - label = "%s (%s)" % (version.verbose_name, version.identifier[:8]) + label = '{} ({})'.format( + version.verbose_name, version.identifier[:8]) else: label = version.verbose_name attrs[field_name] = forms.BooleanField( @@ -355,17 +365,17 @@ def build_versions_form(project): ) attrs[privacy_name] = forms.ChoiceField( # This isn't a real label, but just a slug for the template - label="privacy", + label='privacy', choices=constants.PRIVACY_CHOICES, initial=version.privacy_level, ) - return type('VersionsForm', (BaseVersionsForm,), attrs) + return type(str('VersionsForm'), (BaseVersionsForm,), attrs) class BaseUploadHTMLForm(forms.Form): - content = forms.FileField(label=_("Zip file of HTML")) - overwrite = forms.BooleanField(required=False, - label=_("Overwrite existing HTML?")) + content = forms.FileField(label=_('Zip file of HTML')) + overwrite = forms.BooleanField( + required=False, label=_('Overwrite existing HTML?')) def __init__(self, *args, **kwargs): self.request = kwargs.pop('request', None) @@ -378,15 +388,15 @@ def clean(self): # Validation if version.active and not self.cleaned_data.get('overwrite', False): - raise forms.ValidationError(_("That version is already active!")) + raise forms.ValidationError(_('That version is already active!')) if not filename.name.endswith('zip'): - raise forms.ValidationError(_("Must upload a zip file.")) + raise forms.ValidationError(_('Must upload a zip file.')) return self.cleaned_data def build_upload_html_form(project): - """Upload HTML form with list of versions to upload HTML for""" + """Upload HTML form with list of versions to upload HTML for.""" attrs = { 'project': project, } @@ -395,7 +405,7 @@ def build_upload_html_form(project): choices = [] choices += [(version.slug, version.verbose_name) for version in active] attrs['version'] = forms.ChoiceField( - label=_("Version of the project you are uploading HTML for"), + label=_('Version of the project you are uploading HTML for'), choices=choices, ) return type('UploadHTMLForm', (BaseUploadHTMLForm,), attrs) @@ -403,7 +413,7 @@ def build_upload_html_form(project): class UserForm(forms.Form): - """Project user association form""" + """Project user association form.""" user = forms.CharField() @@ -415,8 +425,8 @@ def clean_user(self): name = self.cleaned_data['user'] user_qs = User.objects.filter(username=name) if not user_qs.exists(): - raise forms.ValidationError(_("User %(name)s does not exist") % - {'name': name}) + raise forms.ValidationError( + _('User {name} does not exist').format(name=name)) self.user = user_qs[0] return name @@ -429,7 +439,7 @@ def save(self): class EmailHookForm(forms.Form): - """Project email notification form""" + """Project email notification form.""" email = forms.EmailField() @@ -449,7 +459,7 @@ def save(self): class WebHookForm(forms.Form): - """Project webhook form""" + """Project webhook form.""" url = forms.URLField() @@ -469,7 +479,7 @@ def save(self): class TranslationForm(forms.Form): - """Project translation form""" + """Project translation form.""" project = forms.CharField() @@ -481,11 +491,14 @@ def clean_project(self): translation_name = self.cleaned_data['project'] translation_qs = Project.objects.filter(slug=translation_name) if not translation_qs.exists(): - raise forms.ValidationError((_("Project %(name)s does not exist") - % {'name': translation_name})) + raise forms.ValidationError(( + _('Project {name} does not exist').format( + name=translation_name))) if translation_qs.first().language == self.parent.language: - err = ("Both projects have a language of `%s`. " - "Please choose one with another language" % self.parent.language) + err = ( + 'Both projects have a language of `{}`. ' + 'Please choose one with another language'.format( + self.parent.language)) raise forms.ValidationError(_(err)) self.translation = translation_qs.first() @@ -493,14 +506,15 @@ def clean_project(self): def save(self): project = self.parent.translations.add(self.translation) - # Run symlinking and other sync logic to make sure we are in a good state. + # Run symlinking and other sync logic to make sure we are in a good + # state. self.parent.save() return project class RedirectForm(forms.ModelForm): - """Form for project redirects""" + """Form for project redirects.""" class Meta(object): model = Redirect @@ -551,15 +565,17 @@ def clean_domain(self): def clean_canonical(self): canonical = self.cleaned_data['canonical'] if canonical and Domain.objects.filter( - project=self.project, canonical=True - ).exclude(domain=self.cleaned_data['domain']).exists(): - raise forms.ValidationError(_(u'Only 1 Domain can be canonical at a time.')) + project=self.project, canonical=True).exclude( + domain=self.cleaned_data['domain']).exists(): + raise forms.ValidationError( + _('Only 1 Domain can be canonical at a time.')) return canonical class IntegrationForm(forms.ModelForm): - """Form to add an integration + """ + Form to add an integration. This limits the choices of the integration type to webhook integration types """ @@ -574,7 +590,7 @@ def __init__(self, *args, **kwargs): self.project = kwargs.pop('project', None) super(IntegrationForm, self).__init__(*args, **kwargs) # Alter the integration type choices to only provider webhooks - self.fields['integration_type'].choices = Integration.WEBHOOK_INTEGRATIONS + self.fields['integration_type'].choices = Integration.WEBHOOK_INTEGRATIONS # yapf: disable # noqa def clean_project(self): return self.project @@ -586,7 +602,7 @@ def save(self, commit=True): class ProjectAdvertisingForm(forms.ModelForm): - """Project promotion opt-out form""" + """Project promotion opt-out form.""" class Meta(object): model = Project @@ -599,7 +615,8 @@ def __init__(self, *args, **kwargs): class FeatureForm(forms.ModelForm): - """Project feature form for dynamic admin choices + """ + Project feature form for dynamic admin choices. This form converts the CharField into a ChoiceField on display. The underlying driver won't attempt to do validation on the choices, and so we diff --git a/readthedocs/projects/views/base.py b/readthedocs/projects/views/base.py index 36aaa24dc8c..81b0c6da333 100644 --- a/readthedocs/projects/views/base.py +++ b/readthedocs/projects/views/base.py @@ -4,9 +4,9 @@ absolute_import, division, print_function, unicode_literals) import logging +from builtins import object from datetime import datetime, timedelta -from builtins import object from django.conf import settings from django.core.urlresolvers import reverse from django.http import HttpResponseRedirect @@ -35,6 +35,8 @@ def get_context_data(self, **kwargs): onboard = {} project = self.get_object() + # TODO: gitlab + # Show for the first few builds, return last build state if project.builds.count() <= 5: onboard['build'] = project.get_latest_build(finished=False) diff --git a/readthedocs/restapi/views/footer_views.py b/readthedocs/restapi/views/footer_views.py index f805ca62d75..ebc96b9ec4b 100644 --- a/readthedocs/restapi/views/footer_views.py +++ b/readthedocs/restapi/views/footer_views.py @@ -1,29 +1,29 @@ +# -*- coding: utf-8 -*- """Endpoint to generate footer HTML.""" -from __future__ import absolute_import +from __future__ import ( + absolute_import, division, print_function, unicode_literals) -from django.shortcuts import get_object_or_404 -from django.template import RequestContext, loader as template_loader +import six from django.conf import settings - - +from django.shortcuts import get_object_or_404 +from django.template import loader as template_loader from rest_framework import decorators, permissions from rest_framework.renderers import JSONRenderer from rest_framework.response import Response from rest_framework_jsonp.renderers import JSONPRenderer -from readthedocs.builds.constants import LATEST -from readthedocs.builds.constants import TAG +from readthedocs.builds.constants import LATEST, TAG from readthedocs.builds.models import Version from readthedocs.projects.models import Project -from readthedocs.projects.version_handling import highest_version -from readthedocs.projects.version_handling import parse_version_failsafe +from readthedocs.projects.version_handling import ( + highest_version, parse_version_failsafe) from readthedocs.restapi.signals import footer_response -import six def get_version_compare_data(project, base_version=None): - """Retrieve metadata about the highest version available for this project. + """ + Retrieve metadata about the highest version available for this project. :param base_version: We assert whether or not the base_version is also the highest version in the resulting "is_highest" value. @@ -69,32 +69,30 @@ def footer_html(request): subproject = request.GET.get('subproject', False) source_suffix = request.GET.get('source_suffix', '.rst') - new_theme = (theme == "sphinx_rtd_theme") - using_theme = (theme == "default") + new_theme = (theme == 'sphinx_rtd_theme') + using_theme = (theme == 'default') project = get_object_or_404(Project, slug=project_slug) version = get_object_or_404( - Version.objects.public(request.user, project=project, only_active=False), + Version.objects.public( + request.user, project=project, only_active=False), slug=version_slug) main_project = project.main_language_project or project - if page_slug and page_slug != "index": - if ( - main_project.documentation_type == "sphinx_htmldir" or - main_project.documentation_type == "mkdocs"): - path = page_slug + "/" - elif main_project.documentation_type == "sphinx_singlehtml": - path = "index.html#document-" + page_slug + if page_slug and page_slug != 'index': + if (main_project.documentation_type == 'sphinx_htmldir' or + main_project.documentation_type == 'mkdocs'): + path = page_slug + '/' + elif main_project.documentation_type == 'sphinx_singlehtml': + path = 'index.html#document-' + page_slug else: - path = page_slug + ".html" + path = page_slug + '.html' else: - path = "" + path = '' if version.type == TAG and version.project.has_pdf(version.slug): print_url = ( - 'https://keminglabs.com/print-the-docs/quote?project={project}&version={version}' - .format( - project=project.slug, - version=version.slug)) + 'https://keminglabs.com/print-the-docs/quote?project={project}&version={version}' # noqa + .format(project=project.slug, version=version.slug)) else: print_url = None @@ -115,14 +113,42 @@ def footer_html(request): 'settings': settings, 'subproject': subproject, 'print_url': print_url, - 'github_edit_url': version.get_github_url(docroot, page_slug, source_suffix, 'edit'), - 'github_view_url': version.get_github_url(docroot, page_slug, source_suffix, 'view'), - 'bitbucket_url': version.get_bitbucket_url(docroot, page_slug, source_suffix), + 'github_edit_url': version.get_github_url( + docroot, + page_slug, + source_suffix, + 'edit', + ), + 'github_view_url': version.get_github_url( + docroot, + page_slug, + source_suffix, + 'view', + ), + 'gitlab_edit_url': version.get_gitlab_url( + docroot, + page_slug, + source_suffix, + 'edit', + ), + 'gitlab_view_url': version.get_gitlab_url( + docroot, + page_slug, + source_suffix, + 'view', + ), + 'bitbucket_url': version.get_bitbucket_url( + docroot, + page_slug, + source_suffix, + ), 'theme': theme, } - html = template_loader.get_template('restapi/footer.html').render(context, - request) + html = template_loader.get_template('restapi/footer.html').render( + context, + request, + ) resp_data = { 'html': html, 'version_active': version.active, @@ -130,8 +156,13 @@ def footer_html(request): 'version_supported': version.supported, } - # Allow folks to hook onto the footer response for various information collection, - # or to modify the resp_data. - footer_response.send(sender=None, request=request, context=context, resp_data=resp_data) + # Allow folks to hook onto the footer response for various information + # collection, or to modify the resp_data. + footer_response.send( + sender=None, + request=request, + context=context, + resp_data=resp_data, + ) return Response(resp_data) From 4eec51ea8ec227f9c8e0f589630bc8797152b962 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 4 Dec 2017 12:51:48 -0500 Subject: [PATCH 35/37] Context for GitLab on templates --- docs/features.rst | 5 +-- docs/getting_started.rst | 6 +-- docs/webhooks.rst | 2 +- readthedocs/builds/models.py | 44 +++++++++++++++++-- readthedocs/doc_builder/backends/sphinx.py | 12 +++++ .../templates/doc_builder/conf.py.tmpl | 4 ++ readthedocs/integrations/models.py | 16 +++++++ readthedocs/projects/constants.py | 8 ++++ readthedocs/projects/views/base.py | 4 +- .../restapi/templates/restapi/footer.html | 12 ++++- 10 files changed, 99 insertions(+), 14 deletions(-) diff --git a/docs/features.rst b/docs/features.rst index 6f1de62f0f4..5d3fd2358c4 100644 --- a/docs/features.rst +++ b/docs/features.rst @@ -3,8 +3,8 @@ Read the Docs features This will serve as a list of all of the features that Read the Docs currently has. Some features are important enough to have their own page in the docs, others will simply be listed here. -GitHub and Bitbucket Integration --------------------------------- +GitHub, Bitbucket and GitLab Integration +---------------------------------------- We now support linking by default in the sidebar. It links to the page on your host, which should help people quickly update typos and send pull requests to contribute to project documentation. @@ -66,4 +66,3 @@ Alternate Domains ----------------- We provide support for CNAMEs, subdomains, and a shorturl for your project as well. This is outlined in the :doc:`alternate_domains` section. - diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 5458619243b..149b668b107 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -88,15 +88,13 @@ Then in your ``conf.py``: Sign Up and Connect an External Account --------------------------------------- -.. TODO Update this with GitLab support later - -If you are going to import a repository from GitHub or Bitbucket, you should +If you are going to import a repository from GitHub, Bitbucket or GitLab, you should connect your account to your provider first. Connecting your account allows for easier importing and enables Read the Docs to configure your repository webhooks automatically. To connect your account, got to your *Settings* dashboard and select *Connected -Services*. From here, you'll be able to connect to your GitHub or Bitbucket +Services*. From here, you'll be able to connect to your GitHub, Bitbucket or GitLab account. This process will ask you to authorize a connection to Read the Docs, that allows us to read information about and clone your repositories. diff --git a/docs/webhooks.rst b/docs/webhooks.rst index dc856726878..91ccdffd65d 100644 --- a/docs/webhooks.rst +++ b/docs/webhooks.rst @@ -3,7 +3,7 @@ Webhooks The primary method that Read the Docs uses to detect changes to your documentation is through the use of *webhooks*. Webhooks are configured with -your repository provider, such as GitHub or Bitbucket, and with each commit, +your repository provider, such as GitHub, Bitbucket or GitLab, and with each commit, merge, or other change to your repository, Read the Docs is notified. When we receive a webhook notification, we determine if the change is related to an active version for your project, and if it is, a build is triggered for that diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 4ccee817446..e960e430e72 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -21,8 +21,8 @@ from readthedocs.core.utils import broadcast from readthedocs.projects.constants import ( - BITBUCKET_REGEXS, BITBUCKET_URL, GITHUB_REGEXS, GITHUB_URL, PRIVACY_CHOICES, - PRIVATE) + BITBUCKET_REGEXS, BITBUCKET_URL, GITHUB_REGEXS, GITHUB_URL, GITLAB_REGEXS, + GITLAB_URL, PRIVACY_CHOICES, PRIVATE) from readthedocs.projects.models import APIProject, Project from .constants import ( @@ -113,7 +113,7 @@ def commit_name(self): Return the branch name, the tag name or the revision identifier. The result could be used as ref in a git repo, e.g. for linking to - GitHub or Bitbucket. + GitHub, Bitbucket or GitLab. """ # LATEST is special as it is usually a branch but does not contain the # name in verbose_name. @@ -295,6 +295,44 @@ def get_github_url( action=action_string, ) + def get_gitlab_url( + self, docroot, filename, source_suffix='.rst', action='view'): + repo_url = self.project.repo + if 'gitlab' not in repo_url: + return '' + + if not docroot: + return '' + else: + if docroot[0] != '/': + docroot = '/{}'.format(docroot) + if docroot[-1] != '/': + docroot = '{}/'.format(docroot) + + if action == 'view': + action_string = 'blob' + elif action == 'edit': + action_string = 'edit' + + for regex in GITLAB_REGEXS: + match = regex.search(repo_url) + if match: + user, repo = match.groups() + break + else: + return '' + repo = repo.rstrip('/') + + return GITLAB_URL.format( + user=user, + repo=repo, + version=self.commit_name, + docroot=docroot, + path=filename, + source_suffix=source_suffix, + action=action_string, + ) + def get_bitbucket_url(self, docroot, filename, source_suffix='.rst'): repo_url = self.project.repo if 'bitbucket' not in repo_url: diff --git a/readthedocs/doc_builder/backends/sphinx.py b/readthedocs/doc_builder/backends/sphinx.py index 17599c8559a..eb127ee1527 100644 --- a/readthedocs/doc_builder/backends/sphinx.py +++ b/readthedocs/doc_builder/backends/sphinx.py @@ -82,6 +82,11 @@ def get_config_params(self): bitbucket_version_is_editable = (self.version.type == 'branch') display_bitbucket = bitbucket_user is not None + gitlab_user, gitlab_repo = version_utils.get_gitlab_username_repo( + url=self.project.repo) + gitlab_version_is_editable = (self.version.type == 'branch') + display_gitlab = gitlab_user is not None + # Avoid hitting database and API if using Docker build environment if getattr(settings, 'DONT_HIT_API', False): versions = self.project.active_versions() @@ -119,6 +124,13 @@ def get_config_params(self): 'bitbucket_version': remote_version, 'bitbucket_version_is_editable': bitbucket_version_is_editable, 'display_bitbucket': display_bitbucket, + + # GitLab + 'gitlab_user': gitlab_user, + 'gitlab_repo': gitlab_repo, + 'gitlab_version': remote_version, + 'gitlab_version_is_editable': gitlab_version_is_editable, + 'display_gitlab': display_gitlab, } finalize_sphinx_context_data.send( diff --git a/readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl b/readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl index 5bfffa39e20..aa95a5bdce0 100644 --- a/readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl +++ b/readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl @@ -105,6 +105,10 @@ context = { 'bitbucket_repo': '{{ bitbucket_repo }}', 'bitbucket_version': '{{ bitbucket_version }}', 'display_bitbucket': {{ display_bitbucket }}, + 'gitlab_user': '{{ gitlab_user }}', + 'gitlab_repo': '{{ gitlab_repo }}', + 'gitlab_version': '{{ gitlab_version }}', + 'display_gitlab': {{ display_gitlab }}, 'READTHEDOCS': True, 'using_theme': (html_theme == "default"), 'new_theme': (html_theme == "sphinx_rtd_theme"), diff --git a/readthedocs/integrations/models.py b/readthedocs/integrations/models.py index 40fd6424598..326430e6e8d 100644 --- a/readthedocs/integrations/models.py +++ b/readthedocs/integrations/models.py @@ -306,6 +306,22 @@ def can_sync(self): return False +class GitLabWebhook(Integration): + + integration_type_id = Integration.GITLAB_WEBHOOK + has_sync = True + + class Meta(object): + proxy = True + + @property + def can_sync(self): + try: + return all((k in self.provider_data) for k in ['id', 'url']) + except (ValueError, TypeError): + return False + + class GenericAPIWebhook(Integration): integration_type_id = Integration.API_WEBHOOK diff --git a/readthedocs/projects/constants.py b/readthedocs/projects/constants.py index 660284ee4c9..2af44ab219e 100644 --- a/readthedocs/projects/constants.py +++ b/readthedocs/projects/constants.py @@ -291,9 +291,17 @@ re.compile('bitbucket.org/(.+)/(.+)/'), re.compile('bitbucket.org/(.+)/(.+)'), ] +GITLAB_REGEXS = [ + re.compile('gitlab.com/(.+)/(.+)(?:\.git){1}'), + re.compile('gitlab.com/(.+)/(.+)'), + re.compile('gitlab.com:(.+)/(.+).git'), +] GITHUB_URL = ( 'https://github.com/{user}/{repo}/' '{action}/{version}{docroot}{path}{source_suffix}') BITBUCKET_URL = ( 'https://bitbucket.org/{user}/{repo}/' 'src/{version}{docroot}{path}{source_suffix}') +GITLAB_URL = ( + 'https://gitlab.com/{user}/{repo}/' + '{action}/{version}{docroot}{path}{source_suffix}') diff --git a/readthedocs/projects/views/base.py b/readthedocs/projects/views/base.py index 81b0c6da333..db6e1195fbe 100644 --- a/readthedocs/projects/views/base.py +++ b/readthedocs/projects/views/base.py @@ -35,8 +35,6 @@ def get_context_data(self, **kwargs): onboard = {} project = self.get_object() - # TODO: gitlab - # Show for the first few builds, return last build state if project.builds.count() <= 5: onboard['build'] = project.get_latest_build(finished=False) @@ -44,6 +42,8 @@ def get_context_data(self, **kwargs): onboard['provider'] = 'github' elif 'bitbucket' in project.repo: onboard['provider'] = 'bitbucket' + elif 'gitlab' in project.repo: + onboard['provider'] = 'gitlab' context['onboard'] = onboard return context diff --git a/readthedocs/restapi/templates/restapi/footer.html b/readthedocs/restapi/templates/restapi/footer.html index 8616af3a5e7..5976976acbc 100644 --- a/readthedocs/restapi/templates/restapi/footer.html +++ b/readthedocs/restapi/templates/restapi/footer.html @@ -93,6 +93,16 @@ Edit + {% elif gitlab_edit_url %} +
+
On GitLab
+
+ View +
+
+ Edit +
+
{% endif %} {% endblock %} @@ -113,7 +123,7 @@
{% block footer %} - Free document hosting provided by Read the Docs. + Free document hosting provided by Read the Docs. {% endblock %} {% if not new_theme %} From 12eae4b27a1a12f8869f30556c86c48e8c47e1ca Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 4 Dec 2017 19:34:35 -0500 Subject: [PATCH 36/37] More style --- readthedocs/projects/forms.py | 36 ++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/readthedocs/projects/forms.py b/readthedocs/projects/forms.py index bd7f7c07786..98dee92b595 100644 --- a/readthedocs/projects/forms.py +++ b/readthedocs/projects/forms.py @@ -183,9 +183,8 @@ class ProjectAdvancedForm(ProjectTriggerBuildMixin, ProjectForm): python_interpreter = forms.ChoiceField( choices=constants.PYTHON_CHOICES, initial='python', - help_text=_( - '(Beta) The Python interpreter used to create the virtual ' - 'environment.'), + help_text=_('(Beta) The Python interpreter used to create the virtual ' + 'environment.'), ) class Meta(object): @@ -323,10 +322,16 @@ def save(self): def save_version(self, version): """Save version if there has been a change, trigger a rebuild.""" - new_value = self.cleaned_data.get('version-%s' % version.slug, None) - privacy_level = self.cleaned_data.get('privacy-%s' % version.slug, None) + new_value = self.cleaned_data.get( + 'version-{}'.format(version.slug), + None, + ) + privacy_level = self.cleaned_data.get( + 'privacy-{}'.format(version.slug), + None, + ) if ((new_value is None or new_value == version.active) and - (privacy_level is None or privacy_level == version.privacy_level)): + (privacy_level is None or privacy_level == version.privacy_level)): # yapf: disable # noqa return version.active = new_value version.privacy_level = privacy_level @@ -354,7 +359,9 @@ def build_versions_form(project): privacy_name = 'privacy-{}'.format(version.slug) if version.type == TAG: label = '{} ({})'.format( - version.verbose_name, version.identifier[:8]) + version.verbose_name, + version.identifier[:8], + ) else: label = version.verbose_name attrs[field_name] = forms.BooleanField( @@ -374,8 +381,8 @@ def build_versions_form(project): class BaseUploadHTMLForm(forms.Form): content = forms.FileField(label=_('Zip file of HTML')) - overwrite = forms.BooleanField( - required=False, label=_('Overwrite existing HTML?')) + overwrite = forms.BooleanField(required=False, + label=_('Overwrite existing HTML?')) def __init__(self, *args, **kwargs): self.request = kwargs.pop('request', None) @@ -491,14 +498,13 @@ def clean_project(self): translation_name = self.cleaned_data['project'] translation_qs = Project.objects.filter(slug=translation_name) if not translation_qs.exists(): - raise forms.ValidationError(( - _('Project {name} does not exist').format( + raise forms.ValidationError( + (_('Project {name} does not exist').format( name=translation_name))) if translation_qs.first().language == self.parent.language: - err = ( - 'Both projects have a language of `{}`. ' - 'Please choose one with another language'.format( - self.parent.language)) + err = ('Both projects have a language of `{}`. ' + 'Please choose one with another language'.format( + self.parent.language)) raise forms.ValidationError(_(err)) self.translation = translation_qs.first() From 3d276dde9304a49bb1b478915a90e4d93d4b81e1 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 6 Dec 2017 09:21:07 -0500 Subject: [PATCH 37/37] Test for get_gitlab_url --- .../rtd_tests/tests/test_repo_parsing.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_repo_parsing.py b/readthedocs/rtd_tests/tests/test_repo_parsing.py index 42ded2a6cb2..33e4ac212d6 100644 --- a/readthedocs/rtd_tests/tests/test_repo_parsing.py +++ b/readthedocs/rtd_tests/tests/test_repo_parsing.py @@ -1,5 +1,6 @@ -from __future__ import absolute_import -import json +# -*- coding: utf-8 -*- +from __future__ import ( + absolute_import, division, print_function, unicode_literals) from django.test import TestCase @@ -7,7 +8,7 @@ class TestRepoParsing(TestCase): - fixtures = ["eric", "test_data"] + fixtures = ['eric', 'test_data'] def setUp(self): self.client.login(username='eric', password='test') @@ -24,6 +25,16 @@ def test_github(self): self.pip.repo = 'https://github.com/user/repo.git' self.assertEqual(self.version.get_github_url(docroot='/docs/', filename='file'), 'https://github.com/user/repo/blob/master/docs/file.rst') + def test_gitlab(self): + self.pip.repo = 'https://gitlab.com/user/repo' + self.assertEqual(self.version.get_gitlab_url(docroot='/foo/bar/', filename='file'), 'https://gitlab.com/user/repo/blob/master/foo/bar/file.rst') + + self.pip.repo = 'https://gitlab.com/user/repo/' + self.assertEqual(self.version.get_gitlab_url(docroot='/foo/bar/', filename='file'), 'https://gitlab.com/user/repo/blob/master/foo/bar/file.rst') + + self.pip.repo = 'https://gitlab.com/user/repo.git' + self.assertEqual(self.version.get_gitlab_url(docroot='/foo/bar/', filename='file'), 'https://gitlab.com/user/repo/blob/master/foo/bar/file.rst') + def test_bitbucket(self): self.pip.repo = 'https://bitbucket.org/user/repo' self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo/src/master/foo/bar/file.rst')