Write github uri backend

ethpm/backends/base.py

@@ -1,7 +1,4 @@
-from abc import (
-    ABC,
-    abstractmethod,
-)
+from abc import ABC, abstractmethod
 
 
 class BaseURIBackend(ABC):
@@ -17,7 +14,7 @@ def can_handle_uri(self, uri: str) -> bool:
         """
         Return a bool indicating whether this backend class can handle the given URI.
         """
-        raise NotImplementedError("Must be implemented by subclass.")
+        pass
 
     @abstractmethod
     def fetch_uri_contents(self, uri: str) -> bytes:

ethpm/backends/http.py

@@ -0,0 +1,26 @@
+import requests
+
+from ethpm.backends.base import BaseURIBackend
+from ethpm.constants import GITHUB_AUTHORITY
+from ethpm.utils.uri import is_valid_github_uri
+from ethpm.validation import validate_github_uri_contents
+
+
+class GithubOverHTTPSBackend(BaseURIBackend):
+    """
+    Base class for all URIs pointing to a content-addressed Github URI.
+    """
+
+    def can_handle_uri(self, uri: str) -> bool:
+        return is_valid_github_uri(uri)
+
+    def fetch_uri_contents(self, uri: str) -> bytes:
+        http_uri, validation_hash = uri.split("#")
+        response = requests.get(http_uri)
+        response.raise_for_status()
+        validate_github_uri_contents(response.content, validation_hash)
+        return response.content
+
+    @property
+    def base_uri(self) -> str:
+        return GITHUB_AUTHORITY

ethpm/backends/ipfs.py

@@ -22,6 +22,7 @@ class BaseIPFSBackend(BaseURIBackend):
     """
     Base class for all URIs with an IPFS scheme.
     """
+
     def can_handle_uri(self, uri: str) -> bool:
         """
         Return a bool indicating whether or not this backend
@@ -65,6 +66,7 @@ class IPFSGatewayBackend(IPFSOverHTTPBackend):
     """
     Backend class for all IPFS URIs served over the IPFS gateway.
     """
+
     @property
     def base_uri(self) -> str:
         return IPFS_GATEWAY_PREFIX

ethpm/constants.py

@@ -8,3 +8,5 @@
 IPFS_GATEWAY_PREFIX = "https://ipfs.io/ipfs/"
 
 INFURA_GATEWAY_PREFIX = "https://ipfs.infura.io"
+
+GITHUB_AUTHORITY = "https://raw.githubusercontent.com/"

ethpm/exceptions.py

@@ -8,8 +8,7 @@ class PyEthPMError(Exception):
 
 class InsufficientAssetsError(PyEthPMError):
     """
-    Raised when a Manifest or Package does not
-    contain the required assets to do something.
+    Raised when a Manifest or Package does not contain the required assets to do something.
     """
 
     pass

ethpm/utils/ipfs.py

@@ -17,7 +17,6 @@ def extract_ipfs_path_from_uri(value: str) -> str:
         return parse_result.path.strip("/")
 
 
-
 def is_ipfs_uri(value: str) -> bool:
     """
     Return a bool indicating whether or not the value is a valid IPFS URI.

ethpm/utils/uri.py

@@ -2,7 +2,7 @@
 from typing import Any, Dict
 from urllib import parse
 
-from eth_utils import to_text
+from eth_utils import is_text, to_text
 
 from ethpm.backends.ipfs import get_ipfs_backend
 from ethpm.exceptions import UriNotSupportedError
@@ -13,6 +13,37 @@
 
 SWARM_SCHEMES = ["bzz", "bzz-immutable", "bzz-raw"]
 
+RAW_GITHUB_AUTHORITY = "raw.githubusercontent.com"
+
+
+def is_valid_github_uri(uri: str) -> bool:
+    """
+    Return a bool indicating whether or not the URI is a valid Github URI.
+    Valid Github URIs *must*:
+    - Have 'http' or 'https' scheme
+    - Have 'raw.githubusercontent.com' authority
+    - Have any path (*should* include a commit hash in path)
+    - Have ending fragment containing any content hash
+    i.e. 'https://raw.githubusercontent.com/any/path#content_hash
+    """
+    if not is_text(uri):
+        return False
+    parse_result = parse.urlparse(uri)
+    path = parse_result.path
+    scheme = parse_result.scheme
+    authority = parse_result.netloc
+    content_hash = parse_result.fragment
+
+    if not path or not scheme or not content_hash:
+        return False
+
+    if scheme not in INTERNET_SCHEMES:
+        return False
+
+    if authority != RAW_GITHUB_AUTHORITY:
+        return False
+    return True
+
 
 def get_manifest_from_content_addressed_uri(uri: str) -> Dict[str, Any]:
     """

ethpm/validation.py

@@ -1,7 +1,7 @@
 import re
 from urllib import parse
 
-from eth_utils import is_checksum_address
+from eth_utils import decode_hex, is_checksum_address, keccak
 
 from ethpm.constants import PACKAGE_NAME_REGEX, REGISTRY_URI_SCHEME
 from ethpm.exceptions import UriNotSupportedError, ValidationError
@@ -84,3 +84,18 @@ def validate_registry_uri_version(query: str) -> None:
         raise UriNotSupportedError(
             "{0} is not a correctly formatted version param.".format(query)
         )
+
+
+def validate_github_uri_contents(contents: bytes, validation_hash: str) -> None:
+    """
+    Validate that the contents match the validation_hash associated with a Github URI.
+    """
+    hashed_contents = keccak(contents)
+    decoded_validation = decode_hex(validation_hash)
+    if hashed_contents != decoded_validation:
+        raise ValidationError(
+            "Invalid Github content-addressed URI. "
+            "Validation hash:{0} does not match the hash of URI contents: {1}.".format(
+                decoded_validation, hashed_contents
+            )
+        )

tests/conftest.py

@@ -73,6 +73,12 @@ def standard_token_manifest():
     return get_manifest("standard-token")
 
 
+@pytest.fixture
+def owned_contract():
+    with open(str(V2_PACKAGES_DIR / "owned" / "contracts" / "Owned.sol")) as file_obj:
+        return file_obj.read()
+
+
 @pytest.fixture
 def invalid_manifest(safe_math_manifest):
     safe_math_manifest["manifest_version"] = 1

tests/ethpm/backends/test_http_backends.py

@@ -0,0 +1,20 @@
+import pytest
+import requests_mock
+
+from ethpm.backends.http import GithubOverHTTPSBackend
+from ethpm.constants import GITHUB_AUTHORITY
+
+
+@pytest.mark.parametrize(
+    "uri",
+    (
+        "https://raw.githubusercontent.com/ethpm/ethpm-spec/481739f6138907db88602558711e9d3c1301c269/examples/owned/contracts/Owned.sol#bfdea1fa5f33c30fee8443c5ffa1020027f8813e0007bb6f82aaa2843a7fdd60",  # noqa: E501
+    ),
+)
+def test_github_over_https_backend_fetch_uri_contents(uri, owned_contract):
+    backend = GithubOverHTTPSBackend()
+    assert backend.base_uri == GITHUB_AUTHORITY
+    with requests_mock.Mocker() as m:
+        m.get(requests_mock.ANY, text=owned_contract)
+        response = backend.fetch_uri_contents(uri)
+        assert response.startswith(b"pragma")

tests/ethpm/test_package_init_from_ipfs.py

@@ -1,5 +1,3 @@
-import os
-
 import pytest
 
 from ethpm import Package
@@ -12,7 +10,6 @@ def test_package_from_ipfs_with_valid_uri(dummy_ipfs_backend):
     package = Package.from_ipfs(VALID_IPFS_PKG)
     assert package.name == "safe-math-lib"
     assert isinstance(package, Package)
-    os.environ.pop('URI_BACKEND_CLASS')
 
 
 @pytest.mark.parametrize(

tests/ethpm/test_package_init_from_registry.py

@@ -1,6 +1,3 @@
-import json
-import os
-
 import pytest
 from solc import compile_source
 

tests/ethpm/utils/test_uri_utils.py

@@ -1,9 +1,8 @@
-import os
-
 import pytest
 
-from ethpm.exceptions import UriNotSupportedError
-from ethpm.utils.uri import get_manifest_from_content_addressed_uri
+from ethpm.exceptions import UriNotSupportedError, ValidationError
+from ethpm.utils.uri import get_manifest_from_content_addressed_uri, is_valid_github_uri
+from ethpm.validation import validate_github_uri_contents
 
 
 @pytest.mark.parametrize(
@@ -41,3 +40,57 @@ def test_get_manfifest_from_content_addressed_uri_raises_exception_for_unsupport
 ):
     with pytest.raises(UriNotSupportedError):
         get_manifest_from_content_addressed_uri(uri)
+
+
+@pytest.mark.parametrize(
+    "uri,expected",
+    (
+        ({}, False),
+        (123, False),
+        ("xxx", False),
+        # no scheme
+        ("raw.githubusercontent.com/any/path#0x123", False),
+        # invalid authority
+        ("http://github.com/any/path#0x123", False),
+        ("https://github.com/any/path#0x123", False),
+        # no path
+        ("http://raw.githubusercontent.com#0x123", False),
+        ("https://raw.githubusercontent.com#0x123", False),
+        # no content hash
+        ("http://raw.githubusercontent.com/any/path", False),
+        ("https://raw.githubusercontent.com/any/path", False),
+        (
+            "http://raw.githubusercontent.com/ethpm/ethpm-spec/481739f6138907db88602558711e9d3c1301c269/examples/owned/contracts/Owned.sol",  # noqa: E501
+            False,
+        ),
+        # valid github urls
+        ("http://raw.githubusercontent.com/any/path#0x123", True),
+        ("https://raw.githubusercontent.com/any/path#0x123", True),
+        (
+            "http://raw.githubusercontent.com/ethpm/ethpm-spec/481739f6138907db88602558711e9d3c1301c269/examples/owned/contracts/Owned.sol#0x123",  # noqa: E501
+            True,
+        ),
+    ),
+)
+def test_is_valid_github_uri(uri, expected):
+    actual = is_valid_github_uri(uri)
+    assert actual is expected
+
+
+@pytest.mark.parametrize(
+    "contents,hashed",
+    (
+        (b"xxx", "bc6bb462e38af7da48e0ae7b5cbae860141c04e5af2cf92328cd6548df111fcb"),
+        (b"xxx", "0xbc6bb462e38af7da48e0ae7b5cbae860141c04e5af2cf92328cd6548df111fcb"),
+    ),
+)
+def test_validate_github_uri_contents(contents, hashed):
+    assert validate_github_uri_contents(contents, hashed) is None
+
+
+@pytest.mark.parametrize(
+    "contents,hashed", ((123, "1234"), (b"xxx", "1234"), (b"123", "0x1234"))
+)
+def test_validate_github_uri_contents_invalidates_incorrect_matches(contents, hashed):
+    with pytest.raises(ValidationError):
+        validate_github_uri_contents(contents, hashed)