Added support for urlparse functions

shublang/shublang.py

@@ -337,6 +337,36 @@ def identity(iterable, element):
     """ Return the same element is passed as parameter."""
     return (element)
 
+@Pipe
+def urlparse_netloc(iterable):
+    return (parse.urlparse(url).netloc for url in iterable)
+
+@Pipe
+def urlparse_params(iterable):
+    return (parse.urlparse(url).params for url in iterable)
+
+@Pipe
+def urlparse_path(iterable):
+    return (parse.urlparse(url).path for url in iterable)
+
+@Pipe
+def urlparse_query(iterable):
+    return (parse.urlparse(url).query for url in iterable)
+
+@Pipe
+def urlparse_scheme(iterable):
+    return (parse.urlparse(url).scheme for url in iterable)
+
+@Pipe
+def urlparse_fragment(iterable):
+    return (parse.urlparse(url).fragment for url in iterable)
+
+@Pipe
+def urlparse(iterable):
+    parsed_iterable = (parse.urlparse(url) for url in iterable)
+    parsed_iterable = ({"scheme": it.scheme, "netloc": it.netloc, "path": it.path,
+                        "params": it.params, "query": it.query, "fragment": it.fragment} for it in parsed_iterable)
+    return parsed_iterable
 
 filter = where
 map = select

tests/test_functions.py

@@ -355,6 +355,36 @@ def test_currency_2():
     assert evaluate('extract_currency', data=['$1,199.00']) == ['$']
 
 
+def test_urlparse_netloc():
+    url = "https://scrapinghub.bamboohr.com/employees/performance/index.php?page=2107&subpage=1#123"
+    assert evaluate('urlparse_netloc', data=[url]) == ['scrapinghub.bamboohr.com']
+
+
+def test_urlparse_path():
+    url = "https://scrapinghub.bamboohr.com/employees/performance/index.php?page=2107&subpage=1#123"
+    assert evaluate('urlparse_path', data=[url]) == ['/employees/performance/index.php']
+
+
+def test_urlparse_query():
+    url = "https://scrapinghub.bamboohr.com/employees/performance/index.php?page=2107&subpage=1#123"
+    assert evaluate('urlparse_query', data=[url]) == ['page=2107&subpage=1']
+
+
+def test_urlparse_scheme():
+    url = "https://scrapinghub.bamboohr.com/employees/performance/index.php?page=2107&subpage=1#123"
+    assert evaluate('urlparse_scheme', data=[url]) == ['https']
+
+
+def test_urlparse_fragment():
+    url = "https://scrapinghub.bamboohr.com/employees/performance/index.php?page=2107&subpage=1#123"
+    assert evaluate('urlparse_fragment', data=[url]) == ['123']
+
+
+def test_urlparse():
+    url = "https://scrapinghub.bamboohr.com/employees/performance/index.php?page=2107&subpage=1#123"
+    assert list(evaluate('urlparse', data=[url])[0].keys()) == ['scheme', 'netloc', 'path', 'params', 'query', 'fragment']
+
+
 @pytest.mark.parametrize(
     "test_input,expected",
     [