Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Dev #54

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Dev #54

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f3b906a
Restructed metadrive
Jun 28, 2019
24e01f9
Restructed metadrive
Jun 28, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Empty file removed .gitmodules
View file
Open in desktop
Empty file.
15 changes: 15 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
clean:
find . -type d -name "__pycache__" -exec rm -rf {} + > /dev/null 2>&1
find . -type f -name "*.pyc" -exec rm -rf {} + > /dev/null 2>&1

isort:
isort -rc metadrive

lint:
flake8 --show-source metadrive
isort --check-only -rc metadrive --diff

test:
pytest metadrive

all: clean lint test
52 changes: 25 additions & 27 deletions metadrive/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,26 +1,22 @@
import yaml
import pkgutil
import inspect
import re
import os
import pkgutil

from metaform import get_schema

from metadrive import utils
from metadrive import drives, utils, config

__all__ = []
for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):
for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):
__all__.append(module_name)
_module = loader.find_module(module_name).load_module(module_name)
globals()[module_name] = _module

import os, config

if not os.path.exists(config.DEFAULT_LOCATION):
os.makedirs(config.DEFAULT_LOCATION)


from metaform import get_schema
from metadrive import drives


def load(data, iterator=False):
'''
takes: saved data item
Expand All @@ -38,6 +34,7 @@ def load(data, iterator=False):
elif isinstance(data, dict):
return instantiate(data)


def search(source, features: dict):
'''
Object discovery by features in source.
Expand All @@ -51,7 +48,8 @@ def search(source, features: dict):

Note: url itself is a feature, too.
'''
raise NotImplemented
raise NotImplementedError


def create(source, features: dict):
'''
Expand All @@ -64,7 +62,7 @@ def create(source, features: dict):
Returns:
object's address, and success status and/or errors.
'''
raise NotImplemented
raise NotImplementedError


def read(term, limit=None):
Expand All @@ -89,7 +87,7 @@ def read(term, limit=None):
if isinstance(readers, list):

for i, reader in enumerate(readers):
print(i+1, reader)
print(i + 1, reader)

reader_id = input("Choose reader [1] ")

Expand All @@ -98,7 +96,7 @@ def read(term, limit=None):
else:
reader_id = int(reader_id)

if reader_id not in range(1, len(readers)+1):
if reader_id not in range(1, len(readers) + 1):
raise Exception("The choice does not exist.")

reader_id -= 1
Expand All @@ -109,10 +107,8 @@ def read(term, limit=None):
else:
raise Exception("Reader defined as anything other than string or list is not supported.")


package = utils.ensure_driver_installed(reader)


module = __import__(package)

# Get method and package:
Expand All @@ -129,29 +125,31 @@ def read(term, limit=None):
else:
return method()


def instantiate(data):
_id = data.get('-')
_drive = data.get('@')

if _id is not None and _drive is not None:

# parsing '@' field: #sample: PyPI::halfbakery_driver==0.0.1:default.api.Topic
# parsing '@' field:
# sample: PyPI::halfbakery_driver==0.0.1:default.api.Topic

# TBD: refactor by importing from metatype
packman = _drive.split('::', 1)[0] #sample: PyPI (Conan, NPM, Paket, etc.)
drivespec = _drive.split('::', 1)[-1] #sample: halfbakery_driver==0.0.1:default.api.Topic
driver_name_version = drivespec.split(':',1)[0] #sample: halfbakery_driver==0.0.1
driver_name = driver_name_version.split('==',1)[0] #sample: halfbakery_driver
driver_version = driver_name_version.rsplit('==',1)[-1] #sample: 0.0.1
profile_name_pkg_path = drivespec.rsplit(':',1)[-1] #sample: default.api.Topic
profile_name = profile_name_pkg_path.split('.',1)[0] #sample: default
pkg_path = profile_name_pkg_path.split('.',1)[-1] #sample: api.Topic
# packman = _drive.split('::', 1)[0] # sample: PyPI (Conan, NPM, Paket, etc.)
drivespec = _drive.split('::', 1)[-1] # sample: halfbakery_driver==0.0.1:default.api.Topic
driver_name_version = drivespec.split(':', 1)[0] # sample: halfbakery_driver==0.0.1
driver_name = driver_name_version.split('==', 1)[0] # sample: halfbakery_driver
# driver_version = driver_name_version.rsplit('==', 1)[-1] # sample: 0.0.1
profile_name_pkg_path = drivespec.rsplit(':', 1)[-1] # sample: default.api.Topic
profile_name = profile_name_pkg_path.split('.', 1)[0] # sample: default
# pkg_path = profile_name_pkg_path.split('.', 1)[-1] # sample: api.Topic

# TBD: refactor by reusing metadrive/api.py# around 90 line
ndriver = driver_name.replace('-', '_')
module = __import__(ndriver)
# module = __import__(ndriver)
api = __import__('{}.api'.format(ndriver), fromlist=[ndriver])
classname = _drive.rsplit('.',1)[-1]
classname = _drive.rsplit('.', 1)[-1]
Klass = getattr(api, classname)

item = Klass(data)
Expand Down
Binary file removed metadrive/_os_static/icons/item.png
View file
Open in desktop
Binary file not shown.
20 changes: 9 additions & 11 deletions metadrive/_requests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
import os
import inspect
import pathlib
import os

import requests
from metadrive import config
from metadrive import utils
from metadrive import mixins

from metadrive import config, mixins, utils

SUBTOOL = os.path.basename(__file__).split('.py')[0]


def get_session(*args, **kwargs):
return requests.Session(*args, **kwargs)


class RequestsDrive(requests.Session):

def __init__(self, *args, **kwargs):
Expand All @@ -30,7 +31,6 @@ def get(self, *args, **kwargs):
'https': 'socks5h://{}'.format(socks)}
kwargs.update({'proxies': proxies})


self.response = super().get(*args, **kwargs)

if hasattr(self, 'profile'):
Expand All @@ -39,7 +39,7 @@ def get(self, *args, **kwargs):
session_prefix_file = os.path.join(
SUBTOOL, '{drive_id}/cookies.json'.format(
drive_id=self.profile
))
))

utils.save_session_data(session_prefix_file, session_data)

Expand All @@ -53,7 +53,7 @@ def get_drive(
recreate_profile=False,
proxies='default'):

## ----------- TO MOVE TO MIXIN --------------- #
# ----------- TO MOVE TO MIXIN --------------- #
proxy = mixins.set_proxies(proxies)
local = mixins.init_profile(profile, porfiles_dir, recreate_profile)

Expand All @@ -66,13 +66,12 @@ def get_drive(
else:
drive = None


if drive is not None:

session_prefix_file = os.path.join(
drive.subtool, '{drive_id}/cookies.json'.format(
drive_id=profile
))
))

if os.path.exists(os.path.join(config.SESSIONS_DIR, session_prefix_file)):
session_data = utils.load_session_data(session_prefix_file)
Expand All @@ -92,7 +91,6 @@ def get_drive(
{'proxy': proxy}
)


# LATER MOVE TO MIXIN
drive.caller_module = inspect.getmodule(inspect.currentframe().f_back)

Expand Down
70 changes: 36 additions & 34 deletions metadrive/_selenium.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,18 @@
# To create selenium driver may use something like:
docker run -d -p 4444:4444 selenium/standalone-chrome:3.7.1-beryllium
'''
import os
import inspect
import os
import pathlib

from deprecated import deprecated
from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.action_chains import ActionChains

from metadrive import config
from deprecated import deprecated

# from selenium.webdriver.support.wait import WebDriverWait


class TabsMixin:

Expand All @@ -29,6 +30,7 @@ def open_tab(self, name, url=None):
raise Exception('Tab not found, and url not provided.')
else:
self.switch_to.window(self.tabs[name])

def current_tab(self):
return next(filter(lambda x: x[1] == self.current_window_handle, self.tabs.items()))

Expand All @@ -47,25 +49,26 @@ def __init__(self, *args, **kwargs):
self.tabs = {'default': self.current_window_handle}
self.metaname = ''


class Remote(webdriver.Remote, TabsMixin):

def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.tabs = {'default': self.current_window_handle}
self.metaname = ''

def get_drive(
driver_location=config.CHROME_DRIVER,
profile='default',
porfiles_dir='.metadrive/sessions/_selenium',
headless=config.SELENIUM.get('headless') == 'True' or False,
load_images=True,
load_adblocker=True,
recreate_profile=False,
download_to='',
proxies='default',
):

def get_drive(
driver_location=config.CHROME_DRIVER,
profile='default',
porfiles_dir='.metadrive/sessions/_selenium',
headless=config.SELENIUM.get('headless') == 'True' or False,
load_images=True,
load_adblocker=True,
recreate_profile=False,
download_to='',
proxies='default',
):
'''
Gets a new browser, with session in specific directory.

Expand Down Expand Up @@ -143,9 +146,8 @@ def get_drive(
# OPTIONS.add_argument('--ssl-protocol=any')
# OPTIONS.add_argument('--web-security=no')


PREFERENCES = {}
#OPTIONS.experimental_options["prefs"] = PREFERENCES
# OPTIONS.experimental_options["prefs"] = PREFERENCES

if headless:
OPTIONS.add_argument('--headless')
Expand All @@ -159,7 +161,7 @@ def get_drive(

if download_to:
PREFERENCES.update(
{'download.default_directory' : download_to,
{'download.default_directory': download_to,
'download.prompt_for_download': False,
'download.directory_upgrade': True,
'safebrowsing.enabled': False,
Expand All @@ -172,7 +174,7 @@ def get_drive(
# OPTIONS.add_extension(os.path.join(os.getcwd(), 'subtools/extensions/ghostery.crx'))
OPTIONS.add_extension(
os.path.join(os.getcwd(), 'subtools/selenium/extensions/ublock_origin.crx'))
except:
except Exception:
pass

# ------------- INITIALIZATION SECTION ------------ #
Expand All @@ -186,7 +188,6 @@ def get_drive(
CHROME_DRIVER_LOCATION = driver_location
local = True


if local:
profile_path = os.path.join(
str(pathlib.Path.home()),
Expand All @@ -196,7 +197,7 @@ def get_drive(

if profile_path is not None:

OPTIONS.add_argument("--user-data-dir={}".format(profile_path));
OPTIONS.add_argument("--user-data-dir={}".format(profile_path))

if not profile_path:
os.makedirs(profile_path)
Expand Down Expand Up @@ -247,18 +248,19 @@ def get_drive(

return browser


@deprecated(reason="Use get_drive() instead.")
def get_driver(
driver_location=config.CHROME_DRIVER,
profile='default',
porfiles_dir='.metadrive/sessions/_selenium',
headless=False,
load_images=True,
load_adblocker=True,
recreate_profile=False,
download_to='',
proxies='default',
):
driver_location=config.CHROME_DRIVER,
profile='default',
porfiles_dir='.metadrive/sessions/_selenium',
headless=False,
load_images=True,
load_adblocker=True,
recreate_profile=False,
download_to='',
proxies='default',
):

return get_drive(
driver_location=driver_location,
Expand All @@ -285,15 +287,15 @@ def save_as(element, driver):
from PIL import Image
from io import BytesIO

as__ = Image.open(
as__ = Image.open( # noqa
BytesIO(
base64.b64decode(
'iVBORw0KGgoAAAANSUhEUgAAACMAAAAQCAIAAAATVVENAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAB3RJTUUH4gsTAjMlHs+8UAAAAB1pVFh0Q29tbWVudAAAAAAAQ3JlYXRlZCB3aXRoIEdJTVBkLmUHAAABrElEQVQ4y2P8//8/A10AEwO9wKhNlAAWZM7SFStXrVn78tUrXl5eTzfXvOwsFhYWiPiSZcs/ffrExsbm7elRUlhAjlX/kcCOXbsfPX7879+/e/fvewcELVm2/P///w8ePDSxsrl3//7///+/fft26cqV/2QBlNBzd3WRlZFhZGRUVFAIDw05efo0AwMDCwvL////r9+4+fnLF05OTl1tbfJCjxE5Px06cnThkiVPnjxlYGD4/uO7nKzckvlzGRgY9h04uHzVqsuXr6ioqKQlJ9nZWFMUem/fvTM0tzx4+Mjfv3////+/eOmy6PhEZO///Plz1dq1RhZWX758oSj0vn/7/u/fPw01VSYmpg8fP67buBEi/vDho+MnT/78+ZONjU1QQJCRkZGJmZmBgeHYiRPzFy2GqHnz9m3vhIlv3r6FcOcvWnzsxAmcaU9aWqogJzs5I0tQUICPl8/R3v7kqdMMDAw/f/2cOn3m/QcPGJmYpKUk+7o6OTk4GBgYLl66vH3XrsS4WAYGhg8fPixauszf10dEWJiBgWH9pk2ebm5WFhY442m0jBi0NgEAiOU/HUCPdjUAAAAASUVORK5CYII=')))
'iVBORw0KGgoAAAANSUhEUgAAACMAAAAQCAIAAAATVVENAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAB3RJTUUH4gsTAjMlHs+8UAAAAB1pVFh0Q29tbWVudAAAAAAAQ3JlYXRlZCB3aXRoIEdJTVBkLmUHAAABrElEQVQ4y2P8//8/A10AEwO9wKhNlAAWZM7SFStXrVn78tUrXl5eTzfXvOwsFhYWiPiSZcs/ffrExsbm7elRUlhAjlX/kcCOXbsfPX7879+/e/fvewcELVm2/P///w8ePDSxsrl3//7///+/fft26cqV/2QBlNBzd3WRlZFhZGRUVFAIDw05efo0AwMDCwvL////r9+4+fnLF05OTl1tbfJCjxE5Px06cnThkiVPnjxlYGD4/uO7nKzckvlzGRgY9h04uHzVqsuXr6ioqKQlJ9nZWFMUem/fvTM0tzx4+Mjfv3////+/eOmy6PhEZO///Plz1dq1RhZWX758oSj0vn/7/u/fPw01VSYmpg8fP67buBEi/vDho+MnT/78+ZONjU1QQJCRkZGJmZmBgeHYiRPzFy2GqHnz9m3vhIlv3r6FcOcvWnzsxAmcaU9aWqogJzs5I0tQUICPl8/R3v7kqdMMDAw/f/2cOn3m/QcPGJmYpKUk+7o6OTk4GBgYLl66vH3XrsS4WAYGhg8fPixauszf10dEWJiBgWH9pk2ebm5WFhY442m0jBi0NgEAiOU/HUCPdjUAAAAASUVORK5CYII='))) # noqa

position = pyautogui.locateOnScreen(as__)

if position:
print(position)
pyautogui.click(x=position[0],y=position[1])
pyautogui.click(x=position[0], y=position[1])
else:
print('Could find "Save as ..." position. Try local, non-headless.')
Loading