Skip to content

Commit

Permalink
Add ignore_env_mismatch in hub.Module.
Browse files Browse the repository at this point in the history
  • Loading branch information
nepeplwu committed Apr 13, 2021
1 parent 98f173e commit 30aace4
Show file tree
Hide file tree
Showing 3 changed files with 65 additions and 21 deletions.
13 changes: 12 additions & 1 deletion paddlehub/commands/install.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,22 @@

@register(name='hub.install', description='Install PaddleHub module.')
class InstallCommand:
def __init__(self):
self.parser = argparse.ArgumentParser(prog='hub install', add_help=True)
self.parser.add_argument(
'--ignore_env_mismatch',
action='store_true',
help='Whether to ignore the environment mismatch when installing the Module.')

def execute(self, argv: List) -> bool:
if not argv:
print("ERROR: You must give at least one module to install.")
return False

options = [arg for arg in argv if arg.startswith('-')]
argv = [arg for arg in argv if not arg.startswith('-')]
args = self.parser.parse_args(options)

manager = LocalModuleManager()
for _arg in argv:
if os.path.exists(_arg) and os.path.isdir(_arg):
Expand All @@ -41,5 +52,5 @@ def execute(self, argv: List) -> bool:
name = _arg[0]
version = None if len(_arg) == 1 else _arg[1]
CacheUpdater("hub_install", name, version).start()
manager.install(name=name, version=version)
manager.install(name=name, version=version, ignore_env_mismatch=args.ignore_env_mismatch)
return True
39 changes: 29 additions & 10 deletions paddlehub/module/manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -155,19 +155,21 @@ def install(self,
version: str = None,
source: str = None,
update: bool = False,
branch: str = None) -> HubModule:
branch: str = None,
ignore_env_mismatch: bool = False) -> HubModule:
'''
Install a HubModule from name or directory or archive file or url. When installing with the name parameter, if a
module that meets the conditions (both name and version) already installed, the installation step will be skipped.
When installing with other parameter, The locally installed modules will be uninstalled.
Args:
name (str|optional): module name to install
directory (str|optional): directory containing module code
archive (str|optional): archive file containing module code
url (str|optional): url points to a archive file containing module code
version (str|optional): module version, use with name parameter
source (str|optional): source containing module code, use with name paramete
name (str|optional): module name to install
directory (str|optional): directory containing module code
archive (str|optional): archive file containing module code
url (str|optional): url points to a archive file containing module code
version (str|optional): module version, use with name parameter
source (str|optional): source containing module code, use with name paramete
ignore_env_mismatch (str|optional): Whether to ignore the environment mismatch when installing the Module.
'''
if name:

Expand All @@ -185,7 +187,7 @@ def install(self,
return hub_module_cls
if source:
return self._install_from_source(name, version, source, update, branch)
return self._install_from_name(name, version)
return self._install_from_name(name, version, ignore_env_mismatch)
elif directory:
return self._install_from_directory(directory)
elif archive:
Expand Down Expand Up @@ -255,7 +257,7 @@ def _install_from_url(self, url: str) -> HubModule:

return self._install_from_archive(file)

def _install_from_name(self, name: str, version: str = None) -> HubModule:
def _install_from_name(self, name: str, version: str = None, ignore_env_mismatch: bool = False) -> HubModule:
'''Install HubModule by name search result'''
result = module_server.search_module(name=name, version=version)
for item in result:
Expand All @@ -277,7 +279,24 @@ def _install_from_name(self, name: str, version: str = None) -> HubModule:

# Cannot find a HubModule that meets the version
if valid_infos:
raise EnvironmentMismatchError(name=name, info=valid_infos, version=version)
if not ignore_env_mismatch:
raise EnvironmentMismatchError(name=name, info=valid_infos, version=version)

# If `ignore_env_mismatch` is set, ignore the problem of environmental mismatch, such as PaddlePaddle or PaddleHub
# version incompatibility. This may cause some unexpected problems during installation or running, but it is useful
# in some cases, for example, the development version of PaddlePaddle(with version number `0.0.0`) is installed
# locally.
if version:
if version in valid_infos:
url = valid_infos[version]['url']
else:
raise HubModuleNotFoundError(name=name, info=module_infos, version=version)
else:
# Get the maximum version number.
version = sorted([utils.Version(_v) for _v in valid_infos.keys()])[-1]
url = valid_infos[str(version)]['url']
log.logger.warning('Ignore environmental mismatch of The Module {}-{}'.format(name, version))
return self._install_from_url(url)
raise HubModuleNotFoundError(name=name, info=module_infos, version=version)

def _install_from_source(self, name: str, version: str, source: str, update: bool = False,
Expand Down
34 changes: 24 additions & 10 deletions paddlehub/module/module.py
Original file line number Diff line number Diff line change
Expand Up @@ -359,16 +359,16 @@ class Module(object):
name(str): Module name.
directory(str|optional): Directory of the module to be loaded, only takes effect when the `name` is not specified.
version(str|optional): The version limit of the module, only takes effect when the `name` is specified. When the local
Module does not meet the specified version conditions, PaddleHub will re-request the server to
download the appropriate Module. Default to None, This means that the local Module will be used.
If the Module does not exist, PaddleHub will download the latest version available from the
server according to the usage environment.
Module does not meet the specified version conditions, PaddleHub will re-request the server to download the
appropriate Module. Default to None, This means that the local Module will be used. If the Module does not exist,
PaddleHub will download the latest version available from the server according to the usage environment.
source(str|optional): Url of a git repository. If this parameter is specified, PaddleHub will no longer download the
specified Module from the default server, but will look for it in the specified repository.
Default to None.
update(bool|optional): Whether to update the locally cached git repository, only takes effect when the `source`
is specified. Default to False.
specified Module from the default server, but will look for it in the specified repository. Default to None.
update(bool|optional): Whether to update the locally cached git repository, only takes effect when the `source` is
specified. Default to False.
branch(str|optional): The branch of the specified git repository. Default to None.
ignore_env_mismatch(bool|optional): Whether to ignore the environment mismatch when installing the Module. Default to
False.
'''

def __new__(cls,
Expand All @@ -379,13 +379,20 @@ def __new__(cls,
source: str = None,
update: bool = False,
branch: str = None,
ignore_env_mismatch: bool = False,
**kwargs):
if cls.__name__ == 'Module':
from paddlehub.server.server import CacheUpdater
# This branch come from hub.Module(name='xxx') or hub.Module(directory='xxx')
if name:
module = cls.init_with_name(
name=name, version=version, source=source, update=update, branch=branch, **kwargs)
name=name,
version=version,
source=source,
update=update,
branch=branch,
ignore_env_mismatch=ignore_env_mismatch,
**kwargs)
CacheUpdater("update_cache", module=name, version=version).start()
elif directory:
module = cls.init_with_directory(directory=directory, **kwargs)
Expand Down Expand Up @@ -470,13 +477,20 @@ def init_with_name(cls,
source: str = None,
update: bool = False,
branch: str = None,
ignore_env_mismatch: bool = False,
**kwargs) -> Union[RunModule, ModuleV1]:
'''Initialize Module according to the specified name.'''
from paddlehub.module.manager import LocalModuleManager
manager = LocalModuleManager()
user_module_cls = manager.search(name, source=source, branch=branch)
if not user_module_cls or not user_module_cls.version.match(version):
user_module_cls = manager.install(name=name, version=version, source=source, update=update, branch=branch)
user_module_cls = manager.install(
name=name,
version=version,
source=source,
update=update,
branch=branch,
ignore_env_mismatch=ignore_env_mismatch)

directory = manager._get_normalized_path(user_module_cls.name)

Expand Down

0 comments on commit 30aace4

Please sign in to comment.