Move code out of __init__.py files, attempt #2.

This includes getting rid of command registration magic,
since it's complicated and commands are going away entirely anyway.

(sapling split of 426a4766355a6b14bed36e0c31cbbdaa32a30d69)

src/python/twitter/pants/bin/pants_exe.py

@@ -27,7 +27,8 @@
 from twitter.pants.base.address import Address
 from twitter.pants.base.config import Config
 from twitter.pants.base.rcfile import RcFile
-from twitter.pants.commands import Command
+from twitter.pants.commands.command import Command
+from twitter.pants.commands.register import register_commands
 from twitter.pants.goal.initialize_reporting import initial_reporting
 from twitter.pants.goal.run_tracker import RunTracker
 from twitter.pants.reporting.report import Report
@@ -83,6 +84,7 @@ def _add_default_options(command, args):
 
 
 def _synthesize_command(root_dir, args):
+  register_commands()
   command = args[0]
 
   if command in Command.all_commands():

src/python/twitter/pants/commands/BUILD

@@ -1,6 +1,6 @@
 python_library(
   name = 'command',
-  sources = ['__init__.py'],
+  sources = ['command.py', 'register.py'],
   dependencies = [
     pants('src/python/twitter/common/collections'),
     pants('src/python/twitter/pants/base:build_file'),

src/python/twitter/pants/commands/__init__.py

@@ -1,120 +0,0 @@
-# ==================================================================================================
-# Copyright 2011 Twitter, Inc.
-# --------------------------------------------------------------------------------------------------
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this work except in compliance with the License.
-# You may obtain a copy of the License in the LICENSE file, or at:
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==================================================================================================
-
-from __future__ import print_function
-
-import inspect
-import sys
-
-from twitter.common.collections import OrderedSet
-from twitter.pants.base.build_file import BuildFile
-from twitter.pants.base.target import Target
-
-
-class Command(object):
-  """Baseclass for all pants subcommands."""
-
-  @staticmethod
-  def get_command(name):
-    return Command._commands.get(name, None)
-
-  @staticmethod
-  def all_commands():
-    return Command._commands.keys()
-
-  _commands = {}
-
-  @staticmethod
-  def _register_modules():
-    """Register all 'Command's from all modules in the current directory."""
-    import pkgutil
-    for _, mod, ispkg in pkgutil.iter_modules(__path__):
-      if ispkg: continue
-      fq_module = '.'.join([__name__, mod])
-      __import__(fq_module)
-      for (_, kls) in inspect.getmembers(sys.modules[fq_module], inspect.isclass):
-        if issubclass(kls, Command):
-          command_name = kls.__dict__.get('__command__', None)
-          if command_name:
-            Command._commands[command_name] = kls
-
-  @staticmethod
-  def scan_addresses(root_dir, base_path=None):
-    """Parses all targets available in BUILD files under base_path and
-    returns their addresses.  If no base_path is specified, root_dir is
-    assumed to be the base_path"""
-
-    addresses = OrderedSet()
-    for buildfile in BuildFile.scan_buildfiles(root_dir, base_path):
-      addresses.update(Target.get_all_addresses(buildfile))
-    return addresses
-
-  @classmethod
-  def serialized(cls):
-    return False
-
-  def __init__(self, run_tracker, root_dir, parser, args):
-    """run_tracker: The (already opened) RunTracker to track this run with
-    root_dir: The root directory of the pants workspace
-    parser: an OptionParser
-    args: the subcommand arguments to parse"""
-    self.run_tracker = run_tracker
-    self.root_dir = root_dir
-
-    # Override the OptionParser's error with more useful output
-    def error(message=None, show_help=True):
-      if message:
-        print(message + '\n')
-      if show_help:
-        parser.print_help()
-      parser.exit(status=1)
-    parser.error = error
-    self.error = error
-
-    self.setup_parser(parser, args)
-    self.options, self.args = parser.parse_args(args)
-    self.parser = parser
-
-  def setup_parser(self, parser, args):
-    """Subclasses should override and confiure the OptionParser to reflect
-    the subcommand option and argument requirements.  Upon successful
-    construction, subcommands will be able to access self.options and
-    self.args."""
-
-    pass
-
-  def error(self, message=None, show_help=True):
-    """Reports the error message, optionally followed by pants help, and then exits."""
-
-  def run(self, lock):
-    """Subcommands that are serialized() should override if they need the ability to interact with
-    the global command lock.
-    The value returned should be an int, 0 indicating success and any other value indicating
-    failure."""
-    return self.execute()
-
-  def execute(self):
-    """Subcommands that do not require serialization should override to perform the command action.
-    The value returned should be an int, 0 indicating success and any other value indicating
-    failure."""
-    raise NotImplementedError('Either run(lock) or execute() must be over-ridden.')
-
-  def cleanup(self):
-    """Called on SIGINT (e.g., when the user hits ctrl-c).
-    Subcommands may override to perform cleanup before exit."""
-    pass
-
-Command._register_modules()

src/python/twitter/pants/commands/build.py

@@ -24,12 +24,10 @@
 from twitter.pants.base.address import Address
 from twitter.pants.base.config import Config
 from twitter.pants.base.target import Target
-from twitter.pants.targets.internal import InternalTarget
+from twitter.pants.commands.command import Command
 from twitter.pants.python.interpreter_cache import PythonInterpreterCache
 from twitter.pants.python.python_builder import PythonBuilder
 
-from . import Command
-
 
 class Build(Command):
   """Builds a specified target."""

src/python/twitter/pants/commands/command.py

@@ -0,0 +1,108 @@
+# ==================================================================================================
+# Copyright 2011 Twitter, Inc.
+# --------------------------------------------------------------------------------------------------
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this work except in compliance with the License.
+# You may obtain a copy of the License in the LICENSE file, or at:
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==================================================================================================
+
+from __future__ import print_function
+
+from twitter.common.collections import OrderedSet
+from twitter.pants.base.build_file import BuildFile
+from twitter.pants.base.target import Target
+
+
+class Command(object):
+  """Baseclass for all pants subcommands."""
+
+  @staticmethod
+  def get_command(name):
+    return Command._commands.get(name, None)
+
+  @staticmethod
+  def all_commands():
+    return Command._commands.keys()
+
+  _commands = {}
+
+  @classmethod
+  def _register(cls):
+    """Register a command class."""
+    command_name = cls.__dict__.get('__command__', None)
+    if command_name:
+      Command._commands[command_name] = cls
+
+  @staticmethod
+  def scan_addresses(root_dir, base_path=None):
+    """Parses all targets available in BUILD files under base_path and
+    returns their addresses.  If no base_path is specified, root_dir is
+    assumed to be the base_path"""
+
+    addresses = OrderedSet()
+    for buildfile in BuildFile.scan_buildfiles(root_dir, base_path):
+      addresses.update(Target.get_all_addresses(buildfile))
+    return addresses
+
+  @classmethod
+  def serialized(cls):
+    return False
+
+  def __init__(self, run_tracker, root_dir, parser, args):
+    """run_tracker: The (already opened) RunTracker to track this run with
+    root_dir: The root directory of the pants workspace
+    parser: an OptionParser
+    args: the subcommand arguments to parse"""
+    self.run_tracker = run_tracker
+    self.root_dir = root_dir
+
+    # Override the OptionParser's error with more useful output
+    def error(message=None, show_help=True):
+      if message:
+        print(message + '\n')
+      if show_help:
+        parser.print_help()
+      parser.exit(status=1)
+    parser.error = error
+    self.error = error
+
+    self.setup_parser(parser, args)
+    self.options, self.args = parser.parse_args(args)
+    self.parser = parser
+
+  def setup_parser(self, parser, args):
+    """Subclasses should override and confiure the OptionParser to reflect
+    the subcommand option and argument requirements.  Upon successful
+    construction, subcommands will be able to access self.options and
+    self.args."""
+
+    pass
+
+  def error(self, message=None, show_help=True):
+    """Reports the error message, optionally followed by pants help, and then exits."""
+
+  def run(self, lock):
+    """Subcommands that are serialized() should override if they need the ability to interact with
+    the global command lock.
+    The value returned should be an int, 0 indicating success and any other value indicating
+    failure."""
+    return self.execute()
+
+  def execute(self):
+    """Subcommands that do not require serialization should override to perform the command action.
+    The value returned should be an int, 0 indicating success and any other value indicating
+    failure."""
+    raise NotImplementedError('Either run(lock) or execute() must be over-ridden.')
+
+  def cleanup(self):
+    """Called on SIGINT (e.g., when the user hits ctrl-c).
+    Subcommands may override to perform cleanup before exit."""
+    pass

src/python/twitter/pants/commands/goal.py

@@ -45,8 +45,9 @@
 from twitter.pants.base.run_info import RunInfo
 from twitter.pants.base.target import Target, TargetDefinitionException
 from twitter.pants.base.workunit import WorkUnit
-from twitter.pants.commands import Command
-from twitter.pants.engine import Engine, GroupEngine
+from twitter.pants.commands.command import Command
+from twitter.pants.engine.engine import Engine
+from twitter.pants.engine.group_engine import GroupEngine
 from twitter.pants.goal import Context, GoalError, Phase
 from twitter.pants.goal import Goal as goal, Group as group
 from twitter.pants.goal.initialize_reporting import update_reporting

src/python/twitter/pants/commands/help.py

@@ -14,10 +14,10 @@
 # limitations under the License.
 # ==================================================================================================
 
-from . import Command
-
 from copy import copy
 
+from twitter.pants.commands.command import Command
+
 
 class Help(Command):
   """Provides help for available commands or a single specified command."""

src/python/twitter/pants/commands/py.py

@@ -21,15 +21,14 @@
 import sys
 import tempfile
 
-from . import Command
-
 from twitter.common.python.pex import PEX
 from twitter.common.python.pex_builder import PEXBuilder
 
 from twitter.pants.base.address import Address
 from twitter.pants.base.config import Config
 from twitter.pants.base.parse_context import ParseContext
 from twitter.pants.base.target import Target
+from twitter.pants.commands.command import Command
 from twitter.pants.python.interpreter_cache import PythonInterpreterCache
 from twitter.pants.python.python_chroot import PythonChroot
 from twitter.pants.targets.python_binary import PythonBinary

src/python/twitter/pants/commands/register.py

@@ -0,0 +1,10 @@
+
+from twitter.pants.commands.build import Build
+from twitter.pants.commands.goal import Goal
+from twitter.pants.commands.help import Help
+from twitter.pants.commands.py import Py
+from twitter.pants.commands.setup_py import SetupPy
+
+def register_commands():
+  for cmd in (Build, Goal, Help, Py, SetupPy):
+    cmd._register()

src/python/twitter/pants/commands/setup_py.py

@@ -29,6 +29,7 @@
 from twitter.pants.base.address import Address
 from twitter.pants.base.config import Config
 from twitter.pants.base.target import Target, TargetDefinitionException
+from twitter.pants.commands.command import Command
 from twitter.pants.python.antlr_builder import PythonAntlrBuilder
 from twitter.pants.python.thrift_builder import PythonThriftBuilder
 from twitter.pants.targets.python_antlr_library import PythonAntlrLibrary
@@ -37,8 +38,6 @@
 from twitter.pants.targets.python_target import PythonTarget
 from twitter.pants.targets.python_thrift_library import PythonThriftLibrary
 
-from . import Command
-
 
 SETUP_BOILERPLATE = """
 # DO NOT EDIT THIS FILE -- AUTOGENERATED BY PANTS

src/python/twitter/pants/engine/__init__.py

@@ -1,24 +0,0 @@
-# ==================================================================================================
-# Copyright 2013 Twitter, Inc.
-# --------------------------------------------------------------------------------------------------
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this work except in compliance with the License.
-# You may obtain a copy of the License in the LICENSE file, or at:
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==================================================================================================
-
-from .engine import Engine, Timer
-from .group_engine import GroupEngine
-
-__all__ = (
-  'Engine',
-  'GroupEngine',
-  'Timer',
-)

tests/python/twitter/pants/engine/test_engine.py

@@ -16,7 +16,7 @@
 
 import mox
 
-from twitter.pants.engine import Engine, Timer
+from twitter.pants.engine.engine import Engine, Timer
 from twitter.pants.tasks import TaskError
 
 from ..base.context_utils import create_context

tests/python/twitter/pants/engine/test_group_engine.py

@@ -19,8 +19,7 @@
 
 from textwrap import dedent
 
-from twitter.pants.engine import GroupEngine
-from twitter.pants.engine.group_engine import GroupIterator, GroupMember
+from twitter.pants.engine.group_engine import GroupEngine, GroupIterator, GroupMember
 from twitter.pants.goal import Goal, Group
 from twitter.pants.tasks import Task
 from twitter.pants.tasks.check_exclusives import ExclusivesMapping