make managers plugins as well and expand ec2 manager

bin/employ

@@ -5,8 +5,8 @@ Employ instance to do your bidding
 Usage:
   employ --help
   employ --version
-  employ help (commands | command <command> | regions)
-  employ run <command> <config_file> [<setup_script> ...]
+  employ help (commands | command <command> | managers | manager <manager>)
+  employ <manager> run <config_file> [<setup_script> ...]
 
 Global Options:
   -h, --help                    Show this message
@@ -15,7 +15,9 @@ Global Options:
 Help Commands:
   commands                      List all available commands
   command <command>             Print the docstring for the provided command
-  regions                       List all available regions names
+  managers                      List all available managers
+  manager <manager>             Print the docstring for the provided manager
+
 """
 from ConfigParser import RawConfigParser
 import sys
@@ -23,11 +25,9 @@ import sys
 from docopt import docopt
 
 import employ
-import employ.manager
-
 
 def command_doc(command):
-    all_commands = employ.manager.Manager.available_commands()
+    all_commands = employ.available_commands()
     if all_commands.get(command):
         print all_commands[command].__doc__
     else:
@@ -35,11 +35,25 @@ def command_doc(command):
 
 
 def list_commands(command=None):
-    all_commands = employ.manager.Manager.available_commands()
+    all_commands = employ.available_commands()
     print "List of Available Commands:"
     for cls in all_commands:
         print "  %s" % cls
 
+def manager_doc(manager):
+    all_managers = employ.available_managers()
+    if all_managers.get(manager):
+        print all_managers[manager].__doc__
+    else:
+        sys.exit("Unknown manager: '%s'" % manager)
+
+
+def list_managers(manager=None):
+    all_managers = employ.available_managers()
+    print "List of Available Managers:"
+    for cls in all_managers:
+        print "  %s" % cls
+
 
 def list_regions():
     print "List of Available Regions:"
@@ -47,19 +61,24 @@ def list_regions():
         print "  %s" % region
 
 
-def run(config_file, setup_scripts):
+def run(manager_cls, config_file, setup_scripts):
     config = RawConfigParser(allow_no_value=True)
     config.read(config_file)
     commands = []
 
-    all_commands = employ.manager.Manager.available_commands()
+    all_commands = employ.available_commands()
     for command in config.sections():
-        if command == "employ":
+        if command == "employ" or command == manager_cls:
             continue
         if command not in all_commands:
             sys.exit("Unknown command '%s'" % command)
         commands.append(all_commands[command].from_config(config))
-    manager = employ.manager.Manager.from_config(config)
+
+    all_managers = employ.available_managers()
+    if manager_cls not in all_managers:
+        sys.exit("Unknown manager: '%s'" % manager_cls)
+
+    manager = all_managers[manager_cls].from_config(config)
     with manager:
         for setup_script in setup_scripts:
             manager.setup(setup_script)
@@ -73,7 +92,9 @@ if arguments["help"]:
         list_commands()
     elif arguments["command"] and arguments["<command>"]:
         command_doc(arguments["<command>"])
-    elif arguments["regions"]:
-        list_regions()
+    elif arguments["managers"]:
+        list_managers()
+    elif arguments["manager"] and arguments["<manager>"]:
+        manager_doc(arguments["<manager>"])
 elif arguments["run"]:
-    run(arguments["<config_file>"], arguments["<setup_script>"])
+    run(arguments["<manager>"], arguments["<config_file>"], arguments["<setup_script>"])

employ/__init__.py

@@ -2,5 +2,25 @@ __version__ = "0.1.0"
 from straight.plugin import load
 
 from employ.commands import Command
+from employ.managers import Manager
 
 commands = load("employ.commands", subclasses=Command)
+managers = load("employ.managers", subclasses=Manager)
+
+
+def available_commands():
+    all_commands = {}
+    for command_cls in commands:
+        if command_cls.name == "command":
+            continue
+        all_commands[command_cls.name] = command_cls
+    return all_commands
+
+
+def available_managers():
+    all_managers = {}
+    for manager_cls in managers:
+        if manager_cls.name == "manager":
+            continue
+        all_managers[manager_cls.name] = manager_cls
+    return all_managers

employ/commands/ab.py

@@ -20,7 +20,7 @@ class ABCommand(Command):
       concurrency=100
       keepalive=False
 
-      employ run ab run_ab.ini
+      employ <manager> run run_ab.ini
     """
     name = "ab"
 

employ/manager.py

@@ -1,75 +0,0 @@
-import boto.ec2
-import boto.manage
-
-from employ import commands
-
-
-class Manager(object):
-    def __init__(
-            self, ami_image_id="ami-da0cf8b3", num_instances=1, instance_name="deployed",
-            region="us-east-1", instance_type="t1.micro", key_name=None, security_group=None
-    ):
-        self.instances = []
-        self.ami_image_id = ami_image_id
-        self.num_instances = num_instances
-        self.instance_name = instance_name
-        self.region = region
-        self.instance_type = instance_type
-        self.key_name = key_name
-        self.security_groups = [security_group] if security_group else []
-        self._connection = None
-
-    @classmethod
-    def from_config(cls, config):
-        settings = {}
-        if config.has_section("employ"):
-            settings = dict(config.items("employ"))
-        return cls(**settings)
-
-    @classmethod
-    def available_regions(cls):
-        for region in boto.ec2.regions():
-            yield region.name
-
-    @classmethod
-    def available_commands(cls):
-        all_commands = {}
-        for command_cls in commands:
-            if command_cls.name == "command":
-                continue
-            all_commands[command_cls.name] = command_cls
-        return all_commands
-
-    def connection(self):
-        if not self._connection:
-            logger.info("Connecting to EC2")
-            self._connection = boto.ec2.connect_to_region(self.region)
-        return self._connection
-
-    def setup_instances(self):
-        connection = self.connection()
-        reservation = connection.run_instances(
-            image_id=self.ami_image_id,
-            min_count=self.num_instances,
-            max_count=self.num_instances,
-            instance_type=self.instance_type,
-            key_name=self.key_name,
-            security_groups=self.security_groups,
-        )
-        self.instances = reservation.instances
-
-    __enter__ = setup_instances
-
-    def cleanup_instances(self):
-        instance_ids = [instance.id for instance in self.instances]
-        connection = self.connection()
-        connection.terminate_instances(instance_ids=instance_ids)
-
-    def __exit__(self, type, value, traceback):
-        self.cleanup_instances()
-
-    def setup(self, script):
-        print "executing setup script: %s" % script
-
-    def run(self, command):
-        print "running command '%s'" % command.command()

employ/managers/__init__.py

@@ -0,0 +1,30 @@
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)
+
+
+class Manager(object):
+    name = "manager"
+
+    @classmethod
+    def from_config(cls, config):
+        settings = {}
+        if config.has_section(cls.name):
+            settings = dict(config.items(cls.name))
+        return cls(**settings)
+
+    def setup_instances(self):
+        raise NotImplementedError()
+
+    __enter__ = setup_instances
+
+    def cleanup_instances(self):
+        raise NotImplementedError()
+
+    def __exit__(self, type, value, traceback):
+        self.cleanup_instances()
+
+    def setup(self, script):
+        raise NotImplementedError()
+
+    def run(self, command):
+        raise NotImplementedError()

employ/managers/ec2.py

@@ -0,0 +1,100 @@
+import time
+
+import boto.ec2
+from boto.manage.cmdshell import sshclient_from_instance
+
+from employ.managers import Manager
+
+
+class EC2Manager(Manager):
+    """
+    Employ Manager which creates instances in EC2
+
+    Config Parameters:
+      [ec2]
+      ami_image_id = ami-da0cf8b3
+      num_instances = 1
+      instance_name = employed
+      region = us-east-1
+      instance_type = t1.micro
+      key_name = some_key
+      security_group = default
+      user_name = root
+      host_key = ~/.ssh/known_hosts
+      ssh_pwd = None
+
+      ; when starting instances this manager will block until
+      ; all instances have the state "running", this interval
+      ; is how long the manager will wait between checking states
+      wait_interval = 5
+    """
+    name = "ec2"
+
+    def __init__(
+            self, ami_image_id="ami-da0cf8b3", num_instances=1, instance_name="employed",
+            region="us-east-1", instance_type="t1.micro", key_name=None, security_group="default",
+            user_name="root", host_key="~/.ssh/known_hosts", ssh_pwd=None, wait_interval=5
+    ):
+        self.instances = []
+        self.ami_image_id = ami_image_id
+        self.num_instances = num_instances
+        self.instance_name = instance_name
+        self.region = region
+        self.instance_type = instance_type
+        self.key_name = key_name
+        self.security_groups = [security_group] if security_group else []
+        self.user_name = user_name
+        self.host_key = host_key
+        self.ssh_pwd = ssh_pwd
+        self.wait_interval = wait_interval
+        self._connection = None
+
+    @classmethod
+    def available_regions(cls):
+        for region in boto.ec2.regions():
+            yield region.name
+
+    def connection(self):
+        if not self._connection:
+            self._connection = boto.ec2.connect_to_region(self.region)
+        return self._connection
+
+    def instance_ids(self):
+        return [instance.id for instance in self.instances]
+
+    def setup_instances(self):
+        print "starting instances"
+        connection = self.connection()
+        reservation = connection.run_instances(
+            image_id=self.ami_image_id,
+            min_count=self.num_instances,
+            max_count=self.num_instances,
+            instance_type=self.instance_type,
+            key_name=self.key_name,
+            security_groups=self.security_groups,
+        )
+        self.instances = reservation.instances
+        connection.create_tags(self.instance_ids(), {"Name": self.instance_name})
+
+        print "waiting until they are all running"
+        while not all(instance.update() == "running" for instance in self.instances):
+            time.sleep(self.wait_interval)
+
+    __enter__ = setup_instances
+
+    def cleanup_instances(self):
+        connection = self.connection()
+        connection.terminate_instances(instance_ids=self.instance_ids())
+
+    def __exit__(self, type, value, traceback):
+        self.cleanup_instances()
+
+    def setup(self, script):
+        print "setup script: %s" % script
+
+    def run(self, command):
+        # shell = sshclient_from_instance(
+        #     self.instances[0], self.host_key, user_name=self.user_name
+        # )
+        # command.aggregate(shell.run(command.command()))
+        print "running command: %s" % command.command()