diff --git a/README.txt b/README.txt index be48ce8..3c12353 100644 --- a/README.txt +++ b/README.txt @@ -9,11 +9,22 @@ Examples are adding additional, pre-configured webservers to a cluster deployments and creating backups - each with just one call from the commandline. Aw(e)some, indeed, if we may say so... +** Requirements ** + Python2.6 + **Installation** mr.awsome is best installed with easy_install, pip or with zc.recipe.egg in a buildout. It installs two scripts, ``aws`` and ``assh``. +A simple directory structure for a project: + ec2-project/ + etc/ + aws.conf + fabfile.py + dbstartup.sh + webstartup.sh + **Configuration** To authorize itself against AWS, mr.awsome uses the following two environment @@ -28,6 +39,10 @@ You can also put them into files and point to them in the ``[aws]`` section with the ``access-key-id`` and ``secret-access-key`` options. It's best to put them in ``~/.aws/`` and make sure only your user can read them. + [aws] + access-key-id = /home/user/.aws/access-file + secret-access-key = /home/user/.aws/secret-file + All other information about server instances is located in ``aws.conf``, which by default is looked up in ``etc/aws.conf``. @@ -129,7 +144,52 @@ Directly after that follows the binary data of the gzipped startup script. **Snapshots** -(Needs description of volumes in "Configuration") +** EBS Volumes ** +To attach EBS volumes : + + [instance:demo-server] + keypair = default + securitygroups = demo-server + region = eu-west-1 + placement = eu-west-1a + # we use images from `http://alestic.com/`_ + image = ami-a62a01d2 + startup_script = startup-demo-server + fabfile = fabfile.py + volumes = + vol-xxxxx /dev/sdh + vol-yyyyy /dev/sdg + +** Elastic IP *** +You have to allocate the new IP and use it to the instance. The tool will associate the Elastic IP to the instance. + + [instance:demo-server] + keypair = default + securitygroups = demo-server + region = eu-west-1 + placement = eu-west-1a + # we use images from `http://alestic.com/`_ + # Ubuntu 9.10 Karmic server 32-bit Europe + image = ami-a62a01d2 + ip = xxx.xxx.xxx.xxx + + +** Cluster *** + By defining a cluster in etc/aws.conf: + + [cluster:lamp-cluster] + servers = + demo-webserver + demo-dbserver + + This will be capable of starting the two servers in one command: + + $ aws cluster_start lamp-cluster + + And terminating them also in one go: + + $ aws cluster_terminate lamp-cluster + **SSH integration** diff --git a/mr/awsome/__init__.py b/mr/awsome/__init__.py index c8d431d..fca05db 100644 --- a/mr/awsome/__init__.py +++ b/mr/awsome/__init__.py @@ -254,7 +254,10 @@ def init_ssh_key(self, user=None): log.error("Can't establish ssh connection.") return if user is None: - user = 'root' + # check user at config file + user = self.config['server_user'] + if user is None: + user = 'root' host = str(instance.public_dns_name) port = 22 client = paramiko.SSHClient() @@ -288,6 +291,13 @@ def snapshot(self, devs=None): volume.create_snapshot(description=description) +class Cluster(object): + def __init__(self,ec2, sid): + self.id = sid + self.ec2 = ec2 + self.config = self.ec2.config['cluster'][sid] + + class Server(object): def __init__(self, ec2, sid): self.id = sid @@ -341,6 +351,9 @@ def __init__(self, configpath): self.all.update(self.instances) self.all.update(self.servers) + self.clusters = {} + for cid in self.config.get('cluster',{}): + self.clusters[cid] = Cluster(self,cid) class AWS(object): def __init__(self, configfile=None): @@ -423,6 +436,29 @@ def cmd_stop(self, argv, help): return log.info("Instance stopped") + def cmd_reboot(self, argv, help): + """Reboot the instance""" + parser = argparse.ArgumentParser( + prog="aws reboot", + description=help, + ) + parser.add_argument("server", nargs=1, + metavar="instance", + help="Name of the instance from the config.", + choices=list(self.ec2.instances)) + args = parser.parse_args(argv) + server = self.ec2.instances[args.server[0]] + instance = server.instance + if instance is None: + return + if instance.state != 'running': + log.info("Instance state: %s", instance.state) + log.info("Instance not terminated") + return + rc = server.conn.reboot_instances([instance.id]) + #instance._update(rc[0]) + log.info("Instance rebooting") + def cmd_terminate(self, argv, help): """Terminates the instance""" parser = argparse.ArgumentParser( @@ -462,6 +498,80 @@ def _parse_overrides(self, options): overrides[key] = value return overrides + def cmd_cluster_list (self, argv, help): + """List servers on a cluster""" + parser = argparse.ArgumentParser( + prog="aws cluster_list", + description=help, + ) + parser.add_argument("cluster", nargs=1, + metavar="cluster", + help="Name of the cluster from the config.", + choices=list(self.ec2.clusters)) + + args = parser.parse_args(argv) + # get cluster + cluster = self.ec2.clusters[args.cluster[0]] + servers = cluster.config.get('servers',[]) + log.info("---Listing servers at cluster %s.",cluster.id) + i = 1 + for s in servers: + log.info("---Cluster %s server %i: %s.---",cluster.id,i,s) + server = self.ec2.instances[s] + self._status(server) + i = i + 1 + + def cmd_cluster_start (self, argv, help): + """Starts the cluster of servers""" + parser = argparse.ArgumentParser( + prog="aws cluster_start", + description=help, + ) + parser.add_argument("cluster", nargs=1, + metavar="cluster", + help="Name of the cluster from the config.", + choices=list(self.ec2.clusters)) + + args = parser.parse_args(argv) + # get cluster + cluster = self.ec2.clusters[args.cluster[0]] + servers = cluster.config.get('servers',[]) + log.info("---Start server at cluster %s",cluster.id) + for s in servers: + log.info("--%s:%s", cluster.id, s) + server = self.ec2.instances[s] + opts = server.config.copy() + instance = server.start(opts) + self._status(server) + + def cmd_cluster_terminate (self, argv, help): + """Terminate the cluster of servers""" + parser = argparse.ArgumentParser( + prog="aws cluster_terminate", + description=help, + ) + parser.add_argument("cluster", nargs=1, + metavar="cluster", + help="Name of the cluster from the config.", + choices=list(self.ec2.clusters)) + + args = parser.parse_args(argv) + cluster = self.ec2.clusters[args.cluster[0]] + servers = cluster.config.get('servers',[]) + log.info("---Terminating servers at cluster %s.",cluster.id) + for s in servers: + server = self.ec2.instances[s] + instance = server.instance + if instance is None: + continue + if instance.state != 'running': + log.info("Instance state: %s", instance.state) + log.info("Instance not terminated") + continue + rc = server.conn.terminate_instances([instance.id]) + instance._update(rc[0]) + log.info("Instance terminated") + def cmd_start(self, argv, help): """Starts the instance""" parser = argparse.ArgumentParser( @@ -538,6 +648,23 @@ def cmd_do(self, argv, help): old_sys_argv = sys.argv old_cwd = os.getcwd() + # check server if active..else return . + tmpserver = None + try: + tmpserver = self.ec2.instances[argv[0]] + except KeyError,e: + log.error("Server not found %s", argv[0]) + parser.parse_args([argv[0]]) + return + + if tmpserver is not None: + instance = tmpserver.instance + if instance is None: + return + if instance.state != 'running': + log.info("Instance state: %s", instance.state) + return + import fabric_integration # this needs to be done before any other fabric module import fabric_integration.patch() @@ -546,11 +673,14 @@ def cmd_do(self, argv, help): import fabric.main hoststr = None + try: fabric_integration.ec2 = self.ec2 fabric_integration.log = log hoststr = argv[0] server = self.ec2.all[hoststr] + + # prepare the connection fabric.state.env.reject_unknown_hosts = True fabric.state.env.disable_known_hosts = True @@ -620,14 +750,51 @@ def cmd_ssh(self, argv, help): if sid_index is None: parser.print_help() return - server = self.ec2.all[argv[sid_index]] - try: - user, host, port, client, known_hosts = server.init_ssh_key() + + hoststr = argv[sid_index] + server = None + try: + server = self.ec2.all[argv[sid_index]] + except KeyError,e: + parser.parse_args([hoststr]) + return + + ## end check running server + tmpserver = None + tmpserver = self.ec2.instances[argv[0]] + + if tmpserver is not None: + instance = tmpserver.instance + if instance is None: + return + if instance.state != 'running': + log.info("Instance state: %s", instance.state) + return + ## end check running server + + try: + if server is not None: + ## end check running server + tmpserver = None + tmpserver = self.ec2.instances[argv[0]] + + if tmpserver is not None: + instance = tmpserver.instance + if instance is None: + return + if instance.state != 'running': + log.info("Instance state: %s", instance.state) + return + ## end check running server + user, host, port, client, known_hosts = server.init_ssh_key() + else: + return except paramiko.SSHException, e: - log.error("Couldn't validate fingerprint for ssh connection.") - log.error(e) - log.error("Is the server finished starting up?") - return + log.error("Couldn't validate fingerprint for ssh connection.") + log.error(e) + log.error("Is the server finished starting up?") + return + client.close() argv[sid_index:sid_index+1] = ['-o', 'UserKnownHostsFile=%s' % known_hosts, '-l', user, @@ -638,7 +805,7 @@ def cmd_ssh(self, argv, help): def cmd_snapshot(self, argv, help): """Creates a snapshot of the volumes specified in the configuration""" parser = argparse.ArgumentParser( - prog="aws status", + prog="aws snapshot", description=help, ) parser.add_argument("server", nargs=1, @@ -690,4 +857,4 @@ def aws_ssh(configpath=None): argv = sys.argv[:] argv.insert(1, "ssh") aws = AWS(configfile=configpath) - return aws(argv) \ No newline at end of file + return aws(argv) diff --git a/mr/awsome/config.py b/mr/awsome/config.py index 05bef03..abba079 100644 --- a/mr/awsome/config.py +++ b/mr/awsome/config.py @@ -10,12 +10,16 @@ def massage_instance_fabfile(self, value): def massage_instance_startup_script(self, value): result = dict() - if value.startswith('gzip:'): - value = value[5:] - result['gzip'] = True - if not os.path.isabs(value): - value = os.path.join(self.path, value) - result['path'] = value + ## value is already a dict, from macro + if isinstance(value,dict) is True: + return value + else: + if value.startswith('gzip:'): + value = value[5:] + result['gzip'] = True + if not os.path.isabs(value): + value = os.path.join(self.path, value) + result['path'] = value return result def massage_instance_securitygroups(self, value): @@ -33,6 +37,15 @@ def massage_instance_volumes(self, value): volumes.append((volume[0], volume[1])) return tuple(volumes) + def massage_cluster_servers(self, value): + servers = [] + for line in value.split('\n'): + server = line.split() + if not len(server): + continue + servers.append((server[0])) + return tuple(servers) + def massage_securitygroup_connections(self, value): connections = [] for line in value.split('\n'): @@ -56,6 +69,7 @@ def _expand(self, sectiongroupname, sectionname, section, seen): raise ValueError("Circular macro expansion.") macrogroupname = sectiongroupname macroname = section['<'] + seen.add((sectiongroupname, sectionname)) if ':' in macroname: macrogroupname, macroname = macroname.split(':')