Adds a basic Vagrant image for Zamboni (bug 689755)

Author: kumar303

.gitignore

@@ -1,4 +1,4 @@
-settings_local.py
+settings_local.py*
 *.py[co]
 *.sw[po]
 .coverage
@@ -22,3 +22,6 @@ vendor
 .nose*
 tmp/*
 tags
+vagrantconfig_local.yaml
+vagrant/manifests/classes/custom.pp
+vagrant/manifests/build

Vagrantfile

@@ -0,0 +1,45 @@
+require "yaml"
+
+# Load up our vagrant config files -- vagrantconfig.yaml first
+_config = YAML.load(File.open(File.join(File.dirname(__FILE__),
+                    "vagrantconfig.yaml"), File::RDONLY).read)
+
+# Local-specific/not-git-managed config -- vagrantconfig_local.yaml
+begin
+    _config.merge!(YAML.load(File.open(File.join(File.dirname(__FILE__),
+                   "vagrantconfig_local.yaml"), File::RDONLY).read))
+rescue Errno::ENOENT # No vagrantconfig_local.yaml found -- that's OK; just
+                     # use the defaults.
+end
+
+CONF = _config
+MOUNT_POINT = '/home/vagrant/project'
+
+Vagrant::Config.run do |config|
+    config.vm.box = "zamboni"
+    config.vm.box_url = "http://files.vagrantup.com/lucid32.box"
+
+    # TODO(Kumar) figure out a way to forward port 80, as requested from the
+    # guest, to 8000 on the host. This apparently doesn't do that :/
+    config.vm.forward_port("web", 80, 8000)
+
+    # Increase vagrant's patience during hang-y CentOS bootup
+    # see: https://github.com/jedi4ever/veewee/issues/14
+    config.ssh.max_tries = 50
+    config.ssh.timeout   = 300
+
+    # nfs needs to be explicitly enabled to run.
+    if CONF['nfs'] == false or RUBY_PLATFORM =~ /mswin(32|64)/
+        config.vm.share_folder("v-root", MOUNT_POINT, ".")
+    else
+        config.vm.share_folder("v-root", MOUNT_POINT, ".", :nfs => true)
+    end
+
+    # For convenience add something like this to /etc/hosts: 33.33.33.24 z.local
+    config.vm.network "33.33.33.24"
+
+    config.vm.provision :puppet do |puppet|
+        puppet.manifests_path = "vagrant/manifests"
+        puppet.manifest_file  = "vagrant.pp"
+    end
+end

vagrant/files/etc/httpd/conf.d/zamboni.conf

@@ -0,0 +1,21 @@
+# HACK: Make the server reload after every hit to refresh django code
+# MaxRequestsPerChild 1
+
+WSGISocketPrefix /var/run/wsgi
+
+<VirtualHost 33.33.33.24:443 33.33.33.24:80>
+    ServerName 33.33.33.24
+
+    DirectoryIndex index.php index.html
+    Options -Indexes
+
+    DocumentRoot "/home/vagrant/project"
+
+    Alias /media/ "/home/vagrant/project/media/"
+    Alias /admin-media/ "/home/vagrant/project/vendor/src/django/django/contrib/admin/media/"
+
+    WSGIDaemonProcess playdoh processes=2 threads=2 maximum-requests=4
+    WSGIProcessGroup playdoh
+
+    WSGIScriptAlias / "/home/vagrant/project/wsgi/zamboni.wsgi"
+</VirtualHost>

vagrant/manifests/classes/apache.pp

@@ -0,0 +1,54 @@
+# Red Hat, CentOS, and Fedora think Apache is the only web server
+# ever, so we have to use a different package on CentOS than Ubuntu.
+class apache {
+    case $operatingsystem {
+        centos: {
+            package { "httpd-devel":
+                ensure => present,
+                before => File['/etc/httpd/conf.d/zamboni.conf'];
+            }
+
+            file { "/etc/httpd/conf.d/zamboni.conf":
+                source => "$PROJ_DIR/vagrant/files/etc/httpd/conf.d/zamboni.conf",
+                owner => "root", group => "root", mode => 0644,
+                require => [
+                    Package['httpd-devel']
+                ];
+            }
+
+            service { "httpd":
+                ensure => running,
+                enable => true,
+                require => [
+                    Package['httpd-devel'],
+                    File['/etc/httpd/conf.d/zamboni.conf']
+                ];
+            }
+
+        }
+        ubuntu: {
+            package { "apache2-threaded-dev":
+                ensure => present,
+                before => File['/etc/apache2/sites-enabled/zamboni.conf']; 
+            }
+
+            file { "/etc/apache2/sites-enabled/zamboni.conf":
+                source => "$PROJ_DIR/vagrant/files/etc/httpd/conf.d/zamboni.conf",
+                owner => "root", group => "root", mode => 0644,
+                require => [
+                    Package['apache2-threaded-dev']
+                ];
+            }
+
+            # DISABLED, using dev server instead.
+            service { "apache2":
+                ensure => stopped,
+                enable => false,
+                require => [
+                    Package['apache2-threaded-dev'],
+                    File['/etc/apache2/sites-enabled/zamboni.conf']
+                ];
+            }
+        }
+    }
+}

vagrant/manifests/classes/custom-dist.pp

@@ -0,0 +1,9 @@
+# Copy this file to custom.pp and declare any commands you want to run on your
+# local machine only. Anything you add to custom.pp will *not* be committed to
+# version control.
+class custom {
+    # exec { "install-nosenicedots":
+    #     command => "sudo pip install nosenicedots",
+    #     require => Package['python-pip']
+    # }
+}

vagrant/manifests/classes/elasticsearch.pp

@@ -0,0 +1,96 @@
+
+# Based loosely on https://gist.github.com/1190526
+# This can probably be simplified when there is a deb package for ES.
+
+class elasticsearch {
+
+    ## NOTE: moved to init.pp so that we don't get a huge pile of updates to
+    ## previously installed packages
+
+    # exec { "add_java_repo":
+    #     # For sun java:
+    #     command => 'sudo add-apt-repository "deb http://archive.canonical.com/ lucid partner"',
+    #     require => Package["python-software-properties"]
+    # }
+    # 
+    # exec { "init_java_repo":
+    #     command => "sudo apt-get update",
+    #     require => Exec['add_java_repo']
+    # }
+
+    exec { "accept_jre_license":
+        command => "echo 'sun-java6-jre shared/accepted-sun-dlj-v1-1 boolean true' | sudo debconf-set-selections",
+        require => Exec["init_java_repo"]
+    }
+
+    exec { "accept_java_license":
+        command => "echo 'sun-java6-bin shared/accepted-sun-dlj-v1-1 boolean true' | sudo debconf-set-selections",
+        require => Exec["init_java_repo"]
+    }
+
+    exec { "accept_java_plugin_license":
+        command => "echo 'sun-java6-plugin shared/accepted-sun-dlj-v1-1 boolean true' | sudo debconf-set-selections",
+        require => Exec["init_java_repo"]
+    }
+
+    package {
+        ["sun-java6-jre", "sun-java6-bin", "sun-java6-plugin", "curl"]:
+            ensure => installed,
+            require => [
+                Exec["accept_jre_license"],
+                Exec["accept_java_plugin_license"],
+                Exec["accept_java_license"]
+            ];
+    }
+
+    exec { "get_source":
+        cwd => "/tmp",
+        command => 'wget https://github.com/downloads/elasticsearch/elasticsearch/elasticsearch-0.17.6.tar.gz -O elasticsearch.tar.gz',
+        unless => 'test -d /usr/local/share/elasticsearch'
+    }
+
+    exec { "untar":
+        cwd => "/tmp",
+        command => 'tar -xzf elasticsearch.tar.gz',
+        require => Exec["get_source"],
+        unless => 'test -d /usr/local/share/elasticsearch'
+    }
+
+    exec { "move_source":
+        cwd => "/tmp",
+        command => 'sudo mv elasticsearch-* /usr/local/share/elasticsearch',
+        require => Exec["untar"],
+        unless => 'test -d /usr/local/share/elasticsearch'
+    }
+
+    exec { "get_service":
+        cwd => "/tmp",
+        command => 'curl -L http://github.com/elasticsearch/elasticsearch-servicewrapper/tarball/master | tar -xz',
+        require => [Package["curl"], Exec["move_source"]],
+        unless => 'test -d /usr/local/share/elasticsearch/bin/service'
+    }
+
+    exec { "move_service":
+        cwd => "/tmp",
+        command => 'sudo mv *servicewrapper*/service /usr/local/share/elasticsearch/bin/',
+        require => [Exec["get_service"], Exec["move_source"]],
+        unless => 'test -d /usr/local/share/elasticsearch/bin/service'
+    }
+
+    exec { "install_service":
+        # Makes /etc/init.d/elasticsearch
+        command => 'sudo /usr/local/share/elasticsearch/bin/service/elasticsearch install',
+        # Install if service is not already running.
+        unless => 'test `cat /usr/local/share/elasticsearch/bin/service/elasticsearch.java.status` = STARTED',
+        require => [
+            Exec["move_service"],
+            Package["sun-java6-jre", "sun-java6-bin", "sun-java6-plugin", "curl"]
+        ]
+    }
+
+    service { "elasticsearch":
+        enable => true,
+        ensure => "running",
+        require => Exec["install_service"]
+    }
+}

vagrant/manifests/classes/init.pp

@@ -0,0 +1,44 @@
+# stage {"pre": before => Stage["main"]} class {'apt': stage => 'pre'}
+
+# Commands to run before all others in puppet.
+class init {
+    group { "puppet":
+        ensure => "present",
+    }
+
+    case $operatingsystem {
+        ubuntu: {
+            exec { "update_apt":
+                command => "sudo apt-get update"
+            }
+
+            # Provides "add-apt-repository" command, useful if you need
+            # to install software from other apt repositories.
+            package { "python-software-properties":
+                ensure => present,
+                require => [
+                    Exec['update_apt'],
+                ];
+            }
+
+            exec { "add_java_repo":
+                # For sun java:
+                command => 'sudo add-apt-repository "deb http://archive.canonical.com/ lucid partner"',
+                require => Package["python-software-properties"]
+            }
+
+            exec { "init_java_repo":
+                # Bah, this is lame.
+                command => "sudo apt-get update",
+                require => Exec['add_java_repo']
+            }
+        }
+    }
+
+    # If you haven't created a custom pp file, create one from dist.
+    file { "$PROJ_DIR/vagrant/manifests/classes/custom.pp":
+        ensure => file,
+        source => "$PROJ_DIR/vagrant/manifests/classes/custom-dist.pp",
+        replace => false;
+    }
+}

vagrant/manifests/classes/mysql.pp

@@ -0,0 +1,26 @@
+# Get mysql up and running
+class mysql {
+    package { "mysql-server":
+        ensure => installed;
+    }
+
+    case $operatingsystem {
+        centos: {
+            package { "mysql-devel":
+                ensure => installed;
+            }
+        }
+
+        ubuntu: {
+            package { "libmysqld-dev":
+                ensure => installed;
+            }
+        }
+    }
+
+    service { "mysql":
+        ensure => running,
+        enable => true,
+        require => Package['mysql-server'];
+    }
+}

vagrant/manifests/classes/python.pp

@@ -0,0 +1,23 @@
+# Install python and compiled modules for project
+class python {
+    package {
+        ["python2.6-dev", "python2.6", "libapache2-mod-wsgi", "python-pip",
+         "libxml2-dev", "libxslt1-dev", "libssl-dev", "swig", "git-core"]:
+            ensure => installed;
+    }
+
+    # Bah. Ubuntu moves at the speed of molasses.
+    # Need the fix for exit statuses: https://github.com/pypa/pip/issues/106
+    exec { "upgrade_pip":
+        command => "sudo easy_install -U pip",
+        require => Package['python-pip']
+    }
+
+    exec { "pip-install-compiled":
+        command => "sudo pip install -r $PROJ_DIR/requirements/compiled.txt",
+        require => [
+            Exec["upgrade_pip"],
+            Package["python2.6", "libxml2-dev", "libxslt1-dev", "libssl-dev", "swig"]
+        ]
+    }
+}

vagrant/manifests/classes/redis.pp

@@ -0,0 +1,14 @@
+class redis {
+    package { "redis-server":
+        ensure => installed;
+    }
+
+    service { "redis-server":
+        ensure => running,
+        enable => true,
+        hasrestart => true,
+        require => [
+            Package['redis-server']
+        ]
+    }
+}

vagrant/manifests/classes/zamboni.pp

@@ -0,0 +1,76 @@
+# playdoh-specific commands that get zamboni all going so you don't
+# have to.
+
+# TODO: Make this rely on things that are not straight-up exec.
+class zamboni {
+    package { "wget":
+        ensure => installed;
+    }
+
+    file { "$PROJ_DIR/settings_local.py":
+        ensure => file,
+        source => "$PROJ_DIR/settings_local.py",
+        replace => false;
+    }
+
+    exec { "create_mysql_database":
+        command => "mysqladmin -uroot create $DB_NAME",
+        unless  => "mysql -uroot -B --skip-column-names -e 'show databases' | /bin/grep '$DB_NAME'",
+        require => File["$PROJ_DIR/settings_local.py"]
+    }
+
+    exec { "grant_mysql_database":
+        command => "mysql -uroot -B -e'GRANT ALL PRIVILEGES ON $DB_NAME.* TO $DB_USER@localhost # IDENTIFIED BY \"$DB_PASS\"'",
+        unless  => "mysql -uroot -B --skip-column-names mysql -e 'select user from user' | grep '$DB_USER'",
+        require => Exec["create_mysql_database"];
+    }
+
+    exec { "fetch_landfill_sql":
+        cwd => "$PROJ_DIR",
+        command => "wget --no-check-certificate -P /tmp https://landfill.addons.allizom.org/db/landfill-`date +%Y-%m-%d`.sql.gz",
+        require => [
+            Package["wget"],
+            Exec["grant_mysql_database"]
+        ];
+    }
+
+    exec { "load_data":
+        cwd => "$PROJ_DIR",
+        command => "zcat /tmp/landfill-`date +%Y-%m-%d`.sql.gz | mysql -u$DB_USER $DB_NAME",
+        require => [
+            Exec["fetch_landfill_sql"]
+        ];
+    }
+
+    # TODO(Kumar) add landfile files as well.
+
+    # Skip this migration because it won't succeed without indexes but you
+    # can't build indexes without running migrations :(
+    file { "$PROJ_DIR/migrations/264-locale-indexes.py":
+        content => "#",
+        replace => true
+    }
+
+    exec { "sql_migrate":
+        cwd => "$PROJ_DIR",
+        command => "python ./vendor/src/schematic/schematic migrations/",
+        require => [
+            Service["mysql"],
+            Package["python2.6"],
+            File["$PROJ_DIR/migrations/264-locale-indexes.py"],
+            Exec["fetch_landfill_sql"],
+            Exec["load_data"]
+        ];
+    }
+
+    exec { "restore_migration_264":
+        cwd => "$PROJ_DIR",
+        command => "git checkout migrations/264-locale-indexes.py",
+        require => [Exec["sql_migrate"], Package["git-core"]]
+    }
+
+    exec { "remove_site_notice":
+        command => "mysql -uroot -e\"delete from config where \\`key\\`='site_notice'\" $DB_NAME",
+        require => Exec["sql_migrate"]
+    }
+}

vagrant/manifests/vagrant.pp

@@ -0,0 +1,31 @@
+#
+# Playdoh puppet magic for dev boxes
+#
+import "classes/*.pp"
+
+$PROJ_DIR = "/home/vagrant/project"
+
+# You can make these less generic if you like, but these are box-specific
+# so it's not required.
+# The db user does not require a password.
+$DB_NAME = "zamboni"
+$DB_USER = "root"
+
+Exec {
+    path => "/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin",
+}
+
+class dev {
+    class {
+        init: before => Class[mysql];
+        mysql: before  => Class[python];
+        python: before => Class[redis];
+        # apache: before => Class[zamboni];
+        redis: before => Class[zamboni];
+        elasticsearch: ;
+        zamboni: ;
+        custom: ;
+    }
+}
+
+include dev

vagrantconfig.yaml

@@ -0,0 +1,5 @@
+# Default config for Vagrant
+
+# Don't change this; use vagrantconfig_local.yaml to override these
+# settings instead.
+nfs: false

vagrantconfig_local.yaml-dist

@@ -0,0 +1,5 @@
+# Configuration for Vagrant
+
+# Change to true if you can use nfs; using nfs significantly
+# improves performance.
+nfs: false