Feature/separate package deploy

fabfile.py

@@ -29,20 +29,28 @@
 LOG_DIR = os.path.join(HOME, "logs/")
 BACKUP_DIR = os.path.join(HOME, "backups/")
 PROJECT_ROOT = os.path.join(HOME, "project/")
-PROJECT_WEB_ROOT = os.path.join(PROJECT_ROOT, "pypln/web/")
-REPO_URL = "https://github.com/NAMD/pypln.git"
+PYPLN_BACKEND_ROOT = os.path.join(PROJECT_ROOT, "backend")
+PYPLN_WEB_ROOT = os.path.join(PROJECT_ROOT, "web/")
+DJANGO_PROJECT_ROOT = os.path.join(PYPLN_WEB_ROOT, "pypln/web/")
+BACKEND_REPO_URL = "https://github.com/NAMD/pypln.backend.git"
+WEB_REPO_URL = "https://github.com/NAMD/pypln.web.git"
 ACTIVATE_SCRIPT = os.path.join(PROJECT_ROOT, "bin/activate")
 
 def _reload_supervisord():
     # XXX: Why does supervisor's init script exit with 1 on "restart"?
     sudo("service supervisor stop")
     sudo("service supervisor start")
 
+def _update_repository(rev):
+    run("git remote update")
+    run("git checkout {}".format(rev))
+    run("git reset --hard {}".format(rev))
+
 def _update_code(rev="master"):
-    with cd(PROJECT_ROOT):
-        run("git remote update")
-        run("git checkout {}".format(rev))
-        run("git reset --hard {}".format(rev))
+    with cd(PYPLN_BACKEND_ROOT):
+        _update_repository(rev)
+    with cd(PYPLN_WEB_ROOT):
+        _update_repository(rev)
 
 def create_db(db_user, db_name, db_host="localhost", db_port=5432):
     # we choose a random password with letters, numbers and some punctuation.
@@ -113,12 +121,13 @@ def initial_setup(rev="master"):
         _create_secret_key_file()
 
     with settings(warn_only=True, user=USER):
-        run("git clone {} {}".format(REPO_URL, PROJECT_ROOT))
+        run("git clone {} {}".format(WEB_REPO_URL, PYPLN_WEB_ROOT))
+        run("git clone {} {}".format(BACKEND_REPO_URL, PYPLN_BACKEND_ROOT))
         _update_code(rev)
         run("virtualenv --system-site-packages {}".format(PROJECT_ROOT))
 
     for daemon in ["router", "pipeliner", "broker", "web"]:
-        config_file_path = os.path.join(PROJECT_ROOT,
+        config_file_path = os.path.join(PYPLN_BACKEND_ROOT,
                 "server_config/pypln-{}.conf".format(daemon))
         sudo("ln -sf {} /etc/supervisor/conf.d/".format(config_file_path))
 
@@ -134,7 +143,7 @@ def initial_setup(rev="master"):
 
     _reload_supervisord()
 
-    nginx_vhost_path = os.path.join(PROJECT_ROOT, "server_config/nginx.conf")
+    nginx_vhost_path = os.path.join(PYPLN_BACKEND_ROOT, "server_config/nginx.conf")
     sudo("ln -sf {} /etc/nginx/sites-enabled/pypln".format(nginx_vhost_path))
     sudo("service nginx restart")
 
@@ -143,12 +152,18 @@ def initial_setup(rev="master"):
 def deploy(rev="master"):
     with prefix("source {}".format(ACTIVATE_SCRIPT)), settings(user=USER), cd(PROJECT_ROOT):
         _update_code(rev)
-        run("python setup.py install")
-        run("python -m nltk.downloader all")
+        with cd(PYPLN_BACKEND_ROOT):
+            run("python setup.py install")
+
+        with cd(PYPLN_WEB_ROOT):
+            run("python setup.py install")
 
-        with cd(PROJECT_WEB_ROOT):
+        #TODO: We need to put all pypln.web requirements in one place.
+        with cd(DJANGO_PROJECT_ROOT):
             run("pip install -r requirements/project.txt")
 
+        run("python -m nltk.downloader all")
+
         manage("syncdb --noinput")
         manage("collectstatic --noinput")
 
@@ -157,7 +172,7 @@ def deploy(rev="master"):
 def manage(command, environment="production"):
     # FIXME: we need to be in the web root because of path issues that should
     # be fixed
-    with prefix("source {}".format(ACTIVATE_SCRIPT)), cd(PROJECT_WEB_ROOT), settings(user=USER):
-        manage_script = os.path.join(PROJECT_WEB_ROOT, "manage.py")
+    with prefix("source {}".format(ACTIVATE_SCRIPT)), cd(DJANGO_PROJECT_ROOT), settings(user=USER):
+        manage_script = os.path.join(DJANGO_PROJECT_ROOT, "manage.py")
         run("python {} {} --settings=settings.{}".format(manage_script,
             command, environment))

pypln/__init__.py

@@ -0,0 +1,2 @@
+import pkg_resources
+pkg_resources.declare_namespace(__name__)

server_config/nginx.conf

@@ -6,7 +6,7 @@ server {
     client_max_body_size 50M;
 
     location /site_media/static/ {
-        alias /srv/pypln/project/pypln/web/site_media/static_files/;
+        alias /srv/pypln/project/web/pypln/web/site_media/static_files/;
     }
 
     # http://docs.gunicorn.org/en/latest/deploy.html has more information about

server_config/pypln-broker.conf

@@ -1,5 +1,5 @@
 [program:pypln-broker]
-command=/srv/pypln/project/exec_in_virtualenv.sh pypln-broker
+command=/srv/pypln/project/backend/exec_in_virtualenv.sh pypln-broker
 directory=/srv/pypln/project/
 user=pypln
 stdout_logfile=/srv/pypln/logs/pypln-broker.out

server_config/pypln-pipeliner.conf

@@ -1,5 +1,5 @@
 [program:pypln-pipeliner]
-command=/srv/pypln/project/exec_in_virtualenv.sh pypln-pipeliner
+command=/srv/pypln/project/backend/exec_in_virtualenv.sh pypln-pipeliner
 directory=/srv/pypln/project/
 user=pypln
 stdout_logfile=/srv/pypln/logs/pypln-pipeliner.out

server_config/pypln-router.conf

@@ -1,5 +1,5 @@
 [program:pypln-router]
-command=/srv/pypln/project/exec_in_virtualenv.sh pypln-router
+command=/srv/pypln/project/backend/exec_in_virtualenv.sh pypln-router
 directory=/srv/pypln/project/
 user=pypln
 stdout_logfile=/srv/pypln/logs/pypln-router.out

server_config/pypln-web.conf

@@ -1,6 +1,6 @@
 [program:pypln-web]
-command=/srv/pypln/project/pypln/web/runwsgi.sh
-directory=/srv/pypln/project/pypln/web/
+command=/srv/pypln/project/web/pypln/web/runwsgi.sh
+directory=/srv/pypln/project/web/pypln/web/
 user=pypln
 stdout_logfile=/srv/pypln/logs/pypln-web.out
 stderr_logfile=/srv/pypln/logs/pypln-web.err

setup.py

@@ -46,6 +46,7 @@ def get_requirements():
                                         'pypln-pipeliner = pypln.backend.pipeliner:main',],
       },
       packages=find_packages(),
+      namespace_packages=["pypln"],
       install_requires=get_requirements(),
       test_suite='nose.collector',
       license='GPL3',