tools added

tools/_venv_common.sh

@@ -0,0 +1,26 @@
+#!/bin/sh -xe
+
+VENV_NAME=${VENV_NAME:-venv}
+
+# .egg-info directories tend to cause bizarre problems (e.g. `pip -e
+# .` might unexpectedly install letshelp-certbot only, in case
+# `python letshelp-certbot/setup.py build` has been called
+# earlier)
+rm -rf *.egg-info
+
+# virtualenv setup is NOT idempotent: shutil.Error:
+# `/home/jakub/dev/letsencrypt/letsencrypt/venv/bin/python2` and
+# `venv/bin/python2` are the same file
+mv $VENV_NAME "$VENV_NAME.$(date +%s).bak" || true
+virtualenv --no-site-packages --setuptools $VENV_NAME $VENV_ARGS
+. ./$VENV_NAME/bin/activate
+
+# Use pipstrap to update Python packaging tools to only update to a well tested
+# version and to work around https://github.com/pypa/pip/issues/4817 on older
+# systems.
+python letsencrypt-auto-source/pieces/pipstrap.py
+./tools/pip_install.sh "$@"
+
+set +x
+echo "Please run the following command to activate developer environment:"
+echo "source $VENV_NAME/bin/activate"

tools/deps.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+#
+# Find all Python imports.
+#
+#  ./tools/deps.sh certbot
+#  ./tools/deps.sh acme
+#  ./tools/deps.sh certbot-apache
+#  ...
+#
+# Manually compare the output with deps in setup.py.
+
+git grep -h -E '^(import|from.*import)' $1/ | \
+    awk '{print $2}' | \
+    grep -vE "^$1" | \
+    sort -u

tools/eff-pubkey.pem

@@ -0,0 +1,9 @@
+-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq
+OzQb2eyW15YFjDDEMI0ZOzt8f504obNs920lDnpPD2/KqgsfjOgw2K7xWDJIj/18
+xUvWPk3LDkrnokNiRkA3KOx3W6fHycKL+zID7zy+xZYBuh2fLyQtWV1VGQ45iNRp
+9+Zo7rH86cdfgkdnWTlNSHyTLW9NbXvyv/E12bppPcEvgCTAQXgnDVJ0/sqmeiij
+n9tTFh03aM+R2V/21h8aTraAS24qiPCz6gkmYGC8yr6mglcnNoYbsLNYZ69zF1XH
+cXPduCPdPdfLlzVlKK1/U7hkA28eG3BIAMh6uJYBRJTpiGgaGdPd7YekUB8S6cy+
+CQIDAQAB
+-----END PUBLIC KEY-----

tools/install_and_test.sh

@@ -0,0 +1,27 @@
+#!/bin/sh -e
+# pip installs the requested packages in editable mode and runs unit tests on
+# them. Each package is installed and tested in the order they are provided
+# before the script moves on to the next package. If CERTBOT_NO_PIN is set not
+# set to 1, packages are installed using pinned versions of all of our
+# dependencies. See pip_install.sh for more information on the versions pinned
+# to.
+
+if [ "$CERTBOT_NO_PIN" = 1 ]; then
+  pip_install="pip install -q -e"
+else
+  pip_install="$(dirname $0)/pip_install_editable.sh"
+fi
+
+set -x
+for requirement in "$@" ; do
+  $pip_install $requirement
+  pkg=$(echo $requirement | cut -f1 -d\[)  # remove any extras such as [dev]
+  if [ $pkg = "." ]; then
+    pkg="certbot"
+  else
+    # Work around a bug in pytest/importlib for the deprecated Python 3.3.
+    # See https://travis-ci.org/certbot/certbot/jobs/308774157#L1333.
+    pkg=$(echo "$pkg" | tr - _)
+  fi
+  pytest --numprocesses auto --quiet --pyargs $pkg
+done

tools/merge_requirements.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+"""Merges multiple Python requirements files into one file.
+
+Requirements files specified later take precedence over earlier ones. Only
+simple SomeProject==1.2.3 format is currently supported.
+
+"""
+
+from __future__ import print_function
+
+import sys
+
+
+def read_file(file_path):
+    """Reads in a Python requirements file.
+
+    :param str file_path: path to requirements file
+
+    :returns: mapping from a project to its pinned version
+    :rtype: dict
+
+    """
+    d = {}
+    with open(file_path) as f:
+        for line in f:
+            line = line.strip()
+            if line and not line.startswith('#'):
+                project, version = line.split('==')
+                if not version:
+                    raise ValueError("Unexpected syntax '{0}'".format(line))
+                d[project] = version
+    return d
+
+
+def print_requirements(requirements):
+    """Prints requirements to stdout.
+
+    :param dict requirements: mapping from a project to its pinned version
+
+    """
+    print('\n'.join('{0}=={1}'.format(k, v)
+          for k, v in sorted(requirements.items())))
+
+
+def merge_requirements_files(*files):
+    """Merges multiple requirements files together and prints the result.
+
+    Requirement files specified later in the list take precedence over earlier
+    files.
+
+    :param tuple files: paths to requirements files
+
+    """
+    d = {}
+    for f in files:
+        d.update(read_file(f))
+    print_requirements(d)
+
+
+if __name__ == '__main__':
+    merge_requirements_files(*sys.argv[1:])

tools/offline-sigrequest.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+set -o errexit
+
+if ! `which festival > /dev/null` ; then
+    echo Please install \'festival\'!
+    exit 1
+fi
+
+function sayhash { # $1 <-- HASH ; $2 <---SIGFILEBALL
+  while read -p "Press Enter to read the hash aloud or type 'done':  " INP && [ "$INP" = "" ] ; do
+    cat $1 | (echo "(Parameter.set 'Duration_Stretch 1.8)"; \
+                echo -n '(SayText "'; \
+                sha256sum | cut -c1-64 | fold -1 | sed 's/^a$/alpha/; s/^b$/bravo/; s/^c$/charlie/; s/^d$/delta/; s/^e$/echo/; s/^f$/foxtrot/'; \
+                echo '")' ) | festival
+  done
+
+  echo 'Paste in the data from the QR code, then type Ctrl-D:'
+  cat > $2
+}
+
+function offlinesign {  # $1 <-- INPFILE ; $2 <---SIGFILE
+  echo HASH FOR SIGNING:
+  SIGFILEBALL="$2.lzma.base64"
+  #echo "(place the resulting raw binary signature in $SIGFILEBALL)"
+  sha256sum $1
+  echo metahash for confirmation only $(sha256sum $1   |cut -d' ' -f1 | tr -d '\n' | sha256sum  | cut -c1-6) ...
+  echo
+  sayhash $1 $SIGFILEBALL
+}
+
+function oncesigned { # $1 <-- INPFILE ; $2 <--SIGFILE
+  SIGFILEBALL="$2.lzma.base64"
+  cat $SIGFILEBALL | tr -d '\r' | base64 -d | unlzma -c > $2 || exit 1
+  if ! [ -f $2 ] ; then
+    echo "Failed to find $2"'!'
+    exit 1
+  fi
+
+  if file $2 | grep -qv " data" ; then
+    echo "WARNING WARNING $2 does not look like a binary signature:"
+    echo `file $2`
+    exit 1
+  fi
+}
+
+HERE=`dirname $0`
+LEAUTO="`realpath $HERE`/../letsencrypt-auto-source/letsencrypt-auto"
+SIGFILE="$LEAUTO".sig
+offlinesign $LEAUTO $SIGFILE
+oncesigned $LEAUTO $SIGFILE

tools/oldest_constraints.txt

@@ -0,0 +1,51 @@
+# This file contains the oldest versions of our dependencies we say we require
+# in our packages or versions we need to support to maintain compatibility with
+# the versions included in the various Linux distros where we are packaged.
+
+# CentOS/RHEL 7 EPEL constraints
+cffi==1.6.0
+chardet==2.2.1
+configobj==4.7.2
+ipaddress==1.0.16
+mock==1.0.1
+ndg-httpsclient==0.3.2
+ply==3.4
+pyasn1==0.1.9
+pycparser==2.14
+pyOpenSSL==0.13.1
+pyparsing==1.5.6
+pyRFC3339==1.0
+python-augeas==0.5.0
+six==1.9.0
+# setuptools 0.9.8 is the actual version packaged, but some other dependencies
+# in this file require setuptools>=1.0 and there are no relevant changes for us
+# between these versions.
+setuptools==1.0.0
+urllib3==1.10.2
+zope.component==4.1.0
+zope.event==4.0.3
+zope.interface==4.0.5
+
+# Debian Jessie Backports constraints
+PyICU==1.8
+colorama==0.3.2
+enum34==1.0.3
+html5lib==0.999
+idna==2.0
+pbr==1.8.0
+pytz==2012rc0
+
+# Our setup.py constraints
+cloudflare==1.5.1
+cryptography==1.2.0
+google-api-python-client==1.5
+oauth2client==2.0
+parsedatetime==1.3
+pyparsing==1.5.5
+python-digitalocean==1.11
+requests[security]==2.4.1
+
+# Ubuntu Xenial constraints
+ConfigArgParse==0.10.0
+funcsigs==0.4
+zope.hookable==4.0.4

tools/pip_install.sh

@@ -0,0 +1,26 @@
+#!/bin/bash -e
+# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set
+# to 1, a combination of tools/oldest_constraints.txt and
+# tools/dev_constraints.txt is used, otherwise, a combination of certbot-auto's
+# requirements file and tools/dev_constraints.txt is used. The other file
+# always takes precedence over tools/dev_constraints.txt.
+
+# get the root of the Certbot repo
+tools_dir=$(dirname $("$(dirname $0)/readlink.py" $0))
+dev_constraints="$tools_dir/dev_constraints.txt"
+merge_reqs="$tools_dir/merge_requirements.py"
+test_constraints=$(mktemp)
+trap "rm -f $test_constraints" EXIT
+
+if [ "$CERTBOT_OLDEST" = 1 ]; then
+    cp "$tools_dir/oldest_constraints.txt" "$test_constraints"
+else
+    repo_root=$(dirname "$tools_dir")
+    certbot_requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt"
+    sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' "$certbot_requirements" > "$test_constraints"
+fi
+
+set -x
+
+# install the requested packages using the pinned requirements as constraints
+pip install -q --constraint <("$merge_reqs" "$dev_constraints" "$test_constraints") "$@"

tools/pip_install_editable.sh

@@ -0,0 +1,10 @@
+#!/bin/sh -e
+# pip installs packages in editable mode using certbot-auto's requirements file
+# as constraints
+
+args=""
+for requirement in "$@" ; do
+  args="$args -e $requirement"
+done
+
+"$(dirname $0)/pip_install.sh" $args

tools/readlink.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+"""Canonicalizes a path and follows any symlinks.
+
+This is the equivalent of `readlink -f` on many Linux systems. This is
+useful as there are often differences in readlink on different
+platforms.
+
+"""
+from __future__ import print_function
+import os
+import sys
+
+print(os.path.realpath(sys.argv[1]))