Implemented new project structure and release process

.decrypt-keys.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+openssl aes-256-cbc -K $encrypted_9df70f2e42de_key -iv $encrypted_9df70f2e42de_iv -in travis-deploy-key.enc -out travis-deploy-key -d
+chmod 600 travis-deploy-key;
+cp travis-deploy-key ~/.ssh/id_rsa;

.gitignore

@@ -1,2 +1,23 @@
 *.class
 *.log
+
+# sbt specific
+.cache
+.history
+.lib/
+dist/*
+target/
+lib_managed/
+src_managed/
+project/boot/
+project/plugins/project/
+
+# Scala-IDE specific
+.scala_dependencies
+.worksheet
+
+#Markdown editing
+.Ulysses-favorites.plist
+
+metastore_db/
+tmp/

.travis.yml

@@ -0,0 +1,62 @@
+language: scala
+
+sudo: false
+
+scala: 2.11.8
+
+addons:
+  apt:
+    packages:
+    - curl
+
+branches:
+  except:
+    - gh-pages
+    - /^v[0-9]/
+
+cache:
+  directories:
+    - $HOME/.ivy2/cache
+    - $HOME/.sbt
+
+env:
+  global:
+    - secure: "dhDQWvkfLqVrcAOH1u8PRxTLM0B+DteVDe1X0zfMWE3Rfmoa7ZNsegvPDePWTyAEEnCDK+27buOSurWxt1wcOtu+7zwnPB5ZB5JPIIJItAtlipWY0o+tiZ046cysn9p9BY3eK5Up9LqaXkPCnvjF19tbHKVKTmTSrd/NGwin/FCdXoLebnG8YU5/+B6hAal92I5gVtKGsbvo6/q4V/8pOkZKjtIUPQg9wHANbefu5eR83CKINkOShUILtSKqzbF79LGttLN4bOCUr+/P54xpJ4GCfnGc6UEBZcQ3HKYjk8KLDnMr0r3oy8yr/RdJbq9MttCxJdbHjlu1Jak068DzUBZIH5QMTLJZdx61aGTmEuNVvItVXFVWOjIUz6vVgLAaKgA5BWe8fIMl5DSbCCs2rUAiaGwq8zuhzDsOeu1wrCKOqUxYrzzq8WFLiGDtD9RmomdXWqVqbw39/Jk4dRPSRJrtsIGGGxCfLV/gPbeAdNDwS+GWSMBEBQe+2f7cSpZcp7vCJ5j5bnTA/0HEqD1tMp1/C1cQ9I3X8HRwc5ivOW1CCOqJlOxBvmns4rjWBIEcRBVN9HbAR3nQQUwJ97q5T45tC9+L2MIlQjuKiYA6VIHweOnO25FMm2I+hAmWnWcHBt0s3IMx3S1yyZcMc7BJLRGsSBytjzb1gL+QhzdvtK0="
+    - secure: "qddU9PPHqKQbMc/+pbX8h9Sy1hR2bjyvyY2isFOsemxi3XMhuPrV15yVju+SGTZklGhPITzbF/zgvVu5hvE5zcCKge/+ODVmcB+lQQNv1/2m2poP7CvT/Qvj1v4BvN6QXVF4/ZA+y/DEoCxV4h8r0uhw8slBj7PkFvGFALPDiaVwoqyiwExPhQJZX/O7UFipdTHCYk38mak3Q1JU5Vq9swBCrTZihWb1hmkyV+Gppa02loxaS/e5JS8UKc0I+kBbTCr8k+yzE9Pi4FqTO3HFYxuhegaDtrWS9gmtBZZXPViS1RtEemtzECUW0tG4xTuPw0EqaCM1Q9Qv4oRb1mtWLqdLytN1OaTW1ZqGlHBUuLZ3HBGsBQoe2NqOt4+g8+LfuDSkrtFGo7MzyikgAE797QkJtJXUNuqcfE5R1IIPpjxHkKlA5u77y2inSPxuA3iCGskbXy/YCWB8gZriKZKPrSXhpLZl9cpH70F4QTuBXDhkqPOVMwP6B7crGhFIzz1+CSWtxoJlERxZ7E2bYmWqySzw8BwvpuWk3ysrPEAAD5/gaWwohESlFgyAdRaexEqqQQG3/JagtxBInErLqcq3PUxp6U7mV63zgPjksmqQNgLaI3wBCMxdBJhl6Ejkaa+pBv+B40BPNczUSZW0vjtO7EDq1jSAhMLZhILQptFKZvE="
+    - secure: "LrhD2u0lM+ZRJ6TvtP2BX3FlQbNf6iRGZ4uTGEGRkylIEAfk1BtRs2Kv9KhjVhlox630vzfSNZxf76yH252JMsqSQJI+3raJThF7tAfLDUK2tdIw8OdHpn940wI01ChpH+3uqKwBIYSLhyU8ryPSCHLeehQeayAvz8ClDgjMhw1OJLA62l26x9nJYEDyOetWtrlqG0N39Lg4Mb722X+jLMmYNOLZRVFb7CjLSaacQ4sRENA1PCQdA87qG/1/wnkJptF/TSXwbdfxr86qVv6n7gqJVqEDjeNABZ4GouX4c3SlJqAZqm26POBysvT7eO6okBOitsoIMM8WEO32tT8PVTRXY17NFnswCVJdarsZFCEv93f7KrRERTtCbHJRuV20LdLMCs9S7NVBnvudcYFT9tcq3dUprON4+9nMd4bGVdKZOuhwY2Xw9P3xur5//pKB9VKsa4Fx6kX7OYbA/dQcTEW2fuXUyQrVbcdfBuM/nkYeJYrya4wsrnzJzW7SqFszcohhJAjM3pklMxE8mlE+pK4N+th/0cxJoQfPyyCde9NEKKpjKZaQhbiuNNXsJCQjyuoLWDSGA/Yh/emJdYWQrjY+sLNRmCOQ4kjuNsVPnatuYUH7TQtX4V2HTmxgdApc0zsMpBGNk+inNvK6xIfTU8saiA5wwC4dZEdmnD7hZ2g="
+    - SWOOP_VERSION_FILE=$TRAVIS_BUILD_DIR/VERSION
+    - SWOOP_RELEASE_BRANCH=release
+    - SWOOP_PROJECT_VERSION="$(grep -Po '\d+\.\d+' $SWOOP_VERSION_FILE).$TRAVIS_BUILD_NUMBER"
+
+before_install:
+  - >
+    if [ ${TRAVIS_PULL_REQUEST} = 'false' -a ${TRAVIS_BRANCH} = ${SWOOP_RELEASE_BRANCH} ]; then
+      bash ./.decrypt-keys.sh &&
+      export PATH=${PATH}:./vendor/bundle &&
+      gem update --system &&
+      gem install sass &&
+      gem install jekyll
+    fi
+
+# Set the version number with the Travis build number as the patch version
+before_script:
+  - >
+    if [ ${TRAVIS_PULL_REQUEST} = 'false' -a ${TRAVIS_BRANCH} = ${SWOOP_RELEASE_BRANCH} ]
+      then echo "${SWOOP_PROJECT_VERSION}" | tee $SWOOP_VERSION_FILE
+      else echo "${SWOOP_PROJECT_VERSION}-SNAPSHOT" | tee $SWOOP_VERSION_FILE
+    fi
+
+after_success:
+  - echo "VERSION NUMBER = $(cat $SWOOP_VERSION_FILE)"
+  - >
+    [ ${TRAVIS_PULL_REQUEST} = 'false' -a ${TRAVIS_BRANCH} = ${SWOOP_RELEASE_BRANCH} ] &&
+    echo "RELEASING: $(cat $SWOOP_VERSION_FILE)" &&
+    sbt publish &&
+    sbt publishMicrosite &&
+    curl -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${GITHUB_TOKEN}" -X POST "https://api.github.com/repos/${TRAVIS_REPO_SLUG}/releases" -d '{
+      "tag_name": "v'"${SWOOP_PROJECT_VERSION}"'",
+      "target_commitish": "'"${TRAVIS_COMMIT}"'",
+      "name": "v'"${SWOOP_PROJECT_VERSION}"'",
+      "body": "https://travis-ci.org/'"${TRAVIS_REPO_SLUG}"'/builds/'"${TRAVIS_BUILD_ID}"'",
+      "prerelease": false
+    }'

README.md

@@ -1,17 +1,19 @@
 # spark-alchemy
 
-Spark Alchemy is a collection of open-source Spark tools & frameworks that have made the data engineering and 
+[![Download](https://api.bintray.com/packages/swoop-inc/maven/spark-alchemy/images/download.svg)](https://bintray.com/swoop-inc/maven/spark-alchemy/_latestVersion)
+
+Spark Alchemy is a collection of open-source Spark tools & frameworks that have made the data engineering and
 data science teams at [Swoop](https://www.swoop.com) highly productive in our demanding petabyte-scale environment
 with rich data (thousands of columns).
 
-We are preparing to release `spark-alchemy`. Click Watch above to be notified when we do. 
+We are preparing to release `spark-alchemy`. Click Watch above to be notified when we do.
 
 Here is a preview of what we'd like to include here:
 
-- Configuration Addressable Production (CAP), Automatic Lifecycle Management (ALM) and Just-in-time Dependency Resolution 
+- Configuration Addressable Production (CAP), Automatic Lifecycle Management (ALM) and Just-in-time Dependency Resolution
 (JDR) as outlined in our Spark+AI Summit talk [Unafraid of Change: Optimizing ETL, ML, and AI in Fast-Paced Environments](https://databricks.com/session/unafraid-of-change-optimizing-etl-ml-ai-in-fast-paced-environments).
 
-- Our extensive set of [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog) (HLL) functions that allow the saving of HLL sketches as binary columns for fast 
+- Our extensive set of [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog) (HLL) functions that allow the saving of HLL sketches as binary columns for fast
 reaggregation as well as HLL interoperability with Postgres. (Spark has an HLL implementation but does not expose the binary HLL sketches,
 which makes its usefulness rather limited.)
 

VERSION

@@ -0,0 +1 @@
+0.1.0-SNAPSHOT

alchemy-test/src/main/scala/com/swoop/alchemy/test/SparkHelpers.scala

@@ -0,0 +1,15 @@
+package com.swoop.alchemy.test
+
+import org.apache.spark.sql.{Column, DataFrame}
+
+trait SparkHelpers {
+
+  def stripExpressionIds(s: String): String = "#\\d+".r.unanchored.replaceAllIn(s, "")
+
+  def simpleSql(column: Column): String = stripExpressionIds(column.expr.sql)
+
+  def simplePlan(df: DataFrame): String = stripExpressionIds(df.queryExecution.logical.toString).stripLineEnd
+
+}
+
+object SparkHelpers extends SparkHelpers

alchemy/src/main/scala/com/swoop/alchemy/utils/AnyExtensions.scala

@@ -0,0 +1,185 @@
+package com.swoop.alchemy.utils
+
+/**
+  * Convenience methods for all types
+  */
+object AnyExtensions {
+
+  /** Sugar for applying functions in a method chain. */
+  implicit class TransformOps[A](val underlying: A) extends AnyVal {
+
+    /** Applies a transformer function in a method chain.
+      *
+      * @param f function to apply
+      * @tparam B the return type
+      * @return the result of applying `f` on `underlying`.
+      */
+    @inline def transform[B](f: A => B): B =
+      f(underlying)
+
+    /** Conditionally applies a transformer function in a method chain.
+      * Use this instead of [[transformWhen()]] when the predicate requires the value of `underlying`.
+      *
+      * @param predicate predicate to evaluate to determine if the function should be applied
+      * @tparam B the return type of the function
+      * @return `underlying` if the predicate evaluates to `false` or the result of function application.
+      */
+    @inline def transformIf[B <: A](predicate: A => Boolean)(f: A => B): A =
+      if (predicate(underlying))
+        f(underlying)
+      else underlying
+
+    /** Conditionally applies a transformer function in a method chain.
+      * Use this instead of [[transformIf()]] when the condition does not require the value of `underlying`.
+      *
+      * @param condition condition to evaluate to determine if the function should be applied
+      * @tparam B the return type of the function
+      * @return `underlying` if the expression evaluates to `false` or the result of function application.
+      */
+    @inline def transformWhen[B <: A](condition: => Boolean)(f: A => B): A =
+      if (condition)
+        f(underlying)
+      else underlying
+
+  }
+
+  /** Sugar for creating side-effects in method chains. */
+  implicit class TapOps[A](val underlying: A) extends AnyVal {
+
+    /** Applies a function for its side-effect as part of a method chain.
+      * Inspired by Ruby's `Object#tap`.
+      *
+      * @param f side-effect function to call
+      * @tparam B the return type of the function; ignored
+      * @return `this`
+      */
+    @inline def tap[B](f: A => B): A = {
+      f(underlying)
+      underlying
+    }
+
+    /** Conditionally applies a function for its side-effect as part of a method chain.
+      * Use this instead of [[tapWhen()]] when the predicate requires the value of `underlying`.
+      *
+      * @param predicate predicate to evaluate to determine if the side-effect should be invoked
+      * @param f         side-effect function to call
+      * @tparam B the return type of the function; ignored
+      * @return `this`
+      */
+    @inline def tapIf[B](predicate: A => Boolean)(f: A => B): A = {
+      if (predicate(underlying))
+        f(underlying)
+      underlying
+    }
+
+    /** Conditionally applies a function for its side-effect as part of a method chain.
+      * Use this instead of [[tapIf()]] when the condition does not require the value of `underlying`.
+      *
+      * @param condition condition to evaluate to determine if the side-effect should be invoked
+      * @param f         side-effect function to call
+      * @tparam B the return type of the function; ignored
+      * @return `this`
+      */
+    @inline def tapWhen[B](condition: => Boolean)(f: A => B): A = {
+      if (condition)
+        f(underlying)
+      underlying
+    }
+  }
+
+  /** Sugar for simple debugging/reporting by printing in a method chain. */
+  implicit class PrintOps[A](val underlying: A) extends AnyVal {
+
+    /** Taps and prints the object in a method chain.
+      * Shorthand for `.tap(println)`.
+      *
+      * @return `underlying`
+      */
+    def tapp: A =
+      underlying.tap(println)
+
+    /** Prints a value as a side effect.
+      *
+      * @param v the value to print
+      * @return `underlying`
+      */
+    def print[B](v: B): A =
+      underlying.tap((_: A) => println(v))
+
+    /** Conditionally taps and prints the object in a method chain.
+      * Use this instead of [[printWhen()]] when the predicate requires the value of `underlying`.
+      *
+      * @param predicate predicate to evaluate to determine if the underlying value should be printed
+      * @return `this`
+      */
+    def printIf(predicate: A => Boolean): A =
+      underlying.tapIf(predicate)(println)
+
+    /** Conditionally prints a value as a side effect.
+      * Use this instead of [[printWhen()]] when the predicate requires the value of `underlying`.
+      *
+      * @param predicate predicate to evaluate to determine if the value should be printed
+      * @param v         side-effect function to call
+      * @tparam B the value type; ignored
+      * @return `this`
+      */
+    def printIf[B](predicate: A => Boolean, v: B): A =
+      underlying.tapIf(predicate)((_: A) => println(v))
+
+    /** Conditionally taps and prints the object in a method chain.
+      * Use this instead of [[printIf()]] when the condition does not require the value of `underlying`.
+      *
+      * @param condition condition to evaluate to determine if the underlying value should be printed
+      * @return `underlying`
+      */
+    def printWhen(condition: => Boolean): A =
+      underlying.tapWhen(condition)(println)
+
+    /** Conditionally prints a value as a side effect.
+      * Use this instead of [[printIf()]] when the condition does not require the value of `underlying`.
+      *
+      * @param condition condition to evaluate to determine if the value should be printed
+      * @tparam B the value type; ignored
+      * @return `underlying`
+      */
+    def printWhen[B](condition: => Boolean, v: B): A =
+      underlying.tapWhen(condition)((_: A) => println(v))
+
+  }
+
+  /** Sugar for conditionally raising exceptions as part of a method chain. */
+  implicit class ThrowOps[A](val underlying: A) extends AnyVal {
+
+    /** Raises an exception if a predicate is satisfied.
+      * Use this instead of [[throwWhen()]] when the predicate requires the value of `underlying`.
+      *
+      * @param predicate predicate to evaluate to determine if the exception should be thrown
+      * @param e         expression that will return an exception
+      * @tparam B the exception type
+      * @return `underlying` if the predicate evaluates to `false`.
+      * @throws B
+      */
+    def throwIf[B <: Throwable](predicate: A => Boolean)(e: => B): A = {
+      if (predicate(underlying))
+        throw e
+      underlying
+    }
+
+    /** Raises an exception if a condition is satisfied.
+      * Use this instead of [[throwIf()]] when the condition does not require the value of `underlying`.
+      *
+      * @param condition condition to evaluate to determine if the exception should be thrown
+      * @param e         expression that will return an exception
+      * @tparam B the exception type
+      * @return `underlying` if the predicate evaluates to `false`.
+      * @throws B
+      */
+    def throwWhen[B <: Throwable](condition: => Boolean, e: => B): A = {
+      if (condition)
+        throw e
+      underlying
+    }
+
+  }
+
+}

alchemy/src/test/scala/com/swoop/alchemy/utils/AnyExtensionsTest.scala

@@ -0,0 +1,27 @@
+package com.swoop.alchemy.utils
+
+import org.scalatest.{BeforeAndAfterEach, FunSuite}
+
+class AnyExtensionsTest extends FunSuite with BeforeAndAfterEach {
+
+  override def beforeEach() {
+
+  }
+
+  test("testTransformOps") {
+
+  }
+
+  test("testThrowOps") {
+
+  }
+
+  test("testTapOps") {
+
+  }
+
+  test("testPrintOps") {
+
+  }
+
+}