prep for org switch

.circleci/config.yml

@@ -1,7 +1,7 @@
 version: '2'
 jobs:
   build:
-    working_directory: /go/src/github.com/archivers-space/content
+    working_directory: /go/src/github.com/datatogether/content
     docker:
       - image: circleci/cci-demo-go-primary:0.0.2
         environment:
@@ -29,32 +29,36 @@ jobs:
             echo Failed waiting for Postgres && exit 1
       - run: 
           name: Make test results directory
-          command: mkdir -p /tmp/test-reports/archivers-space
+          command: mkdir -p /tmp/test-reports/datatogether
       - run:
           name: Install dependencies
           command: go-wrapper download && go-wrapper install && go get -v github.com/jstemmer/go-junit-report
       - run: 
           name: Run tests
-          command: go test -v -race ./... | tee /tmp/test-reports/archivers-space/original.txt ; test ${PIPESTATUS[0]} -eq 0
+          command: go test -v -race ./... | tee /tmp/test-reports/datatogether/original.txt ; test ${PIPESTATUS[0]} -eq 0
       - run: 
           name: Convert test output to junit-style xml
-          command: cat /tmp/test-reports/archivers-space/original.txt | go-junit-report > /tmp/test-reports/archivers-space/junit.xml
+          command: cat /tmp/test-reports/datatogether/original.txt | go-junit-report > /tmp/test-reports/datatogether/junit.xml
       - store_test_results:
-          path: /tmp/test-reports/archivers-space/junit.xml
+          path: /tmp/test-reports/datatogether/junit.xml
       - setup_remote_docker
       - run:
           name: Install Docker client
           command: |
-            set -x
-            VER="17.03.0-ce"
-            curl -L -o /tmp/docker-$VER.tgz https://get.docker.com/builds/Linux/x86_64/docker-$VER.tgz
-            tar -xz -C /tmp -f /tmp/docker-$VER.tgz
-            mv /tmp/docker/* /usr/bin
+            if [ $CIRCLE_BRANCH = 'master' ]; then
+              set -x
+              VER="17.03.0-ce"
+              curl -L -o /tmp/docker-$VER.tgz https://get.docker.com/builds/Linux/x86_64/docker-$VER.tgz
+              tar -xz -C /tmp -f /tmp/docker-$VER.tgz
+              mv /tmp/docker/* /usr/bin
+            fi
       - run:
           name: Publish to Docker Hub
           command: |
-            TAG=0.1.$CIRCLE_BUILD_NUM
-            docker build -t archivers/content:latest -t archivers/content:$TAG .
-            docker login -u $DOCKER_USER -p $DOCKER_PASS
-            docker push archivers/content:$TAG
-            docker push archivers/content:latest
+            if [ $CIRCLE_BRANCH = 'master' ]; then
+              TAG=0.1.$CIRCLE_BUILD_NUM
+              docker build -t datatogether/content:latest -t datatogether/content:$TAG .
+              docker login -u $DOCKER_USER -p $DOCKER_PASS
+              docker push datatogether/content:$TAG
+              docker push datatogether/content:latest
+            fi

Dockerfile

@@ -7,12 +7,12 @@ EXPOSE 3000
 # RUN go-wrapper install github.com/codegangsta/gin
 
 # Copy the local package files to the container’s workspace.
-ADD . /go/src/github.com/archivers-space/content
-# WORKDIR /go/src/github.com/archivers-space/content
+ADD . /go/src/github.com/datatogether/content
+# WORKDIR /go/src/github.com/datatogether/content
 # CMD ["gin", "-i"]
 
 # Install api binary globally within container 
-RUN go install github.com/archivers-space/content
+RUN go install github.com/datatogether/content
 # Set binary as entrypoint
 ENTRYPOINT /go/bin/content
 

config.go

@@ -1,13 +1,11 @@
 package main
 
 import (
-	"encoding/json"
 	"fmt"
-	"github.com/archivers-space/archive"
-	"io/ioutil"
+	"github.com/datatogether/archive"
+	conf "github.com/datatogether/config"
 	"os"
 	"path/filepath"
-	"strings"
 )
 
 // server modes
@@ -30,33 +28,33 @@ const (
 // configuration is read at startup and cannot be alterd without restarting the server.
 type config struct {
 	// port to listen on, will be read from PORT env variable if present.
-	Port string `json:"PORT"`
+	Port string
 
 	// root url for service
-	UrlRoot string `json:"URL_ROOT"`
+	UrlRoot string
 
 	// url of postgres app db
-	PostgresDbUrl string `json:"POSTGRES_DB_URL"`
+	PostgresDbUrl string
 
 	// Public Key to use for signing metablocks. required.
-	PublicKey string `json:"PUBLIC_KEY"`
+	PublicKey string
 
 	// TLS (HTTPS) enable support via LetsEncrypt, default false
 	// should be true in production
-	TLS bool `json:"TLS"`
+	TLS bool
 
 	// read from env variable: AWS_REGION
 	// the region your bucket is in, eg "us-east-1"
-	AwsRegion string `json:"AWS_REGION"`
+	AwsRegion string
 	// read from env variable: AWS_S3_BUCKET_NAME
 	// should be just the name of your bucket, no protocol prefixes or paths
-	AwsS3BucketName string `json:"AWS_S3_BUCKET_NAME"`
+	AwsS3BucketName string
 	// read from env variable: AWS_ACCESS_KEY_ID
-	AwsAccessKeyId string `json:"AWS_ACCESS_KEY_ID"`
+	AwsAccessKeyId string
 	// read from env variable: AWS_SECRET_ACCESS_KEY
-	AwsSecretAccessKey string `json:"AWS_SECRET_ACCESS_KEY"`
+	AwsSecretAccessKey string
 	// path to store & retrieve data from
-	AwsS3BucketPath string `json:"AWS_S3_BUCKET_PATH"`
+	AwsS3BucketPath string
 
 	// seed        = flag.String("seed", "", "seed URL")
 	// cancelAfter = flag.Duration("cancelafter", 0, "automatically cancel the fetchbot after a given time")
@@ -70,43 +68,78 @@ type config struct {
 	// username & password that must be passed in with every request.
 	// leaving these values blank will disable http auth
 	// read from env variable: HTTP_AUTH_USERNAME
-	HttpAuthUsername string `json:"HTTP_AUTH_USERNAME"`
+	HttpAuthUsername string
 	// read from env variable: HTTP_AUTH_PASSWORD
-	HttpAuthPassword string `json:"HTTP_AUTH_PASSWORD"`
+	HttpAuthPassword string
 
 	// if true, requests that have X-Forwarded-Proto: http will be redirected
 	// to their https variant
 	ProxyForceHttps bool
 	// CertbotResponse is only for doing manual SSL certificate generation via LetsEncrypt.
-	CertbotResponse string `json:"CERTBOT_RESPONSE"`
+	CertbotResponse string
 }
 
 // initConfig pulls configuration from config.json
+// func initConfig(mode string) (cfg *config, err error) {
+// 	cfg = &config{}
+
+// 	if err := loadConfigFile(mode, cfg); err != nil {
+// 		return cfg, err
+// 	}
+
+// 	// override config settings with env settings, passing in the current configuration
+// 	// as the default. This has the effect of leaving the config.json value unchanged
+// 	// if the env variable is empty
+// 	cfg.Port = readEnvString("PORT", cfg.Port)
+// 	cfg.UrlRoot = readEnvString("URL_ROOT", cfg.UrlRoot)
+// 	cfg.PublicKey = readEnvString("PUBLIC_KEY", cfg.PublicKey)
+// 	cfg.TLS = readEnvBool("TLS", cfg.TLS)
+// 	cfg.PostgresDbUrl = readEnvString("POSTGRES_DB_URL", cfg.PostgresDbUrl)
+// 	cfg.HttpAuthUsername = readEnvString("HTTP_AUTH_USERNAME", cfg.HttpAuthUsername)
+// 	cfg.HttpAuthPassword = readEnvString("HTTP_AUTH_PASSWORD", cfg.HttpAuthPassword)
+// 	cfg.AwsAccessKeyId = readEnvString("AWS_ACCESS_KEY_ID", cfg.AwsAccessKeyId)
+// 	cfg.AwsSecretAccessKey = readEnvString("AWS_SECRET_ACCESS_KEY", cfg.AwsSecretAccessKey)
+// 	cfg.AwsRegion = readEnvString("AWS_REGION", cfg.AwsRegion)
+// 	cfg.AwsS3BucketName = readEnvString("AWS_S3_BUCKET_NAME", cfg.AwsS3BucketName)
+// 	cfg.AwsS3BucketPath = readEnvString("AWS_S3_BUCKET_PATH", cfg.AwsS3BucketPath)
+// 	cfg.CertbotResponse = readEnvString("CERTBOT_RESPONSE", cfg.CertbotResponse)
+// 	// cfg.StaleDuration = readEnvInt("STALE_DURATION", cfg.StaleDuration)
+
+// 	// make sure port is set
+// 	if cfg.Port == "" {
+// 		cfg.Port = "8080"
+// 	}
+
+// 	err = requireConfigStrings(map[string]string{
+// 		"PORT":            cfg.Port,
+// 		"POSTGRES_DB_URL": cfg.PostgresDbUrl,
+// 		"PUBLIC_KEY":      cfg.PublicKey,
+// 	})
+
+// 	// transfer settings to archive library
+// 	archive.AwsRegion = cfg.AwsRegion
+// 	archive.AwsAccessKeyId = cfg.AwsAccessKeyId
+// 	archive.AwsS3BucketName = cfg.AwsS3BucketName
+// 	archive.AwsS3BucketPath = cfg.AwsS3BucketPath
+// 	archive.AwsSecretAccessKey = cfg.AwsSecretAccessKey
+
+// 	return
+// }
+
 func initConfig(mode string) (cfg *config, err error) {
 	cfg = &config{}
 
-	if err := loadConfigFile(mode, cfg); err != nil {
-		return cfg, err
+	if path := configFilePath(mode, cfg); path != "" {
+		log.Infof("loading config file: %s", filepath.Base(path))
+		if err := conf.Load(cfg, path); err != nil {
+			log.Info("error loading config:", err)
+		}
+	} else {
+		if err := conf.Load(cfg); err != nil {
+			log.Info("error loading config:", err)
+		}
 	}
 
-	// override config settings with env settings, passing in the current configuration
-	// as the default. This has the effect of leaving the config.json value unchanged
-	// if the env variable is empty
-	cfg.Port = readEnvString("PORT", cfg.Port)
-	cfg.UrlRoot = readEnvString("URL_ROOT", cfg.UrlRoot)
-	cfg.PublicKey = readEnvString("PUBLIC_KEY", cfg.PublicKey)
-	cfg.TLS = readEnvBool("TLS", cfg.TLS)
-	cfg.PostgresDbUrl = readEnvString("POSTGRES_DB_URL", cfg.PostgresDbUrl)
-	cfg.HttpAuthUsername = readEnvString("HTTP_AUTH_USERNAME", cfg.HttpAuthUsername)
-	cfg.HttpAuthPassword = readEnvString("HTTP_AUTH_PASSWORD", cfg.HttpAuthPassword)
-	cfg.AwsAccessKeyId = readEnvString("AWS_ACCESS_KEY_ID", cfg.AwsAccessKeyId)
-	cfg.AwsSecretAccessKey = readEnvString("AWS_SECRET_ACCESS_KEY", cfg.AwsSecretAccessKey)
-	cfg.AwsRegion = readEnvString("AWS_REGION", cfg.AwsRegion)
-	cfg.AwsS3BucketName = readEnvString("AWS_S3_BUCKET_NAME", cfg.AwsS3BucketName)
-	cfg.AwsS3BucketPath = readEnvString("AWS_S3_BUCKET_PATH", cfg.AwsS3BucketPath)
-	cfg.CertbotResponse = readEnvString("CERTBOT_RESPONSE", cfg.CertbotResponse)
-	// cfg.StaleDuration = readEnvInt("STALE_DURATION", cfg.StaleDuration)
-
 	// make sure port is set
 	if cfg.Port == "" {
 		cfg.Port = "8080"
@@ -115,7 +148,6 @@ func initConfig(mode string) (cfg *config, err error) {
 	err = requireConfigStrings(map[string]string{
 		"PORT":            cfg.Port,
 		"POSTGRES_DB_URL": cfg.PostgresDbUrl,
-		"PUBLIC_KEY":      cfg.PublicKey,
 	})
 
 	// transfer settings to archive library
@@ -125,11 +157,16 @@ func initConfig(mode string) (cfg *config, err error) {
 	archive.AwsS3BucketPath = cfg.AwsS3BucketPath
 	archive.AwsSecretAccessKey = cfg.AwsSecretAccessKey
 
+	// output to stdout in dev mode
+	if mode == DEVELOP_MODE {
+		log.Out = os.Stdout
+	}
+
 	return
 }
 
 func packagePath(path string) string {
-	return filepath.Join(os.Getenv("GOPATH"), "src/github.com/archivers-space/content", path)
+	return filepath.Join(os.Getenv("GOPATH"), "src/github.com/datatogether/content", path)
 }
 
 // readEnvString reads key from the environment, returns def if empty
@@ -140,22 +177,6 @@ func readEnvString(key, def string) string {
 	return def
 }
 
-// readEnvBool read key form the env, converting to a boolean value. returns def if empty
-func readEnvBool(key string, def bool) bool {
-	if env := os.Getenv(key); env != "" {
-		return env == "true" || env == "TRUE" || env == "t"
-	}
-	return def
-}
-
-// readEnvString reads a slice of strings from key environment var, returns def if empty
-func readEnvStringSlice(key string, def []string) []string {
-	if env := os.Getenv(key); env != "" {
-		return strings.Split(env, ",")
-	}
-	return def
-}
-
 // requireConfigStrings panics if any of the passed in values aren't set
 func requireConfigStrings(values map[string]string) error {
 	for key, value := range values {
@@ -167,33 +188,17 @@ func requireConfigStrings(values map[string]string) error {
 	return nil
 }
 
-// checks for config.[mode].json file to read configuration from if the file exists
-// defaults to config.json, silently fails if no configuration file is present.
-func loadConfigFile(mode string, cfg *config) (err error) {
-	var data []byte
-
-	fileName := packagePath(fmt.Sprintf("config.%s.json", mode))
+// checks for .[mode].env file to read configuration from if the file exists
+// defaults to .env, returns "" if no file is present
+func configFilePath(mode string, cfg *config) string {
+	fileName := packagePath(fmt.Sprintf(".%s.env", mode))
 	if !fileExists(fileName) {
-		fileName = packagePath("config.json")
+		fileName = packagePath(".env")
 		if !fileExists(fileName) {
-			return nil
+			return ""
 		}
 	}
-
-	logger.Printf("reading config file: %s", fileName)
-	data, err = ioutil.ReadFile(fileName)
-	if err != nil {
-		err = fmt.Errorf("error reading %s: %s", fileName, err)
-		return
-	}
-
-	// unmarshal ("decode") config data into a config struct
-	if err = json.Unmarshal(data, cfg); err != nil {
-		err = fmt.Errorf("error parsing %s: %s", fileName, err)
-		return
-	}
-
-	return
+	return fileName
 }
 
 // Does this file exist?

handlers.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	"fmt"
-	"github.com/archivers-space/archive"
+	"github.com/datatogether/archive"
 	"io"
 	"net/http"
 	"strconv"
@@ -30,7 +30,7 @@ func DownloadUrlHandler(w http.ResponseWriter, r *http.Request) {
 
 func DownloadUrl(w http.ResponseWriter, r *http.Request) {
 	u := &archive.Url{Id: r.URL.Path[len("/urls/"):]}
-	if err := u.Read(appDB); err != nil {
+	if err := u.Read(store); err != nil {
 		w.WriteHeader(http.StatusNotFound)
 		io.WriteString(w, err.Error())
 		return