From c02f643b77c60ef1ab95de70697a2677c5a63786 Mon Sep 17 00:00:00 2001 From: Brendan O'Brien Date: Wed, 28 Jun 2017 16:05:34 -0400 Subject: [PATCH] prep for org switch --- .circleci/config.yml | 34 +++++---- Dockerfile | 6 +- Godeps/Godeps.json | 2 +- config.go | 161 ++++++++++++++++++++++--------------------- handlers.go | 4 +- server.go | 44 +++++++++--- transports.go | 4 +- 7 files changed, 143 insertions(+), 112 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 92dc5b6..20337b1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ version: '2' jobs: build: - working_directory: /go/src/github.com/archivers-space/content + working_directory: /go/src/github.com/datatogether/content docker: - image: circleci/cci-demo-go-primary:0.0.2 environment: @@ -29,32 +29,36 @@ jobs: echo Failed waiting for Postgres && exit 1 - run: name: Make test results directory - command: mkdir -p /tmp/test-reports/archivers-space + command: mkdir -p /tmp/test-reports/datatogether - run: name: Install dependencies command: go-wrapper download && go-wrapper install && go get -v github.com/jstemmer/go-junit-report - run: name: Run tests - command: go test -v -race ./... | tee /tmp/test-reports/archivers-space/original.txt ; test ${PIPESTATUS[0]} -eq 0 + command: go test -v -race ./... | tee /tmp/test-reports/datatogether/original.txt ; test ${PIPESTATUS[0]} -eq 0 - run: name: Convert test output to junit-style xml - command: cat /tmp/test-reports/archivers-space/original.txt | go-junit-report > /tmp/test-reports/archivers-space/junit.xml + command: cat /tmp/test-reports/datatogether/original.txt | go-junit-report > /tmp/test-reports/datatogether/junit.xml - store_test_results: - path: /tmp/test-reports/archivers-space/junit.xml + path: /tmp/test-reports/datatogether/junit.xml - setup_remote_docker - run: name: Install Docker client command: | - set -x - VER="17.03.0-ce" - curl -L -o /tmp/docker-$VER.tgz https://get.docker.com/builds/Linux/x86_64/docker-$VER.tgz - tar -xz -C /tmp -f /tmp/docker-$VER.tgz - mv /tmp/docker/* /usr/bin + if [ $CIRCLE_BRANCH = 'master' ]; then + set -x + VER="17.03.0-ce" + curl -L -o /tmp/docker-$VER.tgz https://get.docker.com/builds/Linux/x86_64/docker-$VER.tgz + tar -xz -C /tmp -f /tmp/docker-$VER.tgz + mv /tmp/docker/* /usr/bin + fi - run: name: Publish to Docker Hub command: | - TAG=0.1.$CIRCLE_BUILD_NUM - docker build -t archivers/content:latest -t archivers/content:$TAG . - docker login -u $DOCKER_USER -p $DOCKER_PASS - docker push archivers/content:$TAG - docker push archivers/content:latest \ No newline at end of file + if [ $CIRCLE_BRANCH = 'master' ]; then + TAG=0.1.$CIRCLE_BUILD_NUM + docker build -t datatogether/content:latest -t datatogether/content:$TAG . + docker login -u $DOCKER_USER -p $DOCKER_PASS + docker push datatogether/content:$TAG + docker push datatogether/content:latest + fi \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 0421b88..8092d63 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,12 +7,12 @@ EXPOSE 3000 # RUN go-wrapper install github.com/codegangsta/gin # Copy the local package files to the container’s workspace. -ADD . /go/src/github.com/archivers-space/content -# WORKDIR /go/src/github.com/archivers-space/content +ADD . /go/src/github.com/datatogether/content +# WORKDIR /go/src/github.com/datatogether/content # CMD ["gin", "-i"] # Install api binary globally within container -RUN go install github.com/archivers-space/content +RUN go install github.com/datatogether/content # Set binary as entrypoint ENTRYPOINT /go/bin/content diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 544275b..ddbaedd 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -22,7 +22,7 @@ "Rev": "349dd0209470eabd9514242c688c403c0926d266" }, { - "ImportPath": "github.com/archivers-space/archive", + "ImportPath": "github.com/datatogether/archive", "Rev": "51b99b3cce881aca6f42c00d63d45e1635ddc17b" }, { diff --git a/config.go b/config.go index 0bff513..2d698a4 100644 --- a/config.go +++ b/config.go @@ -1,13 +1,11 @@ package main import ( - "encoding/json" "fmt" - "github.com/archivers-space/archive" - "io/ioutil" + "github.com/datatogether/archive" + conf "github.com/datatogether/config" "os" "path/filepath" - "strings" ) // server modes @@ -30,33 +28,33 @@ const ( // configuration is read at startup and cannot be alterd without restarting the server. type config struct { // port to listen on, will be read from PORT env variable if present. - Port string `json:"PORT"` + Port string // root url for service - UrlRoot string `json:"URL_ROOT"` + UrlRoot string // url of postgres app db - PostgresDbUrl string `json:"POSTGRES_DB_URL"` + PostgresDbUrl string // Public Key to use for signing metablocks. required. - PublicKey string `json:"PUBLIC_KEY"` + PublicKey string // TLS (HTTPS) enable support via LetsEncrypt, default false // should be true in production - TLS bool `json:"TLS"` + TLS bool // read from env variable: AWS_REGION // the region your bucket is in, eg "us-east-1" - AwsRegion string `json:"AWS_REGION"` + AwsRegion string // read from env variable: AWS_S3_BUCKET_NAME // should be just the name of your bucket, no protocol prefixes or paths - AwsS3BucketName string `json:"AWS_S3_BUCKET_NAME"` + AwsS3BucketName string // read from env variable: AWS_ACCESS_KEY_ID - AwsAccessKeyId string `json:"AWS_ACCESS_KEY_ID"` + AwsAccessKeyId string // read from env variable: AWS_SECRET_ACCESS_KEY - AwsSecretAccessKey string `json:"AWS_SECRET_ACCESS_KEY"` + AwsSecretAccessKey string // path to store & retrieve data from - AwsS3BucketPath string `json:"AWS_S3_BUCKET_PATH"` + AwsS3BucketPath string // seed = flag.String("seed", "", "seed URL") // cancelAfter = flag.Duration("cancelafter", 0, "automatically cancel the fetchbot after a given time") @@ -70,43 +68,78 @@ type config struct { // username & password that must be passed in with every request. // leaving these values blank will disable http auth // read from env variable: HTTP_AUTH_USERNAME - HttpAuthUsername string `json:"HTTP_AUTH_USERNAME"` + HttpAuthUsername string // read from env variable: HTTP_AUTH_PASSWORD - HttpAuthPassword string `json:"HTTP_AUTH_PASSWORD"` + HttpAuthPassword string // if true, requests that have X-Forwarded-Proto: http will be redirected // to their https variant ProxyForceHttps bool // CertbotResponse is only for doing manual SSL certificate generation via LetsEncrypt. - CertbotResponse string `json:"CERTBOT_RESPONSE"` + CertbotResponse string } // initConfig pulls configuration from config.json +// func initConfig(mode string) (cfg *config, err error) { +// cfg = &config{} + +// if err := loadConfigFile(mode, cfg); err != nil { +// return cfg, err +// } + +// // override config settings with env settings, passing in the current configuration +// // as the default. This has the effect of leaving the config.json value unchanged +// // if the env variable is empty +// cfg.Port = readEnvString("PORT", cfg.Port) +// cfg.UrlRoot = readEnvString("URL_ROOT", cfg.UrlRoot) +// cfg.PublicKey = readEnvString("PUBLIC_KEY", cfg.PublicKey) +// cfg.TLS = readEnvBool("TLS", cfg.TLS) +// cfg.PostgresDbUrl = readEnvString("POSTGRES_DB_URL", cfg.PostgresDbUrl) +// cfg.HttpAuthUsername = readEnvString("HTTP_AUTH_USERNAME", cfg.HttpAuthUsername) +// cfg.HttpAuthPassword = readEnvString("HTTP_AUTH_PASSWORD", cfg.HttpAuthPassword) +// cfg.AwsAccessKeyId = readEnvString("AWS_ACCESS_KEY_ID", cfg.AwsAccessKeyId) +// cfg.AwsSecretAccessKey = readEnvString("AWS_SECRET_ACCESS_KEY", cfg.AwsSecretAccessKey) +// cfg.AwsRegion = readEnvString("AWS_REGION", cfg.AwsRegion) +// cfg.AwsS3BucketName = readEnvString("AWS_S3_BUCKET_NAME", cfg.AwsS3BucketName) +// cfg.AwsS3BucketPath = readEnvString("AWS_S3_BUCKET_PATH", cfg.AwsS3BucketPath) +// cfg.CertbotResponse = readEnvString("CERTBOT_RESPONSE", cfg.CertbotResponse) +// // cfg.StaleDuration = readEnvInt("STALE_DURATION", cfg.StaleDuration) + +// // make sure port is set +// if cfg.Port == "" { +// cfg.Port = "8080" +// } + +// err = requireConfigStrings(map[string]string{ +// "PORT": cfg.Port, +// "POSTGRES_DB_URL": cfg.PostgresDbUrl, +// "PUBLIC_KEY": cfg.PublicKey, +// }) + +// // transfer settings to archive library +// archive.AwsRegion = cfg.AwsRegion +// archive.AwsAccessKeyId = cfg.AwsAccessKeyId +// archive.AwsS3BucketName = cfg.AwsS3BucketName +// archive.AwsS3BucketPath = cfg.AwsS3BucketPath +// archive.AwsSecretAccessKey = cfg.AwsSecretAccessKey + +// return +// } + func initConfig(mode string) (cfg *config, err error) { cfg = &config{} - if err := loadConfigFile(mode, cfg); err != nil { - return cfg, err + if path := configFilePath(mode, cfg); path != "" { + log.Infof("loading config file: %s", filepath.Base(path)) + if err := conf.Load(cfg, path); err != nil { + log.Info("error loading config:", err) + } + } else { + if err := conf.Load(cfg); err != nil { + log.Info("error loading config:", err) + } } - // override config settings with env settings, passing in the current configuration - // as the default. This has the effect of leaving the config.json value unchanged - // if the env variable is empty - cfg.Port = readEnvString("PORT", cfg.Port) - cfg.UrlRoot = readEnvString("URL_ROOT", cfg.UrlRoot) - cfg.PublicKey = readEnvString("PUBLIC_KEY", cfg.PublicKey) - cfg.TLS = readEnvBool("TLS", cfg.TLS) - cfg.PostgresDbUrl = readEnvString("POSTGRES_DB_URL", cfg.PostgresDbUrl) - cfg.HttpAuthUsername = readEnvString("HTTP_AUTH_USERNAME", cfg.HttpAuthUsername) - cfg.HttpAuthPassword = readEnvString("HTTP_AUTH_PASSWORD", cfg.HttpAuthPassword) - cfg.AwsAccessKeyId = readEnvString("AWS_ACCESS_KEY_ID", cfg.AwsAccessKeyId) - cfg.AwsSecretAccessKey = readEnvString("AWS_SECRET_ACCESS_KEY", cfg.AwsSecretAccessKey) - cfg.AwsRegion = readEnvString("AWS_REGION", cfg.AwsRegion) - cfg.AwsS3BucketName = readEnvString("AWS_S3_BUCKET_NAME", cfg.AwsS3BucketName) - cfg.AwsS3BucketPath = readEnvString("AWS_S3_BUCKET_PATH", cfg.AwsS3BucketPath) - cfg.CertbotResponse = readEnvString("CERTBOT_RESPONSE", cfg.CertbotResponse) - // cfg.StaleDuration = readEnvInt("STALE_DURATION", cfg.StaleDuration) - // make sure port is set if cfg.Port == "" { cfg.Port = "8080" @@ -115,7 +148,6 @@ func initConfig(mode string) (cfg *config, err error) { err = requireConfigStrings(map[string]string{ "PORT": cfg.Port, "POSTGRES_DB_URL": cfg.PostgresDbUrl, - "PUBLIC_KEY": cfg.PublicKey, }) // transfer settings to archive library @@ -125,11 +157,16 @@ func initConfig(mode string) (cfg *config, err error) { archive.AwsS3BucketPath = cfg.AwsS3BucketPath archive.AwsSecretAccessKey = cfg.AwsSecretAccessKey + // output to stdout in dev mode + if mode == DEVELOP_MODE { + log.Out = os.Stdout + } + return } func packagePath(path string) string { - return filepath.Join(os.Getenv("GOPATH"), "src/github.com/archivers-space/content", path) + return filepath.Join(os.Getenv("GOPATH"), "src/github.com/datatogether/content", path) } // readEnvString reads key from the environment, returns def if empty @@ -140,22 +177,6 @@ func readEnvString(key, def string) string { return def } -// readEnvBool read key form the env, converting to a boolean value. returns def if empty -func readEnvBool(key string, def bool) bool { - if env := os.Getenv(key); env != "" { - return env == "true" || env == "TRUE" || env == "t" - } - return def -} - -// readEnvString reads a slice of strings from key environment var, returns def if empty -func readEnvStringSlice(key string, def []string) []string { - if env := os.Getenv(key); env != "" { - return strings.Split(env, ",") - } - return def -} - // requireConfigStrings panics if any of the passed in values aren't set func requireConfigStrings(values map[string]string) error { for key, value := range values { @@ -167,33 +188,17 @@ func requireConfigStrings(values map[string]string) error { return nil } -// checks for config.[mode].json file to read configuration from if the file exists -// defaults to config.json, silently fails if no configuration file is present. -func loadConfigFile(mode string, cfg *config) (err error) { - var data []byte - - fileName := packagePath(fmt.Sprintf("config.%s.json", mode)) +// checks for .[mode].env file to read configuration from if the file exists +// defaults to .env, returns "" if no file is present +func configFilePath(mode string, cfg *config) string { + fileName := packagePath(fmt.Sprintf(".%s.env", mode)) if !fileExists(fileName) { - fileName = packagePath("config.json") + fileName = packagePath(".env") if !fileExists(fileName) { - return nil + return "" } } - - logger.Printf("reading config file: %s", fileName) - data, err = ioutil.ReadFile(fileName) - if err != nil { - err = fmt.Errorf("error reading %s: %s", fileName, err) - return - } - - // unmarshal ("decode") config data into a config struct - if err = json.Unmarshal(data, cfg); err != nil { - err = fmt.Errorf("error parsing %s: %s", fileName, err) - return - } - - return + return fileName } // Does this file exist? diff --git a/handlers.go b/handlers.go index 2bfbf9a..2f1169c 100644 --- a/handlers.go +++ b/handlers.go @@ -2,7 +2,7 @@ package main import ( "fmt" - "github.com/archivers-space/archive" + "github.com/datatogether/archive" "io" "net/http" "strconv" @@ -30,7 +30,7 @@ func DownloadUrlHandler(w http.ResponseWriter, r *http.Request) { func DownloadUrl(w http.ResponseWriter, r *http.Request) { u := &archive.Url{Id: r.URL.Path[len("/urls/"):]} - if err := u.Read(appDB); err != nil { + if err := u.Read(store); err != nil { w.WriteHeader(http.StatusNotFound) io.WriteString(w, err.Error()) return diff --git a/server.go b/server.go index 82813f8..5f1cb4c 100644 --- a/server.go +++ b/server.go @@ -3,7 +3,9 @@ package main import ( "database/sql" "fmt" - "log" + "github.com/datatogether/archive" + "github.com/datatogether/sql_datastore" + "github.com/sirupsen/logrus" "net/http" "os" "time" @@ -18,13 +20,23 @@ var ( // Use this value to avoid bombing alerts lastAlertSent *time.Time - // log output - logger = log.New(os.Stderr, "", log.Ldate|log.Ltime|log.Lshortfile) + // log output via logrus package + log = logrus.New() // application database connection appDB *sql.DB + // elevate default store + store = sql_datastore.DefaultStore ) +func init() { + log.Out = os.Stdout + log.Level = logrus.InfoLevel + log.Formatter = &logrus.TextFormatter{ + ForceColors: true, + } +} + func main() { var err error cfg, err = initConfig(os.Getenv("GOLANG_ENV")) @@ -33,17 +45,16 @@ func main() { panic(fmt.Errorf("server configuration error: %s", err.Error())) } - connectToAppDb() + go connectToAppDb() + sql_datastore.SetDB(appDB) + sql_datastore.Register( + &archive.Url{}, + ) s := &http.Server{} - m := http.NewServeMux() - m.HandleFunc("/.well-known/acme-challenge/", CertbotHandler) - m.Handle("/", middleware(HealthCheckHandler)) - - m.Handle("/urls/", middleware(DownloadUrlHandler)) // connect mux to server - s.Handler = m + s.Handler = NewServerRoutes() // print notable config settings // printConfigInfo() @@ -53,5 +64,16 @@ func main() { // start server wrapped in a log.Fatal b/c http.ListenAndServe will not // return unless there's an error - logger.Fatal(StartServer(cfg, s)) + log.Fatal(StartServer(cfg, s)) +} + +// NewServerRoutes returns a Muxer that has all API routes. +// This makes for easy testing using httptest +func NewServerRoutes() *http.ServeMux { + m := http.NewServeMux() + m.HandleFunc("/.well-known/acme-challenge/", CertbotHandler) + m.Handle("/", middleware(HealthCheckHandler)) + m.Handle("/urls/", middleware(DownloadUrlHandler)) + + return m } diff --git a/transports.go b/transports.go index e29acf5..39936c5 100644 --- a/transports.go +++ b/transports.go @@ -52,7 +52,7 @@ func HttpsRedirect() { return } - logger.Println("TCP Port 80 is available, redirecting traffic to https") + log.Println("TCP Port 80 is available, redirecting traffic to https") srv := &http.Server{ ReadTimeout: 5 * time.Second, @@ -63,5 +63,5 @@ func HttpsRedirect() { http.Redirect(w, req, url, http.StatusMovedPermanently) }), } - logger.Fatal(srv.Serve(ln)) + log.Fatal(srv.Serve(ln)) }