initial commit

Author: Esteban Garcia

.gitignore

@@ -0,0 +1 @@
+lfsproxy.yaml

.lint/config.yml

@@ -0,0 +1,24 @@
+linters:
+  enable:
+  # Enable specific linter
+  # https://golangci-lint.run/usage/linters
+    - asciicheck
+    - durationcheck
+    - errorlint
+    - gocyclo
+    - gofmt
+    - goimports
+    - gosec
+    - maintidx
+    - wastedassign
+    - exportloopref
+issues:
+  exclude-rules:
+    # Exclude some linters from running on tests files.
+    - path: _test\.go
+      linters:
+        - gocyclo
+        - errcheck
+        - gosec
+        - maintidx
+        - errorlint

Dockerfile

@@ -0,0 +1,21 @@
+FROM golang:1.20 AS build
+
+WORKDIR /app
+
+COPY go.* ./
+RUN go mod download
+
+COPY . ./
+
+ENV CGO_ENABLED=0
+
+RUN go build -v -o lfsproxy ./cmd/server.go
+
+FROM alpine:3.17
+
+WORKDIR /app
+
+RUN apk add --no-cache libc6-compat gcompat
+
+COPY --from=build /app/lfsproxy /app/lfsproxy
+CMD ["/app/lfsproxy"]

LICENSE

@@ -0,0 +1,22 @@
+
+MIT License
+
+Copyright (c) 2022 Vela Games Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,30 @@
+# LFS Proxy
+
+This service is a pull-through S3 cache for [Git LFS](https://git-lfs.com/) that caches Upstream LFS objects on S3 to reduce bandwidth costs on Hosted LFS, such as GitHub LFS.
+
+These cached objects are served directly from S3 using Presigned requests for maximum performance.
+
+Requests are cached in-memory using [bigcache](https://github.com/allegro/bigcache) to reduce the amount of HTTP calls to S3
+
+## Installing the LFS Proxy
+
+We are providing a [Helm Chart](https://github.com/vela-games/lfsproxy/tree/main/install/helm/lfsproxy) and an example [terrafom module](https://github.com/vela-games/lfsproxy/tree/main/install/terraform/lfsproxy) for deploying the service 
+
+You'll need to make sure the service account deployed by the chart uses an IAM Role with access to S3 if you are running it on Kubernetes.
+
+We currently don't have any public repositories for the Docker Image or the Helm chart, but is something we are looking into.
+
+## Configurations
+
+All configurations are loaded from environment variables using [envconfig](https://github.com/kelseyhightower/envconfig).
+
+| Configuration Name             | Environment Variable                 | Default Value                                    | Description                                                                                       |
+|--------------------------------|--------------------------------------|--------------------------------------------------|---------------------------------------------------------------------------------------------------|
+| DebugMode                      | APP_DEBUG_MODE                       | false                                            | Enable gin-gonic debug mode                                                                       |
+| UpstreamBaseURL                | APP_UPSTREAM_BASE_URL                |                                                  | The LFS Git Repository base url (Example: https://github.com/vela-games/example.git/info/lfs/)    |
+| S3Bucket                       | APP_S3_BUCKET                        |                                                  | S3 Bucket Name                                                                                    |
+| S3UseAccelerate                | APP_S3_USE_ACCELERATE                | false                                            | If S3 Accelerate URLs should be returned                                                          |
+| S3PresignEnabled               | APP_S3_PRESIGN_ENABLED               | true                                             | If S3 Presign URLs should be used                                                                 |
+| S3PresignExpiration            | APP_S3_PRESIGN_EXPIRATION            | 24h                                              | Presign Expiration                                                                                |
+| CacheEviction                  | APP_CACHE_EVICTION                   | 23h                                              | When to evict cached requests from memory                                                         |
+| EnablePrometheusExporter       | APP_ENABLE_PROMETHEUS_EXPORTER       | false                                            | Enable Prometheus exporter endpoint (/metrics)                                                    |

cache/cache.go

@@ -0,0 +1,23 @@
+package cache
+
+import (
+	"context"
+	"time"
+
+	"github.com/allegro/bigcache/v3"
+)
+
+type Cache interface {
+	Get(key string) ([]byte, error)
+	Set(key string, entry []byte) error
+	Delete(key string) error
+}
+
+func NewCache(ctx context.Context, cacheEviction time.Duration) (Cache, error) {
+	cache, err := bigcache.New(ctx, bigcache.DefaultConfig(cacheEviction))
+	if err != nil {
+		return nil, err
+	}
+
+	return cache, nil
+}

cmd/server.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/vela-games/lfsproxy/config"
+	"github.com/vela-games/lfsproxy/router"
+
+	"context"
+	"os"
+	"os/signal"
+	"syscall"
+)
+
+const PORT = 8080
+
+// NewSigKillContext returns a Context that cancels when os.Interrupt or os.Kill is received
+func NewSigKillContext() context.Context {
+	ctx, cancel := context.WithCancel(context.Background())
+	c := make(chan os.Signal, 2)
+	signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		<-c
+		cancel()
+	}()
+
+	return ctx
+}
+
+func main() {
+	ctx := NewSigKillContext()
+	cfg, err := config.GetConfig()
+	if err != nil {
+		log.Panicf("error getting configuration: %v", err)
+	}
+
+	router := router.NewRouter()
+	err = router.InitRoutes(ctx, cfg)
+	if err != nil {
+		panic(err)
+	}
+
+	err = router.Run(ctx, fmt.Sprintf(":%v", PORT))
+	if err != nil {
+		panic(err)
+	}
+
+}

config/config.go

@@ -0,0 +1,29 @@
+package config
+
+import (
+	"time"
+
+	"github.com/kelseyhightower/envconfig"
+)
+
+type Config struct {
+	DebugMode                bool          `split_words:"true" default:"false"`
+	UpstreamBaseURL          string        `split_words:"true" required:"true"`
+	CacheEviction            time.Duration `split_words:"true" default:"23h"`
+	S3Bucket                 string        `split_words:"true" required:"true"`
+	S3UseAccelerate          bool          `split_words:"true" default:"false"`
+	S3PresignEnabled         bool          `split_words:"true" default:"true"`
+	S3PresignExpiration      time.Duration `split_words:"true" default:"24h"`
+	EnablePrometheusExporter bool          `split_words:"true" default:"false"`
+}
+
+func GetConfig() (*Config, error) {
+	var proxyConfiguration Config
+
+	err := envconfig.Process("app", &proxyConfiguration)
+	if err != nil {
+		return nil, err
+	}
+
+	return &proxyConfiguration, nil
+}

exporter/collector.go

@@ -0,0 +1,49 @@
+package exporter
+
+import (
+	"github.com/gin-gonic/gin"
+	"github.com/go-kit/kit/metrics"
+	"github.com/go-kit/kit/metrics/prometheus"
+	stdprometheus "github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+type LFSProxyCollector struct {
+	CacheHits metrics.Counter
+	CacheMiss metrics.Counter
+	S3Hits    metrics.Counter
+	S3Miss    metrics.Counter
+}
+
+func NewCollector() *LFSProxyCollector {
+	return &LFSProxyCollector{
+		CacheHits: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
+			Namespace: "lfsproxy",
+			Name:      "cache_hit",
+			Help:      "In-memory Cache Hits",
+		}, []string{}),
+		CacheMiss: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
+			Namespace: "lfsproxy",
+			Name:      "cache_miss",
+			Help:      "In-memory Cache Misses",
+		}, []string{}),
+		S3Hits: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
+			Namespace: "lfsproxy",
+			Name:      "s3_hit",
+			Help:      "S3 Cache Hits",
+		}, []string{}),
+		S3Miss: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
+			Namespace: "lfsproxy",
+			Name:      "s3_miss",
+			Help:      "S3 Cache Misses",
+		}, []string{}),
+	}
+}
+
+func PrometheusHandler() gin.HandlerFunc {
+	h := promhttp.Handler()
+
+	return func(c *gin.Context) {
+		h.ServeHTTP(c.Writer, c.Request)
+	}
+}

go.mod

@@ -0,0 +1,64 @@
+module github.com/vela-games/lfsproxy
+
+go 1.20
+
+require (
+	github.com/allegro/bigcache/v3 v3.1.0
+	github.com/aws/aws-sdk-go v1.44.236
+	github.com/gin-contrib/cors v1.4.0
+	github.com/gin-contrib/gzip v0.0.6
+	github.com/gin-gonic/gin v1.9.1
+	github.com/go-kit/kit v0.12.0
+	github.com/jarcoal/httpmock v1.3.1
+	github.com/kelseyhightower/envconfig v1.4.0
+	github.com/prometheus/client_golang v1.14.0
+	github.com/spf13/viper v1.15.0
+	github.com/stretchr/testify v1.8.3
+)
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bytedance/sonic v1.9.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.14.0 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
+	github.com/leodido/go-urn v1.2.4 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/spf13/afero v1.9.3 // indirect
+	github.com/spf13/cast v1.5.0 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/subosito/gotenv v1.4.2 // indirect
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/ugorji/go/codec v1.2.11 // indirect
+	golang.org/x/arch v0.3.0 // indirect
+	golang.org/x/crypto v0.9.0 // indirect
+	golang.org/x/net v0.10.0 // indirect
+	golang.org/x/sys v0.8.0 // indirect
+	golang.org/x/text v0.9.0 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)