diff --git a/benchmarks/wrappers/local/golang/storage.go b/benchmarks/wrappers/local/golang/storage.go new file mode 100644 index 00000000..67941acd --- /dev/null +++ b/benchmarks/wrappers/local/golang/storage.go @@ -0,0 +1,131 @@ +package golang + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/google/uuid" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type Storage struct { + client *minio.Client +} + +var instance *Storage + +func UniqueName(name string) string { + ext := filepath.Ext(name) + basename := name[:len(name)-len(ext)] + uuidStr := strings.Split(uuid.New().String(), "-")[0] + return fmt.Sprintf("%s.%s%s", basename, uuidStr, ext) +} + +func NewStorage() (*Storage, error) { + address, exists := os.LookupEnv("MINIO_ADDRESS") + if !exists { + return nil, nil + } + + accessKey := os.Getenv("MINIO_ACCESS_KEY") + secretKey := os.Getenv("MINIO_SECRET_KEY") + + parts := strings.Split(address, ":") + endpoint := parts[0] + var port int = 9000 // Default port + + if len(parts) > 1 { + var err error + port, err = strconv.Atoi(parts[1]) + if err != nil { + return nil, fmt.Errorf("invalid port in MINIO_ADDRESS: %v", err) + } + } + + client, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKey, secretKey, ""), + Secure: false, + Port: uint16(port), + }) + if err != nil { + return nil, err + } + + return &Storage{client: client}, nil +} + +func (s *Storage) Upload(bucket, file, filepath string) (string, error) { + keyName := UniqueName(file) + _, err := s.client.FPutObject(context.Background(), bucket, keyName, filepath, minio.PutObjectOptions{}) + if err != nil { + return "", err + } + return keyName, nil +} + +func (s *Storage) Download(bucket, file, filepath string) error { + return s.client.FGetObject(context.Background(), bucket, file, filepath, minio.GetObjectOptions{}) +} + +func (s *Storage) DownloadDirectory(bucket, prefix, path string) error { + ctx := context.Background() + objects := s.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: true, + }) + + for object := range objects { + if object.Err != nil { + return object.Err + } + + objectPath := filepath.Join(path, object.Key) + if err := os.MkdirAll(filepath.Dir(objectPath), 0755); err != nil { + return err + } + + if err := s.Download(bucket, object.Key, objectPath); err != nil { + return err + } + } + + return nil +} + +func (s *Storage) UploadStream(bucket, file string, data []byte) (string, error) { + keyName := UniqueName(file) + reader := bytes.NewReader(data) + _, err := s.client.PutObject(context.Background(), bucket, keyName, reader, int64(len(data)), minio.PutObjectOptions{}) + if err != nil { + return "", err + } + return keyName, nil +} + +func (s *Storage) DownloadStream(bucket, file string) ([]byte, error) { + obj, err := s.client.GetObject(context.Background(), bucket, file, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + defer obj.Close() + + return io.ReadAll(obj) +} + +func GetInstance() (*Storage, error) { + if instance == nil { + var err error + instance, err = NewStorage() + if err != nil { + return nil, err + } + } + return instance, nil +} \ No newline at end of file diff --git a/config/systems.json b/config/systems.json index 5a4077a2..7dc62f24 100644 --- a/config/systems.json +++ b/config/systems.json @@ -64,6 +64,25 @@ ], "packages": [] } + }, + "golang": { + "base_images": { + "x64": { + "1.22.0": "golang:1.22", + "1.23.0": "golang:1.23", + "1.24.0": "golang:1.24" + } + }, + "images": [ + "build" + ], + "username": "docker_user", + "deployment": { + "files": [ + "storage.go" + ], + "packages": [] + } } }, "architecture": ["x64"], @@ -121,6 +140,23 @@ "uuid": "3.4.0" } } + }, + "golang": { + "base_images": { + "x64": { + "1" : "amazon/aws-lambda-go:1.2024.10.04.19" + } + }, + "images": [ + "build" + ], + "deployment": { + "files": [ + "handler.go", + "storage.go" + ], + "packages": [] + } } }, "architecture": ["x64", "arm64"], @@ -181,6 +217,26 @@ "uuid": "3.4.0" } } + }, + "golang": { + "base_images": { + "x64": { + "1.22.0": "mcr.microsoft.com/devcontainers/go:1.4-1.22", + "1.23.0": "mcr.microsoft.com/devcontainers/go:1.4-1.23", + "1.24.0": "mcr.microsoft.com/devcontainers/go:1.4-1.24" + } + }, + "images": [ + "build" + ], + "username": "docker_user", + "deployment": { + "files": [ + "handler.go", + "storage.go" + ], + "packages": [] + } } }, "images": { @@ -250,6 +306,26 @@ "uuid": "3.4.0" } } + }, + "golang": { + "base_images": { + "x64": { + "1.22.0": "ubuntu:22.04", + "1.23.0": "ubuntu:22.04", + "1.24.0": "ubuntu:22.04" + } + }, + "images": [ + "build" + ], + "username": "docker_user", + "deployment": { + "files": [ + "handler.go", + "storage.go" + ], + "packages": [] + } } }, "images": { @@ -316,6 +392,26 @@ "minio": "7.0.16" } } + }, + "golang": { + "base_images": { + "x64": { + "1.21.0": "openwhisk/action-golang-v1.21", + "1.22.0": "openwhisk/action-golang-v1.22", + "1.23.0": "openwhisk/action-golang-v1.23" + } + }, + "images": [ + "build" + ], + "username": "docker_user", + "deployment": { + "files": [ + "handler.go", + "storage.go" + ], + "packages": [] + } } }, "architecture": ["x64"], diff --git a/dockerfiles/aws/golang/Dockerfile.build b/dockerfiles/aws/golang/Dockerfile.build new file mode 100644 index 00000000..404b0b43 --- /dev/null +++ b/dockerfiles/aws/golang/Dockerfile.build @@ -0,0 +1,46 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +# Define Go version from VERSION build argument +ARG VERSION +ENV GO_VERSION=${VERSION} +ARG TARGETARCH=amd64 + +####################################### +# Install required utilities +RUN yum install -y shadow-utils wget ca-certificates tar zip + +# Install gosu for running commands as another user +ENV GOSU_VERSION 1.14 +# https://github.com/tianon/gosu/releases/tag/1.14 +# key https://keys.openpgp.org/search?q=tianon%40debian.org +RUN curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-amd64" \ + && chmod +x /usr/local/bin/gosu + +# Download and install Go +RUN wget -q "https://go.dev/dl/go${GO_VERSION}.linux-${TARGETARCH}.tar.gz" -O go.tar.gz && \ + echo "Downloading Go ${GO_VERSION} for ${TARGETARCH}" && \ + # Remove any previous Go installation + rm -rf /usr/local/go && \ + tar -C /usr/local -xzf go.tar.gz && \ + rm go.tar.gz + +# Add Go to PATH permanently +ENV PATH="/usr/local/go/bin:${PATH}" + +# Verify installation +RUN go version + +# Set up SeBS environment +RUN mkdir -p /sebs/ +COPY dockerfiles/golang_installer.sh /sebs/installer.sh +COPY dockerfiles/entrypoint.sh /sebs/entrypoint.sh +RUN chmod +x /sebs/installer.sh /sebs/entrypoint.sh + +# useradd and groupmod is installed in /usr/sbin which is not in PATH +ENV PATH=/usr/sbin:${PATH} +ENV SCRIPT_FILE=/mnt/function/package.sh + +# Set the entry point and command +ENTRYPOINT ["/sebs/entrypoint.sh"] +CMD ["/bin/bash", "/sebs/installer.sh"] \ No newline at end of file diff --git a/dockerfiles/azure/golang/Dockerfile.build b/dockerfiles/azure/golang/Dockerfile.build new file mode 100644 index 00000000..b3810721 --- /dev/null +++ b/dockerfiles/azure/golang/Dockerfile.build @@ -0,0 +1,46 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +# Define Go version from VERSION build argument +ARG VERSION +ENV GO_VERSION=${VERSION} +ARG TARGETARCH=amd64 + +# Install dependencies +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + wget \ + ca-certificates \ + tar \ + zip \ + gosu \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Download and install Go +RUN wget -q "https://go.dev/dl/go${GO_VERSION}.linux-${TARGETARCH}.tar.gz" -O go.tar.gz && \ + echo "Downloading Go ${GO_VERSION} for ${TARGETARCH}" && \ + # Remove any previous Go installation + rm -rf /usr/local/go && \ + tar -C /usr/local -xzf go.tar.gz && \ + rm go.tar.gz + +# Add Go to PATH permanently +ENV PATH="/usr/local/go/bin:${PATH}" + +# Verify installation +RUN go version + +# Set up SeBS environment +RUN mkdir -p /sebs/ +COPY dockerfiles/golang_installer.sh /sebs/installer.sh +COPY dockerfiles/entrypoint.sh /sebs/entrypoint.sh +RUN chmod +x /sebs/installer.sh /sebs/entrypoint.sh + +# useradd and groupmod is installed in /usr/sbin which is not in PATH +ENV PATH=/usr/sbin:${PATH} +ENV SCRIPT_FILE=/mnt/function/package.sh + +# Set the entry point and command +ENTRYPOINT ["/sebs/entrypoint.sh"] +CMD ["/bin/bash", "/sebs/installer.sh"] \ No newline at end of file diff --git a/dockerfiles/gcp/golang/Dockerfile.build b/dockerfiles/gcp/golang/Dockerfile.build new file mode 100644 index 00000000..b3810721 --- /dev/null +++ b/dockerfiles/gcp/golang/Dockerfile.build @@ -0,0 +1,46 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +# Define Go version from VERSION build argument +ARG VERSION +ENV GO_VERSION=${VERSION} +ARG TARGETARCH=amd64 + +# Install dependencies +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + wget \ + ca-certificates \ + tar \ + zip \ + gosu \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Download and install Go +RUN wget -q "https://go.dev/dl/go${GO_VERSION}.linux-${TARGETARCH}.tar.gz" -O go.tar.gz && \ + echo "Downloading Go ${GO_VERSION} for ${TARGETARCH}" && \ + # Remove any previous Go installation + rm -rf /usr/local/go && \ + tar -C /usr/local -xzf go.tar.gz && \ + rm go.tar.gz + +# Add Go to PATH permanently +ENV PATH="/usr/local/go/bin:${PATH}" + +# Verify installation +RUN go version + +# Set up SeBS environment +RUN mkdir -p /sebs/ +COPY dockerfiles/golang_installer.sh /sebs/installer.sh +COPY dockerfiles/entrypoint.sh /sebs/entrypoint.sh +RUN chmod +x /sebs/installer.sh /sebs/entrypoint.sh + +# useradd and groupmod is installed in /usr/sbin which is not in PATH +ENV PATH=/usr/sbin:${PATH} +ENV SCRIPT_FILE=/mnt/function/package.sh + +# Set the entry point and command +ENTRYPOINT ["/sebs/entrypoint.sh"] +CMD ["/bin/bash", "/sebs/installer.sh"] \ No newline at end of file diff --git a/dockerfiles/golang_installer.sh b/dockerfiles/golang_installer.sh new file mode 100644 index 00000000..9e43086c --- /dev/null +++ b/dockerfiles/golang_installer.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +cd /mnt/function +go mod init github.com/spcl/serverless-benchmarks +go get github.com/minio/minio-go/v7 +go get github.com/google/uuid + +go mod tidy diff --git a/dockerfiles/local/golang/Dockerfile.build b/dockerfiles/local/golang/Dockerfile.build new file mode 100644 index 00000000..64200067 --- /dev/null +++ b/dockerfiles/local/golang/Dockerfile.build @@ -0,0 +1,45 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +# Define Go version as build argument with default +ARG VERSION +ENV GO_VERSION=${VERSION} +ARG TARGETARCH=amd64 + +# Install dependencies +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + wget \ + ca-certificates \ + tar \ + zip \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Download and install Go +RUN wget -q "https://go.dev/dl/go${GO_VERSION}.linux-${TARGETARCH}.tar.gz" -O go.tar.gz && \ + echo "Downloading Go ${GO_VERSION} for ${TARGETARCH}" && \ + # Remove any previous Go installation + rm -rf /usr/local/go && \ + tar -C /usr/local -xzf go.tar.gz && \ + rm go.tar.gz + +# Add Go to PATH permanently +ENV PATH="/usr/local/go/bin:${PATH}" + +# Verify installation +RUN go version + +# Set up SeBS environment +RUN mkdir -p /sebs/ +COPY dockerfiles/golang_installer.sh /sebs/installer.sh +COPY dockerfiles/entrypoint.sh /sebs/entrypoint.sh +RUN chmod +x /sebs/installer.sh /sebs/entrypoint.sh + +# Add /usr/sbin to PATH (for useradd and groupmod) +ENV PATH="/usr/sbin:${PATH}" +ENV SCRIPT_FILE="/mnt/function/package.sh" + +# Set the entry point and command +ENTRYPOINT ["/sebs/entrypoint.sh"] +CMD ["/bin/bash", "/sebs/installer.sh"] \ No newline at end of file diff --git a/dockerfiles/openwhisk/golang/Dockerfile.build b/dockerfiles/openwhisk/golang/Dockerfile.build new file mode 100644 index 00000000..a2a8de49 --- /dev/null +++ b/dockerfiles/openwhisk/golang/Dockerfile.build @@ -0,0 +1,46 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +# Define Go version from VERSION build argument +ARG VERSION +ENV GO_VERSION=${VERSION} +ARG TARGETARCH=amd64 + +# Install dependencies +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + wget \ + ca-certificates \ + tar \ + zip \ + gosu \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Download and install Go +RUN wget -q "https://go.dev/dl/go${GO_VERSION}.linux-${TARGETARCH}.tar.gz" -O go.tar.gz && \ + echo "Downloading Go ${GO_VERSION} for ${TARGETARCH}" && \ + # Remove any previous Go installation + rm -rf /usr/local/go && \ + tar -C /usr/local -xzf go.tar.gz && \ + rm go.tar.gz + +# Add Go to PATH permanently +ENV PATH="/usr/local/go/bin:${PATH}" + +# Verify installation +RUN go version + +# Set up SeBS environment +RUN mkdir -p /sebs/ +COPY dockerfiles/golang_installer.sh /sebs/installer.sh +COPY dockerfiles/entrypoint.sh /sebs/entrypoint.sh +RUN chmod +x /sebs/installer.sh /sebs/entrypoint.sh + +# Add /usr/sbin to PATH (for useradd and groupmod) +ENV PATH="/usr/sbin:${PATH}" +ENV SCRIPT_FILE="/mnt/function/package.sh" + +# Set the entry point and command +ENTRYPOINT ["/sebs/entrypoint.sh"] +CMD ["/bin/bash", "/sebs/installer.sh"] \ No newline at end of file diff --git a/dockerfiles/openwhisk/golang/Dockerfile.function b/dockerfiles/openwhisk/golang/Dockerfile.function new file mode 100644 index 00000000..e69de29b diff --git a/sebs/benchmark.py b/sebs/benchmark.py index 42adb4e7..37314614 100644 --- a/sebs/benchmark.py +++ b/sebs/benchmark.py @@ -252,8 +252,9 @@ def hash_directory(directory: str, deployment: str, language: str): FILES = { "python": ["*.py", "requirements.txt*"], "nodejs": ["*.js", "package.json"], + "golang": ["*.go"], } - WRAPPERS = {"python": "*.py", "nodejs": "*.js"} + WRAPPERS = {"python": "*.py", "nodejs": "*.js", "golang": "*.go"} NON_LANG_FILES = ["*.sh", "*.json"] selected_files = FILES[language] + NON_LANG_FILES for file_type in selected_files: @@ -316,6 +317,7 @@ def copy_code(self, output_dir): FILES = { "python": ["*.py", "requirements.txt*"], "nodejs": ["*.js", "package.json"], + "golang": ["*.go", "*.mod"], } path = os.path.join(self.benchmark_path, self.language_name) for file_type in FILES[self.language_name]: @@ -399,6 +401,9 @@ def add_deployment_package_nodejs(self, output_dir): with open(package_config, "w") as package_file: json.dump(package_json, package_file, indent=2) + def add_deployment_package_golang(self, output_dir): + pass + def add_deployment_package(self, output_dir): from sebs.faas.function import Language @@ -406,6 +411,8 @@ def add_deployment_package(self, output_dir): self.add_deployment_package_python(output_dir) elif self.language == Language.NODEJS: self.add_deployment_package_nodejs(output_dir) + elif self.language == Language.GOLANG: + self.add_deployment_package_golang(output_dir) else: raise NotImplementedError @@ -463,7 +470,7 @@ def install_dependencies(self, output_dir): } # run Docker container to install packages - PACKAGE_FILES = {"python": "requirements.txt", "nodejs": "package.json"} + PACKAGE_FILES = {"python": "requirements.txt", "nodejs": "package.json", "golang": "go.mod"} file = os.path.join(output_dir, PACKAGE_FILES[self.language_name]) if os.path.exists(file): try: diff --git a/sebs/config.py b/sebs/config.py index aa20a8e3..c3030ea0 100644 --- a/sebs/config.py +++ b/sebs/config.py @@ -46,9 +46,6 @@ def supported_language_versions( ) -> List[str]: languages = self._system_config.get(deployment_name, {}).get("languages", {}) base_images = languages.get(language_name, {}).get("base_images", {}) - - if deployment_name == "local": - return list(base_images.keys()) return list(base_images.get(architecture, {}).keys()) def supported_architecture(self, deployment_name: str) -> List[str]: diff --git a/sebs/faas/function.py b/sebs/faas/function.py index 0fab7bcf..f53683b0 100644 --- a/sebs/faas/function.py +++ b/sebs/faas/function.py @@ -263,6 +263,7 @@ def deserialize(cached_config: dict) -> "Trigger": class Language(Enum): PYTHON = "python" NODEJS = "nodejs" + GOLANG = "golang" # FIXME: 3.7+ python with future annotations @staticmethod diff --git a/sebs/local/local.py b/sebs/local/local.py index 32b9f9ff..a57bf4f0 100644 --- a/sebs/local/local.py +++ b/sebs/local/local.py @@ -124,6 +124,7 @@ def package_code( CONFIG_FILES = { "python": ["handler.py", "requirements.txt", ".python_packages"], "nodejs": ["handler.js", "package.json", "node_modules"], + "golang": ["handler.go", "go.mod", "go.sum"], } package_config = CONFIG_FILES[language_name] function_dir = os.path.join(directory, "function") diff --git a/tools/build_docker_images.py b/tools/build_docker_images.py index 5336fb48..9fee91c8 100755 --- a/tools/build_docker_images.py +++ b/tools/build_docker_images.py @@ -13,7 +13,7 @@ "--deployment", default=None, choices=["local", "aws", "azure", "gcp"], action="store" ) parser.add_argument("--type", default=None, choices=["build", "run", "manage"], action="store") -parser.add_argument("--language", default=None, choices=["python", "nodejs"], action="store") +parser.add_argument("--language", default=None, choices=["python", "nodejs", "golang"], action="store") parser.add_argument("--language-version", default=None, type=str, action="store") args = parser.parse_args() config = json.load(open(os.path.join(PROJECT_DIR, "config", "systems.json"), "r"))