diff --git a/.github/workflows/dockers-index-exportation-image.yaml b/.github/workflows/dockers-index-exportation-image.yaml new file mode 100644 index 0000000000..0a9c2d124d --- /dev/null +++ b/.github/workflows/dockers-index-exportation-image.yaml @@ -0,0 +1,80 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go + +name: 'Build docker image: index-exportation' +on: + push: + branches: + - main + - release/v*.* + - '!release/v*.*.*' + tags: + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* + pull_request: + paths: + - hack/docker/gen/main.go + - dockers/index/job/exportation/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-index-exportation-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/index/job/exportation/** + - pkg/index/job/exportation/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' + - Makefile + - Makefile.d/** + pull_request_target: + paths: + - hack/docker/gen/main.go + - dockers/index/job/exportation/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-index-exportation-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/index/job/exportation/** + - pkg/index/job/exportation/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' + - Makefile + - Makefile.d/** +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: index-exportation + secrets: inherit diff --git a/.github/workflows/dockers-index-importation-image.yaml b/.github/workflows/dockers-index-importation-image.yaml new file mode 100644 index 0000000000..473ab6fa28 --- /dev/null +++ b/.github/workflows/dockers-index-importation-image.yaml @@ -0,0 +1,80 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go + +name: 'Build docker image: index-importation' +on: + push: + branches: + - main + - release/v*.* + - '!release/v*.*.*' + tags: + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* + pull_request: + paths: + - hack/docker/gen/main.go + - dockers/index/job/importation/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-index-importation-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/index/job/importation/** + - pkg/index/job/importation/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' + - Makefile + - Makefile.d/** + pull_request_target: + paths: + - hack/docker/gen/main.go + - dockers/index/job/importation/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-index-importation-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/index/job/importation/** + - pkg/index/job/importation/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' + - Makefile + - Makefile.d/** +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: index-importation + secrets: inherit diff --git a/Makefile b/Makefile index de1572c19e..cfa14f91c2 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,8 @@ HELM_OPERATOR_IMAGE = $(NAME)-helm-operator INDEX_CORRECTION_IMAGE = $(NAME)-index-correction INDEX_CREATION_IMAGE = $(NAME)-index-creation INDEX_DELETION_IMAGE = $(NAME)-index-deletion +INDEX_EXPORTATION_IMAGE = $(NAME)-index-exportion +INDEX_IMPORTATION_IMAGE = $(NAME)-index-importation INDEX_OPERATOR_IMAGE = $(NAME)-index-operator INDEX_SAVE_IMAGE = $(NAME)-index-save LB_GATEWAY_IMAGE = $(NAME)-lb-gateway diff --git a/Makefile.d/build.mk b/Makefile.d/build.mk index 029ada10fe..562d849784 100644 --- a/Makefile.d/build.mk +++ b/Makefile.d/build.mk @@ -25,6 +25,8 @@ binary/build: \ cmd/index/job/correction/index-correction \ cmd/index/job/creation/index-creation \ cmd/index/job/deletion/index-deletion \ + cmd/index/job/exportation/index-exportation \ + cmd/index/job/importation/index-importation \ cmd/index/job/readreplica/rotate/readreplica-rotate \ cmd/index/job/save/index-save \ cmd/index/operator/index-operator \ @@ -85,6 +87,14 @@ cmd/index/job/deletion/index-deletion: $(eval CGO_ENABLED = 0) $(call go-build,index/job/deletion,,-static,,,$@) +cmd/index/job/exportation/index-exportation: + $(eval CGO_ENABLED = 0) + $(call go-build,index/job/exportation,,-static,,,$@) + +cmd/index/job/importation/index-importation: + $(eval CGO_ENABLED = 0) + $(call go-build,index/job/importation,,-static,,,$@) + cmd/index/job/save/index-save: $(eval CGO_ENABLED = 0) $(call go-build,index/job/save,,-static,,,$@) @@ -134,6 +144,8 @@ binary/build/zip: \ artifacts/vald-index-correction-$(GOOS)-$(GOARCH).zip \ artifacts/vald-index-creation-$(GOOS)-$(GOARCH).zip \ artifacts/vald-index-deletion-$(GOOS)-$(GOARCH).zip \ + artifacts/vald-index-exportation-$(GOOS)-$(GOARCH).zip \ + artifacts/vald-index-importation-$(GOOS)-$(GOARCH).zip \ artifacts/vald-index-operator-$(GOOS)-$(GOARCH).zip \ artifacts/vald-index-save-$(GOOS)-$(GOARCH).zip \ artifacts/vald-lb-gateway-$(GOOS)-$(GOARCH).zip \ @@ -197,6 +209,14 @@ artifacts/vald-index-deletion-$(GOOS)-$(GOARCH).zip: cmd/index/job/deletion/inde $(call mkdir, $(dir $@)) zip --junk-paths $@ $< +artifacts/vald-index-exporation-$(GOOS)-$(GOARCH).zip: cmd/index/job/exportation/index-exportation + $(call mkdir, $(dir $@)) + zip --junk-paths $@ $< + +artifacts/vald-index-importation-$(GOOS)-$(GOARCH).zip: cmd/index/job/importation/index-importation + $(call mkdir, $(dir $@)) + zip --junk-paths $@ $< + artifacts/vald-index-save-$(GOOS)-$(GOARCH).zip: cmd/index/job/save/index-save $(call mkdir, $(dir $@)) zip --junk-paths $@ $< diff --git a/Makefile.d/docker.mk b/Makefile.d/docker.mk index 793ef66e27..d964d6d1eb 100644 --- a/Makefile.d/docker.mk +++ b/Makefile.d/docker.mk @@ -37,6 +37,8 @@ docker/build: \ docker/build/index-correction \ docker/build/index-creation \ docker/build/index-deletion \ + docker/build/index-exportation \ + docker/build/index-importation \ docker/build/index-operator \ docker/build/index-save \ docker/build/loadtest \ @@ -65,6 +67,8 @@ docker/xpanes/build: docker/build/index-correction \ docker/build/index-creation \ docker/build/index-deletion \ + docker/build/index-exportation \ + docker/build/index-importation \ docker/build/index-operator \ docker/build/index-save \ docker/build/loadtest \ @@ -354,6 +358,28 @@ docker/build/index-deletion: IMAGE=$(INDEX_DELETION_IMAGE) \ docker/build/image +.PHONY: docker/name/index-exportation +docker/name/index-exporation: + @echo "$(ORG)/$(INDEX_EXPORTATION_IMAGE)" + +.PHONY: docker/build/index-exportation +## build index-exportation image +docker/build/index-exportation: + @make DOCKERFILE="$(ROOTDIR)/dockers/index/job/exportation/Dockerfile" \ + IMAGE=$(INDEX_EXPORTATION_IMAGE) \ + docker/build/image + +.PHONY: docker/name/index-importation +docker/name/index-importation: + @echo "$(ORG)/$(INDEX_IMPORTATION_IMAGE)" + +.PHONY: docker/build/index-importation +## build index-importation image +docker/build/index-importation: + @make DOCKERFILE="$(ROOTDIR)/dockers/index/job/importation/Dockerfile" \ + IMAGE=$(INDEX_IMPORTATION_IMAGE) \ + docker/build/image + .PHONY: docker/name/index-operator docker/name/index-operator: @echo "$(ORG)/$(INDEX_OPERATOR_IMAGE)" diff --git a/cmd/index/job/exportation/main.go b/cmd/index/job/exportation/main.go new file mode 100644 index 0000000000..c5553f225e --- /dev/null +++ b/cmd/index/job/exportation/main.go @@ -0,0 +1,26 @@ +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import "fmt" + +const ( + maxVersion = "v0.0.10" + minVersion = "v0.0.0" + name = "index exportation job" +) + +func main() { + fmt.Println("hello world") +} diff --git a/cmd/index/job/importation/main.go b/cmd/index/job/importation/main.go new file mode 100644 index 0000000000..ab8fb8146e --- /dev/null +++ b/cmd/index/job/importation/main.go @@ -0,0 +1,26 @@ +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import "fmt" + +const ( + maxVersion = "v0.0.10" + minVersion = "v0.0.0" + name = "index importation job" +) + +func main() { + fmt.Println("hello world") +} diff --git a/dockers/index/job/exportation/Dockerfile b/dockers/index/job/exportation/Dockerfile new file mode 100644 index 0000000000..e31d6c41d9 --- /dev/null +++ b/dockers/index/job/exportation/Dockerfile @@ -0,0 +1,87 @@ +# syntax = docker/dockerfile:latest +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go +ARG UPX_OPTIONS=-9 +# skipcq: DOK-DL3026,DOK-DL3007 +FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder +LABEL maintainer="vdaas.org vald team " +# skipcq: DOK-DL3002 +USER root:root +ARG TARGETARCH +ARG TARGETOS +ARG GO_VERSION +ARG RUST_VERSION +ENV APP_NAME=index-exportation +ENV DEBIAN_FRONTEND=noninteractive +ENV GO111MODULE=on +ENV GOPATH=/go +ENV GOROOT=/opt/go +ENV HOME=/root +ENV INITRD=No +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 +ENV ORG=vdaas +ENV PKG=index/job/exportation +ENV REPO=vald +ENV TZ=Etc/UTC +ENV USER=root +ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH} +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN --mount=type=bind,target=.,rw \ + --mount=type=tmpfs,target=/tmp \ + --mount=type=cache,target=/var/lib/apt,sharing=locked,id=${APP_NAME} \ + --mount=type=cache,target=/var/cache/apt,sharing=locked,id=${APP_NAME} \ + --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ + --mount=type=tmpfs,target="${GOPATH}/src" \ + set -ex \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ + && apt-get update -y \ + && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends --fix-missing \ + build-essential \ + ca-certificates \ + curl \ + tzdata \ + locales \ + git \ + && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ + && apt-get clean \ + && apt-get autoclean -y \ + && apt-get autoremove -y \ + && make GOPATH="${GOPATH}" GOROOT="${GOROOT}" GO_VERSION="${GO_VERSION}" go/install \ + && make GOPATH="${GOPATH}" GOROOT="${GOROOT}" GO_VERSION="${GO_VERSION}" go/download \ + && make GOARCH="${TARGETARCH}" GOOS="${TARGETOS}" REPO="${ORG}" NAME="${REPO}" cmd/${PKG}/${APP_NAME} \ + && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" +# skipcq: DOK-DL3026,DOK-DL3007 +FROM gcr.io/distroless/static:nonroot +LABEL maintainer="vdaas.org vald team " +COPY --from=builder /usr/bin/index-exportation /usr/bin/index-exportation +# skipcq: DOK-DL3002 +USER nonroot:nonroot +ENTRYPOINT ["/usr/bin/index-exportation"] \ No newline at end of file diff --git a/dockers/index/job/importation/Dockerfile b/dockers/index/job/importation/Dockerfile new file mode 100644 index 0000000000..c7eb529861 --- /dev/null +++ b/dockers/index/job/importation/Dockerfile @@ -0,0 +1,87 @@ +# syntax = docker/dockerfile:latest +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go +ARG UPX_OPTIONS=-9 +# skipcq: DOK-DL3026,DOK-DL3007 +FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder +LABEL maintainer="vdaas.org vald team " +# skipcq: DOK-DL3002 +USER root:root +ARG TARGETARCH +ARG TARGETOS +ARG GO_VERSION +ARG RUST_VERSION +ENV APP_NAME=index-importation +ENV DEBIAN_FRONTEND=noninteractive +ENV GO111MODULE=on +ENV GOPATH=/go +ENV GOROOT=/opt/go +ENV HOME=/root +ENV INITRD=No +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 +ENV ORG=vdaas +ENV PKG=index/job/importation +ENV REPO=vald +ENV TZ=Etc/UTC +ENV USER=root +ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH} +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN --mount=type=bind,target=.,rw \ + --mount=type=tmpfs,target=/tmp \ + --mount=type=cache,target=/var/lib/apt,sharing=locked,id=${APP_NAME} \ + --mount=type=cache,target=/var/cache/apt,sharing=locked,id=${APP_NAME} \ + --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ + --mount=type=tmpfs,target="${GOPATH}/src" \ + set -ex \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ + && apt-get update -y \ + && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends --fix-missing \ + build-essential \ + ca-certificates \ + curl \ + tzdata \ + locales \ + git \ + && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ + && apt-get clean \ + && apt-get autoclean -y \ + && apt-get autoremove -y \ + && make GOPATH="${GOPATH}" GOROOT="${GOROOT}" GO_VERSION="${GO_VERSION}" go/install \ + && make GOPATH="${GOPATH}" GOROOT="${GOROOT}" GO_VERSION="${GO_VERSION}" go/download \ + && make GOARCH="${TARGETARCH}" GOOS="${TARGETOS}" REPO="${ORG}" NAME="${REPO}" cmd/${PKG}/${APP_NAME} \ + && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" +# skipcq: DOK-DL3026,DOK-DL3007 +FROM gcr.io/distroless/static:nonroot +LABEL maintainer="vdaas.org vald team " +COPY --from=builder /usr/bin/index-importation /usr/bin/index-importation +# skipcq: DOK-DL3002 +USER nonroot:nonroot +ENTRYPOINT ["/usr/bin/index-importation"] \ No newline at end of file diff --git a/hack/actions/gen/main.go b/hack/actions/gen/main.go index 621807598d..b528c736aa 100644 --- a/hack/actions/gen/main.go +++ b/hack/actions/gen/main.go @@ -335,11 +335,13 @@ const ( managerIndex = "manager-index" - indexCorrection = "index-correction" - indexCreation = "index-creation" - indexDeletion = "index-deletion" - indexSave = "index-save" - indexOperator = "index-operator" + indexCorrection = "index-correction" + indexCreation = "index-creation" + indexDeletion = "index-deletion" + indexExportation = "index-exportation" + indexImportation = "index-importation" + indexSave = "index-save" + indexOperator = "index-operator" readreplicaRotate = "readreplica-rotate" @@ -573,6 +575,14 @@ func main() { AppName: "index-deletion", PackageDir: "index/job/deletion", }, + "vald-index-exportation": { + AppName: "index-exportation", + PackageDir: "index/job/exportation", + }, + "vald-index-importation": { + AppName: "index-importation", + PackageDir: "index/job/importation", + }, "vald-index-save": { AppName: "index-save", PackageDir: "index/job/save", diff --git a/hack/docker/gen/main.go b/hack/docker/gen/main.go index 2e3487a779..0a5f36e20b 100644 --- a/hack/docker/gen/main.go +++ b/hack/docker/gen/main.go @@ -617,6 +617,14 @@ func main() { AppName: "index-deletion", PackageDir: "index/job/deletion", }, + "vald-index-exportation": { + AppName: "index-exportation", + PackageDir: "index/job/exportation", + }, + "vald-index-importation": { + AppName: "index-importation", + PackageDir: "index/job/importation", + }, "vald-readreplica-rotate": { AppName: "readreplica-rotate", PackageDir: "index/job/readreplica/rotate",