Promote Dev

.gitignore

@@ -30,3 +30,9 @@ go.work.sum
 
 # generated docs
 site
+
+# tokenizer lib
+lib
+
+# local configuration files
+.envrc

.golangci.yml

@@ -1,14 +1,15 @@
 run:
   timeout: 5m
   allow-parallel-runners: true
-
+    
 # Settings related to issues
 issues:
   # Report issues on new code only (since we're brining in from upstream)
   new: true
   # Which dirs to exclude: issues from them won't be reported
   exclude-dirs:
     - bin
+
 linters:
   disable-all: true
   enable:
@@ -18,7 +19,7 @@ linters:
     - fatcontext
     - ginkgolinter
     - gocritic
-    - govet
+    # - govet        # do not enable - this causes some metalinter issue
     - loggercheck
     - misspell
     - perfsprint
@@ -27,17 +28,13 @@ linters:
     - makezero
     - errcheck
     - goconst
-    - gofmt
-    - goimports
-    - gosimple
     - ineffassign
     - nakedret
     - prealloc
-    - typecheck
     - unparam
     - unused
-    
+
 linters-settings:
   revive:
     rules:
-      - name: comment-spacings
+      - name: comment-spacings

.tekton/buildah-build.yaml

@@ -44,6 +44,15 @@ spec:
         USERNAME=$(jq -r '.auths["quay.io"].username' /root/.docker/config.json)
         PASSWORD=$(jq -r '.auths["quay.io"].password' /root/.docker/config.json)
 
+        echo "🔐 Extracting Git credentials from workspace..."
+        GIT_USER=$(cat /workspace/git-auth/username)
+        GIT_TOKEN=$(cat /workspace/git-auth/token)
+
+        if [ -z "$GIT_USER" ] || [ -z "$GIT_TOKEN" ]; then
+          echo "❌ Error: Missing git-auth credentials"
+          exit 1
+        fi
+
         if [ "$USERNAME" = "null" ] || [ "$PASSWORD" = "null" ]; then
           echo "❌ Error: Missing registry credentials"
           exit 1
@@ -56,8 +65,10 @@ spec:
         export DOCKER_CONFIG=/root/.docker
         export BUILDER=buildah
         export IMG=$(params.image_tag_base):$(params.dev-version)
-
+        export GIT_NM_USER=$GIT_USER 
+        export NM_TOKEN=$GIT_TOKEN
+
         echo "🚀 Calling make buildah-build with IMG=$IMG..."
-        make buildah-build IMG=$IMG
+        make buildah-build IMG=$IMG 
 
         echo "$IMG" > /tekton/results/image-url

.tekton/go-build-task.yaml

@@ -12,5 +12,24 @@ spec:
       script: |
         #!/bin/bash
         cd $(workspaces.source.path)
+
+        echo "🔐 Extracting Git credentials from workspace..."
+        GIT_USER=$(cat /workspace/git-auth/username)
+        GIT_TOKEN=$(cat /workspace/git-auth/token)
+
+        if [ -z "$GIT_USER" ] || [ -z "$GIT_TOKEN" ]; then
+          echo "❌ Error: Missing git-auth credentials"
+          exit 1
+        fi
+
+        echo "🔐 Configuring Git..."
+        git config --global user.email "[email protected]"
+        git config --global user.name "ci-tag-bot"
+        git config --global url."https://${GIT_USER}:${GIT_TOKEN}@github.com".insteadOf "https://github.com"
+        git config --global --add safe.directory "$(pwd)"
+
+        # required for go build with tokenizer lib linking
+        dnf install -y gcc-c++ libstdc++ libstdc++-devel && dnf clean all
+
         go env -w GOFLAGS=-buildvcs=false
         make build

.tekton/go-lint-task.yaml

@@ -11,6 +11,7 @@ spec:
   steps:
     - name: run-lint
       image: us.icr.io/ibm-hc4ai-operator/golangci-lint:v1.64.8
+      # image: us.icr.io/ibm-hc4ai-operator/golangci-lint:v2.0.3
       imagePullPolicy: IfNotPresent
       script: |
         #!/bin/bash

.tekton/pipelinerun.yaml

@@ -165,6 +165,9 @@ spec:
         workspaces:
           - name: source
             workspace: source
+          - name: git-auth
+            workspace: git-auth
+
 
       - name: extract-version-and-registry
         params:
@@ -328,6 +331,8 @@ spec:
             workspace: registry-secret
           - name: container-storage
             workspace: container-storage
+          - name: git-auth
+            workspace: git-auth
 
       - name: vulnerability-scan
         when:

Dockerfile

@@ -3,28 +3,47 @@ FROM quay.io/projectquay/golang:1.24 AS builder
 ARG TARGETOS
 ARG TARGETARCH
 
-# ENV GOPROXY=https://goproxy.io,direct
+# Install build tools
+RUN dnf install -y gcc-c++ libstdc++ libstdc++-devel && dnf clean all
 
 WORKDIR /workspace
+
+## NeuralMagic internal repos pull config
+ARG GIT_NM_USER
+ARG NM_TOKEN
+### use git token
+RUN echo -e "machine github.com\n\tlogin ${GIT_NM_USER}\n\tpassword ${NM_TOKEN}" >> ~/.netrc
+ENV GOPRIVATE=github.com/neuralmagic
+ENV GIT_TERMINAL_PROMPT=1
+
 # Copy the Go Modules manifests
 COPY go.mod go.mod
 COPY go.sum go.sum
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 RUN go mod download
+RUN rm -rf ~/.netrc # remove git token
 
 # Copy the go source
 COPY cmd ./cmd
 COPY pkg ./pkg
 COPY internal ./internal
 COPY api ./api
 
+# HuggingFace tokenizer bindings
+RUN mkdir -p lib
+RUN curl -L https://github.com/daulet/tokenizers/releases/download/v1.20.2/libtokenizers.${TARGETOS}-${TARGETARCH}.tar.gz | tar -xz -C lib
+RUN ranlib lib/*.a
+
 # Build
 # the GOARCH has not a default value to allow the binary be built according to the host where the command
 # was called. For example, if we call make image-build in a local env which has the Apple Silicon M1 SO
 # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore,
 # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
-RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -o bin/epp cmd/epp/main.go cmd/epp/health.go
+ENV CGO_ENABLED=1
+ENV GOOS=${TARGETOS:-linux}
+ENV GOARCH=${TARGETARCH}
+RUN go build -o bin/epp -ldflags="-extldflags '-L$(pwd)/lib'" cmd/epp/main.go cmd/epp/health.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details

Makefile

@@ -439,11 +439,20 @@ lint: check-golangci-lint ## Run lint
 	golangci-lint run
 
 ##@ Build
+LDFLAGS ?= -extldflags '-L$(shell pwd)/lib'
+CGO_ENABLED=1 # Enable CGO
+
+.PHONY: download-tokenizer
+download-tokenizer: ## Download the HuggingFace tokenizer bindings.
+	@echo "Downloading HuggingFace tokenizer bindings..."
+	mkdir -p lib
+	curl -L https://github.com/daulet/tokenizers/releases/download/v1.20.2/libtokenizers.$(TARGETOS)-$(TARGETARCH).tar.gz | tar -xz -C lib
+	ranlib lib/*.a
 
 .PHONY: build
-build: check-go ##
+build: check-go download-tokenizer ##
 	@printf "\033[33;1m==== Building ====\033[0m\n"
-	go build -o bin/epp cmd/epp/main.go cmd/epp/health.go
+	go build -ldflags="$(LDFLAGS)" -o bin/epp cmd/epp/main.go cmd/epp/health.go
 
 ##@ Container Build/Push
 
@@ -456,7 +465,12 @@ buildah-build: check-builder load-version-json ## Build and push image (multi-ar
 	  for arch in amd64; do \
 	    ARCH_TAG=$$FINAL_TAG-$$arch; \
 	    echo "📦 Building for architecture: $$arch"; \
-		buildah build --arch=$$arch --os=linux --layers -t $(IMG)-$$arch . || exit 1; \
+		buildah build \
+			--arch=$$arch \
+			--build-arg GIT_NM_USER=$(GIT_NM_USER) \
+            --build-arg NM_TOKEN=$(NM_TOKEN) \
+			--os=linux \
+			--layers -t $(IMG)-$$arch . || exit 1; \
 	    echo "🚀 Pushing image: $(IMG)-$$arch"; \
 	    buildah push $(IMG)-$$arch docker://$(IMG)-$$arch || exit 1; \
 	  done; \
@@ -474,7 +488,11 @@ buildah-build: check-builder load-version-json ## Build and push image (multi-ar
 	  sed -e '1 s/\(^FROM\)/FROM --platform=$${BUILDPLATFORM}/' Dockerfile > Dockerfile.cross; \
 	  - docker buildx create --use --name image-builder || true; \
 	  docker buildx use image-builder; \
-	  docker buildx build --push --platform=$(PLATFORMS) --tag $(IMG) -f Dockerfile.cross . || exit 1; \
+	  docker buildx build --push \
+	  			--platform=$(PLATFORMS) \
+	  			--build-arg GIT_NM_USER=$(GIT_NM_USER)\
+                --build-arg NM_TOKEN=$(NM_TOKEN) \
+                --tag $(IMG) -f Dockerfile.cross . || exit 1; \
 	  docker buildx rm image-builder || true; \
 	  rm Dockerfile.cross; \
 	elif [ "$(BUILDER)" = "podman" ]; then \
@@ -489,7 +507,13 @@ buildah-build: check-builder load-version-json ## Build and push image (multi-ar
 .PHONY:	image-build
 image-build: check-container-tool load-version-json ## Build container image using $(CONTAINER_TOOL)
 	@printf "\033[33;1m==== Building container image $(IMG) ====\033[0m\n"
-	$(CONTAINER_TOOL) build --build-arg TARGETOS=$(TARGETOS) --build-arg TARGETARCH=$(TARGETARCH) -t $(IMG) .
+	$(CONTAINER_TOOL) build --platform=$(TARGETOS)/$(TARGETARCH) \
+ 		--build-arg TARGETOS=$(TARGETOS) \
+		--build-arg TARGETARCH=$(TARGETARCH) \
+		--build-arg GIT_NM_USER=$(GIT_NM_USER)\
+        --build-arg NM_TOKEN=$(NM_TOKEN) \
+		--progress=plain \
+ 		-t $(IMG) .
 
 .PHONY: image-push
 image-push: check-container-tool load-version-json ## Push container image $(IMG) to registry

README.md

@@ -6,6 +6,73 @@
 
 This project offers tools for AI Inference, enabling developers to build [Inference Gateways].
 
+---
+## Temporary Fork Configuration
+
+To enable the KVCacheAwareScorer, the following environment variables must be configured:
+```
+export ENABLE_KVCACHE_AWARE_SCORER=true
+export KVCACHE_AWARE_SCORER_WEIGHT=1.0
+export KVCACHE_INDEXER_REDIS_ADDR=<redis-service>
+export HF_TOKEN=<HuggingFace Token that has access to the vLLM models>
+```
+
+To enable the PrefixAwareScorer, the following environment variables must be configured:
+```
+export ENABLE_PREFIX_AWARE_SCORER=true
+export PREFIX_AWARE_SCORER_WEIGHT=1.0
+```
+
+To enable the LoadAwareScorer, the following environment variables must be configured:
+```
+export ENABLE_LOAD_AWARE_SCORER=true
+export LOAD_AWARE_SCORER_WEIGHT=1.0
+```
+
+To enable the SessionAwareScorer, the following environment variables must be configured:
+```
+export ENABLE_SESSION_AWARE_SCORER=true
+export SESSION_AWARE_SCORER_WEIGHT=1.0
+```
+
+To enable Prefill/Decode (PD) processing, the following environment variable must be configured:
+```
+export PD_ENABLED=true
+```
+
+To define the prompt length threshold (requests with a prompt longer than the value defined here will be processed using the prefill-decode process), the following environment variable must be configured:
+```
+export PD_PROMPT_LEN_THRESHOLD=10
+```
+
+Prefill configuration:
+
+To enable and configure the kv cache scorer for prefill, the following environment variables must be configured:
+```
+export PREFILL_ENABLE_KVCACHE_AWARE_SCORER=true
+export PREFILL_KVCACHE_AWARE_SCORER_WEIGHT=1.0
+```
+
+To enable and configure the load aware scorer for prefill, the following environment variables must be configured:
+```
+export PREFILL_ENABLE_LOAD_AWARE_SCORER=true
+export PREFILL_LOAD_AWARE_SCORER_WEIGHT=1.0
+```
+
+Decode configuration:
+
+To enable and configure the kv cache scorer for decode, the following environment variables must be configured:
+```
+export DECODE_ENABLE_KVCACHE_AWARE_SCORER=true
+export DECODE_KVCACHE_AWARE_SCORER_WEIGHT=1.0
+```
+
+To enable and configure the load aware scorer for decode, the following environment variables must be configured:
+```
+export DECODE_ENABLE_LOAD_AWARE_SCORER=true
+export DECODE_LOAD_AWARE_SCORER_WEIGHT=1.0
+```
+---
 [Inference Gateways]:#concepts-and-definitions
 
 ## Concepts and Definitions
@@ -79,8 +146,8 @@ See our website at https://gateway-api-inference-extension.sigs.k8s.io/ for deta
 ## Roadmap
 
 As Inference Gateway builds towards a GA release. We will continue to expand our capabilities, namely:
-1. Prefix-cache aware load balancing with interfaces for remote caches 
-1. Recommended LoRA adapter pipeline for automated rollout 
+1. Prefix-cache aware load balancing with interfaces for remote caches
+1. Recommended LoRA adapter pipeline for automated rollout
 1. Fairness and priority between workloads within the same criticality band
 1. HPA support for autoscaling on aggregate metrics derived from the load balancer
 1. Support for large multi-modal inputs and outputs
@@ -104,4 +171,3 @@ Contributions are readily welcomed, follow the [dev guide](./docs/dev.md) to sta
 ### Code of conduct
 
 Participation in the Kubernetes community is governed by the [Kubernetes Code of Conduct](code-of-conduct.md).
-

deploy/components/vllm-p2p/vllm-deployment.yaml

@@ -31,13 +31,12 @@ spec:
             - "-c"
           args:
             - |
-              export LMCACHE_DISTRIBUTED_URL=$${${POD_IP}}:80 && \
+              export LMCACHE_DISTRIBUTED_URL=$${${POD_IP}} && \
               vllm serve ${MODEL_NAME} \
               --host 0.0.0.0 \
               --port 8000 \
-              --enable-chunked-prefill false \
               --max-model-len ${MAX_MODEL_LEN} \
-              --kv-transfer-config '{"kv_connector":"LMCacheConnector","kv_role":"kv_both"}'
+              --kv-transfer-config '{"kv_connector":"LMCacheConnectorV1","kv_role":"kv_both"}'
           ports:
             - name: http
               containerPort: 8000
@@ -78,6 +77,10 @@ spec:
                 secretKeyRef:
                   name: ${HF_SECRET_NAME}
                   key: ${HF_SECRET_KEY}
+            - name: VLLM_ENABLE_V1_MULTIPROCESSING
+              value: "1"
+            - name: VLLM_WORKER_MULTIPROC_METHOD
+              value: spawn
             - name: LMCACHE_LOOKUP_URL
               value: ${REDIS_HOST}:${REDIS_PORT}
             - name: LMCACHE_ENABLE_DEBUG

deploy/environments/dev/kubernetes-kgateway/patch-deployments.yaml

@@ -29,4 +29,12 @@ spec:
             valueFrom:
               secretKeyRef:
                 name: hf-token
-                key: ${HF_SECRET_KEY}
+                key: ${HF_SECRET_KEY}
+          - name: ENABLE_KVCACHE_AWARE_SCORER
+            value: "true"
+          - name: KVCACHE_AWARE_SCORER_WEIGHT
+            value: "2.0"
+          - name: ENABLE_LOAD_AWARE_SCORER
+            value: "true"
+          - name: LOAD_AWARE_SCORER_WEIGHT
+            value: "1.0"