feat: integrating NIM model UI (#3156)

Signed-off-by: Olga Lavtar <[email protected]>
opendatahub-io · Sep 20, 2024 · 9f1608a · 9f1608a
1 parent 83472f4
commit 9f1608a
Show file tree

Hide file tree

Showing 33 changed files with 1,297 additions and 57 deletions.
diff --git a/backend/src/routes/api/namespaces/const.ts b/backend/src/routes/api/namespaces/const.ts
@@ -11,4 +11,8 @@ export enum NamespaceApplicationCase {
    * Upgrade an existing DSG project to work with model kserve.
    */
   KSERVE_PROMOTION,
+  /**
+   * Nvidia NIMs run on KServe but have different requirements than regular models.
+   */
+  KSERVE_NIM_PROMOTION,
 }
diff --git a/backend/src/routes/api/namespaces/namespaceUtils.ts b/backend/src/routes/api/namespaces/namespaceUtils.ts
@@ -71,6 +71,7 @@ export const applyNamespaceChange = async (
     );
   }
 
+  let annotations = {};
   let labels = {};
   let checkPermissionsFn = null;
   switch (context) {
@@ -92,6 +93,13 @@ export const applyNamespaceChange = async (
         checkPermissionsFn = checkEditNamespacePermission;
       }
       break;
+    case NamespaceApplicationCase.KSERVE_NIM_PROMOTION:
+      {
+        annotations = { 'opendatahub.io/nim-support': 'true' };
+        labels = { 'modelmesh-enabled': 'false' };
+        checkPermissionsFn = checkEditNamespacePermission;
+      }
+      break;
     default:
       throw createCustomError('Unknown configuration', 'Cannot apply namespace change', 400);
   }
@@ -121,9 +129,17 @@ export const applyNamespaceChange = async (
   }
 
   return fastify.kube.coreV1Api
-    .patchNamespace(name, { metadata: { labels } }, undefined, dryRun, undefined, undefined, {
-      headers: { 'Content-type': PatchUtils.PATCH_FORMAT_JSON_MERGE_PATCH },
-    })
+    .patchNamespace(
+      name,
+      { metadata: { annotations, labels } },
+      undefined,
+      dryRun,
+      undefined,
+      undefined,
+      {
+        headers: { 'Content-type': PatchUtils.PATCH_FORMAT_JSON_MERGE_PATCH },
+      },
+    )
     .then(() => ({ applied: true }))
     .catch((e) => {
       fastify.log.error(

diff --git a/backend/src/routes/api/nim-serving/index.ts b/backend/src/routes/api/nim-serving/index.ts
@@ -0,0 +1,25 @@
+import { KubeFastifyInstance, OauthFastifyRequest } from '../../../types';
+import { createCustomError } from '../../../utils/requestUtils';
+import { logRequestDetails } from '../../../utils/fileUtils';
+
+const secretNames = ['nvidia-nim-access', 'nvidia-nim-image-pull'];
+
+export default async (fastify: KubeFastifyInstance): Promise<void> => {
+  fastify.get(
+    '/:secretName',
+    async (
+      request: OauthFastifyRequest<{
+        Params: { secretName: string };
+      }>,
+    ) => {
+      logRequestDetails(fastify, request);
+      const { secretName } = request.params;
+      if (!secretNames.includes(secretName)) {
+        throw createCustomError('Not found', 'Secret not found', 404);
+      }
+      const { coreV1Api, namespace } = fastify.kube;
+
+      return coreV1Api.readNamespacedSecret(secretName, namespace);
+    },
+  );
+};
diff --git a/backend/src/types.ts b/backend/src/types.ts
@@ -41,6 +41,7 @@ export type DashboardConfig = K8sResourceCommon & {
       disableModelRegistry: boolean;
       disableConnectionTypes: boolean;
       disableStorageClasses: boolean;
+      disableNIMModelServing: boolean;
     };
     groupsConfig?: {
       adminGroups: string;

diff --git a/backend/src/utils/constants.ts b/backend/src/utils/constants.ts
@@ -66,6 +66,7 @@ export const blankDashboardCR: DashboardConfig = {
       disableModelRegistry: true,
       disableConnectionTypes: true,
       disableStorageClasses: true,
+      disableNIMModelServing: true,
     },
     notebookController: {
       enabled: true,

diff --git a/docs/dashboard-config.md b/docs/dashboard-config.md
@@ -36,6 +36,7 @@ The following are a list of features that are supported, along with there defaul
 | disableModelRegistry         | true    | Disables Model Registry from the dashboard.                                                          |
 | disableConnectionTypes       | true    | Disables creating custom data connection types from the dashboard.                                   |
 | disableStorageClasses        | true    | Disables storage classes settings nav item from the dashboard.                                       |
+| disableNIMModelServing       | true    | Disables components of NIM Model UI from the dashboard.   
 
 ## Defaults
 
@@ -67,6 +68,7 @@ spec:
     disableDistributedWorkloads: false
     disableConnectionTypes: false
     disableStorageClasses: true
+    disableNIMModelServing: true
 ```
 
 ## Additional fields
@@ -161,6 +163,7 @@ spec:
     disableBiasMetrics: false
     disablePerformanceMetrics: false
     disablePipelineExperiments: false
+    disableNIMModelServing: true
   notebookController:
     enabled: true
     gpuSetting: autodetect

diff --git a/frontend/src/__mocks__/mockDashboardConfig.ts b/frontend/src/__mocks__/mockDashboardConfig.ts
@@ -29,6 +29,7 @@ type MockDashboardConfigType = {
   disableStorageClasses?: boolean;
   disableNotebookController?: boolean;
   notebookSizes?: NotebookSize[];
+  disableNIMModelServing?: boolean;
 };
 
 export const mockDashboardConfig = ({
@@ -58,6 +59,7 @@ export const mockDashboardConfig = ({
   disableConnectionTypes = true,
   disableStorageClasses = false,
   disableNotebookController = false,
+  disableNIMModelServing = true,
   notebookSizes = [
     {
       name: 'XSmall',
@@ -164,6 +166,7 @@ export const mockDashboardConfig = ({
       disableModelRegistry,
       disableConnectionTypes,
       disableStorageClasses,
+      disableNIMModelServing,
     },
     notebookController: {
       enabled: !disableNotebookController,

diff --git a/frontend/src/api/k8s/__tests__/inferenceServices.spec.ts b/frontend/src/api/k8s/__tests__/inferenceServices.spec.ts
@@ -1,9 +1,9 @@
 import {
-  K8sStatus,
   k8sCreateResource,
   k8sDeleteResource,
   k8sGetResource,
   k8sListResource,
+  K8sStatus,
   k8sUpdateResource,
 } from '@openshift/dynamic-plugin-sdk-utils';
 import { mockAcceleratorProfile } from '~/__mocks__/mockAcceleratorProfile';
@@ -186,6 +186,7 @@ describe('assembleInferenceService', () => {
       undefined,
       false,
       undefined,
+      undefined,
       acceleratorProfileState,
       selectedAcceleratorProfile,
     );
@@ -218,6 +219,7 @@ describe('assembleInferenceService', () => {
       undefined,
       true,
       undefined,
+      undefined,
       acceleratorProfileState,
       selectedAcceleratorProfile,
     );
@@ -251,6 +253,7 @@ describe('assembleInferenceService', () => {
       undefined,
       false,
       undefined,
+      undefined,
       acceleratorProfileState,
       selectedAcceleratorProfile,
     );
@@ -279,6 +282,7 @@ describe('assembleInferenceService', () => {
       undefined,
       true,
       undefined,
+      undefined,
       acceleratorProfileState,
       selectedAcceleratorProfile,
     );
@@ -321,6 +325,7 @@ describe('assembleInferenceService', () => {
       undefined,
       false,
       undefined,
+      undefined,
       acceleratorProfileState,
       selectedAcceleratorProfile,
     );
@@ -373,6 +378,7 @@ describe('assembleInferenceService', () => {
       undefined,
       true,
       undefined,
+      undefined,
       acceleratorProfileState,
       selectedAcceleratorProfile,
     );

diff --git a/frontend/src/api/k8s/inferenceServices.ts b/frontend/src/api/k8s/inferenceServices.ts
@@ -4,8 +4,8 @@ import {
   k8sDeleteResource,
   k8sGetResource,
   k8sListResource,
-  k8sUpdateResource,
   K8sStatus,
+  k8sUpdateResource,
 } from '@openshift/dynamic-plugin-sdk-utils';
 import { InferenceServiceModel } from '~/api/models';
 import { InferenceServiceKind, K8sAPIOptions, KnownLabels } from '~/k8sTypes';
@@ -24,6 +24,7 @@ export const assembleInferenceService = (
   editName?: string,
   isModelMesh?: boolean,
   inferenceService?: InferenceServiceKind,
+  isStorageNeeded?: boolean,
   initialAcceleratorProfile?: AcceleratorProfileState,
   selectedAcceleratorProfile?: AcceleratorProfileSelectFieldState,
 ): InferenceServiceKind => {
@@ -162,6 +163,11 @@ export const assembleInferenceService = (
     };
   }
 
+  // If storage is not needed, remove storage from the inference service
+  if (isStorageNeeded !== undefined && !isStorageNeeded) {
+    delete updateInferenceService.spec.predictor.model?.storage;
+  }
+
   return updateInferenceService;
 };
 
@@ -234,13 +240,15 @@ export const createInferenceService = (
   initialAcceleratorProfile?: AcceleratorProfileState,
   selectedAcceleratorProfile?: AcceleratorProfileSelectFieldState,
   dryRun = false,
+  isStorageNeeded?: boolean,
 ): Promise<InferenceServiceKind> => {
   const inferenceService = assembleInferenceService(
     data,
     secretKey,
     undefined,
     isModelMesh,
     undefined,
+    isStorageNeeded,
     initialAcceleratorProfile,
     selectedAcceleratorProfile,
   );
@@ -263,13 +271,15 @@ export const updateInferenceService = (
   initialAcceleratorProfile?: AcceleratorProfileState,
   selectedAcceleratorProfile?: AcceleratorProfileSelectFieldState,
   dryRun = false,
+  isStorageNeeded?: boolean,
 ): Promise<InferenceServiceKind> => {
   const inferenceService = assembleInferenceService(
     data,
     secretKey,
     existingData.metadata.name,
     isModelMesh,
     existingData,
+    isStorageNeeded,
     initialAcceleratorProfile,
     selectedAcceleratorProfile,
   );

diff --git a/frontend/src/api/k8s/pvcs.ts b/frontend/src/api/k8s/pvcs.ts
@@ -3,8 +3,8 @@ import {
   k8sCreateResource,
   k8sDeleteResource,
   k8sListResourceItems,
-  k8sUpdateResource,
   K8sStatus,
+  k8sUpdateResource,
 } from '@openshift/dynamic-plugin-sdk-utils';
 import { K8sAPIOptions, KnownLabels, PersistentVolumeClaimKind } from '~/k8sTypes';
 import { PVCModel } from '~/api/models';
@@ -17,6 +17,7 @@ export const assemblePvc = (
   data: CreatingStorageObject,
   namespace: string,
   editName?: string,
+  hideFromUI?: boolean,
 ): PersistentVolumeClaimKind => {
   const {
     nameDesc: { name: pvcName, description },
@@ -32,9 +33,11 @@ export const assemblePvc = (
     metadata: {
       name,
       namespace,
-      labels: {
-        [KnownLabels.DASHBOARD_RESOURCE]: 'true',
-      },
+      ...(hideFromUI !== true && {
+        labels: {
+          [KnownLabels.DASHBOARD_RESOURCE]: 'true',
+        },
+      }),
       annotations: {
         'openshift.io/display-name': pvcName.trim(),
         'openshift.io/description': description,
@@ -69,8 +72,9 @@ export const createPvc = (
   data: CreatingStorageObject,
   namespace: string,
   opts?: K8sAPIOptions,
+  hideFromUI?: boolean,
 ): Promise<PersistentVolumeClaimKind> => {
-  const pvc = assemblePvc(data, namespace);
+  const pvc = assemblePvc(data, namespace, undefined, hideFromUI);
 
   return k8sCreateResource<PersistentVolumeClaimKind>(
     applyK8sAPIOptions({ model: PVCModel, resource: pvc }, opts),

diff --git a/frontend/src/api/k8s/servingRuntimes.ts b/frontend/src/api/k8s/servingRuntimes.ts
@@ -13,7 +13,10 @@ import {
   ServingRuntimeAnnotations,
   ServingRuntimeKind,
 } from '~/k8sTypes';
-import { CreatingServingRuntimeObject } from '~/pages/modelServing/screens/types';
+import {
+  CreatingServingRuntimeObject,
+  SupportedModelFormatsInfo,
+} from '~/pages/modelServing/screens/types';
 import { ContainerResources } from '~/types';
 import { getModelServingRuntimeName } from '~/pages/modelServing/utils';
 import { getDisplayNameFromK8sResource, translateDisplayNameForK8s } from '~/concepts/k8s/utils';
@@ -33,7 +36,15 @@ export const assembleServingRuntime = (
   selectedAcceleratorProfile?: AcceleratorProfileSelectFieldState,
   isModelMesh?: boolean,
 ): ServingRuntimeKind => {
-  const { name: displayName, numReplicas, modelSize, externalRoute, tokenAuth } = data;
+  const {
+    name: displayName,
+    numReplicas,
+    modelSize,
+    externalRoute,
+    tokenAuth,
+    imageName,
+    supportedModelFormatsInfo,
+  } = data;
   const createName = isCustomServingRuntimesEnabled
     ? translateDisplayNameForK8s(displayName)
     : getModelServingRuntimeName(namespace);
@@ -123,7 +134,12 @@ export const assembleServingRuntime = (
         volumeMounts.push(getshmVolumeMount());
       }
 
-      const containerWithoutResources = _.omit(container, 'resources');
+      const updatedContainer = {
+        ...container,
+        ...(imageName && { image: imageName }),
+      };
+
+      const containerWithoutResources = _.omit(updatedContainer, 'resources');
 
       return {
         ...containerWithoutResources,
@@ -134,6 +150,17 @@ export const assembleServingRuntime = (
     },
   );
 
+  if (supportedModelFormatsInfo) {
+    const supportedModelFormatsObj: SupportedModelFormatsInfo = {
+      name: supportedModelFormatsInfo.name,
+      version: supportedModelFormatsInfo.version,
+      autoSelect: true,
+      priority: 1,
+    };
+
+    updatedServingRuntime.spec.supportedModelFormats = [supportedModelFormatsObj];
+  }
+
   if (isModelMesh) {
     updatedServingRuntime.spec.tolerations = tolerations;
   }

diff --git a/frontend/src/concepts/areas/const.ts b/frontend/src/concepts/areas/const.ts
@@ -29,6 +29,7 @@ export const allFeatureFlags: string[] = Object.keys({
   disableModelRegistry: false,
   disableConnectionTypes: false,
   disableStorageClasses: false,
+  disableNIMModelServing: true,
 } satisfies DashboardCommonConfig);
 
 export const SupportedAreasStateMap: SupportedAreasState = {
@@ -119,4 +120,7 @@ export const SupportedAreasStateMap: SupportedAreasState = {
     requiredComponents: [StackComponent.MODEL_REGISTRY],
     requiredCapabilities: [StackCapability.SERVICE_MESH, StackCapability.SERVICE_MESH_AUTHZ],
   },
+  [SupportedArea.NIM_MODEL]: {
+    featureFlags: ['disableNIMModelServing'],
+  },
 };
diff --git a/frontend/src/concepts/areas/types.ts b/frontend/src/concepts/areas/types.ts
@@ -55,6 +55,7 @@ export enum SupportedArea {
   BIAS_METRICS = 'bias-metrics',
   PERFORMANCE_METRICS = 'performance-metrics',
   TRUSTY_AI = 'trusty-ai',
+  NIM_MODEL = 'nim-model',
 
   /* Distributed Workloads areas */
   DISTRIBUTED_WORKLOADS = 'distributed-workloads',