sc24_k8s_tutorial/h2o-values.yaml at main · mahidhar/sc24_k8s_tutorial · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
h2ogpt:
  enabled: true
  stack:
    enabled: true
  storage:
    size: 15Gi
    class: rook-ceph-block
    useEphemeral: false

  # -- Example configs to use when not using Model Lock and External LLM
  overrideConfig:
    enforce_h2ogpt_api_key: False
    enforce_h2ogpt_ui_key: False
    base_model: h2oai/h2o-danube2-1.8b-chat
    use_safetensors: True
    prompt_type: llama2
    save_dir: /workspace/save/
    use_gpu_id: False
    score_model: None
    max_max_new_tokens: 512
    max_new_tokens: 256

  podSecurityContext:
    fsGroup: null
    runAsGroup: 0
    runAsNonRoot: false
    runAsUser: 0

  securityContext:
    allowPrivilegeEscalation: false
    capabilities:
      drop:
      - ALL
    runAsNonRoot: false
    seccompProfile:
      type: RuntimeDefault

  updateStrategy:
    type: Recreate

  service:
    type: ClusterIP

  resources:
    requests:
      cpu: 1
      memory: 4Gi
    limits:
      cpu: 4
      memory: 4Gi

  extraAffinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
        - matchExpressions:
          - key: nvidia.com/gpu.compute.major
            operator: Gt
            values:
            - "7"
vllm:
  enabled: true

  storage:
    size: 15Gi
    class: rook-ceph-block
    useEphemeral: false

  podSecurityContext:
    fsGroup: null
    runAsGroup: 0
    runAsNonRoot: false
    runAsUser: 0

  securityContext:
    allowPrivilegeEscalation: false
    capabilities:
      drop:
      - ALL
    runAsNonRoot: false
    seccompProfile: null

  extraAffinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
        - matchExpressions:
          - key: nvidia.com/gpu.compute.major
            operator: Gt
            values:
            - "7"

  containerArgs:
    - "--model"
    - h2oai/h2o-danube2-1.8b-chat
    - "--tensor-parallel-size"
    - 1
    - "--seed"
    - 1234
    - "--trust-remote-code"

  updateStrategy:
    type: Recreate
  resources:
    requests:
      cpu: 2
      memory: 8Gi
      nvidia.com/gpu: 1
    limits:
      cpu: 4
      memory: 8Gi
      nvidia.com/gpu: 1