From 605ad894c32c2af660ce1ff35c18f1f70455d4b9 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Fri, 4 Jul 2025 22:15:31 +0100 Subject: [PATCH 01/13] add an experimental helm chart for running legacy IOCs in cluster --- dls-k8s-c7/.helmignore | 23 ++++++ dls-k8s-c7/Chart.yaml | 14 ++++ dls-k8s-c7/README.md | 3 + dls-k8s-c7/templates/_helpers.tpl | 62 ++++++++++++++++ dls-k8s-c7/templates/deployment.yaml | 79 ++++++++++++++++++++ dls-k8s-c7/values.yaml | 104 +++++++++++++++++++++++++++ 6 files changed, 285 insertions(+) create mode 100644 dls-k8s-c7/.helmignore create mode 100644 dls-k8s-c7/Chart.yaml create mode 100644 dls-k8s-c7/README.md create mode 100644 dls-k8s-c7/templates/_helpers.tpl create mode 100644 dls-k8s-c7/templates/deployment.yaml create mode 100644 dls-k8s-c7/values.yaml diff --git a/dls-k8s-c7/.helmignore b/dls-k8s-c7/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/dls-k8s-c7/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/dls-k8s-c7/Chart.yaml b/dls-k8s-c7/Chart.yaml new file mode 100644 index 0000000..7431e97 --- /dev/null +++ b/dls-k8s-c7/Chart.yaml @@ -0,0 +1,14 @@ +apiVersion: v2 +name: dls-k8s-c7 +description: A Helm chart legacy DLS IOCs + +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. It implies the +# the version tag of the container image. +appVersion: 2025.5.1 diff --git a/dls-k8s-c7/README.md b/dls-k8s-c7/README.md new file mode 100644 index 0000000..6b2e8ef --- /dev/null +++ b/dls-k8s-c7/README.md @@ -0,0 +1,3 @@ +A Helm Chart for deploying the dev-c7 container into a DLS Cluster. + +Intended for runnhing legacy IOCs in a RHEL8 or higher DLS worker node. \ No newline at end of file diff --git a/dls-k8s-c7/templates/_helpers.tpl b/dls-k8s-c7/templates/_helpers.tpl new file mode 100644 index 0000000..342aa5c --- /dev/null +++ b/dls-k8s-c7/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "dls-k8s-c7.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "dls-k8s-c7.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "dls-k8s-c7.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "dls-k8s-c7.labels" -}} +helm.sh/chart: {{ include "dls-k8s-c7.chart" . }} +{{ include "dls-k8s-c7.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "dls-k8s-c7.selectorLabels" -}} +app.kubernetes.io/name: {{ include "dls-k8s-c7.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "dls-k8s-c7.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "dls-k8s-c7.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/dls-k8s-c7/templates/deployment.yaml b/dls-k8s-c7/templates/deployment.yaml new file mode 100644 index 0000000..775ad36 --- /dev/null +++ b/dls-k8s-c7/templates/deployment.yaml @@ -0,0 +1,79 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "dls-k8s-c7.fullname" . }} + labels: + {{- include "dls-k8s-c7.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "dls-k8s-c7.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "dls-k8s-c7.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + hostNetwork: {{ .Values.hostNetwork }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: {{ .Values.userId }} + runAsGroup: {{ .Values.groupId | default .Values.userId }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + command: + - bash + args: + - "-c" + - "sleep infinity" + workingDir: /dls_sw/prod/R3.14.12.7/ioc/BL16I/BL16I-MO-IOC-12/21-2 + env: + - name: HOME + value: /home/{{ .Values.userName }} + - name: USER + value: {{ .Values.userName }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/dls-k8s-c7/values.yaml b/dls-k8s-c7/values.yaml new file mode 100644 index 0000000..2e70e73 --- /dev/null +++ b/dls-k8s-c7/values.yaml @@ -0,0 +1,104 @@ +# Default values for dls-k8s-c7. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ +replicaCount: 1 + +# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ +image: + repository: ghcr.io/diamondlightsource/dev-c7 + # This sets the pull policy for images. + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +# these must be filled +userId: 1200288 +userName: hgv27681 + +# set to true for Channel access to be served on the locl subnet. +hostNetwork: false + +# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ +imagePullSecrets: [] +# This is to override the chart name. +nameOverride: "" +fullnameOverride: "" + +# This is for setting Kubernetes Annotations to a Pod. +# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +podAnnotations: {} +# This is for setting Kubernetes Labels to a Pod. +# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +podLabels: {} + +podSecurityContext: + {} + # fsGroup: 2000 + +resources: + {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +# volumes for the Deployment. +volumes: + - name: work + hostPath: + path: /dls_sw/work + - name: prod + hostPath: + path: /dls_sw/prod + - name: epics + hostPath: + path: /dls_sw/epics + - name: apps + hostPath: + path: /dls_sw/apps + - name: targetos + hostPath: + path: /dls_sw/targetOS + - name: etc + hostPath: + path: /dls_sw/etc + - name: home + hostPath: + path: /home/hgv27681 + +# volumeMounts for the Deployment. +volumeMounts: + - name: work + mountPath: /dls_sw/work + readOnly: true + - name: prod + mountPath: /dls_sw/prod + readOnly: true + - name: epics + mountPath: /dls_sw/epics + readOnly: true + - name: apps + mountPath: /dls_sw/apps + readOnly: true + - name: targetos + mountPath: /dls_sw/targetOS + readOnly: true + - name: etc + mountPath: /dls_sw/etc + readOnly: true + - name: home + mountPath: /home/hgv27681 + +nodeSelector: {} + +tolerations: [] + +affinity: {} From 85ec6196b9e1a5b72aed65305074d18034cb243f Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Fri, 4 Jul 2025 22:31:08 +0100 Subject: [PATCH 02/13] better names --- {dls-k8s-c7 => c7-helm-chart}/.helmignore | 0 {dls-k8s-c7 => c7-helm-chart}/Chart.yaml | 2 +- {dls-k8s-c7 => c7-helm-chart}/README.md | 0 {dls-k8s-c7 => c7-helm-chart}/templates/_helpers.tpl | 0 {dls-k8s-c7 => c7-helm-chart}/templates/deployment.yaml | 0 {dls-k8s-c7 => c7-helm-chart}/values.yaml | 0 6 files changed, 1 insertion(+), 1 deletion(-) rename {dls-k8s-c7 => c7-helm-chart}/.helmignore (100%) rename {dls-k8s-c7 => c7-helm-chart}/Chart.yaml (96%) rename {dls-k8s-c7 => c7-helm-chart}/README.md (100%) rename {dls-k8s-c7 => c7-helm-chart}/templates/_helpers.tpl (100%) rename {dls-k8s-c7 => c7-helm-chart}/templates/deployment.yaml (100%) rename {dls-k8s-c7 => c7-helm-chart}/values.yaml (100%) diff --git a/dls-k8s-c7/.helmignore b/c7-helm-chart/.helmignore similarity index 100% rename from dls-k8s-c7/.helmignore rename to c7-helm-chart/.helmignore diff --git a/dls-k8s-c7/Chart.yaml b/c7-helm-chart/Chart.yaml similarity index 96% rename from dls-k8s-c7/Chart.yaml rename to c7-helm-chart/Chart.yaml index 7431e97..2317ffe 100644 --- a/dls-k8s-c7/Chart.yaml +++ b/c7-helm-chart/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -name: dls-k8s-c7 +name: dls-c7 description: A Helm chart legacy DLS IOCs type: application diff --git a/dls-k8s-c7/README.md b/c7-helm-chart/README.md similarity index 100% rename from dls-k8s-c7/README.md rename to c7-helm-chart/README.md diff --git a/dls-k8s-c7/templates/_helpers.tpl b/c7-helm-chart/templates/_helpers.tpl similarity index 100% rename from dls-k8s-c7/templates/_helpers.tpl rename to c7-helm-chart/templates/_helpers.tpl diff --git a/dls-k8s-c7/templates/deployment.yaml b/c7-helm-chart/templates/deployment.yaml similarity index 100% rename from dls-k8s-c7/templates/deployment.yaml rename to c7-helm-chart/templates/deployment.yaml diff --git a/dls-k8s-c7/values.yaml b/c7-helm-chart/values.yaml similarity index 100% rename from dls-k8s-c7/values.yaml rename to c7-helm-chart/values.yaml From 93c14d266350cb541a0585a209ff13d0c0eeb376 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 4 Sep 2025 14:41:16 +0100 Subject: [PATCH 03/13] move helm chart to fastcs standard --- Charts/README.md | 70 +++++++++++++++++++ {c7-helm-chart => Charts/dev-c7}/.helmignore | 0 {c7-helm-chart => Charts/dev-c7}/Chart.yaml | 0 {c7-helm-chart => Charts/dev-c7}/README.md | 0 .../dev-c7}/templates/_helpers.tpl | 0 .../dev-c7}/templates/deployment.yaml | 0 {c7-helm-chart => Charts/dev-c7}/values.yaml | 0 7 files changed, 70 insertions(+) create mode 100644 Charts/README.md rename {c7-helm-chart => Charts/dev-c7}/.helmignore (100%) rename {c7-helm-chart => Charts/dev-c7}/Chart.yaml (100%) rename {c7-helm-chart => Charts/dev-c7}/README.md (100%) rename {c7-helm-chart => Charts/dev-c7}/templates/_helpers.tpl (100%) rename {c7-helm-chart => Charts/dev-c7}/templates/deployment.yaml (100%) rename {c7-helm-chart => Charts/dev-c7}/values.yaml (100%) diff --git a/Charts/README.md b/Charts/README.md new file mode 100644 index 0000000..8d08d85 --- /dev/null +++ b/Charts/README.md @@ -0,0 +1,70 @@ +# Charts + +This directory contains Helm charts for deploying FastCS services. + +## Github Actions + +This folder works in tandem with [_helm.yml](../.github/workflows/_helm.yml) github actions workflow which: + +- Validates the stricter form of SemVer tag that helm requires. +- Finds all subfolders in the `Charts` directory and packages them as Helm charts. +- Publishes the packaged charts to ghcr.io/${{github.repository_owner }}/charts/CHART_NAME +- Uploads the contents of /schemas to artifacts (meaning they will be published to the release) + +This standalone Helm related workflow is independent of the rest of the workflows except that the _pypi workflow has _helm added to its `needs` in `ci.yml`, making sure we only publish to pypi with valid SemVer tags. + +## Schema Generation + +Schema generation for charts' `values.yaml` is handled by [helm-values-schema-json](https://github.com/losisin/helm-values-schema-json). Which is in turn controlled by annotations in the default [values.yaml](fastcs/values.yaml) file. + +The generated schema file will be called `values.schema.json` and will be placed in the same directory as the `values.yaml` file and commited to the repo. This is done automatically by a [pre-commit hook](https://github.com/DiamondLightSource/FastCS/blob/8232393b38cc8e0eee00680e95c2ce06e7983ba6/.pre-commit-config.yaml#L27-L33). Therefore, when developing charts you can update schemas with: + +```bash +git add . ; pre-commit +# or +pre-commit run --all-files +``` + + +Note that this standard name for the schema file means that it is packaged up with the helm chart and available for schema checks in ArgoCD for example. + +## `schemas` folder + +The schemas folder allows us to declare the schemas we want to publish to the release. + +It should contain: + +- A symlink to each of the `values.schema.json` files in the subfolders of `Charts`. The symlink should have a unique name, e.g. `fastcs-values.schema.json`, thus allowing multiple schemas to be published per repo. +- A service schema file which references the above and can be used to validate `values.yaml` in epics-containers services repos, where the these charts will be used as sub-charts. e.g. [fastcs-service.schema.json](../schemas/fastcs-service.schema.json) + +The service schema files are hand coded as they are extremely simple and unlikely to change. + +## Debuging/Development In-Cluster + +The `fastcs` helm chart has two variables to enable debugging/development in-cluster: + +- `editable`: When true: + - a PVC is created. + - The debug version of the container image is referenced. + - The contents of /workspaces and /venv are copied into the PVC. + - The venv from the debug image is an editable install of the project source code in /workspaces. + - The PVC folders are mounted over the corresponding folders in the container. +- `autostart`: + - When false, the container starts with PID 1 as sleep infinity. + - When true, the container starts with its normal entrypoint. + +In combination these flags can be used to debug or develop in-cluster. + +These features will be accessed via `ec`. See https://github.com/epics-containers/edge-containers-cli/issues/207 + +This script will: + +- Inspect the values of `editable` and `autostart` in the `values.yaml` file of the specified IOC (TODO: at present it uses the p47-services source code to do so but this should be determined from the cluster in future). +- Port forward the debugpy port (5678) from the pod to localhost. +- If editable is true, it will mount the PVC locally using pv-mounter and open VSCode to the /workspaces/xxx folder. +- If autostart is false, it will exec into the container and launch debugpy to run the main program. +- If autostart is true, it will exec into the container and attach debugpy to PID 1. + +This then allows you to attach VSCode to debugpy in the cluster container and if 'editable' is true, edit the source code in VSCode and have the changes reflected. + +To attach to debugpy the following launch.json configuration is supplied in the [fastcs-example project](https://github.com/DiamondLightSource/fastcs-example/blob/77daed5f5a2bd01ab4c0e1d8c812e8754b254674/.vscode/launch.json#L7-L22). (this will also go in python-copier-template in future). diff --git a/c7-helm-chart/.helmignore b/Charts/dev-c7/.helmignore similarity index 100% rename from c7-helm-chart/.helmignore rename to Charts/dev-c7/.helmignore diff --git a/c7-helm-chart/Chart.yaml b/Charts/dev-c7/Chart.yaml similarity index 100% rename from c7-helm-chart/Chart.yaml rename to Charts/dev-c7/Chart.yaml diff --git a/c7-helm-chart/README.md b/Charts/dev-c7/README.md similarity index 100% rename from c7-helm-chart/README.md rename to Charts/dev-c7/README.md diff --git a/c7-helm-chart/templates/_helpers.tpl b/Charts/dev-c7/templates/_helpers.tpl similarity index 100% rename from c7-helm-chart/templates/_helpers.tpl rename to Charts/dev-c7/templates/_helpers.tpl diff --git a/c7-helm-chart/templates/deployment.yaml b/Charts/dev-c7/templates/deployment.yaml similarity index 100% rename from c7-helm-chart/templates/deployment.yaml rename to Charts/dev-c7/templates/deployment.yaml diff --git a/c7-helm-chart/values.yaml b/Charts/dev-c7/values.yaml similarity index 100% rename from c7-helm-chart/values.yaml rename to Charts/dev-c7/values.yaml From 1e502e0889410da40332927366d73a20d067f181 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 4 Sep 2025 14:45:41 +0100 Subject: [PATCH 04/13] apply python-copier-template --- .copier-answers.yml | 20 +++ .github/CONTRIBUTING.md | 27 +++ .github/ISSUE_TEMPLATE/bug_report.md | 21 +++ .github/ISSUE_TEMPLATE/issue.md | 13 ++ .../pull_request_template.md | 8 + .../actions/install_requirements/action.yml | 35 ++++ .github/dependabot.yml | 28 ++++ .github/pages/index.html | 2 +- .github/pages/make_switcher.py | 96 +++++++++++ .github/workflows/_container.yml | 64 ++++++++ .github/workflows/_dist.yml | 36 ++++ .github/workflows/_docs.yml | 55 +++++++ .github/workflows/_release.yml | 32 ++++ .github/workflows/_test.yml | 62 +++++++ .github/workflows/_tox.yml | 22 +++ .github/workflows/ci.yml | 57 +++++++ .github/workflows/periodic.yml | 13 ++ .gitignore | 16 +- .pre-commit-config.yaml | 19 ++- .vscode/settings.json | 18 +- Dockerfile | 31 ++++ README.md | 37 +++++ catalog-info.yaml | 10 ++ docs/_api.rst | 16 ++ docs/_templates/custom-module-template.rst | 37 +++++ docs/conf.py | 154 +++++++++++------- docs/explanations.md | 10 ++ docs/explanations/decisions.md | 12 ++ .../0001-record-architecture-decisions.md | 18 ++ ...0002-switched-to-python-copier-template.md | 28 ++++ docs/explanations/decisions/COPYME | 19 +++ docs/genindex.md | 3 + docs/how-to.md | 10 ++ docs/how-to/contribute.md | 2 + docs/how-to/run-container.md | 14 ++ docs/images/dls-logo.svg | 20 +-- docs/index.md | 56 +++++++ docs/reference.md | 12 ++ docs/tutorials.md | 10 ++ docs/tutorials/installation.md | 42 +++++ pyproject.toml | 117 +++++++++++++ src/dev_c7/__init__.py | 11 ++ src/dev_c7/__main__.py | 24 +++ tests/conftest.py | 0 tests/test_cli.py | 9 + 45 files changed, 1258 insertions(+), 88 deletions(-) create mode 100644 .copier-answers.yml create mode 100644 .github/CONTRIBUTING.md create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/issue.md create mode 100644 .github/PULL_REQUEST_TEMPLATE/pull_request_template.md create mode 100644 .github/actions/install_requirements/action.yml create mode 100644 .github/dependabot.yml create mode 100755 .github/pages/make_switcher.py create mode 100644 .github/workflows/_container.yml create mode 100644 .github/workflows/_dist.yml create mode 100644 .github/workflows/_docs.yml create mode 100644 .github/workflows/_release.yml create mode 100644 .github/workflows/_test.yml create mode 100644 .github/workflows/_tox.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/periodic.yml create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 catalog-info.yaml create mode 100644 docs/_api.rst create mode 100644 docs/_templates/custom-module-template.rst create mode 100644 docs/explanations.md create mode 100644 docs/explanations/decisions.md create mode 100644 docs/explanations/decisions/0001-record-architecture-decisions.md create mode 100644 docs/explanations/decisions/0002-switched-to-python-copier-template.md create mode 100644 docs/explanations/decisions/COPYME create mode 100644 docs/genindex.md create mode 100644 docs/how-to.md create mode 100644 docs/how-to/contribute.md create mode 100644 docs/how-to/run-container.md create mode 100644 docs/index.md create mode 100644 docs/reference.md create mode 100644 docs/tutorials.md create mode 100644 docs/tutorials/installation.md create mode 100644 pyproject.toml create mode 100644 src/dev_c7/__init__.py create mode 100644 src/dev_c7/__main__.py create mode 100644 tests/conftest.py create mode 100644 tests/test_cli.py diff --git a/.copier-answers.yml b/.copier-answers.yml new file mode 100644 index 0000000..f4eace6 --- /dev/null +++ b/.copier-answers.yml @@ -0,0 +1,20 @@ +# Changes here will be overwritten by Copier +_commit: 4.2.0 +_src_path: https://github.com/diamondlightsource/python-copier-template +author_email: giles.knap@diamond.ac.uk +author_name: Giles Knap +component_lifecycle: production +component_owner: group:default/controls +component_type: service +description: A Diamond Light Source specific developer container +distribution_name: dev-c7 +docker: true +docker_debug: false +docs_type: sphinx +git_platform: github.com +github_org: DiamondLightSource +package_name: dev_c7 +pypi: false +repo_name: dev-c7 +strict_typing: false +type_checker: pyright diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 0000000..b73e564 --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,27 @@ +# Contribute to the project + +Contributions and issues are most welcome! All issues and pull requests are +handled through [GitHub](https://github.com/DiamondLightSource/dev-c7/issues). Also, please check for any existing issues before +filing a new one. If you have a great idea but it involves big changes, please +file a ticket before making a pull request! We want to make sure you don't spend +your time coding something that might not fit the scope of the project. + +## Issue or Discussion? + +Github also offers [discussions](https://github.com/DiamondLightSource/dev-c7/discussions) as a place to ask questions and share ideas. If +your issue is open ended and it is not obvious when it can be "closed", please +raise it as a discussion instead. + +## Code Coverage + +While 100% code coverage does not make a library bug-free, it significantly +reduces the number of easily caught bugs! Please make sure coverage remains the +same or is improved by a pull request! + +## Developer Information + +It is recommended that developers use a [vscode devcontainer](https://code.visualstudio.com/docs/devcontainers/containers). This repository contains configuration to set up a containerized development environment that suits its own needs. + +This project was created using the [Diamond Light Source Copier Template](https://github.com/DiamondLightSource/python-copier-template) for Python projects. + +For more information on common tasks like setting up a developer environment, running the tests, and setting a pre-commit hook, see the template's [How-to guides](https://diamondlightsource.github.io/python-copier-template/4.2.0/how-to.html). diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..92087f5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,21 @@ +--- +name: Bug Report +about: The template to use for reporting bugs and usability issues +title: " " +labels: 'bug' +assignees: '' + +--- + +Describe the bug, including a clear and concise description of the expected behaviour, the actual behavior and the context in which you encountered it (ideally include details of your environment). + +## Steps To Reproduce +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + + +## Acceptance Criteria +- Specific criteria that will be used to judge if the issue is fixed diff --git a/.github/ISSUE_TEMPLATE/issue.md b/.github/ISSUE_TEMPLATE/issue.md new file mode 100644 index 0000000..52c84dd --- /dev/null +++ b/.github/ISSUE_TEMPLATE/issue.md @@ -0,0 +1,13 @@ +--- +name: Issue +about: The standard template to use for feature requests, design discussions and tasks +title: " " +labels: '' +assignees: '' + +--- + +A brief description of the issue, including specific stakeholders and the business case where appropriate + +## Acceptance Criteria +- Specific criteria that will be used to judge if the issue is fixed diff --git a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md new file mode 100644 index 0000000..8200afe --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md @@ -0,0 +1,8 @@ +Fixes #ISSUE + +### Instructions to reviewer on how to test: +1. Do thing x +2. Confirm thing y happens + +### Checks for reviewer +- [ ] Would the PR title make sense to a user on a set of release notes diff --git a/.github/actions/install_requirements/action.yml b/.github/actions/install_requirements/action.yml new file mode 100644 index 0000000..7a7bda0 --- /dev/null +++ b/.github/actions/install_requirements/action.yml @@ -0,0 +1,35 @@ +name: Install requirements +description: Install a version of python then call pip install and report what was installed +inputs: + python-version: + description: Python version to install, default is from Dockerfile + default: "dev" + pip-install: + description: Parameters to pass to pip install + default: "$([ -f dev-requirements.txt ] && echo '-c dev-requirements.txt') -e .[dev]" + +runs: + using: composite + steps: + - name: Get version of python + run: | + PYTHON_VERSION="${{ inputs.python-version }}" + if [ $PYTHON_VERSION == "dev" ]; then + # python version from Dockerfile, removing potential pinned sha + PYTHON_VERSION=$(sed -Ene "s/ARG PYTHON_VERSION=([0-9\.]+).*/\1/p" Dockerfile) + fi + echo "PYTHON_VERSION=$PYTHON_VERSION" >> "$GITHUB_ENV" + shell: bash + + - name: Setup python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install packages + run: pip install ${{ inputs.pip-install }} + shell: bash + + - name: Report what was installed + run: pip freeze + shell: bash diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..d2c2a0d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,28 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + groups: + actions: + patterns: + - "*" + commit-message: + prefix: "chore" + + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + groups: + dev-dependencies: + patterns: + - "*" + commit-message: + prefix: "chore" diff --git a/.github/pages/index.html b/.github/pages/index.html index 80f0a00..c495f39 100644 --- a/.github/pages/index.html +++ b/.github/pages/index.html @@ -8,4 +8,4 @@ - \ No newline at end of file + diff --git a/.github/pages/make_switcher.py b/.github/pages/make_switcher.py new file mode 100755 index 0000000..c06813a --- /dev/null +++ b/.github/pages/make_switcher.py @@ -0,0 +1,96 @@ +"""Make switcher.json to allow docs to switch between different versions.""" + +import json +import logging +from argparse import ArgumentParser +from pathlib import Path +from subprocess import CalledProcessError, check_output + + +def report_output(stdout: bytes, label: str) -> list[str]: + """Print and return something received frm stdout.""" + ret = stdout.decode().strip().split("\n") + print(f"{label}: {ret}") + return ret + + +def get_branch_contents(ref: str) -> list[str]: + """Get the list of directories in a branch.""" + stdout = check_output(["git", "ls-tree", "-d", "--name-only", ref]) + return report_output(stdout, "Branch contents") + + +def get_sorted_tags_list() -> list[str]: + """Get a list of sorted tags in descending order from the repository.""" + stdout = check_output(["git", "tag", "-l", "--sort=-v:refname"]) + return report_output(stdout, "Tags list") + + +def get_versions(ref: str, add: str | None) -> list[str]: + """Generate the file containing the list of all GitHub Pages builds.""" + # Get the directories (i.e. builds) from the GitHub Pages branch + try: + builds = set(get_branch_contents(ref)) + except CalledProcessError: + builds = set() + logging.warning(f"Cannot get {ref} contents") + + # Add and remove from the list of builds + if add: + builds.add(add) + + # Get a sorted list of tags + tags = get_sorted_tags_list() + + # Make the sorted versions list from main branches and tags + versions: list[str] = [] + for version in ["master", "main"] + tags: + if version in builds: + versions.append(version) + builds.remove(version) + + # Add in anything that is left to the bottom + versions += sorted(builds) + print(f"Sorted versions: {versions}") + return versions + + +def write_json(path: Path, repository: str, versions: list[str]): + """Write the JSON switcher to path.""" + org, repo_name = repository.split("/") + struct = [ + {"version": version, "url": f"https://{org}.github.io/{repo_name}/{version}/"} + for version in versions + ] + text = json.dumps(struct, indent=2) + print(f"JSON switcher:\n{text}") + path.write_text(text, encoding="utf-8") + + +def main(args=None): + """Parse args and write switcher.""" + parser = ArgumentParser( + description="Make a versions.json file from gh-pages directories" + ) + parser.add_argument( + "--add", + help="Add this directory to the list of existing directories", + ) + parser.add_argument( + "repository", + help="The GitHub org and repository name: ORG/REPO", + ) + parser.add_argument( + "output", + type=Path, + help="Path of write switcher.json to", + ) + args = parser.parse_args(args) + + # Write the versions file + versions = get_versions("origin/gh-pages", args.add) + write_json(args.output, args.repository, versions) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/_container.yml b/.github/workflows/_container.yml new file mode 100644 index 0000000..c6cd469 --- /dev/null +++ b/.github/workflows/_container.yml @@ -0,0 +1,64 @@ +on: + workflow_call: + inputs: + publish: + type: boolean + description: If true, pushes image to container registry + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # Need this to get version number from last tag + fetch-depth: 0 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Docker Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and export to Docker local cache + uses: docker/build-push-action@v6 + env: + DOCKER_BUILD_RECORD_UPLOAD: false + with: + context: . + # Need load and tags so we can test it below + load: true + tags: tag_for_testing + + - name: Test cli works in cached runtime image + run: docker run --rm tag_for_testing --version + + - name: Create tags for publishing image + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=ref,event=tag + type=raw,value=latest + + - name: Push cached image to container registry + if: inputs.publish && github.ref_type == 'tag' + uses: docker/build-push-action@v6 + env: + DOCKER_BUILD_RECORD_UPLOAD: false + # This does not build the image again, it will find the image in the + # Docker cache and publish it + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/_dist.yml b/.github/workflows/_dist.yml new file mode 100644 index 0000000..b1c4c93 --- /dev/null +++ b/.github/workflows/_dist.yml @@ -0,0 +1,36 @@ +on: + workflow_call: + +jobs: + build: + runs-on: "ubuntu-latest" + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # Need this to get version number from last tag + fetch-depth: 0 + + - name: Build sdist and wheel + run: > + export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && + pipx run build + + - name: Upload sdist and wheel as artifacts + uses: actions/upload-artifact@v4 + with: + name: dist + path: dist + + - name: Check for packaging errors + run: pipx run twine check --strict dist/* + + - name: Install produced wheel + uses: ./.github/actions/install_requirements + with: + pip-install: dist/*.whl + + - name: Test module --version works using the installed wheel + # If more than one module in src/ replace with module name to test + run: python -m $(ls --hide='*.egg-info' src | head -1) --version diff --git a/.github/workflows/_docs.yml b/.github/workflows/_docs.yml new file mode 100644 index 0000000..1f5491b --- /dev/null +++ b/.github/workflows/_docs.yml @@ -0,0 +1,55 @@ +on: + workflow_call: + + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Avoid git conflicts when tag and branch pushed at same time + if: github.ref_type == 'tag' + run: sleep 60 + + - name: Checkout + uses: actions/checkout@v4 + with: + # Need this to get version number from last tag + fetch-depth: 0 + + - name: Install system packages + run: sudo apt-get install graphviz + + - name: Install python packages + uses: ./.github/actions/install_requirements + + - name: Build docs + run: tox -e docs + + - name: Remove environment.pickle + run: rm build/html/.doctrees/environment.pickle + + - name: Upload built docs artifact + uses: actions/upload-artifact@v4 + with: + name: docs + path: build + + - name: Sanitize ref name for docs version + run: echo "DOCS_VERSION=${GITHUB_REF_NAME//[^A-Za-z0-9._-]/_}" >> $GITHUB_ENV + + - name: Move to versioned directory + run: mv build/html .github/pages/$DOCS_VERSION + + - name: Write switcher.json + run: python .github/pages/make_switcher.py --add $DOCS_VERSION ${{ github.repository }} .github/pages/switcher.json + + - name: Publish Docs to gh-pages + if: github.ref_type == 'tag' || github.ref_name == 'main' + # We pin to the SHA, not the tag, for security reasons. + # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions + uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: .github/pages + keep_files: true diff --git a/.github/workflows/_release.yml b/.github/workflows/_release.yml new file mode 100644 index 0000000..3f949fc --- /dev/null +++ b/.github/workflows/_release.yml @@ -0,0 +1,32 @@ +on: + workflow_call: + +jobs: + artifacts: + runs-on: ubuntu-latest + + steps: + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + merge-multiple: true + + - name: Zip up docs + run: | + set -vxeuo pipefail + if [ -d html ]; then + mv html $GITHUB_REF_NAME + zip -r docs.zip $GITHUB_REF_NAME + rm -rf $GITHUB_REF_NAME + fi + + - name: Create GitHub Release + # We pin to the SHA, not the tag, for security reasons. + # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions + uses: softprops/action-gh-release@da05d552573ad5aba039eaac05058a918a7bf631 # v2.2.2 + with: + prerelease: ${{ contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc') }} + files: "*" + generate_release_notes: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml new file mode 100644 index 0000000..552b29d --- /dev/null +++ b/.github/workflows/_test.yml @@ -0,0 +1,62 @@ +on: + workflow_call: + inputs: + python-version: + type: string + description: The version of python to install + required: true + runs-on: + type: string + description: The runner to run this job on + required: true + secrets: + CODECOV_TOKEN: + required: true + +env: + # https://github.com/pytest-dev/pytest/issues/2042 + PY_IGNORE_IMPORTMISMATCH: "1" + +jobs: + run: + runs-on: ${{ inputs.runs-on }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # Need this to get version number from last tag + fetch-depth: 0 + + - if: inputs.python-version == 'dev' + name: Install dev versions of python packages + uses: ./.github/actions/install_requirements + + - if: inputs.python-version == 'dev' + name: Write the requirements as an artifact + run: pip freeze --exclude-editable > /tmp/dev-requirements.txt + + - if: inputs.python-version == 'dev' + name: Upload dev-requirements.txt + uses: actions/upload-artifact@v4 + with: + name: dev-requirements + path: /tmp/dev-requirements.txt + + - if: inputs.python-version != 'dev' + name: Install latest versions of python packages + uses: ./.github/actions/install_requirements + with: + python-version: ${{ inputs.python-version }} + pip-install: ".[dev]" + + - name: Run tests + run: tox -e tests + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v5 + with: + name: ${{ inputs.python-version }}/${{ inputs.runs-on }} + files: cov.xml + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/_tox.yml b/.github/workflows/_tox.yml new file mode 100644 index 0000000..a13536d --- /dev/null +++ b/.github/workflows/_tox.yml @@ -0,0 +1,22 @@ +on: + workflow_call: + inputs: + tox: + type: string + description: What to run under tox + required: true + + +jobs: + run: + runs-on: "ubuntu-latest" + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install python packages + uses: ./.github/actions/install_requirements + + - name: Run tox + run: tox -e ${{ inputs.tox }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..cb59381 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,57 @@ +name: CI + +on: + push: + branches: + - main + tags: + - '*' + pull_request: + +jobs: + + lint: + uses: ./.github/workflows/_tox.yml + with: + tox: pre-commit,type-checking + + test: + strategy: + matrix: + runs-on: ["ubuntu-latest"] # can add windows-latest, macos-latest + python-version: ["3.11", "3.12", "3.13"] + include: + # Include one that runs in the dev environment + - runs-on: "ubuntu-latest" + python-version: "dev" + fail-fast: false + uses: ./.github/workflows/_test.yml + with: + runs-on: ${{ matrix.runs-on }} + python-version: ${{ matrix.python-version }} + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + container: + needs: test + if: always() + uses: ./.github/workflows/_container.yml + with: + publish: ${{ needs.test.result == 'success' }} + permissions: + contents: read + packages: write + + docs: + uses: ./.github/workflows/_docs.yml + + + dist: + uses: ./.github/workflows/_dist.yml + + release: + needs: [dist, test, docs] + if: github.ref_type == 'tag' + uses: ./.github/workflows/_release.yml + permissions: + contents: write diff --git a/.github/workflows/periodic.yml b/.github/workflows/periodic.yml new file mode 100644 index 0000000..e2a0fd1 --- /dev/null +++ b/.github/workflows/periodic.yml @@ -0,0 +1,13 @@ +name: Periodic + +on: + workflow_dispatch: + schedule: + # Run weekly to check URL links still resolve + - cron: "0 8 * * WED" + +jobs: + linkcheck: + uses: ./.github/workflows/_tox.yml + with: + tox: docs build -- -b linkcheck diff --git a/.gitignore b/.gitignore index e23a2b7..0f33bf2 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,6 @@ __pycache__/ # Distribution / packaging .Python env/ -.venv build/ develop-eggs/ dist/ @@ -56,12 +55,17 @@ cov.xml # Sphinx documentation docs/_build/ +docs/_api # PyBuilder target/ -# DLS build dir and virtual environment -/prefix/ -/venv/ -/lightweight-venv/ -/installed.files +# likely venv names +.venv* +venv* + +# further build artifacts +lockfiles/ + +# ruff cache +.ruff_cache/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a08d6b8..ef68f8a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,24 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.3.0 + rev: v5.0.0 hooks: - id: check-added-large-files - id: check-yaml - id: check-merge-conflict + - id: end-of-file-fixer + + - repo: local + hooks: + - id: ruff + name: lint with ruff + language: system + entry: ruff check --force-exclude --fix + types: [python] + require_serial: true + + - id: ruff-format + name: format with ruff + language: system + entry: ruff format --force-exclude + types: [python] + require_serial: true diff --git a/.vscode/settings.json b/.vscode/settings.json index e998cac..101c75f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,20 +1,12 @@ { - "python.linting.pylintEnabled": false, - "python.linting.flake8Enabled": true, - "python.linting.mypyEnabled": true, - "python.linting.enabled": true, - "python.testing.pytestArgs": [], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true, - "python.formatting.provider": "black", - "python.languageServer": "Pylance", "editor.formatOnSave": true, "editor.codeActionsOnSave": { "source.organizeImports": "explicit" }, - "workbench.colorCustomizations": { - "activityBar.background": "#521D07", - "titleBar.activeBackground": "#73290A", - "titleBar.activeForeground": "#FEFAF8" - } -} \ No newline at end of file + "files.insertFinalNewline": true, + "[python]": { + "editor.defaultFormatter": "charliermarsh.ruff", + }, +} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e77cbc8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +# The devcontainer should use the developer target and run as root with podman +# or docker with user namespaces. +ARG PYTHON_VERSION=3.11 +FROM python:${PYTHON_VERSION} AS developer + +# Add any system dependencies for the developer/build environment here +RUN apt-get update && apt-get install -y --no-install-recommends \ + graphviz \ + && rm -rf /var/lib/apt/lists/* + +# Set up a virtual environment and put it in PATH +RUN python -m venv /venv +ENV PATH=/venv/bin:$PATH + +# The build stage installs the context into the venv +FROM developer AS build +# Requires buildkit 0.17.0 +COPY --chmod=o+wrX . /workspaces/dev-c7 +WORKDIR /workspaces/dev-c7 +RUN touch dev-requirements.txt && pip install -c dev-requirements.txt . + + +# The runtime stage copies the built venv into a slim runtime container +FROM python:${PYTHON_VERSION}-slim AS runtime +# Add apt-get system dependecies for runtime here if needed +COPY --from=build /venv/ /venv/ +ENV PATH=/venv/bin:$PATH + +# change this entrypoint if it is not the same as the repo +ENTRYPOINT ["dev-c7"] +CMD ["--version"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..35a7f15 --- /dev/null +++ b/README.md @@ -0,0 +1,37 @@ +[![CI](https://github.com/DiamondLightSource/dev-c7/actions/workflows/ci.yml/badge.svg)](https://github.com/DiamondLightSource/dev-c7/actions/workflows/ci.yml) +[![Coverage](https://codecov.io/gh/DiamondLightSource/dev-c7/branch/main/graph/badge.svg)](https://codecov.io/gh/DiamondLightSource/dev-c7) + +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://www.apache.org/licenses/LICENSE-2.0) + +# dev_c7 + +A Diamond Light Source specific developer container + +This is where you should write a short paragraph that describes what your module does, +how it does it, and why people should use it. + +Source | +:---: | :---: +Docker | `docker run ghcr.io/diamondlightsource/dev-c7:latest` +Documentation | +Releases | + +This is where you should put some images or code snippets that illustrate +some relevant examples. If it is a library then you might put some +introductory code here: + +```python +from dev_c7 import __version__ + +print(f"Hello dev_c7 {__version__}") +``` + +Or if it is a commandline tool then you might put some example commands here: + +``` +python -m dev_c7 --version +``` + + + +See https://diamondlightsource.github.io/dev-c7 for more detailed documentation. diff --git a/catalog-info.yaml b/catalog-info.yaml new file mode 100644 index 0000000..5953c5b --- /dev/null +++ b/catalog-info.yaml @@ -0,0 +1,10 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: dev-c7 + title: dev-c7 + description: A Diamond Light Source specific developer container +spec: + type: service + lifecycle: production + owner: group:default/controls diff --git a/docs/_api.rst b/docs/_api.rst new file mode 100644 index 0000000..58cad48 --- /dev/null +++ b/docs/_api.rst @@ -0,0 +1,16 @@ +:orphan: + +.. + This page is not included in the TOC tree, but must exist so that the + autosummary pages are generated for dev_c7 and all its + subpackages + +API +=== + +.. autosummary:: + :toctree: _api + :template: custom-module-template.rst + :recursive: + + dev_c7 diff --git a/docs/_templates/custom-module-template.rst b/docs/_templates/custom-module-template.rst new file mode 100644 index 0000000..9aeca54 --- /dev/null +++ b/docs/_templates/custom-module-template.rst @@ -0,0 +1,37 @@ +{{ ('``' + fullname + '``') | underline }} + +{%- set filtered_members = [] %} +{%- for item in members %} + {%- if item in functions + classes + exceptions + attributes %} + {% set _ = filtered_members.append(item) %} + {%- endif %} +{%- endfor %} + +.. automodule:: {{ fullname }} + :members: + + {% block modules %} + {% if modules %} + .. rubric:: Submodules + + .. autosummary:: + :toctree: + :template: custom-module-template.rst + :recursive: + {% for item in modules %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block members %} + {% if filtered_members %} + .. rubric:: Members + + .. autosummary:: + :nosignatures: + {% for item in filtered_members %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} diff --git a/docs/conf.py b/docs/conf.py index f19c428..964feac 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,42 +1,40 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- General configuration ------------------------------------------------ +"""Configuration file for the Sphinx documentation builder. -import re +This file only contains a selection of the most common options. For a full +list see the documentation: +https://www.sphinx-doc.org/en/master/usage/configuration.html +""" -# General information about the project. +import sys +from pathlib import Path from subprocess import check_output +import requests + +import dev_c7 + +# -- General configuration ------------------------------------------------ + +# General information about the project. project = "dev-c7" # The full version, including alpha/beta/rc tags. -try: - # Something like 1.5.0-3-g3ab26eb - release_b = check_output( - ["git", "describe", "--abbrev=7", "--dirty", "--always", "--tags"] - ) - release = release_b.decode() -except BaseException: - # No git, or not under a git repo - release = "0.0" - -print("release is", release) +release = dev_c7.__version__ # The short X.Y version. -# untagged commits look like this 2.0.0-9-g51dab95 -if re.match(r".*-[0-9]+-[a-z0-9]{8}", release): - # Not on a tag - version = "main" +if "+" in release: + # Not on a tag, use branch name + root = Path(__file__).absolute().parent.parent + git_branch = check_output("git branch --show-current".split(), cwd=root) + version = git_branch.decode().strip() else: version = release extensions = [ # Use this for generating API docs "sphinx.ext.autodoc", + # and making summary tables at the top of API docs + "sphinx.ext.autosummary", # This can parse google style docstrings "sphinx.ext.napoleon", # For linking to external sphinx documentation @@ -45,8 +43,17 @@ "sphinx.ext.viewcode", # Adds the inheritance-diagram generation directive "sphinx.ext.inheritance_diagram", + # Add a copy button to each code block + "sphinx_copybutton", + # For the card element + "sphinx_design", + # So we can write markdown files + "myst_parser", ] +# So we can use the ::: syntax +myst_enable_extensions = ["colon_fence"] + # If true, Sphinx will warn about all references where the target cannot # be found. nitpicky = True @@ -63,7 +70,6 @@ ("py:class", "'bool'"), ("py:class", "'object'"), ("py:class", "'id'"), - ("py:class", "apischema.utils.UndefinedType"), ("py:class", "typing_extensions.Literal"), ] @@ -77,6 +83,12 @@ # Don't inherit docstrings from baseclasses autodoc_inherit_docstrings = False +# Document only what is in __all__ +autosummary_ignore_module_all = False + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + # Output graphviz directive produced images in a scalable format graphviz_output_format = "svg" @@ -84,9 +96,6 @@ # role, that is, for text marked up `like this` default_role = "any" -# The suffix of source filenames. -source_suffix = ".rst" - # The master toctree document. master_doc = "index" @@ -100,34 +109,76 @@ # This means you can link things like `str` and `asyncio` to the relevant # docs in the python documentation. -intersphinx_mapping = dict(python=("https://docs.python.org/3/", None)) +intersphinx_mapping = {"python": ("https://docs.python.org/3/", None)} # A dictionary of graphviz graph attributes for inheritance diagrams. -inheritance_graph_attrs = dict(rankdir="TB") +inheritance_graph_attrs = {"rankdir": "TB"} -# Common links that should be available on every page -rst_epilog = """ -.. _Diamond Light Source: - http://www.diamond.ac.uk -""" - -# Ignore localhost links for period check that links in docs are valid +# Ignore localhost links for periodic check that links in docs are valid linkcheck_ignore = [r"http://localhost:\d+/"] +# Set copy-button to ignore python and bash prompts +# https://sphinx-copybutton.readthedocs.io/en/latest/use.html#using-regexp-prompt-identifiers +copybutton_prompt_text = r">>> |\.\.\. |\$ |In \[\d*\]: | {2,5}\.\.\.: | {5,8}: " +copybutton_prompt_is_regexp = True + # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = "sphinx_rtd_theme_github_versions" - -# Options for the sphinx rtd theme, use DLS blue -html_theme_options = dict(style_nav_header_background="rgb(7, 43, 93)") +html_theme = "pydata_sphinx_theme" +github_repo = "dev-c7" +github_user = "DiamondLightSource" +switcher_json = f"https://{github_user}.github.io/{github_repo}/switcher.json" +switcher_exists = requests.get(switcher_json).ok +if not switcher_exists: + print( + "*** Can't read version switcher, is GitHub pages enabled? \n" + " Once Docs CI job has successfully run once, set the " + "Github pages source branch to be 'gh-pages' at:\n" + f" https://github.com/{github_user}/{github_repo}/settings/pages", + file=sys.stderr, + ) -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] +# Theme options for pydata_sphinx_theme +# We don't check switcher because there are 3 possible states for a repo: +# 1. New project, docs are not published so there is no switcher +# 2. Existing project with latest copier template, switcher exists and works +# 3. Existing project with old copier template that makes broken switcher, +# switcher exists but is broken +# Point 3 makes checking switcher difficult, because the updated copier template +# will fix the switcher at the end of the docs workflow, but never gets a chance +# to complete as the docs build warns and fails. +html_theme_options = { + "logo": { + "text": project, + }, + "use_edit_page_button": True, + "github_url": f"https://github.com/{github_user}/{github_repo}", + "icon_links": [ + { + "name": "PyPI", + "url": f"https://pypi.org/project/{project}", + "icon": "fas fa-cube", + } + ], + "switcher": { + "json_url": switcher_json, + "version_match": version, + }, + "check_switcher": False, + "navbar_end": ["theme-switcher", "icon-links", "version-switcher"], + "navigation_with_keys": False, +} + +# A dictionary of values to pass into the template engine’s context for all pages +html_context = { + "github_user": github_user, + "github_repo": github_repo, + "github_version": version, + "doc_path": "docs", +} # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. html_show_sphinx = False @@ -135,17 +186,6 @@ # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. html_show_copyright = False -# Add some CSS classes for columns and other tweaks in a custom css file -html_css_files = ["theme_overrides.css"] - # Logo html_logo = "images/dls-logo.svg" -html_favicon = "images/dls-favicon.ico" - -rst_epilog = """ - -.. |ProjectVersion| replace:: {version} - -""".format( - version=version.strip(), -) +html_favicon = html_logo diff --git a/docs/explanations.md b/docs/explanations.md new file mode 100644 index 0000000..73ab289 --- /dev/null +++ b/docs/explanations.md @@ -0,0 +1,10 @@ +# Explanations + +Explanations of how it works and why it works that way. + +```{toctree} +:maxdepth: 1 +:glob: + +explanations/* +``` diff --git a/docs/explanations/decisions.md b/docs/explanations/decisions.md new file mode 100644 index 0000000..0533b98 --- /dev/null +++ b/docs/explanations/decisions.md @@ -0,0 +1,12 @@ +# Architectural Decision Records + +Architectural decisions are made throughout a project's lifetime. As a way of keeping track of these decisions, we record these decisions in Architecture Decision Records (ADRs) listed below. + +```{toctree} +:glob: true +:maxdepth: 1 + +decisions/* +``` + +For more information on ADRs see this [blog by Michael Nygard](http://thinkrelevance.com/blog/2011/11/15/documenting-architecture-decisions). diff --git a/docs/explanations/decisions/0001-record-architecture-decisions.md b/docs/explanations/decisions/0001-record-architecture-decisions.md new file mode 100644 index 0000000..44d234e --- /dev/null +++ b/docs/explanations/decisions/0001-record-architecture-decisions.md @@ -0,0 +1,18 @@ +# 1. Record architecture decisions + +## Status + +Accepted + +## Context + +We need to record the architectural decisions made on this project. + +## Decision + +We will use Architecture Decision Records, as [described by Michael Nygard](http://thinkrelevance.com/blog/2011/11/15/documenting-architecture-decisions). + +## Consequences + +See Michael Nygard's article, linked above. To create new ADRs we will copy and +paste from existing ones. diff --git a/docs/explanations/decisions/0002-switched-to-python-copier-template.md b/docs/explanations/decisions/0002-switched-to-python-copier-template.md new file mode 100644 index 0000000..66fe5d8 --- /dev/null +++ b/docs/explanations/decisions/0002-switched-to-python-copier-template.md @@ -0,0 +1,28 @@ +# 2. Adopt python-copier-template for project structure + +## Status + +Accepted + +## Context + +We should use the following [python-copier-template](https://github.com/DiamondLightSource/python-copier-template). +The template will ensure consistency in developer +environments and package management. + +## Decision + +We have switched to using the template. + +## Consequences + +This module will use a fixed set of tools as developed in `python-copier-template` +and can pull from this template to update the packaging to the latest techniques. + +As such, the developer environment may have changed, the following could be +different: + +- linting +- formatting +- pip venv setup +- CI/CD diff --git a/docs/explanations/decisions/COPYME b/docs/explanations/decisions/COPYME new file mode 100644 index 0000000..b466c79 --- /dev/null +++ b/docs/explanations/decisions/COPYME @@ -0,0 +1,19 @@ +# 3. Short descriptive title + +Date: Today's date + +## Status + +Accepted + +## Context + +Background to allow us to make the decision, to show how we arrived at our conclusions. + +## Decision + +What decision we made. + +## Consequences + +What we will do as a result of this decision. diff --git a/docs/genindex.md b/docs/genindex.md new file mode 100644 index 0000000..73f1191 --- /dev/null +++ b/docs/genindex.md @@ -0,0 +1,3 @@ +# Index + + diff --git a/docs/how-to.md b/docs/how-to.md new file mode 100644 index 0000000..6b16141 --- /dev/null +++ b/docs/how-to.md @@ -0,0 +1,10 @@ +# How-to Guides + +Practical step-by-step guides for the more experienced user. + +```{toctree} +:maxdepth: 1 +:glob: + +how-to/* +``` diff --git a/docs/how-to/contribute.md b/docs/how-to/contribute.md new file mode 100644 index 0000000..6e41979 --- /dev/null +++ b/docs/how-to/contribute.md @@ -0,0 +1,2 @@ +```{include} ../../.github/CONTRIBUTING.md +``` diff --git a/docs/how-to/run-container.md b/docs/how-to/run-container.md new file mode 100644 index 0000000..59a1e09 --- /dev/null +++ b/docs/how-to/run-container.md @@ -0,0 +1,14 @@ +# Run in a container + +Pre-built containers with dev-c7 and its dependencies already +installed are available on [Github Container Registry](https://ghcr.io/DiamondLightSource/dev-c7). + +## Starting the container + +To pull the container from github container registry and run: + +``` +$ docker run ghcr.io/diamondlightsource/dev-c7:latest --version +``` + +To get a released version, use a numbered release instead of `latest`. diff --git a/docs/images/dls-logo.svg b/docs/images/dls-logo.svg index 0af1a17..4fcaa86 100644 --- a/docs/images/dls-logo.svg +++ b/docs/images/dls-logo.svg @@ -1,11 +1,11 @@ - - - - - - - - - - \ No newline at end of file + + + + + + + + + + diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..730b3fd --- /dev/null +++ b/docs/index.md @@ -0,0 +1,56 @@ +--- +html_theme.sidebar_secondary.remove: true +--- + +```{include} ../README.md +:end-before: + +::::{grid} 2 +:gutter: 4 + +:::{grid-item-card} {material-regular}`directions_walk;2em` +```{toctree} +:maxdepth: 2 +tutorials +``` ++++ +Tutorials for installation and typical usage. New users start here. +::: + +:::{grid-item-card} {material-regular}`directions;2em` +```{toctree} +:maxdepth: 2 +how-to +``` ++++ +Practical step-by-step guides for the more experienced user. +::: + +:::{grid-item-card} {material-regular}`info;2em` +```{toctree} +:maxdepth: 2 +explanations +``` ++++ +Explanations of how it works and why it works that way. +::: + +:::{grid-item-card} {material-regular}`menu_book;2em` +```{toctree} +:maxdepth: 2 +reference +``` ++++ +Technical reference material including APIs and release notes. +::: + +:::: diff --git a/docs/reference.md b/docs/reference.md new file mode 100644 index 0000000..dffd086 --- /dev/null +++ b/docs/reference.md @@ -0,0 +1,12 @@ +# Reference + +Technical reference material including APIs and release notes. + +```{toctree} +:maxdepth: 1 +:glob: + +API <_api/dev_c7> +genindex +Release Notes +``` diff --git a/docs/tutorials.md b/docs/tutorials.md new file mode 100644 index 0000000..1fe66c5 --- /dev/null +++ b/docs/tutorials.md @@ -0,0 +1,10 @@ +# Tutorials + +Tutorials for installation and typical usage. New users start here. + +```{toctree} +:maxdepth: 1 +:glob: + +tutorials/* +``` diff --git a/docs/tutorials/installation.md b/docs/tutorials/installation.md new file mode 100644 index 0000000..ed254a9 --- /dev/null +++ b/docs/tutorials/installation.md @@ -0,0 +1,42 @@ +# Installation + +## Check your version of python + +You will need python 3.11 or later. You can check your version of python by +typing into a terminal: + +``` +$ python3 --version +``` + +## Create a virtual environment + +It is recommended that you install into a “virtual environment” so this +installation will not interfere with any existing Python software: + +``` +$ python3 -m venv /path/to/venv +$ source /path/to/venv/bin/activate +``` + +## Installing the library + +You can now use `pip` to install the library and its dependencies: + +``` +$ python3 -m pip install dev-c7 +``` + +If you require a feature that is not currently released you can also install +from github: + +``` +$ python3 -m pip install git+https://github.com/DiamondLightSource/dev-c7.git +``` + +The library should now be installed and the commandline interface on your path. +You can check the version that has been installed by typing: + +``` +$ dev-c7 --version +``` diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..41a53de --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,117 @@ +[build-system] +requires = ["setuptools>=64", "setuptools_scm[toml]>=8"] +build-backend = "setuptools.build_meta" + +[project] +name = "dev-c7" +classifiers = [ + "Development Status :: 3 - Alpha", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", +] +description = "A Diamond Light Source specific developer container" +dependencies = [] # Add project dependencies here, e.g. ["click", "numpy"] +dynamic = ["version"] +license.file = "LICENSE" +readme = "README.md" +requires-python = ">=3.11" + +[project.optional-dependencies] +dev = [ + "copier", + "myst-parser", + "pipdeptree", + "pre-commit", + "pydata-sphinx-theme>=0.12", + "pyright", + "pytest", + "pytest-cov", + "ruff", + "sphinx-autobuild", + "sphinx-copybutton", + "sphinx-design", + "tox-direct", + "types-mock", +] + +[project.scripts] +dev-c7 = "dev_c7.__main__:main" + +[project.urls] +GitHub = "https://github.com/DiamondLightSource/dev-c7" + +[[project.authors]] # Further authors may be added by duplicating this section +email = "giles.knap@diamond.ac.uk" +name = "Giles Knap" + + +[tool.setuptools_scm] +version_file = "src/dev_c7/_version.py" + +[tool.pyright] +typeCheckingMode = "standard" +reportMissingImports = false # Ignore missing stubs in imported modules + +[tool.pytest.ini_options] +# Run pytest with all our checkers, and don't spam us with massive tracebacks on error +addopts = """ + --tb=native -vv --doctest-modules --doctest-glob="*.rst" + """ +# https://iscinumpy.gitlab.io/post/bound-version-constraints/#watch-for-warnings +filterwarnings = "error" +# Doctest python code in docs, python code in src docstrings, test functions in tests +testpaths = "docs src tests" + +[tool.coverage.run] +data_file = "/tmp/dev_c7.coverage" + +[tool.coverage.paths] +# Tests are run from installed location, map back to the src directory +source = ["src", "**/site-packages/"] + +# tox must currently be configured via an embedded ini string +# See: https://github.com/tox-dev/tox/issues/999 +[tool.tox] +legacy_tox_ini = """ +[tox] +skipsdist=True + +[testenv:{pre-commit,type-checking,tests,docs}] +# Don't create a virtualenv for the command, requires tox-direct plugin +direct = True +passenv = * +allowlist_externals = + pytest + pre-commit + pyright + sphinx-build + sphinx-autobuild +commands = + pre-commit: pre-commit run --all-files --show-diff-on-failure {posargs} + type-checking: pyright src tests {posargs} + tests: pytest --cov=dev_c7 --cov-report term --cov-report xml:cov.xml {posargs} + docs: sphinx-{posargs:build -EW --keep-going} -T docs build/html +""" + +[tool.ruff] +src = ["src", "tests"] +line-length = 88 +lint.select = [ + "B", # flake8-bugbear - https://docs.astral.sh/ruff/rules/#flake8-bugbear-b + "C4", # flake8-comprehensions - https://docs.astral.sh/ruff/rules/#flake8-comprehensions-c4 + "E", # pycodestyle errors - https://docs.astral.sh/ruff/rules/#error-e + "F", # pyflakes rules - https://docs.astral.sh/ruff/rules/#pyflakes-f + "N", # pep8-naming - https://docs.astral.sh/ruff/rules/#pep8-naming-n + "W", # pycodestyle warnings - https://docs.astral.sh/ruff/rules/#warning-w + "I", # isort - https://docs.astral.sh/ruff/rules/#isort-i + "UP", # pyupgrade - https://docs.astral.sh/ruff/rules/#pyupgrade-up + "SLF", # self - https://docs.astral.sh/ruff/settings/#lintflake8-self +] + +[tool.ruff.lint.per-file-ignores] +# By default, private member access is allowed in tests +# See https://github.com/DiamondLightSource/python-copier-template/issues/154 +# Remove this line to forbid private member access in tests +"tests/**/*" = ["SLF001"] diff --git a/src/dev_c7/__init__.py b/src/dev_c7/__init__.py new file mode 100644 index 0000000..a2ffbf3 --- /dev/null +++ b/src/dev_c7/__init__.py @@ -0,0 +1,11 @@ +"""Top level API. + +.. data:: __version__ + :type: str + + Version number as calculated by https://github.com/pypa/setuptools_scm +""" + +from ._version import __version__ + +__all__ = ["__version__"] diff --git a/src/dev_c7/__main__.py b/src/dev_c7/__main__.py new file mode 100644 index 0000000..0a0c083 --- /dev/null +++ b/src/dev_c7/__main__.py @@ -0,0 +1,24 @@ +"""Interface for ``python -m dev_c7``.""" + +from argparse import ArgumentParser +from collections.abc import Sequence + +from . import __version__ + +__all__ = ["main"] + + +def main(args: Sequence[str] | None = None) -> None: + """Argument parser for the CLI.""" + parser = ArgumentParser() + parser.add_argument( + "-v", + "--version", + action="version", + version=__version__, + ) + parser.parse_args(args) + + +if __name__ == "__main__": + main() diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..723c630 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,9 @@ +import subprocess +import sys + +from dev_c7 import __version__ + + +def test_cli_version(): + cmd = [sys.executable, "-m", "dev_c7", "--version"] + assert subprocess.check_output(cmd).decode().strip() == __version__ From 129eda49d32f14bc7b8c7494a918cfb64c5e7ec4 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 4 Sep 2025 15:21:54 +0100 Subject: [PATCH 05/13] fix docs build --- .devcontainer/devcontainer.json | 43 ------------ .github/workflows/_dist.yml | 36 ---------- .github/workflows/_test.yml | 62 ----------------- .github/workflows/ci.yml | 24 +------ .github/workflows/periodic.yml | 13 ---- .pre-commit-config.yaml | 16 ----- .vscode/extensions.json | 2 +- .vscode/launch.json | 2 +- .vscode/tasks.json | 2 +- CONTRIBUTING.rst | 1 - Charts/dev-c7/README.md | 2 +- Dockerfile | 110 ++++++++++++++++++++++--------- README.md | 32 +++++---- README.rst | 43 ------------ build-dev-c7.sh | 2 +- docker/Dockerfile | 79 ---------------------- docs/explanations.md | 5 +- docs/explanations.rst | 13 ---- docs/explanations/why_dev_c7.rst | 2 +- docs/how-to.rst | 15 ----- docs/how-to/Dockerfile | 2 +- docs/how-to/containers.rst | 2 +- docs/how-to/deriving.rst | 4 +- docs/how-to/podman.rst | 1 - docs/index.rst | 48 -------------- docs/make_docs.sh | 16 ----- docs/reference.rst | 12 ---- docs/reference/contributing.rst | 1 - docs/tutorials.md | 5 +- docs/tutorials.rst | 14 ---- docs/tutorials/continue.rst | 1 - docs/tutorials/installation.md | 42 ------------ docs/tutorials/prompt.rst | 1 - edm-fonts/install-fonts.sh | 2 +- requirements.txt | 6 -- src/README.md | 3 + docker/sudoers => sudoers | 0 tests/conftest.py | 0 tests/test_cli.py | 9 --- 39 files changed, 117 insertions(+), 556 deletions(-) delete mode 100644 .devcontainer/devcontainer.json delete mode 100644 .github/workflows/_dist.yml delete mode 100644 .github/workflows/_test.yml delete mode 100644 .github/workflows/periodic.yml delete mode 100644 README.rst delete mode 100644 docker/Dockerfile delete mode 100644 docs/explanations.rst delete mode 100644 docs/how-to.rst delete mode 100644 docs/index.rst delete mode 100755 docs/make_docs.sh delete mode 100644 docs/reference.rst delete mode 100644 docs/reference/contributing.rst delete mode 100644 docs/tutorials.rst delete mode 100644 docs/tutorials/installation.md delete mode 100644 requirements.txt create mode 100644 src/README.md rename docker/sudoers => sudoers (100%) delete mode 100644 tests/conftest.py delete mode 100644 tests/test_cli.py diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json deleted file mode 100644 index 1f51c03..0000000 --- a/.devcontainer/devcontainer.json +++ /dev/null @@ -1,43 +0,0 @@ -// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: -// https://github.com/microsoft/vscode-dev-containers/tree/v0.194.0/containers/ubuntu -{ - "name": "dev-c7", - "image": "ghcr.io/diamondlightsource/dev-c7:latest", - "remoteEnv": { - "DISPLAY": "${localEnv:DISPLAY}", - "HOME": "${localEnv:HOME}", - "SSH_AUTH_SOCK": "${localEnv:SSH_AUTH_SOCK}" - }, - // these two lines mount the local workspace folder so that the path is the same inside and outside the container. - "workspaceMount": "source=${localWorkspaceFolder},target=${localWorkspaceFolder},type=bind,consistency=cached", - "workspaceFolder": "${localWorkspaceFolder}", - // Set *default* container specific settings.json values on container create. - "settings": {}, - // Add the IDs of extensions you want installed when the container is created. - "extensions": [ - "ms-vscode.cpptools", - "ms-python.vscode-pylance", - "ryanluker.vscode-coverage-gutters", - "mhutchie.git-graph", - "eamodio.gitlens", - "redhat.vscode-yaml", - "nsd.vscode-epics" - ], - "mounts": [ - "source=/dls,target=/dls,type=bind,consistency=cached,bind-propagation=shared", - "source=/dls_sw,target=/dls_sw,type=bind,consistency=cached,bind-propagation=shared", - "source=/home,target=/home,type=bind,consistency=cached,bind-propagation=shared", - "source=/scratch/,target=/scratch/,type=bind,consistency=cached", - "source=/tmp,target=/tmp,type=bind,consistency=cached", - "source=${localEnv:SSH_AUTH_SOCK},target=${localEnv:SSH_AUTH_SOCK},type=bind,consistency=cached" - ], - "runArgs": [ - "-e HOME", - "--net=host", - "--security-opt=label=disable", - "--userns=keep-id", - "--annotation=run.oci.keep_original_groups=1", - "--storage-opt=ignore_chown_errors=true", - "--runtime=/usr/bin/crun" - ] -} \ No newline at end of file diff --git a/.github/workflows/_dist.yml b/.github/workflows/_dist.yml deleted file mode 100644 index b1c4c93..0000000 --- a/.github/workflows/_dist.yml +++ /dev/null @@ -1,36 +0,0 @@ -on: - workflow_call: - -jobs: - build: - runs-on: "ubuntu-latest" - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # Need this to get version number from last tag - fetch-depth: 0 - - - name: Build sdist and wheel - run: > - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && - pipx run build - - - name: Upload sdist and wheel as artifacts - uses: actions/upload-artifact@v4 - with: - name: dist - path: dist - - - name: Check for packaging errors - run: pipx run twine check --strict dist/* - - - name: Install produced wheel - uses: ./.github/actions/install_requirements - with: - pip-install: dist/*.whl - - - name: Test module --version works using the installed wheel - # If more than one module in src/ replace with module name to test - run: python -m $(ls --hide='*.egg-info' src | head -1) --version diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml deleted file mode 100644 index 552b29d..0000000 --- a/.github/workflows/_test.yml +++ /dev/null @@ -1,62 +0,0 @@ -on: - workflow_call: - inputs: - python-version: - type: string - description: The version of python to install - required: true - runs-on: - type: string - description: The runner to run this job on - required: true - secrets: - CODECOV_TOKEN: - required: true - -env: - # https://github.com/pytest-dev/pytest/issues/2042 - PY_IGNORE_IMPORTMISMATCH: "1" - -jobs: - run: - runs-on: ${{ inputs.runs-on }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # Need this to get version number from last tag - fetch-depth: 0 - - - if: inputs.python-version == 'dev' - name: Install dev versions of python packages - uses: ./.github/actions/install_requirements - - - if: inputs.python-version == 'dev' - name: Write the requirements as an artifact - run: pip freeze --exclude-editable > /tmp/dev-requirements.txt - - - if: inputs.python-version == 'dev' - name: Upload dev-requirements.txt - uses: actions/upload-artifact@v4 - with: - name: dev-requirements - path: /tmp/dev-requirements.txt - - - if: inputs.python-version != 'dev' - name: Install latest versions of python packages - uses: ./.github/actions/install_requirements - with: - python-version: ${{ inputs.python-version }} - pip-install: ".[dev]" - - - name: Run tests - run: tox -e tests - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v5 - with: - name: ${{ inputs.python-version }}/${{ inputs.runs-on }} - files: cov.xml - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb59381..0679e88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,33 +5,15 @@ on: branches: - main tags: - - '*' + - "*" pull_request: jobs: - lint: uses: ./.github/workflows/_tox.yml with: tox: pre-commit,type-checking - test: - strategy: - matrix: - runs-on: ["ubuntu-latest"] # can add windows-latest, macos-latest - python-version: ["3.11", "3.12", "3.13"] - include: - # Include one that runs in the dev environment - - runs-on: "ubuntu-latest" - python-version: "dev" - fail-fast: false - uses: ./.github/workflows/_test.yml - with: - runs-on: ${{ matrix.runs-on }} - python-version: ${{ matrix.python-version }} - secrets: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - container: needs: test if: always() @@ -45,10 +27,6 @@ jobs: docs: uses: ./.github/workflows/_docs.yml - - dist: - uses: ./.github/workflows/_dist.yml - release: needs: [dist, test, docs] if: github.ref_type == 'tag' diff --git a/.github/workflows/periodic.yml b/.github/workflows/periodic.yml deleted file mode 100644 index e2a0fd1..0000000 --- a/.github/workflows/periodic.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: Periodic - -on: - workflow_dispatch: - schedule: - # Run weekly to check URL links still resolve - - cron: "0 8 * * WED" - -jobs: - linkcheck: - uses: ./.github/workflows/_tox.yml - with: - tox: docs build -- -b linkcheck diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ef68f8a..b086083 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,19 +6,3 @@ repos: - id: check-yaml - id: check-merge-conflict - id: end-of-file-fixer - - - repo: local - hooks: - - id: ruff - name: lint with ruff - language: system - entry: ruff check --force-exclude --fix - types: [python] - require_serial: true - - - id: ruff-format - name: format with ruff - language: system - entry: ruff format --force-exclude - types: [python] - require_serial: true diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 734f215..fee986e 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -4,4 +4,4 @@ "ms-python.python", "ryanluker.vscode-coverage-gutters" ] -} \ No newline at end of file +} diff --git a/.vscode/launch.json b/.vscode/launch.json index f8fcdb4..ce1b374 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -22,4 +22,4 @@ }, } ] -} \ No newline at end of file +} diff --git a/.vscode/tasks.json b/.vscode/tasks.json index ff78a11..82fa1ae 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -13,4 +13,4 @@ "problemMatcher": [], } ] -} \ No newline at end of file +} diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 2e194bf..63c7c1b 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -77,4 +77,3 @@ To make a new release, please follow this checklist: - GitHub Actions will publish the image to ghcr.io and the docs to GitHub Pages .. _release: https://github.com/diamondlightsource/dev-c7/releases - diff --git a/Charts/dev-c7/README.md b/Charts/dev-c7/README.md index 6b2e8ef..8387892 100644 --- a/Charts/dev-c7/README.md +++ b/Charts/dev-c7/README.md @@ -1,3 +1,3 @@ A Helm Chart for deploying the dev-c7 container into a DLS Cluster. -Intended for runnhing legacy IOCs in a RHEL8 or higher DLS worker node. \ No newline at end of file +Intended for runnhing legacy IOCs in a RHEL8 or higher DLS worker node. diff --git a/Dockerfile b/Dockerfile index e77cbc8..e2563be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,31 +1,79 @@ -# The devcontainer should use the developer target and run as root with podman -# or docker with user namespaces. -ARG PYTHON_VERSION=3.11 -FROM python:${PYTHON_VERSION} AS developer - -# Add any system dependencies for the developer/build environment here -RUN apt-get update && apt-get install -y --no-install-recommends \ - graphviz \ - && rm -rf /var/lib/apt/lists/* - -# Set up a virtual environment and put it in PATH -RUN python -m venv /venv -ENV PATH=/venv/bin:$PATH - -# The build stage installs the context into the venv -FROM developer AS build -# Requires buildkit 0.17.0 -COPY --chmod=o+wrX . /workspaces/dev-c7 -WORKDIR /workspaces/dev-c7 -RUN touch dev-requirements.txt && pip install -c dev-requirements.txt . - - -# The runtime stage copies the built venv into a slim runtime container -FROM python:${PYTHON_VERSION}-slim AS runtime -# Add apt-get system dependecies for runtime here if needed -COPY --from=build /venv/ /venv/ -ENV PATH=/venv/bin:$PATH - -# change this entrypoint if it is not the same as the repo -ENTRYPOINT ["dev-c7"] -CMD ["--version"] +# Dockerfile for getting DLS RHEL7 dev environment working in a container +# hosted on RHEL8 +# +# This is a stopgap so that we can use effort to promote the +# kubernetes model https://github.com/diamondlightsource instead of spending +# effort in rebuilding our toolchain for RHEL8 + +FROM centos:7 + +# environment +ENV LANG=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 +ENV LC_CTYPE=en_US.UTF-8 +ENV MODULEPATH=/etc/scl/modulefiles:/etc/scl/modulefiles:/etc/scl/modulefiles:/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/dls_sw/apps/Modules/modulefiles:/dls_sw/etc/modulefiles:/home/hgv27681/privatemodules:/dls_sw/prod/tools/RHEL8-x86_64/defaults/modulefiles + +# switch to the vault for centos packages! +RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \ + sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && \ + sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo + +# make QT not complain +ENV QT_X11_NO_MITSHM=1 +ENV LIBGL_ALWAYS_INDIRECT=1 + +RUN yum update -y && \ + # dev tools and libraries + yum groupinstall -y "Development Tools" && \ + yum install -y glibc.i686 redhat-lsb-core libusbx net-snmp-libs environment-modules git2u net-tools screen cmake lapack-devel readline-devel pcre-devel boost-devel libpcap-devel numactl-libs vim dbus-x11 bind-utils libssh2-devel xorg-x11-server-Xvfb \ + # edm dependencies + epel-release giflib-devel libXmu-devel libpng-devel libXtst-devel zlib-devel xorg-x11-proto-devel motif-devel libX11-devel libXp-devel libXpm-devel libtirpc-devel \ + # areaDetector dependencies + libxml2-devel libjpeg-turbo-devel libtiff-devel glib2-devel \ + # Odin dependencies + hdf5-devel zeromq-devel librdkafka-devel \ + # QT4 dependencies + pyqt4-devel PackageKit-gtk3-module libcanberra-gtk2 mesa-dri-drivers.x86_64 xcb-util xcb-util-devel libxcb.x86_64 libxkbcommon-x11 \ + # dls-remote-desktop-support + xfreerdp zenity \ + # useful tools + sudo meld tk dejavu-sans-mono-fonts gnome-terminal xterm xmessage evince eog firefox java openldap-clients ant && \ + # These packages are not found in the initial install so try again + yum install -y zeromq-devel git2u meld && \ + # install libnet library for EPICS module builds + yum install -y libnet libnet-devel && \ + # clean up caches + yum clean all + +#install the required libusb developer library +RUN yum remove -y libusbx && \ + curl -SL http://rpmfind.net/linux/opensuse/distribution/leap/15.5/repo/oss/x86_64/libusb-1_0-0-1.0.24-150400.3.3.1.x86_64.rpm -o /tmp/libusb-1.rpm && \ + yum install -y /tmp/libusb-1.rpm && \ + rm /tmp/libusb-1.rpm && \ + curl -SL https://rpmfind.net/linux/opensuse/distribution/leap/15.5/repo/oss/x86_64/libusb-1_0-devel-1.0.24-150400.3.3.1.x86_64.rpm -o /tmp/libusb-devel-1.rpm && \ + yum install -y /tmp/libusb-devel-1.rpm && \ + rm /tmp/libusb-devel-1.rpm + +# full sudo rights inside the container +COPY /sudoers /etc/sudoers + +# Workaround to ensure all locales are available +RUN sed -i "/override_install_langs=en_US/d" /etc/yum.conf +RUN yum reinstall -y glibc-common + +# set container time to BST +RUN ln -sf /usr/share/zoneinfo/Europe/London /etc/localtime + +# latest git from the endpoint repo https://computingforgeeks.com/install-git-2-on-centos-7/ +RUN yum remove -y git git-core && \ + # libcurl-devel for git-remote-https + yum -y install libcurl-devel && \ + yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm && \ + yum -y install git git-core + +# change the nobody account and group IDs to match RedHat +RUN sed -i 's/99:99/65534:65534/' /etc/passwd && \ + sed -i 's/:99:/:65534:/' /etc/group + + +RUN diff --git a/README.md b/README.md index 35a7f15..eab3163 100644 --- a/README.md +++ b/README.md @@ -3,35 +3,33 @@ [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://www.apache.org/licenses/LICENSE-2.0) -# dev_c7 +# DLS RHEL7 in a Box Developer Container A Diamond Light Source specific developer container -This is where you should write a short paragraph that describes what your module does, -how it does it, and why people should use it. +This repo defines a container and launch script for re-creating the DLS Controls RHEL7 developer environment on a RHEL8 Workstation. + +This container can also be used for running traditional DLS EPICS IOCs in Kubernetes (i.e. IOCs published in /dls_sw/prod or /dls_sw/work). Source | :---: | :---: -Docker | `docker run ghcr.io/diamondlightsource/dev-c7:latest` +Docker | `podman run ghcr.io/diamondlightsource/dev-c7:latest` Documentation | Releases | -This is where you should put some images or code snippets that illustrate -some relevant examples. If it is a library then you might put some -introductory code here: +## Quick Start -```python -from dev_c7 import __version__ +See the tutorial at `Quick Start `_. -print(f"Hello dev_c7 {__version__}") -``` + -Or if it is a commandline tool then you might put some example commands here: +See for the full documentation. -``` -python -m dev_c7 --version -``` +## Notes - +This project has had the Python Copier Template applied from . It is not a python project but uses the template for the following features:- -See https://diamondlightsource.github.io/dev-c7 for more detailed documentation. +- Building and publishing documentation +- Building and publishing a container image +- Building and publishing a helm chart +- Building schema for the helm chart and releasing that schema diff --git a/README.rst b/README.rst deleted file mode 100644 index 1dfa1c8..0000000 --- a/README.rst +++ /dev/null @@ -1,43 +0,0 @@ -DLS RHEL7 in a Box Developer Container -====================================== - -|code_ci| |docs_ci| |license| - -A Diamond Light Source specific developer container. - -This repo defines a container and launch script for re-creating the -DLS Controls RHEL7 developer environment on a RHEL8 Workstation. - -============== ============================================================== -Source code https://github.com/diamondlightsource/dev-c7 -Documentation https://diamondlightsource.github.io/dev-c7 -Releases https://github.com/diamondlightsource/dev-c7/releases -Images https://ghcr.io/diamondlightsource/dev-c7 -============== ============================================================== - -Quick Start -=========== - -See the tutorial at `Quick Start `_. - - - -.. |code_ci| image:: https://github.com/diamondlightsource/dev-c7/workflows/Code%20CI/badge.svg?branch=main - :target: https://github.com/diamondlightsource/dev-c7/actions?query=workflow%3A%22Code+CI%22 - :alt: Code CI - -.. |docs_ci| image:: https://github.com/diamondlightsource/dev-c7/workflows/Docs%20CI/badge.svg?branch=main - :target: https://github.com/diamondlightsource/dev-c7/actions?query=workflow%3A%22Docs+CI%22 - :alt: Docs CI - -.. |license| image:: https://img.shields.io/badge/License-Apache%202.0-blue.svg - :target: https://opensource.org/licenses/Apache-2.0 - :alt: Apache License - -.. - Anything below this line is used when viewing README.rst and will be replaced - when included in index.rst - -See https://diamondlightsource.github.io/dev-c7 for detailed documentation. - -.. |quick| target:: tutorials/start.html \ No newline at end of file diff --git a/build-dev-c7.sh b/build-dev-c7.sh index c77c92f..7062f20 100755 --- a/build-dev-c7.sh +++ b/build-dev-c7.sh @@ -6,7 +6,7 @@ set -e podman build\ --network host \ -t ${name}:latest \ - ./docker + . # read -r -p "dev-c7 built OK. Push to ${name}:${1} [y/N] " response diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index e29ddc0..0000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,79 +0,0 @@ -# Dockerfile for getting DLS RHEL7 dev environment working in a container -# hosted on RHEL8 -# -# This is a stopgap so that we can use effort to promote the -# kubernetes model https://github.com/diamondlightsource instead of spending -# effort in rebuilding our toolchain for RHEL8 - -FROM centos:7 - -# environment -ENV LANG=en_US.UTF-8 -ENV LC_ALL=en_US.UTF-8 -ENV LC_CTYPE=en_US.UTF-8 -ENV MODULEPATH=/etc/scl/modulefiles:/etc/scl/modulefiles:/etc/scl/modulefiles:/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/dls_sw/apps/Modules/modulefiles:/dls_sw/etc/modulefiles:/home/hgv27681/privatemodules:/dls_sw/prod/tools/RHEL8-x86_64/defaults/modulefiles - -# switch to the vault for centos packages! -RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \ - sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && \ - sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo - -# make QT not complain -ENV QT_X11_NO_MITSHM=1 -ENV LIBGL_ALWAYS_INDIRECT=1 - -RUN yum update -y && \ - # dev tools and libraries - yum groupinstall -y "Development Tools" && \ - yum install -y glibc.i686 redhat-lsb-core libusbx net-snmp-libs environment-modules git2u net-tools screen cmake lapack-devel readline-devel pcre-devel boost-devel libpcap-devel numactl-libs vim dbus-x11 bind-utils libssh2-devel xorg-x11-server-Xvfb \ - # edm dependencies - epel-release giflib-devel libXmu-devel libpng-devel libXtst-devel zlib-devel xorg-x11-proto-devel motif-devel libX11-devel libXp-devel libXpm-devel libtirpc-devel \ - # areaDetector dependencies - libxml2-devel libjpeg-turbo-devel libtiff-devel glib2-devel \ - # Odin dependencies - hdf5-devel zeromq-devel librdkafka-devel \ - # QT4 dependencies - pyqt4-devel PackageKit-gtk3-module libcanberra-gtk2 mesa-dri-drivers.x86_64 xcb-util xcb-util-devel libxcb.x86_64 libxkbcommon-x11 \ - # dls-remote-desktop-support - xfreerdp zenity \ - # useful tools - sudo meld tk dejavu-sans-mono-fonts gnome-terminal xterm xmessage evince eog firefox java openldap-clients ant && \ - # These packages are not found in the initial install so try again - yum install -y zeromq-devel git2u meld && \ - # install libnet library for EPICS module builds - yum install -y libnet libnet-devel && \ - # clean up caches - yum clean all - -#install the required libusb developer library -RUN yum remove -y libusbx && \ - curl -SL http://rpmfind.net/linux/opensuse/distribution/leap/15.5/repo/oss/x86_64/libusb-1_0-0-1.0.24-150400.3.3.1.x86_64.rpm -o /tmp/libusb-1.rpm && \ - yum install -y /tmp/libusb-1.rpm && \ - rm /tmp/libusb-1.rpm && \ - curl -SL https://rpmfind.net/linux/opensuse/distribution/leap/15.5/repo/oss/x86_64/libusb-1_0-devel-1.0.24-150400.3.3.1.x86_64.rpm -o /tmp/libusb-devel-1.rpm && \ - yum install -y /tmp/libusb-devel-1.rpm && \ - rm /tmp/libusb-devel-1.rpm - -# full sudo rights inside the container -COPY /sudoers /etc/sudoers - -# Workaround to ensure all locales are available -RUN sed -i "/override_install_langs=en_US/d" /etc/yum.conf -RUN yum reinstall -y glibc-common - -# set container time to BST -RUN ln -sf /usr/share/zoneinfo/Europe/London /etc/localtime - -# latest git from the endpoint repo https://computingforgeeks.com/install-git-2-on-centos-7/ -RUN yum remove -y git git-core && \ - # libcurl-devel for git-remote-https - yum -y install libcurl-devel && \ - yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm && \ - yum -y install git git-core - -# change the nobody account and group IDs to match RedHat -RUN sed -i 's/99:99/65534:65534/' /etc/passwd && \ - sed -i 's/:99:/:65534:/' /etc/group - - -RUN \ No newline at end of file diff --git a/docs/explanations.md b/docs/explanations.md index 73ab289..ccb51d5 100644 --- a/docs/explanations.md +++ b/docs/explanations.md @@ -6,5 +6,8 @@ Explanations of how it works and why it works that way. :maxdepth: 1 :glob: -explanations/* +explanations/how_it_works +explanations/differences +explanations/why_dev_c7 +explanations/decisions ``` diff --git a/docs/explanations.rst b/docs/explanations.rst deleted file mode 100644 index 888750e..0000000 --- a/docs/explanations.rst +++ /dev/null @@ -1,13 +0,0 @@ -:orphan: - -Explanations -============ - -Explanation of how the library works and why it works that way. - -.. toctree:: - :caption: Explanations - - explanations/how_it_works - explanations/differences - explanations/why_dev_c7 diff --git a/docs/explanations/why_dev_c7.rst b/docs/explanations/why_dev_c7.rst index 7bebcd6..748c016 100644 --- a/docs/explanations/why_dev_c7.rst +++ b/docs/explanations/why_dev_c7.rst @@ -21,4 +21,4 @@ With the new approach under development it would be a waste of effort to port our legacy toolchain to RHEL8 only to replace it shortly afterward. The container based approach decouples the tools and IOCs from the host operating system so any future OS upgrades should be far -smoother. \ No newline at end of file +smoother. diff --git a/docs/how-to.rst b/docs/how-to.rst deleted file mode 100644 index 591026d..0000000 --- a/docs/how-to.rst +++ /dev/null @@ -1,15 +0,0 @@ -:orphan: - -How-to Guides -============= - -Practical step-by-step guides for the more experienced user. - -.. toctree:: - :caption: How-to Guides - - how-to/containers - how-to/troubleshooting - how-to/deriving - how-to/podman - diff --git a/docs/how-to/Dockerfile b/docs/how-to/Dockerfile index 0f65919..bdae04a 100644 --- a/docs/how-to/Dockerfile +++ b/docs/how-to/Dockerfile @@ -6,4 +6,4 @@ RUN yum update -y && \ ENV MY_VAR="useful info" -COPY /my-useful-file.sh /usr/bin \ No newline at end of file +COPY /my-useful-file.sh /usr/bin diff --git a/docs/how-to/containers.rst b/docs/how-to/containers.rst index eaf4af8..db5fe19 100644 --- a/docs/how-to/containers.rst +++ b/docs/how-to/containers.rst @@ -85,7 +85,7 @@ Below is the current Dockerfile, read on for an explanation of the commands used. -.. literalinclude:: ../../docker/Dockerfile +.. literalinclude:: ../../Dockerfile :language: docker diff --git a/docs/how-to/deriving.rst b/docs/how-to/deriving.rst index 2962104..6b17d98 100644 --- a/docs/how-to/deriving.rst +++ b/docs/how-to/deriving.rst @@ -13,7 +13,7 @@ specific requirements e.g. .. note:: If the changes you need are likely to be useful to all developers then you should consider contributing to the base image for dev-c7. - See `../reference/contributing` + See `contribute` To make your own derived container. @@ -30,4 +30,4 @@ that the build is running as root inside the container so it is allowed to write to a system folder. .. literalinclude:: Dockerfile - :language: docker \ No newline at end of file + :language: docker diff --git a/docs/how-to/podman.rst b/docs/how-to/podman.rst index d2bbbb5..788128b 100644 --- a/docs/how-to/podman.rst +++ b/docs/how-to/podman.rst @@ -31,4 +31,3 @@ The additional features are: - provide a much faster container build and run #. put the root of podman scratch files in /scratch//podman - avoid loosing all scratch your files on ``podman system reset`` - diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index 9bde8ad..0000000 --- a/docs/index.rst +++ /dev/null @@ -1,48 +0,0 @@ -.. include:: ../README.rst - :end-before: when included in index.rst - - -How the documentation is structured ------------------------------------ - -Documentation is split into four categories, also accessible from links in the -side-bar. - -.. rst-class:: columns - -`tutorials` -~~~~~~~~~~~ - -.. include:: tutorials.rst - :start-after: ========= - -.. rst-class:: columns - -`how-to` -~~~~~~~~ - -.. include:: how-to.rst - :start-after: ============= - -.. rst-class:: columns - -`explanations` -~~~~~~~~~~~~~~ - -.. include:: explanations.rst - :start-after: ============ - -.. rst-class:: columns - -`reference` -~~~~~~~~~~~ - -.. include:: reference.rst - :start-after: ========= - -.. rst-class:: endcolumns - -About the documentation -~~~~~~~~~~~~~~~~~~~~~~~ - -`Why is the documentation structured this way? `_ diff --git a/docs/make_docs.sh b/docs/make_docs.sh deleted file mode 100755 index 5e4f20e..0000000 --- a/docs/make_docs.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# a script to locally test documentation build. - -root=$(realpath $(dirname ${BASH_SOURCE[0]})/..) - -set -e - -if [[ ! -d "${root}/.venv" ]] ; then - virtualenv "${root}/.venv" --python python3 - source "${root}/.venv/bin/activate" - pip install -r "${root}/requirements.txt" -fi - -source "${root}/.venv/bin/activate" -sphinx-build -EWT --keep-going docs ${root}/build/html "${@}" \ No newline at end of file diff --git a/docs/reference.rst b/docs/reference.rst deleted file mode 100644 index e392b79..0000000 --- a/docs/reference.rst +++ /dev/null @@ -1,12 +0,0 @@ -:orphan: - -Reference -========= - -Practical step-by-step guides for the more experienced user. - -.. toctree:: - :caption: Reference - - reference/contributing - Releases diff --git a/docs/reference/contributing.rst b/docs/reference/contributing.rst deleted file mode 100644 index ac7b6bc..0000000 --- a/docs/reference/contributing.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../../CONTRIBUTING.rst diff --git a/docs/tutorials.md b/docs/tutorials.md index 1fe66c5..938e8eb 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -6,5 +6,8 @@ Tutorials for installation and typical usage. New users start here. :maxdepth: 1 :glob: -tutorials/* +tutorials/start +tutorials/continue +tutorials/vscode +tutorials/prompt ``` diff --git a/docs/tutorials.rst b/docs/tutorials.rst deleted file mode 100644 index 78cc409..0000000 --- a/docs/tutorials.rst +++ /dev/null @@ -1,14 +0,0 @@ -:orphan: - -Tutorials -========= - -Tutorials for installation, library and commandline usage. New users start here. - -.. toctree:: - :caption: Tutorials - - tutorials/start - tutorials/continue - tutorials/vscode - tutorials/prompt diff --git a/docs/tutorials/continue.rst b/docs/tutorials/continue.rst index 5117b67..3ef1796 100644 --- a/docs/tutorials/continue.rst +++ b/docs/tutorials/continue.rst @@ -113,4 +113,3 @@ If the script has acquired new features you may want to update as follows:: Also update your devcontainer.json to match for projects you want to also upgrade. - diff --git a/docs/tutorials/installation.md b/docs/tutorials/installation.md deleted file mode 100644 index ed254a9..0000000 --- a/docs/tutorials/installation.md +++ /dev/null @@ -1,42 +0,0 @@ -# Installation - -## Check your version of python - -You will need python 3.11 or later. You can check your version of python by -typing into a terminal: - -``` -$ python3 --version -``` - -## Create a virtual environment - -It is recommended that you install into a “virtual environment” so this -installation will not interfere with any existing Python software: - -``` -$ python3 -m venv /path/to/venv -$ source /path/to/venv/bin/activate -``` - -## Installing the library - -You can now use `pip` to install the library and its dependencies: - -``` -$ python3 -m pip install dev-c7 -``` - -If you require a feature that is not currently released you can also install -from github: - -``` -$ python3 -m pip install git+https://github.com/DiamondLightSource/dev-c7.git -``` - -The library should now be installed and the commandline interface on your path. -You can check the version that has been installed by typing: - -``` -$ dev-c7 --version -``` diff --git a/docs/tutorials/prompt.rst b/docs/tutorials/prompt.rst index 5fd11b6..048c51a 100644 --- a/docs/tutorials/prompt.rst +++ b/docs/tutorials/prompt.rst @@ -45,4 +45,3 @@ can pass a hostname of your choice as follows. If you are working remotely and launching the container from an ssh session changing the hostname will cause X11 apps to fail. - diff --git a/edm-fonts/install-fonts.sh b/edm-fonts/install-fonts.sh index d7a8848..c188350 100644 --- a/edm-fonts/install-fonts.sh +++ b/edm-fonts/install-fonts.sh @@ -6,4 +6,4 @@ sudo yum install https://github.com/diamondlightsource/dev-c7/raw/main/edm-fonts sudo yum install https://github.com/diamondlightsource/dev-c7/raw/main/edm-fonts/dls-courier-1.0-1.noarch.rpm sudo yum install https://github.com/diamondlightsource/dev-c7/raw/main/edm-fonts/dls-msttcore-1.0-1.noarch.rpm -echo Done \ No newline at end of file +echo Done diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 324276b..0000000 --- a/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -pre-commit -sphinx-rtd-theme-github-versions -sphinx-apischema -black -mypy -flake8 diff --git a/src/README.md b/src/README.md new file mode 100644 index 0000000..3dae230 --- /dev/null +++ b/src/README.md @@ -0,0 +1,3 @@ +# Not a Python Project + +This src is just the skeleton from the python-copier-template. It is a convenient way to extract the current git tag to place in the documentation and serves no other function. diff --git a/docker/sudoers b/sudoers similarity index 100% rename from docker/sudoers rename to sudoers diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_cli.py b/tests/test_cli.py deleted file mode 100644 index 723c630..0000000 --- a/tests/test_cli.py +++ /dev/null @@ -1,9 +0,0 @@ -import subprocess -import sys - -from dev_c7 import __version__ - - -def test_cli_version(): - cmd = [sys.executable, "-m", "dev_c7", "--version"] - assert subprocess.check_output(cmd).decode().strip() == __version__ From bee2e4bbf0f6cef81db9b02819fb769319bc022d Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 4 Sep 2025 15:31:57 +0100 Subject: [PATCH 06/13] get pre-commit working --- .pre-commit-config.yaml | 1 + pyproject.toml | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b086083..1372b54 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,5 +4,6 @@ repos: hooks: - id: check-added-large-files - id: check-yaml + exclude: ^Charts/.*/templates - id: check-merge-conflict - id: end-of-file-fixer diff --git a/pyproject.toml b/pyproject.toml index 41a53de..01f87ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,7 +78,7 @@ legacy_tox_ini = """ [tox] skipsdist=True -[testenv:{pre-commit,type-checking,tests,docs}] +[testenv:{pre-commit,docs}] # Don't create a virtualenv for the command, requires tox-direct plugin direct = True passenv = * @@ -90,8 +90,6 @@ allowlist_externals = sphinx-autobuild commands = pre-commit: pre-commit run --all-files --show-diff-on-failure {posargs} - type-checking: pyright src tests {posargs} - tests: pytest --cov=dev_c7 --cov-report term --cov-report xml:cov.xml {posargs} docs: sphinx-{posargs:build -EW --keep-going} -T docs build/html """ From d2ede2554e774337116b1cd6156a44c67460278f Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 4 Sep 2025 16:05:11 +0100 Subject: [PATCH 07/13] add helm CI --- .github/workflows/_helm.yml | 71 ++++++++++++++++++++++++++++++++ .github/workflows/ci.yml | 10 ++++- .github/workflows/code.yml | 81 ------------------------------------- .github/workflows/docs.yml | 63 ----------------------------- 4 files changed, 79 insertions(+), 146 deletions(-) create mode 100644 .github/workflows/_helm.yml delete mode 100644 .github/workflows/code.yml delete mode 100644 .github/workflows/docs.yml diff --git a/.github/workflows/_helm.yml b/.github/workflows/_helm.yml new file mode 100644 index 0000000..ca1496f --- /dev/null +++ b/.github/workflows/_helm.yml @@ -0,0 +1,71 @@ +name: Package helm charts + +on: + workflow_call: + +env: + HELM_VERSION_TO_INSTALL: 3.14.3 + +jobs: + package-helm-charts: + name: Package and Push Helm Chart + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install helm + uses: Azure/setup-helm@v3 + with: + version: ${{ env.HELM_VERSION_TO_INSTALL }} + + # Check that alpha/beta versions have the form X.Y.Z-alpha.A requried by Helm. + # An early check saves waiting for the entire build before finding a problem. + - name: Check helm version tag + if: ${{ github.ref_type == 'tag' }} + env: + VERSION: "${{ github.ref_name }}" + run: | + if [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-alpha|-beta|-rc).*?$ ]]; then + echo "Valid version format: ${VERSION}" + else + echo "Invalid version: ${VERSION}. Expected: X.Y.Z or X.Y.Z-beta.1 or X.Y.Z-alpha.1" + exit 1 + fi + + - name: Package helm charts + env: + VERSION: "${{ github.ref_type == 'tag' && github.ref_name || '0.0.0' }}" + run: | + set -xe + + mkdir -p charts + for i in $(find Charts -type d -maxdepth 1 -mindepth 1); do + if [[ ${i} =~ ^.*-ioc$ ]]; then + echo "Skipping IOC schema chart: ${i}" + continue + fi + echo "Packaging chart: ${i}" + helm package -u --app-version ${VERSION} --version ${VERSION} ${i} + mv $(basename ${i})-*.tgz charts/ + done + + - name: Upload helm chart values schemas + uses: actions/upload-artifact@v4 + with: + name: helm-chart-schemas + path: schemas/* + + - name: Push tagged helm chart to registry + # TODO - switch to using https://github.com/helm/chart-releaser-action of maybe the docker action? + if: ${{ github.ref_type == 'tag' }} + run: | + set -x + + echo "${{ secrets.GITHUB_TOKEN }}" | helm registry login ghcr.io/${{ github.repository_owner }} --username ${{ github.repository_owner }} --password-stdin + REGISTRY=oci://ghcr.io/${{github.repository_owner }}/charts + for i in charts/*.tgz; do + helm push "${i}" ${REGISTRY,,} + done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0679e88..9f5964c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: tox: pre-commit,type-checking container: - needs: test + needs: helm if: always() uses: ./.github/workflows/_container.yml with: @@ -27,8 +27,14 @@ jobs: docs: uses: ./.github/workflows/_docs.yml + helm: + uses: ./.github/workflows/_helm.yml + permissions: + contents: read + packages: write + release: - needs: [dist, test, docs] + needs: [container, docs, helm] if: github.ref_type == 'tag' uses: ./.github/workflows/_release.yml permissions: diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml deleted file mode 100644 index e393770..0000000 --- a/.github/workflows/code.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: Build and publish image to ghcr.io - -on: - push: - pull_request: - -jobs: - build: - # pull requests are a duplicate of a branch push if within the same repo. - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository - permissions: - contents: read - packages: write - - runs-on: ubuntu-latest - outputs: - developer-tag: ${{ steps.meta-developer.outputs.tags }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Log in to GitHub Docker Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io/${{ github.repository_owner }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build image - uses: docker/build-push-action@v5 - with: - context: ./docker - platforms: ${{ matrix.platform }} - build-args: | - IMAGE_EXT=${{ matrix.extension }} - cache-from: type=gha,scope=${{ matrix.epics-target }} - cache-to: type=gha,mode=max,scope=${{ matrix.epics-target }} - tags: ci_test - load: true - - - name: Developer target metadata - id: meta-developer - uses: docker/metadata-action@v3 - with: - images: | - ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} - tags: | - type=raw,value=${{ github.ref_name }} - type=raw,value=latest - - - name: Push developer image - if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }} - uses: docker/build-push-action@v5 - with: - context: ./docker - platforms: ${{ matrix.platform }} - build-args: | - IMAGE_EXT=${{ matrix.extension }} - tags: ${{ steps.meta-developer.outputs.tags }} - labels: ${{ steps.meta-developer.outputs.labels }} - push: true - - release: - # Release on tag push - publish ioc schema - needs: [build] - if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }} - runs-on: ubuntu-latest - - steps: - - name: Github Release - uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # v0.1.15 - with: - files: | - ibek.ioc.schema.json - generate_release_notes: true diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index 1c49844..0000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Docs CI - -on: - push: - branches: - # Add more branches here to publish docs from other branches - - master - - main - tags: - - "*" - pull_request: - -jobs: - docs: - strategy: - fail-fast: false - matrix: - os: ["ubuntu-latest"] - python: ["3.10"] - - runs-on: ubuntu-latest - - steps: - - name: Avoid git conflicts when tag and branch pushed at same time - if: startsWith(github.ref, 'refs/tags') - run: sleep 60 - - - name: Install python version - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python }} - - - name: Install Packages - # Can delete this if you don't use graphviz in your docs - run: sudo apt-get install graphviz - - - name: checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Install dependencies - run: pip install -r requirements.txt - - - name: Build docs - run: sphinx-build -EWT --keep-going docs build/html - - - name: Move to versioned directory - # e.g. master or 0.1.2 - run: mv build/html ".github/pages/${GITHUB_REF##*/}" - - - name: Write versions.txt - run: sphinx_rtd_theme_github_versions .github/pages - - - name: Publish Docs to gh-pages - if: github.event_name == 'push' - # We pin to the SHA, not the tag, for security reasons. - # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions - uses: peaceiris/actions-gh-pages@068dc23d9710f1ba62e86896f84735d869951305 # v3.8.0 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: .github/pages - keep_files: true From 2d9beaef87e213d5dd58cfd10a4025a7869306df Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 4 Sep 2025 16:17:58 +0100 Subject: [PATCH 08/13] fix CI --- .github/workflows/_helm.yml | 2 +- .github/workflows/ci.yml | 4 ++-- Charts/dev-c7/Chart.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/_helm.yml b/.github/workflows/_helm.yml index ca1496f..56ea1b1 100644 --- a/.github/workflows/_helm.yml +++ b/.github/workflows/_helm.yml @@ -8,7 +8,7 @@ env: jobs: package-helm-charts: - name: Package and Push Helm Chart + name: Helm Chart runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f5964c..2a2fe30 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: lint: uses: ./.github/workflows/_tox.yml with: - tox: pre-commit,type-checking + tox: pre-commit container: needs: helm @@ -34,7 +34,7 @@ jobs: packages: write release: - needs: [container, docs, helm] + needs: [container, docs, helm, lint] if: github.ref_type == 'tag' uses: ./.github/workflows/_release.yml permissions: diff --git a/Charts/dev-c7/Chart.yaml b/Charts/dev-c7/Chart.yaml index 2317ffe..268df46 100644 --- a/Charts/dev-c7/Chart.yaml +++ b/Charts/dev-c7/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -name: dls-c7 +name: dev-c7 description: A Helm chart legacy DLS IOCs type: application From a455640e3e14e6a75f732637063ef4d4d4522026 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Fri, 5 Sep 2025 07:23:24 +0100 Subject: [PATCH 09/13] remove redundant container test from CI --- .github/workflows/_container.yml | 3 --- Dockerfile | 3 --- 2 files changed, 6 deletions(-) diff --git a/.github/workflows/_container.yml b/.github/workflows/_container.yml index c6cd469..dd22fdc 100644 --- a/.github/workflows/_container.yml +++ b/.github/workflows/_container.yml @@ -38,9 +38,6 @@ jobs: load: true tags: tag_for_testing - - name: Test cli works in cached runtime image - run: docker run --rm tag_for_testing --version - - name: Create tags for publishing image id: meta uses: docker/metadata-action@v5 diff --git a/Dockerfile b/Dockerfile index e2563be..27bc402 100644 --- a/Dockerfile +++ b/Dockerfile @@ -74,6 +74,3 @@ RUN yum remove -y git git-core && \ # change the nobody account and group IDs to match RedHat RUN sed -i 's/99:99/65534:65534/' /etc/passwd && \ sed -i 's/:99:/:65534:/' /etc/group - - -RUN From e458b6a113a3b189078a0afe33399c21038e1186 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Fri, 5 Sep 2025 08:15:01 +0100 Subject: [PATCH 10/13] add schema generation for helm --- .pre-commit-config.yaml | 8 ++ Charts/dev-c7/.schema.config.yaml | 25 +++++ Charts/dev-c7/templates/deployment.yaml | 4 + Charts/dev-c7/values.schema.json | 122 ++++++++++++++++++++++++ Charts/dev-c7/values.yaml | 62 ++++++------ schema/dev-c7.values.schema.json | 1 + 6 files changed, 194 insertions(+), 28 deletions(-) create mode 100644 Charts/dev-c7/.schema.config.yaml create mode 100644 Charts/dev-c7/values.schema.json create mode 120000 schema/dev-c7.values.schema.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1372b54..c428efd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,3 +7,11 @@ repos: exclude: ^Charts/.*/templates - id: check-merge-conflict - id: end-of-file-fixer + + - repo: https://github.com/losisin/helm-values-schema-json + rev: v2.2.1 + hooks: + - id: helm-schema + args: + - --config + - Charts/dev-c7/.schema.config.yaml diff --git a/Charts/dev-c7/.schema.config.yaml b/Charts/dev-c7/.schema.config.yaml new file mode 100644 index 0000000..d5950e2 --- /dev/null +++ b/Charts/dev-c7/.schema.config.yaml @@ -0,0 +1,25 @@ +# .schema.yaml + +# Define input, output and source for $refs relative to repository root for pre-commit +values: + - Charts/dev-c7/values.yaml + +output: Charts/dev-c7/values.schema.json + +# bundle up references (don't do this as K8S refs are 500kb+) +bundleRoot: Charts +bundle: false + +# Include comments for the helm-docs plugin into the schema, to allow e.g. documentation in VSCode +useHelmDocs: false + +# Allow additional properties for eg. initResources, different types of volumes/volumeMounts +noAdditionalProperties: false + +schemaRoot: + title: dev-c7 Helm chart + description: Helm chart for deploying a dev-c7 application + # No additional properties in schema root for tighter protection: + additionalProperties: false + +k8sSchemaVersion: v1.33.3 diff --git a/Charts/dev-c7/templates/deployment.yaml b/Charts/dev-c7/templates/deployment.yaml index 775ad36..9da5ff5 100644 --- a/Charts/dev-c7/templates/deployment.yaml +++ b/Charts/dev-c7/templates/deployment.yaml @@ -39,6 +39,10 @@ spec: - "-c" - "sleep infinity" workingDir: /dls_sw/prod/R3.14.12.7/ioc/BL16I/BL16I-MO-IOC-12/21-2 + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} env: - name: HOME value: /home/{{ .Values.userName }} diff --git a/Charts/dev-c7/values.schema.json b/Charts/dev-c7/values.schema.json new file mode 100644 index 0000000..bb86fa3 --- /dev/null +++ b/Charts/dev-c7/values.schema.json @@ -0,0 +1,122 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "dev-c7 Helm chart", + "description": "Helm chart for deploying a dev-c7 application", + "type": "object", + "properties": { + "affinity": { + "$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.33.3/affinity.json", + "type": "object" + }, + "fullnameOverride": { + "type": "string" + }, + "global": { + "description": "shared values for all services", + "type": "object", + "additionalProperties": true + }, + "hostNetwork": { + "description": "enable host networking for the pod", + "type": "boolean" + }, + "image": { + "description": "container image URI", + "type": "object", + "properties": { + "pullPolicy": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "tag": { + "type": "string" + } + } + }, + "imagePullSecrets": { + "type": "array" + }, + "nameOverride": { + "type": "string" + }, + "nodeSelector": { + "$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.33.3/podspec.json#/properties/nodeSelector", + "type": "object" + }, + "podAnnotations": { + "description": "Add annotations to the pod", + "type": "object" + }, + "podLabels": { + "description": "Add labels to the pod", + "type": "object" + }, + "podSecurityContext": { + "$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.33.3/podspec.json#/properties/securityContext", + "type": "object" + }, + "replicaCount": { + "description": "Set to 0 to stop the IOC, 1 to run the IOC", + "type": "integer" + }, + "resources": { + "$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.33.3/container.json#/properties/resources", + "type": "object" + }, + "securityContext": { + "$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.33.3/container.json#/properties/securityContext", + "type": "object" + }, + "tolerations": { + "$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.33.3/podspec.json#/properties/tolerations", + "type": "array" + }, + "userId": { + "type": "integer" + }, + "userName": { + "type": "string" + }, + "volumeMounts": { + "$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.33.3/container.json#/properties/volumeMounts", + "type": "array", + "items": { + "type": "object", + "properties": { + "mountPath": { + "type": "string" + }, + "name": { + "type": "string" + }, + "readOnly": { + "type": "boolean" + } + } + } + }, + "volumes": { + "$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.33.3/podspec.json#/properties/volumes", + "type": "array", + "items": { + "type": "object", + "properties": { + "hostPath": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + }, + "name": { + "type": "string" + } + } + } + } + }, + "additionalProperties": false +} diff --git a/Charts/dev-c7/values.yaml b/Charts/dev-c7/values.yaml index 2e70e73..9fe6ed5 100644 --- a/Charts/dev-c7/values.yaml +++ b/Charts/dev-c7/values.yaml @@ -1,11 +1,21 @@ -# Default values for dls-k8s-c7. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. +# yaml-language-server: $schema=values.schema.json -# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ +# NOTE: the $k8s schema references come from +# https://github.com/yannh/kubernetes-json-schema/tree/master/v1.33.1 +# with the version set in .fastcs.schema.config.yaml (k8sSchemaVersion) + +# Default values for dev-c7 chart. +# With annotations for building the schema using: +# https://github.com/losisin/helm-values-schema-json.git + +# @schema description: shared values for all services +# @schema additionalProperties: true +global: {} + +# @schema description: Set to 0 to stop the IOC, 1 to run the IOC replicaCount: 1 -# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ +# @schema description: container image URI image: repository: ghcr.io/diamondlightsource/dev-c7 # This sets the pull policy for images. @@ -17,7 +27,7 @@ image: userId: 1200288 userName: hgv27681 -# set to true for Channel access to be served on the locl subnet. +# @schema description: enable host networking for the pod hostNetwork: false # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ @@ -26,31 +36,23 @@ imagePullSecrets: [] nameOverride: "" fullnameOverride: "" -# This is for setting Kubernetes Annotations to a Pod. -# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +# @schema description: Add annotations to the pod podAnnotations: {} -# This is for setting Kubernetes Labels to a Pod. -# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + +# @schema description: Add labels to the pod podLabels: {} -podSecurityContext: - {} - # fsGroup: 2000 - -resources: - {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi +# @schema $ref: $k8s/podspec.json#/properties/securityContext +podSecurityContext: {} + +# @schema $ref:$k8s/container.json#/properties/securityContext +securityContext: {} + +# @schema $ref: $k8s/container.json#/properties/resources +resources: {} # volumes for the Deployment. +# @schema $ref: $k8s/podspec.json#/properties/volumes volumes: - name: work hostPath: @@ -75,6 +77,7 @@ volumes: path: /home/hgv27681 # volumeMounts for the Deployment. +# @schema $ref: $k8s/container.json#/properties/volumeMounts volumeMounts: - name: work mountPath: /dls_sw/work @@ -97,8 +100,11 @@ volumeMounts: - name: home mountPath: /home/hgv27681 -nodeSelector: {} - +# @schema $ref: $k8s/podspec.json#/properties/tolerations tolerations: [] +# @schema $ref: $k8s/podspec.json#/properties/nodeSelector +nodeSelector: {} + +# @schema $ref: $k8s/affinity.json affinity: {} diff --git a/schema/dev-c7.values.schema.json b/schema/dev-c7.values.schema.json new file mode 120000 index 0000000..ccabb52 --- /dev/null +++ b/schema/dev-c7.values.schema.json @@ -0,0 +1 @@ +../Charts/dev-c7/values.schema.json \ No newline at end of file From 784b314bd86ca828c24d5e2416f333d136ea074d Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Fri, 5 Sep 2025 08:17:04 +0100 Subject: [PATCH 11/13] fix tag schema in helm action --- .github/workflows/_helm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_helm.yml b/.github/workflows/_helm.yml index 56ea1b1..6b53e9d 100644 --- a/.github/workflows/_helm.yml +++ b/.github/workflows/_helm.yml @@ -28,7 +28,7 @@ jobs: env: VERSION: "${{ github.ref_name }}" run: | - if [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-alpha|-beta|-rc).*?$ ]]; then + if [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+((-alpha|-beta|-rc).*)?$ ]]; then echo "Valid version format: ${VERSION}" else echo "Invalid version: ${VERSION}. Expected: X.Y.Z or X.Y.Z-beta.1 or X.Y.Z-alpha.1" From 186d53b285c74070b51d30a6d62a03c8de50067c Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Fri, 5 Sep 2025 08:19:05 +0100 Subject: [PATCH 12/13] add helm plugin to tox action --- .github/workflows/_tox.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_tox.yml b/.github/workflows/_tox.yml index a13536d..c0abaae 100644 --- a/.github/workflows/_tox.yml +++ b/.github/workflows/_tox.yml @@ -6,7 +6,6 @@ on: description: What to run under tox required: true - jobs: run: runs-on: "ubuntu-latest" @@ -18,5 +17,8 @@ jobs: - name: Install python packages uses: ./.github/actions/install_requirements + - name: Install helm plugins + run: helm plugin install https://github.com/losisin/helm-values-schema-json.git + - name: Run tox run: tox -e ${{ inputs.tox }} From 20176a7bfc546d9b489e696562456bd19364cae5 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Fri, 5 Sep 2025 08:27:45 +0100 Subject: [PATCH 13/13] fix name of schemas folder --- {schema => schemas}/dev-c7.values.schema.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {schema => schemas}/dev-c7.values.schema.json (100%) diff --git a/schema/dev-c7.values.schema.json b/schemas/dev-c7.values.schema.json similarity index 100% rename from schema/dev-c7.values.schema.json rename to schemas/dev-c7.values.schema.json