diff --git a/medcat-trainer/docker-compose-dev.yml b/medcat-trainer/docker-compose-dev.yml index 908093ac7..340941e69 100644 --- a/medcat-trainer/docker-compose-dev.yml +++ b/medcat-trainer/docker-compose-dev.yml @@ -84,5 +84,5 @@ volumes: networks: gateway-auth_gateway-net: + name: ${MCT_GATEWAY_NETWORK_NAME:-gateway-auth_gateway-net} external: true - diff --git a/medcat-trainer/docs/installation.md b/medcat-trainer/docs/installation.md index c97975135..13a1bde92 100644 --- a/medcat-trainer/docs/installation.md +++ b/medcat-trainer/docs/installation.md @@ -19,6 +19,7 @@ services: - Docker Engine - Docker Compose v2 (`docker compose` command) +- `uv` (only needed for the local Django debug script) ## Quick start (prebuilt images) @@ -47,6 +48,56 @@ docker compose -f docker-compose-dev.yml up --build This uses the local `webapp/` source tree and is the recommended setup for development work. +### Local Django debug script + +For backend development where you want to run Django directly on your host +machine, use the local debug helper: + +```bash +./webapp/scripts/run_local_debug.sh +``` + +The script sources `envs/env`, syncs Python dependencies with `uv` when needed, +runs migrations, creates a local admin user, ensures the default user group +exists, and starts Django at `http://127.0.0.1:8001/`. + +By default it also starts only the `solr` service with Docker Compose: + +```bash +docker compose -f docker-compose-dev.yml up -d solr +``` + +Before starting Solr, the script ensures the Compose gateway network exists. It +uses `gateway-auth_gateway-net` by default, creating it if it is missing. If you +are running MedCATtrainer alongside a different gateway stack, set the network +name explicitly: + +```bash +MCT_GATEWAY_NETWORK_NAME=my-gateway-net ./webapp/scripts/run_local_debug.sh +``` + +Available modes: + +```bash +./webapp/scripts/run_local_debug.sh server +./webapp/scripts/run_local_debug.sh worker +./webapp/scripts/run_local_debug.sh shell +./webapp/scripts/run_local_debug.sh bootstrap +``` + +Common overrides: + +| Variable | Description | +|---|---| +| `MCT_DEBUG_HOST` | Django bind host (default `0.0.0.0`). | +| `MCT_DEBUG_PORT` | Django port (default `8001`). | +| `MCT_ENV_FILE` | Env file to source instead of `envs/env`. | +| `MCT_ADMIN_USERNAME` | Local admin username (default `admin`). | +| `MCT_ADMIN_PASSWORD` | Local admin password (default `admin`). | +| `MCT_START_SOLR` | Start Solr through Docker Compose (`1`/`0`, default `1`). | +| `MCT_GATEWAY_NETWORK_NAME` | External Compose network to use/create for Solr. | +| `MCT_SYNC_DEPS` | Run `uv sync --frozen` (`1`/`0`/`auto`, default `auto`). | + ## Legacy MedCAT v0.x support If you still need the legacy MedCAT v0.x-compatible stack: @@ -141,4 +192,4 @@ An example compose file is available at for the selected CDB. ## Next Steps -Now that medcat trainer is installed and running, proceed to [Administrator Setup](admin_setup.md) to create the Admin user. \ No newline at end of file +Now that medcat trainer is installed and running, proceed to [Administrator Setup](admin_setup.md) to create the Admin user. diff --git a/medcat-trainer/envs/env b/medcat-trainer/envs/env index 06c7eb879..1f2a45c2f 100644 --- a/medcat-trainer/envs/env +++ b/medcat-trainer/envs/env @@ -2,7 +2,10 @@ OPENBLAS_NUM_THREADS=1 ### MedCAT cfg ### + +# when in debug mode please use ../../configs/base.txt MEDCAT_CONFIG_FILE=/home/configs/base.txt + # number of MedCAT models that can be cached, run in bg processes at any one time MAX_MEDCAT_MODELS=2 @@ -12,9 +15,12 @@ ENV=non-prod # Complete once this is deployed CSRF_TRUSTED_ORIGINS= +USE_OIDC=0 + ### Django debug setting - to live-reload etc. ### DEBUG=1 + ### Load example CDB, Vocab ### LOAD_EXAMPLES=1 # URL that examples will be sent to @@ -25,7 +31,7 @@ UNIQUE_DOC_NAMES_IN_DATASETS=True MAX_DATASET_SIZE=10000 ### Solr Concept Search Conf ### -CONCEPT_SEARCH_SERVICE_HOST=solr +CONCEPT_SEARCH_SERVICE_HOST=localhost CONCEPT_SEARCH_SERVICE_PORT=8983 ### DB backup dir ### @@ -60,4 +66,4 @@ OTEL_METRICS_EXPORTER=none OTEL_LOGS_EXPORTER=none OTEL_PYTHON_DJANGO_EXCLUDED_URLS=/api/health/*,/metrics OTEL_EXPERIMENTAL_RESOURCE_DETECTORS=containerid,os -OTEL_PYTHON_DISABLED_INSTRUMENTATIONS=sqlite3,psycopg \ No newline at end of file +OTEL_PYTHON_DISABLED_INSTRUMENTATIONS=sqlite3,psycopg diff --git a/medcat-trainer/webapp/frontend/src/mixins/ConceptDetailService.js b/medcat-trainer/webapp/frontend/src/mixins/ConceptDetailService.js index f2d43feeb..09442dbb1 100644 --- a/medcat-trainer/webapp/frontend/src/mixins/ConceptDetailService.js +++ b/medcat-trainer/webapp/frontend/src/mixins/ConceptDetailService.js @@ -19,12 +19,21 @@ export default { } }, fetchConcept (selectedEnt, cdbSearchIndex, callback) { - this.$http.get(`/api/concepts/${cdbSearchIndex}/select?q=cui:${selectedEnt.cui}`).then(resp => { - if (selectedEnt && resp.data.response.docs.length > 0) { - const docEnt = resp.data.response.docs[0] + const cdbs = this.conceptSearchCdbs(cdbSearchIndex) + if (!cdbs) { + if (callback) { + callback() + } + return + } + const query = `search=${encodeURIComponent(selectedEnt.cui)}&cdbs=${encodeURIComponent(cdbs)}` + this.$http.get(`/api/search-concepts/?${query}`).then(resp => { + const results = resp.data?.results || [] + if (selectedEnt && results.length > 0) { + const docEnt = results[0] selectedEnt.desc = docEnt.desc selectedEnt.type_ids = docEnt.type_ids - selectedEnt.pretty_name = docEnt.pretty_name[0] + selectedEnt.pretty_name = Array.isArray(docEnt.pretty_name) ? docEnt.pretty_name[0] : docEnt.pretty_name selectedEnt.synonyms = docEnt.synonyms if ((docEnt.icd10 || []).length > 0) { selectedEnt.icd10 = [] @@ -67,6 +76,17 @@ export default { callback() } }) + }, + conceptSearchCdbs (cdbSearchIndex) { + if (Array.isArray(cdbSearchIndex)) { + return cdbSearchIndex.filter(id => id).join(',') + } + if (!cdbSearchIndex) { + return null + } + const searchIndex = String(cdbSearchIndex) + const collectionMatch = searchIndex.match(/_id_(.+)$/) + return collectionMatch ? collectionMatch[1] : searchIndex } } } diff --git a/medcat-trainer/webapp/frontend/src/tests/components/ConceptSummary.spec.ts b/medcat-trainer/webapp/frontend/src/tests/components/ConceptSummary.spec.ts new file mode 100644 index 000000000..f3fec943d --- /dev/null +++ b/medcat-trainer/webapp/frontend/src/tests/components/ConceptSummary.spec.ts @@ -0,0 +1,161 @@ +import { describe, expect, it, vi } from 'vitest' +import { flushPromises, mount } from '@vue/test-utils' +import ConceptSummary from '@/components/common/ConceptSummary.vue' + +describe('ConceptSummary.vue', () => { + it('shows the CUI after fetching the entity label when no concept search index is configured', async () => { + const mockGet = vi.fn((url: string) => { + if (url === '/api/entities/10/') { + return Promise.resolve({ data: { label: 'C0022660' } }) + } + return Promise.reject(new Error(`Unexpected request: ${url}`)) + }) + + const wrapper = mount(ConceptSummary, { + props: { + project: {}, + selectedEnt: null, + altSearch: false, + searchFilterDBIndex: null + }, + global: { + mocks: { + $http: { get: mockGet } + }, + stubs: { + 'concept-picker': true, + 'font-awesome-icon': true + } + } + }) + + await wrapper.setProps({ + selectedEnt: { + id: 1, + entity: 10, + value: 'acute kidney failure', + start_ind: 10, + end_ind: 30, + acc: 0.99, + assignedValues: { + 'Concept Annotation': null + }, + deleted: false + } + }) + await flushPromises() + + expect(mockGet).toHaveBeenCalledTimes(1) + expect(mockGet).toHaveBeenCalledWith('/api/entities/10/') + expect(wrapper.text()).toContain('C0022660') + }) + + it('shows the CUI when the concept lookup response has no results', async () => { + const mockGet = vi.fn((url: string) => { + if (url === '/api/entities/10/') { + return Promise.resolve({ data: { label: 'C0022660' } }) + } + if (url === '/api/search-concepts/?search=C0022660&cdbs=1') { + return Promise.resolve({ data: { results: [] } }) + } + return Promise.reject(new Error(`Unexpected request: ${url}`)) + }) + + const wrapper = mount(ConceptSummary, { + props: { + project: {}, + selectedEnt: null, + altSearch: false, + searchFilterDBIndex: '1' + }, + global: { + mocks: { + $http: { get: mockGet } + }, + stubs: { + 'concept-picker': true, + 'font-awesome-icon': true + } + } + }) + + await wrapper.setProps({ + selectedEnt: { + id: 1, + entity: 10, + value: 'acute kidney failure', + start_ind: 10, + end_ind: 30, + acc: 0.99, + assignedValues: { + 'Concept Annotation': null + }, + deleted: false + } + }) + await flushPromises() + + expect(mockGet).toHaveBeenCalledTimes(2) + expect(wrapper.text()).toContain('C0022660') + }) + + it('shows concept details returned by the backend concept search endpoint', async () => { + const mockGet = vi.fn((url: string) => { + if (url === '/api/entities/10/') { + return Promise.resolve({ data: { label: 'C0022660' } }) + } + if (url === '/api/search-concepts/?search=C0022660&cdbs=1') { + return Promise.resolve({ + data: { + results: [{ + cui: 'C0022660', + pretty_name: 'Acute kidney failure', + type_ids: ['T047'], + synonyms: ['AKF'] + }] + } + }) + } + return Promise.reject(new Error(`Unexpected request: ${url}`)) + }) + + const wrapper = mount(ConceptSummary, { + props: { + project: {}, + selectedEnt: null, + altSearch: false, + searchFilterDBIndex: '1' + }, + global: { + mocks: { + $http: { get: mockGet } + }, + stubs: { + 'concept-picker': true, + 'font-awesome-icon': true + } + } + }) + + await wrapper.setProps({ + selectedEnt: { + id: 1, + entity: 10, + value: 'acute kidney failure', + start_ind: 10, + end_ind: 30, + acc: 0.99, + assignedValues: { + 'Concept Annotation': null + }, + deleted: false + } + }) + await flushPromises() + + expect(mockGet).toHaveBeenCalledTimes(2) + expect(wrapper.text()).toContain('Acute kidney failure') + expect(wrapper.text()).toContain('T047') + expect(wrapper.text()).toContain('C0022660') + }) +}) diff --git a/medcat-trainer/webapp/frontend/src/views/Demo.vue b/medcat-trainer/webapp/frontend/src/views/Demo.vue index b7d825c75..c36854eca 100644 --- a/medcat-trainer/webapp/frontend/src/views/Demo.vue +++ b/medcat-trainer/webapp/frontend/src/views/Demo.vue @@ -222,11 +222,7 @@ export default { }, fetchCDBSearchIndex () { if (this.selectedProject?.cdb_search_filter?.length > 0) { - this.$http.get(`/api/concept-dbs/${this.selectedProject.cdb_search_filter[0]}/`).then(resp => { - if (resp.data) { - this.searchFilterDBIndex = `${resp.data.name}_id_${this.selectedProject.cdb_search_filter}` - } - }) + this.searchFilterDBIndex = this.selectedProject.cdb_search_filter.join(',') } else { this.searchFilterDBIndex = null } diff --git a/medcat-trainer/webapp/frontend/src/views/TrainAnnotations.vue b/medcat-trainer/webapp/frontend/src/views/TrainAnnotations.vue index 087acb455..a3af3aa3a 100644 --- a/medcat-trainer/webapp/frontend/src/views/TrainAnnotations.vue +++ b/medcat-trainer/webapp/frontend/src/views/TrainAnnotations.vue @@ -466,11 +466,9 @@ export default { }, fetchCDBSearchIndex() { if (this.project.cdb_search_filter.length > 0) { - this.$http.get(`/api/concept-dbs/${this.project.cdb_search_filter[0]}/`).then(resp => { - if (resp.data) { - this.searchFilterDBIndex = `${resp.data.name}_id_${this.project.cdb_search_filter}` - } - }) + this.searchFilterDBIndex = this.project.cdb_search_filter.join(',') + } else { + this.searchFilterDBIndex = null } }, loadDoc(doc) { diff --git a/medcat-trainer/webapp/frontend/vite.config.ts b/medcat-trainer/webapp/frontend/vite.config.ts index 5698c38f2..79b78db1d 100644 --- a/medcat-trainer/webapp/frontend/vite.config.ts +++ b/medcat-trainer/webapp/frontend/vite.config.ts @@ -2,13 +2,11 @@ import { fileURLToPath, URL } from 'node:url' import { defineConfig } from 'vite' import vue from '@vitejs/plugin-vue' -import vueDevTools from 'vite-plugin-vue-devtools' // https://vite.dev/config/ export default defineConfig({ plugins: [ vue(), - vueDevTools(), ], resolve: { alias: { @@ -37,7 +35,7 @@ export default defineConfig({ target: 'http://127.0.0.1:8983/solr', changeOrigin: true, secure: false, - rewrite: (path) => path.replace(/\/api\/concepts/, '/') + rewrite: (path: string) => path.replace(/\/api\/concepts/, '/') }, '^/api/*': { target: 'http://127.0.0.1:8001' diff --git a/medcat-trainer/webapp/pyproject.toml b/medcat-trainer/webapp/pyproject.toml index 727203719..2d467e615 100644 --- a/medcat-trainer/webapp/pyproject.toml +++ b/medcat-trainer/webapp/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "djangorestframework>=3.16,<4", "django-background-tasks-updated>=1.2", "openpyxl>=3.1", - "medcat[meta-cat,spacy,rel-cat,deid]>=2.3", + "medcat[meta-cat,spacy,rel-cat,deid,dict-ner]>=2.7.0", "psycopg[binary,pool]>=3.2", "cryptography>=45", "drf-oidc-auth>=3.0", diff --git a/medcat-trainer/webapp/scripts/run_local_debug.sh b/medcat-trainer/webapp/scripts/run_local_debug.sh new file mode 100755 index 000000000..1939010af --- /dev/null +++ b/medcat-trainer/webapp/scripts/run_local_debug.sh @@ -0,0 +1,175 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WEBAPP_DIR="$(cd "$ROOT_DIR/.." && pwd)" +API_DIR="$WEBAPP_DIR/api" +ENV_FILE="${MCT_ENV_FILE:-$ROOT_DIR/../../envs/env}" + +MODE="${1:-server}" + +usage() { + cat <<'EOF' +Usage: + ./run_local_debug.sh [server|worker|shell|bootstrap] + +Modes: + server Bootstrap local DB, then run Django on 0.0.0.0:8001. + worker Bootstrap local DB, then run background-task worker. + shell Bootstrap local DB, then open Django shell. + bootstrap Run local bootstrap only, then exit. + +App settings are sourced from: + envs/env + +Script-only overrides: + MCT_DEBUG_HOST=127.0.0.1 + MCT_DEBUG_PORT=8001 + MCT_ENV_FILE=/path/to/env + MCT_ADMIN_USERNAME=admin + MCT_ADMIN_PASSWORD=admin + MCT_BOOTSTRAP_ADMIN=0 + MCT_RESET_ADMIN_PASSWORD=0 + MCT_RUN_MIGRATIONS=0 + MCT_START_SOLR=1 + MCT_GATEWAY_NETWORK_NAME=gateway-auth_gateway-net + MCT_SYNC_DEPS=1 +EOF +} + +if [[ "$MODE" == "-h" || "$MODE" == "--help" ]]; then + usage + exit 0 +fi + +if ! command -v uv >/dev/null 2>&1; then + echo "uv is required but was not found on PATH." >&2 + exit 1 +fi + +if [[ ! -f "$ENV_FILE" ]]; then + echo "Env file not found: $ENV_FILE" >&2 + exit 1 +fi + +set -a +source "$ENV_FILE" +set +a + +export UV_PROJECT="$WEBAPP_DIR" +export PYTHONUNBUFFERED="${PYTHONUNBUFFERED:-1}" + +HOST="${MCT_DEBUG_HOST:-0.0.0.0}" +PORT="${MCT_DEBUG_PORT:-8001}" + +ADMIN_USERNAME="${MCT_ADMIN_USERNAME:-admin}" +ADMIN_EMAIL="${MCT_ADMIN_EMAIL:-admin@example.com}" +ADMIN_PASSWORD="${MCT_ADMIN_PASSWORD:-admin}" +BOOTSTRAP_ADMIN="${MCT_BOOTSTRAP_ADMIN:-1}" +RESET_ADMIN_PASSWORD="${MCT_RESET_ADMIN_PASSWORD:-1}" +RUN_MIGRATIONS="${MCT_RUN_MIGRATIONS:-1}" +BOOTSTRAP_GROUP="${MCT_BOOTSTRAP_GROUP:-1}" +SYNC_DEPS="${MCT_SYNC_DEPS:-auto}" +START_SOLR="${MCT_START_SOLR:-1}" +GATEWAY_NETWORK_NAME="${MCT_GATEWAY_NETWORK_NAME:-gateway-auth_gateway-net}" +export MCT_GATEWAY_NETWORK_NAME="$GATEWAY_NETWORK_NAME" + +run_manage() { + (cd "$API_DIR" && uv run python manage.py "$@") +} + +ensure_docker_network() { + local network_name="$1" + + if docker network inspect "$network_name" >/dev/null 2>&1; then + return + fi + + echo "Creating Docker network: $network_name" + docker network create "$network_name" >/dev/null || docker network inspect "$network_name" >/dev/null +} + +if [[ "$SYNC_DEPS" == "1" || ( "$SYNC_DEPS" == "auto" && ! -d "$WEBAPP_DIR/.venv" ) ]]; then + echo "Syncing Python dependencies with uv..." + (cd "$WEBAPP_DIR" && uv sync --frozen) +fi + +if [[ "$START_SOLR" == "1" ]]; then + if ! command -v docker >/dev/null 2>&1; then + echo "MCT_START_SOLR=1 requires docker, but docker was not found on PATH." >&2 + exit 1 + fi + ensure_docker_network "$GATEWAY_NETWORK_NAME" + echo "Starting Solr with docker compose..." + (cd "$ROOT_DIR/../../" && docker compose -f docker-compose-dev.yml up -d solr) +fi + +if command -v curl >/dev/null 2>&1; then + if ! curl -fsS "http://${CONCEPT_SEARCH_SERVICE_HOST}:${CONCEPT_SEARCH_SERVICE_PORT}/solr/admin/info/system?wt=json" >/dev/null 2>&1; then + echo "Warning: Solr did not respond at http://${CONCEPT_SEARCH_SERVICE_HOST}:${CONCEPT_SEARCH_SERVICE_PORT}/solr" >&2 + echo " Concept search/model setup may fail until Solr is running." >&2 + fi +fi + +mkdir -p "$API_DIR/static" +cat > "$API_DIR/static/config.json" <\"}') +" +fi + +if [[ "$BOOTSTRAP_GROUP" == "1" ]]; then + echo "Ensuring default user_group exists..." + (cd "$API_DIR" && uv run python manage.py shell < "$WEBAPP_DIR/scripts/create_group.py") +fi + +case "$MODE" in + bootstrap) + echo "Local bootstrap complete." + ;; + shell) + run_manage shell + ;; + worker) + echo "Starting local background-task worker..." + run_manage process_tasks --log-std + ;; + server) + echo "Starting local Django debug server at http://127.0.0.1:${PORT}/" + echo "Login: ${ADMIN_USERNAME} / ${ADMIN_PASSWORD}" + run_manage runserver "${HOST}:${PORT}" + ;; +esac