From cf01e608c7daebf0896687a2ab1d45de50b7286c Mon Sep 17 00:00:00 2001 From: Jean-Denis Courcol Date: Wed, 18 Jun 2025 15:32:41 +0200 Subject: [PATCH 1/5] fix conflicts --- app/cli/import_data.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/cli/import_data.py b/app/cli/import_data.py index f0d740cb..57052d84 100644 --- a/app/cli/import_data.py +++ b/app/cli/import_data.py @@ -83,6 +83,14 @@ DEFAULT_REGION_ID_FOR_ROOT = uuid.UUID("eb1167b3-67a9-4378-bc65-c1e582e2e662") + +# keep uuid used by core-web-app constant +DEFAULT_HIERARCHY_ID = uuid.UUID("e3e70682-c209-4cac-a29f-6fbed82c07cd") +DEFAULT_REGION_ID_FOR_BASIC_CELL_GROUPS_AND_REGIONS = uuid.UUID("4642cddb-4fbe-4aae-bbf7-0946d6ada066") +DEFAULT_BRAIN_ATLAS_ID = uuid.UUID("55de9d7b-9796-41f9-b719-213c3305ffd7") +DEFAULT_REGION_ID_FOR_ROOT = uuid.UUID("eb1167b3-67a9-4378-bc65-c1e582e2e662") + + BRAIN_ATLAS_NAME = "BlueBrain Atlas" REQUIRED_PATH = click.Path(exists=True, readable=True, dir_okay=False, resolve_path=True) From e40cb98e5c9e481c936b98cbf550e1e386b58498 Mon Sep 17 00:00:00 2001 From: Jean-Denis Courcol Date: Wed, 18 Jun 2025 15:33:15 +0200 Subject: [PATCH 2/5] complete conflict --- app/cli/import_data.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/cli/import_data.py b/app/cli/import_data.py index 57052d84..13dc9aca 100644 --- a/app/cli/import_data.py +++ b/app/cli/import_data.py @@ -83,10 +83,11 @@ DEFAULT_REGION_ID_FOR_ROOT = uuid.UUID("eb1167b3-67a9-4378-bc65-c1e582e2e662") - -# keep uuid used by core-web-app constant +# keep uuid used by core-web-app constant DEFAULT_HIERARCHY_ID = uuid.UUID("e3e70682-c209-4cac-a29f-6fbed82c07cd") -DEFAULT_REGION_ID_FOR_BASIC_CELL_GROUPS_AND_REGIONS = uuid.UUID("4642cddb-4fbe-4aae-bbf7-0946d6ada066") +DEFAULT_REGION_ID_FOR_BASIC_CELL_GROUPS_AND_REGIONS = uuid.UUID( + "4642cddb-4fbe-4aae-bbf7-0946d6ada066" +) DEFAULT_BRAIN_ATLAS_ID = uuid.UUID("55de9d7b-9796-41f9-b719-213c3305ffd7") DEFAULT_REGION_ID_FOR_ROOT = uuid.UUID("eb1167b3-67a9-4378-bc65-c1e582e2e662") From 522933bf753b7ad98c21cd6b619adba40decf685 Mon Sep 17 00:00:00 2001 From: Jean-Denis Courcol Date: Wed, 18 Jun 2025 15:37:22 +0200 Subject: [PATCH 3/5] complete merge --- app/cli/import_data.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/app/cli/import_data.py b/app/cli/import_data.py index 13dc9aca..f0d740cb 100644 --- a/app/cli/import_data.py +++ b/app/cli/import_data.py @@ -83,15 +83,6 @@ DEFAULT_REGION_ID_FOR_ROOT = uuid.UUID("eb1167b3-67a9-4378-bc65-c1e582e2e662") -# keep uuid used by core-web-app constant -DEFAULT_HIERARCHY_ID = uuid.UUID("e3e70682-c209-4cac-a29f-6fbed82c07cd") -DEFAULT_REGION_ID_FOR_BASIC_CELL_GROUPS_AND_REGIONS = uuid.UUID( - "4642cddb-4fbe-4aae-bbf7-0946d6ada066" -) -DEFAULT_BRAIN_ATLAS_ID = uuid.UUID("55de9d7b-9796-41f9-b719-213c3305ffd7") -DEFAULT_REGION_ID_FOR_ROOT = uuid.UUID("eb1167b3-67a9-4378-bc65-c1e582e2e662") - - BRAIN_ATLAS_NAME = "BlueBrain Atlas" REQUIRED_PATH = click.Path(exists=True, readable=True, dir_okay=False, resolve_path=True) From 9be42d92ca035258f378a551a9dd5ca640a0a979 Mon Sep 17 00:00:00 2001 From: Jean-Denis Courcol Date: Wed, 18 Jun 2025 13:41:43 +0200 Subject: [PATCH 4/5] use cell_composition.json to generate leaves uuids --- Makefile | 4 ++-- app/cli/import_data.py | 20 +++++++++++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index b71623b1..b7627a41 100644 --- a/Makefile +++ b/Makefile @@ -55,8 +55,8 @@ import: ## Run the import on a database, assumes mba_hierarchy.json and out are @test -n "$(VIRTUAL_LAB_ID_IMPORT)" || (echo "Please set the variable VIRTUAL_LAB_ID_IMPORT"; exit 1) docker compose up --wait db uv run -m alembic upgrade head - uv run -m app.cli.import_data --seed 1 hierarchy $(HIERARCHY_NAME) mba_hierarchy.json - uv run -m app.cli.import_data --seed 2 run ./out --virtual-lab-id $(VIRTUAL_LAB_ID_IMPORT) --project-id $(PROJECT_ID_IMPORT) --hierarchy-name $(HIERARCHY_NAME) + uv run -m app.cli.import_data --seed 2 hierarchy $(HIERARCHY_NAME) mba_hierarchy.json --cell-composition-file cell_composition.json + uv run -m app.cli.import_data --seed 3 run ./out --virtual-lab-id $(VIRTUAL_LAB_ID_IMPORT) --project-id $(PROJECT_ID_IMPORT) --hierarchy-name $(HIERARCHY_NAME) curate-files: ## Create curated files and save them into the curated directory @$(call load_env,run-local) diff --git a/app/cli/import_data.py b/app/cli/import_data.py index f0d740cb..94ff9b31 100644 --- a/app/cli/import_data.py +++ b/app/cli/import_data.py @@ -77,7 +77,7 @@ # keep uuid used by core-web-app constant DEFAULT_HIERARCHY_ID = uuid.UUID("e3e70682-c209-4cac-a29f-6fbed82c07cd") DEFAULT_REGION_ID_FOR_BASIC_CELL_GROUPS_AND_REGIONS = uuid.UUID( - "4642cddb-4fbe-4aae-bbf7-0946d6ada066" + "fd9953d2-32a4-4a98-859b-64de31d0af99" ) DEFAULT_BRAIN_ATLAS_ID = uuid.UUID("55de9d7b-9796-41f9-b719-213c3305ffd7") DEFAULT_REGION_ID_FOR_ROOT = uuid.UUID("eb1167b3-67a9-4378-bc65-c1e582e2e662") @@ -86,6 +86,7 @@ BRAIN_ATLAS_NAME = "BlueBrain Atlas" REQUIRED_PATH = click.Path(exists=True, readable=True, dir_okay=False, resolve_path=True) +OPTIONAL_PATH = click.Path(exists=False, readable=True, dir_okay=False, resolve_path=True) REQUIRED_PATH_DIR = click.Path( exists=True, readable=True, file_okay=False, dir_okay=True, resolve_path=True ) @@ -1922,7 +1923,12 @@ def run(input_dir, virtual_lab_id, project_id, hierarchy_name): @cli.command() @click.argument("hierarchy_name", type=str) @click.argument("hierarchy_path", type=REQUIRED_PATH) -def hierarchy(hierarchy_name, hierarchy_path): +@click.option( + "--cell-composition-file", + type=OPTIONAL_PATH, + help="a composition json file to preserve brain region ids", +) +def hierarchy(hierarchy_name, hierarchy_path, cell_composition_file=None): """Load a hierarchy.json.""" with open(hierarchy_path) as fd: @@ -1941,6 +1947,13 @@ def recurse(i): regions.append(item) recurse(hierarchy) + brain_region_mapping = {} + if cell_composition_file: + with open(cell_composition_file) as fd: + cell_composition = json.load(fd) + brain_region_mapping = { + v["name"]: uuid.UUID(k) for k, v in cell_composition["hasPart"].items() + } with ( closing(configure_database_session_manager(**SQLA_ENGINE_ARGS)) as database_session_manager, @@ -1982,7 +1995,8 @@ def recurse(i): elif region["id"] == 8: brain_region_id = DEFAULT_REGION_ID_FOR_BASIC_CELL_GROUPS_AND_REGIONS else: - brain_region_id = uuid.uuid4() + brain_region_id = brain_region_mapping.get(region["name"], uuid.uuid4()) + db_br = BrainRegion( id=brain_region_id, annotation_value=region["id"], From 08e993d6d3a5669ed863bfded69d336e40fd0536 Mon Sep 17 00:00:00 2001 From: Jean-Denis Courcol Date: Wed, 18 Jun 2025 13:46:10 +0200 Subject: [PATCH 5/5] add cell composition url --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 28277329..d6dcbaa2 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,11 @@ https://openbraininstitute.sharepoint.com/:u:/s/OpenBrainInstitute/EZATMPEZvN5Cj $ tar xzf out.tar.gz ``` +- optionally, donwload the cell composition from: +``` +https://staging.openbraininstitute.org/api/entitycore/cell-composition +``` + - Import data into the specified database: ```