From df546365ab14a6e628b23d29f0ffb54cb19403a1 Mon Sep 17 00:00:00 2001
From: DriesSchaumont <5946712+DriesSchaumont@users.noreply.github.com>
Date: Tue, 25 Mar 2025 13:36:13 +0000
Subject: [PATCH 1/2] Fix leiden hanging when insufficient shared memory
---
CHANGELOG.md | 2 ++
src/cluster/leiden/config.vsh.yaml | 2 +-
src/cluster/leiden/script.py | 16 +++++++++++++++-
3 files changed, 18 insertions(+), 2 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ca7a0c70875..0d34454025d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -50,6 +50,8 @@
# BUG FIXES
+* `cluster/leiden`: Fix an issue where insufficient shared memory (size of `/dev/shm`) causes the processing to hang.
+
* `utils/subset_vars`: Convert .var column used for subsetting of dtype "boolean" to dtype "bool" when it doesn't contain NaN values (PR #959).
* `resources_test_scripts/annotation_test_data.sh`: Add a layer to the annotation reference dataset with log normalized counts (PR #960).
diff --git a/src/cluster/leiden/config.vsh.yaml b/src/cluster/leiden/config.vsh.yaml
index 1b06cdc6076..e32a35e81e3 100644
--- a/src/cluster/leiden/config.vsh.yaml
+++ b/src/cluster/leiden/config.vsh.yaml
@@ -79,7 +79,7 @@ test_resources:
engines:
- type: docker
- image: python:3.11-slim
+ image: python:3.12-slim
setup:
- type: apt
packages:
diff --git a/src/cluster/leiden/script.py b/src/cluster/leiden/script.py
index 03a39be5802..91137385adc 100644
--- a/src/cluster/leiden/script.py
+++ b/src/cluster/leiden/script.py
@@ -45,7 +45,7 @@
"uns_name": "leiden",
"output_compression": "gzip",
}
-meta = {"cpus": 8, "resources_dir": "."}
+meta = {"cpus": 8, "resources_dir": "src/utils"}
## VIASH END
sys.path.append(meta["resources_dir"])
@@ -55,6 +55,13 @@
_shared_logger_name = "leiden"
+# Function to check available space in /dev/shm
+def get_available_shared_memory():
+ shm_path = "/dev/shm"
+ shm_stats = os.statvfs(shm_path)
+ return shm_stats.f_bsize * shm_stats.f_bavail
+
+
class SharedNumpyMatrix:
def __init__(
self,
@@ -70,6 +77,13 @@ def __init__(
def from_numpy(
cls, memory_manager: managers.SharedMemoryManager, array: npt.ArrayLike
):
+ available_shared_memory = get_available_shared_memory()
+ n_bytes_required = array.nbytes
+ if available_shared_memory < n_bytes_required:
+ raise ValueError(
+ "Not enough shared memory (/dev/shm) is available to load the data. "
+ f"Required amount: {n_bytes_required}, available: {available_shared_memory}."
+ )
shm = memory_manager.SharedMemory(size=array.nbytes)
array_in_shared_memory = np.ndarray(
array.shape, dtype=array.dtype, buffer=shm.buf
From 66a9d3b226cf5a2bb9e97d2ebb1127c1f7e2a3f8 Mon Sep 17 00:00:00 2001
From: DriesSchaumont <5946712+DriesSchaumont@users.noreply.github.com>
Date: Tue, 25 Mar 2025 13:37:31 +0000
Subject: [PATCH 2/2] Undo container image change
---
src/cluster/leiden/config.vsh.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/cluster/leiden/config.vsh.yaml b/src/cluster/leiden/config.vsh.yaml
index e32a35e81e3..1b06cdc6076 100644
--- a/src/cluster/leiden/config.vsh.yaml
+++ b/src/cluster/leiden/config.vsh.yaml
@@ -79,7 +79,7 @@ test_resources:
engines:
- type: docker
- image: python:3.12-slim
+ image: python:3.11-slim
setup:
- type: apt
packages: