From 649094d18eaf7124037a11d578485c879ada8624 Mon Sep 17 00:00:00 2001 From: nur ada Date: Wed, 26 Jul 2023 11:47:28 +0200 Subject: [PATCH 01/27] add gcc/g++ to environment.yml --- environment.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 4bc7f889..c66153f5 100755 --- a/environment.yml +++ b/environment.yml @@ -18,8 +18,9 @@ dependencies: # - pytorch-sparse # From conda-forge and defaults - - python >= 3.9 # (3.6 should also work) + - python # (3.6 should also work) - pip + - cxx-compiler - lemon - vigra - freeglut From 092a3f2d3749e6fea1a2f861f3bd1f36d3d41a93 Mon Sep 17 00:00:00 2001 From: nur ada Date: Wed, 26 Jul 2023 12:10:37 +0200 Subject: [PATCH 02/27] I added comment to environment.yml --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index c66153f5..66cec593 100755 --- a/environment.yml +++ b/environment.yml @@ -20,7 +20,7 @@ dependencies: # From conda-forge and defaults - python # (3.6 should also work) - pip - - cxx-compiler + - cxx-compiler # for g++ - lemon - vigra - freeglut From e0eec5117ad70e444d3052a27f1dbf843c0ec513 Mon Sep 17 00:00:00 2001 From: nur ada Date: Wed, 26 Jul 2023 12:42:17 +0200 Subject: [PATCH 03/27] we commented electonn3 line in environment.yml file , because it failed --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 66cec593..4ac46d86 100755 --- a/environment.yml +++ b/environment.yml @@ -85,7 +85,7 @@ dependencies: - torch-sparse - torch-scatter # Pre-release packages - - git+https://github.com/ELEKTRONN/elektronn3.git@syconn2#egg=elektronn3 + # - git+https://github.com/ELEKTRONN/elektronn3.git@syconn2#egg=elektronn3 - git+https://github.com/knossos-project/knossos_utils.git@syconn2#egg=knossos_utils - git+https://github.com/StructuralNeurobiologyLab/MorphX.git@v0.1#egg=MorphX From c6b3fd56ed473850fa84eb091a9248d046a96793 Mon Sep 17 00:00:00 2001 From: Nur Ada Date: Wed, 26 Jul 2023 16:35:13 +0200 Subject: [PATCH 04/27] save --- environment.yml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/environment.yml b/environment.yml index 4ac46d86..01f832b6 100755 --- a/environment.yml +++ b/environment.yml @@ -7,18 +7,19 @@ channels: - conda-forge - defaults - menpo - - open3d-admin + - open3d-admin + - nvidia # - pyg dependencies: - - cudatoolkit=11.6 + - pytorch-cuda=11.7 # From -c pytorch - - pytorch::pytorch=1.12.1 # can be > 1.9 if glibc >=2.27 is available + - pytorch # can be > 1.9 if glibc >=2.27 is available - torchvision # - pyg # - pytorch-sparse # From conda-forge and defaults - - python # (3.6 should also work) + - python=3.8 # (3.6 should also work) - pip - cxx-compiler # for g++ - lemon @@ -75,17 +76,16 @@ dependencies: - sphinx_rtd_theme - sphinx-autodoc-typehints - # Required dependencies that are not yet available via conda: - pip: - - open3d + - open3d #Note(ada):didn't work - zmesh - plyfile - - torch_geometric == 2.0.2 # 2.0.3 is incompatible with lcp.knn.quantized_sampling + - torch_geometric==2.0.2 # 2.0.3 is incompatible with lcp.knn.quantized_sampling - --find-links https://data.pyg.org/whl/torch-1.12.0+cu116.html - torch-sparse - - torch-scatter + #- torch-scatter # NOTE(ada): we installed it via conda, "conda install pytorch-scatter -c pyg" # Pre-release packages - # - git+https://github.com/ELEKTRONN/elektronn3.git@syconn2#egg=elektronn3 + #- git+https://github.com/ELEKTRONN/elektronn3.git@syconn2#egg=elektronn3 #NOTE(ada): we did download it but with pip install elektronn3 - git+https://github.com/knossos-project/knossos_utils.git@syconn2#egg=knossos_utils - git+https://github.com/StructuralNeurobiologyLab/MorphX.git@v0.1#egg=MorphX @@ -101,3 +101,5 @@ dependencies: #for skeletonisation - fill-voids + + \ No newline at end of file From d3f568f533e1db7290ae860ed9d68c8696972ca9 Mon Sep 17 00:00:00 2001 From: Nur Ada Date: Wed, 26 Jul 2023 17:22:52 +0200 Subject: [PATCH 05/27] deleted a line which gave error compiling --- syconn/extraction/find_object_properties_C.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syconn/extraction/find_object_properties_C.pyx b/syconn/extraction/find_object_properties_C.pyx index 6bc6d3e7..16935efe 100755 --- a/syconn/extraction/find_object_properties_C.pyx +++ b/syconn/extraction/find_object_properties_C.pyx @@ -13,7 +13,7 @@ ctypedef fused n_type: ctypedef vector[int] int_vec ctypedef vector[int_vec] int_vec_vec -ctypedef vector[n_type[:, :, :]] uintarr_vec + ctypedef vector[unordered_map[uint64_t, int_vec]] umvec_rc ctypedef vector[unordered_map[uint64_t, int_vec_vec]] umvec_bb ctypedef vector[unordered_map[uint64_t, int]] umvec_size From 35418c00b8e80e231037cc639baa1382cef8d083 Mon Sep 17 00:00:00 2001 From: Nur Ada Date: Wed, 26 Jul 2023 17:43:56 +0200 Subject: [PATCH 06/27] working environment --- environment.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/environment.yml b/environment.yml index 01f832b6..36d9d7de 100755 --- a/environment.yml +++ b/environment.yml @@ -9,7 +9,7 @@ channels: - menpo - open3d-admin - nvidia - # - pyg + #- pyg dependencies: - pytorch-cuda=11.7 # From -c pytorch @@ -76,6 +76,7 @@ dependencies: - sphinx_rtd_theme - sphinx-autodoc-typehints + #- pyg::torch-scatter - pip: - open3d #Note(ada):didn't work - zmesh @@ -83,9 +84,9 @@ dependencies: - torch_geometric==2.0.2 # 2.0.3 is incompatible with lcp.knn.quantized_sampling - --find-links https://data.pyg.org/whl/torch-1.12.0+cu116.html - torch-sparse - #- torch-scatter # NOTE(ada): we installed it via conda, "conda install pytorch-scatter -c pyg" + - torch-scatter # Pre-release packages - #- git+https://github.com/ELEKTRONN/elektronn3.git@syconn2#egg=elektronn3 #NOTE(ada): we did download it but with pip install elektronn3 + - elektronn3 #- git+https://github.com/ELEKTRONN/elektronn3.git@syconn2#egg=elektronn3 #NOTE(ada): we did download it but with pip install elektronn3 - git+https://github.com/knossos-project/knossos_utils.git@syconn2#egg=knossos_utils - git+https://github.com/StructuralNeurobiologyLab/MorphX.git@v0.1#egg=MorphX From 333d72122e1d90f4745ec63245ea3360ce6bc2b7 Mon Sep 17 00:00:00 2001 From: Nur Ada Date: Fri, 28 Jul 2023 11:12:19 +0200 Subject: [PATCH 07/27] changed open3d version, python version --- environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 36d9d7de..c7470dbd 100755 --- a/environment.yml +++ b/environment.yml @@ -19,7 +19,7 @@ dependencies: # - pytorch-sparse # From conda-forge and defaults - - python=3.8 # (3.6 should also work) + - python=3.7 # (3.6 should also work) NOTE(ada): because open3d was not available for python3.8: glibc 2.18 not found - pip - cxx-compiler # for g++ - lemon @@ -78,7 +78,7 @@ dependencies: #- pyg::torch-scatter - pip: - - open3d #Note(ada):didn't work + - open3d==0.9.0 #Note(ada):>0.9.0 throws /lib64/libm.so.6: version `GLIBC_2.27' not found - zmesh - plyfile - torch_geometric==2.0.2 # 2.0.3 is incompatible with lcp.knn.quantized_sampling From 69833247bd9fe62a61dfa7865b53427cbaf54e06 Mon Sep 17 00:00:00 2001 From: Nur Ada Date: Wed, 2 Aug 2023 11:49:20 +0200 Subject: [PATCH 08/27] we changed scikit-learn version from 0.24.0 to 0.21.3 because of rfc error --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index c7470dbd..ef362e3c 100755 --- a/environment.yml +++ b/environment.yml @@ -43,7 +43,7 @@ dependencies: - scipy < 1.9 # <1.9 because n_jobs parameter in every tree query was removed. Can be >=1.9 if refactored in the whole project - termcolor # fix version due to stored RFC models. - - scikit-learn >= 0.24.1 + - scikit-learn = 0.21.3 - scikit-image - opencv - numba >0.48 # =0.48 led to freeze when importing elektronn3.data From a317ac3e268dc0a10fdf3a66b637f8dbb69f19ba Mon Sep 17 00:00:00 2001 From: Nur Ada Date: Fri, 4 Aug 2023 12:25:41 +0200 Subject: [PATCH 09/27] Bug fix:rfc feature number and name changes the given rfc used 11 features but we provided 14. so we checked the changes is git repository reverted them. --- syconn/extraction/cs_processing_steps.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/syconn/extraction/cs_processing_steps.py b/syconn/extraction/cs_processing_steps.py index 84ce6085..f6c668aa 100755 --- a/syconn/extraction/cs_processing_steps.py +++ b/syconn/extraction/cs_processing_steps.py @@ -1411,16 +1411,17 @@ def synssv_o_features(synssv_o: segmentation.SegmentationObject) -> list: Returns: list """ - features = [synssv_o.size, synssv_o.mesh_area] + # print(synssv_o.attr_dict.keys()) + features = [synssv_o.size, synssv_o.mesh_area, synssv_o.attr_dict["syn_type_sym_ratio"]] #NOTE(ada): we need to delete 3 partner_ids = synssv_o.attr_dict["neuron_partners"] for i_partner_id, partner_id in enumerate(partner_ids): features.append(synssv_o.attr_dict["n_mi_objs_%d" % i_partner_id]) features.append(synssv_o.attr_dict["n_mi_vxs_%d" % i_partner_id]) - features.append(synssv_o.attr_dict["min_dst_mi_nm_%d" % i_partner_id]) + #features.append(synssv_o.attr_dict["min_dst_mi_nm_%d" % i_partner_id]) features.append(synssv_o.attr_dict["n_vc_objs_%d" % i_partner_id]) features.append(synssv_o.attr_dict["n_vc_vxs_%d" % i_partner_id]) - features.append(synssv_o.attr_dict["min_dst_vc_nm_%d" % i_partner_id]) + #features.append(synssv_o.attr_dict["min_dst_vc_nm_%d" % i_partner_id]) return features From cb0dc5813a7e700e1626a0f970ea72bab294a13a Mon Sep 17 00:00:00 2001 From: Nur Ada Date: Fri, 4 Aug 2023 13:11:48 +0200 Subject: [PATCH 10/27] syconn2 uses specific elektronn3 version, which requires specific version of pytorch --- environment.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/environment.yml b/environment.yml index ef362e3c..e295d21d 100755 --- a/environment.yml +++ b/environment.yml @@ -8,13 +8,12 @@ channels: - defaults - menpo - open3d-admin - - nvidia #- pyg dependencies: - - pytorch-cuda=11.7 # From -c pytorch - - pytorch # can be > 1.9 if glibc >=2.27 is available - - torchvision + - cudatoolkit=10.2 + - pytorch==1.5.1 # can be > 1.9 if glibc >=2.27 is available NOTE(ada):pinning version because elektronn3 didnt work + - torchvision==0.6.1 # - pyg # - pytorch-sparse @@ -86,7 +85,7 @@ dependencies: - torch-sparse - torch-scatter # Pre-release packages - - elektronn3 #- git+https://github.com/ELEKTRONN/elektronn3.git@syconn2#egg=elektronn3 #NOTE(ada): we did download it but with pip install elektronn3 + - git+https://github.com/ELEKTRONN/elektronn3.git@syconn2#egg=elektronn3 - git+https://github.com/knossos-project/knossos_utils.git@syconn2#egg=knossos_utils - git+https://github.com/StructuralNeurobiologyLab/MorphX.git@v0.1#egg=MorphX From 8c3641b6aba5ba27daa68014dcc3a27915a737c8 Mon Sep 17 00:00:00 2001 From: Eric Jelli Date: Mon, 28 Aug 2023 13:38:53 +0200 Subject: [PATCH 11/27] Use custom elektronn3 repo with corrected knn.cxx path --- environment.yml | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/environment.yml b/environment.yml index e295d21d..5b42021b 100755 --- a/environment.yml +++ b/environment.yml @@ -7,20 +7,19 @@ channels: - conda-forge - defaults - menpo - - open3d-admin - #- pyg + - open3d-admin + # - pyg dependencies: + - cudatoolkit=11.6 # From -c pytorch - - cudatoolkit=10.2 - - pytorch==1.5.1 # can be > 1.9 if glibc >=2.27 is available NOTE(ada):pinning version because elektronn3 didnt work - - torchvision==0.6.1 + - pytorch::pytorch=1.12.1 # can be > 1.9 if glibc >=2.27 is available + - torchvision # - pyg # - pytorch-sparse # From conda-forge and defaults - - python=3.7 # (3.6 should also work) NOTE(ada): because open3d was not available for python3.8: glibc 2.18 not found + - python > 3.8 # (3.6 should also work) - pip - - cxx-compiler # for g++ - lemon - vigra - freeglut @@ -42,7 +41,7 @@ dependencies: - scipy < 1.9 # <1.9 because n_jobs parameter in every tree query was removed. Can be >=1.9 if refactored in the whole project - termcolor # fix version due to stored RFC models. - - scikit-learn = 0.21.3 + - scikit-learn >= 0.24.1 - scikit-image - opencv - numba >0.48 # =0.48 led to freeze when importing elektronn3.data @@ -75,17 +74,17 @@ dependencies: - sphinx_rtd_theme - sphinx-autodoc-typehints - #- pyg::torch-scatter + # Required dependencies that are not yet available via conda: - pip: - - open3d==0.9.0 #Note(ada):>0.9.0 throws /lib64/libm.so.6: version `GLIBC_2.27' not found + - open3d - zmesh - plyfile - - torch_geometric==2.0.2 # 2.0.3 is incompatible with lcp.knn.quantized_sampling + - torch_geometric == 2.0.2 # 2.0.3 is incompatible with lcp.knn.quantized_sampling - --find-links https://data.pyg.org/whl/torch-1.12.0+cu116.html - torch-sparse - torch-scatter # Pre-release packages - - git+https://github.com/ELEKTRONN/elektronn3.git@syconn2#egg=elektronn3 + - git+https://github.com/mpinb/elektronn3.git@syconn2-mod#egg=elektronn3 - git+https://github.com/knossos-project/knossos_utils.git@syconn2#egg=knossos_utils - git+https://github.com/StructuralNeurobiologyLab/MorphX.git@v0.1#egg=MorphX @@ -101,5 +100,3 @@ dependencies: #for skeletonisation - fill-voids - - \ No newline at end of file From 8715a5426f75a798cf7234b4c67f5fdb2df1f69a Mon Sep 17 00:00:00 2001 From: Eric Jelli Date: Mon, 28 Aug 2023 13:46:14 +0200 Subject: [PATCH 12/27] doc: Mamba installation steps --- docs/instructions.rst | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/docs/instructions.rst b/docs/instructions.rst index cf9eeb11..18d790f9 100644 --- a/docs/instructions.rst +++ b/docs/instructions.rst @@ -20,9 +20,9 @@ Installation - The whole pipeline was designed and tested on Linux systems Before you can set up SyConn, ensure that the -`conda `__ -package manager is installed on your system. Then you can install SyConn -and all of its dependencies into a new conda +`mamba `__ +package manager is installed on your system and that `gcc` and `g++` is available (tested version: 11.2.0). +Then you can install SyConn and all of its dependencies into a new conda `environment `__ named “syconn2” by running: @@ -30,8 +30,8 @@ named “syconn2” by running: git clone https://github.com/StructuralNeurobiologyLab/SyConn cd SyConn - conda env create -f environment.yml -n syconn2 python=3.7 - conda activate syconn2 + CXX=g++; CC=gcc; conda env create -n syconn2 -f environment.yml + mamba activate syconn2 pip install -e . The last command will install SyConn in @@ -43,12 +43,6 @@ command with: pip install . -To update the environment, e.g. if the environment file changed, use: - -:: - - conda env update --name syco --file environment.yml --prune - If you encounter :: From 208dd6b2086a1cc04e0184ea34bb0ddd9532e6db Mon Sep 17 00:00:00 2001 From: Eric Jelli Date: Mon, 28 Aug 2023 13:56:53 +0200 Subject: [PATCH 13/27] Use older open3d version to fix known glibc issue fix error message: OSError: /lib64/libc.so.6: version `GLIBC_2.18' not found (required by ../envs/syconn2/lib/python3.8/site-packages/open3d/libc++abi.so.1) --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 5b42021b..d1a86bd1 100755 --- a/environment.yml +++ b/environment.yml @@ -76,7 +76,7 @@ dependencies: # Required dependencies that are not yet available via conda: - pip: - - open3d + - open3d<=0.9 - zmesh - plyfile - torch_geometric == 2.0.2 # 2.0.3 is incompatible with lcp.knn.quantized_sampling From d88eb0b7475088049c7a0e8b923a898ae58a6e3b Mon Sep 17 00:00:00 2001 From: Eric Jelli Date: Mon, 28 Aug 2023 14:23:09 +0200 Subject: [PATCH 14/27] open3d<=0.9 requires python 3.7 fixes: ERROR: Could not find a version that satisfies the requirement open3d<=0.9 (from versions: 0.10.0.0, 0.11.0, 0.11.1, 0.11.2, 0.12.0, 0.13.0) ERROR: No matching distribution found for open3d<=0.9 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index d1a86bd1..00ada2be 100755 --- a/environment.yml +++ b/environment.yml @@ -18,7 +18,7 @@ dependencies: # - pytorch-sparse # From conda-forge and defaults - - python > 3.8 # (3.6 should also work) + - python=3.7 # (3.6 should also work) - pip - lemon - vigra From 45794af51a286d52af2336707dbf1a3dfd325d0e Mon Sep 17 00:00:00 2001 From: Eric Jelli Date: Tue, 29 Aug 2023 08:41:09 +0200 Subject: [PATCH 15/27] bugfix in knossos_utils for python 3.7 --- environment.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 00ada2be..91e1beab 100755 --- a/environment.yml +++ b/environment.yml @@ -20,6 +20,7 @@ dependencies: # From conda-forge and defaults - python=3.7 # (3.6 should also work) - pip + #- gxx_linux-64 - lemon - vigra - freeglut @@ -85,7 +86,7 @@ dependencies: - torch-scatter # Pre-release packages - git+https://github.com/mpinb/elektronn3.git@syconn2-mod#egg=elektronn3 - - git+https://github.com/knossos-project/knossos_utils.git@syconn2#egg=knossos_utils + - git+https://github.com/mpinb/knossos_utils.git@syconn2-mod#egg=knossos_utils - git+https://github.com/StructuralNeurobiologyLab/MorphX.git@v0.1#egg=MorphX # cloud-volume >=4 throws an error in simple_merge during np.concatenate if any skeleton has no vertices From 3e20171accc6c0be381ce6ffc5b1140d8eb1590d Mon Sep 17 00:00:00 2001 From: Eric Jelli Date: Tue, 29 Aug 2023 10:18:42 +0200 Subject: [PATCH 16/27] Use legacy scikit-learn to prevent error with random forest classifier Error message: Exception: [Errno 2] No such file or directory: '...cwd/models//conn_syn_rfc//rfc_fallback' --- docs/instructions.rst | 6 +++--- environment.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/instructions.rst b/docs/instructions.rst index 18d790f9..77bdacf4 100644 --- a/docs/instructions.rst +++ b/docs/instructions.rst @@ -21,8 +21,8 @@ Installation Before you can set up SyConn, ensure that the `mamba `__ -package manager is installed on your system and that `gcc` and `g++` is available (tested version: 11.2.0). -Then you can install SyConn and all of its dependencies into a new conda +package manager is installed on your system. Then you can install SyConn and +all of its dependencies into a new conda `environment `__ named “syconn2” by running: @@ -30,7 +30,7 @@ named “syconn2” by running: git clone https://github.com/StructuralNeurobiologyLab/SyConn cd SyConn - CXX=g++; CC=gcc; conda env create -n syconn2 -f environment.yml + mamba env create -n syconn2 -f environment.yml mamba activate syconn2 pip install -e . diff --git a/environment.yml b/environment.yml index 91e1beab..81a9f6f3 100755 --- a/environment.yml +++ b/environment.yml @@ -42,7 +42,7 @@ dependencies: - scipy < 1.9 # <1.9 because n_jobs parameter in every tree query was removed. Can be >=1.9 if refactored in the whole project - termcolor # fix version due to stored RFC models. - - scikit-learn >= 0.24.1 + - scikit-learn = 0.21.3 - scikit-image - opencv - numba >0.48 # =0.48 led to freeze when importing elektronn3.data From 5c4c67d65238de20b943a4729eb015c2bee3b6ab Mon Sep 17 00:00:00 2001 From: Eric Jelli Date: Tue, 29 Aug 2023 14:04:32 +0200 Subject: [PATCH 17/27] bugfix: torch-sparse resulted in seg fault --- environment.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/environment.yml b/environment.yml index 81a9f6f3..c958cc35 100755 --- a/environment.yml +++ b/environment.yml @@ -80,10 +80,7 @@ dependencies: - open3d<=0.9 - zmesh - plyfile - - torch_geometric == 2.0.2 # 2.0.3 is incompatible with lcp.knn.quantized_sampling - - --find-links https://data.pyg.org/whl/torch-1.12.0+cu116.html - - torch-sparse - - torch-scatter + - torch_geometric == 2.3.1 # Pre-release packages - git+https://github.com/mpinb/elektronn3.git@syconn2-mod#egg=elektronn3 - git+https://github.com/mpinb/knossos_utils.git@syconn2-mod#egg=knossos_utils From 439d16ee613ea8a4254c6fd5ed8ba2fc30d32514 Mon Sep 17 00:00:00 2001 From: Eric Jelli Date: Tue, 29 Aug 2023 15:55:11 +0200 Subject: [PATCH 18/27] bugfix: Make prediction work with legacy models from example data --- syconn/handler/prediction_pts.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/syconn/handler/prediction_pts.py b/syconn/handler/prediction_pts.py index b48fd469..da29de92 100755 --- a/syconn/handler/prediction_pts.py +++ b/syconn/handler/prediction_pts.py @@ -1926,7 +1926,8 @@ def predict_cmpt_ssd(ssd_kwargs, mpath: Optional[str] = None, ssv_ids: Optional[ mpath = os.path.expanduser(mpath) if os.path.isdir(mpath): # multiple models - mpaths = glob.glob(mpath + '*/state_dict.pth') + #mpaths = glob.glob(mpath + '*/state_dict.pth') + mpaths = glob.glob(mpath + '*.pth') else: # single model mpaths = [mpath] @@ -1967,7 +1968,7 @@ def predict_cmpt_ssd(ssd_kwargs, mpath: Optional[str] = None, ssv_ids: Optional[ batchsizes[ctx] = int(batchsizes[ctx]*default_kwargs['bs']) default_kwargs['bs'] = batchsizes out_dc = predict_pts_plain(ssd_kwargs, - model_loader=get_cmpt_model_pts, + model_loader=get_cpmt_model_pts_OLD, loader_func=pts_loader_cpmt, pred_func=pts_pred_cmpt, postproc_func=pts_postproc_cpmt, @@ -1997,7 +1998,7 @@ def get_cpmt_model_pts_OLD(mpath: Optional[str] = None, device='cuda', pred_type mpath = os.path.expanduser(mpath) if os.path.isdir(mpath): # multiple models - mpaths = glob.glob(mpath + '*/*.pth') + mpaths = glob.glob(mpath + '*.pth') else: # single model, must contain 'cmpt' in its name mpaths = [mpath] @@ -2223,9 +2224,9 @@ def pts_pred_cmpt(m, inp, q_out, d_out, q_cnt, device, bs): high = bs * (ii + 1) with torch.no_grad(): # transpose is required for lcp architectures - g_inp = [torch.from_numpy(i[low:high]).to(device).float().transpose(1, 2) for i in model_inp] + g_inp = [torch.from_numpy(i[low:high]).to(device).float() for i in model_inp] out = m[batch_progress[2]](*g_inp) - out = out.transpose(1, 2).cpu().numpy() + out = out.cpu().numpy() masks = batch_mask[low:high] # filter vertices which belong to sv (discard predictions for cell organelles) out = out[masks] @@ -2435,7 +2436,7 @@ def get_cmpt_kwargs(mdir: str) -> Tuple[dict, dict]: ctx = int(re.findall(r'_ctx(\d+)_', mdir)[-1]) feat_dim = int(re.findall(r'_fdim(\d+)', mdir)[-1]) class_num = int(re.findall(r'_cnum(\d+)', mdir)[-1]) - pred_type = re.findall(r'_types([^_]+)_', mdir)[-1] + pred_type = re.findall(r'_t([^_]+)_', mdir)[-1] batchsize = int(re.findall(r'_bs(\d+)_', mdir)[-1]) # TODO: Fix neighbor_nums or create extra model mkwargs = dict(input_channels=feat_dim, output_channels=class_num, use_norm=use_norm, use_bias=use_bias, From 5ab2eb5e3c24b5a2f231d10ca5cb9f59189c2c5c Mon Sep 17 00:00:00 2001 From: Nur Ada Date: Wed, 30 Aug 2023 17:42:13 +0200 Subject: [PATCH 19/27] eric's change was reversed, changed it back --- syconn/handler/prediction_pts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syconn/handler/prediction_pts.py b/syconn/handler/prediction_pts.py index b48fd469..d28a1f85 100755 --- a/syconn/handler/prediction_pts.py +++ b/syconn/handler/prediction_pts.py @@ -1967,7 +1967,7 @@ def predict_cmpt_ssd(ssd_kwargs, mpath: Optional[str] = None, ssv_ids: Optional[ batchsizes[ctx] = int(batchsizes[ctx]*default_kwargs['bs']) default_kwargs['bs'] = batchsizes out_dc = predict_pts_plain(ssd_kwargs, - model_loader=get_cmpt_model_pts, + model_loader=get_cmpt_model_pts_OLD, loader_func=pts_loader_cpmt, pred_func=pts_pred_cmpt, postproc_func=pts_postproc_cpmt, From afde9286bacc0460b3db3d84f77bf20aff71402a Mon Sep 17 00:00:00 2001 From: Eric Jelli Date: Mon, 4 Sep 2023 12:43:03 +0200 Subject: [PATCH 20/27] bugfix: python2 does not support typing --- syconn/analysis/syconn_knossos_viewer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/syconn/analysis/syconn_knossos_viewer.py b/syconn/analysis/syconn_knossos_viewer.py index 22c56232..35a0ede6 100755 --- a/syconn/analysis/syconn_knossos_viewer.py +++ b/syconn/analysis/syconn_knossos_viewer.py @@ -4,7 +4,6 @@ # Copyright (c) 2016 - now # Max-Planck-Institute of Neurobiology, Munich, Germany # Authors: Philipp Schubert, Joergen Kornfeld -from typing import Dict, Any from PythonQt import QtGui, Qt, QtCore from PythonQt.QtGui import QTableWidget, QTableWidgetItem @@ -27,7 +26,7 @@ class SyConnGateInteraction(object): """ Query the SyConn backend server. """ - ct_from_cache: Dict[Any, Any] + ct_from_cache = {} # type: Dict[Any, Any] def __init__(self, server, synthresh=0.5, axodend_only=True): self.server = server From b1d3d990bff8ad055010d354b66f6b2d7dceb79f Mon Sep 17 00:00:00 2001 From: Eric Jelli Date: Mon, 4 Sep 2023 13:13:32 +0200 Subject: [PATCH 21/27] doc: Write down principal steps for viewer setup --- docs/instructions.rst | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/docs/instructions.rst b/docs/instructions.rst index 77bdacf4..7bc69b4c 100644 --- a/docs/instructions.rst +++ b/docs/instructions.rst @@ -201,6 +201,10 @@ After initialization of the SDs (cell and sub-cellular structures, step SyConn KNOSSOS viewer --------------------- +This setup assumes that you run Linux (or WSL on Linux). If you don't +have requried packages installed, you additional need ``sudo`` rights or +ask your system administrator to install them for you. + The following packages have to be available in the system’s python2 interpreter (will differ from the conda environment): @@ -208,25 +212,49 @@ interpreter (will differ from the conda environment): - lz4 - requests +One approach is to install them via `pip`: +:: + + wget -P ~/.local/lib https://bootstrap.pypa.io/pip/2.7/get-pip.py + python2 ~/.local/lib/get-pip.py --user + python2 -m pip install numpy requests lz4 + In order to inspect the resulting data via the SyConnViewer KNOSSOS-plugin follow these steps: - Wait until ``start.py`` finished. For starting the server manually - run ``syconn.server --working_dir=`` which executes - ``syconn/kplugin/server.py`` and allows to visualize the analysis + run ``syconn.server --working_dir=`` in the syconn conda environment + which executes ``syconn/analysis/server.py`` and allows to visualize the analysis results of the working directory at (````) in KNOSSOS. The server address and port will be printed. -- Download and run the nightly build of KNOSSOS - (https://github.com/knossos-project/knossos/releases/tag/nightly) +- Download and run version 5.1 of KNOSSOS + (https://github.com/knossos-project/knossos/releases/tag/v5.1) + :: + + wget https://github.com/knossos-project/knossos/releases/download/v5.1/linux.KNOSSOS-5.1.AppImage + chmod u+x linux.KNOSSOS-5.1.AppImage + ./linux.KNOSSOS-5.1.AppImage + + Possible pitfalls: + ``libpython2.7.so.1.0: cannot open shared object file: No such file or directory`` + you need to install the python-devtool package on your system: + :: + + sudo apt install libpython2.7 + + If the AppImage complains about missing ``fusermount`` you need to install it (i.e. on Ubuntu 22.04) + :: + + sudo apt install libfuse2 - In KNOSSOS -> File -> Choose Dataset -> browse to your working directory and open ``knossosdatasets/seg/mag1/knossos.conf`` with enabled ‘load_segmentation_overlay’ (at the bottom of the dialog). - Then go to Scripting (top row) -> Run file -> browse to - ``syconn/kplugin/syconn_knossos_viewer.py``, open it and enter the - port and address of the syconn server. + ``syconn/analysis/syconn_knossos_viewer.py``, open it and enter the + port and address of the syconn server as printed in the terminal. - After the SyConnViewer window has opened, the selection of segmentation fragments in the slice-viewports (exploration mode) or From bbc2e5ab05065ed629359cc145d46ef31768b0d1 Mon Sep 17 00:00:00 2001 From: Eric Jelli Date: Mon, 4 Sep 2023 13:17:41 +0200 Subject: [PATCH 22/27] typo --- docs/instructions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/instructions.rst b/docs/instructions.rst index 7bc69b4c..36de212f 100644 --- a/docs/instructions.rst +++ b/docs/instructions.rst @@ -201,7 +201,7 @@ After initialization of the SDs (cell and sub-cellular structures, step SyConn KNOSSOS viewer --------------------- -This setup assumes that you run Linux (or WSL on Linux). If you don't +This setup assumes that you run Linux (or WSL on Windows). If you don't have requried packages installed, you additional need ``sudo`` rights or ask your system administrator to install them for you. From 52b43f2092f125df4e15f4aeff355d756018d7e9 Mon Sep 17 00:00:00 2001 From: Eric Jelli <36883188+erjel@users.noreply.github.com> Date: Thu, 7 Sep 2023 14:54:32 +0200 Subject: [PATCH 23/27] Update instructions.rst New error message about missing libgl occured. Added fix to instructions. --- docs/instructions.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/instructions.rst b/docs/instructions.rst index 36de212f..3e0b1408 100644 --- a/docs/instructions.rst +++ b/docs/instructions.rst @@ -248,6 +248,11 @@ KNOSSOS-plugin follow these steps: sudo apt install libfuse2 + if AppImage complains about ``error while loading shared libraries: libGL.so.1: cannot open shared object file: No such file or directory`` you need to [install](https://stackoverflow.com/a/68666500) it: + :: + + sudo apt install libgl1 + - In KNOSSOS -> File -> Choose Dataset -> browse to your working directory and open ``knossosdatasets/seg/mag1/knossos.conf`` with enabled ‘load_segmentation_overlay’ (at the bottom of the dialog). From 4c434458f4257a556b02137e58743b484fee47ca Mon Sep 17 00:00:00 2001 From: Jeffrey Tong <58666340+spiralsim@users.noreply.github.com> Date: Mon, 1 Jul 2024 18:36:13 -0400 Subject: [PATCH 24/27] Update environment.yml to work again --- docs/instructions.rst | 12 ++++++------ environment.yml | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/instructions.rst b/docs/instructions.rst index 3e0b1408..91293fa0 100644 --- a/docs/instructions.rst +++ b/docs/instructions.rst @@ -16,11 +16,11 @@ More details are linked in the respective chapters. Installation ------------ -- Python 3.7 -- The whole pipeline was designed and tested on Linux systems +- linux-64 or equivalent OS (ex. WSL) + - This is required because the dependency `menpo::osmesa ` is currently only available on linux-64. + - Tested on Ubuntu 22.04.3 LTS -Before you can set up SyConn, ensure that the -`mamba `__ +Before you can set up SyConn, ensure that the latest version of `Anaconda `__ package manager is installed on your system. Then you can install SyConn and all of its dependencies into a new conda `environment `__ @@ -30,8 +30,8 @@ named “syconn2” by running: git clone https://github.com/StructuralNeurobiologyLab/SyConn cd SyConn - mamba env create -n syconn2 -f environment.yml - mamba activate syconn2 + conda env create -n syconn2 -f environment.yml + conda activate syconn2 pip install -e . The last command will install SyConn in diff --git a/environment.yml b/environment.yml index c958cc35..93236086 100755 --- a/environment.yml +++ b/environment.yml @@ -18,8 +18,9 @@ dependencies: # - pytorch-sparse # From conda-forge and defaults - - python=3.7 # (3.6 should also work) + - python >= 3.6 # Setting python=3.7 explicitly may cause the "Solving environment" step to hang - pip + - setuptools < 70 # setuptools >= 70.0.0 throws "cannot import name 'packaging' from 'pkg_resources'" #- gxx_linux-64 - lemon - vigra @@ -42,7 +43,7 @@ dependencies: - scipy < 1.9 # <1.9 because n_jobs parameter in every tree query was removed. Can be >=1.9 if refactored in the whole project - termcolor # fix version due to stored RFC models. - - scikit-learn = 0.21.3 + - scikit-learn >= 0.24.1 - scikit-image - opencv - numba >0.48 # =0.48 led to freeze when importing elektronn3.data @@ -77,10 +78,10 @@ dependencies: # Required dependencies that are not yet available via conda: - pip: - - open3d<=0.9 + - open3d - zmesh - plyfile - - torch_geometric == 2.3.1 + - torch_geometric == 2.0.2 # Pre-release packages - git+https://github.com/mpinb/elektronn3.git@syconn2-mod#egg=elektronn3 - git+https://github.com/mpinb/knossos_utils.git@syconn2-mod#egg=knossos_utils @@ -97,4 +98,3 @@ dependencies: #for skeletonisation - fill-voids - From 92b8e2e2e50a4a608ef0c4375c205ef79388ea6a Mon Sep 17 00:00:00 2001 From: Jeffrey Tong <58666340+spiralsim@users.noreply.github.com> Date: Mon, 1 Jul 2024 18:45:00 -0400 Subject: [PATCH 25/27] Add more clarification for Anaconda and libmamba --- docs/instructions.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/instructions.rst b/docs/instructions.rst index 91293fa0..bed27a3d 100644 --- a/docs/instructions.rst +++ b/docs/instructions.rst @@ -20,9 +20,9 @@ Installation - This is required because the dependency `menpo::osmesa ` is currently only available on linux-64. - Tested on Ubuntu 22.04.3 LTS -Before you can set up SyConn, ensure that the latest version of `Anaconda `__ -package manager is installed on your system. Then you can install SyConn and -all of its dependencies into a new conda +Before you can set up SyConn, ensure that the latest version of the conda package manager is installed on your system. +`Anaconda `__ with `libmamba `__ seems to be the fastest option. +Then you can install SyConn and all of its dependencies into a new conda `environment `__ named “syconn2” by running: @@ -30,6 +30,7 @@ named “syconn2” by running: git clone https://github.com/StructuralNeurobiologyLab/SyConn cd SyConn + conda config --set solver libmamba # If libmamba isn't already set as default conda env create -n syconn2 -f environment.yml conda activate syconn2 pip install -e . From 563a2f648940e93857280e0b26a9bbb71abb02e5 Mon Sep 17 00:00:00 2001 From: Jeffrey Tong <58666340+spiralsim@users.noreply.github.com> Date: Mon, 1 Jul 2024 18:47:14 -0400 Subject: [PATCH 26/27] Add clarification for installing g++ --- docs/instructions.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/instructions.rst b/docs/instructions.rst index bed27a3d..d6473dbf 100644 --- a/docs/instructions.rst +++ b/docs/instructions.rst @@ -22,6 +22,13 @@ Installation Before you can set up SyConn, ensure that the latest version of the conda package manager is installed on your system. `Anaconda `__ with `libmamba `__ seems to be the fastest option. + +You may also need to install g++: + +:: + + sudo apt-get install g++ + Then you can install SyConn and all of its dependencies into a new conda `environment `__ named “syconn2” by running: From 33424d5ba5bf05caa0b96cd5d4f5d3d7d95aafc6 Mon Sep 17 00:00:00 2001 From: Jeffrey Tong <58666340+spiralsim@users.noreply.github.com> Date: Mon, 1 Jul 2024 18:48:55 -0400 Subject: [PATCH 27/27] Fix formatting bug with Anaconda link --- docs/instructions.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/instructions.rst b/docs/instructions.rst index d6473dbf..0c98e008 100644 --- a/docs/instructions.rst +++ b/docs/instructions.rst @@ -17,7 +17,7 @@ Installation ------------ - linux-64 or equivalent OS (ex. WSL) - - This is required because the dependency `menpo::osmesa ` is currently only available on linux-64. + - This is required because the dependency `menpo::osmesa `__ is currently only available on linux-64. - Tested on Ubuntu 22.04.3 LTS Before you can set up SyConn, ensure that the latest version of the conda package manager is installed on your system. @@ -26,7 +26,7 @@ Before you can set up SyConn, ensure that the latest version of the conda packag You may also need to install g++: :: - + sudo apt-get install g++ Then you can install SyConn and all of its dependencies into a new conda