From 898ee1c690e27ad2b91855f64daf62ebcd21a121 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sat, 16 Jan 2021 12:16:10 +0100
Subject: [PATCH 01/54] create test for load_schema

---
 tests/schema/base.json                        |  3 +++
 tests/schema/subfolder_1/sub.json             |  3 +++
 tests/schema/subfolder_2/sub.txt              |  0
 tests/schema/subfolder_2/subfolder_3/sub.json |  3 +++
 tests/schema/subfolder_4/sub.txt              |  0
 tests/test_load_schema.m                      | 13 +++++++++++++
 6 files changed, 22 insertions(+)
 create mode 100644 tests/schema/base.json
 create mode 100644 tests/schema/subfolder_1/sub.json
 create mode 100644 tests/schema/subfolder_2/sub.txt
 create mode 100644 tests/schema/subfolder_2/subfolder_3/sub.json
 create mode 100644 tests/schema/subfolder_4/sub.txt
 create mode 100644 tests/test_load_schema.m

diff --git a/tests/schema/base.json b/tests/schema/base.json
new file mode 100644
index 00000000..73cd85a8
--- /dev/null
+++ b/tests/schema/base.json
@@ -0,0 +1,3 @@
+{
+    "test": 1
+}
\ No newline at end of file
diff --git a/tests/schema/subfolder_1/sub.json b/tests/schema/subfolder_1/sub.json
new file mode 100644
index 00000000..73cd85a8
--- /dev/null
+++ b/tests/schema/subfolder_1/sub.json
@@ -0,0 +1,3 @@
+{
+    "test": 1
+}
\ No newline at end of file
diff --git a/tests/schema/subfolder_2/sub.txt b/tests/schema/subfolder_2/sub.txt
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/schema/subfolder_2/subfolder_3/sub.json b/tests/schema/subfolder_2/subfolder_3/sub.json
new file mode 100644
index 00000000..73cd85a8
--- /dev/null
+++ b/tests/schema/subfolder_2/subfolder_3/sub.json
@@ -0,0 +1,3 @@
+{
+    "test": 1
+}
\ No newline at end of file
diff --git a/tests/schema/subfolder_4/sub.txt b/tests/schema/subfolder_4/sub.txt
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/test_load_schema.m b/tests/test_load_schema.m
new file mode 100644
index 00000000..ace78b0a
--- /dev/null
+++ b/tests/test_load_schema.m
@@ -0,0 +1,13 @@
+function test_load_schema()
+    
+    SCHEMA_DIR = fullfile(fileparts(mfilename('fullpath')), 'schema');
+    
+    schema = bids.internal.load_schema(SCHEMA_DIR);
+    
+    assert(isfield(schema, 'base'));
+    assert(isfield(schema, 'subfolder_1'));
+    assert(isfield(schema.subfolder_1, 'sub'));
+%     assert(isfield(schema.subfolder_2, 'subfolder_3'));
+    assert(~isfield(schema, 'subfolder_4'));
+
+end
\ No newline at end of file

From 0e084104f3d71c29cc5a62fef4f3664d2b375d78 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sat, 16 Jan 2021 12:20:30 +0100
Subject: [PATCH 02/54] add function to load schema

---
 +bids/+internal/load_schema.m | 63 +++++++++++++++++++++++++++++++++++
 1 file changed, 63 insertions(+)
 create mode 100644 +bids/+internal/load_schema.m

diff --git a/+bids/+internal/load_schema.m b/+bids/+internal/load_schema.m
new file mode 100644
index 00000000..3b9d56c2
--- /dev/null
+++ b/+bids/+internal/load_schema.m
@@ -0,0 +1,63 @@
+function schema = load_schema(SCHEMA_DIR)
+  % Loads a json schema by recursively looking through a folder structure.
+  %
+  % The nesting of the output structure reflects a combination of the folder structure and
+  % any eventual nesting within each json.
+  %
+  %
+  % Copyright (C) 2021--, BIDS-MATLAB developers
+
+  % TODO:
+  %  - folders that do not contain json files themselves but contain
+  %  subfolders that do, are not reflected in the output structure (they are
+  %  skipped). This can lead to "name conflicts". See "silenced" unit tests
+  %  for more info.
+
+  if nargin < 1
+    SCHEMA_DIR = fullfile(fileparts(mfilename('fullpath')), '..', '..', 'schema');
+  end
+
+  schema = struct();
+
+  [json_file_list, dirs] = bids.internal.file_utils('FPList', SCHEMA_DIR, '^*.json$');
+
+  schema = append_json_content_to_structure(schema, json_file_list);
+
+  schema = inspect_subdir(schema, dirs);
+
+end
+
+function structure = append_json_content_to_structure(structure, json_file_list)
+
+  for iFile = 1:size(json_file_list, 1)
+
+    file = deblank(json_file_list(iFile, :));
+
+    field_name = bids.internal.file_utils(file, 'basename');
+
+    structure.(field_name) = bids.util.jsondecode(file);
+  end
+
+end
+
+function structure = inspect_subdir(structure, subdir_list)
+  % recursively inspects subdirectory for json files and reflects folder
+  % hierarchy in the output structure.
+
+  for iDir = 1:size(subdir_list, 1)
+
+    directory = deblank(subdir_list(iDir, :));
+
+    [json_file_list, dirs] = bids.internal.file_utils('FPList', directory, '^*.json$');
+
+    if ~isempty(json_file_list)
+      field_name = bids.internal.file_utils(directory, 'basename');
+      structure.(field_name) = struct();
+      structure.(field_name) = append_json_content_to_structure(structure.(field_name), json_file_list);
+    end
+
+    structure = inspect_subdir(structure, dirs);
+
+  end
+
+end

From 9a20988d1080984ca20e68e7df987d39edf41bdd Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sat, 6 Feb 2021 18:21:56 +0100
Subject: [PATCH 03/54] fix conflict

---
 +bids/layout.m | 43 +++++++++++++++++++++++++++++++++----------
 1 file changed, 33 insertions(+), 10 deletions(-)

diff --git a/+bids/layout.m b/+bids/layout.m
index 872fde01..66318485 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -190,16 +190,40 @@ function tolerant_message(tolerant, msg)
   subject.ieeg    = struct([]); % iEEG data
   subject.pet     = struct([]); % PET imaging data
 
-  subject = parse_anat(subject);
-  subject = parse_func(subject);
-  subject = parse_fmap(subject);
-  subject = parse_eeg(subject);
-  subject = parse_meg(subject);
-  subject = parse_beh(subject);
-  subject = parse_dwi(subject);
-  subject = parse_perf(subject);
+  % use BIDS schema to organizing parsing of subject data
+  schema = bids.internal.load_schema();
+  modalities = fieldnames(schema.modalities);
+
+  for iModality = 1:numel(modalities)
+
+    datatypes = schema.modalities.(modalities{iModality}).datatypes;
+
+    for iDatatype = 1:numel(datatypes)
+      switch datatypes{iDatatype}
+        case 'anat'
+          subject = parse_anat(subject);
+        case 'beh'
+          subject = parse_beh(subject);
+        case 'dwi'
+          subject = parse_dwi(subject);
+        case 'eeg'
+          subject = parse_eeg(subject);
+        case 'fmap'
+          subject = parse_fmap(subject);
+        case 'func'
+          subject = parse_func(subject);
+        case 'ieeg'
+          subject = parse_ieeg(subject);
+        case 'meg'
+          subject = parse_meg(subject);
+        case 'perf'
+      end
+    end
+
+  end
+
+  % not covered by schema... yet
   subject = parse_pet(subject);
-  subject = parse_ieeg(subject);
 
 end
 
@@ -241,7 +265,6 @@ function tolerant_message(tolerant, msg)
     for i = 1:numel(file_list)
 
       subject = append_to_structure(file_list{i}, entities, subject, 'func');
-
       subject.func(end).meta = struct([]); % ?
 
     end

From fce12de142c4f63f25cbca70c28638d527ec332a Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sat, 16 Jan 2021 12:57:17 +0100
Subject: [PATCH 04/54] update load_schema test

---
 tests/test_load_schema.m | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/tests/test_load_schema.m b/tests/test_load_schema.m
index ace78b0a..18355b2e 100644
--- a/tests/test_load_schema.m
+++ b/tests/test_load_schema.m
@@ -1,13 +1,14 @@
 function test_load_schema()
-    
-    SCHEMA_DIR = fullfile(fileparts(mfilename('fullpath')), 'schema');
-    
-    schema = bids.internal.load_schema(SCHEMA_DIR);
-    
-    assert(isfield(schema, 'base'));
-    assert(isfield(schema, 'subfolder_1'));
-    assert(isfield(schema.subfolder_1, 'sub'));
-%     assert(isfield(schema.subfolder_2, 'subfolder_3'));
-    assert(~isfield(schema, 'subfolder_4'));
-
-end
\ No newline at end of file
+
+  SCHEMA_DIR = fullfile(fileparts(mfilename('fullpath')), 'schema');
+
+  schema = bids.internal.load_schema(SCHEMA_DIR);
+
+  assert(isfield(schema, 'base'));
+  assert(isfield(schema, 'subfolder_1'));
+  assert(isfield(schema.subfolder_1, 'sub'));
+  %     assert(isfield(schema.subfolder_2, 'subfolder_3'));
+  %     assert(isfield(schema.subfolder_2.subfolder_3, 'sub'));
+  assert(~isfield(schema, 'subfolder_4'));
+
+end

From 4eec143df348d4d71f6fa232a75cd4b22382368b Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sat, 16 Jan 2021 14:32:58 +0100
Subject: [PATCH 05/54] add functions to deal with schema extensions, suffixes
 and entities

---
 +bids/+internal/return_datatype_entities.m   |  5 +++++
 +bids/+internal/return_datatype_extensions.m | 14 +++++++++++++
 +bids/+internal/return_datatype_suffixes.m   | 12 +++++++++++
 tests/test_return_datatype_entities.m        | 21 ++++++++++++++++++++
 tests/test_return_datatype_extensions.m      |  8 ++++++++
 tests/test_return_datatype_suffixes.m        |  8 ++++++++
 6 files changed, 68 insertions(+)
 create mode 100644 +bids/+internal/return_datatype_entities.m
 create mode 100644 +bids/+internal/return_datatype_extensions.m
 create mode 100644 +bids/+internal/return_datatype_suffixes.m
 create mode 100644 tests/test_return_datatype_entities.m
 create mode 100644 tests/test_return_datatype_extensions.m
 create mode 100644 tests/test_return_datatype_suffixes.m

diff --git a/+bids/+internal/return_datatype_entities.m b/+bids/+internal/return_datatype_entities.m
new file mode 100644
index 00000000..3c0b7882
--- /dev/null
+++ b/+bids/+internal/return_datatype_entities.m
@@ -0,0 +1,5 @@
+function entities = return_datatype_entities(datatype)
+
+  entities = fieldnames(datatype.entities);
+
+end
diff --git a/+bids/+internal/return_datatype_extensions.m b/+bids/+internal/return_datatype_extensions.m
new file mode 100644
index 00000000..55a51bcd
--- /dev/null
+++ b/+bids/+internal/return_datatype_extensions.m
@@ -0,0 +1,14 @@
+function extensions = return_datatype_extensions(datatype)
+
+  extensions = '\\(';
+
+  for iExt = 1:numel(datatype.extensions)
+    if ~strcmp(datatype.extensions{iExt}, '.json')
+      extensions = [extensions,  datatype.extensions{iExt}, '|']; %#ok<AGROW>
+    end
+  end
+
+  % Replace final "|" by a ")"
+  extensions(end) = ')';
+
+end
diff --git a/+bids/+internal/return_datatype_suffixes.m b/+bids/+internal/return_datatype_suffixes.m
new file mode 100644
index 00000000..0f64954e
--- /dev/null
+++ b/+bids/+internal/return_datatype_suffixes.m
@@ -0,0 +1,12 @@
+function suffixes = return_datatype_suffixes(datatype)
+
+  suffixes = '\\_(';
+
+  for iExt = 1:numel(datatype.suffixes)
+    suffixes = [suffixes,  datatype.suffixes{iExt}, '|']; %#ok<AGROW>
+  end
+
+  % Replace final "|" by a ")"
+  suffixes(end) = ')';
+
+end
diff --git a/tests/test_return_datatype_entities.m b/tests/test_return_datatype_entities.m
new file mode 100644
index 00000000..e9e588d3
--- /dev/null
+++ b/tests/test_return_datatype_entities.m
@@ -0,0 +1,21 @@
+function test_return_datatype_entities
+
+  schema = bids.internal.load_schema();
+
+  entities = bids.internal.return_datatype_entities(schema.datatypes.func(1));
+
+  expected_output = {
+                     'sub'
+                     'ses'
+                     'task'
+                     'acq'
+                     'ce'
+                     'rec'
+                     'dir'
+                     'run'
+                     'echo'
+                     'part'};
+
+  assert(isequal(entities, expected_output));
+
+end
diff --git a/tests/test_return_datatype_extensions.m b/tests/test_return_datatype_extensions.m
new file mode 100644
index 00000000..4b0be48b
--- /dev/null
+++ b/tests/test_return_datatype_extensions.m
@@ -0,0 +1,8 @@
+function test_return_datatype_extensions
+
+  schema = bids.internal.load_schema();
+
+  extensions = bids.internal.return_datatype_extensions(schema.datatypes.func(1));
+  assert(isequal(extensions, '\\(.nii.gz|.nii)'));
+
+end
diff --git a/tests/test_return_datatype_suffixes.m b/tests/test_return_datatype_suffixes.m
new file mode 100644
index 00000000..36236ef1
--- /dev/null
+++ b/tests/test_return_datatype_suffixes.m
@@ -0,0 +1,8 @@
+function test_return_datatype_suffixes
+
+  schema = bids.internal.load_schema();
+
+  suffixes = bids.internal.return_datatype_suffixes(schema.datatypes.func(1));
+  assert(isequal(suffixes, '\\_(bold|cbv|sbref)'));
+
+end

From aacc8044155882ae50bf910b3b74423643098a20 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sat, 16 Jan 2021 15:06:24 +0100
Subject: [PATCH 06/54] add function to return regular expression for a certain
 data type

---
 +bids/+internal/return_datatype_extensions.m  |  6 +++---
 .../return_datatype_regular_expression.m      |  8 ++++++++
 +bids/+internal/return_datatype_suffixes.m    |  6 +++---
 tests/test_return_datatype_extensions.m       |  2 +-
 .../test_return_datatype_regular_expression.m | 19 +++++++++++++++++++
 tests/test_return_datatype_suffixes.m         |  2 +-
 6 files changed, 35 insertions(+), 8 deletions(-)
 create mode 100644 +bids/+internal/return_datatype_regular_expression.m
 create mode 100644 tests/test_return_datatype_regular_expression.m

diff --git a/+bids/+internal/return_datatype_extensions.m b/+bids/+internal/return_datatype_extensions.m
index 55a51bcd..4a59932a 100644
--- a/+bids/+internal/return_datatype_extensions.m
+++ b/+bids/+internal/return_datatype_extensions.m
@@ -1,6 +1,6 @@
 function extensions = return_datatype_extensions(datatype)
 
-  extensions = '\\(';
+  extensions = '(';
 
   for iExt = 1:numel(datatype.extensions)
     if ~strcmp(datatype.extensions{iExt}, '.json')
@@ -8,7 +8,7 @@
     end
   end
 
-  % Replace final "|" by a ")"
-  extensions(end) = ')';
+  % Replace final "|" by a "){1}"
+  extensions(end:end + 3) = '){1}';
 
 end
diff --git a/+bids/+internal/return_datatype_regular_expression.m b/+bids/+internal/return_datatype_regular_expression.m
new file mode 100644
index 00000000..0bbaebbf
--- /dev/null
+++ b/+bids/+internal/return_datatype_regular_expression.m
@@ -0,0 +1,8 @@
+function regular_expression = return_datatype_regular_expression(datatype)
+
+  suffixes = bids.internal.return_datatype_suffixes(datatype);
+  extensions = bids.internal.return_datatype_extensions(datatype);
+
+  regular_expression = ['^%s.*' suffixes extensions '$'];
+
+end
diff --git a/+bids/+internal/return_datatype_suffixes.m b/+bids/+internal/return_datatype_suffixes.m
index 0f64954e..4ce7476a 100644
--- a/+bids/+internal/return_datatype_suffixes.m
+++ b/+bids/+internal/return_datatype_suffixes.m
@@ -1,12 +1,12 @@
 function suffixes = return_datatype_suffixes(datatype)
 
-  suffixes = '\\_(';
+  suffixes = '_(';
 
   for iExt = 1:numel(datatype.suffixes)
     suffixes = [suffixes,  datatype.suffixes{iExt}, '|']; %#ok<AGROW>
   end
 
-  % Replace final "|" by a ")"
-  suffixes(end) = ')';
+  % Replace final "|" by a "){1}"
+  suffixes(end:end + 3) = '){1}';
 
 end
diff --git a/tests/test_return_datatype_extensions.m b/tests/test_return_datatype_extensions.m
index 4b0be48b..dd9ec4df 100644
--- a/tests/test_return_datatype_extensions.m
+++ b/tests/test_return_datatype_extensions.m
@@ -3,6 +3,6 @@
   schema = bids.internal.load_schema();
 
   extensions = bids.internal.return_datatype_extensions(schema.datatypes.func(1));
-  assert(isequal(extensions, '\\(.nii.gz|.nii)'));
+  assert(isequal(extensions, '(.nii.gz|.nii){1}'));
 
 end
diff --git a/tests/test_return_datatype_regular_expression.m b/tests/test_return_datatype_regular_expression.m
new file mode 100644
index 00000000..7d3fd3db
--- /dev/null
+++ b/tests/test_return_datatype_regular_expression.m
@@ -0,0 +1,19 @@
+function test_return_datatype_regular_expression
+
+  schema = bids.internal.load_schema();
+
+  regular_expression = bids.internal.return_datatype_regular_expression(schema.datatypes.anat(1));
+
+  expected_expression = ['^%s.*', ...
+                         '_(T1w|T2w|PDw|T2starw|FLAIR|inplaneT1|inplaneT2|PDT2|angio){1}', ...
+                         '(.nii.gz|.nii){1}$'];
+
+  assert(isequal(regular_expression, expected_expression));
+
+  data_dir = fullfile(fileparts(mfilename('fullpath')), 'data', 'MoAEpilot', 'sub-01', 'anat');
+  subject_name = 'sub-01';
+  file = bids.internal.file_utils('List', data_dir, sprintf(expected_expression, subject_name));
+
+  assert(isequal(file, 'sub-01_T1w.nii.gz'));
+
+end
diff --git a/tests/test_return_datatype_suffixes.m b/tests/test_return_datatype_suffixes.m
index 36236ef1..749008db 100644
--- a/tests/test_return_datatype_suffixes.m
+++ b/tests/test_return_datatype_suffixes.m
@@ -3,6 +3,6 @@
   schema = bids.internal.load_schema();
 
   suffixes = bids.internal.return_datatype_suffixes(schema.datatypes.func(1));
-  assert(isequal(suffixes, '\\_(bold|cbv|sbref)'));
+  assert(isequal(suffixes, '_(bold|cbv|sbref){1}'));
 
 end

From e14da01f60cd175d58afe404a1a5ec07d7409334 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Thu, 4 Feb 2021 06:02:54 +0100
Subject: [PATCH 07/54] update return_datatype_entities to new schema structure

---
 +bids/+internal/return_datatype_entities.m |  8 +++++++-
 tests/test_return_datatype_entities.m      | 12 +-----------
 2 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/+bids/+internal/return_datatype_entities.m b/+bids/+internal/return_datatype_entities.m
index 3c0b7882..8521d5cf 100644
--- a/+bids/+internal/return_datatype_entities.m
+++ b/+bids/+internal/return_datatype_entities.m
@@ -1,5 +1,11 @@
 function entities = return_datatype_entities(datatype)
 
-  entities = fieldnames(datatype.entities);
+  schema = bids.internal.load_schema();
+
+  entity_names = fieldnames(datatype.entities);
+
+  for i = 1:size(entity_names, 1)
+    entities{1, i} = schema.entities.(entity_names{i}).entity; %#ok<*AGROW>
+  end
 
 end
diff --git a/tests/test_return_datatype_entities.m b/tests/test_return_datatype_entities.m
index e9e588d3..d290dc2f 100644
--- a/tests/test_return_datatype_entities.m
+++ b/tests/test_return_datatype_entities.m
@@ -4,17 +4,7 @@
 
   entities = bids.internal.return_datatype_entities(schema.datatypes.func(1));
 
-  expected_output = {
-                     'sub'
-                     'ses'
-                     'task'
-                     'acq'
-                     'ce'
-                     'rec'
-                     'dir'
-                     'run'
-                     'echo'
-                     'part'};
+  expected_output = {'sub','ses','task','acq','ce','rec','dir','run','echo','part'};
 
   assert(isequal(entities, expected_output));
 

From c76bf485d6f224de7be27b3e8dabaf57e8f479b5 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Thu, 4 Feb 2021 09:57:34 +0100
Subject: [PATCH 08/54] move schema functions

---
 +bids/{+internal => +schema}/load_schema.m              | 0
 +bids/{+internal => +schema}/return_datatype_entities.m | 2 +-
 +bids/layout.m                                          | 2 +-
 tests/test_load_schema.m                                | 2 +-
 tests/test_return_datatype_entities.m                   | 4 ++--
 5 files changed, 5 insertions(+), 5 deletions(-)
 rename +bids/{+internal => +schema}/load_schema.m (100%)
 rename +bids/{+internal => +schema}/return_datatype_entities.m (85%)

diff --git a/+bids/+internal/load_schema.m b/+bids/+schema/load_schema.m
similarity index 100%
rename from +bids/+internal/load_schema.m
rename to +bids/+schema/load_schema.m
diff --git a/+bids/+internal/return_datatype_entities.m b/+bids/+schema/return_datatype_entities.m
similarity index 85%
rename from +bids/+internal/return_datatype_entities.m
rename to +bids/+schema/return_datatype_entities.m
index 8521d5cf..fb15159e 100644
--- a/+bids/+internal/return_datatype_entities.m
+++ b/+bids/+schema/return_datatype_entities.m
@@ -1,6 +1,6 @@
 function entities = return_datatype_entities(datatype)
 
-  schema = bids.internal.load_schema();
+  schema = bids.schema.load_schema();
 
   entity_names = fieldnames(datatype.entities);
 
diff --git a/+bids/layout.m b/+bids/layout.m
index 66318485..95d2fe4d 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -191,7 +191,7 @@ function tolerant_message(tolerant, msg)
   subject.pet     = struct([]); % PET imaging data
 
   % use BIDS schema to organizing parsing of subject data
-  schema = bids.internal.load_schema();
+  schema = bids.schema.load_schema();
   modalities = fieldnames(schema.modalities);
 
   for iModality = 1:numel(modalities)
diff --git a/tests/test_load_schema.m b/tests/test_load_schema.m
index 18355b2e..0a94f39e 100644
--- a/tests/test_load_schema.m
+++ b/tests/test_load_schema.m
@@ -2,7 +2,7 @@ function test_load_schema()
 
   SCHEMA_DIR = fullfile(fileparts(mfilename('fullpath')), 'schema');
 
-  schema = bids.internal.load_schema(SCHEMA_DIR);
+  schema = bids.schema.load_schema(SCHEMA_DIR);
 
   assert(isfield(schema, 'base'));
   assert(isfield(schema, 'subfolder_1'));
diff --git a/tests/test_return_datatype_entities.m b/tests/test_return_datatype_entities.m
index d290dc2f..b439a12a 100644
--- a/tests/test_return_datatype_entities.m
+++ b/tests/test_return_datatype_entities.m
@@ -1,8 +1,8 @@
 function test_return_datatype_entities
 
-  schema = bids.internal.load_schema();
+  schema = bids.schema.load_schema();
 
-  entities = bids.internal.return_datatype_entities(schema.datatypes.func(1));
+  entities = bids.schema.return_datatype_entities(schema.datatypes.func(1));
 
   expected_output = {'sub','ses','task','acq','ce','rec','dir','run','echo','part'};
 

From 84393d68e73ea0d7646c8d1a83282ecdfc9211c4 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Thu, 4 Feb 2021 10:08:01 +0100
Subject: [PATCH 09/54] update tests to moxunit format

---
 tests/test_file_utils.m                       | 10 ++++++++-
 ...ignatures.m => test_function_signatures.m} | 21 ++++++++++++-------
 tests/test_load_schema.m                      | 10 ++++++++-
 tests/test_return_datatype_entities.m         | 11 +++++++++-
 tests/test_return_datatype_extensions.m       | 12 +++++++++--
 .../test_return_datatype_regular_expression.m | 12 +++++++++--
 tests/test_return_datatype_suffixes.m         | 12 +++++++++--
 7 files changed, 72 insertions(+), 16 deletions(-)
 rename tests/{test_functionSignatures.m => test_function_signatures.m} (61%)

diff --git a/tests/test_file_utils.m b/tests/test_file_utils.m
index 6279b2fb..6c3f4c36 100644
--- a/tests/test_file_utils.m
+++ b/tests/test_file_utils.m
@@ -1,4 +1,12 @@
-function test_file_utils()
+function test_suite = test_file_utils %#ok<*STOUT>
+  try % assignment of 'localfunctions' is necessary in Matlab >= 2016
+    test_functions = localfunctions(); %#ok<*NASGU>
+  catch % no problem; early Matlab versions can use initTestSuite fine
+  end
+  initTestSuite;
+end
+
+function test_file_utils_basic()
 
   %% test to get certain part of a filename
   % {'path', 'basename', 'ext', 'filename', 'cpath', 'fpath'}
diff --git a/tests/test_functionSignatures.m b/tests/test_function_signatures.m
similarity index 61%
rename from tests/test_functionSignatures.m
rename to tests/test_function_signatures.m
index 59730c28..5d78cf7b 100644
--- a/tests/test_functionSignatures.m
+++ b/tests/test_function_signatures.m
@@ -1,4 +1,12 @@
-function test_functionSignatures(pth)
+function test_suite = test_function_signatures %#ok<*STOUT>
+  try % assignment of 'localfunctions' is necessary in Matlab >= 2016
+    test_functions = localfunctions(); %#ok<*NASGU>
+  catch % no problem; early Matlab versions can use initTestSuite fine
+  end
+  initTestSuite;
+end
+
+function test_function_signatures_basic()
   % Test functionSignatures.json file
   % The functionSignatures file is used by Matlab to provide code suggestions
   % and completions for functions.
@@ -13,12 +21,11 @@ function test_functionSignatures(pth)
   %
   % Copyright (C) 2020--, BIDS-MATLAB developers
 
-  if ~nargin
-    % parent directory of the tests directory
-    pth = fullfile(pwd, '..');
-  end
-
+  root_dir = fullfile(fileparts(mfilename('fullpath')), '..');
+  
   % Run a smoke test - see if the file is a readable json
   signatures = bids.util.jsondecode( ...
-                                    fullfile(pth, 'functionSignatures.json'));
+                                    fullfile(root_dir, 'functionSignatures.json'));
   assert(isstruct(signatures));
+  
+end
diff --git a/tests/test_load_schema.m b/tests/test_load_schema.m
index 0a94f39e..c0aa145f 100644
--- a/tests/test_load_schema.m
+++ b/tests/test_load_schema.m
@@ -1,4 +1,12 @@
-function test_load_schema()
+function test_suite = test_load_schema %#ok<*STOUT>
+  try % assignment of 'localfunctions' is necessary in Matlab >= 2016
+    test_functions = localfunctions(); %#ok<*NASGU>
+  catch % no problem; early Matlab versions can use initTestSuite fine
+  end
+  initTestSuite;
+end
+
+function test_load_schema_basic()
 
   SCHEMA_DIR = fullfile(fileparts(mfilename('fullpath')), 'schema');
 
diff --git a/tests/test_return_datatype_entities.m b/tests/test_return_datatype_entities.m
index b439a12a..de7bd38d 100644
--- a/tests/test_return_datatype_entities.m
+++ b/tests/test_return_datatype_entities.m
@@ -1,4 +1,13 @@
-function test_return_datatype_entities
+function test_suite = test_return_datatype_entities %#ok<*STOUT>
+  try % assignment of 'localfunctions' is necessary in Matlab >= 2016
+    test_functions = localfunctions(); %#ok<*NASGU>
+  catch % no problem; early Matlab versions can use initTestSuite fine
+  end
+  initTestSuite;
+end
+
+
+function test_return_datatype_entities_basic
 
   schema = bids.schema.load_schema();
 
diff --git a/tests/test_return_datatype_extensions.m b/tests/test_return_datatype_extensions.m
index dd9ec4df..c73515ac 100644
--- a/tests/test_return_datatype_extensions.m
+++ b/tests/test_return_datatype_extensions.m
@@ -1,6 +1,14 @@
-function test_return_datatype_extensions
+function test_suite = test_return_datatype_extensions %#ok<*STOUT>
+  try % assignment of 'localfunctions' is necessary in Matlab >= 2016
+    test_functions = localfunctions(); %#ok<*NASGU>
+  catch % no problem; early Matlab versions can use initTestSuite fine
+  end
+  initTestSuite;
+end
+
+function test_return_datatype_extensions_basic
 
-  schema = bids.internal.load_schema();
+  schema = bids.schema.load_schema();
 
   extensions = bids.internal.return_datatype_extensions(schema.datatypes.func(1));
   assert(isequal(extensions, '(.nii.gz|.nii){1}'));
diff --git a/tests/test_return_datatype_regular_expression.m b/tests/test_return_datatype_regular_expression.m
index 7d3fd3db..05d367c5 100644
--- a/tests/test_return_datatype_regular_expression.m
+++ b/tests/test_return_datatype_regular_expression.m
@@ -1,6 +1,14 @@
-function test_return_datatype_regular_expression
+function test_suite = test_return_datatype_regular_expression %#ok<*STOUT>
+  try % assignment of 'localfunctions' is necessary in Matlab >= 2016
+    test_functions = localfunctions(); %#ok<*NASGU>
+  catch % no problem; early Matlab versions can use initTestSuite fine
+  end
+  initTestSuite;
+end
+
+function test_return_datatype_regular_expression_basic
 
-  schema = bids.internal.load_schema();
+  schema = bids.schema.load_schema();
 
   regular_expression = bids.internal.return_datatype_regular_expression(schema.datatypes.anat(1));
 
diff --git a/tests/test_return_datatype_suffixes.m b/tests/test_return_datatype_suffixes.m
index 749008db..4ff770a9 100644
--- a/tests/test_return_datatype_suffixes.m
+++ b/tests/test_return_datatype_suffixes.m
@@ -1,6 +1,14 @@
-function test_return_datatype_suffixes
+function test_suite = test_return_datatype_suffixes %#ok<*STOUT>
+  try % assignment of 'localfunctions' is necessary in Matlab >= 2016
+    test_functions = localfunctions(); %#ok<*NASGU>
+  catch % no problem; early Matlab versions can use initTestSuite fine
+  end
+  initTestSuite;
+end
+
+function test_return_datatype_suffixes_basic
 
-  schema = bids.internal.load_schema();
+  schema = bids.schema.load_schema();
 
   suffixes = bids.internal.return_datatype_suffixes(schema.datatypes.func(1));
   assert(isequal(suffixes, '_(bold|cbv|sbref){1}'));

From 8dac58498a8e2c0c15a30e6de8c6813920702452 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Thu, 4 Feb 2021 10:31:43 +0100
Subject: [PATCH 10/54] add test for parse_filename

---
 +bids/+internal/parse_filename.m |  3 +-
 tests/test_parse_filename.m      | 74 ++++++++++++++++++++++++++++++++
 2 files changed, 76 insertions(+), 1 deletion(-)
 create mode 100644 tests/test_parse_filename.m

diff --git a/+bids/+internal/parse_filename.m b/+bids/+internal/parse_filename.m
index 681a2c8e..99a3742f 100644
--- a/+bids/+internal/parse_filename.m
+++ b/+bids/+internal/parse_filename.m
@@ -50,7 +50,8 @@
     try
       p = orderfields(p, ['filename', 'ext', 'type', fields]);
     catch
-      warning('Ignoring file ''%s'' not matching template.', filename);
+      warning('bidsMatlab:noMatchingTemplate', ...
+              'Ignoring file %s not matching template.', filename);
       p = struct([]);
     end
   end
diff --git a/tests/test_parse_filename.m b/tests/test_parse_filename.m
new file mode 100644
index 00000000..55cd354e
--- /dev/null
+++ b/tests/test_parse_filename.m
@@ -0,0 +1,74 @@
+function test_suite = test_parse_filename %#ok<*STOUT>
+  try % assignment of 'localfunctions' is necessary in Matlab >= 2016
+    test_functions = localfunctions(); %#ok<*NASGU>
+  catch % no problem; early Matlab versions can use initTestSuite fine
+  end
+  initTestSuite;
+end
+
+function test_parse_filename_basic()
+
+  filename = '../sub-16/anat/sub-16_ses-mri_run-1_echo-2_FLASH.nii.gz';
+  output = bids.internal.parse_filename(filename);
+
+  expected = struct( ...
+                    'filename', 'sub-16_ses-mri_run-1_echo-2_FLASH.nii.gz', ...
+                    'type', 'FLASH', ...
+                    'ext', '.nii.gz', ...
+                    'sub', '16', ...
+                    'ses', 'mri', ...
+                    'run', '1', ...
+                    'echo', '2');
+
+  assertEqual(output, expected);
+
+end
+
+function test_parse_filename_fields()
+
+  filename = '../sub-16/anat/sub-16_ses-mri_run-1_echo-2_FLASH.nii.gz';
+  fields = {'sub', 'ses', 'run', 'echo'};
+  output = bids.internal.parse_filename(filename);
+
+  expected = struct( ...
+                    'filename', 'sub-16_ses-mri_run-1_echo-2_FLASH.nii.gz', ...
+                    'type', 'FLASH', ...
+                    'ext', '.nii.gz', ...
+                    'sub', '16', ...
+                    'ses', 'mri', ...
+                    'run', '1', ...
+                    'echo', '2');
+
+  assertEqual(output, expected);
+
+end
+
+function test_parse_filename_missing_field()
+
+  filename = '../sub-16/anat/sub-16_run-1_echo-2_FLASH.nii.gz';
+  fields = {'sub', 'ses', 'run', 'echo'};
+  output = bids.internal.parse_filename(filename);
+
+  expected = struct( ...
+                    'filename', 'sub-16_run-1_echo-2_FLASH.nii.gz', ...
+                    'type', 'FLASH', ...
+                    'ext', '.nii.gz', ...
+                    'sub', '16', ...
+                    'ses', '', ...
+                    'run', '1', ...
+                    'echo', '2');
+
+  % Would fail as "missing fields" are currently not returned.
+  %     assertEqual(output, expected);
+
+end
+
+function test_parse_filename_wrong_template()
+
+  filename = '../sub-16/anat/sub-16_ses-mri_run-1_echo-2_FLASH.nii.gz';
+
+  assertWarning( ...
+                @()bids.internal.parse_filename(filename, {'echo'}), ...
+                'bidsMatlab:noMatchingTemplate');
+
+end

From 951077cbd6ff8e63abc539d41a42520aa4aef4e5 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Thu, 4 Feb 2021 13:28:08 +0100
Subject: [PATCH 11/54] use schema to parse anat

---
 +bids/+internal/append_to_structure.m | 62 +++++++++++++++++++++++
 +bids/layout.m                        | 11 ++--
 tests/test_append_to_structure.m      | 72 +++++++++++++++++++++++++++
 tests/test_layout.m                   | 14 ++++++
 4 files changed, 154 insertions(+), 5 deletions(-)
 create mode 100644 +bids/+internal/append_to_structure.m
 create mode 100644 tests/test_append_to_structure.m
 create mode 100644 tests/test_layout.m

diff --git a/+bids/+internal/append_to_structure.m b/+bids/+internal/append_to_structure.m
new file mode 100644
index 00000000..3d825d88
--- /dev/null
+++ b/+bids/+internal/append_to_structure.m
@@ -0,0 +1,62 @@
+function subject = append_to_structure(file, subject, modality)
+  % Copyright (C) 2021--, BIDS-MATLAB developers
+
+  p = bids.internal.parse_filename(file);
+  idx = find_suffix_group(modality, p.type);
+  if isempty(idx)
+    warning('append_to_structure:noMatchingSuffix', ...
+            'Skipping file with no valid suffix in schema: %s', file);
+    return
+  end
+
+  schema = bids.schema.load_schema();
+  entities = bids.schema.return_datatype_entities(schema.datatypes.(modality)(idx));
+  p = bids.internal.parse_filename(file, entities);
+
+  if ~isempty(subject.(modality))
+
+    missing_fields = setxor(fieldnames(subject.(modality)), fieldnames(p));
+
+    if ~isempty(missing_fields)
+      for iField = 1:numel(missing_fields)
+        p = add_missing_field(p, ...
+                              missing_fields{iField});
+        subject.(modality) = add_missing_field(subject.(modality), ...
+                                               missing_fields{iField});
+      end
+    end
+
+  end
+
+  subject.(modality) = [subject.(modality) p];
+
+end
+
+function structure = add_missing_field(structure, field)
+  if ~isfield(structure, field)
+    structure.(field) = '';
+  end
+end
+
+function idx = find_suffix_group(modality, suffix)
+
+  idx = [];
+
+  schema = bids.schema.load_schema();
+  suffix_groups = {schema.datatypes.(modality).suffixes}';
+
+  % the following loop could probably be improved with some cellfun magic
+  %   cellfun(@(x, y) any(strcmp(x,y)), {p.type}, suffix_groups)
+  for i = 1:numel(suffix_groups)
+    if any(strcmp(suffix, suffix_groups{i}))
+      idx = i;
+      break
+    end
+  end
+
+  if isempty(idx)
+    warning('findSuffix:noMatchingSuffix', ...
+            'No corresponding suffix in schema for %s for datatype %s', suffix, modality);
+  end
+
+end
diff --git a/+bids/layout.m b/+bids/layout.m
index 95d2fe4d..c5fb7b07 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -229,22 +229,23 @@ function tolerant_message(tolerant, msg)
 
 function subject = parse_anat(subject)
 
+  datatype = 'anat';
+
   % --------------------------------------------------------------------------
   % -Anatomy imaging data
   % --------------------------------------------------------------------------
-  pth = fullfile(subject.path, 'anat');
+  pth = fullfile(subject.path, datatype);
 
   if exist(pth, 'dir')
 
-    entities = return_entities('anat');
-
-    file_list = return_file_list('anat', subject);
+    file_list = return_file_list(datatype, subject);
 
     for i = 1:numel(file_list)
 
-      subject = append_to_structure(file_list{i}, entities, subject, 'anat');
+      subject = bids.internal.append_to_structure(file_list{i}, subject, datatype);
 
     end
+
   end
 
 end
diff --git a/tests/test_append_to_structure.m b/tests/test_append_to_structure.m
new file mode 100644
index 00000000..161727fd
--- /dev/null
+++ b/tests/test_append_to_structure.m
@@ -0,0 +1,72 @@
+function test_suite = test_append_to_structure %#ok<*STOUT>
+  try % assignment of 'localfunctions' is necessary in Matlab >= 2016
+    test_functions = localfunctions(); %#ok<*NASGU>
+  catch % no problem; early Matlab versions can use initTestSuite fine
+  end
+  initTestSuite;
+end
+
+function test_append_to_structure_basic()
+
+  subject = struct('anat', struct([]));
+
+  modality = 'anat';
+
+  file = '../sub-16/anat/sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz';
+  entities = {'sub', 'ses', 'run', 'acq', 'ce', 'rec', 'part'};
+  subject = bids.internal.append_to_structure(file, subject, modality);
+
+  expected.anat = struct( ...
+                         'filename', 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz', ...
+                         'type', 'T1w', ...
+                         'ext', '.nii.gz', ...
+                         'sub', '16', ...
+                         'ses', 'mri', ...
+                         'run', '1', ...
+                         'acq', 'hd', ...
+                         'ce', '', ...
+                         'rec', '', ...
+                         'part', '');
+
+  assertEqual(subject.anat, expected.anat);
+
+end
+
+function test_append_to_structure_basic_test()
+
+  subject = struct('anat', struct([]));
+  modality = 'anat';
+
+  file = '../sub-16/anat/sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz';
+  entities = {'sub', 'ses', 'run', 'acq', 'ce', 'rec', 'part'};
+  subject = bids.internal.append_to_structure(file, subject, modality);
+
+  file = '../sub-16/anat/sub-16_ses-mri_run-1_T1map.nii.gz';
+  entities = {'sub', 'ses', 'run', 'acq', 'ce', 'rec'};
+  subject = bids.internal.append_to_structure(file, subject, modality);
+
+  expected(1).anat = struct( ...
+                            'filename', 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz', ...
+                            'type', 'T1w', ...
+                            'ext', '.nii.gz', ...
+                            'sub', '16', ...
+                            'ses', 'mri', ...
+                            'run', '1', ...
+                            'acq', 'hd', ...
+                            'ce', '', ...
+                            'rec', '', ...
+                            'part', '');
+
+  expected(2).anat = struct( ...
+                            'filename', 'sub-16_ses-mri_run-1_T1map.nii.gz', ...
+                            'type', 'T1map', ...
+                            'ext', '.nii.gz', ...
+                            'sub', '16', ...
+                            'ses', 'mri', ...
+                            'run', '1', ...
+                            'acq', '', ...
+                            'ce', '', ...
+                            'rec', '', ...
+                            'part', '');     %#ok<*STRNU>
+
+end
diff --git a/tests/test_layout.m b/tests/test_layout.m
new file mode 100644
index 00000000..b4757550
--- /dev/null
+++ b/tests/test_layout.m
@@ -0,0 +1,14 @@
+function test_suite = test_layout %#ok<*STOUT>
+  try % assignment of 'localfunctions' is necessary in Matlab >= 2016
+    test_functions = localfunctions(); %#ok<*NASGU>
+  catch % no problem; early Matlab versions can use initTestSuite fine
+  end
+  initTestSuite;
+end
+
+function test_layout_smoke_test()
+
+  pth_bids_example = get_test_data_dir();
+  BIDS = bids.layout(fullfile(pth_bids_example, '7t_trt'));
+
+end

From fe384168dbd8be2ab1cf4c9ccbe6b44e8d179933 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Thu, 4 Feb 2021 13:29:01 +0100
Subject: [PATCH 12/54] linting, cleaning, use valid bids example for
 parse_filename

---
 +bids/+internal/parse_filename.m      |  8 ++---
 tests/test_bids_examples.m            | 10 +++---
 tests/test_function_signatures.m      |  4 +--
 tests/test_parse_filename.m           | 52 ++++++++++++---------------
 tests/test_return_datatype_entities.m |  3 +-
 5 files changed, 34 insertions(+), 43 deletions(-)

diff --git a/+bids/+internal/parse_filename.m b/+bids/+internal/parse_filename.m
index 99a3742f..ea5bf894 100644
--- a/+bids/+internal/parse_filename.m
+++ b/+bids/+internal/parse_filename.m
@@ -4,20 +4,20 @@
   %
   % Example:
   %
-  % >> filename = '../sub-16/anat/sub-16_ses-mri_run-1_echo-2_FLASH.nii.gz';
+  % >> filename = '../sub-16/anat/sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz';
   % >> bids.internal.parse_filename(filename)
   %
   % ans =
   %
   %   struct with fields:
   %
-  %     filename: 'sub-16_ses-mri_run-1_echo-2_FLASH.nii.gz'
-  %         type: 'FLASH'
+  %     filename: 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz'
+  %         type: 'T1w'
   %          ext: '.nii.gz'
   %          sub: '16'
   %          ses: 'mri'
   %          run: '1'
-  %         echo: '2'
+  %         acq: 'hd'
   % __________________________________________________________________________
 
   % Copyright (C) 2016-2018, Guillaume Flandin, Wellcome Centre for Human Neuroimaging
diff --git a/tests/test_bids_examples.m b/tests/test_bids_examples.m
index 1cd30282..8db00981 100644
--- a/tests/test_bids_examples.m
+++ b/tests/test_bids_examples.m
@@ -30,17 +30,17 @@ function test_bids_examples_basic()
 
   % -Try to run bids.layout on each dataset directory and keep track of any
   % failure with a try/catch
-  sts = false(1, numel(d));
+  status = false(1, numel(d));
   msg = cell(1, numel(d));
   for i = 1:numel(d)
     if exist(fullfile(pth_bids_example, d(i).name, '.SKIP_VALIDATION'), 'file')
-      sts(i) = true;
+      status(i) = true;
       fprintf('-');
       continue
     end
     try
       BIDS = bids.layout(fullfile(pth_bids_example, d(i).name));
-      sts(i) = true;
+      status(i) = true;
       fprintf('.');
     catch err
       fprintf('X');
@@ -50,8 +50,8 @@ function test_bids_examples_basic()
   fprintf('\n');
 
   % lists all the folder for which bids.layout failed
-  if ~all(sts)
-    for i = find(~sts)
+  if ~all(status)
+    for i = find(~status)
       fprintf('* %s: %s\n', d(i).name, msg{i});
     end
     error('Parsing of BIDS-compatible datasets failed.');
diff --git a/tests/test_function_signatures.m b/tests/test_function_signatures.m
index 5d78cf7b..f63b4ad8 100644
--- a/tests/test_function_signatures.m
+++ b/tests/test_function_signatures.m
@@ -22,10 +22,10 @@ function test_function_signatures_basic()
   % Copyright (C) 2020--, BIDS-MATLAB developers
 
   root_dir = fullfile(fileparts(mfilename('fullpath')), '..');
-  
+
   % Run a smoke test - see if the file is a readable json
   signatures = bids.util.jsondecode( ...
                                     fullfile(root_dir, 'functionSignatures.json'));
   assert(isstruct(signatures));
-  
+
 end
diff --git a/tests/test_parse_filename.m b/tests/test_parse_filename.m
index 55cd354e..3dc2b324 100644
--- a/tests/test_parse_filename.m
+++ b/tests/test_parse_filename.m
@@ -8,17 +8,29 @@
 
 function test_parse_filename_basic()
 
-  filename = '../sub-16/anat/sub-16_ses-mri_run-1_echo-2_FLASH.nii.gz';
+  filename = '../sub-16/anat/sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz';
   output = bids.internal.parse_filename(filename);
 
   expected = struct( ...
-                    'filename', 'sub-16_ses-mri_run-1_echo-2_FLASH.nii.gz', ...
-                    'type', 'FLASH', ...
+                    'filename', 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz', ...
+                    'type', 'T1w', ...
                     'ext', '.nii.gz', ...
                     'sub', '16', ...
                     'ses', 'mri', ...
                     'run', '1', ...
-                    'echo', '2');
+                    'acq', 'hd');
+
+  %     expected = struct( ...
+  %         'filename', 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz', ...
+  %         'type', 'T1w', ...
+  %         'ext', '.nii.gz', ...
+  %         'sub', '16', ...
+  %         'ses', 'mri', ...
+  %         'run', '1', ...
+  %         'acq', 'hd', ...
+  %         'ce', '', ...
+  %         'rec', '', ...
+  %         'part', '');
 
   assertEqual(output, expected);
 
@@ -26,46 +38,26 @@ function test_parse_filename_basic()
 
 function test_parse_filename_fields()
 
-  filename = '../sub-16/anat/sub-16_ses-mri_run-1_echo-2_FLASH.nii.gz';
-  fields = {'sub', 'ses', 'run', 'echo'};
+  filename = '../sub-16/anat/sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz';
+  fields = {'sub', 'ses', 'run', 'acq', 'ce', 'rec', 'part'};
   output = bids.internal.parse_filename(filename);
 
   expected = struct( ...
-                    'filename', 'sub-16_ses-mri_run-1_echo-2_FLASH.nii.gz', ...
-                    'type', 'FLASH', ...
+                    'filename', 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz', ...
+                    'type', 'T1w', ...
                     'ext', '.nii.gz', ...
                     'sub', '16', ...
                     'ses', 'mri', ...
                     'run', '1', ...
-                    'echo', '2');
+                    'acq', 'hd');
 
   assertEqual(output, expected);
 
 end
 
-function test_parse_filename_missing_field()
-
-  filename = '../sub-16/anat/sub-16_run-1_echo-2_FLASH.nii.gz';
-  fields = {'sub', 'ses', 'run', 'echo'};
-  output = bids.internal.parse_filename(filename);
-
-  expected = struct( ...
-                    'filename', 'sub-16_run-1_echo-2_FLASH.nii.gz', ...
-                    'type', 'FLASH', ...
-                    'ext', '.nii.gz', ...
-                    'sub', '16', ...
-                    'ses', '', ...
-                    'run', '1', ...
-                    'echo', '2');
-
-  % Would fail as "missing fields" are currently not returned.
-  %     assertEqual(output, expected);
-
-end
-
 function test_parse_filename_wrong_template()
 
-  filename = '../sub-16/anat/sub-16_ses-mri_run-1_echo-2_FLASH.nii.gz';
+  filename = '../sub-16/anat/sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz';
 
   assertWarning( ...
                 @()bids.internal.parse_filename(filename, {'echo'}), ...
diff --git a/tests/test_return_datatype_entities.m b/tests/test_return_datatype_entities.m
index de7bd38d..6c02fdb9 100644
--- a/tests/test_return_datatype_entities.m
+++ b/tests/test_return_datatype_entities.m
@@ -6,14 +6,13 @@
   initTestSuite;
 end
 
-
 function test_return_datatype_entities_basic
 
   schema = bids.schema.load_schema();
 
   entities = bids.schema.return_datatype_entities(schema.datatypes.func(1));
 
-  expected_output = {'sub','ses','task','acq','ce','rec','dir','run','echo','part'};
+  expected_output = {'sub', 'ses', 'task', 'acq', 'ce', 'rec', 'dir', 'run', 'echo', 'part'};
 
   assert(isequal(entities, expected_output));
 

From 0e42c5331fbe6e03171a009e75fa3bb41dec3d14 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Thu, 4 Feb 2021 13:55:53 +0100
Subject: [PATCH 13/54] use schema to parse behavioral and refactor

---
 +bids/layout.m | 38 +++-----------------------------------
 1 file changed, 3 insertions(+), 35 deletions(-)

diff --git a/+bids/layout.m b/+bids/layout.m
index c5fb7b07..b3b4d533 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -200,10 +200,8 @@ function tolerant_message(tolerant, msg)
 
     for iDatatype = 1:numel(datatypes)
       switch datatypes{iDatatype}
-        case 'anat'
-          subject = parse_anat(subject);
-        case 'beh'
-          subject = parse_beh(subject);
+        case {'anat', 'beh'}
+          subject = parse_using_schema(subject, datatypes{iDatatype});
         case 'dwi'
           subject = parse_dwi(subject);
         case 'eeg'
@@ -227,9 +225,7 @@ function tolerant_message(tolerant, msg)
 
 end
 
-function subject = parse_anat(subject)
-
-  datatype = 'anat';
+function subject = parse_using_schema(subject, datatype)
 
   % --------------------------------------------------------------------------
   % -Anatomy imaging data
@@ -846,28 +842,6 @@ function tolerant_message(tolerant, msg)
 
 end
 
-function subject = parse_beh(subject)
-  % --------------------------------------------------------------------------
-  % -Behavioral experiments data
-  %
-  % - Event timing, metadata, physiological and other continuous recordings
-  % --------------------------------------------------------------------------
-  pth = fullfile(subject.path, 'beh');
-
-  if exist(pth, 'dir')
-
-    entities = return_entities('beh');
-
-    file_list = return_file_list('beh', subject);
-
-    for i = 1:numel(file_list)
-
-      subject = append_to_structure(file_list{i}, entities, subject, 'beh');
-
-    end
-  end
-end
-
 function subject = parse_dwi(subject)
   % --------------------------------------------------------------------------
   % -Diffusion imaging data
@@ -994,9 +968,6 @@ function tolerant_message(tolerant, msg)
 
   switch modality
 
-    case 'anat'
-      entities = {'sub', 'ses', 'acq', 'ce', 'rec', 'fa', 'echo', 'inv', 'run'};
-
     case 'func'
       entities = {'sub', ...
                   'ses', ...
@@ -1017,9 +988,6 @@ function tolerant_message(tolerant, msg)
     case 'meg'
       entities = {'sub', 'ses', 'task', 'acq', 'run', 'proc', 'meta'};
 
-    case 'beh'
-      entities = {'sub', 'ses', 'task'};
-
     case 'dwi'
       entities = {'sub', 'ses', 'acq', 'run', 'bval', 'bvec'};
 

From 6748545c9d52de01c6886e05d00c08e167fae516 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Thu, 4 Feb 2021 14:33:31 +0100
Subject: [PATCH 14/54] parse ASL with schema

---
 +bids/layout.m | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/+bids/layout.m b/+bids/layout.m
index b3b4d533..bbd14517 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -188,6 +188,7 @@ function tolerant_message(tolerant, msg)
   subject.eeg     = struct([]); % EEG data
   subject.meg     = struct([]); % MEG data
   subject.ieeg    = struct([]); % iEEG data
+  subject.perf     = struct([]); % ASL imaging data
   subject.pet     = struct([]); % PET imaging data
 
   % use BIDS schema to organizing parsing of subject data
@@ -200,7 +201,7 @@ function tolerant_message(tolerant, msg)
 
     for iDatatype = 1:numel(datatypes)
       switch datatypes{iDatatype}
-        case {'anat', 'beh'}
+        case {'anat', 'beh', 'perf'}
           subject = parse_using_schema(subject, datatypes{iDatatype});
         case 'dwi'
           subject = parse_dwi(subject);
@@ -214,7 +215,6 @@ function tolerant_message(tolerant, msg)
           subject = parse_ieeg(subject);
         case 'meg'
           subject = parse_meg(subject);
-        case 'perf'
       end
     end
 
@@ -1041,7 +1041,7 @@ function tolerant_message(tolerant, msg)
 
   switch modality
 
-    case 'anat'
+    case {'anat', 'dwi', 'perf'}
       pattern = '_([a-zA-Z0-9]+){1}\\.nii(\\.gz)?';
 
     case 'func'
@@ -1059,9 +1059,6 @@ function tolerant_message(tolerant, msg)
     case 'beh'
       pattern = '_(events\\.tsv|beh\\.json|physio\\.tsv\\.gz|stim\\.tsv\\.gz)';
 
-    case 'dwi'
-      pattern = '_([a-zA-Z0-9]+){1}\\.nii(\\.gz)?';
-
     case 'pet'
       pattern = '_task-.*_pet\\.nii(\\.gz)?';
 

From 3795b6b29445b7d7d02afc5d76ce3eea4af37053 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Thu, 4 Feb 2021 14:45:46 +0100
Subject: [PATCH 15/54] parse dwi using schema

---
 +bids/layout.m | 74 ++++++++++++++++++++++++--------------------------
 1 file changed, 35 insertions(+), 39 deletions(-)

diff --git a/+bids/layout.m b/+bids/layout.m
index bbd14517..9949cfab 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -246,6 +246,41 @@ function tolerant_message(tolerant, msg)
 
 end
 
+function subject = parse_dwi(subject)
+  % --------------------------------------------------------------------------
+  % -Diffusion imaging data
+  % --------------------------------------------------------------------------
+  datatype = 'dwi';
+  pth = fullfile(subject.path, datatype);
+
+  if exist(pth, 'dir')
+
+    file_list = return_file_list(datatype, subject);
+
+    for i = 1:numel(file_list)
+
+      subject = bids.internal.append_to_structure(file_list{i}, subject, datatype);
+
+      % -bval file
+      % ------------------------------------------------------------------
+      % bval file can also be stored at higher levels (inheritance principle)
+      bvalfile = bids.internal.get_metadata(file_list{i}, '^.*%s\\.bval$');
+      if isfield(bvalfile, 'filename')
+        subject.dwi(end).bval = bids.util.tsvread(bvalfile.filename); % ?
+      end
+
+      % -bvec file
+      % ------------------------------------------------------------------
+      % bvec file can also be stored at higher levels (inheritance principle)
+      bvecfile = bids.internal.get_metadata(file_list{i}, '^.*%s\\.bvec$');
+      if isfield(bvalfile, 'filename')
+        subject.dwi(end).bvec = bids.util.tsvread(bvecfile.filename); % ?
+      end
+
+    end
+  end
+end
+
 function subject = parse_func(subject)
 
   % --------------------------------------------------------------------------
@@ -842,42 +877,6 @@ function tolerant_message(tolerant, msg)
 
 end
 
-function subject = parse_dwi(subject)
-  % --------------------------------------------------------------------------
-  % -Diffusion imaging data
-  % --------------------------------------------------------------------------
-  pth = fullfile(subject.path, 'dwi');
-
-  if exist(pth, 'dir')
-
-    entities = return_entities('dwi');
-
-    file_list = return_file_list('dwi', subject);
-
-    for i = 1:numel(file_list)
-
-      subject = append_to_structure(file_list{i}, entities, subject, 'dwi');
-
-      % -bval file
-      % ------------------------------------------------------------------
-      % bval file can also be stored at higher levels (inheritance principle)
-      bvalfile = bids.internal.get_metadata(file_list{i}, '^.*%s\\.bval$');
-      if isfield(bvalfile, 'filename')
-        subject.dwi(end).bval = bids.util.tsvread(bvalfile.filename); % ?
-      end
-
-      % -bvec file
-      % ------------------------------------------------------------------
-      % bvec file can also be stored at higher levels (inheritance principle)
-      bvecfile = bids.internal.get_metadata(file_list{i}, '^.*%s\\.bvec$');
-      if isfield(bvalfile, 'filename')
-        subject.dwi(end).bvec = bids.util.tsvread(bvecfile.filename); % ?
-      end
-
-    end
-  end
-end
-
 function subject = parse_pet(subject)
   % --------------------------------------------------------------------------
   % -Positron Emission Tomography imaging data
@@ -988,9 +987,6 @@ function tolerant_message(tolerant, msg)
     case 'meg'
       entities = {'sub', 'ses', 'task', 'acq', 'run', 'proc', 'meta'};
 
-    case 'dwi'
-      entities = {'sub', 'ses', 'acq', 'run', 'bval', 'bvec'};
-
     case 'pet'
       entities = {'sub', 'ses', 'task', 'acq', 'rec', 'run'};
 

From c830bd9337f414194bb660b82d374ca1659518e2 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Thu, 4 Feb 2021 14:55:11 +0100
Subject: [PATCH 16/54] linting

---
 +bids/+schema/load_schema.m | 3 ++-
 +bids/layout.m              | 3 ---
 tests/test_layout.m         | 1 +
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/+bids/+schema/load_schema.m b/+bids/+schema/load_schema.m
index 3b9d56c2..ee2bbe39 100644
--- a/+bids/+schema/load_schema.m
+++ b/+bids/+schema/load_schema.m
@@ -53,7 +53,8 @@
     if ~isempty(json_file_list)
       field_name = bids.internal.file_utils(directory, 'basename');
       structure.(field_name) = struct();
-      structure.(field_name) = append_json_content_to_structure(structure.(field_name), json_file_list);
+      structure.(field_name) = append_json_content_to_structure(structure.(field_name), ...
+                                                                json_file_list);
     end
 
     structure = inspect_subdir(structure, dirs);
diff --git a/+bids/layout.m b/+bids/layout.m
index 9949cfab..2bf92837 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -227,9 +227,6 @@ function tolerant_message(tolerant, msg)
 
 function subject = parse_using_schema(subject, datatype)
 
-  % --------------------------------------------------------------------------
-  % -Anatomy imaging data
-  % --------------------------------------------------------------------------
   pth = fullfile(subject.path, datatype);
 
   if exist(pth, 'dir')
diff --git a/tests/test_layout.m b/tests/test_layout.m
index b4757550..619d9a56 100644
--- a/tests/test_layout.m
+++ b/tests/test_layout.m
@@ -10,5 +10,6 @@ function test_layout_smoke_test()
 
   pth_bids_example = get_test_data_dir();
   BIDS = bids.layout(fullfile(pth_bids_example, '7t_trt'));
+  BIDS = bids.layout(fullfile(pth_bids_example, 'asl001'));
 
 end

From 16072de9753a97089f575f653ce934761157ffc7 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Thu, 4 Feb 2021 18:03:51 +0100
Subject: [PATCH 17/54] parse func with schema

---
 +bids/+internal/append_to_structure.m |  2 +-
 +bids/layout.m                        | 83 ++++++---------------------
 tests/test_bids_query.m               |  4 +-
 3 files changed, 23 insertions(+), 66 deletions(-)

diff --git a/+bids/+internal/append_to_structure.m b/+bids/+internal/append_to_structure.m
index 3d825d88..5640756e 100644
--- a/+bids/+internal/append_to_structure.m
+++ b/+bids/+internal/append_to_structure.m
@@ -34,7 +34,7 @@
 
 function structure = add_missing_field(structure, field)
   if ~isfield(structure, field)
-    structure.(field) = '';
+    structure(1).(field) = '';
   end
 end
 
diff --git a/+bids/layout.m b/+bids/layout.m
index 2bf92837..74931c3f 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -283,40 +283,30 @@ function tolerant_message(tolerant, msg)
   % --------------------------------------------------------------------------
   % -Task imaging data
   % --------------------------------------------------------------------------
-  pth = fullfile(subject.path, 'func');
+  datatype = 'func';
+  pth = fullfile(subject.path, datatype);
 
   if exist(pth, 'dir')
 
-    entities = return_entities('func');
-
-    file_list = return_file_list('func', subject);
+    file_list = return_file_list(datatype, subject);
 
     for i = 1:numel(file_list)
 
-      subject = append_to_structure(file_list{i}, entities, subject, 'func');
+      subject = bids.internal.append_to_structure(file_list{i}, subject, datatype);
       subject.func(end).meta = struct([]); % ?
 
-    end
-
-    file_list = return_event_file_list('func', subject);
+      % TODO:
+      %
+      % Events, physiological and other continuous recordings file
+      % can also be stored at higher levels (inheritance principle).
+      %
 
-    for i = 1:numel(file_list)
-
-      subject = append_to_structure(file_list{i}, entities, subject, 'func');
-
-      subject.func(end).meta = bids.util.tsvread(fullfile(pth, file_list{i})); % ?
+      if strcmp(subject.func(end).meta, 'events')
+        subject.func(end).meta = bids.util.tsvread(fullfile(pth, file_list{i})); % ?
+      end
 
     end
 
-    file_list = return_physio_stim_file_list('func', subject);
-
-    for i = 1:numel(file_list)
-
-      subject = append_to_structure(file_list{i}, entities, subject, 'func');
-
-      subject.func(end).meta = struct([]); % ?
-
-    end
   end
 end
 
@@ -964,20 +954,6 @@ function tolerant_message(tolerant, msg)
 
   switch modality
 
-    case 'func'
-      entities = {'sub', ...
-                  'ses', ...
-                  'task', ...
-                  'acq', ...
-                  'rec', ...
-                  'fa', ...
-                  'echo', ...
-                  'dir', ...
-                  'inv', ...
-                  'run', ...
-                  'recording', ...
-                  'meta'};
-
     case {'eeg', 'ieeg'}
       entities = {'sub', 'ses', 'task', 'acq', 'run', 'meta'};
 
@@ -1034,11 +1010,15 @@ function tolerant_message(tolerant, msg)
 
   switch modality
 
+    % TODO
+    % it should be possible to create some of those patterns for the regexp
+    % based on some of the required entities written down in the schema
+
     case {'anat', 'dwi', 'perf'}
       pattern = '_([a-zA-Z0-9]+){1}\\.nii(\\.gz)?';
 
     case 'func'
-      pattern = '_task-.*_bold\\.nii(\\.gz)?';
+      pattern = '_task-.*\\.nii(\\.gz)|events\\.tsv|physio\\.tsv\\.gz|stim\\.tsv\\.gz?';
 
     case 'fmap'
       pattern = '\\.nii(\\.gz)?';
@@ -1050,7 +1030,7 @@ function tolerant_message(tolerant, msg)
       pattern = '_task-.*_meg\\..*[^json]';
 
     case 'beh'
-      pattern = '_(events\\.tsv|beh\\.json|physio\\.tsv\\.gz|stim\\.tsv\\.gz)';
+      pattern = '_task-.*_(events\\.tsv|beh\\.json|physio\\.tsv\\.gz|stim\\.tsv\\.gz)';
 
     case 'pet'
       pattern = '_task-.*_pet\\.nii(\\.gz)?';
@@ -1085,7 +1065,7 @@ function tolerant_message(tolerant, msg)
 
   switch modality
 
-    case {'func', 'eeg', 'meg'}
+    case {'eeg', 'meg'}
       pattern = '_task-.*_events\\.tsv';
 
   end
@@ -1101,31 +1081,6 @@ function tolerant_message(tolerant, msg)
 
 end
 
-function file_list = return_physio_stim_file_list(modality, subject)
-  %
-  % Physiological and other continuous recordings file
-  %
-  % TODO: stim files can also be stored at higher levels (inheritance principle)
-  %
-
-  switch modality
-
-    case {'func'}
-      pattern = '_task-.*_(physio|stim)\\.tsv\\.gz';
-
-  end
-
-  pth = fullfile(subject.path, modality);
-
-  [file_list, d] = bids.internal.file_utils('List', ...
-                                            pth, ...
-                                            sprintf(['^%s.*' pattern '$'], ...
-                                                    subject.name));
-
-  file_list = convert_to_cell(file_list);
-
-end
-
 function metafile = return_fmap_metadata_file(subject, fmap_file)
 
   pth = fullfile(subject.path, 'fmap');
diff --git a/tests/test_bids_query.m b/tests/test_bids_query.m
index 211c04d2..31b4e283 100644
--- a/tests/test_bids_query.m
+++ b/tests/test_bids_query.m
@@ -43,6 +43,9 @@ function test_bids_query_basic()
   types = {'T1w', 'bold', 'events', 'inplaneT2'};
   assert(isequal(bids.query(BIDS, 'types'), types));
 
+  data = bids.query(BIDS, 'data', 'sub', '01', 'task', 'stopsignalwithpseudowordnaming');
+  assertEqual(size(data, 1), 4);
+
   mods = {'anat', 'func'};
   assert(isequal(bids.query(BIDS, 'modalities'), mods));
   assert(isequal(bids.query(BIDS, 'modalities', 'sub', '01'), mods));
@@ -107,7 +110,6 @@ function test_bids_query_modalities()
   assert(isequal(bids.query(BIDS, 'modalities', 'sub', '01'), mods));
   assert(isequal(bids.query(BIDS, 'modalities', 'sub', '01', 'ses', '1'), mods));
 
-  %
   % this now fails on octave 4.2.2 but not on Matlab
   %
   % bids.query(BIDS, 'modalities', 'sub', '01', 'ses', '2')

From c0e1934305f5ad133abd70fd30b5766b0ba375b6 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Thu, 4 Feb 2021 20:12:04 +0100
Subject: [PATCH 18/54] fix issue of regexp on Octave

add a test for it too
fix octave bug in CI


add print for CI


Same
---
 +bids/+internal/append_to_structure.m      | 5 ++---
 +bids/+internal/file_utils.m               | 1 -
 +bids/+internal/return_datatype_suffixes.m | 4 +++-
 +bids/+schema/load_schema.m                | 4 ++--
 tests/test_file_utils.m                    | 5 +++++
 5 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/+bids/+internal/append_to_structure.m b/+bids/+internal/append_to_structure.m
index 5640756e..1101e9e2 100644
--- a/+bids/+internal/append_to_structure.m
+++ b/+bids/+internal/append_to_structure.m
@@ -43,12 +43,11 @@
   idx = [];
 
   schema = bids.schema.load_schema();
-  suffix_groups = {schema.datatypes.(modality).suffixes}';
 
   % the following loop could probably be improved with some cellfun magic
   %   cellfun(@(x, y) any(strcmp(x,y)), {p.type}, suffix_groups)
-  for i = 1:numel(suffix_groups)
-    if any(strcmp(suffix, suffix_groups{i}))
+  for i = 1:size(schema.datatypes.(modality), 1)
+    if any(strcmp(suffix, schema.datatypes.(modality)(i).suffixes))
       idx = i;
       break
     end
diff --git a/+bids/+internal/file_utils.m b/+bids/+internal/file_utils.m
index 70b8083b..c565803f 100644
--- a/+bids/+internal/file_utils.m
+++ b/+bids/+internal/file_utils.m
@@ -282,7 +282,6 @@
     files = dirs;
 
   else
-
     t = regexp(files, expr);
 
     if numel(files) == 1 && ~iscell(t)
diff --git a/+bids/+internal/return_datatype_suffixes.m b/+bids/+internal/return_datatype_suffixes.m
index 4ce7476a..9bb90fc8 100644
--- a/+bids/+internal/return_datatype_suffixes.m
+++ b/+bids/+internal/return_datatype_suffixes.m
@@ -1,8 +1,10 @@
 function suffixes = return_datatype_suffixes(datatype)
 
   suffixes = '_(';
+  
+  datatype
 
-  for iExt = 1:numel(datatype.suffixes)
+  for iExt = 1:numel(datatype(:).suffixes)
     suffixes = [suffixes,  datatype.suffixes{iExt}, '|']; %#ok<AGROW>
   end
 
diff --git a/+bids/+schema/load_schema.m b/+bids/+schema/load_schema.m
index ee2bbe39..ea2e8915 100644
--- a/+bids/+schema/load_schema.m
+++ b/+bids/+schema/load_schema.m
@@ -19,7 +19,7 @@
 
   schema = struct();
 
-  [json_file_list, dirs] = bids.internal.file_utils('FPList', SCHEMA_DIR, '^*.json$');
+  [json_file_list, dirs] = bids.internal.file_utils('FPList', SCHEMA_DIR, '^.*.json$');
 
   schema = append_json_content_to_structure(schema, json_file_list);
 
@@ -48,7 +48,7 @@
 
     directory = deblank(subdir_list(iDir, :));
 
-    [json_file_list, dirs] = bids.internal.file_utils('FPList', directory, '^*.json$');
+    [json_file_list, dirs] = bids.internal.file_utils('FPList', directory, '^.*.json$');
 
     if ~isempty(json_file_list)
       field_name = bids.internal.file_utils(directory, 'basename');
diff --git a/tests/test_file_utils.m b/tests/test_file_utils.m
index 6c3f4c36..36a7c533 100644
--- a/tests/test_file_utils.m
+++ b/tests/test_file_utils.m
@@ -67,6 +67,11 @@ function test_file_utils_basic()
                                   '^test_file_utils.m$');
   assert(isequal(file, 'test_file_utils.m'));
 
+  file = bids.internal.file_utils('List', ...
+                                  test_directory, ...
+                                  '^.*.md$');
+  assert(isequal(file, 'README.md'));
+
   directory = bids.internal.file_utils('List', ...
                                        test_directory, ...
                                        'dir', ...

From e092dad650709ae1d5f87fe3c426f009c546d564 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sat, 6 Feb 2021 18:20:11 +0100
Subject: [PATCH 19/54] prepare for rebase

---
 +bids/layout.m | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/+bids/layout.m b/+bids/layout.m
index 74931c3f..5977e38b 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -188,7 +188,6 @@ function tolerant_message(tolerant, msg)
   subject.eeg     = struct([]); % EEG data
   subject.meg     = struct([]); % MEG data
   subject.ieeg    = struct([]); % iEEG data
-  subject.perf     = struct([]); % ASL imaging data
   subject.pet     = struct([]); % PET imaging data
 
   % use BIDS schema to organizing parsing of subject data
@@ -201,7 +200,7 @@ function tolerant_message(tolerant, msg)
 
     for iDatatype = 1:numel(datatypes)
       switch datatypes{iDatatype}
-        case {'anat', 'beh', 'perf'}
+        case {'anat', 'beh'}
           subject = parse_using_schema(subject, datatypes{iDatatype});
         case 'dwi'
           subject = parse_dwi(subject);
@@ -1014,7 +1013,7 @@ function tolerant_message(tolerant, msg)
     % it should be possible to create some of those patterns for the regexp
     % based on some of the required entities written down in the schema
 
-    case {'anat', 'dwi', 'perf'}
+    case {'anat', 'dwi'}
       pattern = '_([a-zA-Z0-9]+){1}\\.nii(\\.gz)?';
 
     case 'func'

From a3b013485939f98ea213898d13fc2b2d79b7ce5b Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sat, 6 Feb 2021 18:59:02 +0100
Subject: [PATCH 20/54] handle asl file parsing with schema

---
 +bids/layout.m | 64 ++++++++++++++------------------------------------
 1 file changed, 17 insertions(+), 47 deletions(-)

diff --git a/+bids/layout.m b/+bids/layout.m
index 5977e38b..9c2a99ea 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -146,7 +146,7 @@
                                             fullfile(BIDS.dir, sub{iSub}), ...
                                             'dir', ...
                                             '^ses-.*$'));
-                                        
+
     for iSess = 1:numel(sess)
       if isempty(BIDS.subjects)
         BIDS.subjects = parse_subject(BIDS.dir, sub{iSub}, sess{iSess});
@@ -154,7 +154,7 @@
         BIDS.subjects(end + 1) = parse_subject(BIDS.dir, sub{iSub}, sess{iSess});
       end
     end
-    
+
   end
 
 end
@@ -214,6 +214,8 @@ function tolerant_message(tolerant, msg)
           subject = parse_ieeg(subject);
         case 'meg'
           subject = parse_meg(subject);
+        case 'perf'
+          subject = parse_perf(subject);
       end
     end
 
@@ -314,17 +316,18 @@ function tolerant_message(tolerant, msg)
   % --------------------------------------------------------------------------
   % -ASL perfusion imaging data
   % --------------------------------------------------------------------------
+  datatype = 'perf';
   pth = fullfile(subject.path, 'perf');
 
   if exist(pth, 'dir')
 
-    entities = return_entities('perf');
+    file_list = return_file_list(datatype, subject);
 
-    file_list = return_file_list('perf', subject);
+    for i = 1:numel(file_list)
 
-    file_list = convert_to_cell(file_list);
+      subject = bids.internal.append_to_structure(file_list{i}, subject, datatype);
 
-    j = 1;
+    end
 
     % ASL timeseries NIfTI file
     % ----------------------------------------------------------------------
@@ -338,18 +341,9 @@ function tolerant_message(tolerant, msg)
 
       for i = 1:numel(idx)
 
-        % Parse filename
-        % ---------------------------
-        fb = bids.internal.file_utils(bids.internal.file_utils(file_list{idx(i)}, ...
-                                                               'basename'), ...
-                                      'basename');
-
-        p = bids.internal.parse_filename(file_list{idx(i)}, entities);
-
-        subject = append_to_perf(subject, p, j);
+        j = idx(i);
 
-        % add type
-        subject.perf(j).type = 'asl';
+        fb = bids.internal.file_utils(bids.internal.file_utils(file_list{j}, 'basename'), 'basename');
 
         % Manage JSON-sidecar metadata (REQUIRED)
         % ---------------------------
@@ -485,8 +479,8 @@ function tolerant_message(tolerant, msg)
           subject.perf(j).labeling_image_filename = [Ffile '.jpg'];
         end
 
-        j = j + 1;
       end % for i = 1:numel(idx)
+
     end % if any(~cellfun(@isempty, labels))
 
     % -M0scan NIfTI file
@@ -498,12 +492,10 @@ function tolerant_message(tolerant, msg)
     if any(~cellfun(@isempty, labels))
       idx = find(~cellfun(@isempty, labels));
       for i = 1:numel(idx)
-        % Parse filename
-        % ---------------------------
-        fb = bids.internal.file_utils(bids.internal.file_utils(file_list{idx(i)}, 'basename'), 'basename');
-        p = bids.internal.parse_filename(file_list{idx(i)}, entities);
 
-        subject = append_to_perf(subject, p, j);
+        j = idx(i);
+
+        fb = bids.internal.file_utils(bids.internal.file_utils(file_list{j}, 'basename'), 'basename');
 
         % Manage JSON-sidecar metadata (REQUIRED)
         % ---------------------------
@@ -555,31 +547,12 @@ function tolerant_message(tolerant, msg)
           end
         end
       end % for i = 1:numel(idx)
-      j = j + 1;
+
     end  % if any(~cellfun(@isempty, labels))
 
   end % if exist(pth, 'dir')
 end % function subject = parse_perf(subject)
 
-function subject = append_to_perf(subject, p, j)
-
-  if j == 1
-    subject.perf = p;
-  else
-    fields_p = fieldnames(p);
-    for iField = 1:length(fields_p)
-      subject.perf(j).(fields_p{iField}) = p.(fields_p{iField});
-    end
-  end
-
-  % default to run 1 ((!) TODO: but could be that we need to check this
-  % at the end!)
-  if isempty(subject.perf(j).run)
-    subject.perf(j).run = '1';
-  end
-
-end
-
 function subject = parse_fmap(subject)
   %
   % TODO:
@@ -962,9 +935,6 @@ function tolerant_message(tolerant, msg)
     case 'pet'
       entities = {'sub', 'ses', 'task', 'acq', 'rec', 'run'};
 
-    case 'perf'
-      entities = {'sub', 'ses', 'acq', 'dir', 'rec', 'run'};
-
   end
 end
 
@@ -1038,7 +1008,7 @@ function tolerant_message(tolerant, msg)
       pattern = '_task-.*_ieeg\\..*[^json]';
 
     case 'perf'
-      pattern = '_(asl|m0scan)\\.nii(\\.gz)?';
+      pattern = '_(asl|m0scan)\\.nii(\\.gz)|aslcontext\\.tsv|asllabeling\\.jpg';
 
   end
 

From 1d4115cdbd9eaeafea0404c63b3e705ae2623d09 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sat, 6 Feb 2021 20:46:51 +0100
Subject: [PATCH 21/54] refactor asl manage m0scan, json, context, labelling

---
 +bids/layout.m | 308 ++++++++++++++++++++++++++++---------------------
 1 file changed, 177 insertions(+), 131 deletions(-)

diff --git a/+bids/layout.m b/+bids/layout.m
index 9c2a99ea..14c8cce2 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -343,141 +343,16 @@ function tolerant_message(tolerant, msg)
 
         j = idx(i);
 
-        fb = bids.internal.file_utils(bids.internal.file_utils(file_list{j}, 'basename'), 'basename');
-
-        % Manage JSON-sidecar metadata (REQUIRED)
-        % ---------------------------
-        metafile = fullfile(pth, bids.internal.file_utils(fb, 'ext', 'json'));
-
-        if exist(metafile, 'file')
-          [~, Ffile] = fileparts(metafile);
-          subject.perf(j).json_sidecar_filename = [Ffile '.json'];
-          subject.perf(j).meta = bids.util.jsondecode(metafile);
-        else
-          warning(['Missing: ' metafile]);
-
-        end
-
-        % Manage ASLCONTEXT-sidecar metadata (REQUIRED)
-        % ---------------------------
-        metafile = fullfile(pth, bids.internal.file_utils([fb(1:end - 4) '_aslcontext'], 'ext', 'tsv'));
-
-        if exist(metafile, 'file')
-          [~, Ffile] = fileparts(metafile);
-          subject.perf(j).context_sidecar_filename = [Ffile '.tsv'];
-          subject.perf(j).context = bids.util.tsvread(metafile);
-
-        else
-          warning(['Missing: ' metafile]);
-
-        end
-
-        % Manage M0 (REQUIRED)
-        % ---------------------------
-        % M0 field is flexible:
-
-        if ~isfield(subject.perf(j).meta, 'M0Type')
-          warning(['M0Type field missing in ' subject.perf(j).json_sidecar_filename]);
-
-        else
-
-          m0_type = [];
-          m0_explanation = [];
-          m0_volume_index = [];
-          m0_value = [];
-
-          switch subject.perf(j).meta.M0Type
-
-            case 'Separate'
-              % the M0 was obtained as a separate scan
-              m0_type = 'separate_scan';
-              m0_explanation = 'M0 was obtained as a separate scan';
+        subject.perf(j).meta = [];
+        subject.perf(j).dependencies = [];
 
-              % M0scan.nii filename
-              % assuming the (.nii|.nii.gz) extension choice is the same throughout
-              m0_filename = [subject.perf(j).filename(1:end - 10) 'm0scan' subject.perf(j).ext];
-              if ~exist(fullfile(pth, m0_filename), 'file')
-                warning(['Missing: ' m0_filename]);
-              else
-                % subject.perf(j).m0_filename = m0_filename;
-                % -> this is included in the same structure for the m0scan.nii
-              end
-
-              % M0 sidecar filename
-              m0_json_sidecar_filename = [subject.perf(j).filename(1:end - 10) 'm0scan.json'];
-              if ~exist(fullfile(pth, m0_json_sidecar_filename), 'file')
-                warning(['Missing: ' m0_json_sidecar_filename]);
-              else
-                % subject.perf(j).m0_json_sidecar_filename = m0_json_sidecar_filename;
-                % -> this is included in the same structure for the m0scan.nii
-              end
-
-            case 'Included'
-              % M0 is one or more image(s) in the *asl.nii[.gz] timeseries
-              if ~isfield(subject.perf(j), 'context') || ...
-                      ~isfield(subject.perf(j).context, 'volume_type')
-                warning('Cannot find M0 volume in aslcontext, context-information missing');
-
-              else
-                m0indices = find(cellfun(@(x) strcmp(x, 'm0scan'), ...
-                                         subject.perf(j).context.volume_type) == true);
-                if isempty(m0indices)
-                  warning('No M0 volume found in aslcontext');
-
-                else
-                  m0_type = 'within_timeseries';
-                  m0_explanation = 'M0 is one or more image(s) in the *asl.nii[.gz] timeseries';
-                  m0_volume_index = m0indices;
-
-                end
-              end
-
-            case 'Estimate'
-              m0_type = 'single_value';
-              m0_explanation = [ ...
-                                'this is a single estimated M0 value, ', ...
-                                'e.g. when the M0 is obtained from an external scan and/or study'];
-              m0_value = subject.perf(j).meta.M0;
-
-            case 'Absent'
-              m0_type = 'use_control_as_m0';
-              m0_explanation = [ ...
-                                'M0 is absent, so we can use the (average) control volume ', ...
-                                'as pseudo-M0 (if no background suppression was used)'];
-
-              if subject.perf(j).meta.BackgroundSuppression == true
-                warning('Caution when using control as M0, background suppression was applied');
-              end
-
-            otherwise
-              warning(['Unknown M0Type:', ...
-                       subject.perf(j).meta.M0Type, ...
-                       ' in ', ...
-                       subject.perf(j).json_sidecar_filename]);
-          end
+        subject.perf(j) = manage_json_sidecar(subject.perf(j), pth);
 
-          if ~isempty(m0_type)
-            subject.perf(j).m0_type = m0_type;
-            subject.perf(j).m0_explanation = m0_explanation;
-          end
-
-          if ~isempty(m0_volume_index)
-            subject.perf(j).m0_volume_index = m0_volume_index;
-          end
-          if ~isempty(m0_value)
-            subject.perf(j).m0_value = m0_value;
-          end
+        subject.perf(j) = manage_aslcontext(subject.perf(j), pth);
 
-        end
+        subject.perf(j) = manage_asllabeling(subject.perf(j), pth);
 
-        % Manage labeling image metadata (OPTIONAL)
-        % ---------------------------
-        metafile = fullfile(pth, bids.internal.file_utils([fb(1:end - 4) '_labeling'], 'ext', 'jpg'));
-
-        if exist(metafile, 'file')
-          [~, Ffile] = fileparts(metafile);
-          subject.perf(j).labeling_image_filename = [Ffile '.jpg'];
-        end
+        subject.perf(j) = manage_M0(subject.perf(j), pth);
 
       end % for i = 1:numel(idx)
 
@@ -553,6 +428,177 @@ function tolerant_message(tolerant, msg)
   end % if exist(pth, 'dir')
 end % function subject = parse_perf(subject)
 
+function structure = manage_json_sidecar(structure, pth)
+
+  % Manage JSON-sidecar metadata (REQUIRED)
+  % ---------------------------
+  metafile = fullfile(pth, strrep(structure.filename, structure.ext, '.json'));
+
+  if exist(metafile, 'file')
+    [~, Ffile] = fileparts(metafile);
+    structure.dependencies.sidecar = [Ffile '.json'];
+    structure.meta = bids.util.jsondecode(metafile);
+  else
+    warning(['Missing: ' metafile]);
+
+  end
+
+end
+
+function perf = manage_aslcontext(perf, pth)
+
+  % ASLCONTEXT-sidecar metadata (REQUIRED)
+  % ---------------------------
+  metafile = fullfile(pth, strrep(perf.filename, ...
+                                  ['_asl' perf.ext], ...
+                                  '_aslcontext.tsv'));
+
+  if exist(metafile, 'file')
+    [~, Ffile] = fileparts(metafile);
+    perf.dependencies.context.sidecar = [Ffile '.tsv'];
+    perf.dependencies.context.content = bids.util.tsvread(metafile);
+
+  else
+    warning(['Missing: ' metafile]);
+
+  end
+
+end
+
+function perf = manage_asllabeling(perf, pth)
+  % labeling image metadata (OPTIONAL)
+  % ---------------------------
+  metafile = fullfile(pth, strrep(perf.filename, ...
+                                  ['_asl' perf.ext], ...
+                                  '_asllabeling.jpg'));
+
+  if exist(metafile, 'file')
+    [~, Ffile] = fileparts(metafile);
+    perf.dependencies.labeling_image = [Ffile '.jpg'];
+
+  end
+
+end
+
+function perf = manage_M0(perf, pth)
+
+  % M0 field is flexible:
+
+  if ~isfield(perf.meta, 'M0Type')
+    warning(['M0Type field missing in ' perf.dependencies.sidecar]);
+
+  else
+
+    m0_type = [];
+    m0_explanation = [];
+    m0_volume_index = [];
+    m0_value = [];
+    m0_filename = [];
+    m0_sidecar = [];
+
+    switch perf.meta.M0Type
+
+      case 'Separate'
+        % the M0 was obtained as a separate scan
+        m0_type = 'separate_scan';
+        m0_explanation = 'M0 was obtained as a separate scan';
+
+        % M0scan.nii filename
+        % assuming the (.nii|.nii.gz) extension choice is the same throughout
+        m0_filename = strrep(perf.filename, ...
+                             ['_asl' perf.ext], ...
+                             ['_m0scan' perf.ext]);
+
+        if ~exist(fullfile(pth, m0_filename), 'file')
+          warning(['Missing: ' m0_filename]);
+        else
+          % subject.perf(j).m0_filename = m0_filename;
+          % -> this is included in the same structure for the m0scan.nii
+        end
+
+        % M0 sidecar filename
+        m0_sidecar = strrep(perf.filename, ...
+                            ['_asl' perf.ext], ...
+                            '_m0scan.json');
+
+        if ~exist(fullfile(pth, m0_sidecar), 'file')
+          warning(['Missing: ' m0_sidecar]);
+
+        else
+          % subject.perf(j).m0_json_sidecar_filename = m0_json_sidecar_filename;
+          % -> this is included in the same structure for the m0scan.nii
+        end
+
+      case 'Included'
+        % M0 is one or more image(s) in the *asl.nii[.gz] timeseries
+        if ~isfield(perf.dependencies, 'context') || ...
+                ~isfield(perf.dependencies.context.content, 'volume_type')
+          warning('Cannot find M0 volume in aslcontext, context-information missing');
+
+        else
+          m0indices = find(cellfun(@(x) strcmp(x, 'm0scan'), ...
+                                   perf.dependencies.context.content.volume_type) == true);
+
+          if isempty(m0indices)
+            warning('No M0 volume found in aslcontext');
+
+          else
+            m0_type = 'within_timeseries';
+            m0_explanation = 'M0 is one or more image(s) in the *asl.nii[.gz] timeseries';
+            m0_volume_index = m0indices;
+
+          end
+        end
+
+      case 'Estimate'
+        m0_type = 'single_value';
+        m0_explanation = [ ...
+                          'this is a single estimated M0 value, ', ...
+                          'e.g. when the M0 is obtained from an external scan and/or study'];
+        m0_value = perf.meta.M0;
+
+      case 'Absent'
+        m0_type = 'use_control_as_m0';
+        m0_explanation = [ ...
+                          'M0 is absent, so we can use the (average) control volume ', ...
+                          'as pseudo-M0 (if no background suppression was used)'];
+
+        if perf.meta.BackgroundSuppression == true
+          warning('Caution when using control as M0, background suppression was applied');
+        end
+
+      otherwise
+        warning(['Unknown M0Type:', ...
+                 perf.meta.M0Type, ...
+                 ' in ', ...
+                 perf.json_sidecar_filename]);
+    end
+
+    if ~isempty(m0_type)
+      perf.dependencies.m0.type = m0_type;
+      perf.dependencies.m0.explanation = m0_explanation;
+    end
+
+    if ~isempty(m0_volume_index)
+      perf.dependencies.m0.volume_index = m0_volume_index;
+    end
+
+    if ~isempty(m0_value)
+      perf.dependencies.m0.value = m0_value;
+    end
+
+    if ~isempty(m0_filename)
+      perf.dependencies.m0.filename = m0_filename;
+    end
+
+    if ~isempty(m0_sidecar)
+      perf.dependencies.m0.sidecar = m0_sidecar;
+    end
+
+  end
+
+end
+
 function subject = parse_fmap(subject)
   %
   % TODO:

From 2656a5f395df709647e831305b0164d1d9e5de13 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sat, 6 Feb 2021 21:06:47 +0100
Subject: [PATCH 22/54] refactor managing intended_for for ASL

---
 +bids/layout.m              | 109 ++++++++++++++++++------------------
 tests/test_bids_query_asl.m |  16 ++----
 2 files changed, 62 insertions(+), 63 deletions(-)

diff --git a/+bids/layout.m b/+bids/layout.m
index 14c8cce2..0e7b048c 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -354,9 +354,9 @@ function tolerant_message(tolerant, msg)
 
         subject.perf(j) = manage_M0(subject.perf(j), pth);
 
-      end % for i = 1:numel(idx)
+      end
 
-    end % if any(~cellfun(@isempty, labels))
+    end
 
     % -M0scan NIfTI file
     % ---------------------------------------------------------------------
@@ -370,62 +370,16 @@ function tolerant_message(tolerant, msg)
 
         j = idx(i);
 
-        fb = bids.internal.file_utils(bids.internal.file_utils(file_list{j}, 'basename'), 'basename');
-
-        % Manage JSON-sidecar metadata (REQUIRED)
-        % ---------------------------
-        metafile = fullfile(pth, bids.internal.file_utils(fb, 'ext', 'json'));
-
-        if ~exist(metafile, 'file')
-          warning(['Missing: ' metafile]);
-
-        else
-          [~, Ffile] = fileparts(metafile);
-          subject.perf(j).json_sidecar_filename = [Ffile '.json'];
-          subject.perf(j).meta = bids.util.jsondecode(metafile);
-
-          % Manage intended-for (REQUIRED)
-          % ---------------------------
-
-          % Get all NIfTIs that this m0scan is intended for
-          path_intended_for = {};
-          if ~isfield(subject.perf(j).meta, 'IntendedFor')
-            warning(['Missing field IntendedFor in ' metafile]);
-
-          elseif ischar(subject.perf(j).meta.IntendedFor)
-            path_intended_for{1} = subject.perf(j).meta.IntendedFor;
-
-          elseif isstruct(subject.perf(j).meta.IntendedFor)
-            for iPath = 1:length(subject.perf(j).meta.IntendedFor)
-              path_intended_for{iPath} = subject.perf(j).meta.IntendedFor(iPath); %#ok<*AGROW>
-            end
-
-          end
-
-          for iPath = 1:length(path_intended_for)
-            % check if this NIfTI is not missing
-            if ~exist(fullfile(fileparts(pth), path_intended_for{iPath}), 'file')
-              warning(['Missing: ' fullfile(fileparts(pth), path_intended_for{iPath})]);
-
-            else
-              % also check that this NIfTI aims to the same m0scan
-              [~, path2check, ext2check] = fileparts(path_intended_for{iPath});
-              filename_found = max(arrayfun(@(x) strcmp(x.filename, [path2check ext2check]), subject.perf));
-              if ~filename_found
-                warning(['Did not find NIfTI for which is intended: ' subject.perf(j).filename]);
+        subject.perf(j).intended_for = [];
 
-              else
-                subject.perf(j).intended_for = path_intended_for{iPath};
+        subject.perf(j) = manage_intended_for(subject.perf(j), subject, pth);
 
-              end
-            end
-          end
-        end
-      end % for i = 1:numel(idx)
+      end
 
-    end  % if any(~cellfun(@isempty, labels))
+    end
 
   end % if exist(pth, 'dir')
+
 end % function subject = parse_perf(subject)
 
 function structure = manage_json_sidecar(structure, pth)
@@ -599,6 +553,55 @@ function tolerant_message(tolerant, msg)
 
 end
 
+function structure = manage_intended_for(structure, subject, pth)
+
+  structure = manage_json_sidecar(structure, pth);
+
+  if isempty(structure.meta)
+    return
+
+  else
+
+    % Get all NIfTIs that this m0scan is intended for
+    path_intended_for = {};
+    if ~isfield(structure.meta, 'IntendedFor')
+      warning(['Missing field IntendedFor in ' structure.dependencies.sidecar]);
+
+    elseif ischar(structure.meta.IntendedFor)
+      path_intended_for{1} = structure.meta.IntendedFor;
+
+    elseif isstruct(structure.meta.IntendedFor)
+      for iPath = 1:length(structure.meta.IntendedFor)
+        path_intended_for{iPath} = structure.meta.IntendedFor(iPath); %#ok<*AGROW>
+      end
+
+    end
+
+    for iPath = 1:length(path_intended_for)
+      % check if this NIfTI is not missing
+      if ~exist(fullfile(fileparts(pth), path_intended_for{iPath}), 'file')
+        warning(['Missing: ' fullfile(fileparts(pth), path_intended_for{iPath})]);
+
+      else
+        % also check that this NIfTI aims to the same m0scan
+        [~, path2check, ext2check] = fileparts(path_intended_for{iPath});
+        filename_found = max(arrayfun(@(x) strcmp(x.filename, ...
+                                                  [path2check ext2check]), ...
+                                      subject.perf));
+        if ~filename_found
+          warning(['Did not find NIfTI for which is intended: ' structure.filename]);
+
+        else
+          structure.intended_for = path_intended_for{iPath};
+
+        end
+      end
+    end
+
+  end
+
+end
+
 function subject = parse_fmap(subject)
   %
   % TODO:
diff --git a/tests/test_bids_query_asl.m b/tests/test_bids_query_asl.m
index 4211f0ca..a6951f17 100644
--- a/tests/test_bids_query_asl.m
+++ b/tests/test_bids_query_asl.m
@@ -14,23 +14,21 @@ function test_bids_query_asl_basic()
   pth_bids_example = get_test_data_dir();
 
   %% 'asl001'
-  BIDS = bids.layout(fullfile(pth_bids_example, 'asl001 '));
+  BIDS = bids.layout(fullfile(pth_bids_example, 'asl001'));
 
   modalities = {'anat', 'perf'};
   assertEqual(bids.query(BIDS, 'modalities'), modalities);
 
-  types = {'T1w', 'asl'};
-  %   types = {'T1w', 'asl', 'aslcontext', 'asllabelling'};
+  types = {'T1w', 'asl', 'aslcontext', 'asllabeling'};
   assertEqual(bids.query(BIDS, 'types'), types);
 
   %% 'asl002'
-  BIDS = bids.layout(fullfile(pth_bids_example, 'asl002 '));
+  BIDS = bids.layout(fullfile(pth_bids_example, 'asl002'));
 
   modalities = {'anat', 'perf'};
   assertEqual(bids.query(BIDS, 'modalities'), modalities);
 
-  types = {'T1w', 'asl', 'm0scan'};
-  %   types = {'T1w', 'asl', 'aslcontext', 'asllabelling', 'm0scan'};
+  types = {'T1w', 'asl', 'aslcontext', 'asllabeling', 'm0scan'};
   assertEqual(bids.query(BIDS, 'types'), types);
   assertEqual(bids.internal.file_utils(bids.query(BIDS, 'data', 'type', 'm0scan'), 'basename'), ...
               {'sub-Sub103_m0scan.nii'});
@@ -41,8 +39,7 @@ function test_bids_query_asl_basic()
   modalities = {'anat', 'perf'};
   assertEqual(bids.query(BIDS, 'modalities'), modalities);
 
-  types = {'T1w', 'asl', 'm0scan'};
-  %   types = {'T1w', 'asl', 'aslcontext', 'asllabelling', 'm0scan'};
+  types = {'T1w', 'asl', 'aslcontext', 'asllabeling', 'm0scan'};
   assertEqual(bids.query(BIDS, 'types'), types);
 
   %% 'asl004'
@@ -51,8 +48,7 @@ function test_bids_query_asl_basic()
   modalities = {'anat', 'fmap', 'perf'};
   assertEqual(bids.query(BIDS, 'modalities'), modalities);
 
-  types = {'T1w', 'asl', 'm0scan'};
-  %   types = {'T1w', 'asl', 'aslcontext', 'asllabelling', 'm0scan'};
+  types = {'T1w', 'asl', 'aslcontext', 'asllabeling', 'm0scan'};
   assertEqual(bids.query(BIDS, 'types'), types);
 
 end

From b0a4f352ffea593f713bcd1d4668bf955d0db13c Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sat, 6 Feb 2021 21:27:24 +0100
Subject: [PATCH 23/54] pass schema as argument instead of reloading it

---
 +bids/+internal/append_to_structure.m |  9 +++------
 +bids/layout.m                        | 24 ++++++++++++------------
 tests/test_append_to_structure.m      | 10 +++++++---
 3 files changed, 22 insertions(+), 21 deletions(-)

diff --git a/+bids/+internal/append_to_structure.m b/+bids/+internal/append_to_structure.m
index 1101e9e2..efda3537 100644
--- a/+bids/+internal/append_to_structure.m
+++ b/+bids/+internal/append_to_structure.m
@@ -1,15 +1,14 @@
-function subject = append_to_structure(file, subject, modality)
+function subject = append_to_structure(file, subject, modality, schema)
   % Copyright (C) 2021--, BIDS-MATLAB developers
 
   p = bids.internal.parse_filename(file);
-  idx = find_suffix_group(modality, p.type);
+  idx = find_suffix_group(modality, p.type, schema);
   if isempty(idx)
     warning('append_to_structure:noMatchingSuffix', ...
             'Skipping file with no valid suffix in schema: %s', file);
     return
   end
 
-  schema = bids.schema.load_schema();
   entities = bids.schema.return_datatype_entities(schema.datatypes.(modality)(idx));
   p = bids.internal.parse_filename(file, entities);
 
@@ -38,12 +37,10 @@
   end
 end
 
-function idx = find_suffix_group(modality, suffix)
+function idx = find_suffix_group(modality, suffix, schema)
 
   idx = [];
 
-  schema = bids.schema.load_schema();
-
   % the following loop could probably be improved with some cellfun magic
   %   cellfun(@(x, y) any(strcmp(x,y)), {p.type}, suffix_groups)
   for i = 1:size(schema.datatypes.(modality), 1)
diff --git a/+bids/layout.m b/+bids/layout.m
index 0e7b048c..5e8aa2cd 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -201,21 +201,21 @@ function tolerant_message(tolerant, msg)
     for iDatatype = 1:numel(datatypes)
       switch datatypes{iDatatype}
         case {'anat', 'beh'}
-          subject = parse_using_schema(subject, datatypes{iDatatype});
+          subject = parse_using_schema(subject, datatypes{iDatatype}, schema);
         case 'dwi'
-          subject = parse_dwi(subject);
+          subject = parse_dwi(subject, schema);
         case 'eeg'
           subject = parse_eeg(subject);
         case 'fmap'
           subject = parse_fmap(subject);
         case 'func'
-          subject = parse_func(subject);
+          subject = parse_func(subject, schema);
         case 'ieeg'
           subject = parse_ieeg(subject);
         case 'meg'
           subject = parse_meg(subject);
         case 'perf'
-          subject = parse_perf(subject);
+          subject = parse_perf(subject, schema);
       end
     end
 
@@ -226,7 +226,7 @@ function tolerant_message(tolerant, msg)
 
 end
 
-function subject = parse_using_schema(subject, datatype)
+function subject = parse_using_schema(subject, datatype, schema)
 
   pth = fullfile(subject.path, datatype);
 
@@ -236,7 +236,7 @@ function tolerant_message(tolerant, msg)
 
     for i = 1:numel(file_list)
 
-      subject = bids.internal.append_to_structure(file_list{i}, subject, datatype);
+      subject = bids.internal.append_to_structure(file_list{i}, subject, datatype, schema);
 
     end
 
@@ -244,7 +244,7 @@ function tolerant_message(tolerant, msg)
 
 end
 
-function subject = parse_dwi(subject)
+function subject = parse_dwi(subject, schema)
   % --------------------------------------------------------------------------
   % -Diffusion imaging data
   % --------------------------------------------------------------------------
@@ -257,7 +257,7 @@ function tolerant_message(tolerant, msg)
 
     for i = 1:numel(file_list)
 
-      subject = bids.internal.append_to_structure(file_list{i}, subject, datatype);
+      subject = bids.internal.append_to_structure(file_list{i}, subject, datatype, schema);
 
       % -bval file
       % ------------------------------------------------------------------
@@ -279,7 +279,7 @@ function tolerant_message(tolerant, msg)
   end
 end
 
-function subject = parse_func(subject)
+function subject = parse_func(subject, schema)
 
   % --------------------------------------------------------------------------
   % -Task imaging data
@@ -293,7 +293,7 @@ function tolerant_message(tolerant, msg)
 
     for i = 1:numel(file_list)
 
-      subject = bids.internal.append_to_structure(file_list{i}, subject, datatype);
+      subject = bids.internal.append_to_structure(file_list{i}, subject, datatype, schema);
       subject.func(end).meta = struct([]); % ?
 
       % TODO:
@@ -311,7 +311,7 @@ function tolerant_message(tolerant, msg)
   end
 end
 
-function subject = parse_perf(subject)
+function subject = parse_perf(subject, schema)
 
   % --------------------------------------------------------------------------
   % -ASL perfusion imaging data
@@ -325,7 +325,7 @@ function tolerant_message(tolerant, msg)
 
     for i = 1:numel(file_list)
 
-      subject = bids.internal.append_to_structure(file_list{i}, subject, datatype);
+      subject = bids.internal.append_to_structure(file_list{i}, subject, datatype, schema);
 
     end
 
diff --git a/tests/test_append_to_structure.m b/tests/test_append_to_structure.m
index 161727fd..fcc8cf8d 100644
--- a/tests/test_append_to_structure.m
+++ b/tests/test_append_to_structure.m
@@ -8,13 +8,15 @@
 
 function test_append_to_structure_basic()
 
+  schema = bids.schema.load_schema();
+
   subject = struct('anat', struct([]));
 
   modality = 'anat';
 
   file = '../sub-16/anat/sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz';
   entities = {'sub', 'ses', 'run', 'acq', 'ce', 'rec', 'part'};
-  subject = bids.internal.append_to_structure(file, subject, modality);
+  subject = bids.internal.append_to_structure(file, subject, modality, schema);
 
   expected.anat = struct( ...
                          'filename', 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz', ...
@@ -34,16 +36,18 @@ function test_append_to_structure_basic()
 
 function test_append_to_structure_basic_test()
 
+  schema = bids.schema.load_schema();
+
   subject = struct('anat', struct([]));
   modality = 'anat';
 
   file = '../sub-16/anat/sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz';
   entities = {'sub', 'ses', 'run', 'acq', 'ce', 'rec', 'part'};
-  subject = bids.internal.append_to_structure(file, subject, modality);
+  subject = bids.internal.append_to_structure(file, subject, modality, schema);
 
   file = '../sub-16/anat/sub-16_ses-mri_run-1_T1map.nii.gz';
   entities = {'sub', 'ses', 'run', 'acq', 'ce', 'rec'};
-  subject = bids.internal.append_to_structure(file, subject, modality);
+  subject = bids.internal.append_to_structure(file, subject, modality, schema);
 
   expected(1).anat = struct( ...
                             'filename', 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz', ...

From 60c4cd4dabc705cbca0bc286a29a7e20b6af7cd5 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sat, 6 Feb 2021 23:19:53 +0100
Subject: [PATCH 24/54] refactor handling of loading of metadata in  layout

---
 +bids/+internal/get_metadata.m | 13 +++--
 +bids/layout.m                 | 99 +++++++++++++++-------------------
 2 files changed, 52 insertions(+), 60 deletions(-)

diff --git a/+bids/+internal/get_metadata.m b/+bids/+internal/get_metadata.m
index 612658f1..1c822eda 100644
--- a/+bids/+internal/get_metadata.m
+++ b/+bids/+internal/get_metadata.m
@@ -73,9 +73,15 @@
 
   end
 
-  % ==========================================================================
-  % -Inheritance principle
-  % ==========================================================================
+  if isempty(meta)
+    warning('No metadata for %s', filename);
+  end
+
+end
+
+% ==========================================================================
+% -Inheritance principle
+% ==========================================================================
 function s1 = update_metadata(s1, s2, file)
   if isempty(s2)
     return
@@ -88,3 +94,4 @@
       s1.(fn{i}) = s2.(fn{i});
     end
   end
+end
diff --git a/+bids/layout.m b/+bids/layout.m
index 5e8aa2cd..6b48e7cd 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -346,7 +346,11 @@ function tolerant_message(tolerant, msg)
         subject.perf(j).meta = [];
         subject.perf(j).dependencies = [];
 
-        subject.perf(j) = manage_json_sidecar(subject.perf(j), pth);
+        subject.perf(j).meta = bids.internal.get_metadata( ...
+                                                          fullfile( ...
+                                                                   subject.path, ...
+                                                                   datatype, ...
+                                                                   file_list{j}));
 
         subject.perf(j) = manage_aslcontext(subject.perf(j), pth);
 
@@ -372,6 +376,12 @@ function tolerant_message(tolerant, msg)
 
         subject.perf(j).intended_for = [];
 
+        subject.perf(j).meta = bids.internal.get_metadata( ...
+                                                          fullfile( ...
+                                                                   subject.path, ...
+                                                                   datatype, ...
+                                                                   file_list{j}));
+
         subject.perf(j) = manage_intended_for(subject.perf(j), subject, pth);
 
       end
@@ -380,23 +390,6 @@ function tolerant_message(tolerant, msg)
 
   end % if exist(pth, 'dir')
 
-end % function subject = parse_perf(subject)
-
-function structure = manage_json_sidecar(structure, pth)
-
-  % Manage JSON-sidecar metadata (REQUIRED)
-  % ---------------------------
-  metafile = fullfile(pth, strrep(structure.filename, structure.ext, '.json'));
-
-  if exist(metafile, 'file')
-    [~, Ffile] = fileparts(metafile);
-    structure.dependencies.sidecar = [Ffile '.json'];
-    structure.meta = bids.util.jsondecode(metafile);
-  else
-    warning(['Missing: ' metafile]);
-
-  end
-
 end
 
 function perf = manage_aslcontext(perf, pth)
@@ -439,7 +432,7 @@ function tolerant_message(tolerant, msg)
   % M0 field is flexible:
 
   if ~isfield(perf.meta, 'M0Type')
-    warning(['M0Type field missing in ' perf.dependencies.sidecar]);
+    warning('M0Type field missing for %s', perf.filename);
 
   else
 
@@ -555,8 +548,6 @@ function tolerant_message(tolerant, msg)
 
 function structure = manage_intended_for(structure, subject, pth)
 
-  structure = manage_json_sidecar(structure, pth);
-
   if isempty(structure.meta)
     return
 
@@ -565,7 +556,7 @@ function tolerant_message(tolerant, msg)
     % Get all NIfTIs that this m0scan is intended for
     path_intended_for = {};
     if ~isfield(structure.meta, 'IntendedFor')
-      warning(['Missing field IntendedFor in ' structure.dependencies.sidecar]);
+      warning('Missing field IntendedFor for %s', structure.filename);
 
     elseif ischar(structure.meta.IntendedFor)
       path_intended_for{1} = structure.meta.IntendedFor;
@@ -613,11 +604,13 @@ function tolerant_message(tolerant, msg)
   % --------------------------------------------------------------------------
   % -Fieldmap data
   % --------------------------------------------------------------------------
-  pth = fullfile(subject.path, 'fmap');
+  datatype = 'fmap';
+
+  pth = fullfile(subject.path, datatype);
 
   if exist(pth, 'dir')
 
-    file_list = return_file_list('fmap', subject);
+    file_list = return_file_list(datatype, subject);
 
     j = 1;
 
@@ -643,12 +636,11 @@ function tolerant_message(tolerant, msg)
 
         subject = append_common_fmap_fields_to_structure(subject, labels{idx(i)}, j);
 
-        metafile = return_fmap_metadata_file(subject, file_list{idx(i)});
-        subject.fmap(j).meta = struct([]);
-        % (!) TODO: file can also be stored at higher levels (inheritance principle)
-        if ~isempty(metafile)
-          subject.fmap(j).meta = bids.util.jsondecode(metafile);
-        end
+        subject.fmap(j).meta = bids.internal.get_metadata( ...
+                                                          fullfile( ...
+                                                                   subject.path, ...
+                                                                   datatype, ...
+                                                                   file_list{j}));
 
         j = j + 1;
 
@@ -681,13 +673,13 @@ function tolerant_message(tolerant, msg)
 
         subject = append_common_fmap_fields_to_structure(subject, labels{idx(i)}, j);
 
-        metafile = return_fmap_metadata_file(subject, file_list{idx(i)});
+        json_file = return_jsonfile(subject, file_list{idx(i)}, datatype);
         subject.fmap(j).meta = struct([]);
         % (!) TODO: file can also be stored at higher levels (inheritance principle)
-        if ~isempty(metafile)
+        if ~isempty(json_file)
           subject.fmap(j).meta = { ...
-                                  bids.util.jsondecode(metafile), ...
-                                  bids.util.jsondecode(strrep(metafile, ...
+                                  bids.util.jsondecode(json_file), ...
+                                  bids.util.jsondecode(strrep(json_file, ...
                                                               '_phase1.json', ...
                                                               '_phase2.json'))};
         end
@@ -716,12 +708,11 @@ function tolerant_message(tolerant, msg)
 
         subject = append_common_fmap_fields_to_structure(subject, labels{idx(i)}, j);
 
-        metafile = return_fmap_metadata_file(subject, file_list{idx(i)});
-        subject.fmap(j).meta = struct([]);
-        % (!) TODO: file can also be stored at higher levels (inheritance principle)
-        if ~isempty(metafile)
-          subject.fmap(j).meta = bids.util.jsondecode(metafile);
-        end
+        subject.fmap(j).meta = bids.internal.get_metadata( ...
+                                                          fullfile( ...
+                                                                   subject.path, ...
+                                                                   datatype, ...
+                                                                   file_list{j}));
 
         j = j + 1;
 
@@ -748,12 +739,11 @@ function tolerant_message(tolerant, msg)
 
         subject = append_common_fmap_fields_to_structure(subject, labels{idx(i)}, j);
 
-        metafile = return_fmap_metadata_file(subject, file_list{idx(i)});
-        subject.fmap(j).meta = struct([]);
-        % (!) TODO: file can also be stored at higher levels (inheritance principle)
-        if ~isempty(metafile)
-          subject.fmap(j).meta = bids.util.jsondecode(metafile);
-        end
+        subject.fmap(j).meta = bids.internal.get_metadata( ...
+                                                          fullfile( ...
+                                                                   subject.path, ...
+                                                                   datatype, ...
+                                                                   file_list{j}));
 
         j = j + 1;
 
@@ -1019,11 +1009,6 @@ function tolerant_message(tolerant, msg)
 
 end
 
-% TODO
-%
-% more refactoring can be done across the several 'return_X_file_list' functions
-%
-
 function file_list = return_file_list(modality, subject)
 
   switch modality
@@ -1099,18 +1084,18 @@ function tolerant_message(tolerant, msg)
 
 end
 
-function metafile = return_fmap_metadata_file(subject, fmap_file)
+function json_file = return_jsonfile(subject, filename, modality)
 
-  pth = fullfile(subject.path, 'fmap');
+  pth = fullfile(subject.path, modality);
 
   fb = bids.internal.file_utils(bids.internal.file_utils( ...
-                                                         fmap_file, ...
+                                                         filename, ...
                                                          'basename'), ...
                                 'basename');
-  metafile = fullfile(pth, bids.internal.file_utils(fb, 'ext', 'json'));
+  json_file = fullfile(pth, bids.internal.file_utils(fb, 'ext', 'json'));
 
-  if ~exist(metafile, 'file')
-    metafile = [];
+  if ~exist(json_file, 'file')
+    json_file = [];
   end
 
 end

From accf9d3310387ccd8ce60d552f8d89c8c0139796 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sun, 7 Feb 2021 00:14:56 +0100
Subject: [PATCH 25/54] refactor fmap parsing to use schema

---
 +bids/layout.m | 269 ++++++++++---------------------------------------
 1 file changed, 52 insertions(+), 217 deletions(-)

diff --git a/+bids/layout.m b/+bids/layout.m
index 6b48e7cd..4e24effc 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -207,7 +207,7 @@ function tolerant_message(tolerant, msg)
         case 'eeg'
           subject = parse_eeg(subject);
         case 'fmap'
-          subject = parse_fmap(subject);
+          subject = parse_fmap(subject, schema);
         case 'func'
           subject = parse_func(subject, schema);
         case 'ieeg'
@@ -593,162 +593,59 @@ function tolerant_message(tolerant, msg)
 
 end
 
-function subject = parse_fmap(subject)
-  %
-  % TODO:
-  %
-  % 20210114 - From Remi:
-  % For other modalities, metadata are fetched upon query.
-  % It is unclear why we do it differently for fmaps
+function subject = parse_fmap(subject, schema)
 
-  % --------------------------------------------------------------------------
-  % -Fieldmap data
-  % --------------------------------------------------------------------------
   datatype = 'fmap';
-
   pth = fullfile(subject.path, datatype);
 
   if exist(pth, 'dir')
 
     file_list = return_file_list(datatype, subject);
 
-    j = 1;
-
-    % -Phase difference image and at least one magnitude image
-    % ----------------------------------------------------------------------
-    labels = return_labels_fieldmap(file_list, 'phase_difference_image');
-
-    if any(~cellfun(@isempty, labels))
-
-      idx = find(~cellfun(@isempty, labels));
-
-      for i = 1:numel(idx)
-
-        subject.fmap(j).type = 'phasediff';
-        subject.fmap(j).filename = file_list{idx(i)};
-        subject.fmap(j).magnitude = { ...
-                                     strrep(file_list{idx(i)}, ...
-                                            '_phasediff.nii', ...
-                                            '_magnitude1.nii'), ...
-                                     strrep(file_list{idx(i)}, ...
-                                            '_phasediff.nii', ...
-                                            '_magnitude2.nii')}; % optional
-
-        subject = append_common_fmap_fields_to_structure(subject, labels{idx(i)}, j);
-
-        subject.fmap(j).meta = bids.internal.get_metadata( ...
-                                                          fullfile( ...
-                                                                   subject.path, ...
-                                                                   datatype, ...
-                                                                   file_list{j}));
-
-        j = j + 1;
-
-      end
-    end
-
-    % -Two phase images and two magnitude images
-    % ----------------------------------------------------------------------
-    labels = return_labels_fieldmap(file_list, 'two_phase_image');
-
-    if any(~cellfun(@isempty, labels))
-
-      idx = find(~cellfun(@isempty, labels));
-
-      for i = 1:numel(idx)
+    for i = 1:numel(file_list)
 
-        subject.fmap(j).type = 'phase12';
-        subject.fmap(j).filename = { ...
-                                    file_list{idx(i)}, ...
-                                    strrep(file_list{idx(i)}, ...
-                                           '_phase1.nii', ...
-                                           '_phase2.nii')};
-        subject.fmap(j).magnitude = { ...
-                                     strrep(file_list{idx(i)}, ...
-                                            '_phase1.nii', ...
-                                            '_magnitude1.nii'), ...
-                                     strrep(file_list{idx(i)}, ...
-                                            '_phase1.nii', ...
-                                            '_magnitude2.nii')};
-
-        subject = append_common_fmap_fields_to_structure(subject, labels{idx(i)}, j);
-
-        json_file = return_jsonfile(subject, file_list{idx(i)}, datatype);
-        subject.fmap(j).meta = struct([]);
-        % (!) TODO: file can also be stored at higher levels (inheritance principle)
-        if ~isempty(json_file)
-          subject.fmap(j).meta = { ...
-                                  bids.util.jsondecode(json_file), ...
-                                  bids.util.jsondecode(strrep(json_file, ...
-                                                              '_phase1.json', ...
-                                                              '_phase2.json'))};
-        end
+      subject = bids.internal.append_to_structure(file_list{i}, subject, datatype, schema);
 
-        j = j + 1;
+      subject.fmap(i).meta = bids.internal.get_metadata( ...
+                                                        fullfile( ...
+                                                                 subject.path, ...
+                                                                 datatype, ...
+                                                                 file_list{i}));
+      %       subject.perf(i).intended_for = [];
+      %       subject.fmap(i) = manage_intended_for(subject.fmap(i), subject, pth);
+
+      switch subject.fmap(i).type
+
+        % -A single, real fieldmap image
+        case {'fieldmap', 'magnitude'}
+          subject.fmap(i).dependencies.magnitude = strrep(file_list{idx(i)}, ...
+                                                          '_fieldmap.nii', ...
+                                                          '_magnitude.nii');
+
+          % -Phase difference image and at least one magnitude image
+        case {'phasediff'}
+          subject.fmap(i).dependencies.magnitude = { ...
+                                                    strrep(file_list{i}, ...
+                                                           '_phasediff.nii', ...
+                                                           '_magnitude1.nii'), ...
+                                                    strrep(file_list{i}, ...
+                                                           '_phasediff.nii', ...
+                                                           '_magnitude2.nii')}; % optional
+
+          % -Two phase images and two magnitude images
+        case {'phase1', 'phase2'}
+          subject.fmap(i).dependencies.magnitude = { ...
+                                                    strrep(file_list{i}, ...
+                                                           '_phase1.nii', ...
+                                                           '_magnitude1.nii'), ...
+                                                    strrep(file_list{i}, ...
+                                                           '_phase1.nii', ...
+                                                           '_magnitude2.nii')};
 
       end
 
     end
 
-    % -A single, real fieldmap image
-    % ----------------------------------------------------------------------
-    labels = return_labels_fieldmap(file_list, 'fieldmap_image');
-
-    if any(~cellfun(@isempty, labels))
-
-      idx = find(~cellfun(@isempty, labels));
-
-      for i = 1:numel(idx)
-
-        subject.fmap(j).type = 'fieldmap';
-        subject.fmap(j).filename = file_list{idx(i)};
-        subject.fmap(j).magnitude = strrep(file_list{idx(i)}, ...
-                                           '_fieldmap.nii', ...
-                                           '_magnitude.nii');
-
-        subject = append_common_fmap_fields_to_structure(subject, labels{idx(i)}, j);
-
-        subject.fmap(j).meta = bids.internal.get_metadata( ...
-                                                          fullfile( ...
-                                                                   subject.path, ...
-                                                                   datatype, ...
-                                                                   file_list{j}));
-
-        j = j + 1;
-
-      end
-    end
-
-    % -Multiple phase encoded directions (topup)
-    % ----------------------------------------------------------------------
-    labels = return_labels_fieldmap(file_list, 'phase_encoded_direction_image');
-
-    if any(~cellfun(@isempty, labels))
-
-      idx = find(~cellfun(@isempty, labels));
-
-      for i = 1:numel(idx)
-
-        subject.fmap(j).filename = file_list{idx(i)};
-        if ~isempty(regexp(subject.fmap(j).filename, 'm0scan', 'ONCE'))
-          subject.fmap(j).type = 'm0scan';
-        else
-          subject.fmap(j).type = 'epi';
-        end
-        subject.fmap(j).dir = labels{idx(i)}.dir;
-
-        subject = append_common_fmap_fields_to_structure(subject, labels{idx(i)}, j);
-
-        subject.fmap(j).meta = bids.internal.get_metadata( ...
-                                                          fullfile( ...
-                                                                   subject.path, ...
-                                                                   datatype, ...
-                                                                   file_list{j}));
-
-        j = j + 1;
-
-      end
-    end
   end
 
 end
@@ -945,14 +842,6 @@ function tolerant_message(tolerant, msg)
 
 end
 
-function subject = append_common_fmap_fields_to_structure(subject, labels, idx)
-
-  subject.fmap(idx).ses = regexprep(labels.ses, '^_[a-zA-Z0-9]+-', '');
-  subject.fmap(idx).acq = regexprep(labels.acq, '^_[a-zA-Z0-9]+-', '');
-  subject.fmap(idx).run = regexprep(labels.run, '^_[a-zA-Z0-9]+-', '');
-
-end
-
 function f = convert_to_cell(f)
   if isempty(f)
     f = {};
@@ -977,38 +866,6 @@ function tolerant_message(tolerant, msg)
   end
 end
 
-function labels = return_labels_fieldmap(file_list, fiefmap_type)
-
-  direction_pattern = '';
-
-  switch fiefmap_type
-
-    case 'phase_difference_image'
-      suffix = 'phasediff';
-
-    case 'two_phase_image'
-      suffix = 'phase1';
-
-    case 'fieldmap_image'
-      suffix = 'fieldmap';
-
-    case 'phase_encoded_direction_image'
-      suffix = '(epi|m0scan)';
-
-      direction_pattern = '_dir-(?<dir>[a-zA-Z0-9]+)?';
-
-  end
-
-  labels = regexp(file_list, [ ...
-                              '^sub-[a-zA-Z0-9]+', ...          % sub-<participant_label>
-                              '(?<ses>_ses-[a-zA-Z0-9]+)?', ... % ses-<label>
-                              '(?<acq>_acq-[a-zA-Z0-9]+)?', ... % acq-<label>
-                              direction_pattern, ...            % dir-<index>
-                              '(?<run>_run-[a-zA-Z0-9]+)?', ... % run-<index>
-                              '_' suffix '\.nii(\.gz)?$'], 'names');   % NIfTI file extension
-
-end
-
 function file_list = return_file_list(modality, subject)
 
   switch modality
@@ -1017,15 +874,12 @@ function tolerant_message(tolerant, msg)
     % it should be possible to create some of those patterns for the regexp
     % based on some of the required entities written down in the schema
 
-    case {'anat', 'dwi'}
+    case {'anat', 'dwi', 'fmap', 'pet'}
       pattern = '_([a-zA-Z0-9]+){1}\\.nii(\\.gz)?';
 
     case 'func'
       pattern = '_task-.*\\.nii(\\.gz)|events\\.tsv|physio\\.tsv\\.gz|stim\\.tsv\\.gz?';
 
-    case 'fmap'
-      pattern = '\\.nii(\\.gz)?';
-
     case 'eeg'
       pattern = '_task-.*_eeg\\..*[^json]';
 
@@ -1035,9 +889,6 @@ function tolerant_message(tolerant, msg)
     case 'beh'
       pattern = '_task-.*_(events\\.tsv|beh\\.json|physio\\.tsv\\.gz|stim\\.tsv\\.gz)';
 
-    case 'pet'
-      pattern = '_task-.*_pet\\.nii(\\.gz)?';
-
     case 'ieeg'
       pattern = '_task-.*_ieeg\\..*[^json]';
 
@@ -1075,31 +926,15 @@ function tolerant_message(tolerant, msg)
 
   pth = fullfile(subject.path, modality);
 
-  [file_list, d] = bids.internal.file_utils('List', ...
-                                            pth, ...
-                                            sprintf(['^%s.*' pattern '$'], ...
-                                                    subject.name));
+  file_list = bids.internal.file_utils('List', ...
+                                       pth, ...
+                                       sprintf(['^%s.*' pattern '$'], ...
+                                               subject.name));
 
   file_list = convert_to_cell(file_list);
 
 end
 
-function json_file = return_jsonfile(subject, filename, modality)
-
-  pth = fullfile(subject.path, modality);
-
-  fb = bids.internal.file_utils(bids.internal.file_utils( ...
-                                                         filename, ...
-                                                         'basename'), ...
-                                'basename');
-  json_file = fullfile(pth, bids.internal.file_utils(fb, 'ext', 'json'));
-
-  if ~exist(json_file, 'file')
-    json_file = [];
-  end
-
-end
-
 function file_list = return_channel_description_file_list(modality, subject)
   %
   % Channel description table
@@ -1116,10 +951,10 @@ function tolerant_message(tolerant, msg)
 
   pth = fullfile(subject.path, modality);
 
-  [file_list, d] = bids.internal.file_utils('List', ...
-                                            pth, ...
-                                            sprintf(['^%s.*' pattern '$'], ...
-                                                    subject.name));
+  file_list = bids.internal.file_utils('List', ...
+                                       pth, ...
+                                       sprintf(['^%s.*' pattern '$'], ...
+                                               subject.name));
 
   file_list = convert_to_cell(file_list);
 
@@ -1138,10 +973,10 @@ function tolerant_message(tolerant, msg)
 
   pth = fullfile(subject.path, modality);
 
-  [file_list, d] = bids.internal.file_utils('List', ...
-                                            pth, ...
-                                            sprintf(['^%s' pattern '$'], ...
-                                                    subject.name));
+  file_list = bids.internal.file_utils('List', ...
+                                       pth, ...
+                                       sprintf(['^%s' pattern '$'], ...
+                                               subject.name));
 
   file_list = convert_to_cell(file_list);
 

From cc8281822da58266642ee627815185f4b1912191 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sun, 7 Feb 2021 00:25:06 +0100
Subject: [PATCH 26/54] parse ieeg with schema

---
 +bids/layout.m | 55 ++++----------------------------------------------
 1 file changed, 4 insertions(+), 51 deletions(-)

diff --git a/+bids/layout.m b/+bids/layout.m
index 4e24effc..e6056925 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -200,7 +200,7 @@ function tolerant_message(tolerant, msg)
 
     for iDatatype = 1:numel(datatypes)
       switch datatypes{iDatatype}
-        case {'anat', 'beh'}
+        case {'anat', 'beh', 'ieeg'}
           subject = parse_using_schema(subject, datatypes{iDatatype}, schema);
         case 'dwi'
           subject = parse_dwi(subject, schema);
@@ -210,8 +210,6 @@ function tolerant_message(tolerant, msg)
           subject = parse_fmap(subject, schema);
         case 'func'
           subject = parse_func(subject, schema);
-        case 'ieeg'
-          subject = parse_ieeg(subject);
         case 'meg'
           subject = parse_meg(subject);
         case 'perf'
@@ -792,45 +790,6 @@ function tolerant_message(tolerant, msg)
   end
 end
 
-function subject = parse_ieeg(subject)
-  % --------------------------------------------------------------------------
-  % -Human intracranial electrophysiology
-  % --------------------------------------------------------------------------
-  pth = fullfile(subject.path, 'ieeg');
-
-  if exist(pth, 'dir')
-
-    entities = return_entities('ieeg');
-
-    file_list = return_file_list('ieeg', subject);
-
-    for i = 1:numel(file_list)
-
-      % European Data Format (.edf)
-      % BrainVision Core Data Format (.vhdr, .eeg, .vmrk) by Brain Products GmbH
-      % The format used by the MATLAB toolbox EEGLAB (.set and .fdt files)
-      % Neurodata Without Borders (.nwb)
-      % MEF3 (.mef)
-
-      p = bids.internal.parse_filename(file_list{i}, entities);
-      switch p.ext
-        case {'.edf', '.vhdr', '.set', '.nwb', '.mef'}
-          % each recording is described with a single file,
-          % even though the data can consist of multiple
-          subject.ieeg = [subject.ieeg p];
-          subject.ieeg(end).meta = struct([]); % ?
-        case {'.vmrk', '.eeg', '.fdt'}
-          % skip the additional files that come with certain data formats
-        otherwise
-          % skip unknown files
-      end
-
-    end
-
-  end
-
-end
-
 % --------------------------------------------------------------------------
 %                            HELPER FUNCTIONS
 % --------------------------------------------------------------------------
@@ -854,7 +813,7 @@ function tolerant_message(tolerant, msg)
 
   switch modality
 
-    case {'eeg', 'ieeg'}
+    case 'eeg'
       entities = {'sub', 'ses', 'task', 'acq', 'run', 'meta'};
 
     case 'meg'
@@ -880,18 +839,12 @@ function tolerant_message(tolerant, msg)
     case 'func'
       pattern = '_task-.*\\.nii(\\.gz)|events\\.tsv|physio\\.tsv\\.gz|stim\\.tsv\\.gz?';
 
-    case 'eeg'
-      pattern = '_task-.*_eeg\\..*[^json]';
-
-    case 'meg'
-      pattern = '_task-.*_meg\\..*[^json]';
+    case {'eeg', 'meg', 'ieeg'}
+      pattern = ['_task-.*_' modality '\\..*[^json]'];
 
     case 'beh'
       pattern = '_task-.*_(events\\.tsv|beh\\.json|physio\\.tsv\\.gz|stim\\.tsv\\.gz)';
 
-    case 'ieeg'
-      pattern = '_task-.*_ieeg\\..*[^json]';
-
     case 'perf'
       pattern = '_(asl|m0scan)\\.nii(\\.gz)|aslcontext\\.tsv|asllabeling\\.jpg';
 

From 327adad8cbe75b8c8a2e90c42f82f0aa9b936e67 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sun, 7 Feb 2021 00:49:55 +0100
Subject: [PATCH 27/54] use schema to parse eeg

---
 +bids/layout.m | 91 +++++++++++++++-----------------------------------
 1 file changed, 27 insertions(+), 64 deletions(-)

diff --git a/+bids/layout.m b/+bids/layout.m
index e6056925..65ba169f 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -205,7 +205,7 @@ function tolerant_message(tolerant, msg)
         case 'dwi'
           subject = parse_dwi(subject, schema);
         case 'eeg'
-          subject = parse_eeg(subject);
+          subject = parse_eeg(subject, schema);
         case 'fmap'
           subject = parse_fmap(subject, schema);
         case 'func'
@@ -243,9 +243,11 @@ function tolerant_message(tolerant, msg)
 end
 
 function subject = parse_dwi(subject, schema)
+
   % --------------------------------------------------------------------------
-  % -Diffusion imaging data
+  %  Diffusion imaging data
   % --------------------------------------------------------------------------
+
   datatype = 'dwi';
   pth = fullfile(subject.path, datatype);
 
@@ -262,7 +264,7 @@ function tolerant_message(tolerant, msg)
       % bval file can also be stored at higher levels (inheritance principle)
       bvalfile = bids.internal.get_metadata(file_list{i}, '^.*%s\\.bval$');
       if isfield(bvalfile, 'filename')
-        subject.dwi(end).bval = bids.util.tsvread(bvalfile.filename); % ?
+        subject.dwi(end).bval = bids.util.tsvread(bvalfile.filename);
       end
 
       % -bvec file
@@ -270,7 +272,7 @@ function tolerant_message(tolerant, msg)
       % bvec file can also be stored at higher levels (inheritance principle)
       bvecfile = bids.internal.get_metadata(file_list{i}, '^.*%s\\.bvec$');
       if isfield(bvalfile, 'filename')
-        subject.dwi(end).bvec = bids.util.tsvread(bvecfile.filename); % ?
+        subject.dwi(end).bvec = bids.util.tsvread(bvecfile.filename);
       end
 
     end
@@ -280,8 +282,9 @@ function tolerant_message(tolerant, msg)
 function subject = parse_func(subject, schema)
 
   % --------------------------------------------------------------------------
-  % -Task imaging data
+  %  Task imaging data
   % --------------------------------------------------------------------------
+
   datatype = 'func';
   pth = fullfile(subject.path, datatype);
 
@@ -300,8 +303,8 @@ function tolerant_message(tolerant, msg)
       % can also be stored at higher levels (inheritance principle).
       %
 
-      if strcmp(subject.func(end).meta, 'events')
-        subject.func(end).meta = bids.util.tsvread(fullfile(pth, file_list{i})); % ?
+      if strcmp(subject.func(end).type, 'events')
+        subject.func(end).meta = bids.util.tsvread(fullfile(pth, file_list{i}));
       end
 
     end
@@ -312,8 +315,9 @@ function tolerant_message(tolerant, msg)
 function subject = parse_perf(subject, schema)
 
   % --------------------------------------------------------------------------
-  % -ASL perfusion imaging data
+  % ASL perfusion imaging data
   % --------------------------------------------------------------------------
+
   datatype = 'perf';
   pth = fullfile(subject.path, 'perf');
 
@@ -648,67 +652,28 @@ function tolerant_message(tolerant, msg)
 
 end
 
-function subject = parse_eeg(subject)
-  % --------------------------------------------------------------------------
-  % -EEG data
-  % --------------------------------------------------------------------------
-  pth = fullfile(subject.path, 'eeg');
-
-  if exist(pth, 'dir')
-
-    entities = return_entities('eeg');
+function subject = parse_eeg(subject, schema)
 
-    file_list = return_file_list('eeg', subject);
-
-    for i = 1:numel(file_list)
-
-      % European data format (.edf)
-      % BrainVision Core Data Format (.vhdr, .vmrk, .eeg) by Brain Products GmbH
-      % The format used by the MATLAB toolbox EEGLAB (.set and .fdt files)
-      % Biosemi data format (.bdf)
-
-      p = bids.internal.parse_filename(file_list{i}, entities);
-      switch p.ext
-        case {'.edf', '.vhdr', '.set', '.bdf'}
-          % each recording is described with a single file,
-          % even though the data can consist of multiple
-          subject.eeg = [subject.eeg p];
-          subject.eeg(end).meta = struct([]); % ?
-        case {'.vmrk', '.eeg', '.fdt'}
-          % skip the additional files that come with certain data formats
-        otherwise
-          % skip unknown files
-      end
-
-    end
-
-    file_list = return_event_file_list('eeg', subject);
-
-    for i = 1:numel(file_list)
+  datatype = 'eeg';
 
-      subject = append_to_structure(file_list{i}, entities, subject, 'eeg');
-
-      subject.eeg(end).meta = bids.util.tsvread(fullfile(pth, file_list{i})); % ?
+  pth = fullfile(subject.path, datatype);
 
-    end
+  if exist(pth, 'dir')
 
-    file_list = return_channel_description_file_list('eeg', subject);
+    file_list = return_file_list(datatype, subject);
 
     for i = 1:numel(file_list)
 
-      subject = append_to_structure(file_list{i}, entities, subject, 'eeg');
-
-      subject.eeg(end).meta = bids.util.tsvread(fullfile(pth, file_list{i})); % ?
+      subject = bids.internal.append_to_structure(file_list{i}, subject, datatype, schema);
 
-    end
+      switch subject.eeg(end).type
 
-    file_list = return_session_specific_file_list('eeg', subject);
+        case {'events', 'channels', 'electrodes'}
+          subject.eeg(end).meta = bids.util.tsvread(fullfile(pth, file_list{i}));
 
-    for i = 1:numel(file_list)
+        case {'photo', 'coordsystem'}
 
-      subject = append_to_structure(file_list{i}, entities, subject, 'eeg');
-
-      subject.eeg(end).meta = struct([]); % ?
+      end
 
     end
 
@@ -717,9 +682,7 @@ function tolerant_message(tolerant, msg)
 end
 
 function subject = parse_meg(subject)
-  % --------------------------------------------------------------------------
-  % -MEG data
-  % --------------------------------------------------------------------------
+
   pth = fullfile(subject.path, 'meg');
 
   if exist(pth, 'dir')
@@ -872,7 +835,7 @@ function tolerant_message(tolerant, msg)
 
   switch modality
 
-    case {'eeg', 'meg'}
+    case 'meg'
       pattern = '_task-.*_events\\.tsv';
 
   end
@@ -897,7 +860,7 @@ function tolerant_message(tolerant, msg)
 
   switch modality
 
-    case {'eeg', 'meg'}
+    case 'meg'
       pattern = '_task-.*_channels\\.tsv';
 
   end
@@ -917,7 +880,7 @@ function tolerant_message(tolerant, msg)
 
   switch modality
 
-    case {'eeg', 'meg'}
+    case 'meg'
       pattern = [ ...
                  '(_ses-[a-zA-Z0-9]+)?.*_', ...
                  '(electrodes\\.tsv|photo\\.jpg|coordsystem\\.json|headshape\\..*)'];

From 8de1143cb4640bab24aeb3ca704f94555f96434c Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sun, 7 Feb 2021 01:09:46 +0100
Subject: [PATCH 28/54] parse meg with schema and refactor

---
 +bids/layout.m | 147 +++----------------------------------------------
 1 file changed, 9 insertions(+), 138 deletions(-)

diff --git a/+bids/layout.m b/+bids/layout.m
index 65ba169f..950d9194 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -204,14 +204,12 @@ function tolerant_message(tolerant, msg)
           subject = parse_using_schema(subject, datatypes{iDatatype}, schema);
         case 'dwi'
           subject = parse_dwi(subject, schema);
-        case 'eeg'
-          subject = parse_eeg(subject, schema);
+        case {'eeg', 'meg'}
+          subject = parse_meeg(subject, datatypes{iDatatype}, schema);
         case 'fmap'
           subject = parse_fmap(subject, schema);
         case 'func'
           subject = parse_func(subject, schema);
-        case 'meg'
-          subject = parse_meg(subject);
         case 'perf'
           subject = parse_perf(subject, schema);
       end
@@ -652,9 +650,7 @@ function tolerant_message(tolerant, msg)
 
 end
 
-function subject = parse_eeg(subject, schema)
-
-  datatype = 'eeg';
+function subject = parse_meeg(subject, datatype, schema)
 
   pth = fullfile(subject.path, datatype);
 
@@ -666,10 +662,13 @@ function tolerant_message(tolerant, msg)
 
       subject = bids.internal.append_to_structure(file_list{i}, subject, datatype, schema);
 
-      switch subject.eeg(end).type
+      switch subject.(datatype)(end).type
 
-        case {'events', 'channels', 'electrodes'}
-          subject.eeg(end).meta = bids.util.tsvread(fullfile(pth, file_list{i}));
+        case {'events', 'channels', 'electrodes'}  %
+          % TODO: events / channels file can also be stored
+          % at higher levels (inheritance principle)
+          %
+          subject.(datatype)(end).meta = bids.util.tsvread(fullfile(pth, file_list{i}));
 
         case {'photo', 'coordsystem'}
 
@@ -681,58 +680,6 @@ function tolerant_message(tolerant, msg)
 
 end
 
-function subject = parse_meg(subject)
-
-  pth = fullfile(subject.path, 'meg');
-
-  if exist(pth, 'dir')
-
-    entities = return_entities('meg');
-
-    file_list = return_file_list('meg', subject);
-
-    for i = 1:numel(file_list)
-
-      subject = append_to_structure(file_list{i}, entities, subject, 'meg');
-
-      subject.meg(end).meta = struct([]); % ?
-
-    end
-
-    file_list = return_event_file_list('meg', subject);
-
-    for i = 1:numel(file_list)
-
-      subject = append_to_structure(file_list{i}, entities, subject, 'meg');
-
-      subject.meg(end).meta = bids.util.tsvread(fullfile(pth, file_list{i})); % ?
-
-    end
-
-    file_list = return_channel_description_file_list('meg', subject);
-
-    for i = 1:numel(file_list)
-
-      subject = append_to_structure(file_list{i}, entities, subject, 'meg');
-
-      subject.meg(end).meta = bids.util.tsvread(fullfile(pth, file_list{i})); % ?
-
-    end
-
-    file_list = return_session_specific_file_list('meg', subject);
-
-    for i = 1:numel(file_list)
-
-      subject = append_to_structure(file_list{i}, entities, subject, 'meg');
-
-      subject.meg(end).meta = struct([]); % ?
-
-    end
-
-  end
-
-end
-
 function subject = parse_pet(subject)
   % --------------------------------------------------------------------------
   % -Positron Emission Tomography imaging data
@@ -776,12 +723,6 @@ function tolerant_message(tolerant, msg)
 
   switch modality
 
-    case 'eeg'
-      entities = {'sub', 'ses', 'task', 'acq', 'run', 'meta'};
-
-    case 'meg'
-      entities = {'sub', 'ses', 'task', 'acq', 'run', 'proc', 'meta'};
-
     case 'pet'
       entities = {'sub', 'ses', 'task', 'acq', 'rec', 'run'};
 
@@ -827,73 +768,3 @@ function tolerant_message(tolerant, msg)
   file_list = convert_to_cell(file_list);
 
 end
-
-function file_list = return_event_file_list(modality, subject)
-  %
-  % TODO: events file can also be stored at higher levels (inheritance principle)
-  %
-
-  switch modality
-
-    case 'meg'
-      pattern = '_task-.*_events\\.tsv';
-
-  end
-
-  pth = fullfile(subject.path, modality);
-
-  file_list = bids.internal.file_utils('List', ...
-                                       pth, ...
-                                       sprintf(['^%s.*' pattern '$'], ...
-                                               subject.name));
-
-  file_list = convert_to_cell(file_list);
-
-end
-
-function file_list = return_channel_description_file_list(modality, subject)
-  %
-  % Channel description table
-  %
-  % TODO: those files can also be stored at higher levels (inheritance principle)
-  %
-
-  switch modality
-
-    case 'meg'
-      pattern = '_task-.*_channels\\.tsv';
-
-  end
-
-  pth = fullfile(subject.path, modality);
-
-  file_list = bids.internal.file_utils('List', ...
-                                       pth, ...
-                                       sprintf(['^%s.*' pattern '$'], ...
-                                               subject.name));
-
-  file_list = convert_to_cell(file_list);
-
-end
-
-function file_list = return_session_specific_file_list(modality, subject)
-
-  switch modality
-
-    case 'meg'
-      pattern = [ ...
-                 '(_ses-[a-zA-Z0-9]+)?.*_', ...
-                 '(electrodes\\.tsv|photo\\.jpg|coordsystem\\.json|headshape\\..*)'];
-
-  end
-
-  pth = fullfile(subject.path, modality);
-
-  file_list = bids.internal.file_utils('List', ...
-                                       pth, ...
-                                       sprintf(['^%s' pattern '$'], ...
-                                               subject.name));
-
-  file_list = convert_to_cell(file_list);
-
-end

From 831843e69da7eb0cf88b1939f69298883f385116 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sun, 7 Feb 2021 12:04:37 +0100
Subject: [PATCH 29/54] reorganize tests and make list of target files to parse
 for meeg more general

---
 +bids/layout.m               |  2 +-
 +bids/query.m                | 13 +++++---
 tests/test_bids_query.m      | 27 +++++++--------
 tests/test_bids_query_asl.m  |  2 +-
 tests/test_bids_query_eeg.m  | 64 ++++++++++++++++++++++++++++++++++++
 tests/test_bids_query_ieeg.m | 36 ++++++++++++++++++++
 tests/test_bids_query_meg.m  | 28 ++++++++++++++++
 tests/test_layout.m          |  6 ++--
 8 files changed, 154 insertions(+), 24 deletions(-)
 create mode 100644 tests/test_bids_query_eeg.m
 create mode 100644 tests/test_bids_query_ieeg.m
 create mode 100644 tests/test_bids_query_meg.m

diff --git a/+bids/layout.m b/+bids/layout.m
index 950d9194..ba68e773 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -744,7 +744,7 @@ function tolerant_message(tolerant, msg)
       pattern = '_task-.*\\.nii(\\.gz)|events\\.tsv|physio\\.tsv\\.gz|stim\\.tsv\\.gz?';
 
     case {'eeg', 'meg', 'ieeg'}
-      pattern = ['_task-.*_' modality '\\..*[^json]'];
+      pattern = '_([a-zA-Z0-9]+){1}\\..*[^json]';
 
     case 'beh'
       pattern = '_task-.*_(events\\.tsv|beh\\.json|physio\\.tsv\\.gz|stim\\.tsv\\.gz)';
diff --git a/+bids/query.m b/+bids/query.m
index 80bbe10f..90d41e8c 100644
--- a/+bids/query.m
+++ b/+bids/query.m
@@ -2,12 +2,17 @@
   % QUERY Query a directory structure formatted according to the BIDS standard
   % FORMAT result = bids.query(BIDS,query,...)
   % BIDS   - BIDS directory name or BIDS structure (from bids.layout)
-  % query  - type of query: {'data', 'metadata', 'sessions', 'subjects',
-  %          'runs', 'tasks', 'runs', 'types', 'modalities'}
+  % query  - type of query:
+  %                          - 'data',
+  %                          - 'metadata',
+  %                          - 'sessions',
+  %                          - 'subjects',
+  %                          - 'runs',
+  %                          - 'tasks',
+  %                          - 'types',
+  %                          - 'modalities'
   % result - outcome of query
   %
-  % See also:
-  % bids
 
   % __________________________________________________________________________
   %
diff --git a/tests/test_bids_query.m b/tests/test_bids_query.m
index 31b4e283..48b55957 100644
--- a/tests/test_bids_query.m
+++ b/tests/test_bids_query.m
@@ -7,7 +7,7 @@
 end
 
 function test_bids_query_basic()
-  % Test BIDS queries on ds007
+  % Test BIDS queries
   % This dataset comes from https://github.com/bids-standard/bids-examples
   % and is downloaded automatically by the continuous integration framework
   % and is required for the tests to be run.
@@ -30,8 +30,6 @@ function test_bids_query_basic()
   subjs = arrayfun(@(x) sprintf('%02d', x), 1:20, 'UniformOutput', false);
   assert(isequal(bids.query(BIDS, 'subjects'), subjs));
 
-  assert(isempty(bids.query(BIDS, 'sessions')));
-
   assert(isequal(bids.query(BIDS, 'runs'), {'01', '02'}));
 
   tasks = { ...
@@ -46,10 +44,6 @@ function test_bids_query_basic()
   data = bids.query(BIDS, 'data', 'sub', '01', 'task', 'stopsignalwithpseudowordnaming');
   assertEqual(size(data, 1), 4);
 
-  mods = {'anat', 'func'};
-  assert(isequal(bids.query(BIDS, 'modalities'), mods));
-  assert(isequal(bids.query(BIDS, 'modalities', 'sub', '01'), mods));
-
   assert(isempty(bids.query(BIDS, 'runs', 'type', 'T1w')));
 
   runs = {'01', '02'};
@@ -79,31 +73,32 @@ function test_bids_query_basic()
 end
 
 function test_bids_query_sessions()
-  %
-  %   parse a folder with sessions
-  %
 
   pth_bids_example = get_test_data_dir();
 
   BIDS = bids.layout(fullfile(pth_bids_example, 'synthetic'));
-
-  %   test
   sessions = {'01', '02'};
   assert(isequal(bids.query(BIDS, 'sessions'), sessions));
   assert(isequal(bids.query(BIDS, 'sessions', 'sub', '02'), sessions));
 
+  BIDS = bids.layout(fullfile(pth_bids_example, 'ds007'));
+
+  assert(isempty(bids.query(BIDS, 'sessions')));
+
 end
 
 function test_bids_query_modalities()
-  %
-  %   parse a folder with different modalities per session
-  %
 
   pth_bids_example = get_test_data_dir();
 
+  BIDS = bids.layout(fullfile(pth_bids_example, 'ds007'));
+
+  mods = {'anat', 'func'};
+  assert(isequal(bids.query(BIDS, 'modalities'), mods));
+  assert(isequal(bids.query(BIDS, 'modalities', 'sub', '01'), mods));
+
   BIDS = bids.layout(fullfile(pth_bids_example, '7t_trt'));
 
-  %   test
   mods = {'anat', 'fmap', 'func'};
 
   assert(isequal(bids.query(BIDS, 'modalities'), mods));
diff --git a/tests/test_bids_query_asl.m b/tests/test_bids_query_asl.m
index a6951f17..624ce626 100644
--- a/tests/test_bids_query_asl.m
+++ b/tests/test_bids_query_asl.m
@@ -8,7 +8,7 @@
 
 function test_bids_query_asl_basic()
   %
-  %   asl specific queries
+  %   asl queries
   %
 
   pth_bids_example = get_test_data_dir();
diff --git a/tests/test_bids_query_eeg.m b/tests/test_bids_query_eeg.m
new file mode 100644
index 00000000..f407b992
--- /dev/null
+++ b/tests/test_bids_query_eeg.m
@@ -0,0 +1,64 @@
+function test_suite = test_bids_query_eeg %#ok<*STOUT>
+  try % assignment of 'localfunctions' is necessary in Matlab >= 2016
+    test_functions = localfunctions(); %#ok<*NASGU>
+  catch % no problem; early Matlab versions can use initTestSuite fine
+  end
+  initTestSuite;
+end
+
+function test_bids_query_eeg_basic()
+  %
+  %   eeg queries
+  %
+
+  pth_bids_example = get_test_data_dir();
+
+  %%
+  BIDS = bids.layout(fullfile(pth_bids_example, 'eeg_cbm'));
+
+  modalities = {'eeg'};
+  assertEqual(bids.query(BIDS, 'modalities'), modalities);
+
+  types = {'channels', 'eeg', 'events'};
+  assertEqual(bids.query(BIDS, 'types'), types);
+
+  %%
+  BIDS = bids.layout(fullfile(pth_bids_example, 'eeg_ds000117'));
+
+  modalities = {'anat', 'eeg'};
+  assertEqual(bids.query(BIDS, 'modalities'), modalities);
+
+  types = {'T1w', 'eeg', 'electrodes', 'events'};
+  % Missing: 'coordsystem', 'channels'
+  assertEqual(bids.query(BIDS, 'types'), types);
+
+  %%
+  BIDS = bids.layout(fullfile(pth_bids_example, 'eeg_face13'));
+
+  modalities = {'eeg'};
+  assertEqual(bids.query(BIDS, 'modalities'), modalities);
+
+  types = {'channels', 'eeg', 'events'};
+  % Missing: 'electrodes'
+  % skipped as it contains a task entity and thus does not match the schema
+  assertEqual(bids.query(BIDS, 'types'), types);
+
+  %%
+  BIDS = bids.layout(fullfile(pth_bids_example, 'eeg_matchingpennies'));
+
+  modalities = {'eeg'};
+  assertEqual(bids.query(BIDS, 'modalities'), modalities);
+
+  types = {'channels', 'eeg', 'events'};
+  assertEqual(bids.query(BIDS, 'types'), types);
+
+  %%
+  BIDS = bids.layout(fullfile(pth_bids_example, 'eeg_rishikesh'));
+
+  modalities = {'eeg'};
+  assertEqual(bids.query(BIDS, 'modalities'), modalities);
+
+  types = {'channels', 'eeg', 'events'};
+  assertEqual(bids.query(BIDS, 'types'), types);
+
+end
diff --git a/tests/test_bids_query_ieeg.m b/tests/test_bids_query_ieeg.m
new file mode 100644
index 00000000..d72099a4
--- /dev/null
+++ b/tests/test_bids_query_ieeg.m
@@ -0,0 +1,36 @@
+function test_suite = test_bids_query_ieeg %#ok<*STOUT>
+  try % assignment of 'localfunctions' is necessary in Matlab >= 2016
+    test_functions = localfunctions(); %#ok<*NASGU>
+  catch % no problem; early Matlab versions can use initTestSuite fine
+  end
+  initTestSuite;
+end
+
+function test_bids_query_ieeg_basic()
+  %
+  %   eeg queries
+  %
+
+  pth_bids_example = get_test_data_dir();
+
+  %%
+  BIDS = bids.layout(fullfile(pth_bids_example, 'ieeg_epilepsy'));
+
+  modalities = {'anat', 'ieeg'};
+  assertEqual(bids.query(BIDS, 'modalities'), modalities);
+
+  types = {'T1w', 'channels', 'electrodes', 'events', 'ieeg'};
+  % Missing: 'coordsystem'
+  assertEqual(bids.query(BIDS, 'types'), types);
+
+  %%
+  BIDS = bids.layout(fullfile(pth_bids_example, 'ieeg_epilepsy_ecog'));
+
+  modalities = {'anat', 'ieeg'};
+  assertEqual(bids.query(BIDS, 'modalities'), modalities);
+
+  types = {'T1w', 'channels', 'electrodes', 'events', 'ieeg', 'photo'};
+  % Missing: 'coordsystem'
+  assertEqual(bids.query(BIDS, 'types'), types);
+
+end
diff --git a/tests/test_bids_query_meg.m b/tests/test_bids_query_meg.m
new file mode 100644
index 00000000..ac7f7608
--- /dev/null
+++ b/tests/test_bids_query_meg.m
@@ -0,0 +1,28 @@
+function test_suite = test_bids_query_meg %#ok<*STOUT>
+  try % assignment of 'localfunctions' is necessary in Matlab >= 2016
+    test_functions = localfunctions(); %#ok<*NASGU>
+  catch % no problem; early Matlab versions can use initTestSuite fine
+  end
+  initTestSuite;
+end
+
+function test_bids_query_meg_basic()
+  %
+  %   meg queries
+  %
+
+  pth_bids_example = get_test_data_dir();
+
+  BIDS = bids.layout(fullfile(pth_bids_example, 'ds000246'));
+
+  modalities = {'anat', 'meg'};
+  assertEqual(bids.query(BIDS, 'modalities'), modalities);
+
+  types = {'T1w', 'channels', 'photo'};
+  % missing:  'meg' 'coordsystem', 'headshape'
+  assertEqual(bids.query(BIDS, 'types'), types);
+
+  BIDS = bids.layout(fullfile(pth_bids_example, 'ds000247'));
+  BIDS = bids.layout(fullfile(pth_bids_example, 'ds000248'));
+
+end
diff --git a/tests/test_layout.m b/tests/test_layout.m
index 619d9a56..30eb4c83 100644
--- a/tests/test_layout.m
+++ b/tests/test_layout.m
@@ -9,7 +9,9 @@
 function test_layout_smoke_test()
 
   pth_bids_example = get_test_data_dir();
-  BIDS = bids.layout(fullfile(pth_bids_example, '7t_trt'));
-  BIDS = bids.layout(fullfile(pth_bids_example, 'asl001'));
+
+  BIDS = bids.layout(fullfile(pth_bids_example, 'genetics_ukbb'));
+
+  BIDS = bids.layout(fullfile(pth_bids_example, 'ds210'));
 
 end

From 888ad556a21bdb4a9b4ea7ece41a563c9e0da4b3 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sun, 7 Feb 2021 13:08:33 +0100
Subject: [PATCH 30/54] add test dwi

---
 tests/test_bids_query_dwi.m | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 tests/test_bids_query_dwi.m

diff --git a/tests/test_bids_query_dwi.m b/tests/test_bids_query_dwi.m
new file mode 100644
index 00000000..435bbbf2
--- /dev/null
+++ b/tests/test_bids_query_dwi.m
@@ -0,0 +1,26 @@
+function test_suite = test_bids_query_dwi %#ok<*STOUT>
+  try % assignment of 'localfunctions' is necessary in Matlab >= 2016
+    test_functions = localfunctions(); %#ok<*NASGU>
+  catch % no problem; early Matlab versions can use initTestSuite fine
+  end
+  initTestSuite;
+end
+
+function test_bids_query_dwi_basic()
+  %
+  %   eeg queries
+  %
+
+  pth_bids_example = get_test_data_dir();
+
+  %%
+  BIDS = bids.layout(fullfile(pth_bids_example, 'ds000117'));
+
+  modalities = {'anat',    'beh',    'dwi',    'fmap',    'func',    'meg'};
+  assertEqual(bids.query(BIDS, 'modalities'), modalities);
+
+  types = {'T1w', 'bold', 'dwi', 'events', 'magnitude1', 'magnitude2', 'meg', 'phasediff'};
+  % Missing: 'headshappe', 'FLASH'
+  assertEqual(bids.query(BIDS, 'types'), types);
+
+end

From 236e3c218ed6f6d5c7d6f3909ad8efbd0410d7b7 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sun, 7 Feb 2021 13:09:20 +0100
Subject: [PATCH 31/54] make file listing most general for all modalities

---
 +bids/layout.m | 40 +++++++++++-----------------------------
 1 file changed, 11 insertions(+), 29 deletions(-)

diff --git a/+bids/layout.m b/+bids/layout.m
index ba68e773..60dd6acd 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -256,21 +256,18 @@ function tolerant_message(tolerant, msg)
     for i = 1:numel(file_list)
 
       subject = bids.internal.append_to_structure(file_list{i}, subject, datatype, schema);
-
-      % -bval file
+      
+      % bval & bvec file
       % ------------------------------------------------------------------
-      % bval file can also be stored at higher levels (inheritance principle)
+      % TODO: they can also be stored at higher levels (inheritance principle)
       bvalfile = bids.internal.get_metadata(file_list{i}, '^.*%s\\.bval$');
       if isfield(bvalfile, 'filename')
-        subject.dwi(end).bval = bids.util.tsvread(bvalfile.filename);
+          subject.dwi(end).bval = bids.util.tsvread(bvalfile.filename);
       end
-
-      % -bvec file
-      % ------------------------------------------------------------------
-      % bvec file can also be stored at higher levels (inheritance principle)
+      
       bvecfile = bids.internal.get_metadata(file_list{i}, '^.*%s\\.bvec$');
       if isfield(bvalfile, 'filename')
-        subject.dwi(end).bvec = bids.util.tsvread(bvecfile.filename);
+          subject.dwi(end).bvec = bids.util.tsvread(bvecfile.filename);
       end
 
     end
@@ -731,29 +728,14 @@ function tolerant_message(tolerant, msg)
 
 function file_list = return_file_list(modality, subject)
 
-  switch modality
-
+    % We list anything but json files
+    
     % TODO
     % it should be possible to create some of those patterns for the regexp
     % based on some of the required entities written down in the schema
-
-    case {'anat', 'dwi', 'fmap', 'pet'}
-      pattern = '_([a-zA-Z0-9]+){1}\\.nii(\\.gz)?';
-
-    case 'func'
-      pattern = '_task-.*\\.nii(\\.gz)|events\\.tsv|physio\\.tsv\\.gz|stim\\.tsv\\.gz?';
-
-    case {'eeg', 'meg', 'ieeg'}
-      pattern = '_([a-zA-Z0-9]+){1}\\..*[^json]';
-
-    case 'beh'
-      pattern = '_task-.*_(events\\.tsv|beh\\.json|physio\\.tsv\\.gz|stim\\.tsv\\.gz)';
-
-    case 'perf'
-      pattern = '_(asl|m0scan)\\.nii(\\.gz)|aslcontext\\.tsv|asllabeling\\.jpg';
-
-  end
-
+    
+  pattern = '_([a-zA-Z0-9]+){1}\\..*[^json]';
+    
   pth = fullfile(subject.path, modality);
 
   [file_list, d] = bids.internal.file_utils('List', ...

From 641c8891e4e79cdc9ff45680fd385a3b3602f647 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sun, 7 Feb 2021 13:29:35 +0100
Subject: [PATCH 32/54] make sure .pos files and meg.ds folders are parsed

---
 +bids/layout.m              | 27 +++++++++++++++------------
 tests/test_bids_query_eeg.m |  3 ++-
 tests/test_bids_query_meg.m |  4 ++--
 3 files changed, 19 insertions(+), 15 deletions(-)

diff --git a/+bids/layout.m b/+bids/layout.m
index 60dd6acd..aee946eb 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -728,14 +728,15 @@ function tolerant_message(tolerant, msg)
 
 function file_list = return_file_list(modality, subject)
 
-    % We list anything but json files
-    
-    % TODO
-    % it should be possible to create some of those patterns for the regexp
-    % based on some of the required entities written down in the schema
-    
-  pattern = '_([a-zA-Z0-9]+){1}\\..*[^json]';
-    
+  % We list anything but json files
+
+  % TODO
+  % it should be possible to create some of those patterns for the regexp
+  % based on some of the required entities written down in the schema
+
+  % jn to omit json but not .pos file for headshape.pos
+  pattern = '_([a-zA-Z0-9]+){1}\\..*[^jn]';
+
   pth = fullfile(subject.path, modality);
 
   [file_list, d] = bids.internal.file_utils('List', ...
@@ -743,10 +744,12 @@ function tolerant_message(tolerant, msg)
                                             sprintf(['^%s.*' pattern '$'], ...
                                                     subject.name));
 
-  if strcmp(modality, 'meg') && isempty(file_list)
-    file_list = d;
-  end
-
   file_list = convert_to_cell(file_list);
 
+  if strcmp(modality, 'meg') && ~isempty(d)
+    for i = 1:size(d, 1)
+      file_list{end + 1, 1} = d(i, :);
+    end
+  end
+
 end
diff --git a/tests/test_bids_query_eeg.m b/tests/test_bids_query_eeg.m
index f407b992..9e989001 100644
--- a/tests/test_bids_query_eeg.m
+++ b/tests/test_bids_query_eeg.m
@@ -29,7 +29,8 @@ function test_bids_query_eeg_basic()
   assertEqual(bids.query(BIDS, 'modalities'), modalities);
 
   types = {'T1w', 'eeg', 'electrodes', 'events'};
-  % Missing: 'coordsystem', 'channels'
+  % Missing: 'coordsystem',
+  % Missing: 'channels' in root folder
   assertEqual(bids.query(BIDS, 'types'), types);
 
   %%
diff --git a/tests/test_bids_query_meg.m b/tests/test_bids_query_meg.m
index ac7f7608..8e51674b 100644
--- a/tests/test_bids_query_meg.m
+++ b/tests/test_bids_query_meg.m
@@ -18,8 +18,8 @@ function test_bids_query_meg_basic()
   modalities = {'anat', 'meg'};
   assertEqual(bids.query(BIDS, 'modalities'), modalities);
 
-  types = {'T1w', 'channels', 'photo'};
-  % missing:  'meg' 'coordsystem', 'headshape'
+  types = {'T1w', 'channels', 'headshape', 'meg', 'photo'};
+  % missing: 'coordsystem'
   assertEqual(bids.query(BIDS, 'types'), types);
 
   BIDS = bids.layout(fullfile(pth_bids_example, 'ds000247'));

From 6008e33de06630b25be706f2dddb1f4f76094794 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sun, 7 Feb 2021 14:16:32 +0100
Subject: [PATCH 33/54] refactor and linting

simplify perf parsing
---
 +bids/layout.m | 100 ++++++++++++++++++-------------------------------
 1 file changed, 36 insertions(+), 64 deletions(-)

diff --git a/+bids/layout.m b/+bids/layout.m
index aee946eb..aaa7633f 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -112,7 +112,8 @@
   p = bids.internal.file_utils('FPList', BIDS.dir, '^participants\.tsv$');
   if ~isempty(p)
     try
-      BIDS.participants = bids.util.tsvread(p);
+      BIDS.participants = struct('content', [], 'meta', []);
+      BIDS.participants.content = bids.util.tsvread(p);
     catch
       msg = ['unable to read ' p];
       tolerant_message(tolerant, msg);
@@ -256,18 +257,18 @@ function tolerant_message(tolerant, msg)
     for i = 1:numel(file_list)
 
       subject = bids.internal.append_to_structure(file_list{i}, subject, datatype, schema);
-      
+
       % bval & bvec file
       % ------------------------------------------------------------------
       % TODO: they can also be stored at higher levels (inheritance principle)
       bvalfile = bids.internal.get_metadata(file_list{i}, '^.*%s\\.bval$');
       if isfield(bvalfile, 'filename')
-          subject.dwi(end).bval = bids.util.tsvread(bvalfile.filename);
+        subject.dwi(end).bval = bids.util.tsvread(bvalfile.filename);
       end
-      
+
       bvecfile = bids.internal.get_metadata(file_list{i}, '^.*%s\\.bvec$');
       if isfield(bvalfile, 'filename')
-          subject.dwi(end).bvec = bids.util.tsvread(bvecfile.filename);
+        subject.dwi(end).bvec = bids.util.tsvread(bvecfile.filename);
       end
 
     end
@@ -290,7 +291,6 @@ function tolerant_message(tolerant, msg)
     for i = 1:numel(file_list)
 
       subject = bids.internal.append_to_structure(file_list{i}, subject, datatype, schema);
-      subject.func(end).meta = struct([]); % ?
 
       % TODO:
       %
@@ -299,7 +299,7 @@ function tolerant_message(tolerant, msg)
       %
 
       if strcmp(subject.func(end).type, 'events')
-        subject.func(end).meta = bids.util.tsvread(fullfile(pth, file_list{i}));
+        subject.func(i).content = bids.util.tsvread(fullfile(pth, file_list{i}));
       end
 
     end
@@ -324,68 +324,42 @@ function tolerant_message(tolerant, msg)
 
       subject = bids.internal.append_to_structure(file_list{i}, subject, datatype, schema);
 
-    end
+      switch subject.perf(i).type
 
-    % ASL timeseries NIfTI file
-    % ----------------------------------------------------------------------
-    labels = regexp(file_list, [ ...
-                                '^sub-[a-zA-Z0-9]+' ...              % sub-<participant_label>
-                                '_asl\.nii(\.gz)?$'], 'names'); % NIfTI file suffix/extension
+        case 'asl'
 
-    if any(~cellfun(@isempty, labels))
+          subject.perf(i).meta = [];
+          subject.perf(i).dependencies = [];
 
-      idx = find(~cellfun(@isempty, labels));
+          subject.perf(i).meta = bids.internal.get_metadata( ...
+                                                            fullfile( ...
+                                                                     subject.path, ...
+                                                                     datatype, ...
+                                                                     file_list{i}));
 
-      for i = 1:numel(idx)
+          subject.perf(i) = manage_aslcontext(subject.perf(i), pth);
 
-        j = idx(i);
+          subject.perf(i) = manage_asllabeling(subject.perf(i), pth);
 
-        subject.perf(j).meta = [];
-        subject.perf(j).dependencies = [];
+          subject.perf(i) = manage_M0(subject.perf(i), pth);
 
-        subject.perf(j).meta = bids.internal.get_metadata( ...
-                                                          fullfile( ...
-                                                                   subject.path, ...
-                                                                   datatype, ...
-                                                                   file_list{j}));
+        case 'm0scan'
 
-        subject.perf(j) = manage_aslcontext(subject.perf(j), pth);
+          subject.perf(i).intended_for = [];
 
-        subject.perf(j) = manage_asllabeling(subject.perf(j), pth);
+          subject.perf(i).meta = bids.internal.get_metadata( ...
+                                                            fullfile( ...
+                                                                     subject.path, ...
+                                                                     datatype, ...
+                                                                     file_list{i}));
 
-        subject.perf(j) = manage_M0(subject.perf(j), pth);
+          subject.perf(i) = manage_intended_for(subject.perf(i), subject, pth);
 
       end
 
     end
 
-    % -M0scan NIfTI file
-    % ---------------------------------------------------------------------
-    labels = regexp(file_list, [ ...
-                                '^sub-[a-zA-Z0-9]+' ...              % sub-<participant_label>
-                                '_m0scan\.nii(\.gz)?$'], 'names'); % NIfTI file suffix/extension
-
-    if any(~cellfun(@isempty, labels))
-      idx = find(~cellfun(@isempty, labels));
-      for i = 1:numel(idx)
-
-        j = idx(i);
-
-        subject.perf(j).intended_for = [];
-
-        subject.perf(j).meta = bids.internal.get_metadata( ...
-                                                          fullfile( ...
-                                                                   subject.path, ...
-                                                                   datatype, ...
-                                                                   file_list{j}));
-
-        subject.perf(j) = manage_intended_for(subject.perf(j), subject, pth);
-
-      end
-
-    end
-
-  end % if exist(pth, 'dir')
+  end
 
 end
 
@@ -399,7 +373,7 @@ function tolerant_message(tolerant, msg)
 
   if exist(metafile, 'file')
     [~, Ffile] = fileparts(metafile);
-    perf.dependencies.context.sidecar = [Ffile '.tsv'];
+    perf.dependencies.context.filename = [Ffile '.tsv'];
     perf.dependencies.context.content = bids.util.tsvread(metafile);
 
   else
@@ -418,7 +392,7 @@ function tolerant_message(tolerant, msg)
 
   if exist(metafile, 'file')
     [~, Ffile] = fileparts(metafile);
-    perf.dependencies.labeling_image = [Ffile '.jpg'];
+    perf.dependencies.labeling_image.filename = [Ffile '.jpg'];
 
   end
 
@@ -512,10 +486,8 @@ function tolerant_message(tolerant, msg)
         end
 
       otherwise
-        warning(['Unknown M0Type:', ...
-                 perf.meta.M0Type, ...
-                 ' in ', ...
-                 perf.json_sidecar_filename]);
+        warning(['Unknown M0Type:', perf.meta.M0Type]);
+
     end
 
     if ~isempty(m0_type)
@@ -619,7 +591,7 @@ function tolerant_message(tolerant, msg)
                                                           '_fieldmap.nii', ...
                                                           '_magnitude.nii');
 
-          % -Phase difference image and at least one magnitude image
+          % Phase difference image and at least one magnitude image
         case {'phasediff'}
           subject.fmap(i).dependencies.magnitude = { ...
                                                     strrep(file_list{i}, ...
@@ -629,7 +601,7 @@ function tolerant_message(tolerant, msg)
                                                            '_phasediff.nii', ...
                                                            '_magnitude2.nii')}; % optional
 
-          % -Two phase images and two magnitude images
+          % Two phase images and two magnitude images
         case {'phase1', 'phase2'}
           subject.fmap(i).dependencies.magnitude = { ...
                                                     strrep(file_list{i}, ...
@@ -659,13 +631,13 @@ function tolerant_message(tolerant, msg)
 
       subject = bids.internal.append_to_structure(file_list{i}, subject, datatype, schema);
 
-      switch subject.(datatype)(end).type
+      switch subject.(datatype)(i).type
 
         case {'events', 'channels', 'electrodes'}  %
           % TODO: events / channels file can also be stored
           % at higher levels (inheritance principle)
           %
-          subject.(datatype)(end).meta = bids.util.tsvread(fullfile(pth, file_list{i}));
+          subject.(datatype)(i).content = bids.util.tsvread(fullfile(pth, file_list{i}));
 
         case {'photo', 'coordsystem'}
 

From 910d193f8b144908c1cfe6992ae7723f25230259 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sun, 7 Feb 2021 15:33:37 +0100
Subject: [PATCH 34/54] refactor layout to manage most of the parsing through
 the same subfunction

improve parsing of tsv files
---
 +bids/layout.m              | 544 ++++++++++++++++--------------------
 tests/test_bids_query_dwi.m |   5 +-
 2 files changed, 239 insertions(+), 310 deletions(-)

diff --git a/+bids/layout.m b/+bids/layout.m
index aaa7633f..b1bf0dc4 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -42,8 +42,6 @@
   % BIDS.dir          -- BIDS directory
   % BIDS.description  -- content of dataset_description.json
   % BIDS.sessions     -- cellstr of sessions
-  % BIDS.scans        -- for sub-<participant_label>_scans.tsv (should go within subjects)
-  % BIDS.sess         -- for sub-<participants_label>_sessions.tsv (should go within subjects)
   % BIDS.participants -- for participants.tsv
   % BIDS.subjects'    -- structure array of subjects
 
@@ -51,8 +49,6 @@
                 'dir', root, ...
                 'description', struct([]), ...
                 'sessions', {{}}, ...
-                'scans', struct([]), ...
-                'sess', struct([]), ...
                 'participants', struct([]), ...
                 'subjects', struct([]));
 
@@ -99,41 +95,8 @@
   % [sourcedata/]
   % [phenotype/]
 
-  % -Scans key file
-  % ==========================================================================
-
-  % sub-<participant_label>/[ses-<session_label>/]
-  %     sub-<participant_label>_scans.tsv
-
-  % See also optional README and CHANGES files
-
-  % -Participant key file
-  % ==========================================================================
-  p = bids.internal.file_utils('FPList', BIDS.dir, '^participants\.tsv$');
-  if ~isempty(p)
-    try
-      BIDS.participants = struct('content', [], 'meta', []);
-      BIDS.participants.content = bids.util.tsvread(p);
-    catch
-      msg = ['unable to read ' p];
-      tolerant_message(tolerant, msg);
-    end
-  end
-  p = bids.internal.file_utils('FPList', BIDS.dir, '^participants\.json$');
-  if ~isempty(p)
-    BIDS.participants.meta = bids.util.jsondecode(p);
-  end
-
-  % -Sessions file
-  % ==========================================================================
-
-  % sub-<participant_label>/[ses-<session_label>/]
-  %      sub-<participant_label>[_ses-<session_label>]_sessions.tsv
-
-  % -Tasks: JSON files are accessed through metadata
-  % ==========================================================================
-  % t = bids.internal.file_utils('FPList',BIDS.dir,...
-  %    '^task-.*_(beh|bold|events|channels|physio|stim|meg)\.(json|tsv)$');
+  BIDS.participants = [];
+  BIDS.participants = manage_tsv(BIDS.participants, BIDS.dir, 'participants.tsv');
 
   % -Subjects
   % ==========================================================================
@@ -160,14 +123,6 @@
 
 end
 
-function tolerant_message(tolerant, msg)
-  if tolerant
-    warning(msg);
-  else
-    error(msg);
-  end
-end
-
 % ==========================================================================
 % -Parse a subject's directory
 % ==========================================================================
@@ -179,7 +134,9 @@ function tolerant_message(tolerant, msg)
 
   subject.name    = subjname;   % subject name ('sub-<participant_label>')
   subject.path    = fullfile(pth, subjname, sesname); % full path to subject directory
-  subject.session = sesname; % session name ('' or 'ses-<label>')
+  subject.session = sesname;    % session name ('' or 'ses-<label>')
+  subject.scans   = struct([]); % for sub-<participant_label>_scans.tsv
+  subject.sess    = struct([]); % for sub-<participants_label>_sessions.tsv
   subject.anat    = struct([]); % anatomy imaging data
   subject.func    = struct([]); % task imaging data
   subject.fmap    = struct([]); % fieldmap data
@@ -201,16 +158,16 @@ function tolerant_message(tolerant, msg)
 
     for iDatatype = 1:numel(datatypes)
       switch datatypes{iDatatype}
-        case {'anat', 'beh', 'ieeg'}
+        case {'anat', 'func', 'beh', 'meg', 'eeg', 'ieeg'}
           subject = parse_using_schema(subject, datatypes{iDatatype}, schema);
         case 'dwi'
           subject = parse_dwi(subject, schema);
-        case {'eeg', 'meg'}
-          subject = parse_meeg(subject, datatypes{iDatatype}, schema);
+          %         case {'eeg', 'meg'}
+          %           subject = parse_meeg(subject, datatypes{iDatatype}, schema);
         case 'fmap'
           subject = parse_fmap(subject, schema);
-        case 'func'
-          subject = parse_func(subject, schema);
+          %         case 'func'
+          %           subject = parse_func(subject, schema);
         case 'perf'
           subject = parse_perf(subject, schema);
       end
@@ -235,6 +192,26 @@ function tolerant_message(tolerant, msg)
 
       subject = bids.internal.append_to_structure(file_list{i}, subject, datatype, schema);
 
+      if ~isempty(subject.(datatype)) && strcmp(subject.(datatype)(end).ext, '.tsv')
+        % events
+        % stim
+        % channels
+        % electrodes
+        %
+        % does not cover physio.tsv.gz or stim.tsv.gz
+
+        subject.(datatype)(end).content = [];
+        subject.(datatype)(end).meta = [];
+
+        subject.(datatype)(end) = manage_tsv( ...
+                                             subject.(datatype)(end), ...
+                                             pth, ...
+                                             subject.(datatype)(end).filename);
+
+      end
+
+      % case {'photo', 'coordsystem'}
+
     end
 
   end
@@ -275,38 +252,6 @@ function tolerant_message(tolerant, msg)
   end
 end
 
-function subject = parse_func(subject, schema)
-
-  % --------------------------------------------------------------------------
-  %  Task imaging data
-  % --------------------------------------------------------------------------
-
-  datatype = 'func';
-  pth = fullfile(subject.path, datatype);
-
-  if exist(pth, 'dir')
-
-    file_list = return_file_list(datatype, subject);
-
-    for i = 1:numel(file_list)
-
-      subject = bids.internal.append_to_structure(file_list{i}, subject, datatype, schema);
-
-      % TODO:
-      %
-      % Events, physiological and other continuous recordings file
-      % can also be stored at higher levels (inheritance principle).
-      %
-
-      if strcmp(subject.func(end).type, 'events')
-        subject.func(i).content = bids.util.tsvread(fullfile(pth, file_list{i}));
-      end
-
-    end
-
-  end
-end
-
 function subject = parse_perf(subject, schema)
 
   % --------------------------------------------------------------------------
@@ -337,7 +282,13 @@ function tolerant_message(tolerant, msg)
                                                                      datatype, ...
                                                                      file_list{i}));
 
-          subject.perf(i) = manage_aslcontext(subject.perf(i), pth);
+          aslcontext_file = strrep(subject.perf(i).filename, ...
+                                   ['_asl' subject.perf(i).ext], ...
+                                   '_aslcontext.tsv');
+          subject.perf(i).dependencies.context = manage_tsv( ...
+                                                            struct('content', [], 'meta', []), ...
+                                                            pth, ...
+                                                            aslcontext_file);
 
           subject.perf(i) = manage_asllabeling(subject.perf(i), pth);
 
@@ -363,21 +314,209 @@ function tolerant_message(tolerant, msg)
 
 end
 
-function perf = manage_aslcontext(perf, pth)
+function subject = parse_fmap(subject, schema)
 
-  % ASLCONTEXT-sidecar metadata (REQUIRED)
-  % ---------------------------
-  metafile = fullfile(pth, strrep(perf.filename, ...
-                                  ['_asl' perf.ext], ...
-                                  '_aslcontext.tsv'));
+  datatype = 'fmap';
+  pth = fullfile(subject.path, datatype);
 
-  if exist(metafile, 'file')
-    [~, Ffile] = fileparts(metafile);
-    perf.dependencies.context.filename = [Ffile '.tsv'];
-    perf.dependencies.context.content = bids.util.tsvread(metafile);
+  if exist(pth, 'dir')
+
+    file_list = return_file_list(datatype, subject);
+
+    for i = 1:numel(file_list)
+
+      subject = bids.internal.append_to_structure(file_list{i}, subject, datatype, schema);
+
+      subject.fmap(i).meta = bids.internal.get_metadata( ...
+                                                        fullfile( ...
+                                                                 subject.path, ...
+                                                                 datatype, ...
+                                                                 file_list{i}));
+      %       subject.perf(i).intended_for = [];
+      %       subject.fmap(i) = manage_intended_for(subject.fmap(i), subject, pth);
+
+      switch subject.fmap(i).type
+
+        % -A single, real fieldmap image
+        case {'fieldmap', 'magnitude'}
+          subject.fmap(i).dependencies.magnitude = strrep(file_list{idx(i)}, ...
+                                                          '_fieldmap.nii', ...
+                                                          '_magnitude.nii');
+
+          % Phase difference image and at least one magnitude image
+        case {'phasediff'}
+          subject.fmap(i).dependencies.magnitude = { ...
+                                                    strrep(file_list{i}, ...
+                                                           '_phasediff.nii', ...
+                                                           '_magnitude1.nii'), ...
+                                                    strrep(file_list{i}, ...
+                                                           '_phasediff.nii', ...
+                                                           '_magnitude2.nii')}; % optional
+
+          % Two phase images and two magnitude images
+        case {'phase1', 'phase2'}
+          subject.fmap(i).dependencies.magnitude = { ...
+                                                    strrep(file_list{i}, ...
+                                                           '_phase1.nii', ...
+                                                           '_magnitude1.nii'), ...
+                                                    strrep(file_list{i}, ...
+                                                           '_phase1.nii', ...
+                                                           '_magnitude2.nii')};
+
+      end
+
+    end
+
+  end
+
+end
+
+function subject = parse_pet(subject)
+  % --------------------------------------------------------------------------
+  % -Positron Emission Tomography imaging data
+  % --------------------------------------------------------------------------
+  pth = fullfile(subject.path, 'pet');
+
+  if exist(pth, 'dir')
+
+    entities = return_entities('pet');
+
+    file_list = return_file_list('pet', subject);
+
+    for i = 1:numel(file_list)
+
+      subject = append_to_structure(file_list{i}, entities, subject, 'pet');
+
+    end
+  end
+end
+
+% --------------------------------------------------------------------------
+%                            HELPER FUNCTIONS
+% --------------------------------------------------------------------------
+
+function tolerant_message(tolerant, msg)
+  if tolerant
+    warning(msg);
+  else
+    error(msg);
+  end
+end
+
+function subject = append_to_structure(file, entities, subject, modality)
+
+  p = bids.internal.parse_filename(file, entities);
+  subject.(modality) = [subject.(modality) p];
+
+end
+
+function f = convert_to_cell(f)
+  if isempty(f)
+    f = {};
+  else
+    f = cellstr(f);
+  end
+end
+
+function entities = return_entities(modality)
+
+  switch modality
+
+    case 'pet'
+      entities = {'sub', 'ses', 'task', 'acq', 'rec', 'run'};
+
+  end
+end
+
+function file_list = return_file_list(modality, subject)
+
+  % We list anything but json files
+
+  % TODO
+  % it should be possible to create some of those patterns for the regexp
+  % based on some of the required entities written down in the schema
+
+  % jn to omit json but not .pos file for headshape.pos
+  pattern = '_([a-zA-Z0-9]+){1}\\..*[^jn]';
+
+  pth = fullfile(subject.path, modality);
+
+  [file_list, d] = bids.internal.file_utils('List', ...
+                                            pth, ...
+                                            sprintf(['^%s.*' pattern '$'], ...
+                                                    subject.name));
+
+  file_list = convert_to_cell(file_list);
+
+  if strcmp(modality, 'meg') && ~isempty(d)
+    for i = 1:size(d, 1)
+      file_list{end + 1, 1} = d(i, :);
+    end
+  end
+
+end
+
+function structure = manage_tsv(structure, pth, filename)
+
+  p = bids.internal.file_utils('FPList', pth,  ['^' strrep(filename, '.tsv', '\.tsv') '$']);
+
+  if isempty(p)
+    warning('Missing: %s', fullfile(pth, filename));
 
   else
-    warning(['Missing: ' metafile]);
+    structure.content = bids.util.tsvread(p);
+
+    p = bids.internal.file_utils('FPList', pth,  ['^' strrep(filename, '.tsv', '\.json') '$']);
+    if ~isempty(p)
+      structure.meta = bids.util.jsondecode(p);
+    end
+
+  end
+
+end
+
+function structure = manage_intended_for(structure, subject, pth)
+
+  if isempty(structure.meta)
+    return
+
+  else
+
+    % Get all NIfTIs that this m0scan is intended for
+    path_intended_for = {};
+    if ~isfield(structure.meta, 'IntendedFor')
+      warning('Missing field IntendedFor for %s', structure.filename);
+
+    elseif ischar(structure.meta.IntendedFor)
+      path_intended_for{1} = structure.meta.IntendedFor;
+
+    elseif isstruct(structure.meta.IntendedFor)
+      for iPath = 1:length(structure.meta.IntendedFor)
+        path_intended_for{iPath} = structure.meta.IntendedFor(iPath); %#ok<*AGROW>
+      end
+
+    end
+
+    for iPath = 1:length(path_intended_for)
+      % check if this NIfTI is not missing
+      if ~exist(fullfile(fileparts(pth), path_intended_for{iPath}), 'file')
+        warning(['Missing: ' fullfile(fileparts(pth), path_intended_for{iPath})]);
+
+      else
+        % also check that this NIfTI aims to the same m0scan
+        [~, path2check, ext2check] = fileparts(path_intended_for{iPath});
+        filename_found = max(arrayfun(@(x) strcmp(x.filename, ...
+                                                  [path2check ext2check]), ...
+                                      subject.perf));
+        if ~filename_found
+          warning(['Did not find NIfTI for which is intended: ' structure.filename]);
+
+        else
+          structure.intended_for = path_intended_for{iPath};
+
+        end
+      end
+    end
 
   end
 
@@ -514,214 +653,3 @@ function tolerant_message(tolerant, msg)
   end
 
 end
-
-function structure = manage_intended_for(structure, subject, pth)
-
-  if isempty(structure.meta)
-    return
-
-  else
-
-    % Get all NIfTIs that this m0scan is intended for
-    path_intended_for = {};
-    if ~isfield(structure.meta, 'IntendedFor')
-      warning('Missing field IntendedFor for %s', structure.filename);
-
-    elseif ischar(structure.meta.IntendedFor)
-      path_intended_for{1} = structure.meta.IntendedFor;
-
-    elseif isstruct(structure.meta.IntendedFor)
-      for iPath = 1:length(structure.meta.IntendedFor)
-        path_intended_for{iPath} = structure.meta.IntendedFor(iPath); %#ok<*AGROW>
-      end
-
-    end
-
-    for iPath = 1:length(path_intended_for)
-      % check if this NIfTI is not missing
-      if ~exist(fullfile(fileparts(pth), path_intended_for{iPath}), 'file')
-        warning(['Missing: ' fullfile(fileparts(pth), path_intended_for{iPath})]);
-
-      else
-        % also check that this NIfTI aims to the same m0scan
-        [~, path2check, ext2check] = fileparts(path_intended_for{iPath});
-        filename_found = max(arrayfun(@(x) strcmp(x.filename, ...
-                                                  [path2check ext2check]), ...
-                                      subject.perf));
-        if ~filename_found
-          warning(['Did not find NIfTI for which is intended: ' structure.filename]);
-
-        else
-          structure.intended_for = path_intended_for{iPath};
-
-        end
-      end
-    end
-
-  end
-
-end
-
-function subject = parse_fmap(subject, schema)
-
-  datatype = 'fmap';
-  pth = fullfile(subject.path, datatype);
-
-  if exist(pth, 'dir')
-
-    file_list = return_file_list(datatype, subject);
-
-    for i = 1:numel(file_list)
-
-      subject = bids.internal.append_to_structure(file_list{i}, subject, datatype, schema);
-
-      subject.fmap(i).meta = bids.internal.get_metadata( ...
-                                                        fullfile( ...
-                                                                 subject.path, ...
-                                                                 datatype, ...
-                                                                 file_list{i}));
-      %       subject.perf(i).intended_for = [];
-      %       subject.fmap(i) = manage_intended_for(subject.fmap(i), subject, pth);
-
-      switch subject.fmap(i).type
-
-        % -A single, real fieldmap image
-        case {'fieldmap', 'magnitude'}
-          subject.fmap(i).dependencies.magnitude = strrep(file_list{idx(i)}, ...
-                                                          '_fieldmap.nii', ...
-                                                          '_magnitude.nii');
-
-          % Phase difference image and at least one magnitude image
-        case {'phasediff'}
-          subject.fmap(i).dependencies.magnitude = { ...
-                                                    strrep(file_list{i}, ...
-                                                           '_phasediff.nii', ...
-                                                           '_magnitude1.nii'), ...
-                                                    strrep(file_list{i}, ...
-                                                           '_phasediff.nii', ...
-                                                           '_magnitude2.nii')}; % optional
-
-          % Two phase images and two magnitude images
-        case {'phase1', 'phase2'}
-          subject.fmap(i).dependencies.magnitude = { ...
-                                                    strrep(file_list{i}, ...
-                                                           '_phase1.nii', ...
-                                                           '_magnitude1.nii'), ...
-                                                    strrep(file_list{i}, ...
-                                                           '_phase1.nii', ...
-                                                           '_magnitude2.nii')};
-
-      end
-
-    end
-
-  end
-
-end
-
-function subject = parse_meeg(subject, datatype, schema)
-
-  pth = fullfile(subject.path, datatype);
-
-  if exist(pth, 'dir')
-
-    file_list = return_file_list(datatype, subject);
-
-    for i = 1:numel(file_list)
-
-      subject = bids.internal.append_to_structure(file_list{i}, subject, datatype, schema);
-
-      switch subject.(datatype)(i).type
-
-        case {'events', 'channels', 'electrodes'}  %
-          % TODO: events / channels file can also be stored
-          % at higher levels (inheritance principle)
-          %
-          subject.(datatype)(i).content = bids.util.tsvread(fullfile(pth, file_list{i}));
-
-        case {'photo', 'coordsystem'}
-
-      end
-
-    end
-
-  end
-
-end
-
-function subject = parse_pet(subject)
-  % --------------------------------------------------------------------------
-  % -Positron Emission Tomography imaging data
-  % --------------------------------------------------------------------------
-  pth = fullfile(subject.path, 'pet');
-
-  if exist(pth, 'dir')
-
-    entities = return_entities('pet');
-
-    file_list = return_file_list('pet', subject);
-
-    for i = 1:numel(file_list)
-
-      subject = append_to_structure(file_list{i}, entities, subject, 'pet');
-
-    end
-  end
-end
-
-% --------------------------------------------------------------------------
-%                            HELPER FUNCTIONS
-% --------------------------------------------------------------------------
-
-function subject = append_to_structure(file, entities, subject, modality)
-
-  p = bids.internal.parse_filename(file, entities);
-  subject.(modality) = [subject.(modality) p];
-
-end
-
-function f = convert_to_cell(f)
-  if isempty(f)
-    f = {};
-  else
-    f = cellstr(f);
-  end
-end
-
-function entities = return_entities(modality)
-
-  switch modality
-
-    case 'pet'
-      entities = {'sub', 'ses', 'task', 'acq', 'rec', 'run'};
-
-  end
-end
-
-function file_list = return_file_list(modality, subject)
-
-  % We list anything but json files
-
-  % TODO
-  % it should be possible to create some of those patterns for the regexp
-  % based on some of the required entities written down in the schema
-
-  % jn to omit json but not .pos file for headshape.pos
-  pattern = '_([a-zA-Z0-9]+){1}\\..*[^jn]';
-
-  pth = fullfile(subject.path, modality);
-
-  [file_list, d] = bids.internal.file_utils('List', ...
-                                            pth, ...
-                                            sprintf(['^%s.*' pattern '$'], ...
-                                                    subject.name));
-
-  file_list = convert_to_cell(file_list);
-
-  if strcmp(modality, 'meg') && ~isempty(d)
-    for i = 1:size(d, 1)
-      file_list{end + 1, 1} = d(i, :);
-    end
-  end
-
-end
diff --git a/tests/test_bids_query_dwi.m b/tests/test_bids_query_dwi.m
index 435bbbf2..26b530cf 100644
--- a/tests/test_bids_query_dwi.m
+++ b/tests/test_bids_query_dwi.m
@@ -19,8 +19,9 @@ function test_bids_query_dwi_basic()
   modalities = {'anat',    'beh',    'dwi',    'fmap',    'func',    'meg'};
   assertEqual(bids.query(BIDS, 'modalities'), modalities);
 
-  types = {'T1w', 'bold', 'dwi', 'events', 'magnitude1', 'magnitude2', 'meg', 'phasediff'};
-  % Missing: 'headshappe', 'FLASH'
+  types = {'T1w', 'bold', 'dwi', 'events', 'headshape', ...
+           'magnitude1', 'magnitude2', 'meg', 'phasediff'};
+  % Missing: 'FLASH'
   assertEqual(bids.query(BIDS, 'types'), types);
 
 end

From 3208ebbfc574de860d85adeec8b208d0bd2debef Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sun, 7 Feb 2021 16:41:09 +0100
Subject: [PATCH 35/54] rename function and variables

- modality is used consistently instead of datatype
- suffix is used instead of type
---
 +bids/+internal/append_to_structure.m         |  4 +-
 +bids/+internal/get_metadata.m                |  2 +-
 +bids/+internal/parse_filename.m              |  6 +-
 +bids/+internal/return_datatype_extensions.m  | 14 ----
 .../return_datatype_regular_expression.m      |  8 ---
 +bids/+internal/return_datatype_suffixes.m    | 14 ----
 +bids/+internal/return_modality_extensions.m  | 14 ++++
 .../return_modality_regular_expression.m      |  8 +++
 +bids/+internal/return_modality_suffixes.m    | 12 ++++
 ..._entities.m => return_modality_entities.m} |  0
 +bids/layout.m                                | 68 +++++++++----------
 +bids/query.m                                 | 39 ++++++-----
 tests/test_append_to_structure.m              |  6 +-
 tests/test_bids_query.m                       | 28 ++++----
 tests/test_bids_query_asl.m                   | 19 +++---
 tests/test_bids_query_dwi.m                   |  6 +-
 tests/test_bids_query_eeg.m                   | 20 +++---
 tests/test_bids_query_ieeg.m                  |  8 +--
 tests/test_bids_query_meg.m                   |  4 +-
 tests/test_get_metadata.m                     |  8 +--
 tests/test_parse_filename.m                   |  6 +-
 ...ties.m => test_return_modality_entities.m} |  6 +-
 ...ns.m => test_return_modality_extensions.m} |  6 +-
 ...test_return_modality_regular_expression.m} |  6 +-
 ...ixes.m => test_return_modality_suffixes.m} |  6 +-
 25 files changed, 160 insertions(+), 158 deletions(-)
 delete mode 100644 +bids/+internal/return_datatype_extensions.m
 delete mode 100644 +bids/+internal/return_datatype_regular_expression.m
 delete mode 100644 +bids/+internal/return_datatype_suffixes.m
 create mode 100644 +bids/+internal/return_modality_extensions.m
 create mode 100644 +bids/+internal/return_modality_regular_expression.m
 create mode 100644 +bids/+internal/return_modality_suffixes.m
 rename +bids/+schema/{return_datatype_entities.m => return_modality_entities.m} (100%)
 rename tests/{test_return_datatype_entities.m => test_return_modality_entities.m} (70%)
 rename tests/{test_return_datatype_extensions.m => test_return_modality_extensions.m} (66%)
 rename tests/{test_return_datatype_regular_expression.m => test_return_modality_regular_expression.m} (81%)
 rename tests/{test_return_datatype_suffixes.m => test_return_modality_suffixes.m} (65%)

diff --git a/+bids/+internal/append_to_structure.m b/+bids/+internal/append_to_structure.m
index efda3537..35e9c5f7 100644
--- a/+bids/+internal/append_to_structure.m
+++ b/+bids/+internal/append_to_structure.m
@@ -2,14 +2,14 @@
   % Copyright (C) 2021--, BIDS-MATLAB developers
 
   p = bids.internal.parse_filename(file);
-  idx = find_suffix_group(modality, p.type, schema);
+  idx = find_suffix_group(modality, p.suffix, schema);
   if isempty(idx)
     warning('append_to_structure:noMatchingSuffix', ...
             'Skipping file with no valid suffix in schema: %s', file);
     return
   end
 
-  entities = bids.schema.return_datatype_entities(schema.datatypes.(modality)(idx));
+  entities = bids.schema.return_modality_entities(schema.datatypes.(modality)(idx));
   p = bids.internal.parse_filename(file, entities);
 
   if ~isempty(subject.(modality))
diff --git a/+bids/+internal/get_metadata.m b/+bids/+internal/get_metadata.m
index 1c822eda..5d436f4d 100644
--- a/+bids/+internal/get_metadata.m
+++ b/+bids/+internal/get_metadata.m
@@ -31,7 +31,7 @@
 
     % -List the potential metadata files associated with this file suffix type
     % Default is to assume it is a JSON file
-    metafile = bids.internal.file_utils('FPList', pth, sprintf(pattern, p.type));
+    metafile = bids.internal.file_utils('FPList', pth, sprintf(pattern, p.suffix));
 
     if isempty(metafile)
       metafile = {};
diff --git a/+bids/+internal/parse_filename.m b/+bids/+internal/parse_filename.m
index ea5bf894..a638338e 100644
--- a/+bids/+internal/parse_filename.m
+++ b/+bids/+internal/parse_filename.m
@@ -12,7 +12,7 @@
   %   struct with fields:
   %
   %     filename: 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz'
-  %         type: 'T1w'
+  %         suffix: 'T1w'
   %          ext: '.nii.gz'
   %          sub: '16'
   %          ses: 'mri'
@@ -32,7 +32,7 @@
 
   % -Identify the suffix and extension of this file
   % https://bids-specification.readthedocs.io/en/stable/02-common-principles.html#file-name-structure
-  [p.type, p.ext] = strtok(parts{end}, '.');
+  [p.suffix, p.ext] = strtok(parts{end}, '.');
 
   % -Separate the entity from the label for each pair identified above
   for i = 1:numel(parts) - 1
@@ -48,7 +48,7 @@
       end
     end
     try
-      p = orderfields(p, ['filename', 'ext', 'type', fields]);
+      p = orderfields(p, ['filename', 'ext', 'suffix', fields]);
     catch
       warning('bidsMatlab:noMatchingTemplate', ...
               'Ignoring file %s not matching template.', filename);
diff --git a/+bids/+internal/return_datatype_extensions.m b/+bids/+internal/return_datatype_extensions.m
deleted file mode 100644
index 4a59932a..00000000
--- a/+bids/+internal/return_datatype_extensions.m
+++ /dev/null
@@ -1,14 +0,0 @@
-function extensions = return_datatype_extensions(datatype)
-
-  extensions = '(';
-
-  for iExt = 1:numel(datatype.extensions)
-    if ~strcmp(datatype.extensions{iExt}, '.json')
-      extensions = [extensions,  datatype.extensions{iExt}, '|']; %#ok<AGROW>
-    end
-  end
-
-  % Replace final "|" by a "){1}"
-  extensions(end:end + 3) = '){1}';
-
-end
diff --git a/+bids/+internal/return_datatype_regular_expression.m b/+bids/+internal/return_datatype_regular_expression.m
deleted file mode 100644
index 0bbaebbf..00000000
--- a/+bids/+internal/return_datatype_regular_expression.m
+++ /dev/null
@@ -1,8 +0,0 @@
-function regular_expression = return_datatype_regular_expression(datatype)
-
-  suffixes = bids.internal.return_datatype_suffixes(datatype);
-  extensions = bids.internal.return_datatype_extensions(datatype);
-
-  regular_expression = ['^%s.*' suffixes extensions '$'];
-
-end
diff --git a/+bids/+internal/return_datatype_suffixes.m b/+bids/+internal/return_datatype_suffixes.m
deleted file mode 100644
index 9bb90fc8..00000000
--- a/+bids/+internal/return_datatype_suffixes.m
+++ /dev/null
@@ -1,14 +0,0 @@
-function suffixes = return_datatype_suffixes(datatype)
-
-  suffixes = '_(';
-  
-  datatype
-
-  for iExt = 1:numel(datatype(:).suffixes)
-    suffixes = [suffixes,  datatype.suffixes{iExt}, '|']; %#ok<AGROW>
-  end
-
-  % Replace final "|" by a "){1}"
-  suffixes(end:end + 3) = '){1}';
-
-end
diff --git a/+bids/+internal/return_modality_extensions.m b/+bids/+internal/return_modality_extensions.m
new file mode 100644
index 00000000..d3f50fe6
--- /dev/null
+++ b/+bids/+internal/return_modality_extensions.m
@@ -0,0 +1,14 @@
+function extensions = return_modality_extensions(modality)
+
+  extensions = '(';
+
+  for iExt = 1:numel(modality.extensions)
+    if ~strcmp(modality.extensions{iExt}, '.json')
+      extensions = [extensions,  modality.extensions{iExt}, '|']; %#ok<AGROW>
+    end
+  end
+
+  % Replace final "|" by a "){1}"
+  extensions(end:end + 3) = '){1}';
+
+end
diff --git a/+bids/+internal/return_modality_regular_expression.m b/+bids/+internal/return_modality_regular_expression.m
new file mode 100644
index 00000000..938e8f72
--- /dev/null
+++ b/+bids/+internal/return_modality_regular_expression.m
@@ -0,0 +1,8 @@
+function regular_expression = return_modality_regular_expression(modality)
+
+  suffixes = bids.internal.return_modality_suffixes(modality);
+  extensions = bids.internal.return_modality_extensions(modality);
+
+  regular_expression = ['^%s.*' suffixes extensions '$'];
+
+end
diff --git a/+bids/+internal/return_modality_suffixes.m b/+bids/+internal/return_modality_suffixes.m
new file mode 100644
index 00000000..4bcf1a74
--- /dev/null
+++ b/+bids/+internal/return_modality_suffixes.m
@@ -0,0 +1,12 @@
+function suffixes = return_modality_suffixes(modality)
+
+  suffixes = '_(';
+
+  for iExt = 1:numel(modality(:).suffixes)
+    suffixes = [suffixes,  modality.suffixes{iExt}, '|']; %#ok<AGROW>
+  end
+
+  % Replace final "|" by a "){1}"
+  suffixes(end:end + 3) = '){1}';
+
+end
diff --git a/+bids/+schema/return_datatype_entities.m b/+bids/+schema/return_modality_entities.m
similarity index 100%
rename from +bids/+schema/return_datatype_entities.m
rename to +bids/+schema/return_modality_entities.m
diff --git a/+bids/layout.m b/+bids/layout.m
index b1bf0dc4..7e8ff04b 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -150,24 +150,20 @@
 
   % use BIDS schema to organizing parsing of subject data
   schema = bids.schema.load_schema();
-  modalities = fieldnames(schema.modalities);
+  modality_groups = fieldnames(schema.modalities);
 
-  for iModality = 1:numel(modalities)
+  for iGroup = 1:numel(modality_groups)
 
-    datatypes = schema.modalities.(modalities{iModality}).datatypes;
+    modality = schema.modalities.(modality_groups{iGroup}).datatypes;
 
-    for iDatatype = 1:numel(datatypes)
-      switch datatypes{iDatatype}
+    for iModality = 1:numel(modality)
+      switch modality{iModality}
         case {'anat', 'func', 'beh', 'meg', 'eeg', 'ieeg'}
-          subject = parse_using_schema(subject, datatypes{iDatatype}, schema);
+          subject = parse_using_schema(subject, modality{iModality}, schema);
         case 'dwi'
           subject = parse_dwi(subject, schema);
-          %         case {'eeg', 'meg'}
-          %           subject = parse_meeg(subject, datatypes{iDatatype}, schema);
         case 'fmap'
           subject = parse_fmap(subject, schema);
-          %         case 'func'
-          %           subject = parse_func(subject, schema);
         case 'perf'
           subject = parse_perf(subject, schema);
       end
@@ -180,19 +176,19 @@
 
 end
 
-function subject = parse_using_schema(subject, datatype, schema)
+function subject = parse_using_schema(subject, modality, schema)
 
-  pth = fullfile(subject.path, datatype);
+  pth = fullfile(subject.path, modality);
 
   if exist(pth, 'dir')
 
-    file_list = return_file_list(datatype, subject);
+    file_list = return_file_list(modality, subject);
 
     for i = 1:numel(file_list)
 
-      subject = bids.internal.append_to_structure(file_list{i}, subject, datatype, schema);
+      subject = bids.internal.append_to_structure(file_list{i}, subject, modality, schema);
 
-      if ~isempty(subject.(datatype)) && strcmp(subject.(datatype)(end).ext, '.tsv')
+      if ~isempty(subject.(modality)) && strcmp(subject.(modality)(end).ext, '.tsv')
         % events
         % stim
         % channels
@@ -200,13 +196,13 @@
         %
         % does not cover physio.tsv.gz or stim.tsv.gz
 
-        subject.(datatype)(end).content = [];
-        subject.(datatype)(end).meta = [];
+        subject.(modality)(end).content = [];
+        subject.(modality)(end).meta = [];
 
-        subject.(datatype)(end) = manage_tsv( ...
-                                             subject.(datatype)(end), ...
+        subject.(modality)(end) = manage_tsv( ...
+                                             subject.(modality)(end), ...
                                              pth, ...
-                                             subject.(datatype)(end).filename);
+                                             subject.(modality)(end).filename);
 
       end
 
@@ -224,16 +220,16 @@
   %  Diffusion imaging data
   % --------------------------------------------------------------------------
 
-  datatype = 'dwi';
-  pth = fullfile(subject.path, datatype);
+  modality = 'dwi';
+  pth = fullfile(subject.path, modality);
 
   if exist(pth, 'dir')
 
-    file_list = return_file_list(datatype, subject);
+    file_list = return_file_list(modality, subject);
 
     for i = 1:numel(file_list)
 
-      subject = bids.internal.append_to_structure(file_list{i}, subject, datatype, schema);
+      subject = bids.internal.append_to_structure(file_list{i}, subject, modality, schema);
 
       % bval & bvec file
       % ------------------------------------------------------------------
@@ -258,18 +254,18 @@
   % ASL perfusion imaging data
   % --------------------------------------------------------------------------
 
-  datatype = 'perf';
+  modality = 'perf';
   pth = fullfile(subject.path, 'perf');
 
   if exist(pth, 'dir')
 
-    file_list = return_file_list(datatype, subject);
+    file_list = return_file_list(modality, subject);
 
     for i = 1:numel(file_list)
 
-      subject = bids.internal.append_to_structure(file_list{i}, subject, datatype, schema);
+      subject = bids.internal.append_to_structure(file_list{i}, subject, modality, schema);
 
-      switch subject.perf(i).type
+      switch subject.perf(i).suffix
 
         case 'asl'
 
@@ -279,7 +275,7 @@
           subject.perf(i).meta = bids.internal.get_metadata( ...
                                                             fullfile( ...
                                                                      subject.path, ...
-                                                                     datatype, ...
+                                                                     modality, ...
                                                                      file_list{i}));
 
           aslcontext_file = strrep(subject.perf(i).filename, ...
@@ -301,7 +297,7 @@
           subject.perf(i).meta = bids.internal.get_metadata( ...
                                                             fullfile( ...
                                                                      subject.path, ...
-                                                                     datatype, ...
+                                                                     modality, ...
                                                                      file_list{i}));
 
           subject.perf(i) = manage_intended_for(subject.perf(i), subject, pth);
@@ -316,26 +312,26 @@
 
 function subject = parse_fmap(subject, schema)
 
-  datatype = 'fmap';
-  pth = fullfile(subject.path, datatype);
+  modality = 'fmap';
+  pth = fullfile(subject.path, modality);
 
   if exist(pth, 'dir')
 
-    file_list = return_file_list(datatype, subject);
+    file_list = return_file_list(modality, subject);
 
     for i = 1:numel(file_list)
 
-      subject = bids.internal.append_to_structure(file_list{i}, subject, datatype, schema);
+      subject = bids.internal.append_to_structure(file_list{i}, subject, modality, schema);
 
       subject.fmap(i).meta = bids.internal.get_metadata( ...
                                                         fullfile( ...
                                                                  subject.path, ...
-                                                                 datatype, ...
+                                                                 modality, ...
                                                                  file_list{i}));
       %       subject.perf(i).intended_for = [];
       %       subject.fmap(i) = manage_intended_for(subject.fmap(i), subject, pth);
 
-      switch subject.fmap(i).type
+      switch subject.fmap(i).suffix
 
         % -A single, real fieldmap image
         case {'fieldmap', 'magnitude'}
diff --git a/+bids/query.m b/+bids/query.m
index 90d41e8c..dfc09538 100644
--- a/+bids/query.m
+++ b/+bids/query.m
@@ -9,7 +9,7 @@
   %                          - 'subjects',
   %                          - 'runs',
   %                          - 'tasks',
-  %                          - 'types',
+  %                          - 'suffixes',
   %                          - 'modalities'
   % result - outcome of query
   %
@@ -33,14 +33,15 @@
   opts = parse_query(varargin);
 
   switch query
-    case {'sessions', 'subjects', 'modalities', 'tasks', 'runs', 'types', 'data', 'metadata'}
-      % -Initialise output variable
+    case {'sessions', 'subjects', 'modalities', 'tasks', 'runs', 'suffixes', 'data', 'metadata'}
+
+      % Initialise output variable
       result = {};
 
-      % -For subjects and modality we pass only the subjects/modalities asked for
+      % For subjects and modality we pass only the subjects/modalities asked for
       % otherwise we pass all of them
 
-      % -Filter according to subjects
+      % Filter according to subjects
       if any(ismember(opts(:, 1), 'sub'))
         subs = opts{ismember(opts(:, 1), 'sub'), 2};
         opts(ismember(opts(:, 1), 'sub'), :) = [];
@@ -48,7 +49,8 @@
         subs = unique({BIDS.subjects.name});
         subs = regexprep(subs, '^[a-zA-Z0-9]+-', '');
       end
-      % -Filter according to modality
+
+      % Filter according to modality
       if any(ismember(opts(:, 1), 'modality'))
         mods = opts{ismember(opts(:, 1), 'modality'), 2};
         opts(ismember(opts(:, 1), 'modality'), :) = [];
@@ -60,7 +62,7 @@
         mods   = mods(hasmod);
       end
 
-      % -Get optional target option for metadata query
+      % Get optional target option for metadata query
       if strcmp(query, 'metadata') && any(ismember(opts(:, 1), 'target'))
         target = opts{ismember(opts(:, 1), 'target'), 2};
         opts(ismember(opts(:, 1), 'target'), :) = [];
@@ -71,7 +73,8 @@
         target = [];
       end
 
-      % -Perform query
+      %% Perform query
+
       % Loop through all the subjects and modalities filtered previously
       for i = 1:numel(BIDS.subjects)
         % -Only continue if this subject is one of those filtered
@@ -132,16 +135,16 @@
                 if sts && isfield(d(k), 'task')
                   result{end + 1} = d(k).task;
                 end
-              case 'types'
-                if sts && isfield(d(k), 'type')
-                  result{end + 1} = d(k).type;
+              case 'suffixes'
+                if sts && isfield(d(k), 'suffix')
+                  result{end + 1} = d(k).suffix;
                 end
             end
           end
         end
       end
 
-      % -Postprocessing output variable
+      %% Postprocessing output variable
       switch query
         case 'subjects'
           result = unique(result);
@@ -156,7 +159,7 @@
           if numel(result) == 1
             result = result{1};
           end
-        case {'tasks', 'runs', 'types'}
+        case {'tasks', 'runs', 'suffixes'}
           result = unique(result);
           result(cellfun('isempty', result)) = [];
       end
@@ -164,9 +167,11 @@
       error('Invalid query input: ''%s''', query);
   end
 
-  % ==========================================================================
-  % -Parse BIDS query
-  % ==========================================================================
+end
+
+% ==========================================================================
+% -Parse BIDS query
+% ==========================================================================
 function query = parse_query(query)
   if numel(query) == 1 && isstruct(query{1})
     query = [fieldnames(query{1}), struct2cell(query{1})];
@@ -186,3 +191,5 @@
       end
     end
   end
+
+end
diff --git a/tests/test_append_to_structure.m b/tests/test_append_to_structure.m
index fcc8cf8d..012b0ced 100644
--- a/tests/test_append_to_structure.m
+++ b/tests/test_append_to_structure.m
@@ -20,7 +20,7 @@ function test_append_to_structure_basic()
 
   expected.anat = struct( ...
                          'filename', 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz', ...
-                         'type', 'T1w', ...
+                         'suffix', 'T1w', ...
                          'ext', '.nii.gz', ...
                          'sub', '16', ...
                          'ses', 'mri', ...
@@ -51,7 +51,7 @@ function test_append_to_structure_basic_test()
 
   expected(1).anat = struct( ...
                             'filename', 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz', ...
-                            'type', 'T1w', ...
+                            'suffix', 'T1w', ...
                             'ext', '.nii.gz', ...
                             'sub', '16', ...
                             'ses', 'mri', ...
@@ -63,7 +63,7 @@ function test_append_to_structure_basic_test()
 
   expected(2).anat = struct( ...
                             'filename', 'sub-16_ses-mri_run-1_T1map.nii.gz', ...
-                            'type', 'T1map', ...
+                            'suffix', 'T1map', ...
                             'ext', '.nii.gz', ...
                             'sub', '16', ...
                             'ses', 'mri', ...
diff --git a/tests/test_bids_query.m b/tests/test_bids_query.m
index 48b55957..5b5ebcf8 100644
--- a/tests/test_bids_query.m
+++ b/tests/test_bids_query.m
@@ -38,22 +38,22 @@ function test_bids_query_basic()
            'stopsignalwithpseudowordnaming'};
   assert(isequal(bids.query(BIDS, 'tasks'), tasks));
 
-  types = {'T1w', 'bold', 'events', 'inplaneT2'};
-  assert(isequal(bids.query(BIDS, 'types'), types));
+  suffixes = {'T1w', 'bold', 'events', 'inplaneT2'};
+  assert(isequal(bids.query(BIDS, 'suffixes'), suffixes));
 
   data = bids.query(BIDS, 'data', 'sub', '01', 'task', 'stopsignalwithpseudowordnaming');
   assertEqual(size(data, 1), 4);
 
-  assert(isempty(bids.query(BIDS, 'runs', 'type', 'T1w')));
+  assert(isempty(bids.query(BIDS, 'runs', 'suffix', 'T1w')));
 
   runs = {'01', '02'};
-  assert(isequal(bids.query(BIDS, 'runs', 'type', 'bold'), runs));
+  assert(isequal(bids.query(BIDS, 'runs', 'suffix', 'bold'), runs));
 
   bold = bids.query(BIDS, 'data', ...
                     'sub', '05', ...
                     'run', '02', ...
                     'task', 'stopsignalwithmanualresponse', ...
-                    'type', 'bold');
+                    'suffix', 'bold');
   assert(iscellstr(bold));
   assert(numel(bold) == 1);
 
@@ -61,12 +61,12 @@ function test_bids_query_basic()
                   'sub', '05', ...
                   'run', '02', ...
                   'task', 'stopsignalwithmanualresponse', ...
-                  'type', 'bold');
+                  'suffix', 'bold');
   assert(isstruct(md) & isfield(md, 'RepetitionTime') & isfield(md, 'TaskName'));
   assert(md.RepetitionTime == 2);
   assert(strcmp(md.TaskName, 'stop signal with manual response'));
 
-  t1 = bids.query(BIDS, 'data', 'type', 'T1w');
+  t1 = bids.query(BIDS, 'data', 'suffix', 'T1w');
   assert(iscellstr(t1));
   assert(numel(t1) == numel(bids.query(BIDS, 'subjects')));
 
@@ -93,17 +93,17 @@ function test_bids_query_modalities()
 
   BIDS = bids.layout(fullfile(pth_bids_example, 'ds007'));
 
-  mods = {'anat', 'func'};
-  assert(isequal(bids.query(BIDS, 'modalities'), mods));
-  assert(isequal(bids.query(BIDS, 'modalities', 'sub', '01'), mods));
+  modalities = {'anat', 'func'};
+  assert(isequal(bids.query(BIDS, 'modalities'), modalities));
+  assert(isequal(bids.query(BIDS, 'modalities', 'sub', '01'), modalities));
 
   BIDS = bids.layout(fullfile(pth_bids_example, '7t_trt'));
 
-  mods = {'anat', 'fmap', 'func'};
+  modalities = {'anat', 'fmap', 'func'};
 
-  assert(isequal(bids.query(BIDS, 'modalities'), mods));
-  assert(isequal(bids.query(BIDS, 'modalities', 'sub', '01'), mods));
-  assert(isequal(bids.query(BIDS, 'modalities', 'sub', '01', 'ses', '1'), mods));
+  assert(isequal(bids.query(BIDS, 'modalities'), modalities));
+  assert(isequal(bids.query(BIDS, 'modalities', 'sub', '01'), modalities));
+  assert(isequal(bids.query(BIDS, 'modalities', 'sub', '01', 'ses', '1'), modalities));
 
   % this now fails on octave 4.2.2 but not on Matlab
   %
diff --git a/tests/test_bids_query_asl.m b/tests/test_bids_query_asl.m
index 624ce626..b891ff48 100644
--- a/tests/test_bids_query_asl.m
+++ b/tests/test_bids_query_asl.m
@@ -19,8 +19,8 @@ function test_bids_query_asl_basic()
   modalities = {'anat', 'perf'};
   assertEqual(bids.query(BIDS, 'modalities'), modalities);
 
-  types = {'T1w', 'asl', 'aslcontext', 'asllabeling'};
-  assertEqual(bids.query(BIDS, 'types'), types);
+  suffixes = {'T1w', 'asl', 'aslcontext', 'asllabeling'};
+  assertEqual(bids.query(BIDS, 'suffixes'), suffixes);
 
   %% 'asl002'
   BIDS = bids.layout(fullfile(pth_bids_example, 'asl002'));
@@ -28,9 +28,10 @@ function test_bids_query_asl_basic()
   modalities = {'anat', 'perf'};
   assertEqual(bids.query(BIDS, 'modalities'), modalities);
 
-  types = {'T1w', 'asl', 'aslcontext', 'asllabeling', 'm0scan'};
-  assertEqual(bids.query(BIDS, 'types'), types);
-  assertEqual(bids.internal.file_utils(bids.query(BIDS, 'data', 'type', 'm0scan'), 'basename'), ...
+  suffixes = {'T1w', 'asl', 'aslcontext', 'asllabeling', 'm0scan'};
+  assertEqual(bids.query(BIDS, 'suffixes'), suffixes);
+  assertEqual(bids.internal.file_utils(bids.query(BIDS, 'data', 'suffix', 'm0scan'), ...
+                                       'basename'), ...
               {'sub-Sub103_m0scan.nii'});
 
   %% 'asl003'
@@ -39,8 +40,8 @@ function test_bids_query_asl_basic()
   modalities = {'anat', 'perf'};
   assertEqual(bids.query(BIDS, 'modalities'), modalities);
 
-  types = {'T1w', 'asl', 'aslcontext', 'asllabeling', 'm0scan'};
-  assertEqual(bids.query(BIDS, 'types'), types);
+  suffixes = {'T1w', 'asl', 'aslcontext', 'asllabeling', 'm0scan'};
+  assertEqual(bids.query(BIDS, 'suffixes'), suffixes);
 
   %% 'asl004'
   BIDS = bids.layout(fullfile(pth_bids_example, 'asl004'));
@@ -48,7 +49,7 @@ function test_bids_query_asl_basic()
   modalities = {'anat', 'fmap', 'perf'};
   assertEqual(bids.query(BIDS, 'modalities'), modalities);
 
-  types = {'T1w', 'asl', 'aslcontext', 'asllabeling', 'm0scan'};
-  assertEqual(bids.query(BIDS, 'types'), types);
+  suffixes = {'T1w', 'asl', 'aslcontext', 'asllabeling', 'm0scan'};
+  assertEqual(bids.query(BIDS, 'suffixes'), suffixes);
 
 end
diff --git a/tests/test_bids_query_dwi.m b/tests/test_bids_query_dwi.m
index 26b530cf..59475244 100644
--- a/tests/test_bids_query_dwi.m
+++ b/tests/test_bids_query_dwi.m
@@ -19,9 +19,9 @@ function test_bids_query_dwi_basic()
   modalities = {'anat',    'beh',    'dwi',    'fmap',    'func',    'meg'};
   assertEqual(bids.query(BIDS, 'modalities'), modalities);
 
-  types = {'T1w', 'bold', 'dwi', 'events', 'headshape', ...
-           'magnitude1', 'magnitude2', 'meg', 'phasediff'};
+  suffixes = {'T1w', 'bold', 'dwi', 'events', 'headshape', ...
+              'magnitude1', 'magnitude2', 'meg', 'phasediff'};
   % Missing: 'FLASH'
-  assertEqual(bids.query(BIDS, 'types'), types);
+  assertEqual(bids.query(BIDS, 'suffixes'), suffixes);
 
 end
diff --git a/tests/test_bids_query_eeg.m b/tests/test_bids_query_eeg.m
index 9e989001..7421638c 100644
--- a/tests/test_bids_query_eeg.m
+++ b/tests/test_bids_query_eeg.m
@@ -19,8 +19,8 @@ function test_bids_query_eeg_basic()
   modalities = {'eeg'};
   assertEqual(bids.query(BIDS, 'modalities'), modalities);
 
-  types = {'channels', 'eeg', 'events'};
-  assertEqual(bids.query(BIDS, 'types'), types);
+  suffixes = {'channels', 'eeg', 'events'};
+  assertEqual(bids.query(BIDS, 'suffixes'), suffixes);
 
   %%
   BIDS = bids.layout(fullfile(pth_bids_example, 'eeg_ds000117'));
@@ -28,10 +28,10 @@ function test_bids_query_eeg_basic()
   modalities = {'anat', 'eeg'};
   assertEqual(bids.query(BIDS, 'modalities'), modalities);
 
-  types = {'T1w', 'eeg', 'electrodes', 'events'};
+  suffixes = {'T1w', 'eeg', 'electrodes', 'events'};
   % Missing: 'coordsystem',
   % Missing: 'channels' in root folder
-  assertEqual(bids.query(BIDS, 'types'), types);
+  assertEqual(bids.query(BIDS, 'suffixes'), suffixes);
 
   %%
   BIDS = bids.layout(fullfile(pth_bids_example, 'eeg_face13'));
@@ -39,10 +39,10 @@ function test_bids_query_eeg_basic()
   modalities = {'eeg'};
   assertEqual(bids.query(BIDS, 'modalities'), modalities);
 
-  types = {'channels', 'eeg', 'events'};
+  suffixes = {'channels', 'eeg', 'events'};
   % Missing: 'electrodes'
   % skipped as it contains a task entity and thus does not match the schema
-  assertEqual(bids.query(BIDS, 'types'), types);
+  assertEqual(bids.query(BIDS, 'suffixes'), suffixes);
 
   %%
   BIDS = bids.layout(fullfile(pth_bids_example, 'eeg_matchingpennies'));
@@ -50,8 +50,8 @@ function test_bids_query_eeg_basic()
   modalities = {'eeg'};
   assertEqual(bids.query(BIDS, 'modalities'), modalities);
 
-  types = {'channels', 'eeg', 'events'};
-  assertEqual(bids.query(BIDS, 'types'), types);
+  suffixes = {'channels', 'eeg', 'events'};
+  assertEqual(bids.query(BIDS, 'suffixes'), suffixes);
 
   %%
   BIDS = bids.layout(fullfile(pth_bids_example, 'eeg_rishikesh'));
@@ -59,7 +59,7 @@ function test_bids_query_eeg_basic()
   modalities = {'eeg'};
   assertEqual(bids.query(BIDS, 'modalities'), modalities);
 
-  types = {'channels', 'eeg', 'events'};
-  assertEqual(bids.query(BIDS, 'types'), types);
+  suffixes = {'channels', 'eeg', 'events'};
+  assertEqual(bids.query(BIDS, 'suffixes'), suffixes);
 
 end
diff --git a/tests/test_bids_query_ieeg.m b/tests/test_bids_query_ieeg.m
index d72099a4..89c22396 100644
--- a/tests/test_bids_query_ieeg.m
+++ b/tests/test_bids_query_ieeg.m
@@ -19,9 +19,9 @@ function test_bids_query_ieeg_basic()
   modalities = {'anat', 'ieeg'};
   assertEqual(bids.query(BIDS, 'modalities'), modalities);
 
-  types = {'T1w', 'channels', 'electrodes', 'events', 'ieeg'};
+  suffixes = {'T1w', 'channels', 'electrodes', 'events', 'ieeg'};
   % Missing: 'coordsystem'
-  assertEqual(bids.query(BIDS, 'types'), types);
+  assertEqual(bids.query(BIDS, 'suffixes'), suffixes);
 
   %%
   BIDS = bids.layout(fullfile(pth_bids_example, 'ieeg_epilepsy_ecog'));
@@ -29,8 +29,8 @@ function test_bids_query_ieeg_basic()
   modalities = {'anat', 'ieeg'};
   assertEqual(bids.query(BIDS, 'modalities'), modalities);
 
-  types = {'T1w', 'channels', 'electrodes', 'events', 'ieeg', 'photo'};
+  suffixes = {'T1w', 'channels', 'electrodes', 'events', 'ieeg', 'photo'};
   % Missing: 'coordsystem'
-  assertEqual(bids.query(BIDS, 'types'), types);
+  assertEqual(bids.query(BIDS, 'suffixes'), suffixes);
 
 end
diff --git a/tests/test_bids_query_meg.m b/tests/test_bids_query_meg.m
index 8e51674b..55c27b32 100644
--- a/tests/test_bids_query_meg.m
+++ b/tests/test_bids_query_meg.m
@@ -18,9 +18,9 @@ function test_bids_query_meg_basic()
   modalities = {'anat', 'meg'};
   assertEqual(bids.query(BIDS, 'modalities'), modalities);
 
-  types = {'T1w', 'channels', 'headshape', 'meg', 'photo'};
+  suffixes = {'T1w', 'channels', 'headshape', 'meg', 'photo'};
   % missing: 'coordsystem'
-  assertEqual(bids.query(BIDS, 'types'), types);
+  assertEqual(bids.query(BIDS, 'suffixes'), suffixes);
 
   BIDS = bids.layout(fullfile(pth_bids_example, 'ds000247'));
   BIDS = bids.layout(fullfile(pth_bids_example, 'ds000248'));
diff --git a/tests/test_get_metadata.m b/tests/test_get_metadata.m
index be0e32be..45d502b1 100644
--- a/tests/test_get_metadata.m
+++ b/tests/test_get_metadata.m
@@ -39,19 +39,19 @@ function test_get_metadata_basic()
   BIDS = bids.layout(pth);
 
   %% test func metadata base directory
-  metadata = bids.query(BIDS, 'metadata', 'type', 'bold');
+  metadata = bids.query(BIDS, 'metadata', 'suffix', 'bold');
   % assert(metadata.RepetitionTime == func.RepetitionTime);
 
   %% test func metadata subject 01
-  metadata = bids.query(BIDS, 'metadata', 'sub', '01', 'type', 'bold');
+  metadata = bids.query(BIDS, 'metadata', 'sub', '01', 'suffix', 'bold');
   % assert(metadata.RepetitionTime == func_sub_01.RepetitionTime);
 
   %% test anat metadata base directory
-  metadata = bids.query(BIDS, 'metadata', 'type', 'T1w');
+  metadata = bids.query(BIDS, 'metadata', 'suffix', 'T1w');
   % assert(metadata.FlipAngle == anat.FlipAngle);
 
   %% test anat metadata subject 01
-  metadata = bids.query(BIDS, 'metadata', 'sub', '01', 'type', 'T1w');
+  metadata = bids.query(BIDS, 'metadata', 'sub', '01', 'suffix', 'T1w');
   assert(metadata.FlipAngle == anat_sub_01.FlipAngle);
   assert(strcmp(metadata.Manufacturer, anat_sub_01.Manufacturer));
 
diff --git a/tests/test_parse_filename.m b/tests/test_parse_filename.m
index 3dc2b324..2a721434 100644
--- a/tests/test_parse_filename.m
+++ b/tests/test_parse_filename.m
@@ -13,7 +13,7 @@ function test_parse_filename_basic()
 
   expected = struct( ...
                     'filename', 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz', ...
-                    'type', 'T1w', ...
+                    'suffix', 'T1w', ...
                     'ext', '.nii.gz', ...
                     'sub', '16', ...
                     'ses', 'mri', ...
@@ -22,7 +22,7 @@ function test_parse_filename_basic()
 
   %     expected = struct( ...
   %         'filename', 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz', ...
-  %         'type', 'T1w', ...
+  %         'suffix', 'T1w', ...
   %         'ext', '.nii.gz', ...
   %         'sub', '16', ...
   %         'ses', 'mri', ...
@@ -44,7 +44,7 @@ function test_parse_filename_fields()
 
   expected = struct( ...
                     'filename', 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz', ...
-                    'type', 'T1w', ...
+                    'suffix', 'T1w', ...
                     'ext', '.nii.gz', ...
                     'sub', '16', ...
                     'ses', 'mri', ...
diff --git a/tests/test_return_datatype_entities.m b/tests/test_return_modality_entities.m
similarity index 70%
rename from tests/test_return_datatype_entities.m
rename to tests/test_return_modality_entities.m
index 6c02fdb9..6f43ff0a 100644
--- a/tests/test_return_datatype_entities.m
+++ b/tests/test_return_modality_entities.m
@@ -1,4 +1,4 @@
-function test_suite = test_return_datatype_entities %#ok<*STOUT>
+function test_suite = test_return_modality_entities %#ok<*STOUT>
   try % assignment of 'localfunctions' is necessary in Matlab >= 2016
     test_functions = localfunctions(); %#ok<*NASGU>
   catch % no problem; early Matlab versions can use initTestSuite fine
@@ -6,11 +6,11 @@
   initTestSuite;
 end
 
-function test_return_datatype_entities_basic
+function test_return_modality_entities_basic
 
   schema = bids.schema.load_schema();
 
-  entities = bids.schema.return_datatype_entities(schema.datatypes.func(1));
+  entities = bids.schema.return_modality_entities(schema.datatypes.func(1));
 
   expected_output = {'sub', 'ses', 'task', 'acq', 'ce', 'rec', 'dir', 'run', 'echo', 'part'};
 
diff --git a/tests/test_return_datatype_extensions.m b/tests/test_return_modality_extensions.m
similarity index 66%
rename from tests/test_return_datatype_extensions.m
rename to tests/test_return_modality_extensions.m
index c73515ac..cfa14bcc 100644
--- a/tests/test_return_datatype_extensions.m
+++ b/tests/test_return_modality_extensions.m
@@ -1,4 +1,4 @@
-function test_suite = test_return_datatype_extensions %#ok<*STOUT>
+function test_suite = test_return_modality_extensions %#ok<*STOUT>
   try % assignment of 'localfunctions' is necessary in Matlab >= 2016
     test_functions = localfunctions(); %#ok<*NASGU>
   catch % no problem; early Matlab versions can use initTestSuite fine
@@ -6,11 +6,11 @@
   initTestSuite;
 end
 
-function test_return_datatype_extensions_basic
+function test_return_modality_extensions_basic
 
   schema = bids.schema.load_schema();
 
-  extensions = bids.internal.return_datatype_extensions(schema.datatypes.func(1));
+  extensions = bids.internal.return_modality_extensions(schema.datatypes.func(1));
   assert(isequal(extensions, '(.nii.gz|.nii){1}'));
 
 end
diff --git a/tests/test_return_datatype_regular_expression.m b/tests/test_return_modality_regular_expression.m
similarity index 81%
rename from tests/test_return_datatype_regular_expression.m
rename to tests/test_return_modality_regular_expression.m
index 05d367c5..131e76ca 100644
--- a/tests/test_return_datatype_regular_expression.m
+++ b/tests/test_return_modality_regular_expression.m
@@ -1,4 +1,4 @@
-function test_suite = test_return_datatype_regular_expression %#ok<*STOUT>
+function test_suite = test_return_modality_regular_expression %#ok<*STOUT>
   try % assignment of 'localfunctions' is necessary in Matlab >= 2016
     test_functions = localfunctions(); %#ok<*NASGU>
   catch % no problem; early Matlab versions can use initTestSuite fine
@@ -6,11 +6,11 @@
   initTestSuite;
 end
 
-function test_return_datatype_regular_expression_basic
+function test_return_modality_regular_expression_basic
 
   schema = bids.schema.load_schema();
 
-  regular_expression = bids.internal.return_datatype_regular_expression(schema.datatypes.anat(1));
+  regular_expression = bids.internal.return_modality_regular_expression(schema.datatypes.anat(1));
 
   expected_expression = ['^%s.*', ...
                          '_(T1w|T2w|PDw|T2starw|FLAIR|inplaneT1|inplaneT2|PDT2|angio){1}', ...
diff --git a/tests/test_return_datatype_suffixes.m b/tests/test_return_modality_suffixes.m
similarity index 65%
rename from tests/test_return_datatype_suffixes.m
rename to tests/test_return_modality_suffixes.m
index 4ff770a9..2018bfcc 100644
--- a/tests/test_return_datatype_suffixes.m
+++ b/tests/test_return_modality_suffixes.m
@@ -1,4 +1,4 @@
-function test_suite = test_return_datatype_suffixes %#ok<*STOUT>
+function test_suite = test_return_modality_suffixes %#ok<*STOUT>
   try % assignment of 'localfunctions' is necessary in Matlab >= 2016
     test_functions = localfunctions(); %#ok<*NASGU>
   catch % no problem; early Matlab versions can use initTestSuite fine
@@ -6,11 +6,11 @@
   initTestSuite;
 end
 
-function test_return_datatype_suffixes_basic
+function test_return_modality_suffixes_basic
 
   schema = bids.schema.load_schema();
 
-  suffixes = bids.internal.return_datatype_suffixes(schema.datatypes.func(1));
+  suffixes = bids.internal.return_modality_suffixes(schema.datatypes.func(1));
   assert(isequal(suffixes, '_(bold|cbv|sbref){1}'));
 
 end

From 9bbbc87038ff52b01ba6bda9673b67f984db84cf Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sun, 7 Feb 2021 17:20:04 +0100
Subject: [PATCH 36/54] rename function

---
 ...pend_to_structure.m => append_to_layout.m} |  2 +-
 +bids/layout.m                                | 46 ++++++-------------
 ...to_structure.m => test_append_to_layout.m} | 10 ++--
 3 files changed, 21 insertions(+), 37 deletions(-)
 rename +bids/+internal/{append_to_structure.m => append_to_layout.m} (95%)
 rename tests/{test_append_to_structure.m => test_append_to_layout.m} (87%)

diff --git a/+bids/+internal/append_to_structure.m b/+bids/+internal/append_to_layout.m
similarity index 95%
rename from +bids/+internal/append_to_structure.m
rename to +bids/+internal/append_to_layout.m
index 35e9c5f7..a5e06868 100644
--- a/+bids/+internal/append_to_structure.m
+++ b/+bids/+internal/append_to_layout.m
@@ -1,4 +1,4 @@
-function subject = append_to_structure(file, subject, modality, schema)
+function subject = append_to_layout(file, subject, modality, schema)
   % Copyright (C) 2021--, BIDS-MATLAB developers
 
   p = bids.internal.parse_filename(file);
diff --git a/+bids/layout.m b/+bids/layout.m
index 7e8ff04b..ca713da1 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -166,14 +166,14 @@
           subject = parse_fmap(subject, schema);
         case 'perf'
           subject = parse_perf(subject, schema);
+        case 'pet'
+          % not covered by schema... yet
+          subject = parse_pet(subject, schema);
       end
     end
 
   end
 
-  % not covered by schema... yet
-  subject = parse_pet(subject);
-
 end
 
 function subject = parse_using_schema(subject, modality, schema)
@@ -186,7 +186,7 @@
 
     for i = 1:numel(file_list)
 
-      subject = bids.internal.append_to_structure(file_list{i}, subject, modality, schema);
+      subject = bids.internal.append_to_layout(file_list{i}, subject, modality, schema);
 
       if ~isempty(subject.(modality)) && strcmp(subject.(modality)(end).ext, '.tsv')
         % events
@@ -229,7 +229,7 @@
 
     for i = 1:numel(file_list)
 
-      subject = bids.internal.append_to_structure(file_list{i}, subject, modality, schema);
+      subject = bids.internal.append_to_layout(file_list{i}, subject, modality, schema);
 
       % bval & bvec file
       % ------------------------------------------------------------------
@@ -263,7 +263,7 @@
 
     for i = 1:numel(file_list)
 
-      subject = bids.internal.append_to_structure(file_list{i}, subject, modality, schema);
+      subject = bids.internal.append_to_layout(file_list{i}, subject, modality, schema);
 
       switch subject.perf(i).suffix
 
@@ -321,7 +321,7 @@
 
     for i = 1:numel(file_list)
 
-      subject = bids.internal.append_to_structure(file_list{i}, subject, modality, schema);
+      subject = bids.internal.append_to_layout(file_list{i}, subject, modality, schema);
 
       subject.fmap(i).meta = bids.internal.get_metadata( ...
                                                         fullfile( ...
@@ -368,20 +368,18 @@
 end
 
 function subject = parse_pet(subject)
-  % --------------------------------------------------------------------------
-  % -Positron Emission Tomography imaging data
-  % --------------------------------------------------------------------------
+
+  modality = 'pet';
+
   pth = fullfile(subject.path, 'pet');
 
   if exist(pth, 'dir')
 
-    entities = return_entities('pet');
-
-    file_list = return_file_list('pet', subject);
+    file_list = return_file_list(modality, subject);
 
     for i = 1:numel(file_list)
 
-      subject = append_to_structure(file_list{i}, entities, subject, 'pet');
+      subject = bids.internal.append_to_layout(file_list{i}, subject, modality, schema);
 
     end
   end
@@ -399,13 +397,6 @@ function tolerant_message(tolerant, msg)
   end
 end
 
-function subject = append_to_structure(file, entities, subject, modality)
-
-  p = bids.internal.parse_filename(file, entities);
-  subject.(modality) = [subject.(modality) p];
-
-end
-
 function f = convert_to_cell(f)
   if isempty(f)
     f = {};
@@ -414,16 +405,6 @@ function tolerant_message(tolerant, msg)
   end
 end
 
-function entities = return_entities(modality)
-
-  switch modality
-
-    case 'pet'
-      entities = {'sub', 'ses', 'task', 'acq', 'rec', 'run'};
-
-  end
-end
-
 function file_list = return_file_list(modality, subject)
 
   % We list anything but json files
@@ -432,6 +413,9 @@ function tolerant_message(tolerant, msg)
   % it should be possible to create some of those patterns for the regexp
   % based on some of the required entities written down in the schema
 
+  % TODO
+  % this does not cover coordsystem.json
+
   % jn to omit json but not .pos file for headshape.pos
   pattern = '_([a-zA-Z0-9]+){1}\\..*[^jn]';
 
diff --git a/tests/test_append_to_structure.m b/tests/test_append_to_layout.m
similarity index 87%
rename from tests/test_append_to_structure.m
rename to tests/test_append_to_layout.m
index 012b0ced..2af05c35 100644
--- a/tests/test_append_to_structure.m
+++ b/tests/test_append_to_layout.m
@@ -1,4 +1,4 @@
-function test_suite = test_append_to_structure %#ok<*STOUT>
+function test_suite = test_append_to_layout %#ok<*STOUT>
   try % assignment of 'localfunctions' is necessary in Matlab >= 2016
     test_functions = localfunctions(); %#ok<*NASGU>
   catch % no problem; early Matlab versions can use initTestSuite fine
@@ -6,7 +6,7 @@
   initTestSuite;
 end
 
-function test_append_to_structure_basic()
+function test_append_to_layout_basic()
 
   schema = bids.schema.load_schema();
 
@@ -16,7 +16,7 @@ function test_append_to_structure_basic()
 
   file = '../sub-16/anat/sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz';
   entities = {'sub', 'ses', 'run', 'acq', 'ce', 'rec', 'part'};
-  subject = bids.internal.append_to_structure(file, subject, modality, schema);
+  subject = bids.internal.append_to_layout(file, subject, modality, schema);
 
   expected.anat = struct( ...
                          'filename', 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz', ...
@@ -43,11 +43,11 @@ function test_append_to_structure_basic_test()
 
   file = '../sub-16/anat/sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz';
   entities = {'sub', 'ses', 'run', 'acq', 'ce', 'rec', 'part'};
-  subject = bids.internal.append_to_structure(file, subject, modality, schema);
+  subject = bids.internal.append_to_layout(file, subject, modality, schema);
 
   file = '../sub-16/anat/sub-16_ses-mri_run-1_T1map.nii.gz';
   entities = {'sub', 'ses', 'run', 'acq', 'ce', 'rec'};
-  subject = bids.internal.append_to_structure(file, subject, modality, schema);
+  subject = bids.internal.append_to_layout(file, subject, modality, schema);
 
   expected(1).anat = struct( ...
                             'filename', 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz', ...

From d43ca198f88acafe38ad6a0422a94c007c137fc2 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sun, 7 Feb 2021 18:50:24 +0100
Subject: [PATCH 37/54] refactor query

---
 +bids/query.m               | 285 +++++++++++++++++++++---------------
 tests/test_bids_query_asl.m |   4 +
 2 files changed, 173 insertions(+), 116 deletions(-)

diff --git a/+bids/query.m b/+bids/query.m
index dfc09538..970d707c 100644
--- a/+bids/query.m
+++ b/+bids/query.m
@@ -30,164 +30,217 @@
 
   BIDS = bids.layout(BIDS);
 
-  opts = parse_query(varargin);
+  options = parse_query(varargin);
 
   switch query
-    case {'sessions', 'subjects', 'modalities', 'tasks', 'runs', 'suffixes', 'data', 'metadata'}
 
-      % Initialise output variable
-      result = {};
+    case {'sessions', 'subjects', 'modalities', 'tasks', 'runs', 'suffixes', 'data', 'metadata'}
 
       % For subjects and modality we pass only the subjects/modalities asked for
       % otherwise we pass all of them
 
-      % Filter according to subjects
-      if any(ismember(opts(:, 1), 'sub'))
-        subs = opts{ismember(opts(:, 1), 'sub'), 2};
-        opts(ismember(opts(:, 1), 'sub'), :) = [];
-      else
-        subs = unique({BIDS.subjects.name});
-        subs = regexprep(subs, '^[a-zA-Z0-9]+-', '');
-      end
+      [subjects, options] = get_subjects(BIDS, options);
 
-      % Filter according to modality
-      if any(ismember(opts(:, 1), 'modality'))
-        mods = opts{ismember(opts(:, 1), 'modality'), 2};
-        opts(ismember(opts(:, 1), 'modality'), :) = [];
-      else
-        hasmod = arrayfun(@(y) structfun(@(x) isstruct(x) & ~isempty(x), y), ...
-                          BIDS.subjects, 'UniformOutput', false);
-        hasmod = any([hasmod{:}], 2);
-        mods   = fieldnames(BIDS.subjects)';
-        mods   = mods(hasmod);
-      end
+      [modalities, options] = get_modalities(BIDS, options);
 
       % Get optional target option for metadata query
-      if strcmp(query, 'metadata') && any(ismember(opts(:, 1), 'target'))
-        target = opts{ismember(opts(:, 1), 'target'), 2};
-        opts(ismember(opts(:, 1), 'target'), :) = [];
-        if iscellstr(target)
-          target = substruct('.', target{1});
-        end
-      else
-        target = [];
-      end
+      [target, options] = get_target(query, options);
 
-      %% Perform query
-
-      % Loop through all the subjects and modalities filtered previously
-      for i = 1:numel(BIDS.subjects)
-        % -Only continue if this subject is one of those filtered
-        if ~ismember(BIDS.subjects(i).name(5:end), subs)
-          continue
-        end
-        for j = 1:numel(mods)
-          d = BIDS.subjects(i).(mods{j});
-          for k = 1:numel(d)
-            % -sts is kept true only if this modality is one of those filtered
-            sts = true;
-            for l = 1:size(opts, 1)
-              if ~isfield(d(k), opts{l, 1}) || ~ismember(d(k).(opts{l, 1}), opts{l, 2})
-                sts = false;
-              end
-            end
-            switch query
-              case 'subjects'
-                if sts
-                  result{end + 1} = BIDS.subjects(i).name;
-                end
-              case 'sessions'
-                if sts
-                  result{end + 1} = BIDS.subjects(i).session;
-                end
-              case 'modalities'
-                if sts
-                  hasmod = structfun(@(x) isstruct(x) & ~isempty(x), ...
-                                     BIDS.subjects(i));
-                  allmods = fieldnames(BIDS.subjects(i))';
-                  result = union(result, allmods(hasmod));
-                end
-              case 'data'
-                if sts && isfield(d(k), 'filename')
-                  result{end + 1} = fullfile(BIDS.subjects(i).path, mods{j}, d(k).filename);
-                end
-              case 'metadata'
-                if sts && isfield(d(k), 'filename')
-                  f = fullfile(BIDS.subjects(i).path, mods{j}, d(k).filename);
-                  result{end + 1} = bids.internal.get_metadata(f);
-                  if ~isempty(target)
-                    try
-                      result{end} = subsref(result{end}, target);
-                    catch
-                      warning('Non-existent field for metadata.');
-                      result{end} = [];
-                    end
-                  end
-                end
-                %                             if sts && isfield(d(k),'meta')
-                %                                 result{end+1} = d(k).meta;
-                %                             end
-              case 'runs'
-                if sts && isfield(d(k), 'run')
-                  result{end + 1} = d(k).run;
-                end
-              case 'tasks'
-                if sts && isfield(d(k), 'task')
-                  result{end + 1} = d(k).task;
-                end
-              case 'suffixes'
-                if sts && isfield(d(k), 'suffix')
-                  result{end + 1} = d(k).suffix;
-                end
-            end
-          end
-        end
-      end
+      result = perform_query(BIDS, query, options, subjects, modalities, target);
 
       %% Postprocessing output variable
       switch query
+
         case 'subjects'
           result = unique(result);
           result = regexprep(result, '^[a-zA-Z0-9]+-', '');
+
         case 'sessions'
           result = unique(result);
           result = regexprep(result, '^[a-zA-Z0-9]+-', '');
           result(cellfun('isempty', result)) = [];
+
         case {'modalities', 'data'}
           result = result';
+
         case 'metadata'
           if numel(result) == 1
             result = result{1};
           end
+
         case {'tasks', 'runs', 'suffixes'}
           result = unique(result);
           result(cellfun('isempty', result)) = [];
       end
+
     otherwise
       error('Invalid query input: ''%s''', query);
+
   end
 
 end
 
-% ==========================================================================
-% -Parse BIDS query
-% ==========================================================================
-function query = parse_query(query)
-  if numel(query) == 1 && isstruct(query{1})
-    query = [fieldnames(query{1}), struct2cell(query{1})];
+function options = parse_query(options)
+
+  if numel(options) == 1 && isstruct(options{1})
+    options = [fieldnames(options{1}), struct2cell(options{1})];
+
   else
-    if mod(numel(query), 2)
+    if mod(numel(options), 2)
       error('Invalid input syntax: each BIDS entity requires an associated label');
     end
-    query = reshape(query, 2, [])';
+    options = reshape(options, 2, [])';
+
   end
-  for i = 1:size(query, 1)
-    if ischar(query{i, 2})
-      query{i, 2} = cellstr(query{i, 2});
+
+  for i = 1:size(options, 1)
+
+    if ischar(options{i, 2})
+      options{i, 2} = cellstr(options{i, 2});
+    end
+
+    for j = 1:numel(options{i, 2})
+      if iscellstr(options{i, 2})
+        options{i, 2}{j} = regexprep(options{i, 2}{j}, sprintf('^%s-', options{i, 1}), '');
+      end
     end
-    for j = 1:numel(query{i, 2})
-      if iscellstr(query{i, 2})
-        query{i, 2}{j} = regexprep(query{i, 2}{j}, sprintf('^%s-', query{i, 1}), '');
+
+  end
+
+end
+
+function [subjects, options] = get_subjects(BIDS, options)
+
+  if any(ismember(options(:, 1), 'sub'))
+    subjects = options{ismember(options(:, 1), 'sub'), 2};
+    options(ismember(options(:, 1), 'sub'), :) = [];
+  else
+    subjects = unique({BIDS.subjects.name});
+    subjects = regexprep(subjects, '^[a-zA-Z0-9]+-', '');
+  end
+
+end
+
+function [modalities, options] = get_modalities(BIDS, options)
+
+  if any(ismember(options(:, 1), 'modality'))
+    modalities = options{ismember(options(:, 1), 'modality'), 2};
+    options(ismember(options(:, 1), 'modality'), :) = [];
+  else
+    hasmod = arrayfun(@(y) structfun(@(x) isstruct(x) & ~isempty(x), y), ...
+                      BIDS.subjects, 'UniformOutput', false);
+    hasmod = any([hasmod{:}], 2);
+    modalities   = fieldnames(BIDS.subjects)';
+    modalities   = modalities(hasmod);
+  end
+
+end
+
+function [target, options] = get_target(query, options)
+
+  target = [];
+
+  if strcmp(query, 'metadata') && any(ismember(options(:, 1), 'target'))
+
+    target = options{ismember(options(:, 1), 'target'), 2};
+    options(ismember(options(:, 1), 'target'), :) = [];
+
+    if iscellstr(target)
+      target = substruct('.', target{1});
+    end
+
+  end
+
+end
+
+function result = perform_query(BIDS, query, options, subjects, modalities, target)
+
+  % Initialise output variable
+  result = {};
+
+  % Loop through all the subjects and modalities filtered previously
+  for i = 1:numel(BIDS.subjects)
+
+    % -Only continue if this subject is one of those filtered
+    if ~ismember(BIDS.subjects(i).name(5:end), subjects)
+      continue
+    end
+
+    for j = 1:numel(modalities)
+
+      d = BIDS.subjects(i).(modalities{j});
+
+      for k = 1:numel(d)
+
+        % status is kept true only if this modality is one of those filtered
+        status = true;
+
+        for l = 1:size(options, 1)
+          if ~isfield(d(k), options{l, 1}) || ~ismember(d(k).(options{l, 1}), options{l, 2})
+            status = false;
+          end
+        end
+
+        switch query
+
+          case 'subjects'
+            if status
+              result{end + 1} = BIDS.subjects(i).name;
+            end
+
+          case 'sessions'
+            if status
+              result{end + 1} = BIDS.subjects(i).session;
+            end
+
+          case 'modalities'
+            if status
+              hasmod = structfun(@(x) isstruct(x) & ~isempty(x), ...
+                                 BIDS.subjects(i));
+              allmods = fieldnames(BIDS.subjects(i))';
+              result = union(result, allmods(hasmod));
+            end
+
+          case 'data'
+            if status && isfield(d(k), 'filename')
+              result{end + 1} = fullfile(BIDS.subjects(i).path, modalities{j}, d(k).filename);
+            end
+
+          case 'metadata'
+            if status && isfield(d(k), 'filename')
+
+              f = fullfile(BIDS.subjects(i).path, modalities{j}, d(k).filename);
+              result{end + 1} = bids.internal.get_metadata(f);
+              if ~isempty(target)
+                try
+                  result{end} = subsref(result{end}, target);
+                catch
+                  warning('Non-existent field for metadata.');
+                  result{end} = [];
+                end
+              end
+
+            end
+            % if status && isfield(d(k),'meta')
+            %   result{end+1} = d(k).meta;
+            % end
+
+          case 'runs'
+            if status && isfield(d(k), 'run')
+              result{end + 1} = d(k).run;
+            end
+
+          case 'tasks'
+            if status && isfield(d(k), 'task')
+              result{end + 1} = d(k).task;
+            end
+
+          case 'suffixes'
+            if status && isfield(d(k), 'suffix')
+              result{end + 1} = d(k).suffix;
+            end
+
+        end
       end
     end
   end
diff --git a/tests/test_bids_query_asl.m b/tests/test_bids_query_asl.m
index b891ff48..a6e87284 100644
--- a/tests/test_bids_query_asl.m
+++ b/tests/test_bids_query_asl.m
@@ -52,4 +52,8 @@ function test_bids_query_asl_basic()
   suffixes = {'T1w', 'asl', 'aslcontext', 'asllabeling', 'm0scan'};
   assertEqual(bids.query(BIDS, 'suffixes'), suffixes);
 
+  filename = bids.query(BIDS, 'data', 'suffix', 'm0scan', 'dir', 'pa');
+  basename = bids.internal.file_utils(filename, 'basename');
+  assertEqual(basename, {'sub-Sub1_dir-pa_m0scan.nii'});
+
 end

From 1332dec6adbd315817226897b3d92e7d433910a4 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Sun, 7 Feb 2021 23:10:09 +0100
Subject: [PATCH 38/54] allow schemaless parsing and test on derivatives

---
 +bids/+internal/append_to_layout.m |  49 +++++++++++--
 +bids/layout.m                     | 113 +++++++++++++++++++----------
 +bids/query.m                      |   2 +-
 tests/test_append_to_layout.m      |  24 +++++-
 tests/test_layout_derivatives.m    |  59 +++++++++++++++
 5 files changed, 199 insertions(+), 48 deletions(-)
 create mode 100644 tests/test_layout_derivatives.m

diff --git a/+bids/+internal/append_to_layout.m b/+bids/+internal/append_to_layout.m
index a5e06868..c937ad8d 100644
--- a/+bids/+internal/append_to_layout.m
+++ b/+bids/+internal/append_to_layout.m
@@ -1,17 +1,48 @@
 function subject = append_to_layout(file, subject, modality, schema)
+  %
+  % appends a file to the BIDS layout by parsing it according to the schema
+  %
+  % USAGE::
+  %
+  %   subject = append_to_layout(file, subject, modality, schema == [])
+  %
+  % :param file:
+  % :type  file: string
+  % :param subject: subject sub-structure from the BIDS layout
+  % :type  subject: strcture
+  % :param modality:
+  % :type  modality: string
+  % :param schema:
+  % :type  schema: strcture
+  %
+  %
   % Copyright (C) 2021--, BIDS-MATLAB developers
 
+  if ~exist('schema', 'var')
+    schema = [];
+  end
+
+  % Parse file fist to identify the suffix group in the template.
+  % Then reparse the file using the entity-label pairs defined in the schema.
   p = bids.internal.parse_filename(file);
+
   idx = find_suffix_group(modality, p.suffix, schema);
-  if isempty(idx)
-    warning('append_to_structure:noMatchingSuffix', ...
-            'Skipping file with no valid suffix in schema: %s', file);
-    return
-  end
 
-  entities = bids.schema.return_modality_entities(schema.datatypes.(modality)(idx));
-  p = bids.internal.parse_filename(file, entities);
+  if ~isempty(schema)
+
+    if isempty(idx)
+      warning('append_to_structure:noMatchingSuffix', ...
+              'Skipping file with no valid suffix in schema: %s', file);
+      return
+    end
+
+    entities = bids.schema.return_modality_entities(schema.datatypes.(modality)(idx));
+    p = bids.internal.parse_filename(file, entities);
+
+  end
 
+  % Check any new entity field that needs to be added into the layout or the output
+  % of the parsing to make sure the 2 structures can be concatenated
   if ~isempty(subject.(modality))
 
     missing_fields = setxor(fieldnames(subject.(modality)), fieldnames(p));
@@ -41,6 +72,10 @@
 
   idx = [];
 
+  if isempty(schema)
+    return
+  end
+
   % the following loop could probably be improved with some cellfun magic
   %   cellfun(@(x, y) any(strcmp(x,y)), {p.type}, suffix_groups)
   for i = 1:size(schema.datatypes.(modality), 1)
diff --git a/+bids/layout.m b/+bids/layout.m
index ca713da1..a6c74abd 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -1,9 +1,16 @@
 function BIDS = layout(root, tolerant)
+  %
   % Parse a directory structure formated according to the BIDS standard
-  % FORMAT BIDS = bids.layout(root)
-  % root     - directory formated according to BIDS [Default: pwd]
-  % tolerant - if set to 0 (default) only files g
-  % BIDS     - structure containing the BIDS file layout
+  %
+  % USAGE::
+  %
+  %   BIDS = bids.layout(root==pwd, tolerant==false)
+  %
+  % :param root:     directoryof the dataset formated according to BIDS [default: ``pwd``]
+  % :type  root:     string
+  % :param tolerant: if set to ``false`` (default) only valid BIDS datasets will be parsed
+  %                  following the bids-schema
+  % :type  tolerant: boolean
   % __________________________________________________________________________
   %
   % BIDS (Brain Imaging Data Structure): https://bids.neuroimaging.io/
@@ -19,15 +26,17 @@
   % ==========================================================================
   if ~nargin
     root = pwd;
+
   elseif nargin == 1
     if ischar(root)
       root = bids.internal.file_utils(root, 'CPath');
     elseif isstruct(root)
-      BIDS = root; % or BIDS = bids.layout(root.root);
+      BIDS = root;
       return
     else
       error('Invalid syntax.');
     end
+
   elseif nargin > 2
     error('Too many input arguments.');
   end
@@ -36,7 +45,7 @@
     tolerant = false;
   end
 
-  % -BIDS structure
+  % BIDS structure
   % ==========================================================================
 
   % BIDS.dir          -- BIDS directory
@@ -52,7 +61,8 @@
                 'participants', struct([]), ...
                 'subjects', struct([]));
 
-  % -Validation of BIDS root directory
+  % ==========================================================================
+  % REFACTOR STARTS - Validation of BIDS root directory
   % ==========================================================================
   if ~exist(BIDS.dir, 'dir')
     error('BIDS directory does not exist: ''%s''', BIDS.dir);
@@ -65,8 +75,12 @@
     tolerant_message(tolerant, msg);
 
   end
+  % ==========================================================================
+  % REFACTOR ENDS - Validation of BIDS root directory
+  % ==========================================================================
 
-  % -Dataset description
+  % ==========================================================================
+  % REFACTOR STARTS - Dataset description
   % ==========================================================================
   try
     BIDS.description = bids.util.jsondecode(fullfile(BIDS.dir, 'dataset_description.json'));
@@ -86,36 +100,45 @@
     end
 
   end
+  % ==========================================================================
+  % REFACTOR ENDS - Dataset description
+  % ==========================================================================
 
-  % -Optional directories
+  % Optional directories
   % ==========================================================================
-  % [code/]
+  % [code/] - ignore
   % [derivatives/]
-  % [stimuli/]
-  % [sourcedata/]
+  % [stimuli/] - ingore
+  % [sourcedata/] - ignore
   % [phenotype/]
 
   BIDS.participants = [];
   BIDS.participants = manage_tsv(BIDS.participants, BIDS.dir, 'participants.tsv');
 
-  % -Subjects
+  % Subjects
   % ==========================================================================
-  sub = cellstr(bids.internal.file_utils('List', BIDS.dir, 'dir', '^sub-.*$'));
-  if isequal(sub, {''})
+  subjects = cellstr(bids.internal.file_utils('List', BIDS.dir, 'dir', '^sub-.*$'));
+  if isequal(subjects, {''})
     error('No subjects found in BIDS directory.');
   end
 
-  for iSub = 1:numel(sub)
-    sess = cellstr(bids.internal.file_utils('List', ...
-                                            fullfile(BIDS.dir, sub{iSub}), ...
-                                            'dir', ...
-                                            '^ses-.*$'));
+  % if in tolerant mode we go schema-less
+  schema = bids.schema.load_schema();
+  if tolerant
+    schema = [];
+  end
+
+  for iSub = 1:numel(subjects)
+    sessions = cellstr(bids.internal.file_utils('List', ...
+                                                fullfile(BIDS.dir, subjects{iSub}), ...
+                                                'dir', ...
+                                                '^ses-.*$'));
 
-    for iSess = 1:numel(sess)
+    for iSess = 1:numel(sessions)
       if isempty(BIDS.subjects)
-        BIDS.subjects = parse_subject(BIDS.dir, sub{iSub}, sess{iSess});
+        BIDS.subjects = parse_subject(BIDS.dir, subjects{iSub}, sessions{iSess}, schema);
       else
-        BIDS.subjects(end + 1) = parse_subject(BIDS.dir, sub{iSub}, sess{iSess});
+        BIDS.subjects(end + 1) = parse_subject(BIDS.dir, subjects{iSub}, sessions{iSess}, schema);
       end
     end
 
@@ -123,14 +146,13 @@
 
 end
 
-% ==========================================================================
-% -Parse a subject's directory
-% ==========================================================================
-function subject = parse_subject(pth, subjname, sesname)
-
+function subject = parse_subject(pth, subjname, sesname, schema)
+  %
+  % Parse a subject's directory
+  %
   % For each modality (anat, func, eeg...) all the files from the
-  % corresponding directory are listed and their filenames parsed with extra
-  % BIDS valid entities listed (e.g. 'acq','ce','rec','fa'...).
+  % corresponding directory are listed and their filenames parsed with
+  % BIDS valid entities as listed in the schema (if the schema is not empty).
 
   subject.name    = subjname;   % subject name ('sub-<participant_label>')
   subject.path    = fullfile(pth, subjname, sesname); % full path to subject directory
@@ -148,18 +170,30 @@
   subject.ieeg    = struct([]); % iEEG data
   subject.pet     = struct([]); % PET imaging data
 
-  % use BIDS schema to organizing parsing of subject data
-  schema = bids.schema.load_schema();
-  modality_groups = fieldnames(schema.modalities);
+  % dummy variable if we go schema less
+  modality_groups = true;
+  if ~isempty(schema)
+    modality_groups = fieldnames(schema.modalities);
+  end
 
   for iGroup = 1:numel(modality_groups)
 
-    modality = schema.modalities.(modality_groups{iGroup}).datatypes;
+    % if we go schema-less we list directories in the subject/session folder
+    % as proxy of the modalities that we have to parse
+    modalities = cellstr(bids.internal.file_utils('List', ...
+                                                  subject.path, ...
+                                                  'dir', ...
+                                                  '.*'));
+    if ~isempty(schema)
+      modalities = schema.modalities.(modality_groups{iGroup}).datatypes;
+    end
 
-    for iModality = 1:numel(modality)
-      switch modality{iModality}
+    % if we go schema-less, we pass an empty schema to all the parsing functions
+    % so the parsing is unconstrained
+    for iModality = 1:numel(modalities)
+      switch modalities{iModality}
         case {'anat', 'func', 'beh', 'meg', 'eeg', 'ieeg'}
-          subject = parse_using_schema(subject, modality{iModality}, schema);
+          subject = parse_using_schema(subject, modalities{iModality}, schema);
         case 'dwi'
           subject = parse_dwi(subject, schema);
         case 'fmap'
@@ -169,6 +203,11 @@
         case 'pet'
           % not covered by schema... yet
           subject = parse_pet(subject, schema);
+        otherwise
+          % in case we are going schemaless and the modality is not one of the
+          % usual suspect
+          subject.(modalities{iModality}) = struct([]);
+          subject = parse_using_schema(subject, modalities{iModality}, []);
       end
     end
 
diff --git a/+bids/query.m b/+bids/query.m
index 970d707c..843bd2d6 100644
--- a/+bids/query.m
+++ b/+bids/query.m
@@ -161,7 +161,7 @@
   % Loop through all the subjects and modalities filtered previously
   for i = 1:numel(BIDS.subjects)
 
-    % -Only continue if this subject is one of those filtered
+    % Only continue if this subject is one of those filtered
     if ~ismember(BIDS.subjects(i).name(5:end), subjects)
       continue
     end
diff --git a/tests/test_append_to_layout.m b/tests/test_append_to_layout.m
index 2af05c35..279212b1 100644
--- a/tests/test_append_to_layout.m
+++ b/tests/test_append_to_layout.m
@@ -15,7 +15,6 @@ function test_append_to_layout_basic()
   modality = 'anat';
 
   file = '../sub-16/anat/sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz';
-  entities = {'sub', 'ses', 'run', 'acq', 'ce', 'rec', 'part'};
   subject = bids.internal.append_to_layout(file, subject, modality, schema);
 
   expected.anat = struct( ...
@@ -42,11 +41,9 @@ function test_append_to_structure_basic_test()
   modality = 'anat';
 
   file = '../sub-16/anat/sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz';
-  entities = {'sub', 'ses', 'run', 'acq', 'ce', 'rec', 'part'};
   subject = bids.internal.append_to_layout(file, subject, modality, schema);
 
   file = '../sub-16/anat/sub-16_ses-mri_run-1_T1map.nii.gz';
-  entities = {'sub', 'ses', 'run', 'acq', 'ce', 'rec'};
   subject = bids.internal.append_to_layout(file, subject, modality, schema);
 
   expected(1).anat = struct( ...
@@ -74,3 +71,24 @@ function test_append_to_structure_basic_test()
                             'part', '');     %#ok<*STRNU>
 
 end
+
+function test_append_to_layout_schemaless()
+
+  subject = struct('newmod', struct([]));
+
+  modality = 'newmod';
+
+  file = '../sub-16/newmod/sub-16_schema-less_anything-goes_newsuffix.EXT';
+  subject = bids.internal.append_to_layout(file, subject, modality);
+
+  expected.newmod = struct( ...
+                           'filename', 'sub-16_schema-less_anything-goes_newsuffix.EXT', ...
+                           'suffix', 'newsuffix', ...
+                           'ext', '.EXT', ...
+                           'sub', '16', ...
+                           'schema', 'less', ...
+                           'anything', 'goes');
+
+  assertEqual(subject.newmod, expected.newmod);
+
+end
diff --git a/tests/test_layout_derivatives.m b/tests/test_layout_derivatives.m
new file mode 100644
index 00000000..e893edbf
--- /dev/null
+++ b/tests/test_layout_derivatives.m
@@ -0,0 +1,59 @@
+function test_suite = test_layout_derivatives %#ok<*STOUT>
+  try % assignment of 'localfunctions' is necessary in Matlab >= 2016
+    test_functions = localfunctions(); %#ok<*NASGU>
+  catch % no problem; early Matlab versions can use initTestSuite fine
+  end
+  initTestSuite;
+end
+
+function test_layout_schemaless()
+
+  pth_bids_example = get_test_data_dir();
+
+  tolerant = true();
+
+  BIDS = bids.layout(fullfile(pth_bids_example, ...
+                              'ds000001-fmriprep'), tolerant);
+
+  modalities = {'anat', 'figures', 'func'};
+  assertEqual(bids.query(BIDS, 'modalities'), modalities);
+
+  % Those fail for now
+  %   data = bids.query(BIDS, 'data', ...
+  %                     'sub', '10', ...
+  %                     'modality', 'func', ...
+  %                     'suffix', 'bold', ...
+  %                     'run', '1', ...
+  %                     'res', '2');
+  %   data = bids.query(BIDS, 'data', ...
+  %                     'sub', '10', ...
+  %                     'modality', 'func', ...
+  %                     'suffix', 'bold', ...
+  %                     'run', '1', ...
+  %                     'space', 'MNI152NLin6Asym');
+
+end
+
+function test_layout_nested_derivatives()
+
+  pth_bids_example = get_test_data_dir();
+
+  tolerant = true();
+
+  BIDS = bids.layout(fullfile(pth_bids_example, ...
+                              'ds000117', ...
+                              'derivatives', ...
+                              'meg_derivatives'), tolerant);
+
+  modalities = {'meg'};
+  assertEqual(bids.query(BIDS, 'modalities'), modalities);
+
+  data = bids.query(BIDS, 'data', ...
+                    'sub', '01', ...
+                    'run', '01', ...
+                    'proc', 'sss', ...
+                    'suffix', 'meg');
+  basename = bids.internal.file_utils(data, 'basename');
+  assertEqual(basename, {'sub-01_ses-meg_task-facerecognition_run-01_proc-sss_meg'});
+
+end

From 2fc8d7797f612a074060d58115bcd29db6e803f0 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Mon, 8 Feb 2021 09:10:14 +0100
Subject: [PATCH 39/54] refactor schema handling

---
 +bids/+internal/append_to_layout.m       |  4 +-
 +bids/+schema/load_schema.m              | 21 ++++++--
 +bids/+schema/return_modalities.m        | 13 +++++
 +bids/+schema/return_modality_entities.m |  6 +--
 +bids/+schema/return_modality_groups.m   |  9 ++++
 +bids/layout.m                           | 63 ++++++++++--------------
 tests/test_layout_derivatives.m          |  8 +--
 tests/test_load_schema.m                 | 14 +++++-
 tests/test_return_modality_entities.m    |  2 +-
 9 files changed, 88 insertions(+), 52 deletions(-)
 create mode 100644 +bids/+schema/return_modalities.m
 create mode 100644 +bids/+schema/return_modality_groups.m

diff --git a/+bids/+internal/append_to_layout.m b/+bids/+internal/append_to_layout.m
index c937ad8d..36bdbf16 100644
--- a/+bids/+internal/append_to_layout.m
+++ b/+bids/+internal/append_to_layout.m
@@ -1,6 +1,6 @@
 function subject = append_to_layout(file, subject, modality, schema)
   %
-  % appends a file to the BIDS layout by parsing it according to the schema
+  % appends a file to the BIDS layout by parsing it according to the provided schema
   %
   % USAGE::
   %
@@ -36,7 +36,7 @@
       return
     end
 
-    entities = bids.schema.return_modality_entities(schema.datatypes.(modality)(idx));
+    entities = bids.schema.return_modality_entities(schema.datatypes.(modality)(idx), schema);
     p = bids.internal.parse_filename(file, entities);
 
   end
diff --git a/+bids/+schema/load_schema.m b/+bids/+schema/load_schema.m
index ea2e8915..3f656e69 100644
--- a/+bids/+schema/load_schema.m
+++ b/+bids/+schema/load_schema.m
@@ -1,4 +1,4 @@
-function schema = load_schema(SCHEMA_DIR)
+function schema = load_schema(use_schema)
   % Loads a json schema by recursively looking through a folder structure.
   %
   % The nesting of the output structure reflects a combination of the folder structure and
@@ -14,12 +14,27 @@
   %  for more info.
 
   if nargin < 1
-    SCHEMA_DIR = fullfile(fileparts(mfilename('fullpath')), '..', '..', 'schema');
+    use_schema = true();
+  end
+
+  if ~use_schema
+    schema = [];
+    return
+  end
+
+  if ischar(use_schema)
+    schema_dir = use_schema;
+  else
+    schema_dir = fullfile(fileparts(mfilename('fullpath')), '..', '..', 'schema');
+  end
+
+  if ~exist(schema_dir, 'dir')
+    error('The schema directory %s does not exist.', schema_dir);
   end
 
   schema = struct();
 
-  [json_file_list, dirs] = bids.internal.file_utils('FPList', SCHEMA_DIR, '^.*.json$');
+  [json_file_list, dirs] = bids.internal.file_utils('FPList', schema_dir, '^.*.json$');
 
   schema = append_json_content_to_structure(schema, json_file_list);
 
diff --git a/+bids/+schema/return_modalities.m b/+bids/+schema/return_modalities.m
new file mode 100644
index 00000000..07643c2b
--- /dev/null
+++ b/+bids/+schema/return_modalities.m
@@ -0,0 +1,13 @@
+function modalities = return_modalities(subject, schema, modality_group)
+
+  % if we go schema-less we list directories in the subject/session folder
+  % as proxy of the modalities that we have to parse
+  modalities = cellstr(bids.internal.file_utils('List', ...
+                                                subject.path, ...
+                                                'dir', ...
+                                                '.*'));
+  if ~isempty(schema)
+    modalities = schema.modalities.(modality_group).datatypes;
+  end
+
+end
diff --git a/+bids/+schema/return_modality_entities.m b/+bids/+schema/return_modality_entities.m
index fb15159e..f879b23f 100644
--- a/+bids/+schema/return_modality_entities.m
+++ b/+bids/+schema/return_modality_entities.m
@@ -1,8 +1,6 @@
-function entities = return_datatype_entities(datatype)
+function entities = return_modality_entities(suffix_group, schema)
 
-  schema = bids.schema.load_schema();
-
-  entity_names = fieldnames(datatype.entities);
+  entity_names = fieldnames(suffix_group.entities);
 
   for i = 1:size(entity_names, 1)
     entities{1, i} = schema.entities.(entity_names{i}).entity; %#ok<*AGROW>
diff --git a/+bids/+schema/return_modality_groups.m b/+bids/+schema/return_modality_groups.m
new file mode 100644
index 00000000..3651f232
--- /dev/null
+++ b/+bids/+schema/return_modality_groups.m
@@ -0,0 +1,9 @@
+function modality_groups = return_modality_groups(schema)
+
+  % dummy variable if we go schema less
+  modality_groups = {nan()};
+  if ~isempty(schema)
+    modality_groups = fieldnames(schema.modalities);
+  end
+
+end
diff --git a/+bids/layout.m b/+bids/layout.m
index a6c74abd..9f490e21 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -1,16 +1,23 @@
-function BIDS = layout(root, tolerant)
+function BIDS = layout(root, use_schema)
   %
   % Parse a directory structure formated according to the BIDS standard
   %
   % USAGE::
   %
-  %   BIDS = bids.layout(root==pwd, tolerant==false)
+  %   BIDS = bids.layout(root==pwd, use_schema==false)
   %
-  % :param root:     directoryof the dataset formated according to BIDS [default: ``pwd``]
-  % :type  root:     string
-  % :param tolerant: if set to ``false`` (default) only valid BIDS datasets will be parsed
-  %                  following the bids-schema
-  % :type  tolerant: boolean
+  % :param root:       directory of the dataset formated according to BIDS [default: ``pwd``]
+  % :type  root:       string
+  % :param use_schema: If set to ``true`` (default), the parsing of the dataset
+  %                    will follow the bids-schema provided with bids-matlab.
+  %                    If set to ``false`` files just have to be of the form
+  %                    ``sub-label_[entity-label]_suffix.ext`` to be parsed.
+  %                    If a folder path is provided, then the schema contained
+  %                    in that folder willl be used for parsing.
+  % :type  use_schema: boolean
+  %
+  %
+
   % __________________________________________________________________________
   %
   % BIDS (Brain Imaging Data Structure): https://bids.neuroimaging.io/
@@ -18,7 +25,7 @@
   %   describing outputs of neuroimaging experiments.
   %   K. J. Gorgolewski et al, Scientific Data, 2016.
   % __________________________________________________________________________
-
+  %
   % Copyright (C) 2016-2018, Guillaume Flandin, Wellcome Centre for Human Neuroimaging
   % Copyright (C) 2018--, BIDS-MATLAB developers
 
@@ -41,8 +48,8 @@
     error('Too many input arguments.');
   end
 
-  if ~exist('tolerant', 'var')
-    tolerant = false;
+  if ~exist('use_schema', 'var')
+    use_schema = true;
   end
 
   % BIDS structure
@@ -72,7 +79,7 @@
     msg = sprintf('BIDS directory not valid: missing dataset_description.json: ''%s''', ...
                   BIDS.dir);
 
-    tolerant_message(tolerant, msg);
+    tolerant_message(use_schema, msg);
 
   end
   % ==========================================================================
@@ -86,7 +93,7 @@
     BIDS.description = bids.util.jsondecode(fullfile(BIDS.dir, 'dataset_description.json'));
   catch err
     msg = sprintf('BIDS dataset description could not be read: %s', err.message);
-    tolerant_message(tolerant, msg);
+    tolerant_message(use_schema, msg);
   end
 
   fields_to_check = {'BIDSVersion', 'Name'};
@@ -96,7 +103,7 @@
       msg = sprintf( ...
                     'BIDS dataset description not valid: missing %s field.', ...
                     fields_to_check{iField});
-      tolerant_message(tolerant, msg);
+      tolerant_message(use_schema, msg);
     end
 
   end
@@ -122,11 +129,7 @@
     error('No subjects found in BIDS directory.');
   end
 
-  % if in tolerant mode we go schema-less
-  schema = bids.schema.load_schema();
-  if tolerant
-    schema = [];
-  end
+  schema = bids.schema.load_schema(use_schema);
 
   for iSub = 1:numel(subjects)
     sessions = cellstr(bids.internal.file_utils('List', ...
@@ -170,23 +173,11 @@
   subject.ieeg    = struct([]); % iEEG data
   subject.pet     = struct([]); % PET imaging data
 
-  % dummy variable if we go schema less
-  modality_groups = true;
-  if ~isempty(schema)
-    modality_groups = fieldnames(schema.modalities);
-  end
+  modality_groups = bids.schema.return_modality_groups(schema);
 
   for iGroup = 1:numel(modality_groups)
 
-    % if we go schema-less we list directories in the subject/session folder
-    % as proxy of the modalities that we have to parse
-    modalities = cellstr(bids.internal.file_utils('List', ...
-                                                  subject.path, ...
-                                                  'dir', ...
-                                                  '.*'));
-    if ~isempty(schema)
-      modalities = schema.modalities.(modality_groups{iGroup}).datatypes;
-    end
+    modalities = bids.schema.return_modalities(subject, schema, modality_groups{iGroup});
 
     % if we go schema-less, we pass an empty schema to all the parsing functions
     % so the parsing is unconstrained
@@ -428,11 +419,11 @@
 %                            HELPER FUNCTIONS
 % --------------------------------------------------------------------------
 
-function tolerant_message(tolerant, msg)
-  if tolerant
-    warning(msg);
-  else
+function tolerant_message(use_schema, msg)
+  if use_schema
     error(msg);
+  else
+    warning(msg);
   end
 end
 
diff --git a/tests/test_layout_derivatives.m b/tests/test_layout_derivatives.m
index e893edbf..5c74ffce 100644
--- a/tests/test_layout_derivatives.m
+++ b/tests/test_layout_derivatives.m
@@ -10,10 +10,10 @@ function test_layout_schemaless()
 
   pth_bids_example = get_test_data_dir();
 
-  tolerant = true();
+  use_schema = false();
 
   BIDS = bids.layout(fullfile(pth_bids_example, ...
-                              'ds000001-fmriprep'), tolerant);
+                              'ds000001-fmriprep'), use_schema);
 
   modalities = {'anat', 'figures', 'func'};
   assertEqual(bids.query(BIDS, 'modalities'), modalities);
@@ -38,12 +38,12 @@ function test_layout_nested_derivatives()
 
   pth_bids_example = get_test_data_dir();
 
-  tolerant = true();
+  use_schema = false();
 
   BIDS = bids.layout(fullfile(pth_bids_example, ...
                               'ds000117', ...
                               'derivatives', ...
-                              'meg_derivatives'), tolerant);
+                              'meg_derivatives'), use_schema);
 
   modalities = {'meg'};
   assertEqual(bids.query(BIDS, 'modalities'), modalities);
diff --git a/tests/test_load_schema.m b/tests/test_load_schema.m
index c0aa145f..fc5933ba 100644
--- a/tests/test_load_schema.m
+++ b/tests/test_load_schema.m
@@ -8,9 +8,9 @@
 
 function test_load_schema_basic()
 
-  SCHEMA_DIR = fullfile(fileparts(mfilename('fullpath')), 'schema');
+  use_schema = fullfile(fileparts(mfilename('fullpath')), 'schema');
 
-  schema = bids.schema.load_schema(SCHEMA_DIR);
+  schema = bids.schema.load_schema(use_schema);
 
   assert(isfield(schema, 'base'));
   assert(isfield(schema, 'subfolder_1'));
@@ -20,3 +20,13 @@ function test_load_schema_basic()
   assert(~isfield(schema, 'subfolder_4'));
 
 end
+
+function test_load_schema_schemaless()
+
+  use_schema = false();
+
+  schema = bids.schema.load_schema(use_schema);
+
+  assertEqual(schema, []);
+
+end
diff --git a/tests/test_return_modality_entities.m b/tests/test_return_modality_entities.m
index 6f43ff0a..3a0e3930 100644
--- a/tests/test_return_modality_entities.m
+++ b/tests/test_return_modality_entities.m
@@ -10,7 +10,7 @@
 
   schema = bids.schema.load_schema();
 
-  entities = bids.schema.return_modality_entities(schema.datatypes.func(1));
+  entities = bids.schema.return_modality_entities(schema.datatypes.func(1), schema);
 
   expected_output = {'sub', 'ses', 'task', 'acq', 'ce', 'rec', 'dir', 'run', 'echo', 'part'};
 

From 52e919a3372e933b159e0adc238356c3e2dad286 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Mon, 8 Feb 2021 09:31:37 +0100
Subject: [PATCH 40/54] refactor description light validation

---
 +bids/layout.m | 88 ++++++++++++++++++++++++++------------------------
 1 file changed, 45 insertions(+), 43 deletions(-)

diff --git a/+bids/layout.m b/+bids/layout.m
index 9f490e21..91ed027f 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -35,23 +35,32 @@
     root = pwd;
 
   elseif nargin == 1
+
     if ischar(root)
       root = bids.internal.file_utils(root, 'CPath');
+
     elseif isstruct(root)
-      BIDS = root;
+      BIDS = root; % ?????
       return
+
     else
       error('Invalid syntax.');
+
     end
 
   elseif nargin > 2
     error('Too many input arguments.');
+
   end
 
   if ~exist('use_schema', 'var')
     use_schema = true;
   end
 
+  if ~exist(root, 'dir')
+    error('BIDS directory does not exist: ''%s''', root);
+  end
+
   % BIDS structure
   % ==========================================================================
 
@@ -68,48 +77,7 @@
                 'participants', struct([]), ...
                 'subjects', struct([]));
 
-  % ==========================================================================
-  % REFACTOR STARTS - Validation of BIDS root directory
-  % ==========================================================================
-  if ~exist(BIDS.dir, 'dir')
-    error('BIDS directory does not exist: ''%s''', BIDS.dir);
-
-  elseif ~exist(fullfile(BIDS.dir, 'dataset_description.json'), 'file')
-
-    msg = sprintf('BIDS directory not valid: missing dataset_description.json: ''%s''', ...
-                  BIDS.dir);
-
-    tolerant_message(use_schema, msg);
-
-  end
-  % ==========================================================================
-  % REFACTOR ENDS - Validation of BIDS root directory
-  % ==========================================================================
-
-  % ==========================================================================
-  % REFACTOR STARTS - Dataset description
-  % ==========================================================================
-  try
-    BIDS.description = bids.util.jsondecode(fullfile(BIDS.dir, 'dataset_description.json'));
-  catch err
-    msg = sprintf('BIDS dataset description could not be read: %s', err.message);
-    tolerant_message(use_schema, msg);
-  end
-
-  fields_to_check = {'BIDSVersion', 'Name'};
-  for iField = 1:numel(fields_to_check)
-
-    if ~isfield(BIDS.description, fields_to_check{iField})
-      msg = sprintf( ...
-                    'BIDS dataset description not valid: missing %s field.', ...
-                    fields_to_check{iField});
-      tolerant_message(use_schema, msg);
-    end
-
-  end
-  % ==========================================================================
-  % REFACTOR ENDS - Dataset description
-  % ==========================================================================
+  BIDS = validate_description(BIDS, use_schema);
 
   % Optional directories
   % ==========================================================================
@@ -419,6 +387,40 @@
 %                            HELPER FUNCTIONS
 % --------------------------------------------------------------------------
 
+function BIDS = validate_description(BIDS, use_schema)
+
+  if ~exist(fullfile(BIDS.dir, 'dataset_description.json'), 'file')
+
+    msg = sprintf('BIDS directory not valid: missing dataset_description.json: ''%s''', ...
+                  BIDS.dir);
+
+    tolerant_message(use_schema, msg);
+
+  end
+  try
+    BIDS.description = bids.util.jsondecode(fullfile(BIDS.dir, 'dataset_description.json'));
+  catch err
+    msg = sprintf('BIDS dataset description could not be read: %s', err.message);
+    tolerant_message(use_schema, msg);
+  end
+
+  fields_to_check = {'BIDSVersion', 'Name'};
+  for iField = 1:numel(fields_to_check)
+
+    if ~isfield(BIDS.description, fields_to_check{iField})
+      msg = sprintf( ...
+                    'BIDS dataset description not valid: missing %s field.', ...
+                    fields_to_check{iField});
+      tolerant_message(use_schema, msg);
+    end
+
+    % TODO
+    % Add warning if bids version does not match schema version
+
+  end
+
+end
+
 function tolerant_message(use_schema, msg)
   if use_schema
     error(msg);

From d50ef3ec7c767f7a15cadf8c2b8d20a923ad09f5 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Mon, 8 Feb 2021 10:24:01 +0100
Subject: [PATCH 41/54] start implementing querying for dependencies

---
 +bids/layout.m              |  2 +-
 +bids/query.m               | 77 +++++++++++++++++++++----------------
 tests/test_bids_query_asl.m | 20 ++++++++--
 3 files changed, 62 insertions(+), 37 deletions(-)

diff --git a/+bids/layout.m b/+bids/layout.m
index 91ed027f..371a10ae 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -40,7 +40,7 @@
       root = bids.internal.file_utils(root, 'CPath');
 
     elseif isstruct(root)
-      BIDS = root; % ?????
+      BIDS = root; % for bids.query
       return
 
     else
diff --git a/+bids/query.m b/+bids/query.m
index 843bd2d6..3163d934 100644
--- a/+bids/query.m
+++ b/+bids/query.m
@@ -28,54 +28,60 @@
   %#ok<*AGROW>
   narginchk(2, Inf);
 
+  VALID_QUERIES = { ...
+                   'sessions', ...
+                   'subjects', ...
+                   'modalities', ...
+                   'tasks', ...
+                   'runs', ...
+                   'suffixes', ...
+                   'data', ...
+                   'metadata', ...
+                   'dependencies'};
+
+  if ~any(strcmp(query, VALID_QUERIES))
+    error('Invalid query input: ''%s''', query);
+  end
+
   BIDS = bids.layout(BIDS);
 
   options = parse_query(varargin);
 
-  switch query
-
-    case {'sessions', 'subjects', 'modalities', 'tasks', 'runs', 'suffixes', 'data', 'metadata'}
-
-      % For subjects and modality we pass only the subjects/modalities asked for
-      % otherwise we pass all of them
-
-      [subjects, options] = get_subjects(BIDS, options);
+  % For subjects and modality we pass only the subjects/modalities asked for
+  % otherwise we pass all of them
 
-      [modalities, options] = get_modalities(BIDS, options);
+  [subjects, options] = get_subjects(BIDS, options);
 
-      % Get optional target option for metadata query
-      [target, options] = get_target(query, options);
+  [modalities, options] = get_modalities(BIDS, options);
 
-      result = perform_query(BIDS, query, options, subjects, modalities, target);
+  % Get optional target option for metadata query
+  [target, options] = get_target(query, options);
 
-      %% Postprocessing output variable
-      switch query
+  result = perform_query(BIDS, query, options, subjects, modalities, target);
 
-        case 'subjects'
-          result = unique(result);
-          result = regexprep(result, '^[a-zA-Z0-9]+-', '');
+  %% Postprocessing output variable
+  switch query
 
-        case 'sessions'
-          result = unique(result);
-          result = regexprep(result, '^[a-zA-Z0-9]+-', '');
-          result(cellfun('isempty', result)) = [];
+    case 'subjects'
+      result = unique(result);
+      result = regexprep(result, '^[a-zA-Z0-9]+-', '');
 
-        case {'modalities', 'data'}
-          result = result';
+    case 'sessions'
+      result = unique(result);
+      result = regexprep(result, '^[a-zA-Z0-9]+-', '');
+      result(cellfun('isempty', result)) = [];
 
-        case 'metadata'
-          if numel(result) == 1
-            result = result{1};
-          end
+    case {'modalities', 'data'}
+      result = result';
 
-        case {'tasks', 'runs', 'suffixes'}
-          result = unique(result);
-          result(cellfun('isempty', result)) = [];
+    case 'metadata'
+      if numel(result) == 1
+        result = result{1};
       end
 
-    otherwise
-      error('Invalid query input: ''%s''', query);
-
+    case {'tasks', 'runs', 'suffixes'}
+      result = unique(result);
+      result(cellfun('isempty', result)) = [];
   end
 
 end
@@ -240,6 +246,11 @@
               result{end + 1} = d(k).suffix;
             end
 
+          case 'dependencies'
+            if status && isfield(d(k), 'dependencies')
+              result{end + 1} = d(k).dependencies;
+            end
+
         end
       end
     end
diff --git a/tests/test_bids_query_asl.m b/tests/test_bids_query_asl.m
index a6e87284..a8740b77 100644
--- a/tests/test_bids_query_asl.m
+++ b/tests/test_bids_query_asl.m
@@ -22,6 +22,19 @@ function test_bids_query_asl_basic()
   suffixes = {'T1w', 'asl', 'aslcontext', 'asllabeling'};
   assertEqual(bids.query(BIDS, 'suffixes'), suffixes);
 
+  BIDS.subjects(1).perf(1);
+
+  filename = bids.query(BIDS, 'data', 'sub', 'Sub103', 'suffix', 'asl');
+  basename = bids.internal.file_utils(filename, 'basename');
+  assertEqual(basename, {'sub-Sub103_asl.nii'});
+
+  meta = bids.query(BIDS, 'metadata', 'sub', 'Sub103', 'suffix', 'asl');
+
+  dependencies = bids.query(BIDS, 'dependencies', 'sub', 'Sub103', 'suffix', 'asl');
+  dependencies{1}.labeling_image;
+  dependencies{1}.context;
+  dependencies{1}.m0;
+
   %% 'asl002'
   BIDS = bids.layout(fullfile(pth_bids_example, 'asl002'));
 
@@ -30,9 +43,10 @@ function test_bids_query_asl_basic()
 
   suffixes = {'T1w', 'asl', 'aslcontext', 'asllabeling', 'm0scan'};
   assertEqual(bids.query(BIDS, 'suffixes'), suffixes);
-  assertEqual(bids.internal.file_utils(bids.query(BIDS, 'data', 'suffix', 'm0scan'), ...
-                                       'basename'), ...
-              {'sub-Sub103_m0scan.nii'});
+
+  filename = bids.query(BIDS, 'data', 'sub', 'Sub103', 'suffix', 'm0scan');
+  basename = bids.internal.file_utils(filename, 'basename');
+  assertEqual(basename, {'sub-Sub103_m0scan.nii'});
 
   %% 'asl003'
   BIDS = bids.layout(fullfile(pth_bids_example, 'asl003'));

From f1b924bd5eb920c3d98ba9be0abcbd9bb82dc389 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Mon, 8 Feb 2021 11:32:12 +0100
Subject: [PATCH 42/54] fix CI

fix CI bug - attempt 1


fix CI bug - attempt 2
---
 +bids/+internal/append_to_layout.m           | 11 ++++++++++-
 +bids/+internal/return_modality_extensions.m |  5 +++++
 +bids/+internal/return_modality_suffixes.m   |  5 +++++
 +bids/+schema/return_modality_entities.m     |  5 +++++
 tests/test_load_schema.m                     |  2 +-
 5 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/+bids/+internal/append_to_layout.m b/+bids/+internal/append_to_layout.m
index 36bdbf16..2952ce5a 100644
--- a/+bids/+internal/append_to_layout.m
+++ b/+bids/+internal/append_to_layout.m
@@ -79,10 +79,19 @@
   % the following loop could probably be improved with some cellfun magic
   %   cellfun(@(x, y) any(strcmp(x,y)), {p.type}, suffix_groups)
   for i = 1:size(schema.datatypes.(modality), 1)
-    if any(strcmp(suffix, schema.datatypes.(modality)(i).suffixes))
+
+    this_suffix_group = schema.datatypes.(modality)(i);
+
+    % for CI
+    if iscell(this_suffix_group)
+      this_suffix_group = this_suffix_group{1};
+    end
+
+    if any(strcmp(suffix, this_suffix_group.suffixes))
       idx = i;
       break
     end
+
   end
 
   if isempty(idx)
diff --git a/+bids/+internal/return_modality_extensions.m b/+bids/+internal/return_modality_extensions.m
index d3f50fe6..8a1d4eea 100644
--- a/+bids/+internal/return_modality_extensions.m
+++ b/+bids/+internal/return_modality_extensions.m
@@ -2,6 +2,11 @@
 
   extensions = '(';
 
+  % for CI
+  if iscell(modality)
+    modality = modality{1};
+  end
+
   for iExt = 1:numel(modality.extensions)
     if ~strcmp(modality.extensions{iExt}, '.json')
       extensions = [extensions,  modality.extensions{iExt}, '|']; %#ok<AGROW>
diff --git a/+bids/+internal/return_modality_suffixes.m b/+bids/+internal/return_modality_suffixes.m
index 4bcf1a74..10d81750 100644
--- a/+bids/+internal/return_modality_suffixes.m
+++ b/+bids/+internal/return_modality_suffixes.m
@@ -2,6 +2,11 @@
 
   suffixes = '_(';
 
+  % For CI
+  if iscell(modality)
+    modality = modality{1};
+  end
+
   for iExt = 1:numel(modality(:).suffixes)
     suffixes = [suffixes,  modality.suffixes{iExt}, '|']; %#ok<AGROW>
   end
diff --git a/+bids/+schema/return_modality_entities.m b/+bids/+schema/return_modality_entities.m
index f879b23f..d90a9720 100644
--- a/+bids/+schema/return_modality_entities.m
+++ b/+bids/+schema/return_modality_entities.m
@@ -1,5 +1,10 @@
 function entities = return_modality_entities(suffix_group, schema)
 
+  % for CI
+  if iscell(suffix_group)
+    suffix_group = suffix_group{1};
+  end
+
   entity_names = fieldnames(suffix_group.entities);
 
   for i = 1:size(entity_names, 1)
diff --git a/tests/test_load_schema.m b/tests/test_load_schema.m
index fc5933ba..941d916d 100644
--- a/tests/test_load_schema.m
+++ b/tests/test_load_schema.m
@@ -6,7 +6,7 @@
   initTestSuite;
 end
 
-function test_load_schema_basic()
+function test_load_schema_path()
 
   use_schema = fullfile(fileparts(mfilename('fullpath')), 'schema');
 

From f1e186b863a25baa9de2d0fd5c039e40d3ed54bf Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Mon, 8 Feb 2021 12:33:40 +0100
Subject: [PATCH 43/54] increase depth fetch (in the hope that codecov will be
 happy)

---
 .github/workflows/run_tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index c64684d5..1b11cf9c 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -29,7 +29,7 @@ jobs:
       uses: actions/checkout@v2
       with:
         submodules: true
-        fetch-depth: 1
+        fetch-depth: 2
 
     - name: Install BIDS validator
       run: |        

From f6986a67f7c7d51a32088224243f004812f5aa49 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Mon, 8 Feb 2021 21:59:50 +0100
Subject: [PATCH 44/54] upload CI coverage

---
 +bids/layout.m                  |  2 +-
 +bids/query.m                   | 17 ++++++++++++-----
 .github/workflows/run_tests.yml |  6 ++++++
 3 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/+bids/layout.m b/+bids/layout.m
index 371a10ae..064ea0ac 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -4,7 +4,7 @@
   %
   % USAGE::
   %
-  %   BIDS = bids.layout(root==pwd, use_schema==false)
+  %   BIDS = bids.layout(root = pwd, use_schema = false)
   %
   % :param root:       directory of the dataset formated according to BIDS [default: ``pwd``]
   % :type  root:       string
diff --git a/+bids/query.m b/+bids/query.m
index 3163d934..00e238e4 100644
--- a/+bids/query.m
+++ b/+bids/query.m
@@ -1,8 +1,14 @@
 function result = query(BIDS, query, varargin)
-  % QUERY Query a directory structure formatted according to the BIDS standard
-  % FORMAT result = bids.query(BIDS,query,...)
-  % BIDS   - BIDS directory name or BIDS structure (from bids.layout)
-  % query  - type of query:
+  %
+  % Queries a directory structure formatted according to the BIDS standard
+  %
+  % USAGE::
+  %
+  %   result = bids.query(BIDS, query, ...)
+  %
+  % :param BIDS: BIDS directory name or BIDS structure (from bids.layout)
+  % :type  BIDS: (strcuture or string)
+  % :param query: type of query:
   %                          - 'data',
   %                          - 'metadata',
   %                          - 'sessions',
@@ -11,7 +17,8 @@
   %                          - 'tasks',
   %                          - 'suffixes',
   %                          - 'modalities'
-  % result - outcome of query
+  % :type  query: string
+  %
   %
 
   % __________________________________________________________________________
diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index 1b11cf9c..88e50fee 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -58,6 +58,12 @@ jobs:
         with_coverage: true
         cover_xml_file: coverage.xml
 
+    - name: Upload coverage
+      uses: actions/upload-artifact@v1 
+      with: 
+        name: coverage_file 
+        path: coverage.xml         
+
     - name: Code coverage
       uses: codecov/codecov-action@v1
       with:

From a0af0d3861f978fbeba37e163576cf30f1a687ff Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Wed, 10 Feb 2021 07:08:13 +0100
Subject: [PATCH 45/54] move all entities to sub-structure

---
 +bids/+internal/get_metadata.m   |  12 ++--
 +bids/+internal/keep_file.m      |  32 +++++++++
 +bids/+internal/parse_filename.m |   9 +--
 +bids/query.m                    | 109 ++++++++++++++----------------
 tests/test_append_to_layout.m    |  48 ++++++-------
 tests/test_bids_query.m          | 111 ++++++++++++++++++++-----------
 tests/test_keep_file.m           |  57 ++++++++++++++++
 tests/test_parse_filename.m      |  16 ++---
 8 files changed, 257 insertions(+), 137 deletions(-)
 create mode 100644 +bids/+internal/keep_file.m
 create mode 100644 tests/test_keep_file.m

diff --git a/+bids/+internal/get_metadata.m b/+bids/+internal/get_metadata.m
index 5d436f4d..106ddf39 100644
--- a/+bids/+internal/get_metadata.m
+++ b/+bids/+internal/get_metadata.m
@@ -21,7 +21,7 @@
   N = 3;
 
   % -There is a session level in the hierarchy
-  if isfield(p, 'ses') && ~isempty(p.ses)
+  if isfield(p.entities, 'ses') && ~isempty(p.entities.ses)
     N = N + 1;
   end
 
@@ -44,13 +44,17 @@
     for i = 1:numel(metafile)
 
       p2 = bids.internal.parse_filename(metafile{i});
-      fn = setdiff(fieldnames(p2), {'filename', 'ext', 'type'});
+      entities = {};
+      if isfield(p2, 'entities')
+        entities = fieldnames(p2.entities);
+      end
 
       % -Check if this metadata file contains the same entity-label pairs as its
       % data file counterpart
       ismeta = true;
-      for j = 1:numel(fn)
-        if ~isfield(p, fn{j}) || ~strcmp(p.(fn{j}), p2.(fn{j}))
+      for j = 1:numel(entities)
+        if ~isfield(p.entities, entities{j}) || ...
+                ~strcmp(p.entities.(entities{j}), p2.entities.(entities{j}))
           ismeta = false;
           break
         end
diff --git a/+bids/+internal/keep_file.m b/+bids/+internal/keep_file.m
new file mode 100644
index 00000000..391116ef
--- /dev/null
+++ b/+bids/+internal/keep_file.m
@@ -0,0 +1,32 @@
+function status = keep_file(file_struct, options)
+
+  status = true;
+
+  % suffix is treated separately as it is not one of the entities
+  for l = 1:size(options, 1)
+    if strcmp(options{l, 1}, 'suffix') && ~ismember(file_struct.suffix, options{l, 2})
+      status = false;
+      return
+    end
+  end
+
+  for l = 1:size(options, 1)
+
+    if ~strcmp(options{l, 1}, 'suffix')
+
+      if ~ismember(options{l, 1}, fieldnames(file_struct.entities))
+        status = false;
+        break
+      end
+
+      if isfield(file_struct.entities, options{l, 1}) && ...
+              ~ismember(file_struct.entities.(options{l, 1}), options{l, 2})
+        status = false;
+        break
+      end
+
+    end
+
+  end
+
+end
diff --git a/+bids/+internal/parse_filename.m b/+bids/+internal/parse_filename.m
index a638338e..c8375d5f 100644
--- a/+bids/+internal/parse_filename.m
+++ b/+bids/+internal/parse_filename.m
@@ -37,18 +37,19 @@
   % -Separate the entity from the label for each pair identified above
   for i = 1:numel(parts) - 1
     [d, dummy] = regexp(parts{i}, '(?:\-)+', 'split', 'match'); %#ok<ASGLU>
-    p.(d{1}) = d{2};
+    p.entities.(d{1}) = d{2};
   end
 
   % -Extra fields can be added to the structure and ordered specifically.
   if nargin == 2
     for i = 1:numel(fields)
-      if ~isfield(p, fields{i})
-        p.(fields{i}) = '';
+      if ~isfield(p.entities, fields{i})
+        p.entities.(fields{i}) = '';
       end
     end
     try
-      p = orderfields(p, ['filename', 'ext', 'suffix', fields]);
+      p = orderfields(p, {'filename', 'ext', 'suffix', 'entities'});
+      p.entities = orderfields(p.entities, fields);
     catch
       warning('bidsMatlab:noMatchingTemplate', ...
               'Ignoring file %s not matching template.', filename);
diff --git a/+bids/query.m b/+bids/query.m
index 00e238e4..56fec96b 100644
--- a/+bids/query.m
+++ b/+bids/query.m
@@ -185,78 +185,71 @@
 
       for k = 1:numel(d)
 
-        % status is kept true only if this modality is one of those filtered
-        status = true;
+        % status is kept true only if this file matches
+        % the options of the query
+        status = bids.internal.keep_file(d(k), options);
 
-        for l = 1:size(options, 1)
-          if ~isfield(d(k), options{l, 1}) || ~ismember(d(k).(options{l, 1}), options{l, 2})
-            status = false;
-          end
-        end
+        if status
 
-        switch query
+          switch query
 
-          case 'subjects'
-            if status
+            case 'subjects'
               result{end + 1} = BIDS.subjects(i).name;
-            end
 
-          case 'sessions'
-            if status
+            case 'sessions'
               result{end + 1} = BIDS.subjects(i).session;
-            end
 
-          case 'modalities'
-            if status
+            case 'modalities'
               hasmod = structfun(@(x) isstruct(x) & ~isempty(x), ...
                                  BIDS.subjects(i));
               allmods = fieldnames(BIDS.subjects(i))';
               result = union(result, allmods(hasmod));
-            end
-
-          case 'data'
-            if status && isfield(d(k), 'filename')
-              result{end + 1} = fullfile(BIDS.subjects(i).path, modalities{j}, d(k).filename);
-            end
-
-          case 'metadata'
-            if status && isfield(d(k), 'filename')
-
-              f = fullfile(BIDS.subjects(i).path, modalities{j}, d(k).filename);
-              result{end + 1} = bids.internal.get_metadata(f);
-              if ~isempty(target)
-                try
-                  result{end} = subsref(result{end}, target);
-                catch
-                  warning('Non-existent field for metadata.');
-                  result{end} = [];
+
+            case 'data'
+              if isfield(d(k), 'filename')
+                result{end + 1} = fullfile(BIDS.subjects(i).path, modalities{j}, d(k).filename);
+              end
+
+            case 'metadata'
+              if isfield(d(k), 'filename')
+
+                f = fullfile(BIDS.subjects(i).path, modalities{j}, d(k).filename);
+                result{end + 1} = bids.internal.get_metadata(f);
+                if ~isempty(target)
+                  try
+                    result{end} = subsref(result{end}, target);
+                  catch
+                    warning('Non-existent field for metadata.');
+                    result{end} = [];
+                  end
                 end
+
+              end
+              % if status && isfield(d(k),'meta')
+              %   result{end+1} = d(k).meta;
+              % end
+
+            case 'runs'
+              if isfield(d(k).entities, 'run')
+                result{end + 1} = d(k).entities.run;
               end
 
-            end
-            % if status && isfield(d(k),'meta')
-            %   result{end+1} = d(k).meta;
-            % end
-
-          case 'runs'
-            if status && isfield(d(k), 'run')
-              result{end + 1} = d(k).run;
-            end
-
-          case 'tasks'
-            if status && isfield(d(k), 'task')
-              result{end + 1} = d(k).task;
-            end
-
-          case 'suffixes'
-            if status && isfield(d(k), 'suffix')
-              result{end + 1} = d(k).suffix;
-            end
-
-          case 'dependencies'
-            if status && isfield(d(k), 'dependencies')
-              result{end + 1} = d(k).dependencies;
-            end
+            case 'tasks'
+              if isfield(d(k).entities, 'task')
+                result{end + 1} = d(k).entities.task;
+              end
+
+            case 'suffixes'
+              if isfield(d(k), 'suffix')
+                result{end + 1} = d(k).suffix;
+              end
+
+            case 'dependencies'
+              if isfield(d(k), 'dependencies')
+                result{end + 1} = d(k).dependencies;
+              end
+
+          end
 
         end
       end
diff --git a/tests/test_append_to_layout.m b/tests/test_append_to_layout.m
index 279212b1..5abe9576 100644
--- a/tests/test_append_to_layout.m
+++ b/tests/test_append_to_layout.m
@@ -21,13 +21,13 @@ function test_append_to_layout_basic()
                          'filename', 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz', ...
                          'suffix', 'T1w', ...
                          'ext', '.nii.gz', ...
-                         'sub', '16', ...
-                         'ses', 'mri', ...
-                         'run', '1', ...
-                         'acq', 'hd', ...
-                         'ce', '', ...
-                         'rec', '', ...
-                         'part', '');
+                         'entities', struct('sub', '16', ...
+                                            'ses', 'mri', ...
+                                            'run', '1', ...
+                                            'acq', 'hd', ...
+                                            'ce', '', ...
+                                            'rec', '', ...
+                                            'part', ''));
 
   assertEqual(subject.anat, expected.anat);
 
@@ -50,25 +50,25 @@ function test_append_to_structure_basic_test()
                             'filename', 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz', ...
                             'suffix', 'T1w', ...
                             'ext', '.nii.gz', ...
-                            'sub', '16', ...
-                            'ses', 'mri', ...
-                            'run', '1', ...
-                            'acq', 'hd', ...
-                            'ce', '', ...
-                            'rec', '', ...
-                            'part', '');
+                            'entities', struct('sub', '16', ...
+                                               'ses', 'mri', ...
+                                               'run', '1', ...
+                                               'acq', 'hd', ...
+                                               'ce', '', ...
+                                               'rec', '', ...
+                                               'part', ''));
 
   expected(2).anat = struct( ...
                             'filename', 'sub-16_ses-mri_run-1_T1map.nii.gz', ...
                             'suffix', 'T1map', ...
                             'ext', '.nii.gz', ...
-                            'sub', '16', ...
-                            'ses', 'mri', ...
-                            'run', '1', ...
-                            'acq', '', ...
-                            'ce', '', ...
-                            'rec', '', ...
-                            'part', '');     %#ok<*STRNU>
+                            'entities', struct('sub', '16', ...
+                                               'ses', 'mri', ...
+                                               'run', '1', ...
+                                               'acq', '', ...
+                                               'ce', '', ...
+                                               'rec', '', ...
+                                               'part', ''));     %#ok<*STRNU>
 
 end
 
@@ -85,9 +85,9 @@ function test_append_to_layout_schemaless()
                            'filename', 'sub-16_schema-less_anything-goes_newsuffix.EXT', ...
                            'suffix', 'newsuffix', ...
                            'ext', '.EXT', ...
-                           'sub', '16', ...
-                           'schema', 'less', ...
-                           'anything', 'goes');
+                           'entities', struct('sub', '16', ...
+                                              'schema', 'less', ...
+                                              'anything', 'goes'));
 
   assertEqual(subject.newmod, expected.newmod);
 
diff --git a/tests/test_bids_query.m b/tests/test_bids_query.m
index 5b5ebcf8..a8979afc 100644
--- a/tests/test_bids_query.m
+++ b/tests/test_bids_query.m
@@ -4,9 +4,7 @@
   catch % no problem; early Matlab versions can use initTestSuite fine
   end
   initTestSuite;
-end
 
-function test_bids_query_basic()
   % Test BIDS queries
   % This dataset comes from https://github.com/bids-standard/bids-examples
   % and is downloaded automatically by the continuous integration framework
@@ -23,32 +21,21 @@ function test_bids_query_basic()
   % Copyright (C) 2019, Guillaume Flandin, Wellcome Centre for Human Neuroimaging
   % Copyright (C) 2019--, BIDS-MATLAB developers
 
-  pth_bids_example = get_test_data_dir();
-
-  BIDS = bids.layout(fullfile(pth_bids_example, 'ds007'));
+end
 
-  subjs = arrayfun(@(x) sprintf('%02d', x), 1:20, 'UniformOutput', false);
-  assert(isequal(bids.query(BIDS, 'subjects'), subjs));
+function test_bids_query_data()
 
-  assert(isequal(bids.query(BIDS, 'runs'), {'01', '02'}));
+  pth_bids_example = get_test_data_dir();
 
-  tasks = { ...
-           'stopsignalwithletternaming', ...
-           'stopsignalwithmanualresponse', ...
-           'stopsignalwithpseudowordnaming'};
-  assert(isequal(bids.query(BIDS, 'tasks'), tasks));
+  BIDS = bids.layout(fullfile(pth_bids_example, 'ds007'));
 
-  suffixes = {'T1w', 'bold', 'events', 'inplaneT2'};
-  assert(isequal(bids.query(BIDS, 'suffixes'), suffixes));
+  t1 = bids.query(BIDS, 'data', 'suffix', 'T1w');
+  assert(iscellstr(t1));
+  assert(numel(t1) == numel(bids.query(BIDS, 'subjects')));
 
   data = bids.query(BIDS, 'data', 'sub', '01', 'task', 'stopsignalwithpseudowordnaming');
   assertEqual(size(data, 1), 4);
 
-  assert(isempty(bids.query(BIDS, 'runs', 'suffix', 'T1w')));
-
-  runs = {'01', '02'};
-  assert(isequal(bids.query(BIDS, 'runs', 'suffix', 'bold'), runs));
-
   bold = bids.query(BIDS, 'data', ...
                     'sub', '05', ...
                     'run', '02', ...
@@ -57,6 +44,14 @@ function test_bids_query_basic()
   assert(iscellstr(bold));
   assert(numel(bold) == 1);
 
+end
+
+function test_bids_query_metadata()
+
+  pth_bids_example = get_test_data_dir();
+
+  BIDS = bids.layout(fullfile(pth_bids_example, 'ds007'));
+
   md = bids.query(BIDS, 'metadata', ...
                   'sub', '05', ...
                   'run', '02', ...
@@ -66,25 +61,6 @@ function test_bids_query_basic()
   assert(md.RepetitionTime == 2);
   assert(strcmp(md.TaskName, 'stop signal with manual response'));
 
-  t1 = bids.query(BIDS, 'data', 'suffix', 'T1w');
-  assert(iscellstr(t1));
-  assert(numel(t1) == numel(bids.query(BIDS, 'subjects')));
-
-end
-
-function test_bids_query_sessions()
-
-  pth_bids_example = get_test_data_dir();
-
-  BIDS = bids.layout(fullfile(pth_bids_example, 'synthetic'));
-  sessions = {'01', '02'};
-  assert(isequal(bids.query(BIDS, 'sessions'), sessions));
-  assert(isequal(bids.query(BIDS, 'sessions', 'sub', '02'), sessions));
-
-  BIDS = bids.layout(fullfile(pth_bids_example, 'ds007'));
-
-  assert(isempty(bids.query(BIDS, 'sessions')));
-
 end
 
 function test_bids_query_modalities()
@@ -121,3 +97,60 @@ function test_bids_query_modalities()
   % assert(isequal(bids.query(BIDS, 'modalities', 'sub', '01', 'ses', '2'), mods(2:3)));
 
 end
+
+function test_bids_query_basic()
+
+  pth_bids_example = get_test_data_dir();
+
+  BIDS = bids.layout(fullfile(pth_bids_example, 'ds007'));
+
+  tasks = { ...
+           'stopsignalwithletternaming', ...
+           'stopsignalwithmanualresponse', ...
+           'stopsignalwithpseudowordnaming'};
+  assert(isequal(bids.query(BIDS, 'tasks'), tasks));
+
+  assert(isempty(bids.query(BIDS, 'runs', 'suffix', 'T1w')));
+
+  runs = {'01', '02'};
+  assert(isequal(bids.query(BIDS, 'runs'), runs));
+  assert(isequal(bids.query(BIDS, 'runs', 'suffix', 'bold'), runs));
+
+end
+
+function test_bids_query_subjects()
+
+  pth_bids_example = get_test_data_dir();
+
+  BIDS = bids.layout(fullfile(pth_bids_example, 'ds007'));
+
+  subjs = arrayfun(@(x) sprintf('%02d', x), 1:20, 'UniformOutput', false);
+  assert(isequal(bids.query(BIDS, 'subjects'), subjs));
+
+end
+
+function test_bids_query_sessions()
+
+  pth_bids_example = get_test_data_dir();
+
+  BIDS = bids.layout(fullfile(pth_bids_example, 'synthetic'));
+  sessions = {'01', '02'};
+  assert(isequal(bids.query(BIDS, 'sessions'), sessions));
+  assert(isequal(bids.query(BIDS, 'sessions', 'sub', '02'), sessions));
+
+  BIDS = bids.layout(fullfile(pth_bids_example, 'ds007'));
+
+  assert(isempty(bids.query(BIDS, 'sessions')));
+
+end
+
+function test_bids_query_suffixes()
+
+  pth_bids_example = get_test_data_dir();
+
+  BIDS = bids.layout(fullfile(pth_bids_example, 'ds007'));
+
+  suffixes = {'T1w', 'bold', 'events', 'inplaneT2'};
+  assert(isequal(bids.query(BIDS, 'suffixes'), suffixes));
+
+end
diff --git a/tests/test_keep_file.m b/tests/test_keep_file.m
new file mode 100644
index 00000000..a47670f5
--- /dev/null
+++ b/tests/test_keep_file.m
@@ -0,0 +1,57 @@
+function test_suite = test_keep_file %#ok<*STOUT>
+  try % assignment of 'localfunctions' is necessary in Matlab >= 2016
+    test_functions = localfunctions(); %#ok<*NASGU>
+  catch % no problem; early Matlab versions can use initTestSuite fine
+  end
+  initTestSuite;
+
+end
+
+function test_keep_file_basic()
+
+  file_struct = struct('suffix', 'bold', ...
+                       'entities', struct('task', 'balloon', ...
+                                          'ses', '01', ...
+                                          'sub', '02'));
+
+  options = {'ses', {'01'}};
+  assertEqual(bids.internal.keep_file(file_struct, options), true);
+
+  options = {'ses', {'02'}};
+  assertEqual(bids.internal.keep_file(file_struct, options), false);
+
+  options = {'ses', {'01', '02'}};
+  assertEqual(bids.internal.keep_file(file_struct, options), true);
+
+  options = {'suffix', {'T1w'}};
+  assertEqual(bids.internal.keep_file(file_struct, options), false);
+
+  options = {'suffix', {'bold'}};
+  assertEqual(bids.internal.keep_file(file_struct, options), true);
+
+  options = {'suffix', {'T1w', 'bold'}};
+  assertEqual(bids.internal.keep_file(file_struct, options), true);
+
+  options = {'sub', {'02'}
+             'ses', {'01'}};
+  assertEqual(bids.internal.keep_file(file_struct, options), true);
+
+  options = {'sub', {'02'}
+             'ses', {'02'}};
+  assertEqual(bids.internal.keep_file(file_struct, options), false);
+
+  options = {'sub', {'02'}
+             'ses', {'01'}
+             'suffix', {'T1w'}};
+  assertEqual(bids.internal.keep_file(file_struct, options), false);
+
+  options = {'sub', {'02'}
+             'task', {'balloon'}};
+  assertEqual(bids.internal.keep_file(file_struct, options), true);
+
+  file_struct = struct('suffix', 'T1w', ...
+                       'entities', struct('ses', '01', ...
+                                          'sub', '02'));
+  assertEqual(bids.internal.keep_file(file_struct, options), false);
+
+end
diff --git a/tests/test_parse_filename.m b/tests/test_parse_filename.m
index 2a721434..1630de5e 100644
--- a/tests/test_parse_filename.m
+++ b/tests/test_parse_filename.m
@@ -15,10 +15,10 @@ function test_parse_filename_basic()
                     'filename', 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz', ...
                     'suffix', 'T1w', ...
                     'ext', '.nii.gz', ...
-                    'sub', '16', ...
-                    'ses', 'mri', ...
-                    'run', '1', ...
-                    'acq', 'hd');
+                    'entities', struct('sub', '16', ...
+                                       'ses', 'mri', ...
+                                       'run', '1', ...
+                                       'acq', 'hd'));
 
   %     expected = struct( ...
   %         'filename', 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz', ...
@@ -46,10 +46,10 @@ function test_parse_filename_fields()
                     'filename', 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz', ...
                     'suffix', 'T1w', ...
                     'ext', '.nii.gz', ...
-                    'sub', '16', ...
-                    'ses', 'mri', ...
-                    'run', '1', ...
-                    'acq', 'hd');
+                    'entities', struct('sub', '16', ...
+                                       'ses', 'mri', ...
+                                       'run', '1', ...
+                                       'acq', 'hd'));
 
   assertEqual(output, expected);
 

From 1659acd681b12441cf8bad200afd2827e34c9b32 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Wed, 10 Feb 2021 10:46:09 +0100
Subject: [PATCH 46/54] rename keep file function

---
 .../{keep_file.m => keep_file_for_query.m}    |  0
 +bids/query.m                                 |  2 +-
 tests/test_keep_file.m                        | 22 +++++++++----------
 3 files changed, 12 insertions(+), 12 deletions(-)
 rename +bids/+internal/{keep_file.m => keep_file_for_query.m} (100%)

diff --git a/+bids/+internal/keep_file.m b/+bids/+internal/keep_file_for_query.m
similarity index 100%
rename from +bids/+internal/keep_file.m
rename to +bids/+internal/keep_file_for_query.m
diff --git a/+bids/query.m b/+bids/query.m
index 56fec96b..cbed700d 100644
--- a/+bids/query.m
+++ b/+bids/query.m
@@ -187,7 +187,7 @@
 
         % status is kept true only if this file matches
         % the options of the query
-        status = bids.internal.keep_file(d(k), options);
+        status = bids.internal.keep_file_for_query(d(k), options);
 
         if status
 
diff --git a/tests/test_keep_file.m b/tests/test_keep_file.m
index a47670f5..71226937 100644
--- a/tests/test_keep_file.m
+++ b/tests/test_keep_file.m
@@ -15,43 +15,43 @@ function test_keep_file_basic()
                                           'sub', '02'));
 
   options = {'ses', {'01'}};
-  assertEqual(bids.internal.keep_file(file_struct, options), true);
+  assertEqual(bids.internal.keep_file_for_query(file_struct, options), true);
 
   options = {'ses', {'02'}};
-  assertEqual(bids.internal.keep_file(file_struct, options), false);
+  assertEqual(bids.internal.keep_file_for_query(file_struct, options), false);
 
   options = {'ses', {'01', '02'}};
-  assertEqual(bids.internal.keep_file(file_struct, options), true);
+  assertEqual(bids.internal.keep_file_for_query(file_struct, options), true);
 
   options = {'suffix', {'T1w'}};
-  assertEqual(bids.internal.keep_file(file_struct, options), false);
+  assertEqual(bids.internal.keep_file_for_query(file_struct, options), false);
 
   options = {'suffix', {'bold'}};
-  assertEqual(bids.internal.keep_file(file_struct, options), true);
+  assertEqual(bids.internal.keep_file_for_query(file_struct, options), true);
 
   options = {'suffix', {'T1w', 'bold'}};
-  assertEqual(bids.internal.keep_file(file_struct, options), true);
+  assertEqual(bids.internal.keep_file_for_query(file_struct, options), true);
 
   options = {'sub', {'02'}
              'ses', {'01'}};
-  assertEqual(bids.internal.keep_file(file_struct, options), true);
+  assertEqual(bids.internal.keep_file_for_query(file_struct, options), true);
 
   options = {'sub', {'02'}
              'ses', {'02'}};
-  assertEqual(bids.internal.keep_file(file_struct, options), false);
+  assertEqual(bids.internal.keep_file_for_query(file_struct, options), false);
 
   options = {'sub', {'02'}
              'ses', {'01'}
              'suffix', {'T1w'}};
-  assertEqual(bids.internal.keep_file(file_struct, options), false);
+  assertEqual(bids.internal.keep_file_for_query(file_struct, options), false);
 
   options = {'sub', {'02'}
              'task', {'balloon'}};
-  assertEqual(bids.internal.keep_file(file_struct, options), true);
+  assertEqual(bids.internal.keep_file_for_query(file_struct, options), true);
 
   file_struct = struct('suffix', 'T1w', ...
                        'entities', struct('ses', '01', ...
                                           'sub', '02'));
-  assertEqual(bids.internal.keep_file(file_struct, options), false);
+  assertEqual(bids.internal.keep_file_for_query(file_struct, options), false);
 
 end

From b000789cd39ed6e47208d31ba45f0f6007fd64fa Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Wed, 10 Feb 2021 10:59:04 +0100
Subject: [PATCH 47/54] remove parse_pet

---
 +bids/layout.m | 20 +-------------------
 1 file changed, 1 insertion(+), 19 deletions(-)

diff --git a/+bids/layout.m b/+bids/layout.m
index 064ea0ac..0a4d6215 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -151,7 +151,7 @@
     % so the parsing is unconstrained
     for iModality = 1:numel(modalities)
       switch modalities{iModality}
-        case {'anat', 'func', 'beh', 'meg', 'eeg', 'ieeg'}
+        case {'anat', 'func', 'beh', 'meg', 'eeg', 'ieeg', 'pet'}
           subject = parse_using_schema(subject, modalities{iModality}, schema);
         case 'dwi'
           subject = parse_dwi(subject, schema);
@@ -365,24 +365,6 @@
 
 end
 
-function subject = parse_pet(subject)
-
-  modality = 'pet';
-
-  pth = fullfile(subject.path, 'pet');
-
-  if exist(pth, 'dir')
-
-    file_list = return_file_list(modality, subject);
-
-    for i = 1:numel(file_list)
-
-      subject = bids.internal.append_to_layout(file_list{i}, subject, modality, schema);
-
-    end
-  end
-end
-
 % --------------------------------------------------------------------------
 %                            HELPER FUNCTIONS
 % --------------------------------------------------------------------------

From 7f34319c851a390de5674a537d7568c5fc2bcf79 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Wed, 10 Feb 2021 14:38:50 +0100
Subject: [PATCH 48/54] small refactoring

---
 +bids/+internal/add_missing_field.m      |  5 ++++
 +bids/+internal/append_to_layout.m       | 29 ++++++++++------------
 +bids/+internal/match_structure_fields.m | 14 +++++++++++
 +bids/+internal/parse_filename.m         |  6 ++---
 +bids/layout.m                           | 31 ++++++++++++------------
 5 files changed, 50 insertions(+), 35 deletions(-)
 create mode 100644 +bids/+internal/add_missing_field.m
 create mode 100644 +bids/+internal/match_structure_fields.m

diff --git a/+bids/+internal/add_missing_field.m b/+bids/+internal/add_missing_field.m
new file mode 100644
index 00000000..9e7ae6f1
--- /dev/null
+++ b/+bids/+internal/add_missing_field.m
@@ -0,0 +1,5 @@
+function structure = add_missing_field(structure, field)
+  if ~isfield(structure, field)
+    structure(1).(field) = '';
+  end
+end
diff --git a/+bids/+internal/append_to_layout.m b/+bids/+internal/append_to_layout.m
index 2952ce5a..a2a88c8e 100644
--- a/+bids/+internal/append_to_layout.m
+++ b/+bids/+internal/append_to_layout.m
@@ -45,16 +45,19 @@
   % of the parsing to make sure the 2 structures can be concatenated
   if ~isempty(subject.(modality))
 
-    missing_fields = setxor(fieldnames(subject.(modality)), fieldnames(p));
-
-    if ~isempty(missing_fields)
-      for iField = 1:numel(missing_fields)
-        p = add_missing_field(p, ...
-                              missing_fields{iField});
-        subject.(modality) = add_missing_field(subject.(modality), ...
-                                               missing_fields{iField});
-      end
-    end
+    [subject.(modality), p] = bids.internal.match_structure_fields(subject.(modality), p);
+
+    %     missing_fields = setxor(fieldnames(subject.(modality)), fieldnames(p));
+
+    %     if ~isempty(missing_fields)
+    %       for iField = 1:numel(missing_fields)
+    %
+    %         p = bids.internal.add_missing_field(p, missing_fields{iField});
+    %
+    %         subject.(modality) = bids.internal.add_missing_field(subject.(modality), ...
+    %                                                missing_fields{iField});
+    %       end
+    %     end
 
   end
 
@@ -62,12 +65,6 @@
 
 end
 
-function structure = add_missing_field(structure, field)
-  if ~isfield(structure, field)
-    structure(1).(field) = '';
-  end
-end
-
 function idx = find_suffix_group(modality, suffix, schema)
 
   idx = [];
diff --git a/+bids/+internal/match_structure_fields.m b/+bids/+internal/match_structure_fields.m
new file mode 100644
index 00000000..1234f84a
--- /dev/null
+++ b/+bids/+internal/match_structure_fields.m
@@ -0,0 +1,14 @@
+function [s1, s2] = match_structure_fields(s1, s2)
+
+  missing_fields = setxor(fieldnames(s1), fieldnames(s2));
+
+  if ~isempty(missing_fields)
+    for iField = 1:numel(missing_fields)
+
+      s1 = bids.internal.add_missing_field(s1, missing_fields{iField});
+      s2 = bids.internal.add_missing_field(s2, missing_fields{iField});
+
+    end
+  end
+
+end
diff --git a/+bids/+internal/parse_filename.m b/+bids/+internal/parse_filename.m
index c8375d5f..cb81f388 100644
--- a/+bids/+internal/parse_filename.m
+++ b/+bids/+internal/parse_filename.m
@@ -43,9 +43,7 @@
   % -Extra fields can be added to the structure and ordered specifically.
   if nargin == 2
     for i = 1:numel(fields)
-      if ~isfield(p.entities, fields{i})
-        p.entities.(fields{i}) = '';
-      end
+      p.entities = bids.internal.add_missing_field(p.entities, fields{i});
     end
     try
       p = orderfields(p, {'filename', 'ext', 'suffix', 'entities'});
@@ -56,3 +54,5 @@
       p = struct([]);
     end
   end
+
+end
diff --git a/+bids/layout.m b/+bids/layout.m
index 0a4d6215..848a0037 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -109,7 +109,11 @@
       if isempty(BIDS.subjects)
         BIDS.subjects = parse_subject(BIDS.dir, subjects{iSub}, sessions{iSess}, schema);
       else
-        BIDS.subjects(end + 1) = parse_subject(BIDS.dir, subjects{iSub}, sessions{iSess}, schema);
+        new_subject = parse_subject(BIDS.dir, subjects{iSub}, sessions{iSess}, schema);
+        [BIDS.subjects, new_subject] = bids.internal.match_structure_fields(BIDS.subjects, ...
+                                                                            new_subject);
+        BIDS.subjects(end + 1) = new_subject;
+
       end
     end
 
@@ -130,16 +134,6 @@
   subject.session = sesname;    % session name ('' or 'ses-<label>')
   subject.scans   = struct([]); % for sub-<participant_label>_scans.tsv
   subject.sess    = struct([]); % for sub-<participants_label>_sessions.tsv
-  subject.anat    = struct([]); % anatomy imaging data
-  subject.func    = struct([]); % task imaging data
-  subject.fmap    = struct([]); % fieldmap data
-  subject.beh     = struct([]); % behavioral experiment data
-  subject.dwi     = struct([]); % diffusion imaging data
-  subject.perf    = struct([]); % ASL perfusion imaging data
-  subject.eeg     = struct([]); % EEG data
-  subject.meg     = struct([]); % MEG data
-  subject.ieeg    = struct([]); % iEEG data
-  subject.pet     = struct([]); % PET imaging data
 
   modality_groups = bids.schema.return_modality_groups(schema);
 
@@ -159,12 +153,9 @@
           subject = parse_fmap(subject, schema);
         case 'perf'
           subject = parse_perf(subject, schema);
-        case 'pet'
-          % not covered by schema... yet
-          subject = parse_pet(subject, schema);
         otherwise
-          % in case we are going schemaless and the modality is not one of the
-          % usual suspect
+          % in case we are going schemaless
+          % and the modality is not one of the usual suspect
           subject.(modalities{iModality}) = struct([]);
           subject = parse_using_schema(subject, modalities{iModality}, []);
       end
@@ -180,6 +171,8 @@
 
   if exist(pth, 'dir')
 
+    subject = bids.internal.add_missing_field(subject, modality);
+
     file_list = return_file_list(modality, subject);
 
     for i = 1:numel(file_list)
@@ -223,6 +216,8 @@
 
   if exist(pth, 'dir')
 
+    subject = bids.internal.add_missing_field(subject, modality);
+
     file_list = return_file_list(modality, subject);
 
     for i = 1:numel(file_list)
@@ -257,6 +252,8 @@
 
   if exist(pth, 'dir')
 
+    subject = bids.internal.add_missing_field(subject, modality);
+
     file_list = return_file_list(modality, subject);
 
     for i = 1:numel(file_list)
@@ -315,6 +312,8 @@
 
   if exist(pth, 'dir')
 
+    subject = bids.internal.add_missing_field(subject, modality);
+
     file_list = return_file_list(modality, subject);
 
     for i = 1:numel(file_list)

From c9b3af8b1167baaa7ec516d5ed09ee9da95c9972 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Mon, 15 Feb 2021 10:55:06 +0100
Subject: [PATCH 49/54] refactor matching structure content

---
 +bids/+internal/append_to_layout.m | 12 ------------
 +bids/layout.m                     |  4 +++-
 2 files changed, 3 insertions(+), 13 deletions(-)

diff --git a/+bids/+internal/append_to_layout.m b/+bids/+internal/append_to_layout.m
index a2a88c8e..e2d7d56d 100644
--- a/+bids/+internal/append_to_layout.m
+++ b/+bids/+internal/append_to_layout.m
@@ -47,18 +47,6 @@
 
     [subject.(modality), p] = bids.internal.match_structure_fields(subject.(modality), p);
 
-    %     missing_fields = setxor(fieldnames(subject.(modality)), fieldnames(p));
-
-    %     if ~isempty(missing_fields)
-    %       for iField = 1:numel(missing_fields)
-    %
-    %         p = bids.internal.add_missing_field(p, missing_fields{iField});
-    %
-    %         subject.(modality) = bids.internal.add_missing_field(subject.(modality), ...
-    %                                                missing_fields{iField});
-    %       end
-    %     end
-
   end
 
   subject.(modality) = [subject.(modality) p];
diff --git a/+bids/layout.m b/+bids/layout.m
index 848a0037..ab8c4f26 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -110,9 +110,11 @@
         BIDS.subjects = parse_subject(BIDS.dir, subjects{iSub}, sessions{iSess}, schema);
       else
         new_subject = parse_subject(BIDS.dir, subjects{iSub}, sessions{iSess}, schema);
+       
         [BIDS.subjects, new_subject] = bids.internal.match_structure_fields(BIDS.subjects, ...
                                                                             new_subject);
-        BIDS.subjects(end + 1) = new_subject;
+        % TODO: this can be added to "match_structure_fields"
+        BIDS.subjects(end + 1) = new_subject; 
 
       end
     end

From 3fa4c136ebd074bdc1fe79369a966f813d5b23be Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Mon, 15 Feb 2021 11:26:30 +0100
Subject: [PATCH 50/54] Attempt to fix CI failures

---
 +bids/+internal/append_to_layout.m | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/+bids/+internal/append_to_layout.m b/+bids/+internal/append_to_layout.m
index e2d7d56d..d54b88ac 100644
--- a/+bids/+internal/append_to_layout.m
+++ b/+bids/+internal/append_to_layout.m
@@ -49,7 +49,12 @@
 
   end
 
-  subject.(modality) = [subject.(modality) p];
+  if isempty(subject.(modality))
+      subject.(modality) = p;
+  else
+      subject.(modality)(end+1,1) = p;
+  end
+  
 
 end
 

From 814a9a7c7be1870422b9b2ede8eb4f4f1d162384 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Mon, 15 Feb 2021 11:54:25 +0100
Subject: [PATCH 51/54] fix assert in test

---
 tests/test_return_modality_extensions.m | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_return_modality_extensions.m b/tests/test_return_modality_extensions.m
index cfa14bcc..551adc49 100644
--- a/tests/test_return_modality_extensions.m
+++ b/tests/test_return_modality_extensions.m
@@ -11,6 +11,6 @@
   schema = bids.schema.load_schema();
 
   extensions = bids.internal.return_modality_extensions(schema.datatypes.func(1));
-  assert(isequal(extensions, '(.nii.gz|.nii){1}'));
+  assertEqual(extensions, '(.nii.gz|.nii){1}');
 
 end

From 849dfa9ba5382d710c43b164ad9ba680423972d2 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Mon, 15 Feb 2021 12:34:18 +0100
Subject: [PATCH 52/54] update tests and lint

---
 +bids/+internal/append_to_layout.m            |  5 +-
 +bids/+internal/parse_filename.m              | 36 +++++++++-----
 +bids/layout.m                                |  4 +-
 tests/test_append_to_layout.m                 | 49 ++++++++++---------
 tests/test_bids_query.m                       | 26 +++++-----
 tests/test_file_utils.m                       | 38 +++++++-------
 tests/test_get_metadata.m                     |  4 +-
 tests/test_return_modality_entities.m         |  2 +-
 .../test_return_modality_regular_expression.m |  4 +-
 tests/test_return_modality_suffixes.m         |  2 +-
 tests/test_tsvread.m                          |  4 +-
 tests/test_tsvwrite.m                         | 12 ++---
 12 files changed, 98 insertions(+), 88 deletions(-)

diff --git a/+bids/+internal/append_to_layout.m b/+bids/+internal/append_to_layout.m
index d54b88ac..11c3a76e 100644
--- a/+bids/+internal/append_to_layout.m
+++ b/+bids/+internal/append_to_layout.m
@@ -50,11 +50,10 @@
   end
 
   if isempty(subject.(modality))
-      subject.(modality) = p;
+    subject.(modality) = p;
   else
-      subject.(modality)(end+1,1) = p;
+    subject.(modality)(end + 1, 1) = p;
   end
-  
 
 end
 
diff --git a/+bids/+internal/parse_filename.m b/+bids/+internal/parse_filename.m
index cb81f388..ebaba2bf 100644
--- a/+bids/+internal/parse_filename.m
+++ b/+bids/+internal/parse_filename.m
@@ -1,23 +1,35 @@
 function p = parse_filename(filename, fields)
+  %
   % Split a filename into its building constituents
-  % FORMAT p = bids.internal.parse_filename(filename, fields)
+  %
+  % USAGE::
+  %
+  %   p = bids.internal.parse_filename(filename, fields)
+  %
+  % :param filename: fielname to parse that follows the pattern
+  %                  ``sub-label[_entity-label]*_suffix.extension``
+  % :type  filename: string
+  % :param fields:   cell of strings of the entities to use for parsing
+  % :type  fields:   cell
   %
   % Example:
   %
-  % >> filename = '../sub-16/anat/sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz';
-  % >> bids.internal.parse_filename(filename)
+  %   filename = '../sub-16/anat/sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz';
+  %
+  %   bids.internal.parse_filename(filename)
   %
-  % ans =
+  %   ans =
   %
   %   struct with fields:
   %
-  %     filename: 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz'
-  %         suffix: 'T1w'
-  %          ext: '.nii.gz'
-  %          sub: '16'
-  %          ses: 'mri'
-  %          run: '1'
-  %         acq: 'hd'
+  %     'filename', 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz', ...
+  %     'suffix', 'T1w', ...
+  %     'ext', '.nii.gz', ...
+  %     'entities', struct('sub', '16', ...
+  %                        'ses', 'mri', ...
+  %                        'run', '1', ...
+  %                        'acq', 'hd');
+  %
   % __________________________________________________________________________
 
   % Copyright (C) 2016-2018, Guillaume Flandin, Wellcome Centre for Human Neuroimaging
@@ -26,12 +38,10 @@
   filename = bids.internal.file_utils(filename, 'filename');
 
   % -Identify all the BIDS entity-label pairs present in the filename (delimited by "_")
-  % https://bids-specification.readthedocs.io/en/stable/99-appendices/04-entity-table.html
   [parts, dummy] = regexp(filename, '(?:_)+', 'split', 'match'); %#ok<ASGLU>
   p.filename = filename;
 
   % -Identify the suffix and extension of this file
-  % https://bids-specification.readthedocs.io/en/stable/02-common-principles.html#file-name-structure
   [p.suffix, p.ext] = strtok(parts{end}, '.');
 
   % -Separate the entity from the label for each pair identified above
diff --git a/+bids/layout.m b/+bids/layout.m
index ab8c4f26..4d3f0fcf 100644
--- a/+bids/layout.m
+++ b/+bids/layout.m
@@ -110,11 +110,11 @@
         BIDS.subjects = parse_subject(BIDS.dir, subjects{iSub}, sessions{iSess}, schema);
       else
         new_subject = parse_subject(BIDS.dir, subjects{iSub}, sessions{iSess}, schema);
-       
+
         [BIDS.subjects, new_subject] = bids.internal.match_structure_fields(BIDS.subjects, ...
                                                                             new_subject);
         % TODO: this can be added to "match_structure_fields"
-        BIDS.subjects(end + 1) = new_subject; 
+        BIDS.subjects(end + 1) = new_subject;
 
       end
     end
diff --git a/tests/test_append_to_layout.m b/tests/test_append_to_layout.m
index 5abe9576..492b81e3 100644
--- a/tests/test_append_to_layout.m
+++ b/tests/test_append_to_layout.m
@@ -29,7 +29,7 @@ function test_append_to_layout_basic()
                                             'rec', '', ...
                                             'part', ''));
 
-  assertEqual(subject.anat, expected.anat);
+  assertEqual(subject, expected);
 
 end
 
@@ -46,29 +46,30 @@ function test_append_to_structure_basic_test()
   file = '../sub-16/anat/sub-16_ses-mri_run-1_T1map.nii.gz';
   subject = bids.internal.append_to_layout(file, subject, modality, schema);
 
-  expected(1).anat = struct( ...
-                            'filename', 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz', ...
-                            'suffix', 'T1w', ...
-                            'ext', '.nii.gz', ...
-                            'entities', struct('sub', '16', ...
-                                               'ses', 'mri', ...
-                                               'run', '1', ...
-                                               'acq', 'hd', ...
-                                               'ce', '', ...
-                                               'rec', '', ...
-                                               'part', ''));
-
-  expected(2).anat = struct( ...
-                            'filename', 'sub-16_ses-mri_run-1_T1map.nii.gz', ...
-                            'suffix', 'T1map', ...
-                            'ext', '.nii.gz', ...
-                            'entities', struct('sub', '16', ...
-                                               'ses', 'mri', ...
-                                               'run', '1', ...
-                                               'acq', '', ...
-                                               'ce', '', ...
-                                               'rec', '', ...
-                                               'part', ''));     %#ok<*STRNU>
+  expected.anat(1, 1) = struct( ...
+                               'filename', 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz', ...
+                               'suffix', 'T1w', ...
+                               'ext', '.nii.gz', ...
+                               'entities', struct('sub', '16', ...
+                                                  'ses', 'mri', ...
+                                                  'run', '1', ...
+                                                  'acq', 'hd', ...
+                                                  'ce', '', ...
+                                                  'rec', '', ...
+                                                  'part', ''));
+
+  expected.anat(2, 1) = struct( ...
+                               'filename', 'sub-16_ses-mri_run-1_T1map.nii.gz', ...
+                               'suffix', 'T1map', ...
+                               'ext', '.nii.gz', ...
+                               'entities', struct('sub', '16', ...
+                                                  'ses', 'mri', ...
+                                                  'run', '1', ...
+                                                  'acq', '', ...
+                                                  'ce', '', ...
+                                                  'rec', ''));     %#ok<*STRNU>
+
+  assertEqual(subject, expected);
 
 end
 
diff --git a/tests/test_bids_query.m b/tests/test_bids_query.m
index a8979afc..07e9f287 100644
--- a/tests/test_bids_query.m
+++ b/tests/test_bids_query.m
@@ -70,16 +70,16 @@ function test_bids_query_modalities()
   BIDS = bids.layout(fullfile(pth_bids_example, 'ds007'));
 
   modalities = {'anat', 'func'};
-  assert(isequal(bids.query(BIDS, 'modalities'), modalities));
-  assert(isequal(bids.query(BIDS, 'modalities', 'sub', '01'), modalities));
+  assertEqual(bids.query(BIDS, 'modalities'), modalities);
+  assertEqual(bids.query(BIDS, 'modalities', 'sub', '01'), modalities);
 
   BIDS = bids.layout(fullfile(pth_bids_example, '7t_trt'));
 
   modalities = {'anat', 'fmap', 'func'};
 
-  assert(isequal(bids.query(BIDS, 'modalities'), modalities));
-  assert(isequal(bids.query(BIDS, 'modalities', 'sub', '01'), modalities));
-  assert(isequal(bids.query(BIDS, 'modalities', 'sub', '01', 'ses', '1'), modalities));
+  assertEqual(bids.query(BIDS, 'modalities'), modalities);
+  assertEqual(bids.query(BIDS, 'modalities', 'sub', '01'), modalities);
+  assertEqual(bids.query(BIDS, 'modalities', 'sub', '01', 'ses', '1'), modalities);
 
   % this now fails on octave 4.2.2 but not on Matlab
   %
@@ -94,7 +94,7 @@ function test_bids_query_modalities()
   %
   % when it should return
 
-  % assert(isequal(bids.query(BIDS, 'modalities', 'sub', '01', 'ses', '2'), mods(2:3)));
+  % assertEqual(bids.query(BIDS, 'modalities', 'sub', '01', 'ses', '2'), mods(2:3)));
 
 end
 
@@ -108,13 +108,13 @@ function test_bids_query_basic()
            'stopsignalwithletternaming', ...
            'stopsignalwithmanualresponse', ...
            'stopsignalwithpseudowordnaming'};
-  assert(isequal(bids.query(BIDS, 'tasks'), tasks));
+  assertEqual(bids.query(BIDS, 'tasks'), tasks);
 
   assert(isempty(bids.query(BIDS, 'runs', 'suffix', 'T1w')));
 
   runs = {'01', '02'};
-  assert(isequal(bids.query(BIDS, 'runs'), runs));
-  assert(isequal(bids.query(BIDS, 'runs', 'suffix', 'bold'), runs));
+  assertEqual(bids.query(BIDS, 'runs'), runs);
+  assertEqual(bids.query(BIDS, 'runs', 'suffix', 'bold'), runs);
 
 end
 
@@ -125,7 +125,7 @@ function test_bids_query_subjects()
   BIDS = bids.layout(fullfile(pth_bids_example, 'ds007'));
 
   subjs = arrayfun(@(x) sprintf('%02d', x), 1:20, 'UniformOutput', false);
-  assert(isequal(bids.query(BIDS, 'subjects'), subjs));
+  assertEqual(bids.query(BIDS, 'subjects'), subjs);
 
 end
 
@@ -135,8 +135,8 @@ function test_bids_query_sessions()
 
   BIDS = bids.layout(fullfile(pth_bids_example, 'synthetic'));
   sessions = {'01', '02'};
-  assert(isequal(bids.query(BIDS, 'sessions'), sessions));
-  assert(isequal(bids.query(BIDS, 'sessions', 'sub', '02'), sessions));
+  assertEqual(bids.query(BIDS, 'sessions'), sessions);
+  assertEqual(bids.query(BIDS, 'sessions', 'sub', '02'), sessions);
 
   BIDS = bids.layout(fullfile(pth_bids_example, 'ds007'));
 
@@ -151,6 +151,6 @@ function test_bids_query_suffixes()
   BIDS = bids.layout(fullfile(pth_bids_example, 'ds007'));
 
   suffixes = {'T1w', 'bold', 'events', 'inplaneT2'};
-  assert(isequal(bids.query(BIDS, 'suffixes'), suffixes));
+  assertEqual(bids.query(BIDS, 'suffixes'), suffixes);
 
 end
diff --git a/tests/test_file_utils.m b/tests/test_file_utils.m
index 36a7c533..f7607873 100644
--- a/tests/test_file_utils.m
+++ b/tests/test_file_utils.m
@@ -14,21 +14,21 @@ function test_file_utils_basic()
   str = fullfile('folder', 'filename.extension');
 
   p = bids.internal.file_utils(str, 'path');
-  assert(isequal(p, 'folder'));
+  assertEqual(p, 'folder');
 
   p = bids.internal.file_utils(str, 'fpath');
-  assert(isequal(p, fullfile(pwd, 'folder')));
+  assertEqual(p, fullfile(pwd, 'folder'));
 
   filename = bids.internal.file_utils(str, 'basename');
-  assert(isequal(filename, 'filename'));
+  assertEqual(filename, 'filename');
 
   ext = bids.internal.file_utils(str, 'ext');
-  assert(isequal(ext, 'extension'));
+  assertEqual(ext, 'extension');
 
   str = fullfile('folder', 'subfolder', '..', 'filename.extension');
   cpath = bids.internal.file_utils(str, 'cpath');
-  assert(isequal(cpath, ...
-                 fullfile(pwd, 'folder', 'filename.extension')));
+  assertEqual(cpath, ...
+              fullfile(pwd, 'folder', 'filename.extension'));
 
   %% test to set certain part of a filename
   % {'path', 'basename', 'ext', 'filename', 'prefix', 'suffix'}
@@ -36,27 +36,27 @@ function test_file_utils_basic()
   str = fullfile('folder', 'filename.extension');
 
   new_str = bids.internal.file_utils(str, 'ext', 'newext');
-  assert(isequal(new_str, fullfile('folder', 'filename.newext')));
+  assertEqual(new_str, fullfile('folder', 'filename.newext'));
 
   new_str = bids.internal.file_utils(str, 'basename', 'new_name');
-  assert(isequal(new_str, fullfile('folder', 'new_name.extension')));
+  assertEqual(new_str, fullfile('folder', 'new_name.extension'));
 
   new_str = bids.internal.file_utils(str, 'filename', 'new_name.newext');
-  assert(isequal(new_str, fullfile('folder', 'new_name.newext')));
+  assertEqual(new_str, fullfile('folder', 'new_name.newext'));
 
   new_str = bids.internal.file_utils(str, 'prefix', 'pre_');
-  assert(isequal(new_str, fullfile('folder', 'pre_filename.extension')));
+  assertEqual(new_str, fullfile('folder', 'pre_filename.extension'));
 
   new_str = bids.internal.file_utils(str, 'suffix', '_suffix');
-  assert(isequal(new_str, fullfile('folder', 'filename_suffix.extension')));
+  assertEqual(new_str, fullfile('folder', 'filename_suffix.extension'));
 
   new_str = bids.internal.file_utils(str, 'path', fullfile(pwd, 'new_folder'));
-  assert(isequal(new_str, fullfile(pwd, 'new_folder', 'filename.extension')));
+  assertEqual(new_str, fullfile(pwd, 'new_folder', 'filename.extension'));
 
   new_str = bids.internal.file_utils(str, ...
                                      'prefix', 'pre_', ...
                                      'suffix', '_suffix');
-  assert(isequal(new_str, fullfile('folder', 'pre_filename_suffix.extension')));
+  assertEqual(new_str, fullfile('folder', 'pre_filename_suffix.extension'));
 
   %% test to list files
 
@@ -65,29 +65,29 @@ function test_file_utils_basic()
   file = bids.internal.file_utils('List', ...
                                   test_directory, ...
                                   '^test_file_utils.m$');
-  assert(isequal(file, 'test_file_utils.m'));
+  assertEqual(file, 'test_file_utils.m');
 
   file = bids.internal.file_utils('List', ...
                                   test_directory, ...
                                   '^.*.md$');
-  assert(isequal(file, 'README.md'));
+  assertEqual(file, 'README.md');
 
   directory = bids.internal.file_utils('List', ...
                                        test_directory, ...
                                        'dir', ...
                                        '^data$');
-  assert(isequal(directory, 'data'));
+  assertEqual(directory, 'data');
 
   fp_file = bids.internal.file_utils('FPList', ...
                                      test_directory, ...
                                      '^test_file_utils.m$');
-  assert(isequal(fp_file, [mfilename('fullpath') '.m']));
+  assertEqual(fp_file, [mfilename('fullpath') '.m']);
 
   fp_directory = bids.internal.file_utils('FPList', ...
                                           test_directory, ...
                                           'dir', ...
                                           '^data$');
-  assert(isequal(fp_directory, ...
-                 fullfile(test_directory, 'data')));
+  assertEqual(fp_directory, ...
+              fullfile(test_directory, 'data'));
 
 end
diff --git a/tests/test_get_metadata.m b/tests/test_get_metadata.m
index 45d502b1..383c3860 100644
--- a/tests/test_get_metadata.m
+++ b/tests/test_get_metadata.m
@@ -52,7 +52,7 @@ function test_get_metadata_basic()
 
   %% test anat metadata subject 01
   metadata = bids.query(BIDS, 'metadata', 'sub', '01', 'suffix', 'T1w');
-  assert(metadata.FlipAngle == anat_sub_01.FlipAngle);
-  assert(strcmp(metadata.Manufacturer, anat_sub_01.Manufacturer));
+  assertEqual(metadata.FlipAngle, anat_sub_01.FlipAngle);
+  assertEqual(metadata.Manufacturer, anat_sub_01.Manufacturer);
 
 end
diff --git a/tests/test_return_modality_entities.m b/tests/test_return_modality_entities.m
index 3a0e3930..d9a38cd6 100644
--- a/tests/test_return_modality_entities.m
+++ b/tests/test_return_modality_entities.m
@@ -14,6 +14,6 @@
 
   expected_output = {'sub', 'ses', 'task', 'acq', 'ce', 'rec', 'dir', 'run', 'echo', 'part'};
 
-  assert(isequal(entities, expected_output));
+  assertEqual(entities, expected_output);
 
 end
diff --git a/tests/test_return_modality_regular_expression.m b/tests/test_return_modality_regular_expression.m
index 131e76ca..668451b2 100644
--- a/tests/test_return_modality_regular_expression.m
+++ b/tests/test_return_modality_regular_expression.m
@@ -16,12 +16,12 @@
                          '_(T1w|T2w|PDw|T2starw|FLAIR|inplaneT1|inplaneT2|PDT2|angio){1}', ...
                          '(.nii.gz|.nii){1}$'];
 
-  assert(isequal(regular_expression, expected_expression));
+  assertEqual(regular_expression, expected_expression);
 
   data_dir = fullfile(fileparts(mfilename('fullpath')), 'data', 'MoAEpilot', 'sub-01', 'anat');
   subject_name = 'sub-01';
   file = bids.internal.file_utils('List', data_dir, sprintf(expected_expression, subject_name));
 
-  assert(isequal(file, 'sub-01_T1w.nii.gz'));
+  assertEqual(file, 'sub-01_T1w.nii.gz');
 
 end
diff --git a/tests/test_return_modality_suffixes.m b/tests/test_return_modality_suffixes.m
index 2018bfcc..19212185 100644
--- a/tests/test_return_modality_suffixes.m
+++ b/tests/test_return_modality_suffixes.m
@@ -11,6 +11,6 @@
   schema = bids.schema.load_schema();
 
   suffixes = bids.internal.return_modality_suffixes(schema.datatypes.func(1));
-  assert(isequal(suffixes, '_(bold|cbv|sbref){1}'));
+  assertEqual(suffixes, '_(bold|cbv|sbref){1}');
 
 end
diff --git a/tests/test_tsvread.m b/tests/test_tsvread.m
index fec8e190..d5ca578a 100644
--- a/tests/test_tsvread.m
+++ b/tests/test_tsvread.m
@@ -30,13 +30,13 @@ function test_tsvread_basic()
   %% test tsvread on tsv file
   tsv_file = fullfile(pth, 'sub-01', 'func', 'sub-01_task-auditory_events.tsv');
   output = bids.util.tsvread(tsv_file);
-  assert(isequal(output.onset', events.onset));
+  assertEqual(output.onset', events.onset);
 
   %% test tsvread on zipped tsv file
   output = bids.util.tsvread(fullfile( ...
                                       fileparts(mfilename('fullpath')), ....
                                       'data', ...
                                       'sub-01_task-auditory_events.tsv.gz'));
-  assert(isequal(output.onset', events.onset));
+  assertEqual(output.onset', events.onset);
 
 end
diff --git a/tests/test_tsvwrite.m b/tests/test_tsvwrite.m
index 3285000b..751685eb 100644
--- a/tests/test_tsvwrite.m
+++ b/tests/test_tsvwrite.m
@@ -51,19 +51,19 @@ function test_tsvwrite_basic()
   C = textscan(FID, '%s%s%s%s%s', 'Delimiter', '\t', 'EndOfLine', '\n');
 
   % check header
-  assert(isequal(C{4}{1}, 'speed'));
+  assertEqual(C{4}{1}, 'speed');
 
   % check that empty values are entered as NaN: logFile(1,1).speed
-  assert(isequal(C{4}{2}, 'n/a'));
+  assertEqual(C{4}{2}, 'n/a');
 
   % check that missing fields are entered as NaN: logFile(2,1).speed
-  assert(isequal(C{4}{3}, 'n/a'));
+  assertEqual(C{4}{3}, 'n/a');
 
   % check that NaN are written as : logFile(2,1).onset
-  assert(isequal(C{1}{3}, 'n/a')); %
+  assertEqual(C{1}{3}, 'n/a'); %
 
   % check values entered properly: logFile(2,1).is_fixation
-  assert(isequal(str2double(C{5}{3}), 3));
+  assertEqual(str2double(C{5}{3}), 3);
 
   %% test tsvread on tsv file using cell input
   % TO DO?
@@ -79,6 +79,6 @@ function test_tsvwrite_basic()
 
   FID = fopen(tsv_file, 'r');
   C = textscan(FID, '%s%s', 'Delimiter', '\t', 'EndOfLine', '\n');
-  assert(isequal(C{1}{2}, 'n/a')); %
+  assertEqual(C{1}{2}, 'n/a'); %
 
 end

From e8ea5722d654d3b95ce9f91e3758cc431d2ee87e Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Mon, 15 Feb 2021 13:08:53 +0100
Subject: [PATCH 53/54] update test to new schema

---
 tests/test_return_modality_regular_expression.m | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/test_return_modality_regular_expression.m b/tests/test_return_modality_regular_expression.m
index 668451b2..b3984f5b 100644
--- a/tests/test_return_modality_regular_expression.m
+++ b/tests/test_return_modality_regular_expression.m
@@ -13,7 +13,9 @@
   regular_expression = bids.internal.return_modality_regular_expression(schema.datatypes.anat(1));
 
   expected_expression = ['^%s.*', ...
-                         '_(T1w|T2w|PDw|T2starw|FLAIR|inplaneT1|inplaneT2|PDT2|angio){1}', ...
+                         '_(T1w|T2w|PDw|T2starw|FLAIR|inplaneT1|inplaneT2|PDT2|angio|', ...
+                         'T2star|FLASH|PD', ... % deprecated suffixes
+                         '){1}', ...
                          '(.nii.gz|.nii){1}$'];
 
   assertEqual(regular_expression, expected_expression);

From e318a430a8b61f5e53ac1238ce8a5d3051f64575 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Mon, 15 Feb 2021 13:27:15 +0100
Subject: [PATCH 54/54] update test for parsing and querying derivatives

---
 tests/test_layout_derivatives.m | 36 ++++++++++++++++++++++-----------
 tests/test_load_schema.m        |  4 +++-
 tests/test_parse_filename.m     | 12 -----------
 3 files changed, 27 insertions(+), 25 deletions(-)

diff --git a/tests/test_layout_derivatives.m b/tests/test_layout_derivatives.m
index 5c74ffce..ae6923ad 100644
--- a/tests/test_layout_derivatives.m
+++ b/tests/test_layout_derivatives.m
@@ -19,19 +19,31 @@ function test_layout_schemaless()
   assertEqual(bids.query(BIDS, 'modalities'), modalities);
 
   % Those fail for now
-  %   data = bids.query(BIDS, 'data', ...
-  %                     'sub', '10', ...
-  %                     'modality', 'func', ...
-  %                     'suffix', 'bold', ...
-  %                     'run', '1', ...
-  %                     'res', '2');
-  %   data = bids.query(BIDS, 'data', ...
-  %                     'sub', '10', ...
-  %                     'modality', 'func', ...
-  %                     'suffix', 'bold', ...
-  %                     'run', '1', ...
-  %                     'space', 'MNI152NLin6Asym');
+  data = bids.query(BIDS, 'data', ...
+                    'sub', '10', ...
+                    'modality', 'func', ...
+                    'suffix', 'bold', ...
+                    'run', '1', ...
+                    'res', '2');
+
+  basename = bids.internal.file_utils(data, 'basename');
+  assertEqual(basename, {
+                         ['sub-10_task-balloonanalogrisktask_run-1', ...
+                          '_space-MNI152NLin2009cAsym_res-2_desc-preproc_bold.nii']
+                        });
 
+  data = bids.query(BIDS, 'data', ...
+                    'sub', '10', ...
+                    'modality', 'func', ...
+                    'suffix', 'bold', ...
+                    'run', '1', ...
+                    'space', 'MNI152NLin6Asym');
+
+  basename = bids.internal.file_utils(data, 'basename');
+  assertEqual(basename, {
+                         ['sub-10_task-balloonanalogrisktask_run-1', ...
+                          '_space-MNI152NLin6Asym_desc-smoothAROMAnonaggr_bold.nii']
+                        });
 end
 
 function test_layout_nested_derivatives()
diff --git a/tests/test_load_schema.m b/tests/test_load_schema.m
index 941d916d..6b267a06 100644
--- a/tests/test_load_schema.m
+++ b/tests/test_load_schema.m
@@ -15,9 +15,11 @@ function test_load_schema_path()
   assert(isfield(schema, 'base'));
   assert(isfield(schema, 'subfolder_1'));
   assert(isfield(schema.subfolder_1, 'sub'));
+  assert(~isfield(schema, 'subfolder_4'));
+
+  % some recursive aspects are not implemented yet
   %     assert(isfield(schema.subfolder_2, 'subfolder_3'));
   %     assert(isfield(schema.subfolder_2.subfolder_3, 'sub'));
-  assert(~isfield(schema, 'subfolder_4'));
 
 end
 
diff --git a/tests/test_parse_filename.m b/tests/test_parse_filename.m
index 1630de5e..36ebdfe2 100644
--- a/tests/test_parse_filename.m
+++ b/tests/test_parse_filename.m
@@ -20,18 +20,6 @@ function test_parse_filename_basic()
                                        'run', '1', ...
                                        'acq', 'hd'));
 
-  %     expected = struct( ...
-  %         'filename', 'sub-16_ses-mri_run-1_acq-hd_T1w.nii.gz', ...
-  %         'suffix', 'T1w', ...
-  %         'ext', '.nii.gz', ...
-  %         'sub', '16', ...
-  %         'ses', 'mri', ...
-  %         'run', '1', ...
-  %         'acq', 'hd', ...
-  %         'ce', '', ...
-  %         'rec', '', ...
-  %         'part', '');
-
   assertEqual(output, expected);
 
 end