From cac015027f5bf9878fd6af1e9b5e5d248d58de01 Mon Sep 17 00:00:00 2001
From: Dung Truong <dt.young112@gmail.com>
Date: Tue, 3 Oct 2023 14:17:49 -0700
Subject: [PATCH] nemar.json file

---
 run_pipeline.m          | 12 +++++++++++-
 web/aggregate_status.py | 30 +++++++++++++++++++++++++++++-
 web/pipeline_status.php |  6 ++++++
 3 files changed, 46 insertions(+), 2 deletions(-)

diff --git a/run_pipeline.m b/run_pipeline.m
index 61373f4..b037148 100644
--- a/run_pipeline.m
+++ b/run_pipeline.m
@@ -40,19 +40,29 @@ function run_pipeline(dsnumber, varargin)
 
 eeg_logdir = fullfile(opt.logdir, 'eeg_logs');
 log_file = fullfile(opt.logdir, 'matlab_log');
-codeDir = fullfile(opt.logdir, "code");
+codeDir = fullfile(opt.outputdir, "code");
 if strcmp(opt.modeval, "new")
     % create output directories
     if opt.verbose
         fprintf('Output dir: %s\n', opt.outputdir);
     end
     if exist(opt.outputdir, 'dir')
+	if exist(codeDir, 'dir') && exist(fullfile(codeDir, 'nemar.json'))
+	    if ~exist(fullfile(pipelineroot, 'temp-nemar-json'), 'dir')
+		mkdir(fullfile(pipelineroot, 'temp-nemar-json'));
+	    end
+	    [status, msg] = copyfile(fullfile(codeDir, 'nemar.json'), fullfile(pipelineroot, 'temp-nemar-json', [dsnumber '_nemar.json']))
+	    if status ~= 1
+		error('Error backing up nemar.json file');
+	    end
+	end
         rmdir(opt.outputdir, 's');
     end
     status = mkdir(opt.outputdir);
     if ~status
         error('Could not create output directory');
     else
+	status = copyfile(fullfile(pipelineroot, 'temp-nemar-json', [dsnumber '_nemar.json'], fullfile(codeDir, 'nemar.json')));
         disp("Output directory created");
     end
 
diff --git a/web/aggregate_status.py b/web/aggregate_status.py
index b88c2bf..4f5ad18 100755
--- a/web/aggregate_status.py
+++ b/web/aggregate_status.py
@@ -8,6 +8,7 @@
 import argparse
 import subprocess
 import re
+import json
 
 raw_dir = "/data/qumulo/openneuro"
 processed_dir = "/data/qumulo/openneuro/processed"
@@ -38,6 +39,31 @@ def get_known_errors(matlab_log, batcherr_log):
     
     return errors
 
+def write_nemar_json(df, is_processed):
+    path = os.path.join(processed_dir, df['dsnumber'][0]) 
+    code_dir = path + '/code'
+    if not os.path.exists(code_dir):
+        os.mkdir(code_dir)
+
+    status_file = code_dir + "/nemar.json"
+    if os.path.exists(status_file):
+        with open(status_file, 'r') as fin:
+            status = json.load(fin)
+    else:  
+        status = {
+            "error": "",
+            "warning": "",
+            "has_visualization": None,
+        }
+    status["has_visualization"] = is_processed
+    with open(status_file, 'w') as fout:
+        json.dump(status, fout, indent=4)
+    try:
+        os.chmod(status_file, 0o664) # add write permission to group
+    except:
+        print(f'Cannot change permission for {status_file}')
+
+
 def append_modality(df):
     '''
     Get the modality of the dataset
@@ -103,6 +129,8 @@ def append_debug(df, processing):
             except:
                 print(f'Cannot change permission for {debug_note}')
         df['debug_note'] = notes
+        # write nemar.json with processed status based on value of notes
+        write_nemar_json(df, is_processed=(notes == "ok"))
 
         # manual debug note
         manual_debug_note = os.path.join(path, "logs", "debug", "manual_debug_note")
@@ -300,4 +328,4 @@ def get_pipeline_status(dsnumbers):
         final_df.to_html(out, index=False, na_rep="")
         logfile.write('writing html\n')
 
-    logfile.close()
\ No newline at end of file
+    logfile.close()
diff --git a/web/pipeline_status.php b/web/pipeline_status.php
index 15641f6..3124ae3 100644
--- a/web/pipeline_status.php
+++ b/web/pipeline_status.php
@@ -75,6 +75,11 @@
                             <input class="form-control" id="sbatch" name="sbatch"></input>
 			    <button type="button" onclick="copyText('sbatch')">Copy path</button>
                         </div>
+                        <div class="form-group">
+                            <label for="doc" class="col-form-label">nemar.json:</label>
+                            <input class="form-control" id="nemarjson" name="nemarjson"></input>
+			    <button type="button" onclick="copyText('nemarjson')">Copy path</button>
+                        </div>
                         <button type="button" class="btn btn-secondary" data-dismiss="modal">Cancel</button>
                     </form>
 		</div>
@@ -179,6 +184,7 @@
 		modal.find('#indLogDir').val('/expanse/projects/nemar/openneuro/processed/' + dsnumber + '/logs/eeg_logs');
 		modal.find('#note').val('/expanse/projects/nemar/openneuro/processed/' + dsnumber + '/logs/debug/manual_debug_note');
 		modal.find('#sbatch').val('/expanse/projects/nemar/openneuro/processed/logs/' + dsnumber + 'sbatch');
+		modal.find('#nemarjson').val('/expanse/projects/nemar/openneuro/processed/' + dsnumber + '/code/nemar.json');
 	    });
 	    $('#dsMatlabLogs').on('show.bs.modal', function (event) {
 		var clicked = $(event.relatedTarget);