diff --git a/run_pipeline.m b/run_pipeline.m index 61373f4..b037148 100644 --- a/run_pipeline.m +++ b/run_pipeline.m @@ -40,19 +40,29 @@ function run_pipeline(dsnumber, varargin) eeg_logdir = fullfile(opt.logdir, 'eeg_logs'); log_file = fullfile(opt.logdir, 'matlab_log'); -codeDir = fullfile(opt.logdir, "code"); +codeDir = fullfile(opt.outputdir, "code"); if strcmp(opt.modeval, "new") % create output directories if opt.verbose fprintf('Output dir: %s\n', opt.outputdir); end if exist(opt.outputdir, 'dir') + if exist(codeDir, 'dir') && exist(fullfile(codeDir, 'nemar.json')) + if ~exist(fullfile(pipelineroot, 'temp-nemar-json'), 'dir') + mkdir(fullfile(pipelineroot, 'temp-nemar-json')); + end + [status, msg] = copyfile(fullfile(codeDir, 'nemar.json'), fullfile(pipelineroot, 'temp-nemar-json', [dsnumber '_nemar.json'])) + if status ~= 1 + error('Error backing up nemar.json file'); + end + end rmdir(opt.outputdir, 's'); end status = mkdir(opt.outputdir); if ~status error('Could not create output directory'); else + status = copyfile(fullfile(pipelineroot, 'temp-nemar-json', [dsnumber '_nemar.json'], fullfile(codeDir, 'nemar.json'))); disp("Output directory created"); end diff --git a/web/aggregate_status.py b/web/aggregate_status.py index b88c2bf..4f5ad18 100755 --- a/web/aggregate_status.py +++ b/web/aggregate_status.py @@ -8,6 +8,7 @@ import argparse import subprocess import re +import json raw_dir = "/data/qumulo/openneuro" processed_dir = "/data/qumulo/openneuro/processed" @@ -38,6 +39,31 @@ def get_known_errors(matlab_log, batcherr_log): return errors +def write_nemar_json(df, is_processed): + path = os.path.join(processed_dir, df['dsnumber'][0]) + code_dir = path + '/code' + if not os.path.exists(code_dir): + os.mkdir(code_dir) + + status_file = code_dir + "/nemar.json" + if os.path.exists(status_file): + with open(status_file, 'r') as fin: + status = json.load(fin) + else: + status = { + "error": "", + "warning": "", + "has_visualization": None, + } + status["has_visualization"] = is_processed + with open(status_file, 'w') as fout: + json.dump(status, fout, indent=4) + try: + os.chmod(status_file, 0o664) # add write permission to group + except: + print(f'Cannot change permission for {status_file}') + + def append_modality(df): ''' Get the modality of the dataset @@ -103,6 +129,8 @@ def append_debug(df, processing): except: print(f'Cannot change permission for {debug_note}') df['debug_note'] = notes + # write nemar.json with processed status based on value of notes + write_nemar_json(df, is_processed=(notes == "ok")) # manual debug note manual_debug_note = os.path.join(path, "logs", "debug", "manual_debug_note") @@ -300,4 +328,4 @@ def get_pipeline_status(dsnumbers): final_df.to_html(out, index=False, na_rep="") logfile.write('writing html\n') - logfile.close() \ No newline at end of file + logfile.close() diff --git a/web/pipeline_status.php b/web/pipeline_status.php index 15641f6..3124ae3 100644 --- a/web/pipeline_status.php +++ b/web/pipeline_status.php @@ -75,6 +75,11 @@ +