WIP: introducing OmexExecSummary to unify reporting of errors

vcell-cli/src/main/java/org/vcell/cli/CLIStandalone.java

@@ -2,6 +2,7 @@
 
 import org.vcell.cli.biosimulation.BiosimulationsCommand;
 import org.vcell.cli.run.ExecuteCommand;
+import org.vcell.cli.run.ExecuteOmexCommand;
 import org.vcell.cli.sbml.ModelCommand;
 import org.vcell.cli.vcml.*;
 
@@ -22,6 +23,7 @@
         ExportOmexBatchCommand.class,
         ImportOmexCommand.class,
         ImportOmexBatchCommand.class,
+        ExecuteOmexCommand.class,
         ExecuteCommand.class,
         VersionCommand.class,
         ModelCommand.class,

vcell-cli/src/main/java/org/vcell/cli/CliTracer.java

@@ -0,0 +1,31 @@
+package org.vcell.cli;
+
+import org.vcell.trace.Tracer;
+
+public class CliTracer implements CLIRecordable {
+
+    @Override
+    public void writeDetailedErrorList(Exception e, String message) {
+        Tracer.failure(e, "writeDetailedErrorList(): "+message);
+    }
+    @Override
+    public void writeFullSuccessList(String message) {
+        Tracer.success("writeFullSuccessList(): " + message);
+    }
+    @Override
+    public void writeErrorList(Exception e, String message) {
+        Tracer.failure(e, "writeErrorList(): " + message);
+    }
+    @Override
+    public void writeDetailedResultList(String message) {
+        Tracer.log("writeDetailedResultList(): "+message);
+    }
+    @Override
+    public void writeSpatialList(String message) {
+        Tracer.log("writeSpatialList(): "+message);
+    }
+    @Override
+    public void writeImportErrorList(Exception e, String message) {
+        Tracer.failure(e, "writeImportErrorList(): " + message);
+    }
+}

vcell-cli/src/main/java/org/vcell/cli/run/ExecuteImpl.java

@@ -4,7 +4,6 @@
 import cbit.vcell.solver.ode.ODESolverResultSet;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.jlibsedml.XMLException;
 import org.vcell.cli.CLIRecordable;
 import org.vcell.cli.PythonStreamException;
 import org.vcell.cli.exceptions.ExecutionException;

vcell-cli/src/main/java/org/vcell/cli/run/ExecuteOmexCommand.java

@@ -0,0 +1,127 @@
+package org.vcell.cli.run;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.vcell.cli.CLIPythonManager;
+import org.vcell.cli.CLIRecordable;
+import org.vcell.cli.CliTracer;
+import org.vcell.cli.testsupport.OmexExecSummary;
+import org.vcell.cli.testsupport.OmexTestingDatabase;
+import org.vcell.trace.Tracer;
+import org.vcell.util.FileUtils;
+import org.vcell.util.exe.Executable;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.Date;
+import java.util.concurrent.Callable;
+
+@Command(name = "execute-omex", description = "run .vcml or .omex files via Python API")
+public class ExecuteOmexCommand implements Callable<Integer> {
+
+    private final static Logger logger = LogManager.getLogger(ExecuteOmexCommand.class);
+
+    @Option(names = { "-i", "--inputFilePath" }, required = true, description = "Path to a COMBINE/OMEX archive file")
+    private File inputFilePath;
+
+    @Option(names = { "-o", "--outputFilePath"}, required = true, description = "Directory to save outputs")
+    private File outputFilePath;
+
+    @Option(names = {"--keepTempFiles"}, defaultValue = "false")
+    private boolean bKeepTempFiles = false;
+
+    @Option(names = {"--exactMatchOnly"}, defaultValue = "false")
+    private boolean bExactMatchOnly = false;
+
+    @Option(names = "--small-mesh", defaultValue = "false", description = "force spatial simulations to have a very small mesh to make execution faster")
+    private boolean bSmallMeshOverride = false;
+
+    @Option(names = {"--encapsulateOutput"}, defaultValue = "true", description =
+        "VCell will encapsulate output results in a sub directory when executing with a single input archive; has no effect when providing an input directory")
+    private boolean bEncapsulateOutput = true;
+
+    @Option(names = {"--timeout_ms"}, defaultValue = "600000", description = "executable wall clock timeout in milliseconds")
+    // timeout for compiled solver running long jobs; default 10 minutes
+    private long EXECUTABLE_MAX_WALLCLOCK_MILLIS = 10 * 60 * 1000;
+
+    @Option(names = {"-h", "--help"}, description = "show this help message and exit", usageHelp = true)
+    private boolean help = false;
+
+    @Option(names = {"-d", "--debug"}, description = "full application debug mode")
+    private boolean bDebug = false;
+
+    @Option(names = {"-q", "--quiet"}, description = "suppress all console output")
+    private boolean bQuiet = false;
+
+
+    public Integer call() {
+
+        CLIRecordable cliTracer = new CliTracer();
+        try {
+            if (bDebug && bQuiet) {
+                System.err.println("cannot specify both debug and quiet, try --help for usage");
+                return 1;
+            }
+            Level logLevel;
+            if (!bQuiet && bDebug) {
+                logLevel = Level.DEBUG;
+            } else if (bQuiet) {
+                logLevel = Level.OFF;
+            } else {
+                logLevel = logger.getLevel();
+            }
+
+            if (this.inputFilePath.exists() && !this.inputFilePath.isFile()) {
+                System.err.println("Input path must be a file");
+                return 1;
+            }
+            if (this.outputFilePath.exists() && !this.outputFilePath.isDirectory()) {
+                System.err.println("Output path must be a directory");
+                return 1;
+            }
+
+            LoggerContext config = (LoggerContext)(LogManager.getContext(false));
+            config.getConfiguration().getLoggerConfig(LogManager.getLogger("org.vcell").getName()).setLevel(logLevel);
+            config.getConfiguration().getLoggerConfig(LogManager.getLogger("cbit").getName()).setLevel(logLevel);
+            config.updateLoggers();
+
+
+            CLIPythonManager.getInstance().instantiatePythonProcess();
+
+            Executable.setGlobalTimeoutMS(EXECUTABLE_MAX_WALLCLOCK_MILLIS);
+            logger.info("Beginning execution");
+            File tmpDir = Files.createTempDirectory("VCell_CLI_" + Long.toHexString(new Date().getTime())).toFile();
+
+            Tracer.clearTraceEvents();
+            ExecuteImpl.singleMode(inputFilePath, tmpDir, cliTracer, bKeepTempFiles, bExactMatchOnly,
+                            bEncapsulateOutput, bSmallMeshOverride);
+            CLIPythonManager.getInstance().closePythonProcess();
+            // WARNING: Python needs re-instantiation once the above line is called!
+            FileUtils.copyDirectoryContents(tmpDir, outputFilePath, true, null);
+            OmexExecSummary omexExecSummary = new OmexExecSummary();
+            omexExecSummary.file_path = String.valueOf(inputFilePath);
+            omexExecSummary.status = OmexExecSummary.ActualStatus.PASSED;
+            new ObjectMapper().writeValue(new File(outputFilePath, "exec_summary.json"), omexExecSummary);
+            new ObjectMapper().writeValue(new File(outputFilePath, "logs.json"), Tracer.getTraceEvents());
+            return 0;
+        } catch (Exception e) { ///TODO: Break apart into specific exceptions to maximize logging.
+            LogManager.getLogger(this.getClass()).error(e.getMessage(), e);
+            OmexExecSummary omexExecSummary = OmexTestingDatabase.summarize(inputFilePath,e,Tracer.getErrors());
+            try {
+                new ObjectMapper().writeValue(new File(outputFilePath, "exec_summary.json"), omexExecSummary);
+                new ObjectMapper().writeValue(new File(outputFilePath, "logs.json"), Tracer.getTraceEvents());
+            } catch (IOException ex) {
+                logger.error("Failed to write exec summary and structured logs", ex);
+            }
+            return 1;
+        } finally {
+            logger.debug("Completed all execution");
+        }
+    }
+}

vcell-cli/src/main/java/org/vcell/cli/run/ExecutionJob.java

@@ -6,19 +6,17 @@
 import org.vcell.cli.PythonStreamException;
 import org.vcell.cli.exceptions.ExecutionException;
 import org.vcell.cli.run.hdf5.BiosimulationsHdf5Writer;
+import org.vcell.cli.run.hdf5.BiosimulationsHdfWriterException;
 import org.vcell.cli.run.hdf5.HDF5ExecutionResults;
 import org.vcell.sedml.log.BiosimulationLog;
 import org.vcell.util.FileUtils;
 
 import java.io.File;
 import java.io.IOException;
-import java.math.BigDecimal;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.List;
 
-import org.vcell.cli.run.hdf5.BiosimulationsHdfWriterException;
-
 /**
  * Contains the code necessary to execute an Omex archive in VCell
  */

vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexExecSummary.java

@@ -0,0 +1,24 @@
+package org.vcell.cli.testsupport;
+
+public class OmexExecSummary {
+
+    public enum ActualStatus {
+        PASSED,
+        FAILED
+    }
+
+    public String file_path;
+    public ActualStatus status;
+    public FailureType failure_type;
+    public String failure_desc;
+
+    @Override
+    public String toString() {
+        return "OmexExecSummary{" +
+                ", file_path='" + file_path + '\'' +
+                ", status=" + status +
+                ", failure_type=" + failure_type +
+                ", failure_desc="+((failure_desc!=null)?('\'' + failure_desc + '\''):null) +
+                '}';
+    }
+}

vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestingDatabase.java

@@ -1,12 +1,18 @@
 package org.vcell.cli.testsupport;
 
+import cbit.vcell.mapping.MappingException;
 import com.fasterxml.jackson.databind.MappingIterator;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import org.vcell.sbml.vcell.SBMLImportException;
+import org.vcell.trace.Span;
+import org.vcell.trace.TraceEvent;
+import org.vcell.trace.Tracer;
 
 import java.io.*;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.NoSuchElementException;
 
 public class OmexTestingDatabase {
 
@@ -29,10 +35,30 @@ public enum TestCollection {
         }
     }
 
-    public static OmexTestCase queryOmexTestCase(List<OmexTestCase> omexTestCases, String path) throws NoSuchElementException {
-        return omexTestCases.stream().filter(tc -> tc.matchFileSuffix(path)).findFirst().orElseThrow();
+    public static List<OmexTestCase> queryOmexTestCase(List<OmexTestCase> omexTestCases, String path) {
+        return omexTestCases.stream().filter(tc -> tc.matchFileSuffix(path)).toList();
     }
 
+    public String generateReport(List<OmexTestCase> omexTestCases, List<OmexExecSummary> execSummaries) {
+        StringBuilder report = new StringBuilder();
+        for (OmexTestCase testCase : omexTestCases) {
+            report.append(testCase.toString()).append("\n");
+        }
+        return report.toString();
+    }
+
+    public static List<OmexExecSummary> loadOmexExecSummaries(String execSummariesNdjson) throws IOException {
+        List<OmexExecSummary> execSummaries = new ArrayList<>();
+        ObjectMapper objectMapper = new ObjectMapper();
+        try (MappingIterator<OmexExecSummary> it = objectMapper.readerFor(OmexExecSummary.class).readValues(execSummariesNdjson)) {
+            while (it.hasNext()) {
+                execSummaries.add(it.next());
+            }
+        }
+        return execSummaries;
+    }
+
+
     // read a newline-delimited json file into a list of OmexTextCase objects
     public static List<OmexTestCase> loadOmexTestCases() throws IOException {
         List<OmexTestCase> testCases = new ArrayList<>();
@@ -53,4 +79,65 @@ public static List<OmexTestCase> loadOmexTestCases() throws IOException {
         return testCases;
     }
 
+    public static OmexExecSummary summarize(File inputFilePath, Exception exception, List<TraceEvent> errorEvents) {
+        OmexExecSummary execSummary = new OmexExecSummary();
+        execSummary.file_path = inputFilePath.toString();
+        execSummary.status = OmexExecSummary.ActualStatus.PASSED;
+        if (exception != null || !errorEvents.isEmpty()) {
+            execSummary.failure_type = determineFault(exception, errorEvents);
+            execSummary.failure_desc = null;
+            if (exception != null) {
+                execSummary.failure_desc = exception.getMessage();
+            }
+            if (!errorEvents.isEmpty()) {
+                execSummary.failure_desc = errorEvents.get(0).message + " " + errorEvents.get(0).exception;
+            }
+        }
+        return execSummary;
+    }
+
+    public static FailureType determineFault(Exception caughtException, List<TraceEvent> errorEvents){ // Throwable because Assertion Error
+        String errorMessage = "";
+        if (caughtException != null) {
+            errorMessage = caughtException.getMessage();
+        }
+
+        if (errorMessage.contains("refers to either a non-existent model")) { //"refers to either a non-existent model (invalid SED-ML) or to another model with changes (not supported yet)"
+            return FailureType.SEDML_UNSUPPORTED_MODEL_REFERENCE;
+        } else if (errorMessage.contains("System IO encountered a fatal error")){
+            Throwable subException = caughtException.getCause();
+            //String subMessage = (subException == null) ? "" : subException.getMessage();
+            if (subException instanceof FileAlreadyExistsException){
+                return FailureType.HDF5_FILE_ALREADY_EXISTS;
+            }
+        } else if (errorMessage.contains("error while processing outputs: null")){
+            Throwable subException = caughtException.getCause();
+            if (subException instanceof ArrayIndexOutOfBoundsException){
+                return FailureType.ARRAY_INDEX_OUT_OF_BOUNDS;
+            }
+        } else if (errorMessage.contains("nconsistent unit system in SBML model") ||
+                errorMessage.contains("ust be of type")){
+            return FailureType.SEDML_ERRONEOUS_UNIT_SYSTEM;
+        } else if (errorMessage.contains("There are no SED-MLs in the archive to execute")) {
+            return FailureType.SEDML_NO_SEDMLS_TO_EXECUTE;
+        } else if (errorMessage.contains("MappingException occurred: failed to generate math")) {
+            return FailureType.MATH_GENERATION_FAILURE;
+        }
+
+        // else check Tracer error events for known faults
+        for (TraceEvent event : errorEvents) {
+            if (event.hasException(SBMLImportException.class)) {
+                return FailureType.SBML_IMPORT_FAILURE;
+            }
+            if (event.span.getNestedContextName().contains(Span.ContextType.PROCESSING_SEDML.name()+"(preProcessDoc)")){
+                return FailureType.SEDML_PREPROCESS_FAILURE;
+            }
+            if (event.hasException(MappingException.class)) {
+                return FailureType.MATH_GENERATION_FAILURE;
+            }
+        }
+
+        return FailureType.UNCATETORIZED_FAULT;
+    }
+
 }