Updating cicd for cli

.github/workflows/CI-full.yml

@@ -43,43 +43,44 @@ jobs:
         cache: "pip"
 
     - name: Install Dependencies
-      run: pip install -r requirements.txt
+      # We want to make sure the poetry installation is attached to the correct python version, or it may not find said version.
+      run: python${{ env.python-version }} -m pip install -r requirements.txt
 
     - name: build vcell-cli-utils package
       run: |
         cd vcell-cli-utils
-        poetry install
-        poetry run python -m pytest
+        python${{ env.python-version }} -m poetry install
+        python${{ env.python-version }} -m poetry run python -m pytest
 
     - name: build vcell-admin package
       run: |
         cd docker/swarm/vcell-admin
-        poetry install
-        poetry run python -m pytest
+        python${{ env.python-version }} -m poetry install
+        python${{ env.python-version }} -m poetry run python -m pytest
 
     - name: build pythonCopasiOpt package
       run: |
         cd pythonCopasiOpt/vcell-opt
-        poetry install
-        poetry run python -m pytest
+        python${{ env.python-version }} -m poetry install
+        python${{ env.python-version }} -m poetry run python -m pytest
 
     - name: build pythonVtk package
       run: |
         cd pythonVtk
-        poetry install
-        poetry run python -m pytest
+        python${{ env.python-version }} -m poetry install
+        python${{ env.python-version }} -m poetry run python -m pytest
 
     - name: build vcutils package
       run: |
         cd python-utils
-        poetry install
-        poetry run python -m pytest
+        python${{ env.python-version }} -m poetry install
+        python${{ env.python-version }} -m poetry run python -m pytest
 
     - name: build python-restclient package
       run: |
         cd python-restclient
-        poetry install
-        poetry run python -m pytest
+        python${{ env.python-version }} -m poetry install
+        python${{ env.python-version }} -m poetry run python -m pytest
 
     - name: build vcelldata package
       run: |

docker/build/Dockerfile-api-dev

@@ -10,12 +10,12 @@ RUN $JAVA_HOME/bin/jlink \
          --output /javaruntime
 
 # Define base image and copy in jlink created minimal Java 17 environment
-FROM python:3.9.7-slim
+FROM python:3.10.14-slim
 ENV JAVA_HOME=/opt/java/openjdk
 ENV PATH "${JAVA_HOME}/bin:${PATH}"
 COPY --from=jre-build /javaruntime $JAVA_HOME
 
-# now we have Java 17 and Python 3.9
+# now we have Java 17 and Python 3.10
 
 RUN apt-get -y update && \
     apt-get install -y --no-install-recommends curl dnsutils apt-utils libfreetype6 fontconfig fonts-dejavu && \

docker/build/Dockerfile-data-dev

@@ -10,12 +10,12 @@ RUN $JAVA_HOME/bin/jlink \
          --output /javaruntime
 
 # Define base image and copy in jlink created minimal Java 17 environment
-FROM python:3.9.7-slim
+FROM python:3.10.14-slim
 ENV JAVA_HOME=/opt/java/openjdk
 ENV PATH "${JAVA_HOME}/bin:${PATH}"
 COPY --from=jre-build /javaruntime $JAVA_HOME
 
-# now we have Java 17 and Python 3.9
+# now we have Java 17 and Python 3.10
 
 RUN apt-get -y update && \
 	apt-get -y install openssh-client screen apt-utils libfreetype6 fontconfig fonts-dejavu && \

python-restclient/pyproject.toml

@@ -10,7 +10,7 @@ keywords = ["OpenAPI", "OpenAPI-Generator", "VCell API"]
 include = ["vcell_client/py.typed"]
 
 [tool.poetry.dependencies]
-python = "^3.9"
+python = "~3.10"
 
 urllib3 = ">= 1.25.3"
 python-dateutil = ">=2.8.2"

python-utils/pyproject.toml

@@ -19,7 +19,7 @@ classifiers = [
 ]
 
 [tool.poetry.dependencies]
-python = "^3.9"
+python = "~3.10"
 requests = "^2.31.0"
 pydantic = "^2.5.2"
 python-dotenv = "^1.0.0"

pythonCopasiOpt/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-slim-buster
+FROM python:3.10.9-slim-buster
 
 RUN apt -y update && apt -y upgrade &&  \
     apt-get install -y curl

pythonCopasiOpt/vcell-opt/pyproject.toml

@@ -6,7 +6,7 @@ authors = ["Jim Schaff <[email protected]>"]
 readme = "README.md"
 
 [tool.poetry.dependencies]
-python = "^3.9"
+python = "~3.10"
 copasi-basico = "0.40"
 python-copasi = "4.37.264"
 jupyter = "^1.0.0"

pythonData/pyproject.toml

@@ -8,7 +8,7 @@ packages = [{ include = "vcelldata" }]
 
 
 [tool.poetry.dependencies]
-python = "^3.9"
+python = "~3.10"
 typer = "^0.12.3"
 numexpr = "^2.10.0"
 zarr = "^2.17.2"

pythonVtk/pyproject.toml

@@ -6,7 +6,7 @@ authors = ["Jim Schaff <[email protected]>"]
 readme = "README.md"
 
 [tool.poetry.dependencies]
-python = "^3.9"
+python = "~3.10"
 vtk = "^9.2.6"
 thrift = "^0.16.0"
 

vcell-cli-utils/pyproject.toml

@@ -25,7 +25,7 @@ biosimulators_utils = "^0.1.180"
 python-libcombine = "0.2.19"
 fire = "^0.5.0"
 PyYAML = "^6.0"
-pandas = "^1.5.3"
+pandas = "^2.2.2"
 matplotlib = "^3.5.2"
 seaborn = "^0.12.2"
 python-libsbml-experimental = "^5.20.1"

vcell-cli/src/main/java/org/vcell/cli/biosimulation/BiosimulationsCommand.java

@@ -106,14 +106,14 @@ public Integer call() {
                 try {
                     CLIPythonManager.getInstance().closePythonProcess(); // WARNING: Python will need reinstantiation after this is called
                 } catch (Exception e) {
-                    logger.error(e.getMessage(), e);
+                    logger.error("Error closing python process:", e);
                 }
                 logger.debug("Finished all execution.");
             }
         } catch (Exception e) {
             if (!bQuiet) {
                 Tracer.reportErrors(bDebug);
-                logger.error(e.getMessage(), e);
+                logger.error("Unexpected Error in BioSimulations execution:", e);
             }
             return 1;
         }

vcell-cli/src/main/java/org/vcell/cli/run/ExecuteImpl.java

@@ -14,8 +14,7 @@
 import java.io.File;
 import java.io.FilenameFilter;
 import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
+import java.nio.file.*;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -137,9 +136,24 @@ public static void singleMode(File inputFile, File rootOutputDir, CLIRecordable
         if (!inputFile.getParentFile().getCanonicalPath().contains(adjustedOutputDir.getCanonicalPath()))
             RunUtils.removeAndMakeDirs(adjustedOutputDir);
 
-        PythonCalls.generateStatusYaml(inputFile.getAbsolutePath(), targetOutputDir);    // generate Status YAML
+        File targetInputFile;
 
-        ExecuteImpl.singleExecOmex(inputFile, rootOutputDir, cliLogger, bKeepTempFiles, bExactMatchOnly,
+        if(inputFile.canWrite()){
+            targetInputFile = inputFile;
+        } else {
+            String namePrefix = inputFile.getName().substring(0, inputFile.getName().lastIndexOf('.'));
+            Path tempFile = Files.createTempFile(namePrefix, ".omex");
+            File writableCopy = Files.copy(inputFile.toPath(), tempFile, StandardCopyOption.REPLACE_EXISTING).toFile();
+            if (!writableCopy.canWrite()
+                    && !writableCopy.setWritable(true) // Main Attempt
+                    && !writableCopy.setWritable(true, true)) // Backup Attempt
+                throw new RuntimeException("Unable to create writable file");
+            targetInputFile = writableCopy;
+        }
+
+        PythonCalls.generateStatusYaml(targetInputFile.getAbsolutePath(), targetOutputDir);    // generate Status YAML
+
+        ExecuteImpl.singleExecOmex(targetInputFile, rootOutputDir, cliLogger, bKeepTempFiles, bExactMatchOnly,
                 bEncapsulateOutput, bSmallMeshOverride, bBioSimMode);
     }
 

vcell-cli/src/main/java/org/vcell/cli/run/ExecutionJob.java

@@ -91,13 +91,13 @@ public void preprocessArchive() throws PythonStreamException, IOException {
             this.omexHandler.extractOmex();
             this.sedmlLocations = omexHandler.getSedmlLocationsAbsolute();
         } catch (IOException e){
-            String error = e.getMessage() + ", error for OmexHandler with " + inputFilePath;
+            String error = "Error for OmexHandler with " + inputFilePath;
             this.cliRecorder.writeErrorList(bioModelBaseName);
             this.cliRecorder.writeDetailedResultList(bioModelBaseName + ", " + "IO error with OmexHandler");
             logger.error(error);
             throw new RuntimeException(error, e);
         } catch (Exception e) {
-            String error = e.getMessage() + ", error for archive " + inputFilePath;
+            String error = "Error for archive " + inputFilePath;
             logger.error(error);
             if (omexHandler!=null) omexHandler.deleteExtractedOmex();
             this.cliRecorder.writeErrorList(bioModelBaseName);

vcell-cli/src/main/java/org/vcell/cli/run/OmexHandler.java

@@ -5,15 +5,18 @@
 import de.unirostock.sems.cbarchive.CombineArchiveException;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.jdom2.IllegalNameException;
 import org.jdom2.JDOMException;
 import org.vcell.cli.CLIUtils;
 
+import javax.xml.transform.TransformerException;
 import java.io.File;
 import java.io.IOException;
 import java.net.URI;
 import java.nio.file.*;
 import java.text.ParseException;
 import java.util.*;
+import java.util.stream.Stream;
 
 public class OmexHandler {
     String tempPath;
@@ -23,14 +26,15 @@ public class OmexHandler {
     CombineArchive archive;
 
     private final static Logger logger = LogManager.getLogger(OmexHandler.class);
-    
+
     // Assuming omexPath will always be absolute path
     // NB: Need to convert class to use Log4j2
     public OmexHandler(String omexPath, String outDir) throws IOException {
         this.omexPath = omexPath;
         this.outDirPath = outDir;
 
-        if (!new File(omexPath).exists()) {
+        File omexFile = new File(omexPath);
+        if (!omexFile.exists()) {
             String[] omexNameArray = omexPath.split("/", -2);
             String omexName = omexNameArray[omexNameArray.length - 1];
             IOException e = new IOException("Provided OMEX `" + omexName + "` is not present at path: " + omexPath);
@@ -40,48 +44,78 @@ public OmexHandler(String omexPath, String outDir) throws IOException {
         int indexOfLastSlash = omexPath.lastIndexOf("/");
         this.omexName = omexPath.substring(indexOfLastSlash + 1);
         this.tempPath = RunUtils.getTempDir();
-        try {
-            replaceMetadataRdfFiles(Paths.get(omexPath));
-            this.archive = new CombineArchive(new File(omexPath));
-            if (this.archive.hasErrors()){
-                String message = "Unable to initialise OMEX archive "+this.omexName+": "+this.archive.getErrors();
-                logger.error(message);
-                throw new IOException(message);
+        boolean initiallyEncounteredErrors = false;
+        try (CombineArchive initialArchive = new CombineArchive(omexFile, true)){
+            // With the `continueOnError` flag set, an exception won't be thrown.
+            initiallyEncounteredErrors = initialArchive.hasErrors();
+            // We want to try and repair any RDF issue the archive has, because we don't care that much about RDF validity.
+            // so, if (1) We have RDF errors; OR (2) have no rdf file, then we add / find & replace
+            if ((initiallyEncounteredErrors && omexFile.canWrite() && initialArchive.getErrors().contains(".rdf"))
+                || initialArchive.getEntries().stream().map(ArchiveEntry::getFileName).noneMatch(name->name.endsWith(".rdf")))
+                this.replaceMetadataRdfFiles(Paths.get(omexPath)); // Replace the offending files
+        } catch (CombineArchiveException | JDOMException | IllegalNameException | ParseException e) {
+            // Alright, we got errors, AND an exception. Let's try a hail mary.
+            if (omexFile.canWrite())
+                this.replaceMetadataRdfFiles(Paths.get(omexPath));
+        }
+
+        try { // If we still have problems here, we're bombing out.
+            this.archive = new CombineArchive(omexFile);
+            if (!this.archive.hasErrors()){
+                if (initiallyEncounteredErrors) logger.info("Successfully Repaired `" + this.omexName + "`");
+                return;
             }
-        }catch (CombineArchiveException | JDOMException | ParseException e) {
+            // We're unable to solve the remaining errors, so we're officially rejecting this archive.
+            String message = "Unable to initialise OMEX archive " + this.omexName + ": " + this.archive.getErrors();
+            logger.error(message);
+            throw new IOException(message);
+        } catch (CombineArchiveException | JDOMException | ParseException e) {
             String message = String.format("Unable to initialise OMEX archive \"%s\", archive maybe corrupted", this.omexName);
             logger.error(message+": "+e.getMessage(), e);
             throw new IOException(e);
         }
     }
 
-    private void replaceMetadataRdfFiles(Path zipFilePath) throws IOException {
-        try( FileSystem fs = FileSystems.newFileSystem(zipFilePath) ) {
+    private void replaceMetadataRdfFiles(Path zipFilePath) {
+        try (FileSystem fs = FileSystems.newFileSystem(zipFilePath)) {
             for (Path root : fs.getRootDirectories()) {
-                Files.walk(root)
+                try (Stream<Path> rdfFiles = Files.walk(root)
                         .filter(Files::isRegularFile)
-                        .filter(path -> path.toString().endsWith(".rdf"))
-                        .forEach(path -> {
-                            try {
-                                // write empty RDF file to temp file and replace the file inside the zip
-                                Path tempFile = Files.createTempFile("temp", ".rdf");
-                                String new_rdf_content =
+                        .filter(path -> path.toString().endsWith(".rdf"))){
+
+                    if (rdfFiles.findAny().isEmpty()){
+                        Path targetPath = Paths.get(root.toString(), "metadata.rdf");
+                        Files.write(targetPath, "fileToBeReplaced".getBytes());
+                    }
+                }
+
+                try (Stream<Path> rdfFiles = Files.walk(root)
+                        .filter(Files::isRegularFile)
+                        .filter(path -> path.toString().endsWith(".rdf"))){
+
+                    rdfFiles.forEach(path -> {
+                        try {
+                            // write empty RDF file to temp file and replace the file inside the zip
+                            Path tempFile = Files.createTempFile("temp", ".rdf");
+                            String new_rdf_content =
                                     """
                                     <?xml version='1.0' encoding='UTF-8'?>
                                     <rdf:RDF xmlns:rdf='http://www.w3.org/1999/02/22-rdf-syntax-ns#'>
                                     </rdf:RDF>
                                     """;
-                                Files.write(tempFile, new_rdf_content.getBytes());
-                                // replace fileInsideZipPath with temp file
-                                Files.delete(path);
-                                Files.copy(tempFile, path);
-                                Files.delete(tempFile);
-                            } catch (IOException e) {
-                                logger.error("Unable to delete metadata.rdf file from OMEX archive: " + e.getMessage(), e);
-                            }
-                        });
+                            Files.write(tempFile, new_rdf_content.getBytes());
+                            // replace fileInsideZipPath with temp file
+                            Files.delete(path);
+                            Files.copy(tempFile, path);
+                            Files.delete(tempFile);
+                        } catch (IOException e) {
+                            logger.error("Unable to delete metadata.rdf file from OMEX archive: " + e.getMessage(), e);
+                        }
+                    });
+                }
             }
-
+        } catch (IOException | ReadOnlyFileSystemException e) {
+            throw new RuntimeException("Unable to delete metadata.rdf file from OMEX archive: ", e);
         }
     }