Investigating issues with loading extended data

Author: hohonuuli

build.sbt

@@ -3,7 +3,7 @@ import Dependencies._
 Global / onChangedBuildSource := ReloadOnSourceChanges
 Laika / sourceDirectories     := Seq(baseDirectory.value / "docs")
 
-ThisBuild / scalaVersion     := "3.7.1"
+ThisBuild / scalaVersion     := "3.7.3"
 ThisBuild / organization     := "org.fathomnet"
 ThisBuild / organizationName := "MBARI"
 ThisBuild / startYear        := Some(2021)

project/Dependencies.scala

@@ -2,29 +2,29 @@ import sbt._
 
 object Dependencies {
 
-    private val circeVersion = "0.14.14"
+    private val circeVersion = "0.14.15"
     lazy val circeCore       = "io.circe" %% "circe-core"    % circeVersion
     lazy val circeGeneric    = "io.circe" %% "circe-generic" % circeVersion
     lazy val circeParser     = "io.circe" %% "circe-parser"  % circeVersion
 
     lazy val jansi = "org.fusesource.jansi" % "jansi" % "2.4.2"
 
-    lazy val logback  = "ch.qos.logback"               % "logback-classic" % "1.5.18"
+    lazy val logback  = "ch.qos.logback"               % "logback-classic" % "1.5.19"
     lazy val methanol = "com.github.mizosoft.methanol" % "methanol"        % "1.8.3"
     lazy val munit    = "org.scalameta"               %% "munit"           % "1.1.1"
     lazy val picocli  = "info.picocli"                 % "picocli"         % "4.7.7"
 
     lazy val slf4jJdk = "org.slf4j" % "slf4j-jdk-platform-logging" % "2.0.17"
 
-    private val tapirVersion  = "1.11.35"
+    private val tapirVersion  = "1.11.48"
     lazy val tapirStubServer  = "com.softwaremill.sttp.tapir"   %% "tapir-sttp-stub-server"  % tapirVersion
     lazy val tapirSwagger     = "com.softwaremill.sttp.tapir"   %% "tapir-swagger-ui-bundle" % tapirVersion
     lazy val tapirCirce       = "com.softwaremill.sttp.tapir"   %% "tapir-json-circe"        % tapirVersion
     lazy val tapirCirceClient = "com.softwaremill.sttp.client3" %% "circe"                   % "3.11.0"
     lazy val tapirNetty       = "com.softwaremill.sttp.tapir"   %% "tapir-netty-server"      % tapirVersion
     lazy val tapirVertx       = "com.softwaremill.sttp.tapir"   %% "tapir-vertx-server"      % tapirVersion
 
-    lazy val typesafeConfig = "com.typesafe" % "config" % "1.4.3"
-    lazy val zio            = "dev.zio"     %% "zio"    % "2.1.19"
+    lazy val typesafeConfig = "com.typesafe" % "config" % "1.4.5"
+    lazy val zio            = "dev.zio"     %% "zio"    % "2.1.21"
 
 }

project/build.properties

@@ -1 +1 @@
-sbt.version=1.11.2
+sbt.version=1.11.7

src/main/scala/org/fathomnet/worms/Main.scala

@@ -13,6 +13,8 @@ import org.fathomnet.worms.etc.jdk.CustomExecutors
 import org.fathomnet.worms.etc.jdk.CustomExecutors.*
 import org.fathomnet.worms.etc.jdk.Logging.given
 import org.fathomnet.worms.io.WormsLoader
+
+import scala.util.control.NonFatal
 // import org.fathomnet.worms.io.extended.CombineTrees.combine
 import org.fathomnet.worms.io.extended.{CombineTrees, ExtendedLoader}
 import picocli.CommandLine
@@ -129,13 +131,20 @@ object Main:
         val (wormsConcepts, rootOpt) = WormsLoader.load(wormsDir)
         val newRoot                  = rootOpt.map { root =>
             if (treeFiles.nonEmpty)
-                // Our new base. We use 0 as aphiaId so that the real aphiaIds are not incremented when the trees are combined
-                val newRoot      = WormsNode("object", "", 0L, 0L, Nil, Nil)
-                val newBranches  = treeFiles.flatMap(ExtendedLoader.load(_)).toSeq
-                val trees        = root +: newBranches
-                val combinedRoot = CombineTrees.combine(newRoot, trees, root.maxAphiaId)
-                // We need to reset the aphiaId to -1 so that it's obivious that the root is not a real aphiaId
-                combinedRoot.copy(aphiaId = -1L)
+                try {
+                    // Our new base. We use 0 as aphiaId so that the real aphiaIds are not incremented when the trees are combined
+                    val newRoot = WormsNode("object", "", 0L, 0L, Nil, Nil)
+                    val newBranches = treeFiles.flatMap(ExtendedLoader.load).toSeq
+                    val trees = root +: newBranches
+                    val combinedRoot = CombineTrees.combine(newRoot, trees, root.maxAphiaId)
+                    // We need to reset the aphiaId to -1 so that it's obivious that the root is not a real aphiaId
+                    combinedRoot.copy(aphiaId = -1L)
+                }
+                catch {
+                    case NonFatal(e) =>
+                        log.atWarn.withCause(e).log("Error combining trees from " + treeFiles.mkString(", "))
+                        root
+                }
             else root
         }
         (wormsConcepts, newRoot)

src/main/scala/org/fathomnet/worms/io/MutableWormsNode.scala

@@ -43,7 +43,7 @@ object MutableWormsNodeBuilder:
         do parentNode.children.append(conceptNode)
         // In worms, the root node has an aphiaId of 1
         val minAphiaId = wormsConcepts.map(_.id).min
-        map.get(minAphiaId).get
+        map(minAphiaId)
 
     /**
      * Given the full tree, return only the animalia node

src/main/scala/org/fathomnet/worms/io/extended/ExtendedLoader.scala

@@ -11,7 +11,7 @@ import org.fathomnet.worms.io.{MutableWormsNodeBuilder, WormsConcept, WormsConce
 import org.fathomnet.worms.{WormsNode, WormsNodeBuilder}
 
 import java.nio.file.Path
-import scala.util.Using
+import scala.util.{Failure, Success, Try, Using}
 import scala.util.control.NonFatal
 
 object ExtendedLoader:
@@ -39,17 +39,22 @@ object ExtendedLoader:
      * @return
      */
     def load(extendedFile: Path): Option[WormsNode] =
-        val wormsConcepts = read(extendedFile.toString)
-        val tree          = MutableWormsNodeBuilder.buildTree(wormsConcepts)
-        Option(WormsNodeBuilder.from(tree))
+        Try {
+            val wormsConcepts = read(extendedFile.toString)
+            val tree = MutableWormsNodeBuilder.buildTree(wormsConcepts)
+            Option(WormsNodeBuilder.from(tree))
+        } match
+            case Success(s) => s
+            case Failure(exception) =>
+                log.atError.withCause(exception).log(s"Failed to load extended file: $extendedFile")
+                None
 
     def read(file: String): Seq[WormsConcept] =
-        val t = Using(scala.io.Source.fromFile(file)) { source =>
+        Using(scala.io.Source.fromFile(file)) { source =>
             source.getLines().flatMap(from).toSeq
-        }
-        t match
-            case scala.util.Success(s) => s
-            case scala.util.Failure(e) =>
+        } match
+            case Success(s) => s
+            case Failure(e) =>
                 log.atError.withCause(e).log(s"Failed to read file: $file")
                 Seq.empty
 

src/main/scala/org/fathomnet/worms/io/model.scala

@@ -7,7 +7,7 @@
 package org.fathomnet.worms.io
 
 import scala.io.Source
-import scala.util.Try
+import scala.util.{Success, Try, Using}
 
 /*
  * This file contains code to read the 3 relevant files from the Worms
@@ -20,12 +20,14 @@ import scala.util.Try
 def taxonIDToKey(taxonID: String): Long =
     taxonID.split(":").last.toLong
 
-def readFile[A](file: String, rowMapper: String => Option[A]): List[A] =
-    Source
-        .fromFile(file)
-        .getLines
-        .flatMap(rowMapper)
-        .toList
+def readFile[A](file: String, rowMapper: String => Option[A]): List[A] = {
+    Using(Source.fromFile(file)) { source =>
+        source.getLines().flatMap(rowMapper).toList
+    } match
+        case Success(list) => list
+        case _             => List.empty[A]
+
+}
 
 final case class Taxon(
     taxonID: String,
@@ -34,9 +36,9 @@ final case class Taxon(
     rank: String,
     acceptedNameUsageID: Option[String]
 ):
-    val id         = taxonIDToKey(taxonID)
-    val parentId   = parentNameUsageID.map(taxonIDToKey)
-    val acceptedId = acceptedNameUsageID.map(taxonIDToKey)
+    val id: Long = taxonIDToKey(taxonID)
+    val parentId: Option[Long] = parentNameUsageID.map(taxonIDToKey)
+    val acceptedId: Option[Long] = acceptedNameUsageID.map(taxonIDToKey)
 
 object Taxon:
     def from(row: String): Option[Taxon] =
@@ -72,7 +74,7 @@ final case class SpeciesProfile(
     isExtinct: Option[Boolean],
     isBrackish: Option[Boolean]
 ):
-    val id = taxonIDToKey(taxonID)
+    val id: Long = taxonIDToKey(taxonID)
 
 object SpeciesProfile: