[WIP] Live m3 follow up

frontend/javascripts/admin/api/mesh.ts

@@ -15,7 +15,7 @@ export type MeshLodInfo = {
   transform: [Vector4, Vector4, Vector4]; // 4x3 matrix
 };
 
-type MeshSegmentInfo = {
+export type MeshSegmentInfo = {
   meshFormat: "draco";
   lods: Array<MeshLodInfo>;
   chunkScale: Vector3;
@@ -71,7 +71,7 @@ type MeshChunkDataRequest = {
   segmentId: number | null; // Only relevant for neuroglancer precomputed meshes
 };
 
-type MeshChunkDataRequestList = {
+export type MeshChunkDataRequestList = {
   meshFileName: string;
   requests: MeshChunkDataRequest[];
 };

frontend/javascripts/libs/diffable_map.ts

@@ -1,3 +1,5 @@
+import _ from "lodash";
+
 const defaultItemsPerBatch = 1000;
 let idCounter = 0;
 const idSymbol = Symbol("id");
@@ -417,9 +419,11 @@ class DiffableMap<K extends number, V> implements NotEnumerableByObject {
               ? "null"
               : v === undefined
                 ? "undefined"
-                : typeof v === "object"
-                  ? JSON.stringify(v)
-                  : String(v);
+                : v instanceof DiffableMap
+                  ? v.toString()
+                  : typeof v === "object"
+                    ? JSON.stringify(v)
+                    : String(v);
           return `${k} => ${vStr}`;
         })
         .join(", ");
@@ -468,11 +472,15 @@ function shallowCopy<K extends number, V>(template: DiffableMap<K, V>): Diffable
 export function diffDiffableMaps<K extends number, V>(
   mapA: DiffableMap<K, V>,
   mapB: DiffableMap<K, V>,
+  useDeepEqualityCheck: boolean = false,
 ): {
   changed: Array<K>;
   onlyA: Array<K>;
   onlyB: Array<K>;
 } {
+  const areDifferent = (valueA: V | undefined, valueB: V | undefined) =>
+    useDeepEqualityCheck ? !_.isEqual(valueA, valueB) : valueA !== valueB;
+
   // For the edge case that one of the maps is empty, we will consider them dependent, anyway
   const areDiffsDependent = mapA.getId() === mapB.getId() || mapA.size() === 0 || mapB.size() === 0;
   let idx = 0;
@@ -506,7 +514,7 @@ export function diffDiffableMaps<K extends number, V>(
 
       for (const key of setA.values()) {
         if (setB.has(key)) {
-          if (currentMapA.get(key) !== currentMapB.get(key)) {
+          if (areDifferent(currentMapA.get(key), currentMapB.get(key))) {
             changed.push(key);
           } else {
             // The key exists in both chunks, do not emit this key.
@@ -530,32 +538,32 @@ export function diffDiffableMaps<K extends number, V>(
     idx++;
   }
 
-  if (!areDiffsDependent) {
-    // Since, the DiffableMaps don't share the same structure, we might have
-    // aggregated false-positives, meaning onlyA and onlyB can include the same
-    // keys, which might or might not belong to the changed set.
-    // Construct a set for fast lookup
-    const setA = new Set(onlyA);
-    // Intersection of onlyA and onlyB:
-    const missingChangedIds = onlyB.filter((id) => setA.has(id));
-    const missingChangedIdSet = new Set(missingChangedIds);
-    const newOnlyA = onlyA.filter((id) => !missingChangedIdSet.has(id));
-    const newOnlyB = onlyB.filter((id) => !missingChangedIdSet.has(id));
-    // Ensure that these elements are not equal before adding them to "changed"
-    const newChanged = changed.concat(
-      missingChangedIds.filter((id) => mapA.getOrThrow(id) !== mapB.getOrThrow(id)),
-    );
+  if (areDiffsDependent) {
     return {
-      changed: newChanged,
-      onlyA: newOnlyA,
-      onlyB: newOnlyB,
+      changed,
+      onlyA,
+      onlyB,
     };
   }
 
+  // Since, the DiffableMaps don't share the same structure, we might have
+  // aggregated false-positives, meaning onlyA and onlyB can include the same
+  // keys, which might or might not belong to the changed set.
+  // Construct a set for fast lookup
+  const setA = new Set(onlyA);
+  // Intersection of onlyA and onlyB:
+  const missingChangedIds = onlyB.filter((id) => setA.has(id));
+  const missingChangedIdSet = new Set(missingChangedIds);
+  const newOnlyA = onlyA.filter((id) => !missingChangedIdSet.has(id));
+  const newOnlyB = onlyB.filter((id) => !missingChangedIdSet.has(id));
+  // Ensure that these elements are not equal before adding them to "changed"
+  const newChanged = changed.concat(
+    missingChangedIds.filter((id) => areDifferent(mapA.getOrThrow(id), mapB.getOrThrow(id))),
+  );
   return {
-    changed,
-    onlyA,
-    onlyB,
+    changed: newChanged,
+    onlyA: newOnlyA,
+    onlyB: newOnlyB,
   };
 }
 

frontend/javascripts/libs/utils.ts

@@ -621,6 +621,46 @@ export function diffArrays<T>(
   };
 }
 
+export function diffNumberArrays(a: number[], b: number[]) {
+  // Create sorted copies to avoid mutating inputs
+  const A = [...a].sort((x, y) => x - y);
+  const B = [...b].sort((x, y) => x - y);
+
+  const both: number[] = [];
+  const onlyA: number[] = [];
+  const onlyB: number[] = [];
+
+  let indexA = 0;
+  let indexB = 0;
+
+  while (indexA < A.length && indexB < B.length) {
+    if (A[indexA] === B[indexB]) {
+      both.push(A[indexA]);
+      indexA++;
+      indexB++;
+    } else if (A[indexA] < B[indexB]) {
+      onlyA.push(A[indexA]);
+      indexA++;
+    } else {
+      onlyB.push(B[indexB]);
+      indexB++;
+    }
+  }
+
+  // Remaining elements
+  while (indexA < A.length) {
+    onlyA.push(A[indexA]);
+    indexA++;
+  }
+
+  while (indexB < B.length) {
+    onlyB.push(B[indexB]);
+    indexB++;
+  }
+
+  return { both, onlyA, onlyB };
+}
+
 export function diffMaps<K, V>(
   stateA: Map<K, V>,
   stateB: Map<K, V>,

frontend/javascripts/test/backend-snapshot-tests/annotations.e2e.ts

@@ -242,7 +242,9 @@ describe("Annotation API (E2E)", () => {
         ],
       },
     ];
-    const createTreesUpdateActions = Array.from(diffTrees(tracingId, new DiffableMap(), trees));
+    const createTreesUpdateActions = Array.from(
+      diffTrees(tracingId, new DiffableMap(), trees, false),
+    );
     const updateTreeGroupsUpdateAction = UpdateActions.updateTreeGroups(treeGroups, tracingId);
     const [saveQueue] = addVersionNumbers(
       createSaveQueueFromUpdateActions(
@@ -266,7 +268,9 @@ describe("Annotation API (E2E)", () => {
     const createdExplorational = await api.createExplorational(datasetId, "skeleton", false, null);
     const { tracingId } = createdExplorational.annotationLayers[0];
     let trees = createTreeMapFromTreeArray(generateDummyTrees(5, 6));
-    const createTreesUpdateActions = Array.from(diffTrees(tracingId, new DiffableMap(), trees));
+    const createTreesUpdateActions = Array.from(
+      diffTrees(tracingId, new DiffableMap(), trees, false),
+    );
     const metadata = [
       {
         key: "city",

frontend/javascripts/test/fixtures/dataset_server_object.ts

@@ -23,6 +23,7 @@ const sampleColorLayer: APIColorLayer = {
 };
 
 export const sampleHdf5AgglomerateName = "sampleHdf5Mapping";
+export const sampleMappingFileName = "sampleMappingFile";
 // this is a uint32 segmentation layer
 const sampleSegmentationLayer: APISegmentationLayer = {
   name: "segmentation",

frontend/javascripts/test/helpers/agglomerate_mapping_helper.ts

@@ -153,6 +153,12 @@ export class AgglomerateMapping {
     return this.versions[version];
   }
 
+  // Returns a copy of the current adjacency list. It is a deep clone to avoid direct manipulation from outside.
+  // AdjacencyList is currently unversioned.
+  getAdjacencyList(): Map<number, Set<number>> {
+    return _.cloneDeep(this.adjacencyList);
+  }
+
   private resetVersionCounter(initialVersion: number) {
     /*
      * Reset the most recent version to be stored as version `initialVersion`.