more tests

Author: normen662

fdb-extensions/src/main/java/com/apple/foundationdb/async/hnsw/HNSW.java

@@ -141,21 +141,6 @@ public static class Config {
         private final boolean useRaBitQ;
         private final int raBitQNumExBits;
 
-        protected Config(final int numDimensions) {
-            this.random = DEFAULT_RANDOM;
-            this.metric = DEFAULT_METRIC;
-            this.numDimensions = numDimensions;
-            this.useInlining = DEFAULT_USE_INLINING;
-            this.m = DEFAULT_M;
-            this.mMax = DEFAULT_M_MAX;
-            this.mMax0 = DEFAULT_M_MAX_0;
-            this.efConstruction = DEFAULT_EF_CONSTRUCTION;
-            this.extendCandidates = DEFAULT_EXTEND_CANDIDATES;
-            this.keepPrunedConnections = DEFAULT_KEEP_PRUNED_CONNECTIONS;
-            this.useRaBitQ = DEFAULT_USE_RABITQ;
-            this.raBitQNumExBits = DEFAULT_RABITQ_NUM_EX_BITS;
-        }
-
         protected Config(@Nonnull final Random random, @Nonnull final Metric metric, final int numDimensions,
                          final boolean useInlining, final int m, final int mMax, final int mMax0,
                          final int efConstruction, final boolean extendCandidates, final boolean keepPrunedConnections,
@@ -231,6 +216,37 @@ public ConfigBuilder toBuilder() {
                     getRaBitQNumExBits());
         }
 
+        @Override
+        public final boolean equals(final Object o) {
+            if (!(o instanceof Config)) {
+                return false;
+            }
+
+            final Config config = (Config)o;
+            return numDimensions == config.numDimensions && useInlining == config.useInlining && m == config.m &&
+                    mMax == config.mMax && mMax0 == config.mMax0 && efConstruction == config.efConstruction &&
+                    extendCandidates == config.extendCandidates &&
+                    keepPrunedConnections == config.keepPrunedConnections && useRaBitQ == config.useRaBitQ &&
+                    raBitQNumExBits == config.raBitQNumExBits && random.equals(config.random) &&
+                    metric == config.metric;
+        }
+
+        @Override
+        public int hashCode() {
+            int result = metric.name().hashCode();
+            result = 31 * result + numDimensions;
+            result = 31 * result + Boolean.hashCode(useInlining);
+            result = 31 * result + m;
+            result = 31 * result + mMax;
+            result = 31 * result + mMax0;
+            result = 31 * result + efConstruction;
+            result = 31 * result + Boolean.hashCode(extendCandidates);
+            result = 31 * result + Boolean.hashCode(keepPrunedConnections);
+            result = 31 * result + Boolean.hashCode(useRaBitQ);
+            result = 31 * result + raBitQNumExBits;
+            return result;
+        }
+
         @Override
         @Nonnull
         public String toString() {
@@ -409,6 +425,17 @@ public static ConfigBuilder newConfigBuilder() {
         return new ConfigBuilder();
     }
 
+    /**
+     * Returns a default {@link Config}.
+     * @param numDimensions number of dimensions
+     * @return a new default {@code Config}.
+     * @see ConfigBuilder#build
+     */
+    @Nonnull
+    public static Config defaultConfig(int numDimensions) {
+        return new ConfigBuilder().build(numDimensions);
+    }
+
     /**
      * Creates a new {@code HNSW} instance using the default configuration, write listener, and read listener.
      * <p>

fdb-extensions/src/main/java/com/apple/foundationdb/async/hnsw/InliningStorageAdapter.java

@@ -176,13 +176,26 @@ private Node<NodeReferenceWithVector> nodeFromRaw(final int layer,
     @Nonnull
     private NodeReferenceWithVector neighborFromRaw(final int layer, final @Nonnull byte[] key, final byte[] value) {
         final OnReadListener onReadListener = getOnReadListener();
-
         onReadListener.onKeyValueRead(layer, key, value);
+
         final Tuple neighborKeyTuple = getDataSubspace().unpack(key);
         final Tuple neighborValueTuple = Tuple.fromBytes(value);
 
-        final Tuple neighborPrimaryKey = neighborKeyTuple.getNestedTuple(2); // neighbor primary key
-        final RealVector neighborVector = StorageAdapter.vectorFromTuple(getConfig(), neighborValueTuple); // the entire value is the vector
+        return neighborFromTuples(neighborKeyTuple, neighborValueTuple);
+    }
+
+    /**
+     * Constructs a {@code NodeReferenceWithVector} from tuples retrieved from storage.
+     * <p>
+     * @param keyTuple the key tuple from the database, which contains the neighbor's primary key.
+     * @param valueTuple the value tuple from the database, which represents the neighbor's vector.
+     * @return a new {@link NodeReferenceWithVector} instance representing the deserialized neighbor.
+     * @throws IllegalArgumentException if the key or value byte arrays are malformed and cannot be unpacked.
+     */
+    @Nonnull
+    private NodeReferenceWithVector neighborFromTuples(final @Nonnull Tuple keyTuple, final Tuple valueTuple) {
+        final Tuple neighborPrimaryKey = keyTuple.getNestedTuple(2); // neighbor primary key
+        final RealVector neighborVector = StorageAdapter.vectorFromTuple(getConfig(), valueTuple); // the entire value is the vector
         return new NodeReferenceWithVector(neighborPrimaryKey, neighborVector);
     }
 
@@ -308,6 +321,7 @@ private byte[] getNeighborKey(final int layer,
     @Override
     public Iterable<Node<NodeReferenceWithVector>> scanLayer(@Nonnull final ReadTransaction readTransaction, int layer,
                                                              @Nullable final Tuple lastPrimaryKey, int maxNumRead) {
+        final OnReadListener onReadListener = getOnReadListener();
         final byte[] layerPrefix = getDataSubspace().pack(Tuple.from(layer));
         final Range range =
                 lastPrimaryKey == null
@@ -317,30 +331,29 @@ public Iterable<Node<NodeReferenceWithVector>> scanLayer(@Nonnull final ReadTran
         final AsyncIterable<KeyValue> itemsIterable =
                 readTransaction.getRange(range,
                         maxNumRead, false, StreamingMode.ITERATOR);
-        int numRead = 0;
         Tuple nodePrimaryKey = null;
         ImmutableList.Builder<Node<NodeReferenceWithVector>> nodeBuilder = ImmutableList.builder();
-        ImmutableList.Builder<NodeReferenceWithVector> neighborsBuilder = ImmutableList.builder();
+        ImmutableList.Builder<NodeReferenceWithVector> neighborsBuilder = null;
         for (final KeyValue item: itemsIterable) {
-            final NodeReferenceWithVector neighbor =
-                    neighborFromRaw(layer, item.getKey(), item.getValue());
-            final Tuple primaryKeyFromNodeReference = neighbor.getPrimaryKey();
-            if (nodePrimaryKey == null) {
-                nodePrimaryKey = primaryKeyFromNodeReference;
-            } else {
-                if (!nodePrimaryKey.equals(primaryKeyFromNodeReference)) {
+            final byte[] key = item.getKey();
+            final byte[] value = item.getValue();
+            onReadListener.onKeyValueRead(layer, key, value);
+
+            final Tuple neighborKeyTuple = getDataSubspace().unpack(key);
+            final Tuple neighborValueTuple = Tuple.fromBytes(value);
+            final NodeReferenceWithVector neighbor = neighborFromTuples(neighborKeyTuple, neighborValueTuple);
+            final Tuple nodePrimaryKeyFromNeighbor = neighborKeyTuple.getNestedTuple(1);
+            if (nodePrimaryKey == null || !nodePrimaryKey.equals(nodePrimaryKeyFromNeighbor)) {
+                if (nodePrimaryKey != null) {
                     nodeBuilder.add(getNodeFactory().create(nodePrimaryKey, null, neighborsBuilder.build()));
                 }
+                nodePrimaryKey = nodePrimaryKeyFromNeighbor;
+                neighborsBuilder = ImmutableList.builder();
             }
             neighborsBuilder.add(neighbor);
-            numRead ++;
-        }
-
-        // there may be a rest
-        if (numRead > 0 && numRead < maxNumRead) {
-            nodeBuilder.add(getNodeFactory().create(nodePrimaryKey, null, neighborsBuilder.build()));
         }
 
+        // there may be a rest; throw it away
         return nodeBuilder.build();
     }
 }

fdb-extensions/src/main/java/com/apple/foundationdb/async/hnsw/StorageAdapter.java

@@ -168,7 +168,7 @@ void writeNode(@Nonnull Transaction transaction, @Nonnull Node<N> node, int laye
                    @Nonnull NeighborsChangeSet<N> changeSet);
 
     /**
-     * Scans a specified layer of the directory, returning an iterable sequence of nodes.
+     * Scans a specified layer of the structure, returning an iterable sequence of nodes.
      * <p>
      * This method allows for paginated scanning of a layer. The scan can be started from the beginning of the layer by
      * passing {@code null} for the {@code lastPrimaryKey}, or it can be resumed from a previous point by providing the