Skip to content

QA-589: Feature/vector index metrics #64

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

QA-589: Feature/vector index metrics #64

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
664ea21
add first draft vector index test
dothebart Jun 27, 2025
560be7e
revert merge issue
dothebart Jun 27, 2025
cf67d4e
reduce document size
dothebart Jun 27, 2025
fb9dfd8
reduce memory footprint
dothebart Jun 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions simple/run-small-indexes.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
function main () {
require("./simple/test").test({
outputCsv: true,
small: true,
indexes: true
});
}
if (typeof arango !== "undefined") {
main();
}
137 changes: 101 additions & 36 deletions simple/test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
/* jshint globalstrict:false, strict:false, maxlen: 500 */
/* global aql, GLOBAL, makeGraph, makeTree, */
"use strict";
const internal = require("internal");
const arango = internal.arango;
Expand Down Expand Up @@ -96,7 +98,7 @@ function toAsciiTable (title, out) {

return table.toString();
}

let randomPoint;
exports.test = function (testParams) {
testParams.tiny = testParams.tiny || false;
testParams.small = testParams.small || false;
Expand Down Expand Up @@ -135,7 +137,7 @@ exports.test = function (testParams) {

// Substring first 5 characters to limit to A.B.C format and not use any `nightly`, `rc`, `preview` etc.
const serverVersion = (((typeof arango) !== "undefined") ? arango.getVersion() : internal.version).split("-")[0];
testParams.zkdMdiRenamed = semver.satisfies(serverVersion, ">3.11.99") ;
testParams.zkdMdiRenamed = semver.satisfies(serverVersion, ">3.11.99");
const isEnterprise = internal.isEnterprise();
const isCluster = internal.isCluster();

Expand Down Expand Up @@ -214,14 +216,14 @@ exports.test = function (testParams) {
let errors = [];
for (let i = 0; i < tests.length; ++i) {
let test = tests[i];
print(test)
print(test);
try {
if (!(test['version'] === undefined || semver.satisfies(serverVersion, test['version']))) {
print(`skipping test ${test['name']}, requires version ${test['version']}`);
} else if (test['legacy'] && !testParams.legacy) {
print(`skipping legacy test ${test['name']}`);
if (!(test["version"] === undefined || semver.satisfies(serverVersion, test["version"]))) {
print(`skipping test ${test["name"]}, requires version ${test["version"]}`);
} else if (test["legacy"] && !testParams.legacy) {
print(`skipping legacy test ${test["name"]}`);
} else {
print(`running test ${test['name']}`);
print(`running test ${test["name"]}`);
for (let j = 0; j < options.collections.length; ++j) {
let collection = options.collections[j];

Expand All @@ -231,7 +233,7 @@ exports.test = function (testParams) {
const stats = calc(results, options);

const result = {
name: test['name'],
name: test["name"],
runs: options.runs,
min: stats.min.toFixed(options.digits),
max: stats.max.toFixed(options.digits),
Expand All @@ -253,8 +255,8 @@ exports.test = function (testParams) {
} // for j
}
} catch (ex) {
print(`exception in test ${test['name']}: ${String(ex)}\n${String(ex.stack)}`);
errors.push({ name: test['name'], error: ex });
print(`exception in test ${test["name"]}: ${String(ex)}\n${String(ex.stack)}`);
errors.push({ name: test["name"], error: ex });
GLOBAL.returnValue = 1;
}
} // for i
Expand Down Expand Up @@ -810,7 +812,6 @@ exports.test = function (testParams) {
createArangoSearch(viewParams);
}
},

fill = function (params) {
let c = db._collection(params.collection),
n = parseInt(params.collection.replace(/[a-z]+/g, ""), 10),
Expand All @@ -819,34 +820,67 @@ exports.test = function (testParams) {
for (let i = 0; i < docSize; ++i) {
doc["value" + i] = i;
}
const seed = 12132390894;
const randomNumberGeneratorFloat = function (seed) {
const rng = (function *(seed) {
while (true) {
const nextVal = Math.cos(seed++);
yield nextVal;
}
})(seed);

return function () {
return rng.next().value;
};
};
let getVector = function () {
let gen = randomNumberGeneratorFloat(seed);

return Array.from({
length: params.dimension
}, () => gen());
};

const batchSize = params.batchSize;
if (batchSize) {
// perform babies operations
for (let i = 0; i < n / batchSize; ++i) {
let docs = [];
for (let j = 0; j < batchSize; ++j) {
docs.push({ _key: "test" + (i * batchSize + j), ...doc });
let oneDoc = { _key: "test" + (i * batchSize + j), ...doc };
if (params.type === "vector") {
oneDoc["vector"] = getVector();
if (n / 2 === i) {
randomPoint = oneDoc["vector"];
}
}
docs.push(oneDoc);
}
c.insert(docs);
}
} else {
// perform single document operations
for (let i = 0; i < n; ++i) {
doc._key = "test" + i;
if (params.type === "vector") {
doc["vector"] = getVector();
if (n / 2 === i) {
randomPoint = doc["vector"];
}
}
c.insert(doc);
}
}
},

// /////////////////////////////////////////////////////////////////////////////
// indexes tests
// /////////////////////////////////////////////////////////////////////////////

insertIndexOne = function (params) {
let c = db._collection(params.collection),
n = parseInt(params.collection.replace(/[a-z]+/g, ""), 10);

// perform small batch document operations
const batchSize = params.batchSize || 100;
let docs = [];
Expand All @@ -868,11 +902,11 @@ exports.test = function (testParams) {
}
}
},

insertIndexTwo = function (params) {
let c = db._collection(params.collection),
n = parseInt(params.collection.replace(/[a-z]+/g, ""), 10);

// perform small batch document operations
const batchSize = params.batchSize || 100;
let docs = [];
Expand Down Expand Up @@ -1002,10 +1036,10 @@ exports.test = function (testParams) {
// edgeTests
// /////////////////////////////////////////////////////////////////////////////

traversalProjections = function(params) {
traversalProjections = function () {
// Note that depth 8 is good for all three sizes small (6), medium (7)
// and big (8). Depending on the size, we create a different tree.
db._query(`FOR v IN 0..8 OUTBOUND "TreeV/S1:K1" GRAPH "Tree" RETURN v.data`, {}, {}, {silent});
db._query("FOR v IN 0..8 OUTBOUND \"TreeV/S1:K1\" GRAPH \"Tree\" RETURN v.data", {}, {}, {silent});
},

outbound = function (params) {
Expand Down Expand Up @@ -2390,7 +2424,7 @@ exports.test = function (testParams) {
setup: function (params) {
drop(params);
create(params);
db[params.collection].ensureIndex({ type: "persistent", fields: ["value1"] });
db[params.collection].ensureIndex({ type: "persistent", fields: ["value1"] });
},
type: "number",
teardown: drop
Expand All @@ -2403,7 +2437,7 @@ exports.test = function (testParams) {
setup: function (params) {
drop(params);
create(params);
db[params.collection].ensureIndex({ type: "persistent", fields: ["value1"], estimates: false });
db[params.collection].ensureIndex({ type: "persistent", fields: ["value1"], estimates: false });
},
type: "number",
teardown: drop
Expand All @@ -2416,7 +2450,7 @@ exports.test = function (testParams) {
setup: function (params) {
drop(params);
create(params);
db[params.collection].ensureIndex({ type: "persistent", fields: ["value1"] });
db[params.collection].ensureIndex({ type: "persistent", fields: ["value1"] });
},
type: "string",
teardown: drop
Expand All @@ -2429,7 +2463,7 @@ exports.test = function (testParams) {
setup: function (params) {
drop(params);
create(params);
db[params.collection].ensureIndex({ type: "persistent", fields: ["value1"], estimates: false });
db[params.collection].ensureIndex({ type: "persistent", fields: ["value1"], estimates: false });
},
type: "string",
teardown: drop
Expand All @@ -2442,8 +2476,8 @@ exports.test = function (testParams) {
setup: function (params) {
drop(params);
create(params);
db[params.collection].ensureIndex({ type: "persistent", fields: ["value1"] });
db[params.collection].ensureIndex({ type: "persistent", fields: ["value2"] });
db[params.collection].ensureIndex({ type: "persistent", fields: ["value1"] });
db[params.collection].ensureIndex({ type: "persistent", fields: ["value2"] });
},
type: "number",
teardown: drop
Expand All @@ -2456,8 +2490,8 @@ exports.test = function (testParams) {
setup: function (params) {
drop(params);
create(params);
db[params.collection].ensureIndex({ type: "persistent", fields: ["value1"], estimates: false });
db[params.collection].ensureIndex({ type: "persistent", fields: ["value2"], estimates: false });
db[params.collection].ensureIndex({ type: "persistent", fields: ["value1"], estimates: false });
db[params.collection].ensureIndex({ type: "persistent", fields: ["value2"], estimates: false });
},
type: "number",
teardown: drop
Expand All @@ -2470,8 +2504,8 @@ exports.test = function (testParams) {
setup: function (params) {
drop(params);
create(params);
db[params.collection].ensureIndex({ type: "persistent", fields: ["value1"] });
db[params.collection].ensureIndex({ type: "persistent", fields: ["value2"] });
db[params.collection].ensureIndex({ type: "persistent", fields: ["value1"] });
db[params.collection].ensureIndex({ type: "persistent", fields: ["value2"] });
},
type: "string",
teardown: drop
Expand All @@ -2484,13 +2518,44 @@ exports.test = function (testParams) {
setup: function (params) {
drop(params);
create(params);
db[params.collection].ensureIndex({ type: "persistent", fields: ["value1"], estimates: false });
db[params.collection].ensureIndex({ type: "persistent", fields: ["value2"], estimates: false });
db[params.collection].ensureIndex({ type: "persistent", fields: ["value1"], estimates: false });
db[params.collection].ensureIndex({ type: "persistent", fields: ["value2"], estimates: false });
},
type: "string",
teardown: drop
}
},
{
name: "indexes-vector",
params: {
vector: true,
dimension: 500,
setup: function (params) {
drop(params);
create(params);
fill(params);
db[params.collection].ensureIndex({
name: "vector_l2",
type: "vector",
fields: ["vector"],
inBackground: false,
params: {
metric: "l2",
dimension: params.dimension,
nLists: 10,
trainingIterations: 10
}
});
},
func: function (params) {
db._query(aql`FOR d IN
${db[params.collection]}
SORT APPROX_NEAR_L2(d.vector, ${randomPoint}) LIMIT 5 RETURN {key: d._key}`).toArray();
},
type: "vector",
teardown: drop
}
}
];

// Tests without collections/IO, to focus on aql block performance.
Expand Down Expand Up @@ -2549,7 +2614,7 @@ exports.test = function (testParams) {
{
name: "traversal-projections",
params: { func: traversalProjections }
},
},
{
name: "traversal-outbound-1",
params: { func: outbound, minDepth: 1, maxDepth: 1, loops: 1000 }
Expand Down Expand Up @@ -3276,7 +3341,7 @@ exports.test = function (testParams) {
}
db._query(
params.queryString,
bindParam,
bindParam
);
}

Expand Down Expand Up @@ -3330,7 +3395,7 @@ exports.test = function (testParams) {
FILTER d.x == 9 and d.y >= 52
RETURN d`
}
},
}
];

const runSatelliteGraphTests = (testParams.satelliteGraphTests && isEnterprise && isCluster);
Expand Down Expand Up @@ -3414,7 +3479,7 @@ exports.test = function (testParams) {
setup: function (params) {
db._drop(params.collection);
let col = db._create(params.collection);
let type = (testParams.zkdMdiRenamed) ? "mdi":"zkd";
let type = (testParams.zkdMdiRenamed) ? "mdi" : "zkd";
col.ensureIndex({type: type, name: "mdiIndex", fields: ["x", "y"], fieldValueTypes: "double"});
db._query(`
FOR i IN 0..${params.collectionSize}
Expand Down Expand Up @@ -3586,7 +3651,7 @@ exports.test = function (testParams) {

runTestSuite("Arango Search No Materialization", arangosearchNoMaterializationTests, options);
}

// indexes tests
if (testParams.indexes) {
options = {
Expand Down