Skip to content

Commit aa3ea87

Browse files
Copilotmathiasrw
Copilot
and
mathiasrw
authored
Update indexes when bulk loading data to fix #1027 (#2209)
Co-authored-by: copilot-swe-agent[bot] <[email protected]> Co-authored-by: mathiasrw <[email protected]> Co-authored-by: Mathias Wulff <[email protected]>
1 parent 30bc356 commit aa3ea87

File tree

6 files changed

+737
-0
lines changed

6 files changed

+737
-0
lines changed

‎src/60createtable.js‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -448,6 +448,25 @@ yy.CreateTable.prototype.execute = function (databaseid, params, cb) {
448448
table.uniqs[uk.hh][ukaddr] = r;
449449
});
450450
}
451+
452+
// Update regular indices (non-unique)
453+
if (table.inddefs) {
454+
for (var indexid in table.inddefs) {
455+
var indexdef = table.inddefs[indexid];
456+
var hh = indexdef.hh;
457+
if (table.indices[hh]) {
458+
var addr = new Function('r,params,alasql', 'return ' + indexdef.rightfns)(
459+
r,
460+
params,
461+
alasql
462+
);
463+
if (!table.indices[hh][addr]) {
464+
table.indices[hh][addr] = [];
465+
}
466+
table.indices[hh][addr].push(r);
467+
}
468+
}
469+
}
451470
}
452471

453472
// Trigger prevent functionality

‎test/performance/#1027/README.md‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
# Performance Test for Issue #1027
2+
3+
## Issue Description
4+
Users report performance issues when querying AlaSQL database with joins. Response times of several minutes occur when:
5+
1. Loading JSON files into database tables (build time)
6+
2. Creating tables with ~2500 total records across 3 tables
7+
3. Running synchronous queries with 2 joins at runtime
8+
9+
## Root Cause
10+
The performance issue was caused by regular indices (created with `CREATE INDEX`) not being updated when records are inserted via:
11+
1. `INSERT` statements using `table.insert()` function
12+
2. Direct data loading by pushing to `table.data` array
13+
14+
In `src/60createtable.js`, the `table.insert()` function only updated:
15+
- Primary key indices (`table.pk`)
16+
- Unique indices (`table.uk`)
17+
18+
But it did NOT update regular indices stored in `table.indices[hh]`.
19+
20+
## Performance Impact
21+
Without working indices:
22+
- Queries with joins perform full table scans (O(n*m) complexity)
23+
- With 1000 records in each of 3 tables, a 2-join query does ~1M comparisons
24+
- Performance degrades dramatically with larger datasets
25+
26+
With working indices:
27+
- Index lookups are O(1) on average
28+
- Query performance improves by orders of magnitude
29+
30+
## Fix Implemented
31+
The fix updates regular indices during INSERT operations by iterating through all index definitions in `table.inddefs` and adding the inserted record to the appropriate index bucket.
32+
33+
## Test Files
34+
35+
### Performance Comparison Test (Recommended)
36+
**`perf-comparison.js`** - Unified test that compares both loading methods side-by-side:
37+
- Uses functions to run the same setup twice
38+
- Tests both INSERT statements and direct table.data.push()
39+
- Prints clear comparison showing indices are populated differently
40+
- Can run individual tests or both with command line flags
41+
42+
Run it with:
43+
```bash
44+
# Compare both methods
45+
node test/performance/#1027/perf-comparison.js both
46+
# or just:
47+
node test/performance/#1027/perf-comparison.js
48+
49+
# Test INSERT method only
50+
node test/performance/#1027/perf-comparison.js insert
51+
52+
# Test direct loading only
53+
node test/performance/#1027/perf-comparison.js direct
54+
```
55+
56+
### Individual Performance Tests
57+
These scripts test each method separately:
58+
59+
- `perf-join-index.js` - Tests join performance with indices using INSERT statements
60+
- `perf-direct-load.js` - Tests the scenario of loading data directly via table.data.push()
61+
62+
Run them with:
63+
```bash
64+
node test/performance/#1027/perf-join-index.js
65+
node test/performance/#1027/perf-direct-load.js
66+
```
67+
68+
### Correctness Tests (Mocha)
69+
The main test file `test/test1027.js` uses mocha to verify:
70+
- Indices are created and populated correctly
71+
- Indices are updated during INSERT operations
72+
- Composite indices work correctly
73+
- JOIN queries use indices properly
74+
- Multiple indices on the same table work
75+
76+
Run with the full test suite:
77+
```bash
78+
yarn test
79+
```
80+
81+
## Expected Behavior
82+
After the fix:
83+
- INSERT statements properly maintain indices during data loading
84+
- Queries complete in milliseconds with either loading method
85+
- Direct data loading builds indices on-the-fly during first query
86+
- Both methods produce identical results with similar query performance

‎test/performance/#1027/perf-comparison.js‎

Lines changed: 239 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,239 @@
1+
// Performance test for issue #1027 - Comparing data loading methods
2+
// This script tests the same query with two different data loading approaches:
3+
// 1. Using INSERT statements (recommended)
4+
// 2. Using direct table.data.push() (simulating JSON data loading)
5+
6+
var alasql = require('../../..');
7+
8+
// Configuration
9+
var config = {
10+
numCompanies: 10,
11+
numDepartments: 100, // 10 departments per company
12+
numUsers: 1000, // 10 users per department
13+
queryIterations: 100,
14+
};
15+
16+
// Parse command line argument for loading method
17+
var loadMethod = process.argv[2] || 'both'; // 'insert', 'direct', or 'both'
18+
19+
console.log('=== Performance Test #1027 - Data Loading Comparison ===\n');
20+
console.log('Configuration:');
21+
console.log(' Companies: ' + config.numCompanies);
22+
console.log(' Departments: ' + config.numDepartments);
23+
console.log(' Users: ' + config.numUsers);
24+
console.log(' Query iterations: ' + config.queryIterations);
25+
console.log();
26+
27+
// Function to setup tables with indices
28+
function setupTables(dbName) {
29+
alasql('CREATE DATABASE ' + dbName);
30+
alasql('USE ' + dbName);
31+
32+
alasql('CREATE TABLE users (id INT PRIMARY KEY, name STRING, department_id INT)');
33+
alasql('CREATE TABLE departments (id INT PRIMARY KEY, name STRING, company_id INT)');
34+
alasql('CREATE TABLE companies (id INT PRIMARY KEY, name STRING)');
35+
36+
// Create indices on join columns
37+
alasql('CREATE INDEX idx_users_dept ON users(department_id)');
38+
alasql('CREATE INDEX idx_dept_company ON departments(company_id)');
39+
}
40+
41+
// Function to load data using INSERT statements
42+
function loadDataWithInsert(dbName) {
43+
// Insert companies
44+
for (var i = 1; i <= config.numCompanies; i++) {
45+
alasql('INSERT INTO companies VALUES (?, ?)', [i, 'Company ' + i]);
46+
}
47+
48+
// Insert departments
49+
for (var i = 1; i <= config.numDepartments; i++) {
50+
var companyId = ((i - 1) % config.numCompanies) + 1;
51+
alasql('INSERT INTO departments VALUES (?, ?, ?)', [i, 'Department ' + i, companyId]);
52+
}
53+
54+
// Insert users
55+
for (var i = 1; i <= config.numUsers; i++) {
56+
var deptId = ((i - 1) % config.numDepartments) + 1;
57+
alasql('INSERT INTO users VALUES (?, ?, ?)', [i, 'User ' + i, deptId]);
58+
}
59+
}
60+
61+
// Function to load data by directly pushing to table.data array
62+
function loadDataDirect(dbName) {
63+
var db = alasql.databases[dbName];
64+
var companies = db.tables.companies;
65+
var departments = db.tables.departments;
66+
var users = db.tables.users;
67+
68+
// Push data directly to arrays
69+
for (var i = 1; i <= config.numCompanies; i++) {
70+
companies.data.push({id: i, name: 'Company ' + i});
71+
}
72+
73+
for (var i = 1; i <= config.numDepartments; i++) {
74+
var companyId = ((i - 1) % config.numCompanies) + 1;
75+
departments.data.push({id: i, name: 'Department ' + i, company_id: companyId});
76+
}
77+
78+
for (var i = 1; i <= config.numUsers; i++) {
79+
var deptId = ((i - 1) % config.numDepartments) + 1;
80+
users.data.push({id: i, name: 'User ' + i, department_id: deptId});
81+
}
82+
}
83+
84+
// Function to verify index population
85+
function verifyIndices(dbName) {
86+
var db = alasql.databases[dbName];
87+
var usersTable = db.tables.users;
88+
var deptsTable = db.tables.departments;
89+
90+
var userIndexDef = usersTable.inddefs['idx_users_dept'];
91+
var userIndexData = usersTable.indices[userIndexDef.hh];
92+
var deptIndexDef = deptsTable.inddefs['idx_dept_company'];
93+
var deptIndexData = deptsTable.indices[deptIndexDef.hh];
94+
95+
return {
96+
userIndexKeys: Object.keys(userIndexData).length,
97+
deptIndexKeys: Object.keys(deptIndexData).length,
98+
};
99+
}
100+
101+
// Function to run query performance test
102+
function runQueryTest(dbName) {
103+
var query = `
104+
SELECT u.name as user_name, d.name as dept_name, c.name as company_name
105+
FROM users u
106+
INNER JOIN departments d ON u.department_id = d.id
107+
INNER JOIN companies c ON d.company_id = c.id
108+
WHERE c.id = 5
109+
`;
110+
111+
var times = [];
112+
var result;
113+
114+
// Run query multiple times
115+
for (var i = 0; i < config.queryIterations; i++) {
116+
var start = Date.now();
117+
result = alasql(query);
118+
times.push(Date.now() - start);
119+
}
120+
121+
// Calculate statistics
122+
var totalTime = times.reduce(function (sum, t) {
123+
return sum + t;
124+
}, 0);
125+
var avgTime = totalTime / times.length;
126+
var minTime = Math.min.apply(null, times);
127+
var maxTime = Math.max.apply(null, times);
128+
129+
return {
130+
resultCount: result.length,
131+
totalTime: totalTime,
132+
avgTime: avgTime,
133+
minTime: minTime,
134+
maxTime: maxTime,
135+
iterations: times.length,
136+
};
137+
}
138+
139+
// Function to print results
140+
function printResults(method, loadTime, indices, queryStats) {
141+
console.log('='.repeat(70));
142+
console.log('Method: ' + method);
143+
console.log('='.repeat(70));
144+
console.log();
145+
console.log('Data Loading:');
146+
console.log(' Time: ' + loadTime + 'ms');
147+
console.log();
148+
console.log('Index Status After Loading:');
149+
console.log(' idx_users_dept keys: ' + indices.userIndexKeys + ' (expected: 100)');
150+
console.log(' idx_dept_company keys: ' + indices.deptIndexKeys + ' (expected: 10)');
151+
if (method === 'INSERT statements') {
152+
if (indices.userIndexKeys === 100 && indices.deptIndexKeys === 10) {
153+
console.log(' ✓ Indices properly populated during INSERT');
154+
} else {
155+
console.log(' ✗ Indices NOT populated (BUG!)');
156+
}
157+
} else {
158+
if (indices.userIndexKeys === 0 && indices.deptIndexKeys === 0) {
159+
console.log(' ℹ Indices empty (expected - built on first query)');
160+
} else {
161+
console.log(' ℹ Indices have data');
162+
}
163+
}
164+
console.log();
165+
console.log('Query Performance (' + queryStats.iterations + ' iterations):');
166+
console.log(' Results returned: ' + queryStats.resultCount + ' (expected: 100)');
167+
console.log(' Total time: ' + queryStats.totalTime + 'ms');
168+
console.log(' Average time: ' + queryStats.avgTime.toFixed(2) + 'ms');
169+
console.log(' Min time: ' + queryStats.minTime + 'ms');
170+
console.log(' Max time: ' + queryStats.maxTime + 'ms');
171+
console.log(' Queries/sec: ' + (1000 / queryStats.avgTime).toFixed(2));
172+
console.log();
173+
}
174+
175+
// Run tests based on command line argument
176+
if (loadMethod === 'insert' || loadMethod === 'both') {
177+
console.log('Testing: INSERT statements (recommended approach)');
178+
console.log('-'.repeat(70));
179+
180+
setupTables('test_insert');
181+
var startLoad = Date.now();
182+
loadDataWithInsert('test_insert');
183+
var loadTime = Date.now() - startLoad;
184+
185+
var indices = verifyIndices('test_insert');
186+
var queryStats = runQueryTest('test_insert');
187+
188+
printResults('INSERT statements', loadTime, indices, queryStats);
189+
190+
alasql('DROP DATABASE test_insert');
191+
}
192+
193+
if (loadMethod === 'direct' || loadMethod === 'both') {
194+
console.log('Testing: Direct table.data.push() (simulating JSON load)');
195+
console.log('-'.repeat(70));
196+
197+
setupTables('test_direct');
198+
var startLoad = Date.now();
199+
loadDataDirect('test_direct');
200+
var loadTime = Date.now() - startLoad;
201+
202+
var indices = verifyIndices('test_direct');
203+
var queryStats = runQueryTest('test_direct');
204+
205+
printResults('Direct table.data.push()', loadTime, indices, queryStats);
206+
207+
alasql('DROP DATABASE test_direct');
208+
}
209+
210+
if (loadMethod === 'both') {
211+
console.log('='.repeat(70));
212+
console.log('SUMMARY');
213+
console.log('='.repeat(70));
214+
console.log();
215+
console.log('Both methods produce the same query results and similar performance.');
216+
console.log();
217+
console.log('Key differences:');
218+
console.log(' 1. INSERT statements populate indices during data loading');
219+
console.log(' 2. Direct loading requires indices to be built on first query');
220+
console.log(' 3. After fix, INSERT method maintains indices properly');
221+
console.log();
222+
console.log('Recommendation: Use INSERT statements for best index maintenance.');
223+
console.log();
224+
}
225+
226+
console.log('=== Test Complete ===');
227+
228+
// Print usage
229+
if (loadMethod !== 'insert' && loadMethod !== 'direct' && loadMethod !== 'both') {
230+
console.log();
231+
console.log('Usage: node perf-comparison.js [method]');
232+
console.log(' method: insert, direct, or both (default: both)');
233+
console.log();
234+
console.log('Examples:');
235+
console.log(' node test/performance/#1027/perf-comparison.js');
236+
console.log(' node test/performance/#1027/perf-comparison.js insert');
237+
console.log(' node test/performance/#1027/perf-comparison.js direct');
238+
console.log(' node test/performance/#1027/perf-comparison.js both');
239+
}

0 commit comments

Comments
 (0)