Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(#9542): implement CouchDB Nouveau #9541

Draft
wants to merge 24 commits into
base: 9542_freetext_tco
Choose a base branch
from
Draft

feat(#9542): implement CouchDB Nouveau #9541

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
55ba2ba
build image for nouveau
m5r Oct 14, 2024
653c77c
nouveau-backed fulltext search index
m5r Oct 14, 2024
76c34b8
couchdb & nouveau: 3.4.1 => 3.4.2
m5r Nov 7, 2024
6502ea1
`contacts_by_type_freetext` & `reports_by_freetext` nouveau-style
m5r Nov 7, 2024
e6c0a59
remove freetext views 👋
m5r Nov 18, 2024
950e3c5
all 3 indexes indexing properly
m5r Nov 20, 2024
b9b64fb
debug problematic keys
m5r Nov 20, 2024
90a1599
debug problematic keys with numbers...
m5r Nov 25, 2024
9fa0db7
ok contacts_by_freetext done
m5r Nov 27, 2024
d507f20
ok contacts_by_type_freetext
m5r Nov 27, 2024
123cb7b
ok reports_by_freetext, excluding `_attachments`
m5r Nov 27, 2024
ae8f4c8
get rid of debugging logs
m5r Nov 27, 2024
6fbb492
got ordering to work with the following request parameters:
m5r Nov 27, 2024
ce98950
`?q={search}+AND+cht_contact_type:district_hospital`
m5r Nov 28, 2024
f333b28
patch eslint couchdb plugin
m5r Dec 2, 2024
8d19d32
skip freetext views unit tests
m5r Dec 2, 2024
31d274d
fix monitoring unit test
m5r Dec 2, 2024
0d48625
fix monitoring unit test
m5r Dec 2, 2024
a713e34
add nouveau service to compose file for couchdb cluster
m5r Dec 12, 2024
db53828
feat(#9690): monitor nouveau (#9700)
m5r Dec 16, 2024
e1d1701
Merge branch '9542_freetext_tco' into couchdb-nouveau
jkuester Feb 21, 2025
3e93668
Remove webapp unit tests for views that do not exist
jkuester Feb 21, 2025
94c2104
Add custom `contact` index to reports_by_freetext
jkuester Feb 26, 2025
88facc1
Hack the CI to just publish the images without waiting for tests
jkuester Feb 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -370,7 +370,7 @@ jobs:
if: ${{ failure() }}

publish:
needs: [tests, config-tests, test-cht-form, translations]
needs: [build]
name: Publish branch build
runs-on: ubuntu-22.04
timeout-minutes: 60
Expand Down
50 changes: 45 additions & 5 deletions api/src/services/monitoring.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,15 @@ const VIEW_INDEXES_TO_MONITOR = {
users: ['users'],
};

const NOUVEAU_INDEXES_TO_MONITOR = {
medic: {
'medic-nouveau': [
'contacts_by_freetext',
'reports_by_freetext',
],
},
};

const MESSAGE_QUEUE_STATUS_KEYS = ['due', 'scheduled', 'muted', 'failed', 'delivered'];
const fromEntries = (keys, value) => {
// "shim" of Object.fromEntries
Expand Down Expand Up @@ -115,7 +124,7 @@ const getFragmentation = ({ sizes }, viewIndexInfos) => {
return totalFile / totalActive;
};

const mapDbInfo = (dbInfo, viewIndexInfos) => {
const mapDbInfo = (dbInfo, viewIndexInfos, nouveauIndexInfos) => {
return {
name: dbInfo.db_name || '',
update_sequence: getSequenceNumber(dbInfo.update_seq),
Expand All @@ -131,8 +140,13 @@ const mapDbInfo = (dbInfo, viewIndexInfos) => {
sizes: {
active: defaultNumber(viewIndexInfo.view_index?.sizes?.active),
file: defaultNumber(viewIndexInfo.view_index?.sizes?.file),
}
}))
},
})),
nouveau_indexes: nouveauIndexInfos?.map(nouveauIndexInfo => ({
name: nouveauIndexInfo.name || '',
num_docs: defaultNumber(nouveauIndexInfo.search_index.num_docs),
disk_size: defaultNumber(nouveauIndexInfo.search_index.disk_size),
})),
};
};

Expand Down Expand Up @@ -167,11 +181,37 @@ const fetchViewIndexInfosForDb = (db) => Promise.all(

const fetchAllViewIndexInfos = () => Promise.all(Object.keys(VIEW_INDEXES_TO_MONITOR).map(fetchViewIndexInfosForDb));

const fetchNouveauIndexInfo = (db, designDoc, indexName) => request
.get({
url: `${environment.serverUrl}/${db}/_design/${designDoc}/_nouveau_info/${indexName}`,
json: true
})
.catch(err => {
logger.error('Error fetching nouveau index info: %o', err);
return null;
});

const fetchNouveauIndexInfosForDdoc = (db, ddoc) => NOUVEAU_INDEXES_TO_MONITOR[db][ddoc].map(
indexName => fetchNouveauIndexInfo(DBS_TO_MONITOR[db], ddoc, indexName),
);

const fetchNouveauIndexInfosForDb = (db) => Promise.all(Object.keys(NOUVEAU_INDEXES_TO_MONITOR[db]).flatMap(
ddoc => fetchNouveauIndexInfosForDdoc(db, ddoc),
)).then((nouveauIndexInfos) => nouveauIndexInfos.filter(info => info));

const fetchAllNouveauIndexInfos = () => Promise.all(
Object.keys(NOUVEAU_INDEXES_TO_MONITOR).map(fetchNouveauIndexInfosForDb),
);

const getDbInfos = async () => {
const [dbInfos, viewIndexInfos] = await Promise.all([fetchDbsInfo(), fetchAllViewIndexInfos()]);
const [dbInfos, viewIndexInfos, nouveauIndexInfos] = await Promise.all([
fetchDbsInfo(),
fetchAllViewIndexInfos(),
fetchAllNouveauIndexInfos(),
]);
const result = {};
Object.keys(DBS_TO_MONITOR).forEach((dbKey, i) => {
result[dbKey] = mapDbInfo(dbInfos[i], viewIndexInfos[i]);
result[dbKey] = mapDbInfo(dbInfos[i], viewIndexInfos[i], nouveauIndexInfos[i]);
});
return result;
};
Expand Down
144 changes: 127 additions & 17 deletions api/tests/mocha/services/monitoring.spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,35 @@ const VIEW_INDEX_INFO_BY_DESIGN = {
}
};

const NOUVEAU_DDOCS_BY_DB = {
[environment.db]: ['medic-nouveau'],
};

const NOUVEAU_INDEX_INFO_BY_DDOC = {
'medic-nouveau': {
reports_by_freetext: {
name: '_design/medic-nouveau/reports_by_freetext',
search_index: {
update_seq: 1956891,
purge_seq: 0,
num_docs: 183741,
disk_size: 157258510,
signature: 'cfd67cbb4800308021b6547bcf21cbf99b9476186b5251f317b221225714c5d3',
},
},
contacts_by_freetext: {
name: '_design/medic-nouveau/contacts_by_freetext',
search_index: {
update_seq: 1956891,
purge_seq: 0,
num_docs: 207734,
disk_size: 76815351,
signature: '46de1dfc576838494f798264571dc59658db7ea164915dd459a7752c31591ae6',
},
},
},
};

const setUpMocks = () => {
sinon.stub(deployInfo, 'get').resolves({ version: '5.3.2' });
sinon.stub(request, 'get')
Expand All @@ -173,6 +202,16 @@ const setUpMocks = () => {
.resolves(VIEW_INDEX_INFO_BY_DESIGN[designDoc]);
});
});
Object.keys(NOUVEAU_DDOCS_BY_DB).forEach(dbName => {
NOUVEAU_DDOCS_BY_DB[dbName].forEach(designDoc => {
Object.keys(NOUVEAU_INDEX_INFO_BY_DDOC[designDoc]).forEach(indexName => {
request.get
.withArgs(
sinon.match({ url: `${environment.serverUrl}/${dbName}/_design/${designDoc}/_nouveau_info/${indexName}` }),
).resolves(NOUVEAU_INDEX_INFO_BY_DDOC[designDoc][indexName]);
});
});
});
sinon.stub(request, 'post').withArgs(sinon.match({ url: `${environment.serverUrl}/_dbs_info` }))
.resolves(dbInfos);
sinon.stub(db.sentinel, 'get').withArgs('_local/transitions-seq')
Expand Down Expand Up @@ -250,6 +289,7 @@ const getExpectedViewIndexes = (dbName) => {

const getCurrentDdocNames = (db) => getBundledDdocs(db)
.then(ddocs => ddocs
.filter(ddoc => !!ddoc.views)
.map(ddoc => ddoc._id)
.map(ddocId => ddocId.split('/')[1]));

Expand Down Expand Up @@ -283,9 +323,21 @@ describe('Monitoring service', () => {
update_sequence: 100,
sizes: {
active: 600,
file: 700
file: 700,
},
view_indexes: getExpectedViewIndexes(environment.db)
view_indexes: getExpectedViewIndexes(environment.db),
nouveau_indexes: [
{
disk_size: 76815351,
name: '_design/medic-nouveau/contacts_by_freetext',
num_docs: 207734,
},
{
disk_size: 157258510,
name: '_design/medic-nouveau/reports_by_freetext',
num_docs: 183741,
},
],
},
sentinel: {
doc_count: 30,
Expand All @@ -297,7 +349,8 @@ describe('Monitoring service', () => {
active: 500,
file: 500
},
view_indexes: getExpectedViewIndexes(`${environment.db}-sentinel`)
view_indexes: getExpectedViewIndexes(`${environment.db}-sentinel`),
nouveau_indexes: undefined,
},
users: {
doc_count: 50,
Expand All @@ -309,7 +362,8 @@ describe('Monitoring service', () => {
active: 500,
file: 501
},
view_indexes: getExpectedViewIndexes('_users')
view_indexes: getExpectedViewIndexes('_users'),
nouveau_indexes: undefined,
},
usersmeta: {
doc_count: 40,
Expand All @@ -321,7 +375,8 @@ describe('Monitoring service', () => {
active: 500,
file: 5000
},
view_indexes: getExpectedViewIndexes(`${environment.db}-users-meta`)
view_indexes: getExpectedViewIndexes(`${environment.db}-users-meta`),
nouveau_indexes: undefined,
}
});
chai.expect(actual.messaging).to.deep.equal({
Expand All @@ -347,6 +402,14 @@ describe('Monitoring service', () => {
[{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-admin/_info` }],
[{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-client/_info` }],
[{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-conflicts/_info` }],
[{
json: true,
url: `${environment.serverUrl}/${environment.db}/_design/medic-nouveau/_nouveau_info/contacts_by_freetext`,
}],
[{
json: true,
url: `${environment.serverUrl}/${environment.db}/_design/medic-nouveau/_nouveau_info/reports_by_freetext`,
}],
[{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-scripts/_info` }],
[{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-sms/_info` }],
[{ json: true, url: `${environment.serverUrl}/${environment.db}-sentinel/_design/sentinel/_info` }],
Expand Down Expand Up @@ -394,7 +457,19 @@ describe('Monitoring service', () => {
active: 600,
file: 700
},
view_indexes: getExpectedViewIndexes(environment.db)
view_indexes: getExpectedViewIndexes(environment.db),
nouveau_indexes: [
{
disk_size: 76815351,
name: '_design/medic-nouveau/contacts_by_freetext',
num_docs: 207734,
},
{
disk_size: 157258510,
name: '_design/medic-nouveau/reports_by_freetext',
num_docs: 183741,
},
],
},
sentinel: {
doc_count: 30,
Expand All @@ -406,7 +481,8 @@ describe('Monitoring service', () => {
active: 500,
file: 500
},
view_indexes: getExpectedViewIndexes(`${environment.db}-sentinel`)
view_indexes: getExpectedViewIndexes(`${environment.db}-sentinel`),
nouveau_indexes: undefined,
},
users: {
doc_count: 50,
Expand All @@ -418,7 +494,8 @@ describe('Monitoring service', () => {
active: 500,
file: 501
},
view_indexes: getExpectedViewIndexes('_users')
view_indexes: getExpectedViewIndexes('_users'),
nouveau_indexes: undefined,
},
usersmeta: {
doc_count: 40,
Expand All @@ -430,7 +507,8 @@ describe('Monitoring service', () => {
active: 500,
file: 5000
},
view_indexes: getExpectedViewIndexes(`${environment.db}-users-meta`)
view_indexes: getExpectedViewIndexes(`${environment.db}-users-meta`),
nouveau_indexes: undefined,
}
});
chai.expect(actual.messaging).to.deep.equal({
Expand Down Expand Up @@ -483,6 +561,14 @@ describe('Monitoring service', () => {
[{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-admin/_info` }],
[{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-client/_info` }],
[{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-conflicts/_info` }],
[{
json: true,
url: `${environment.serverUrl}/${environment.db}/_design/medic-nouveau/_nouveau_info/contacts_by_freetext`,
}],
[{
json: true,
url: `${environment.serverUrl}/${environment.db}/_design/medic-nouveau/_nouveau_info/reports_by_freetext`,
}],
[{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-scripts/_info` }],
[{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-sms/_info` }],
[{ json: true, url: `${environment.serverUrl}/${environment.db}-sentinel/_design/sentinel/_info` }],
Expand Down Expand Up @@ -531,7 +617,8 @@ describe('Monitoring service', () => {
active: -1,
file: -1
},
view_indexes: []
view_indexes: [],
nouveau_indexes: [],
},
sentinel: {
doc_count: -1,
Expand All @@ -543,7 +630,8 @@ describe('Monitoring service', () => {
active: -1,
file: -1
},
view_indexes: []
view_indexes: [],
nouveau_indexes: undefined,
},
users: {
doc_count: -1,
Expand All @@ -555,7 +643,8 @@ describe('Monitoring service', () => {
active: -1,
file: -1
},
view_indexes: []
view_indexes: [],
nouveau_indexes: undefined,
},
usersmeta: {
doc_count: -1,
Expand All @@ -567,7 +656,8 @@ describe('Monitoring service', () => {
active: -1,
file: -1
},
view_indexes: []
view_indexes: [],
nouveau_indexes: undefined,
}
});
chai.expect(actual.messaging).to.deep.equal({
Expand All @@ -591,6 +681,14 @@ describe('Monitoring service', () => {
[{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-admin/_info` }],
[{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-client/_info` }],
[{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-conflicts/_info` }],
[{
json: true,
url: `${environment.serverUrl}/${environment.db}/_design/medic-nouveau/_nouveau_info/contacts_by_freetext`,
}],
[{
json: true,
url: `${environment.serverUrl}/${environment.db}/_design/medic-nouveau/_nouveau_info/reports_by_freetext`,
}],
[{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-scripts/_info` }],
[{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-sms/_info` }],
[{ json: true, url: `${environment.serverUrl}/${environment.db}-sentinel/_design/sentinel/_info` }],
Expand Down Expand Up @@ -627,7 +725,8 @@ describe('Monitoring service', () => {
active: -1,
file: -1
},
view_indexes: []
view_indexes: [],
nouveau_indexes: [],
},
sentinel: {
doc_count: -1,
Expand All @@ -639,7 +738,8 @@ describe('Monitoring service', () => {
active: -1,
file: -1
},
view_indexes: []
view_indexes: [],
nouveau_indexes: undefined,
},
users: {
doc_count: -1,
Expand All @@ -651,7 +751,8 @@ describe('Monitoring service', () => {
active: -1,
file: -1
},
view_indexes: []
view_indexes: [],
nouveau_indexes: undefined,
},
usersmeta: {
doc_count: -1,
Expand All @@ -663,7 +764,8 @@ describe('Monitoring service', () => {
active: -1,
file: -1
},
view_indexes: []
view_indexes: [],
nouveau_indexes: undefined,
}
});
chai.expect(actual.messaging).to.deep.equal({
Expand Down Expand Up @@ -714,6 +816,14 @@ describe('Monitoring service', () => {
[{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-admin/_info` }],
[{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-client/_info` }],
[{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-conflicts/_info` }],
[{
json: true,
url: `${environment.serverUrl}/${environment.db}/_design/medic-nouveau/_nouveau_info/contacts_by_freetext`,
}],
[{
json: true,
url: `${environment.serverUrl}/${environment.db}/_design/medic-nouveau/_nouveau_info/reports_by_freetext`,
}],
[{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-scripts/_info` }],
[{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-sms/_info` }],
[{ json: true, url: `${environment.serverUrl}/${environment.db}-sentinel/_design/sentinel/_info` }],
Expand Down
Loading
Loading