Skip to content

Commit 7f37382

Browse files
MichaelSolatirobdodson
MichaelSolati
and
robdodson
authored
feat(search): add support for search in different languages, fixes GoogleChrome#4644, fixes GoogleChrome#4643 (GoogleChrome#4922)
* feat(search): add support for search in different languages, fixes GoogleChrome#4644 * Update src/site/_collections/algolia.js Co-authored-by: Rob Dodson <[email protected]> * Update src/site/_collections/algolia.js Co-authored-by: Rob Dodson <[email protected]> * Update src/site/_filters/urls.js Co-authored-by: Rob Dodson <[email protected]> * refactor: make changes Co-authored-by: Rob Dodson <[email protected]>
1 parent d1cbdaa commit 7f37382

21 files changed

+619
-324
lines changed

‎.eleventy.js

+9-6
Original file line numberDiff line numberDiff line change
@@ -56,15 +56,17 @@ const Tooltip = require(`./${componentsDir}/Tooltip`);
5656
const {Video} = require(`./${componentsDir}/Video`);
5757
const YouTube = require(`./${componentsDir}/YouTube`);
5858

59-
const collectionsDir = 'src/site/_collections';
60-
const authors = require(`./${collectionsDir}/authors`);
61-
const blogPostsDescending = require(`./${collectionsDir}/blog-posts-descending`);
62-
const newsletters = require(`./${collectionsDir}/newsletters`);
59+
// Collections
60+
const algolia = require('./src/site/_collections/algolia');
61+
const authors = require(`./src/site/_collections/authors`);
62+
const blogPostsDescending = require(`./src/site/_collections/blog-posts-descending`);
63+
const newsletters = require(`./src/site/_collections/newsletters`);
6364
const {
6465
postsWithLighthouse,
65-
} = require(`./${collectionsDir}/posts-with-lighthouse`);
66-
const tags = require(`./${collectionsDir}/tags`);
66+
} = require(`./src/site/_collections/posts-with-lighthouse`);
67+
const tags = require(`./src/site/_collections/tags`);
6768

69+
// Filters
6870
const filtersDir = 'src/site/_filters';
6971
const consoleDump = require(`./${filtersDir}/console-dump`);
7072
const {i18n} = require(`./${filtersDir}/i18n`);
@@ -168,6 +170,7 @@ module.exports = function (config) {
168170
// ----------------------------------------------------------------------------
169171
// COLLECTIONS
170172
// ----------------------------------------------------------------------------
173+
config.addCollection('algolia', algolia);
171174
config.addCollection('authors', authors);
172175
config.addCollection('blogPosts', blogPostsDescending);
173176
config.addCollection('newsletters', newsletters);

‎algolia.js

+67-30
Original file line numberDiff line numberDiff line change
@@ -13,57 +13,94 @@
1313
* See the License for the specific language governing permissions and
1414
* limitations under the License.
1515
*/
16-
1716
require('dotenv').config();
18-
1917
const algoliasearch = require('algoliasearch');
2018
const fs = require('fs');
21-
const log = require('fancy-log');
19+
const {sizeof} = require('sizeof');
2220

23-
const raw = fs.readFileSync('dist/algolia.json', 'utf-8');
24-
const indexed = JSON.parse(raw);
21+
const maxChunkSizeInBytes = 10000000; // 10,000,000
2522

26-
// Revision will look like "YYYYMMDDHHMM".
27-
const revision = new Date().toISOString().substring(0, 16).replace(/\D/g, '');
28-
const primaryIndexName = 'webdev';
29-
const deployIndexName = `webdev_deploy_${revision}`;
23+
/**
24+
* Chunks array of AlgoliaCollectionItem into array of array of AlgoliaCollectionItem smaller than 10 MB.
25+
*
26+
* @param {AlgoliaCollectionItem[]} arr
27+
* @return {AlgoliaCollectionItem[][]}
28+
*/
29+
const chunkAlgolia = (arr) => {
30+
const chunked = [];
31+
let tempSizeInBytes = 0;
32+
let temp = [];
33+
for (const current of arr) {
34+
const currentSizeInBytes = sizeof(current);
35+
if (tempSizeInBytes + currentSizeInBytes < maxChunkSizeInBytes) {
36+
temp.push(current);
37+
tempSizeInBytes += currentSizeInBytes;
38+
} else {
39+
chunked.push(temp);
40+
temp = [current];
41+
tempSizeInBytes = currentSizeInBytes;
42+
}
43+
}
44+
chunked.push(temp);
45+
return chunked;
46+
};
3047

3148
async function index() {
32-
if (!process.env.ALGOLIA_APP || !process.env.ALGOLIA_KEY) {
49+
const indexedOn = new Date();
50+
51+
if (!process.env.ALGOLIA_APP_ID || !process.env.ALGOLIA_API_KEY) {
3352
console.warn('Missing Algolia environment variables, skipping indexing.');
3453
return;
3554
}
3655

56+
const raw = fs.readFileSync('dist/algolia.json', 'utf-8');
57+
/** @type {AlgoliaCollection} */
58+
const algoliaData = JSON.parse(raw);
59+
60+
// Set date of when object is being added to algolia
61+
algoliaData.map((e) => {
62+
e.indexedOn = indexedOn.getTime();
63+
return e;
64+
});
65+
66+
const chunkedAlgoliaData = chunkAlgolia(algoliaData);
67+
const postsCount = algoliaData.length;
68+
69+
// @ts-ignore
3770
const client = algoliasearch(
38-
process.env.ALGOLIA_APP,
39-
process.env.ALGOLIA_KEY,
71+
process.env.ALGOLIA_APP_ID,
72+
process.env.ALGOLIA_API_KEY,
4073
);
74+
const index = client.initIndex('prod_web_dev');
4175

42-
const primaryIndex = client.initIndex(primaryIndexName); // nb. not actually used, just forces init
43-
const deployIndex = client.initIndex(deployIndexName);
44-
45-
log(
46-
`Indexing ${indexed.length} articles with temporary index ${deployIndex.indexName}...`,
76+
console.log(
77+
`Indexing ${postsCount} articles amongst ${chunkedAlgoliaData.length} chunk(s).`,
4778
);
4879

49-
// TODO(samthor): This is from https://www.algolia.com/doc/api-reference/api-methods/replace-all-objects/#examples,
50-
// are there more things that should be copied?
51-
const scope = ['settings', 'synonyms', 'rules'];
52-
await client.copyIndex(primaryIndex.indexName, deployIndex.indexName, {
53-
scope,
80+
// When indexing data we mark these two fields as fields that can be filtered by.
81+
await index.setSettings({
82+
attributesForFaceting: ['locales', 'tags'],
5483
});
5584

56-
// TODO(samthor): Batch uploads so that we don't send more than 10mb.
57-
// As of September 2019, the JSON itself is only ~70k. \shrug/
58-
await deployIndex.saveObjects(indexed);
59-
log(`Indexed, replacing existing index ${primaryIndex.indexName}`);
85+
// Update algolia index with new data
86+
for (let i = 0; i < chunkedAlgoliaData.length; i++) {
87+
await index.saveObjects(chunkedAlgoliaData[i], {
88+
autoGenerateObjectIDIfNotExist: true,
89+
});
90+
}
91+
92+
console.log('Updated algolia data.');
93+
94+
console.log('Deleting old data no longer in algolia.json.');
95+
await index.deleteBy({
96+
filters: `indexedOn < ${indexedOn.getTime()}`,
97+
});
98+
console.log('Deleted old data.');
6099

61-
// Move our temporary deploy index on-top of the primary index, atomically.
62-
await client.moveIndex(deployIndex.indexName, primaryIndex.indexName);
63-
log('Done!');
100+
console.log('Done!');
64101
}
65102

66103
index().catch((err) => {
67-
log.error(err);
104+
console.error(err);
68105
throw err;
69106
});

‎gulpfile.js

+1
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ const compressImagesTransform = (pngQuality, jpegQuality) => {
3838
return through2.obj();
3939
}
4040
return imagemin([
41+
// @ts-ignore
4142
pngquant({quality: [pngQuality, pngQuality]}),
4243
mozjpeg({quality: jpegQuality * 100}),
4344
]);

0 commit comments

Comments
 (0)