@@ -204,44 +204,227 @@ export type Prepared<R = never> = {
204204
205205export type SearchEntry = {
206206 readonly description : ToolDescription
207- /** The path split into words, so `zones` matches `get_zones` as a word rather than as a substring of `timezones`. */
208- readonly pathWords : ReadonlyArray < string >
209- readonly searchText : string
207+ /** Position in the full search index, for scoring through the word index. */
208+ readonly position : number
209+ /** The tool's searchable text by field, read when the word index is built. */
210+ readonly text : ( ) => Readonly < Record < SearchField , ReadonlyArray < string > > >
211+ /** Words in the tool's own name (the last path segment). */
212+ readonly nameLength : number
210213}
211214
212- const tokenize = ( query : string ) : Array < string > =>
213- query
215+ type SearchField = "namespace" | "name" | "description" | "parameters" | "namespaceDescription"
216+
217+ // Standard field-weighted BM25 (k1 1.2, b 0.75) plus a preference for short names. Compared against other rankers on
218+ // model-written and real search queries over an ~11k-tool catalog (anomalyco/evals, dumber/evals/tool-search).
219+ const fieldWeights : Record < SearchField , number > = {
220+ name : 3 ,
221+ namespace : 2 ,
222+ description : 1 ,
223+ namespaceDescription : 0.4 ,
224+ parameters : 0.3 ,
225+ }
226+ const searchFields = Object . keys ( fieldWeights ) as ReadonlyArray < SearchField >
227+ const k1 = 1.2
228+ const lengthNormalization = 0.75
229+ /** score × (1 + brevityBoost / words in the tool name): "users" prefers `list_users` over `list_user_groups`. */
230+ const brevityBoost = 0.5
231+ /** A query word of 4+ letters also matches longer words it starts with, at this weight: `term` → `terminal`. */
232+ const prefixWeight = 0.3
233+
234+ /** Words split at case changes and non-alphanumerics: `listUserGroups` → list, user, groups. */
235+ const splitWords = ( text : string ) : Array < string > =>
236+ text
214237 . replace ( / ( [ a - z 0 - 9 ] ) ( [ A - Z ] ) / g, "$1 $2" )
215238 . toLowerCase ( )
216239 . split ( / [ ^ a - z 0 - 9 ] + / )
217- . filter ( ( term ) => term . length > 0 && term !== "*" )
240+ . filter ( ( word ) => word . length > 0 )
241+
242+ /** Plural endings only, applied the same everywhere: `settings` → `setting`, `statuses` → `status`, `vms` → `vm`. */
243+ const stem = ( word : string ) => {
244+ if ( word . length > 4 && word . endsWith ( "ies" ) ) return word . slice ( 0 , - 3 ) + "y"
245+ if ( word . length > 4 && / (?: s s | s h | c h | x | z | u s ) e s $ / . test ( word ) ) return word . slice ( 0 , - 2 )
246+ if ( word . length > 2 && word . endsWith ( "s" ) && ! / (?: s s | u s | i s ) $ / . test ( word ) ) return word . slice ( 0 , - 1 )
247+ return word
248+ }
218249
219- const termForms = ( term : string ) : Array < string > => {
220- const forms = [ term ]
221- if ( term . endsWith ( "es" ) && term . length > 3 ) forms . push ( term . slice ( 0 , - 2 ) )
222- if ( term . endsWith ( "s" ) && term . length > 2 ) forms . push ( term . slice ( 0 , - 1 ) )
223- return forms
250+ /** Searchable words of a text: its split words, plus each camelCase word whole (`CronJob` → cron, job, cronjob). */
251+ const searchWords = ( text : string ) =>
252+ splitWords ( text )
253+ . concat ( ( text . match ( / [ A - Z a - z 0 - 9 ] * [ a - z 0 - 9 ] [ A - Z ] [ A - Z a - z 0 - 9 ] * / g) ?? [ ] ) . map ( ( word ) => word . toLowerCase ( ) ) )
254+ . map ( stem )
255+
256+ const fieldWords = ( text : string , cache : Map < string , ReadonlyArray < string > > ) => {
257+ const cached = cache . get ( text )
258+ if ( cached ) return cached
259+ const words = searchWords ( text )
260+ cache . set ( text , words )
261+ return words
224262}
225263
226- const rank = ( entries : ReadonlyArray < SearchEntry > , query : string ) : Array < SearchEntry > => {
227- const terms = tokenize ( query ) . map ( termForms )
228- return entries
229- . map ( ( entry ) => {
230- const path = entry . description . path . toLowerCase ( )
231- const description = entry . description . description . toLowerCase ( )
232- const score = terms . reduce (
233- ( total , forms ) =>
234- total +
235- ( forms . some ( ( form ) => path === form || path . endsWith ( `.${ form } ` ) ) ? 20 : 0 ) +
236- ( forms . some ( ( form ) => entry . pathWords . includes ( form ) ) ? 12 : 0 ) +
237- ( forms . some ( ( form ) => path . includes ( form ) ) ? 8 : 0 ) +
238- ( forms . some ( ( form ) => description . includes ( form ) ) ? 4 : 0 ) +
239- ( forms . some ( ( form ) => entry . searchText . includes ( form ) ) ? 2 : 0 ) ,
240- 0 ,
241- )
242- return { entry, score }
243- } )
244- . filter ( ( { score } ) => terms . length === 0 || score > 0 )
264+ /** For each word, the tools containing it and their field-weighted, length-normalized term frequency. */
265+ type WordIndex = {
266+ readonly postings : ReadonlyMap < string , { readonly positions : Int32Array ; readonly weights : Float32Array } >
267+ readonly documents : number
268+ /** Sorted indexed words, for partial-word matching. */
269+ readonly words : ReadonlyArray < string >
270+ }
271+
272+ // Built once per search index on the first query.
273+ const wordIndexes = new WeakMap < ReadonlyArray < SearchEntry > , WordIndex > ( )
274+
275+ // Locations with the same tools (global MCP servers) share one word index, keyed by a hash of the indexed text. An
276+ // index lives while some search index uses it; once none does it can be collected and its key is removed. A hash
277+ // collision (about 1 in 37 trillion with 1,000 distinct catalogs alive) would only misrank results: they still come
278+ // from the caller's own tools, and the tool count in the key keeps positions in range.
279+ const shared =
280+ typeof WeakRef === "undefined" || typeof FinalizationRegistry === "undefined"
281+ ? undefined
282+ : {
283+ indexes : new Map < string , WeakRef < WordIndex > > ( ) ,
284+ cleanup : new FinalizationRegistry < { key : string ; ref : WeakRef < WordIndex > } > ( ( { key, ref } ) => {
285+ // A newer index may have been stored under the same key since this one was dropped.
286+ if ( shared ?. indexes . get ( key ) === ref ) shared . indexes . delete ( key )
287+ } ) ,
288+ }
289+
290+ const wordIndexFor = ( index : ReadonlyArray < SearchEntry > ) : WordIndex => {
291+ const cached = wordIndexes . get ( index )
292+ if ( cached ) return cached
293+ const texts = index . map ( ( entry ) => entry . text ( ) )
294+ const key = shared ? textKey ( texts ) : undefined
295+ const reused = key === undefined ? undefined : shared ?. indexes . get ( key ) ?. deref ( )
296+ if ( reused ) {
297+ wordIndexes . set ( index , reused )
298+ return reused
299+ }
300+ const built = buildWordIndex ( texts )
301+ wordIndexes . set ( index , built )
302+ if ( shared && key !== undefined ) {
303+ const ref = new WeakRef ( built )
304+ shared . indexes . set ( key , ref )
305+ shared . cleanup . register ( built , { key, ref } )
306+ }
307+ return built
308+ }
309+
310+ /** Tool count plus two independent 32-bit FNV-1a hashes of every indexed field, in index order. */
311+ const textKey = ( texts : ReadonlyArray < Readonly < Record < SearchField , ReadonlyArray < string > > > > ) => {
312+ let first = 0x811c9dc5
313+ let second = 0x9e3779b9
314+ const add = ( text : string ) => {
315+ for ( let i = 0 ; i < text . length ; i ++ ) {
316+ const code = text . charCodeAt ( i )
317+ first = Math . imul ( first ^ code , 0x01000193 )
318+ second = Math . imul ( second ^ code , 0x5bd1e995 )
319+ second ^= second >>> 15
320+ }
321+ // Separators keep field boundaries distinct: ["ab", "c"] and ["a", "bc"] hash differently.
322+ first = Math . imul ( first ^ 0x1f , 0x01000193 )
323+ second = Math . imul ( second ^ 0x1f , 0x5bd1e995 )
324+ }
325+ for ( const text of texts )
326+ for ( const field of searchFields ) {
327+ for ( const part of text [ field ] ) add ( part )
328+ add ( "\u0000" )
329+ }
330+ return `${ texts . length } :${ ( first >>> 0 ) . toString ( 36 ) } :${ ( second >>> 0 ) . toString ( 36 ) } `
331+ }
332+
333+ const buildWordIndex = ( texts : ReadonlyArray < Readonly < Record < SearchField , ReadonlyArray < string > > > > ) : WordIndex => {
334+ // Field texts are split once here and dropped afterwards; descriptions repeat across tools, so each text is split
335+ // once and shared while building.
336+ const cache = new Map < string , ReadonlyArray < string > > ( )
337+ const words = ( texts : ReadonlyArray < string > ) => texts . flatMap ( ( text ) => fieldWords ( text , cache ) )
338+ const fields = texts . map ( ( text ) => {
339+ return {
340+ namespace : words ( text . namespace ) ,
341+ name : words ( text . name ) ,
342+ description : words ( text . description ) ,
343+ parameters : words ( text . parameters ) ,
344+ namespaceDescription : words ( text . namespaceDescription ) ,
345+ }
346+ } )
347+ const averageLength = Object . fromEntries (
348+ searchFields . map ( ( field ) => [
349+ field ,
350+ fields . reduce ( ( sum , entry ) => sum + entry [ field ] . length , 0 ) / texts . length || 1 ,
351+ ] ) ,
352+ ) as Record < SearchField , number >
353+ const building = new Map < string , { positions : Array < number > ; weights : Array < number > } > ( )
354+ const weights = new Map < string , number > ( )
355+ fields . forEach ( ( entry , position ) => {
356+ weights . clear ( )
357+ for ( const field of searchFields ) {
358+ const weight =
359+ fieldWeights [ field ] /
360+ ( 1 - lengthNormalization + ( lengthNormalization * entry [ field ] . length ) / averageLength [ field ] )
361+ for ( const word of entry [ field ] ) weights . set ( word , ( weights . get ( word ) ?? 0 ) + weight )
362+ }
363+ for ( const [ word , weight ] of weights ) {
364+ let posting = building . get ( word )
365+ if ( ! posting ) building . set ( word , ( posting = { positions : [ ] , weights : [ ] } ) )
366+ posting . positions . push ( position )
367+ posting . weights . push ( weight )
368+ }
369+ } )
370+ const postings = new Map (
371+ [ ...building ] . map ( ( [ word , posting ] ) => [
372+ word ,
373+ { positions : Int32Array . from ( posting . positions ) , weights : Float32Array . from ( posting . weights ) } ,
374+ ] ) ,
375+ )
376+ return { postings, documents : texts . length , words : [ ...postings . keys ( ) ] . sort ( ) }
377+ }
378+
379+ /** Indexed words longer than a 4+ letter query word that start with it. */
380+ const prefixMatches = ( words : WordIndex , term : string ) : Array < readonly [ string , number ] > => {
381+ if ( term . length < 4 ) return [ ]
382+ let low = 0
383+ let high = words . words . length
384+ while ( low < high ) {
385+ const middle = ( low + high ) >> 1
386+ if ( words . words [ middle ] ! < term ) low = middle + 1
387+ else high = middle
388+ }
389+ const matches : Array < readonly [ string , number ] > = [ ]
390+ for ( let i = low ; i < words . words . length && words . words [ i ] ! . startsWith ( term ) ; i ++ )
391+ if ( words . words [ i ] !== term ) matches . push ( [ words . words [ i ] ! , prefixWeight ] )
392+ return matches
393+ }
394+
395+ const rank = (
396+ entries : ReadonlyArray < SearchEntry > ,
397+ query : string ,
398+ index : ReadonlyArray < SearchEntry > ,
399+ ) : Array < SearchEntry > => {
400+ const terms = searchWords ( query )
401+ if ( terms . length === 0 )
402+ return [ ...entries ] . sort ( ( left , right ) => compareText ( left . description . path , right . description . path ) )
403+ const words = wordIndexFor ( index )
404+ const allowed = entries === index ? undefined : new Set ( entries . map ( ( entry ) => entry . position ) )
405+ const scores = new Map < number , number > ( )
406+ for ( const term of terms ) {
407+ // Each tool counts a query word once, through its best-scoring match.
408+ const best = new Map < number , number > ( )
409+ for ( const [ word , factor ] of [ [ term , 1 ] as const , ...prefixMatches ( words , term ) ] ) {
410+ const posting = words . postings . get ( word )
411+ if ( ! posting ) continue
412+ const idf = Math . log ( 1 + ( words . documents - posting . positions . length + 0.5 ) / ( posting . positions . length + 0.5 ) )
413+ for ( let i = 0 ; i < posting . positions . length ; i ++ ) {
414+ const position = posting . positions [ i ] !
415+ if ( allowed && ! allowed . has ( position ) ) continue
416+ const weight = posting . weights [ i ] !
417+ const score = ( factor * idf * weight * ( k1 + 1 ) ) / ( weight + k1 )
418+ if ( score > ( best . get ( position ) ?? 0 ) ) best . set ( position , score )
419+ }
420+ }
421+ for ( const [ position , score ] of best ) scores . set ( position , ( scores . get ( position ) ?? 0 ) + score )
422+ }
423+ return [ ...scores ]
424+ . map ( ( [ position , score ] ) => ( {
425+ entry : index [ position ] ! ,
426+ score : score * ( 1 + brevityBoost / Math . max ( 1 , index [ position ] ! . nameLength ) ) ,
427+ } ) )
245428 . sort (
246429 ( left , right ) =>
247430 right . score - left . score || compareText ( left . entry . description . path , right . entry . description . path ) ,
@@ -276,7 +459,7 @@ const makeSearchTool = (searchIndex: ReadonlyArray<SearchEntry>): Tool => ({
276459 : scoped . find (
277460 ( entry ) => entry . description . path === pathQuery || toolExpression ( entry . description . path ) === trimmed ,
278461 )
279- const ranked = exact !== undefined ? [ exact ] : rank ( scoped , query )
462+ const ranked = exact !== undefined ? [ exact ] : rank ( scoped , pathQuery , searchIndex )
280463 const items = ranked . slice ( offset , offset + ( request . limit ?? defaultSearchLimit ) ) . map ( ( { description } ) => ( {
281464 ...description ,
282465 path : toolExpression ( description . path ) ,
@@ -296,20 +479,23 @@ export const searchSignature = (() => {
296479 return `search(${ inputTypeScript ( tool , true ) } ): ${ outputTypeScript ( tool , true ) } `
297480} ) ( )
298481
299- const toSearchEntry = < R > ( visible : VisibleTool < R > ) : SearchEntry => ( {
300- description : describeTool ( visible ) ,
301- pathWords : tokenize ( visible . path ) ,
302- searchText : [
303- visible . path ,
304- visible . tool . description ,
305- ...visible . namespaces . flatMap ( ( namespace ) => ( namespace . description === undefined ? [ ] : [ namespace . description ] ) ) ,
306- ...inputProperties ( visible . tool ) . flatMap ( ( { name, description : property } ) =>
307- property === undefined ? [ name ] : [ name , property ] ,
308- ) ,
309- ]
310- . join ( "\n" )
311- . toLowerCase ( ) ,
312- } )
482+ const toSearchEntry = < R > ( visible : VisibleTool < R > , position : number ) : SearchEntry => {
483+ const segments = visible . path . split ( "." )
484+ const name = segments . at ( - 1 ) ?? ""
485+ return {
486+ description : describeTool ( visible ) ,
487+ position,
488+ text : ( ) => ( {
489+ namespace : segments . slice ( 0 , - 1 ) ,
490+ name : [ name ] ,
491+ description : [ visible . tool . description ] ,
492+ // Parameter names only: their descriptions are half of all words and didn't improve ranking.
493+ parameters : inputProperties ( visible . tool ) . map ( ( property ) => property . name ) ,
494+ namespaceDescription : visible . namespaces . map ( ( namespace ) => namespace . description ?? "" ) ,
495+ } ) ,
496+ nameLength : splitWords ( name ) . length ,
497+ }
498+ }
313499
314500export const prepare = < R > ( tools : Tools < R > ) : Prepared < R > => {
315501 const root = toolTrie ( tools )
@@ -351,6 +537,7 @@ const resolve = <R>(root: ToolNode<R>, path: ReadonlyArray<string>, index: Reado
351537 const closest = rank (
352538 ns ? index . filter ( ( entry ) => entry . description . path . startsWith ( `${ ns } .` ) ) : index ,
353539 ns ? segments . slice ( 1 ) . join ( " " ) : name ,
540+ index ,
354541 ) [ 0 ]
355542 throw new ToolRuntimeError (
356543 "UnknownTool" ,
0 commit comments