diff --git a/.code-samples.meilisearch.yaml b/.code-samples.meilisearch.yaml index 0714bb4b8..5b49a0fce 100644 --- a/.code-samples.meilisearch.yaml +++ b/.code-samples.meilisearch.yaml @@ -262,7 +262,17 @@ update_filterable_attributes_1: |- -H 'Content-Type: application/json' \ --data-binary '[ "genres", - "director" + "director", + { + "attributePatterns": ["*_ratings"], + "features": { + "facetSearch": false, + "filters": { + "equality": true, + "comparison": false + } + } + } ]' reset_filterable_attributes_1: |- curl \ diff --git a/assets/misc/meilisearch-collection-postman.json b/assets/misc/meilisearch-collection-postman.json index 5e0ca9ded..0773dccbc 100644 --- a/assets/misc/meilisearch-collection-postman.json +++ b/assets/misc/meilisearch-collection-postman.json @@ -1,7 +1,7 @@ { "info": { - "_postman_id": "11c3d322-31f0-4b54-b0c7-138b5efc7f6b", - "name": "Meilisearch v1.13", + "_postman_id": "b321d331-d809-4909-8e49-e29c8d749d32", + "name": "Meilisearch v1.14", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", "_exporter_id": "25294324" }, @@ -101,6 +101,11 @@ "key": "filter", "value": "genre=\"adventure\"", "disabled": true + }, + { + "key": "ids", + "value": "1,2", + "disabled": true } ] } diff --git a/guides/docker.mdx b/guides/docker.mdx index 728aba3bd..29ad5d5e0 100644 --- a/guides/docker.mdx +++ b/guides/docker.mdx @@ -14,7 +14,7 @@ Docker is a tool that bundles applications into containers. Docker containers en Docker containers are distributed in images. To use Meilisearch, use the `docker pull` command to download a Meilisearch image: ```sh -docker pull getmeili/meilisearch:v1.13 +docker pull getmeili/meilisearch:v1.14 ``` Meilisearch deploys a new Docker image with every release of the engine. Each image is tagged with the corresponding Meilisearch version, indicated in the above example by the text following the `:` symbol. You can see [the full list of available Meilisearch Docker images](https://hub.docker.com/r/getmeili/meilisearch/tags#!) on Docker Hub. @@ -31,7 +31,7 @@ After completing the previous step, use `docker run` to launch the Meilisearch i docker run -it --rm \ -p 7700:7700 \ -v $(pwd)/meili_data:/meili_data \ - getmeili/meilisearch:v1.13 + getmeili/meilisearch:v1.14 ``` ### Configure Meilisearch @@ -47,7 +47,7 @@ docker run -it --rm \ -p 7700:7700 \ -e MEILI_MASTER_KEY='MASTER_KEY'\ -v $(pwd)/meili_data:/meili_data \ - getmeili/meilisearch:v1.13 + getmeili/meilisearch:v1.14 ``` #### Passing instance options with CLI arguments @@ -58,7 +58,7 @@ If you want to pass command-line arguments to Meilisearch with Docker, you must docker run -it --rm \ -p 7700:7700 \ -v $(pwd)/meili_data:/meili_data \ - getmeili/meilisearch:v1.13 \ + getmeili/meilisearch:v1.14 \ meilisearch --master-key="MASTER_KEY" ``` @@ -76,7 +76,7 @@ To keep your data intact between reboots, specify a dedicated volume by running docker run -it --rm \ -p 7700:7700 \ -v $(pwd)/meili_data:/meili_data \ - getmeili/meilisearch:v1.13 + getmeili/meilisearch:v1.14 ``` The example above uses `$(pwd)/meili_data`, which is a directory in the host machine. Depending on your OS, mounting volumes from the host to the container might result in performance loss and is only recommended when developing your application. @@ -91,7 +91,7 @@ To import a dump, use Meilisearch's `--import-dump` command-line option and spec docker run -it --rm \ -p 7700:7700 \ -v $(pwd)/meili_data:/meili_data \ - getmeili/meilisearch:v1.13 \ + getmeili/meilisearch:v1.14 \ meilisearch --import-dump /meili_data/dumps/20200813-042312213.dump ``` @@ -111,7 +111,7 @@ To generate a Meilisearch snapshot with Docker, launch Meilisearch with `--sched docker run -it --rm \ -p 7700:7700 \ -v $(pwd)/meili_data:/meili_data \ - getmeili/meilisearch:v1.13 \ + getmeili/meilisearch:v1.14 \ meilisearch --schedule-snapshot --snapshot-dir /meili_data/snapshots ``` @@ -123,7 +123,7 @@ To import a snapshot, launch Meilisearch with the `--import-snapshot` option: docker run -it --rm \ -p 7700:7700 \ -v $(pwd)/meili_data:/meili_data \ - getmeili/meilisearch:v1.13 \ + getmeili/meilisearch:v1.14 \ meilisearch --import-snapshot /meili_data/snapshots/data.ms.snapshot ``` diff --git a/learn/resources/experimental_features_overview.mdx b/learn/resources/experimental_features_overview.mdx index 74dc280ef..9c0a60514 100644 --- a/learn/resources/experimental_features_overview.mdx +++ b/learn/resources/experimental_features_overview.mdx @@ -54,3 +54,5 @@ Activating or deactivating experimental features this way does not require you t | [Edit documents with function](/reference/api/documents#update-documents-with-function) | Use a RHAI function to edit documents directly in the Meilisearch database | API route | | [`/network` route](/reference/api/network) | Enable `/network` route | API route | | [Dumpless upgrade](/learn/self_hosted/configure_meilisearch_at_launch#dumpless-upgrade) | Upgrade Meilisearch without generating a dump | API route | +| [Composite embedders](/reference/api/settings#composite-embedders) | Enable composite embedders | API route | +| [Search query embedding cache](/learn/self_hosted/configure_meilisearch_at_launch#search-query-embedding-cache) | Enable a cache for search query embeddings | CLI flag or environment variable | diff --git a/learn/resources/telemetry.mdx b/learn/resources/telemetry.mdx index bf77cca5e..ddc076409 100644 --- a/learn/resources/telemetry.mdx +++ b/learn/resources/telemetry.mdx @@ -176,6 +176,7 @@ This list is liable to change with every new version of Meilisearch. It's not be | `sortable_attributes.has_geo` | `true` if `_geo` is set as a sortable attribute | true | `filterable_attributes.total` | Number of filterable attributes | 3 | `filterable_attributes.has_geo` | `true` if `_geo` is set as a filterable attribute | false +| `filterable_attributes.has_patterns` | `true` if `filterableAttributes` uses `attributePatterns` | true | `searchable_attributes.total` | Number of searchable attributes | 4 | `searchable_attributes.with_wildcard` | `true` if `*` is specified as a searchable attribute | false | `per_task_uid` | `true` if a `uids` is used to fetch a particular task resource | true @@ -220,9 +221,11 @@ This list is liable to change with every new version of Meilisearch. It's not be | `indexes.total_single_index` | The total number of calls when only one index is queried | 2007 | `matching_strategy.most_used_strategy` | Most used word matching strategy | last | `infos.with_configuration_file` | `true` if the instance is launched with a configuration file | false +| `infos.experimental_composite_embedders` | `true` if the `compositeEmbedders` feature is set to `true` for this instance | false | `infos.experimental_contains_filter` | `true` if the `containsFilter` experimental feature is enabled | false | `infos.experimental_edit_documents_by_function` | `true` if the `editDocumentsByFunction` experimental feature is enabled | false | `infos.experimental_enable_metrics` | `true` if `--experimental-enable-metrics` is specified at launch | false +| `infos.experimental_embedding_cache_entries` | Size of configured embedding cache | 100 | `infos.experimental_replication_parameters` | `true` if `--experimental-replication-parameters` is specified at launch | false | `infos.experimental_reduce_indexing_memory_usage` | `true` if `--experimental-reduce-indexing-memory-usage` is specified at launch | false | `infos.experimental_logs_mode` | `human` or `json` depending on the value specified | human @@ -243,7 +246,7 @@ This list is liable to change with every new version of Meilisearch. It's not be | `scoring.ranking_score_threshold` | `true` if rankingScoreThreshold was specified in an aggregated list of requests | true | `attributes_to_search_on.total_number_of_uses` | Total number of queries where `attributesToSearchOn` is set | 5 | `vector.max_vector_size` | Highest number of dimensions given for the `vector` parameter in this batch | 1536 -| `vector.retrieve_vectors` | `true` if the retrieve_vectors parameter has been used in this batch. | false +| `vector.retrieve_vectors` | `true` if the retrieve_vectors parameter has been used in this batch. | false | `hybrid.enabled` | `true` if hybrid search been used in the aggregated event | true | `hybrid.semantic_ratio` | `true` if semanticRatio was used in this batch, otherwise false | false | `embedders.total` | Numbers of defined embedders | 2 diff --git a/learn/self_hosted/configure_meilisearch_at_launch.mdx b/learn/self_hosted/configure_meilisearch_at_launch.mdx index d15eacdd5..52310c14e 100644 --- a/learn/self_hosted/configure_meilisearch_at_launch.mdx +++ b/learn/self_hosted/configure_meilisearch_at_launch.mdx @@ -393,6 +393,17 @@ Sets the maximum size of [accepted payloads](/learn/getting_started/documents#da Configure the maximum amount of simultaneous search requests. By default, Meilisearch queues up to 1000 search requests at any given moment. This limit exists to prevent Meilisearch from consuming an unbounded amount of RAM. +### Search query embedding cache + +**Environment variable**: `MEILI_EXPERIMENTAL_EMBEDDING_CACHE_ENTRIES`
+**CLI option**: `--experimental-embedding-cache-entries`
+**Default value**: `0`
+**Expected value**: an integer + +Sets the size of the search query embedding cache. By default, Meilisearch generates an embedding for every new search query. When this option is set to an integer bigger than 0, Meilisearch returns a previously generated embedding if it recently performed the same query. + +The least recently used entries are evicted first. Embedders with the same configuration share the same cache, even if they were declared in distinct indexes. + ### Schedule snapshot creation **Environment variable**: `MEILI_SCHEDULE_SNAPSHOT`
diff --git a/learn/self_hosted/install_meilisearch_locally.mdx b/learn/self_hosted/install_meilisearch_locally.mdx index 4ec7e6aa6..1c85a8c17 100644 --- a/learn/self_hosted/install_meilisearch_locally.mdx +++ b/learn/self_hosted/install_meilisearch_locally.mdx @@ -54,14 +54,14 @@ These commands launch the **latest stable release** of Meilisearch. ```bash # Fetch the latest version of Meilisearch image from DockerHub -docker pull getmeili/meilisearch:v1.13 +docker pull getmeili/meilisearch:v1.14 # Launch Meilisearch in development mode with a master key docker run -it --rm \ -p 7700:7700 \ -e MEILI_ENV='development' \ -v $(pwd)/meili_data:/meili_data \ - getmeili/meilisearch:v1.13 + getmeili/meilisearch:v1.14 # Use ${pwd} instead of $(pwd) in PowerShell ``` diff --git a/learn/self_hosted/supported_os.mdx b/learn/self_hosted/supported_os.mdx index 6d63a578f..844b5d3cf 100644 --- a/learn/self_hosted/supported_os.mdx +++ b/learn/self_hosted/supported_os.mdx @@ -15,7 +15,7 @@ Use [Meilisearch Cloud](https://www.meilisearch.com/cloud?utm_campaign=oss&utm_s ## Linux -The Meilisearch binary works on all Linux distributions with `amd64/x86_64` or `aarch64/arm64` architecture using glibc 2.31 and later. You can check your glibc version using: +The Meilisearch binary works on all Linux distributions with `amd64/x86_64` or `aarch64/arm64` architecture using glibc 2.35 and later. You can check your glibc version using: ``` ldd --version diff --git a/package.json b/package.json index 93362c54f..10312bd06 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "documentation", - "version": "1.13.0", + "version": "1.14.0", "description": "", "main": "index.js", "dependencies": {}, diff --git a/reference/api/batches.mdx b/reference/api/batches.mdx index fbdb56582..22eb254a1 100644 --- a/reference/api/batches.mdx +++ b/reference/api/batches.mdx @@ -26,7 +26,10 @@ The `/batches` route gives information about the progress of batches of [asynchr }, "indexUids": { "INDEX_NAME": 1 - } + }, + "progressTrace": { … }, + "writeChannelCongestion": { … }, + "internalDatabaseSizes": { … } }, "duration": "PT0.250518S", "startedAt": "2024-12-10T15:20:30.18182Z", @@ -103,6 +106,18 @@ List with the `types` of tasks contained in the batch. List of the number of tasks in the batch separated by the indexes they affect. +#### `progressTrace` + +List with full paths for each operation performed in the batch, together with the processing time in human-readable format. + +#### `writeChannelCongestion` + +Object containing information on write operations computed during indexing. Can be useful when diagnosing performance issues associated with write speeds. + +#### `internalDatabaseSizes` + +Size of each internal database, including by how much it changed after a batch was processed. + ### `duration` **Type**: String
@@ -191,7 +206,10 @@ For example, `?uids=0` returns a batch containing the task with a `taskUid` equa }, "indexUids": { "INDEX_NAME": 1 - } + }, + "progressTrace": { … }, + "writeChannelCongestion": { … }, + "internalDatabaseSizes": { … } }, "duration": "PT0.110083S", "startedAt": "2024-12-10T15:49:04.995321Z", diff --git a/reference/api/documents.mdx b/reference/api/documents.mdx index ee796856d..e65bb5839 100644 --- a/reference/api/documents.mdx +++ b/reference/api/documents.mdx @@ -36,6 +36,7 @@ Use `offset` and `limit` to browse through documents. | **`fields`** | Array of strings/`null` | `*` | Document attributes to show (case-sensitive, comma-separated) | | **`filter`** | String/Array of array of strings/`null` | N/A | Refine results based on attributes in the `filterableAttributes` list | | **`retrieveVectors`** | Boolean | `false` | Return document vector data with search result | +| **`ids`** | Array of primary keys | `null` | Return documents based on their primary keys | Sending an empty payload (`--data-binary '{}'`) will return all documents in the index. @@ -50,6 +51,10 @@ Sending an empty payload (`--data-binary '{}'`) will return all documents in the | **`limit`** | Integer | Number of documents returned | | **`total`** | Integer | Total number of documents in the index | + +`/indexes/{index_uid}/documents/fetch` and `/indexes/{index_uid}/documents` responses do not return documents following the order of their primary keys. + + ### Example @@ -118,6 +123,7 @@ Using the query parameters `offset` and `limit`, you can browse through all your | **`fields`** | `*` | Document attributes to show (case-sensitive, comma-separated) | | **`filter`** | N/A | Refine results based on attributes in the `filterableAttributes` list | | **`retrieveVectors`** | `false` | Return document vector data with search result | +| **`ids`** | `null` | Return documents based on their primary keys | ### Response @@ -128,6 +134,10 @@ Using the query parameters `offset` and `limit`, you can browse through all your | **`limit`** | Integer | Number of documents returned | | **`total`** | Integer | Total number of documents in the index | + +`/indexes/{index_uid}/documents/fetch` and `/indexes/{index_uid}/documents` responses do not return documents following the order of their primary keys. + + ### Example diff --git a/reference/api/facet_search.mdx b/reference/api/facet_search.mdx index 09c5975f0..f99099ddc 100644 --- a/reference/api/facet_search.mdx +++ b/reference/api/facet_search.mdx @@ -31,14 +31,15 @@ For example, searching for `Jane` will return `Jane Austen`, but searching for ` ### Body -| Name | Type | Default value | Description | -| :---------------------------------------------------------------------------------------------------- | :----------------------------------------------------------------- | :------------ | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **`facetName`** * | String | `null` | Facet name to search values on | -| **`facetQuery`** | String | `null` | Search query for a given facet value. If `facetQuery` isn't specified, Meilisearch performs a [placeholder search](/reference/api/search#placeholder-search) which returns all facet values for the searched facet, limited to 100 | -| [**`q`**](/reference/api/search#query-q) | String | `""` | Query string | -| **[`filter`](/reference/api/search#filter)** | [String*](/learn/filtering_and_sorting/filter_expression_reference) | `null` | Filter queries by an attribute's value | -| **[`matchingStrategy`](/reference/api/search#matching-strategy)** | String | `last` | Strategy used to match query terms within documents | -| **[`attributesToSearchOn`](/reference/api/search##customize-attributes-to-search-on-at-search-time)** | Array of strings | `null` | Restrict search to the specified attributes | +| Name | Type | Default value | Description | +| ----------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------- | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | +| **`facetName`** * | String | `null` | Facet name to search values on | +| **`facetQuery`** | String | `null` | Search query for a given facet value. If `facetQuery` isn't specified, Meilisearch returns all facet values for the searched facet, limited to 100 | +| **[`q`](/reference/api/search#query-q)** | String | `""` | Query string | +| **[`filter`](/reference/api/search#filter)** | [String*](/learn/filtering_and_sorting/filter_expression_reference) | `null` | Filter queries by an attribute's value | +| **[`matchingStrategy`](/reference/api/search#matching-strategy)** | String | `"last"` | Strategy used to match query terms within documents | +| **[`attributesToSearchOn`](/reference/api/search##customize-attributes-to-search-on-at-search-time)** | Array of strings | `null` | Restrict search to the specified attributes | +| **`exhaustiveFacetCount`** | Boolean | `false` | Return an exhaustive count of facets, up to the limit defined by [`maxTotalHits`](/reference/api/settings#pagination) | ### Response diff --git a/reference/api/settings.mdx b/reference/api/settings.mdx index ba54f2dc7..a3c3c65f0 100644 --- a/reference/api/settings.mdx +++ b/reference/api/settings.mdx @@ -162,7 +162,7 @@ If the provided index does not exist, it will be created. | **[`displayedAttributes`](#displayed-attributes)** | Array of strings | All attributes: `["*"]` | Fields displayed in the returned documents | | **[`distinctAttribute`](#distinct-attribute)** | String | `null` | Search returns documents with distinct (different) values of the given field | | **[`faceting`](#faceting)** | Object | [Default object](#faceting-object) | Faceting settings | -| **[`filterableAttributes`](#filterable-attributes)** | Array of strings | Empty | Attributes to use as filters and facets | +| **[`filterableAttributes`](#filterable-attributes)** | Array of strings or objects | Empty | Attributes to use as filters and facets | | **[`pagination`](#pagination)** | Object | [Default object](#pagination-object) | Pagination settings | | **[`proximityPrecision`](#proximity-precision)** | String | `"byWord"` | Precision level when calculating the proximity ranking rule | | **[`facetSearch`](#facet-search)** | Boolean | `true` | Enable or disable [facet search](/reference/api/facet_search) functionality | @@ -684,13 +684,43 @@ You can use the returned `taskUid` to get more details on [the status of the tas ## Filterable attributes -Attributes in the `filterableAttributes` list can be used as filters or facets. +Attributes in the `filterableAttributes` list can be used as [filters](/learn/filtering_and_sorting/filter_search_results) or [facets](/learn/filtering_and_sorting/search_with_facet_filters). -Updating filterable attributes will re-index all documents in the index, which can take some time. We recommend updating your index settings first and then adding documents as this reduces RAM consumption. +Updating filterable attributes will re-index all documents in the index, which can take some time. To reduce RAM consumption, first update your index settings and then add documents. -[To learn more about filterable attributes, refer to our dedicated guide.](/learn/filtering_and_sorting/filter_search_results) +### Filterable attribute object + +`filterableAttributes` may be an array of either strings filterable attribute objects. + +Filterable attribute objects must contain the following fields: + +| Name | Type | Default value | Description | +| ----------------------- | ---------------- | ------------- | -------------------------------------------------------- | +| **`attributePatterns`** | Array of strings | `[]` | A list of strings indicating filterable fields | +| **`features`** | Object | `{"facetSearch": false, "filters": {"equality": true, "comparison": false}` | A list outlining filter types enabled for the specified attributes | + +#### `attributePatterns` + +Attribute patterns may begin or end with a * wildcard to match multiple fields: `customer_*`, `attribute*`. + +#### `features` + +`features` allows you to decide which filter features are allowed for the specified attributes. It accepts the following fields: + +- `facetSearch`: Whether facet search should be enabled for the specified attributes. Boolean, defaults to `false` +- `filter`: A list outlining the filter types for the specified attributes. Must be an object and accepts the following fields: + - `equality`: Enables `=`, `!=`, `IN`, `EXISTS`, `IS NULL`, `IS EMPTY`, `NOT`, `AND`, and `OR`. Boolean, defaults to `true` + - `comparison`: Enables `>`, `>=`, `<`, `<=`, `TO`, `EXISTS`, `IS NULL`, `IS EMPTY`, `NOT`, `AND`, and `OR`. Boolean, defaults to `false` + +Calculating `comparison` filters is a resource-intensive operation. Disabling them may lead to better search and indexing performance. `equality` filters use fewer resources and have limited impact on performance. + + +Use the simple string syntax to match reserved attributes. Reserved Meilisearch fields are always prefixed with an underscore (`_`), such as `_geo` and `_vector`. + +If set as a filterable attribute, reserved attributes ignore the `features` field and automatically activate all search features. Reserved fields will not be matched by wildcard `attributePatterns` such as `_*`. + ### Get filterable attributes @@ -736,15 +766,28 @@ Update an index's filterable attributes list. [, , …] ``` -An array of strings containing the attributes that can be used as filters at query time. +An array of strings containing the attributes that can be used as filters at query time. All filter types are enabled for the specified attributes when using the array of strings format. -If an attribute contains an object, you can use dot notation to set one or more of its keys as a value for this setting: `"filterableAttributes": ["release_date.year"]`. +You may also use an array of objects: - -If the field does not exist, no error will be thrown. - +``` +[ + { + "attributePatterns": [, , …], + "features": { + "facetSearch": , + "filter": { + "equality": , + "comparison": + } + } + } +] +``` -[To learn more about filterable attributes, refer to our dedicated guide.](/learn/filtering_and_sorting/filter_search_results) +If the specified field does not exist, Meilisearch will silently ignore it. + +If an attribute contains an object, you can use dot notation to set one or more of its keys as a value for this setting: `"filterableAttributes": ["release_date.year"]` or `"attributePatterns": ["release_date.year"]`. #### Example @@ -821,7 +864,7 @@ Locale objects must have the following fields: | Name | Type | Default value | Description | | :----------------- | :------ | :------------ | :---------------------------------------------------------- | | **`locales`** | Array of strings | `[]` | A list of strings indicating one or more ISO-639 locales | -| **`attribute_patterns`** | Array of strings | `[]` | A list of strings indicating which fields correspond to the specified locales | +| **`attributePatterns`** | Array of strings | `[]` | A list of strings indicating which fields correspond to the specified locales | #### `locales` @@ -2380,20 +2423,23 @@ The embedders object may contain up to 256 embedder objects. Each embedder objec These embedder objects may contain the following fields: -| Name | Type | Default Value | Description | -| :---------------------| :---------------| :-----------------------------------------------------------------------| :-------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **`source`** | String | Empty | The third-party tool that will generate embeddings from documents. Must be `openAi`, `huggingFace`, `ollama`, `rest`, or `userProvided` | -| **`url`** | String | `http://localhost:11434/api/embeddings` | The URL Meilisearch contacts when querying the embedder | -| **`apiKey`** | String | Empty | Authentication token Meilisearch should send with each request to the embedder. If not present, Meilisearch will attempt to read it from environment variables | -| **`model`** | String | Empty | The model your embedder uses when generating vectors | -| **`documentTemplate`** | String | `{% for field in fields %} {% if field.is_searchable and not field.value == nil %}{{ field.name }}: {{ field.value }} {% endif %} {% endfor %}` | Template defining the data Meilisearch sends to the embedder | -| **`documentTemplateMaxBytes`** | Integer | `400` | Maximum allowed size of rendered document template | -| **`dimensions`** | Integer | Empty | Number of dimensions in the chosen model. If not supplied, Meilisearch tries to infer this value | -| **`revision`** | String | Empty | Model revision hash | -| **`distribution`** | Object | Empty | Describes the natural distribution of search results. Must contain two fields, `mean` and `sigma`, each containing a numeric value between `0` and `1` | -| **`request`** | Object | Empty | A JSON value representing the request Meilisearch makes to the remote embedder | -| **`response`** | Object | Empty | A JSON value representing the request Meilisearch expects from the remote embedder | -| **`binaryQuantized`** | Boolean | Empty | Once set to `true`, irreversibly converts all vector dimensions to 1-bit values | +| Name | Type | Default Value | Description | +| ------------------------------ | ------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **`source`** | String | Empty | The third-party tool that will generate embeddings from documents. Must be `openAi`, `huggingFace`, `ollama`, `rest`, or `userProvided` | +| **`url`** | String | `http://localhost:11434/api/embeddings` | The URL Meilisearch contacts when querying the embedder | +| **`apiKey`** | String | Empty | Authentication token Meilisearch should send with each request to the embedder. If not present, Meilisearch will attempt to read it from environment variables | +| **`model`** | String | Empty | The model your embedder uses when generating vectors | +| **`documentTemplate`** | String | `{% for field in fields %} {% if field.is_searchable and not field.value == nil %}{{ field.name }}: {{ field.value }} {% endif %} {% endfor %}` | Template defining the data Meilisearch sends to the embedder | +| **`documentTemplateMaxBytes`** | Integer | `400` | Maximum allowed size of rendered document template | +| **`dimensions`** | Integer | Empty | Number of dimensions in the chosen model. If not supplied, Meilisearch tries to infer this value | +| **`revision`** | String | Empty | Model revision hash | +| **`distribution`** | Object | Empty | Describes the natural distribution of search results. Must contain two fields, `mean` and `sigma`, each containing a numeric value between `0` and `1` | +| **`request`** | Object | Empty | A JSON value representing the request Meilisearch makes to the remote embedder | +| **`response`** | Object | Empty | A JSON value representing the response Meilisearch expects from the remote embedder | +| **`binaryQuantized`** | Boolean | Empty | Once set to `true`, irreversibly converts all vector dimensions to 1-bit values | +| **`indexingEmbedder`** | Object | Empty | Configures embedder to vectorize documents during indexing | +| **`searchEmbedder`** | Object | Empty | Configures embedder to vectorize search queries | +| **`pooling`** | String | `"useModel"` | Pooling method for Hugging Face embedders | ### Get embedder settings @@ -2405,7 +2451,7 @@ Get the embedders configured for an index. | Name | Type | Description | | :---------------- | :----- | :------------------------------------------------------------------------ | -| **`index_uid`** * | String | [`uid`](/learn/getting_started/indexes#index-uid) of the requested index | +| **`index_uid`** * | String | [`uid`](/learn/getting_started/indexes#index-uid) of the requested index | #### Example @@ -2457,7 +2503,10 @@ Partially update the embedder settings for an index. When this setting is update "request": { … }, "response": { … }, "headers": { … }, - "binaryQuantized": + "binaryQuantized": , + "pooling": , + "indexingEmbedder": { … }, + "searchEmbedder": { … } } } ``` @@ -2466,18 +2515,39 @@ Set an embedder to `null` to remove it from the embedders list. ##### `source` -Use `source` to configure an embedder's source. The following embedders can auto-generate vectors for documents and queries: +Use `source` to configure an embedder's source. The source corresponds to a service that generates embeddings from your documents. +Meilisearch supports the following sources: - `openAi` - `huggingFace` - `ollama` +- `rest` +- `userProvided` +- `composite` -Additionally, use `rest` to auto-generate embeddings with any embedder offering a REST API. +`rest` is a generic source compatible with any embeddings provider offering a REST API. -You may also configure a `userProvided` embedder. In this case, you must manually include vector data in your documents' `_vectors` field. You must also manually generate vectors for search queries. +Use `userProvided` when you want to generate embeddings manually. In this case, you must include vector data in your documents' `_vectors` field. You must also generate vectors for search queries. This field is mandatory. +###### Composite embedders + +Choose `composite` to use one embedder during indexing time, and another embedder at search time. Must be used together with [`indexingEmbedder` and `searchEmbedder`](#indexingembedder-and-searchembedder). + + +This is an experimental feature. Use the experimental features endpoint to activate it: + +```sh +curl \ + -X PATCH 'MEILISEARCH_URL/experimental-features/' \ + -H 'Content-Type: application/json' \ + --data-binary '{ + "compositeEmbedders": true + }' +``` + + ##### `url` Meilisearch queries `url` to generate vector embeddings for queries and documents. `url` must point to a REST-compatible embedder. You may also use `url` to work with proxies, such as when targeting `openAi` from behind a proxy. @@ -2543,7 +2613,6 @@ This field is incompatible with `userProvided` embedders. This field is optional for all other embedders. - ##### `dimensions` Number of dimensions in the chosen model. If not supplied, Meilisearch tries to infer this value. @@ -2695,6 +2764,42 @@ This option can be useful when working with large Meilisearch projects. Consider **Activating `binaryQuantized` is irreversible.** Once enabled, Meilisearch converts all vectors and discards all vector data that does fit within 1-bit. The only way to recover the vectors' original values is to re-vectorize the whole index in a new embedder. +##### `pooling` + +Configure how Meilisearch should merge individual tokens into a single embedding. + +`pooling` must be one of the following strings: + +- `"useModel"`: Meilisearch will fetch the pooling method from the model configuration. Default value for new embedders +- `"forceMean"`: always use mean pooling. Default value for embedders created in Meilisearch \<=v1.13 +- `"forceCls"`: always use CLS pooling + +If in doubt, use `"useModel"`. `"forceMean"` and `"forceCls"` are compatibility options that might be necessary for certain embedders and models. + +`pooling` is optional for embedders with the `huggingFace` source. + +`pooling` is invalid for all other embedder sources. + +##### `indexingEmbedder` and `searchEmbedder` + +When using a [composite embedder](#composite-embedders), configure separate embedders Meilisearch should use when vectorizing documents and search queries. + +`indexingEmbedder` often benefits from the higher bandwidth and speed of remote providers so it can vectorize large batches of documents quickly. `searchEmbedder` may often benefits from the lower latency of processing queries locally. + +Both fields must be an object and accept the same fields as a regular embedder, with the following exceptions: + +- `indexingEmbedder` and `searchEmbedder` must use the same model for generating embeddings +- `indexingEmbedder` and `searchEmbedder` must have identical `dimension`s and `pooling` methods +- `source` is mandatory for both `indexingEmbedder` and `searchEmbedder` +- Neither sub-embedder can set `source` to `composite` or `userProvided` +- Neither `binaryQuantized` and `distribution` are valid sub-embedder fields and must always be declared in the main embedder +- `documentTemplate` and `documentTemplateMaxBytes` are invalid fields for `searchEmbedder` +- `documentTemplate` and `documentTemplateMaxBytes` are mandatory for `indexingEmbedder`, if applicable to its source + +`indexingEmbedder` and `searchEmbedder` are mandatory when using the `composite` source. + +`indexingEmbedder` and `searchEmbedder` are incompatible with all other embedder sources. + #### Example diff --git a/reference/errors/error_codes.mdx b/reference/errors/error_codes.mdx index e21a48b6e..0bf485c7d 100644 --- a/reference/errors/error_codes.mdx +++ b/reference/errors/error_codes.mdx @@ -340,6 +340,10 @@ The [`limit`](/reference/api/search#limit) parameter is invalid. It should be an The [`locales`](/reference/api/search#query-locales) parameter is invalid. +## `invalid_settings_embedder` + +The [`embedders`](/reference/api/settings#embedders) index setting value is invalid. + ## `invalid_settings_facet_search` The [`facetSearch`](/reference/api/settings#facet-search) index setting value is invalid.