- Updated /search endpoint to only escape searches on lower bounds le…

…ngth limit for "asyoutype" searches. This fixes issues with running non-AsYouType searches for single or two letter searches such as "w2".

- Added check for indexes already existing before we create them so that we don't create duplicate indexes and incur the overhead of maintaining all the duplicates on the database.
- Added @types/node to the dev packages.
- Updated README adding terminology section to help with developers getting on track with troubleshooting the service.
- Upated qual.sh build script to better handle versioning updates overlooked by developers.

README.md

@@ -8,6 +8,16 @@ In addition featured-search-results provides directory listings associated with
 maintaining, updating, and searching directory information, including active `department` listings, is provided
 with this code and the API endpoints they implement.
 
+## terminology
+
+* __`Result`__ - A result is URL we associate with different types of search _`Matchings`_ that can be found in searches submitted to this service. A part of each result's definition is a _`Display Title`_, the _`Target URL`_, a list of _`Matchings`_, and a percentile _`Weight`_ for use in prioritizing which matchings take precedence when there's multiple Matchings with hits (possibly across multiple Results) - higher weight takes precedence.
+  * _`Matchings`_ - These associate individual or sets of words to be matched with the search by a _`Matching Type`_ which determines the matching logic to use with the words defined for the Matching.
+  * _`Matching Types`_ - All comparisons are done case insensitive.
+    * _`Exact`_ - The words in the matching's definition must exactly match the search submitted.
+    * _`Phrase`_ - The words in the matching's definition must be present anywhere in the search submitted but in the order found in the matching's definition.
+    * _`Keyword`_ - The words in the matching's definition must be present in the search submitted but the order in which they're found doesn't matter.
+* __`Query` or `Visitor Searches`__ - The search requests submitted to the service. These are logged along with the timestamp of submission and any matching `Results` that the system responded with. An interval task runs roughly every 27 minutes to expire logged Queries that are over 6 months old and recalculate associated hit counts.
+
 ## api endpoints
 
 ### public endpoints

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "featured-search-sveltekit",
-	"version": "0.0.1",
+	"version": "2.0.36",
 	"private": true,
 	"scripts": {
 		"dev": "vite dev --host 0.0.0.0 --port 80",
@@ -35,6 +35,7 @@
 		"@txstate-mws/svelte-components": "^1.5.12",
 		"@txstate-mws/svelte-forms": "^1.3.17",
 		"@types/luxon": "^3.4.2",
+		"@types/node": "^22.13.0",
 		"eslint-config-standard-with-typescript": "^40.0.0",
 		"eslint-plugin-svelte": "^2.35.1",
 		"sass": "^1.70.0",

qual.sh

@@ -1,14 +1,37 @@
-#!/bin/sh
+#!/bin/bash
 # Pass version as first argument - REQUIRED.
 #VER="${1:-`git symbolic-ref -q --short HEAD || git describe --tags --exact-match`}"
 VER=$1
-GITBRANCH=`git symbolic-ref -q --short HEAD`
-GITTAG=`git describe --tags --exact-match 2>/dev/null`
-COMMIT=$(git rev-parse HEAD)
+getNPMVer () {
+  npm pkg get version | sed -e 's/^"//' -e 's/"$//'
+}
 integrityExit () {
-  echo $1
+  echo "$1"
   echo "Aborting build to preserve build tag integrity with commits." && exit 1
 }
+GITBRANCH=`git symbolic-ref -q --short HEAD`
+GITTAG=`git describe --tags --exact-match 2>/dev/null`
+COMMIT=$(git rev-parse HEAD)
+cd `git rev-parse --show-toplevel` # cd to repo root.
+IMAGENAME=`basename $(pwd)`2
+if [ -f ./package.json ]; then
+  NPMVER=`getNPMVer`
+fi
+echo "Build Tag: $VER"
+echo "NPM-Ver:   $NPMVER"
+echo "Branch:    $GITBRANCH"
+echo "Git Tag:   $GITTAG"
+echo Commit: $COMMIT
+echo Image Name: $IMAGENAME
+echo
+# Ensure they've already updated their npm version to match what they want to tag the build with.
+if [[ "$NPMVER" != "" && "$VER" != "$NPMVER" ]]; then
+  integrityExit "The version in your npm package.json file does not match the build tag passed.
+Please run the following command to update your package version and commit those changes before attempting to build:
+
+    npm version -no-git-tag-version $VER
+"
+fi
 # Ensure we're building from committed changes.
 if `git diff-index --quiet HEAD --`; then
   echo "Active branch $GITBRANCH is clean. Continuing with checks..."
@@ -19,33 +42,64 @@ fi
 if [ "$VER" = "$GITBRANCH" ]; then
    integrityExit "Not tagging commit $COMMIT on branch $GITBRANCH with tag $GITBRANCH. Please provide a versioning argument."
 fi
-# Ensure our version paramenter is not in conflict with an existing commit tag.
-if [[ "$GITTAG" != "" && "$GITTAG" != "$VER" ]]; then
-  integrityExit "Not overwriting existing tag $GITTAG on commit $COMMIT with new tag $VER."
-fi
+## Ensure our version paramenter is not in conflict with an existing commit tag.
+#if [[ "$GITTAG" != "" && "$GITTAG" != "v$VER" ]]; then
+#  integrityExit "Not overwriting existing tag $GITTAG on commit $COMMIT with new tag $VER."
+#fi
 # Ensure we're able to tag the commit with the version.
-if [ "$VER" = "$GITTAG" ]; then
+if [[ "$VER" = "$GITTAG" || "v$VER" = "$GITTAG" ]]; then
   echo "Commit $COMMIT is already tagged as $GITTAG."
 else
-  if `git tag $VER $COMMIT`; then
-    echo "Tagged commit $COMMIT on branch $GITBRANCH with tag $VER."
-  else
-    integrityExit "Failed to tag commit $COMMIT on branch $GITBRANCH with tag $VER."
-  fi
-fi
-# Ensure our package version is in sync with the tags.
-if `npm version $VER`; then
-  echo "Updated package.json version to $VER."
-else
-  integrityExit "Failed to update package.json version to $VER."
+  integrityExit "Please tag current commit"
+#  if `git tag $VER $COMMIT`; then
+#    echo "Tagged commit $COMMIT on branch $GITBRANCH with tag $VER."
+#  else
+#    integrityExit "Failed to tag commit $COMMIT on branch $GITBRANCH with tag $VER."
+#  fi
 fi
 echo "Proceeding with versioned builds..."
-docker build -t registry.its.txstate.edu/search-featured-results .
-docker tag registry.its.txstate.edu/search-featured-results registry.its.txstate.edu/search-featured-results:$VER
-docker push registry.its.txstate.edu/search-featured-results
-docker push registry.its.txstate.edu/search-featured-results:$VER
-docker build -t registry.its.txstate.edu/search-featured-results-cron -f Dockerfile.cron .
-docker tag registry.its.txstate.edu/search-featured-results-cron registry.its.txstate.edu/search-featured-results-cron:$VER
-docker push registry.its.txstate.edu/search-featured-results-cron
-docker push registry.its.txstate.edu/search-featured-results-cron:$VER
+docker build -t registry.its.txstate.edu/$IMAGENAME .
+docker tag registry.its.txstate.edu/$IMAGENAME registry.its.txstate.edu/$IMAGENAME:$VER
+PUSH_STATUS=$(docker push registry.its.txstate.edu/$IMAGENAME:latest --quiet)
+if [[ "$PUSH_STATUS" = "registry.its.txstate.edu/$IMAGENAME:latest" ]]; then
+  echo "Pushed latest build to registry.its.txstate.edu/$IMAGENAME."
+else
+  integrityExit "Failed to push latest build to registry.its.txstate.edu/$IMAGENAME.
+Please resolve the issues preventing the push and try again with the following command:
+
+    docker push registry.its.txstate.edu/$IMAGENAME:latest
+"
+fi
+PUSH_STATUS=$(docker push registry.its.txstate.edu/$IMAGENAME:$VER --quiet)
+if [[ "$PUSH_STATUS" = "registry.its.txstate.edu/$IMAGENAME:$VER" ]]; then
+  echo "Pushed tagged build to registry.its.txstate.edu/$IMAGENAME:$VER."
+else
+  integrityExit "Failed to push tagged build to registry.its.txstate.edu/$IMAGENAME:$VER.
+Please resolve the issues preventing the push and try again with the following command:
+
+    docker push registry.its.txstate.edu/$IMAGENAME:$VER
+"
+fi
+docker build -t registry.its.txstate.edu/$IMAGENAME-cron -f Dockerfile.cron .
+docker tag registry.its.txstate.edu/$IMAGENAME-cron registry.its.txstate.edu/$IMAGENAME-cron:$VER
+PUSH_STATUS=$(docker push registry.its.txstate.edu/$IMAGENAME-cron:latest --quiet)
+if [[ "$PUSH_STATUS" = "registry.its.txstate.edu/$IMAGENAME-cron:latest" ]]; then
+  echo "Pushed latest build to registry.its.txstate.edu/$IMAGENAME-cron."
+else
+  integrityExit "Failed to push latest build to registry.its.txstate.edu/$IMAGENAME-cron.
+Please resolve the issues preventing the push and try again with the following command:
+
+    docker push registry.its.txstate.edu/$IMAGENAME-cron:latest
+"
+fi
+PUSH_STATUS=$(docker push registry.its.txstate.edu/$IMAGENAME-cron:$VER --quiet)
+if [[ "$PUSH_STATUS" = "registry.its.txstate.edu/$IMAGENAME-cron:$VER" ]]; then
+  echo "Pushed tagged build to registry.its.txstate.edu/$IMAGENAME-cron:$VER."
+else
+  integrityExit "Failed to push tagged build to registry.its.txstate.edu/$IMAGENAME-cron:$VER.
+Please resolve the issues preventing the push and try again with the following command:
+
+    docker push registry.its.txstate.edu/$IMAGENAME-cron:$VER
+"
+fi
 git push origin --tags

src/lib/models/query.ts

@@ -167,10 +167,11 @@ const QuerySchema = new Schema<IQuery, QueryModel, IQueryMethods>({
   lasthit: { type: Date },
   results: [{ type: Schema.Types.ObjectId, ref: 'Result' }]
 })
-QuerySchema.index({ query: 1 })
+const existingIndexes = QuerySchema.indexes()
+existingIndexes.includes([{ query: 1 }, {}]) ? console.log('QuerySchema index { query: 1 } already exists.') : QuerySchema.index({ query: 1 })
 // we always push later dates on the end of hits, so hits[0] is the minimum and the
 // only index we need - luckily mongo supports this with dot notation
-QuerySchema.index({ 'hits.0': 1 })
+existingIndexes.includes([{ 'hits.0': 1 }, {}]) ? console.log('QuerySchema index { hits.0: 1 } already exists.') : QuerySchema.index({ 'hits.0': 1 })
 
 QuerySchema.methods.basic = function () {
   return {

src/lib/models/result.ts

@@ -23,7 +23,7 @@ import axios from 'axios'
 import { DateTime } from 'luxon'
 import mongoose from 'mongoose'
 const { Schema, models, model, Error, deleteModel } = mongoose
-import type { Model, Document, ObjectId } from 'mongoose'
+import type { Model, Document, ObjectId, IndexDefinition } from 'mongoose'
 import { isBlank, isNotNull, sortby, eachConcurrent } from 'txstate-utils'
 import { getUrlEquivalencies, isValidHttpUrl, normalizeUrl, querysplit, getMongoStages, getFields } from '../util/helpers.js'
 import type { Paging, AdvancedSearchResult, AggregateResult, SearchMappings, MappingType, SortParam } from '../util/helpers.js'
@@ -284,9 +284,10 @@ const ResultSchema = new Schema<IResult, ResultModel, IResultMethods>({
 })
 // ResultSchema.plugin(paginate)
 
-ResultSchema.index({ 'entries.keywords': 1 })
-ResultSchema.index({ 'currency.tested': 1 })
-ResultSchema.index({ url: 1 })
+const existingIndexes = ResultSchema.indexes()
+existingIndexes.includes([{ 'entries.keywords': 1 }, {}]) ? console.log('ResultSchema index { \'entries.keywords\': 1 } already exists.') : ResultSchema.index({ 'entries.keywords': 1 })
+existingIndexes.includes([{ 'currency.tested': 1 }, {}]) ? console.log('ResultSchema index { \'currency.tested\': 1 } already exists.') : ResultSchema.index({ 'currency.tested': 1 })
+existingIndexes.includes([{ url: 1 }, {}]) ? console.log('ResultSchema index { url: 1 } already exists.') : ResultSchema.index({ url: 1 })
 
 ResultSchema.methods.basic = function () {
   return {

src/routes/(api)/search/+server.ts

@@ -4,13 +4,16 @@ import { Query } from '$lib/models/query.js'
 
 /** @type {import('./$types').RequestHandler} */
 export async function GET ({ url, setHeaders }) {
-  const query = url.searchParams.get('q') ?? undefined
-  if (!query?.length || query.length < 3 || query.length > 1024) return json([])
+  const query = (url.searchParams.get('q') as string) ?? undefined
+  if (!query?.length || query.length > 1024) return json([])
   const asyoutype = !!url.searchParams.get('asyoutype')
+  // We don't want to flood the api with `asyoutype` requests, so limit by lower length if such.
+  if (asyoutype && query.length < 3) return json([])
   /* I think we should make `asyoutype` a number parameter that indicates which word in query
      is the selection word in the input being typed to. We could then update `findByQuery`'s
      call stack to match with the corresponding word in query being the one that gets the `startsWith()`
-     evaluation on matching. */
+     evaluation on matching. We could default to current behavior if there are no words to correspond
+     with the number or a non-numeric value is passed. */
   const results = await Result.findByQuery(query)
   const ret = results.map(result => result.basic())
   if (!asyoutype) Query.record(query, results)