From b18fe5ff5df01aa0f9fb74705cd66a454f35f7dd Mon Sep 17 00:00:00 2001 From: pal155 Date: Wed, 10 Jun 2020 11:48:34 +1000 Subject: [PATCH 01/15] Fix for #297 Include automatically added search filters (no identifiers or taxon variants) to the download query. Add some more information to allow non-taxon downloads to make sense. --- grails-app/conf/application.yml | 2 +- grails-app/services/au/org/ala/bie/DownloadService.groovy | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/grails-app/conf/application.yml b/grails-app/conf/application.yml index e72c56e6..350b3863 100644 --- a/grails-app/conf/application.yml +++ b/grails-app/conf/application.yml @@ -189,7 +189,7 @@ vernacularListsUrl: /default-vernacular-lists.json # Location of locality keywords (null for default) localityKeywordsUrl: /default-locality-keywords.json #nationalSpeciesDatasets: dr2699,dr2700,dr2702,dr2704,dr2703,dr3118 -defaultDownloadFields: guid,rank,scientificName,scientificNameAuthorship,taxonomicStatus,establishmentMeans,rk_genus,rk_family,rk_order,rk_class,rk_phylum,rk_kingdom,datasetName,parentGuid,acceptedConceptName,acceptedConceptID +defaultDownloadFields: guid,rank,scientificName,scientificNameAuthorship,taxonomicStatus,establishmentMeans,rk_genus,rk_family,rk_order,rk_class,rk_phylum,rk_kingdom,datasetName,parentGuid,acceptedConceptName,acceptedConceptID,idxtype,name additionalResultFields: "" #toggle for the population of occurrence counts diff --git a/grails-app/services/au/org/ala/bie/DownloadService.groovy b/grails-app/services/au/org/ala/bie/DownloadService.groovy index dac73d39..31f9df64 100644 --- a/grails-app/services/au/org/ala/bie/DownloadService.groovy +++ b/grails-app/services/au/org/ala/bie/DownloadService.groovy @@ -13,6 +13,7 @@ package au.org.ala.bie +import au.org.ala.bie.search.IndexDocType import au.org.ala.bie.util.Encoder class DownloadService { @@ -31,10 +32,11 @@ class DownloadService { */ def download(params, OutputStream outputStream, Locale locale){ def q = Encoder.escapeQuery(params.q ?: "*:*") - def fq = params.list('fq') - def fields = params.fields ?: grailsApplication.config.defaultDownloadFields ?: "guid,rank,scientificName,rk_genus,rk_family,rk_order,rk_class,rk_phylum,rk_kingdom,datasetName" + def fq = new ArrayList<>(params.list('fq')) // Make modifiable + def fields = params.fields ?: grailsApplication.config.defaultDownloadFields def fqs = '' + grailsApplication.config.solr.search.fq.each { fq << it } if (fq) { fqs = '&fq=' + fq.collect({ Encoder.escapeQuery(it) }).join("&fq=") } From 2fb6ce546015355a7c71458c666c40fc23910ab8 Mon Sep 17 00:00:00 2001 From: pal155 Date: Wed, 10 Jun 2020 12:03:30 +1000 Subject: [PATCH 02/15] Add titles and clean up left-overs --- grails-app/i18n/messages.properties | 2 ++ grails-app/services/au/org/ala/bie/DownloadService.groovy | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/grails-app/i18n/messages.properties b/grails-app/i18n/messages.properties index 0bcf1bf1..06804d34 100755 --- a/grails-app/i18n/messages.properties +++ b/grails-app/i18n/messages.properties @@ -82,6 +82,8 @@ download.rk_phylum=phylum download.rk_subkingdom=subkingdom download.rk_kingdom=kingdom download.datasetName=datasetName +download.idxtype=type +download.name=title admin.ala.label=ALA admin functions admin.ala.lead=view config, build info & set banner message diff --git a/grails-app/services/au/org/ala/bie/DownloadService.groovy b/grails-app/services/au/org/ala/bie/DownloadService.groovy index 31f9df64..7232414d 100644 --- a/grails-app/services/au/org/ala/bie/DownloadService.groovy +++ b/grails-app/services/au/org/ala/bie/DownloadService.groovy @@ -13,13 +13,11 @@ package au.org.ala.bie -import au.org.ala.bie.search.IndexDocType import au.org.ala.bie.util.Encoder class DownloadService { def grailsApplication - def indexService def messageSource /** From 6f64f4cca622de242794fce38b52ecb2b95b1db8 Mon Sep 17 00:00:00 2001 From: pal155 Date: Wed, 10 Jun 2020 21:04:29 +1000 Subject: [PATCH 03/15] Change occurrence import to a simple scan --- .../au/org/ala/bie/ImportController.groovy | 20 +- .../au/org/ala/bie/ImportService.groovy | 258 +++++------------- 2 files changed, 82 insertions(+), 196 deletions(-) diff --git a/grails-app/controllers/au/org/ala/bie/ImportController.groovy b/grails-app/controllers/au/org/ala/bie/ImportController.groovy index 257eefa1..e7585c2b 100755 --- a/grails-app/controllers/au/org/ala/bie/ImportController.groovy +++ b/grails-app/controllers/au/org/ala/bie/ImportController.groovy @@ -15,7 +15,6 @@ package au.org.ala.bie import au.org.ala.bie.util.Job import grails.converters.JSON -import org.apache.commons.lang.BooleanUtils import au.org.ala.web.AlaSecured /** * Controller for data import into the system. @@ -70,7 +69,7 @@ class ImportController { return } - def clearIndex = BooleanUtils.toBooleanObject(params.clear_index ?: "false") + def clearIndex = params.getBoolean('clear_index', false) def dwcDir = params.dwca_dir if(new File(dwcDir).exists()){ @@ -212,7 +211,8 @@ class ImportController { */ // Documented in openapi.yml def importOccurrences(){ - def job = execute("importOccurrences", "admin.button.loadoccurrence", { importService.importOccurrenceData() }) + def online = params.getBoolean('online', false) + def job = execute("importOccurrences", "admin.button.loadoccurrence", { importService.importOccurrenceData(online) }) asJson (job.status()) } @@ -242,7 +242,7 @@ class ImportController { // Documented in openapi.yml def buildLinkIdentifiers() { - def online = BooleanUtils.toBooleanObject(params.online ?: "false") + def online = params.getBoolean('online', false) def job = execute("buildLinkIdentifiers", "admin.button.buildLinks", { importService.buildLinkIdentifiers(online) }) asJson (job.status()) @@ -250,7 +250,7 @@ class ImportController { // Documented in openapi.yml def denormaliseTaxa() { - def online = BooleanUtils.toBooleanObject(params.online ?: "false") + def online = params.getBoolean('online', false) def job = execute("denormaliseTaxa", "admin.button.denormalise", { importService.denormaliseTaxa(online) }) asJson (job.status()) @@ -258,7 +258,7 @@ class ImportController { // Documented in openapi.yml def buildFavourites() { - def online = BooleanUtils.toBooleanObject(params.online ?: "false") + def online = params.getBoolean('online', false) def job = execute("buildFavourites", "admin.button.buildfavourites", { importService.buildFavourites(online) }) asJson (job.status()) @@ -266,7 +266,7 @@ class ImportController { // Documented in openapi.yml def buildWeights() { - def online = BooleanUtils.toBooleanObject(params.online ?: "false") + def online = params.getBoolean('online', false) def job = execute("buildWeights", "admin.button.buildweights", { importService.buildWeights(online) }) asJson (job.status()) @@ -274,7 +274,7 @@ class ImportController { // Documented in openapi.yml def buildSuggestIndex() { - def online = BooleanUtils.toBooleanObject(params.online ?: "false") + def online = params.getBoolean('online', false) def job = execute("buildSuggestIndex", "admin.button.buildsuggestindex", { importService.buildSuggestIndex(online) }) asJson (job.status()) @@ -288,14 +288,14 @@ class ImportController { */ // Documented in openapi.yml def loadPreferredImages() { - def online = BooleanUtils.toBooleanObject(params.online ?: "false") + def online = params.getBoolean('online', false) def job = execute("loadImages", "admin.button.loadimagespref", { importService.loadPreferredImages(online) }) asJson (job.status()) } // Documented in openapi.yml def loadImages() { - def online = BooleanUtils.toBooleanObject(params.online ?: "false") + def online = params.getBoolean('online', false) def job = execute("loadImages", "admin.button.loadimagesall", { importService.loadImages(online) }) asJson (job.status()) } diff --git a/grails-app/services/au/org/ala/bie/ImportService.groovy b/grails-app/services/au/org/ala/bie/ImportService.groovy index ee1fcedd..e56759ef 100644 --- a/grails-app/services/au/org/ala/bie/ImportService.groovy +++ b/grails-app/services/au/org/ala/bie/ImportService.groovy @@ -100,6 +100,8 @@ class ImportService implements GrailsConfigurationAware { static BATCH_SIZE = 5000 // Buffer size for commits static BUFFER_SIZE = 1000 + // The count size for count requests + static COUNT_SIZE = 20 // Accepted status static ACCEPTED_STATUS = TaxonomicType.values().findAll({ it.accepted }).collect({ "taxonomicStatus:${it.term}" }).join(' OR ') // Synonym status @@ -239,7 +241,7 @@ class ImportService implements GrailsConfigurationAware { importLocalities() break case 'occurrences': - importOccurrenceData() + importOccurrenceData(false) break case 'regions': importRegions() @@ -726,202 +728,86 @@ class ImportService implements GrailsConfigurationAware { } /** - * Paginate through taxon docs in SOLR and update their occurrence status via either: - * - checking against a list of datasetID codes (config) - * OR - * - searching for occurrence records with the (taxon concept) GUID - * - * Example cursor search - * http://bie-dev.ala.org.au/solr/bie/select?q=idxtype:TAXON+AND+taxonomicStatus:accepted&wt=json&rows=100&indent=true&sort=id+asc&cursorMark=* - * Pagination via cursor: https://cwiki.apache.org/confluence/display/solr/Pagination+of+Results - **/ - def importOccurrenceData() throws Exception { - def pageSize = BATCH_SIZE - def paramsMap = [ - q : "idxtype:${IndexDocType.TAXON.name()} AND (${ACCEPTED_STATUS})", - //fq: "datasetID:dr2699", // testing only with AFD - cursorMark: CursorMarkParams.CURSOR_MARK_START, // gets updated by subsequent searches - fl : "id,idxtype,guid,scientificName,datasetID", // will restrict results to dos with these fields (bit like fq) - rows : pageSize, - sort : "id asc" // needed for cursor searching - ] - - // first get a count of results so we can determine number of pages to process - Map countMap = paramsMap.clone(); // shallow clone is OK - countMap.rows = 0 - countMap.remove("cursorMark") - def searchCount = searchService.getCursorSearchResults(new MapSolrParams(countMap), true) - // could throw exception - def totalDocs = searchCount?.results?.numFound ?: 0 - int totalPages = (totalDocs + pageSize - 1) / pageSize - log.debug "totalDocs = ${totalDocs} || totalPages = ${totalPages}" - log("Processing " + String.format("%,d", totalDocs) + " taxa (via ${paramsMap.q})...
") // send to browser - - def promiseList = new PromiseList() // for biocache queries - Queue commitQueue = new ConcurrentLinkedQueue() // queue to put docs to be indexes - ExecutorService executor = Executors.newSingleThreadExecutor() // consumer of queue - single blocking thread - executor.execute { - indexDocInQueue(commitQueue, "initialised") // will keep polling the queue until terminated via cancel() - } - - // iterate over pages - (1..totalPages).each { page -> - try { - MapSolrParams solrParams = new MapSolrParams(paramsMap) - log.debug "${page}. paramsMap = ${paramsMap}" - def searchResults = searchService.getCursorSearchResults(solrParams, true) - // use offline index to search - def resultsDocs = searchResults?.results ?: [] + * Go through the index and get occurrence counts for accepted taxa + */ + def importOccurrenceData(online) { + int pageSize = BATCH_SIZE + int processed = 0 + def typeQuery = "idxtype:\"${IndexDocType.TAXON.name()}\" AND (${ACCEPTED_STATUS})" + def prevCursor + def cursor - // buckets to group results into - def taxaLocatedInHubCountry = [] // automatically get included - def taxaToSearchOccurrences = [] // need to search biocache to see if they are located in hub country + log("Starting occurrence count scan for ${online ? 'online' : 'offline'} index") + try { + prevCursor = "" + cursor = CursorMarkParams.CURSOR_MARK_START + processed = 0 + while (prevCursor != cursor) { + def startTime = System.currentTimeMillis() + SolrQuery query = new SolrQuery(typeQuery) + query.setParam('cursorMark', cursor) + query.setSort("id", SolrQuery.ORDER.asc) + query.setRows(pageSize) + def response = indexService.query(query, online) + def docs = response.results + int total = docs.numFound + def buffer = [] + def guids = [] + def updates = [:] - // iterate over the result set - resultsDocs.each { doc -> + docs.each { doc -> + def taxonID = doc.guid + def update = [id: doc.id, idxtype: [set: doc.idxtype], guid: [set: taxonID], occurrenceCount: [set: 0]] if (nationalSpeciesDatasets && nationalSpeciesDatasets.contains(doc.datasetID)) { - taxaLocatedInHubCountry.add(doc) - // in national list so _assume_ it is located in host/hub county - } else { - taxaToSearchOccurrences.add(doc) - // search occurrence records to determine if it is located in host/hub county + update.locatedInHubCountry = ["set": true] } - } - - // update national list without occurrence record lookup - updateTaxaWithLocationInfo(taxaLocatedInHubCountry, commitQueue) - // update the rest via occurrence search (non blocking via promiseList) - promiseList << { searchOccurrencesWithGuids(resultsDocs, commitQueue) } - // update cursor - paramsMap.cursorMark = searchResults?.nextCursorMark ?: "" - // update view via via JS - updateProgressBar(totalPages, page) - log("${page}. taxaLocatedInHubCountry = ${taxaLocatedInHubCountry.size()} | taxaToSearchOccurrences = ${taxaToSearchOccurrences.size()}") - } catch (Exception ex) { - log.warn "Error calling BIE SOLR: ${ex.message}", ex - log("ERROR calling SOLR: ${ex.message}") - } - } - - log("Waiting for all occurrence searches and SOLR commits to finish (could take some time)") - - //promiseList.get() // block until all promises are complete - promiseList.onComplete { List results -> - //executor.shutdownNow() - isKeepIndexing = false // stop indexing thread - executor.shutdown() - log("Total taxa found with occurrence records = ${results.sum()}") - log("waiting for indexing to finish...") - } - } - - /** - * Batch update of SOLR docs for occurrence/location info - * TODO extract field name into config: "locatedInHubCountry" - * - * @param docs - * @param commitQueue - * @return - */ - def updateTaxaWithLocationInfo(List docs, Queue commitQueue) { - def totalDocumentsUpdated = 0 - - docs.each { Map doc -> - if (doc.containsKey("id") && doc.containsKey("guid") && doc.containsKey("idxtype")) { - Map updateDoc = [:] - updateDoc["id"] = doc.id // doc key - updateDoc["idxtype"] = ["set": doc.idxtype] // required field - updateDoc["guid"] = ["set": doc.guid] // required field - updateDoc["locatedInHubCountry"] = ["set": true] - if (doc.containsKey("occurrenceCount")) { - updateDoc["occurrenceCount"] = ["set": doc["occurrenceCount"]] - } - commitQueue.offer(updateDoc) // throw it on the queue - totalDocumentsUpdated++ - } else { - log.warn "Updating doc error: missing keys ${doc}" - } - } - - totalDocumentsUpdated - } - - /** - * Poll the queue of docs and index in batches - * - * @param updateDocs - * @return - */ - def indexDocInQueue(Queue updateDocs, msg) { - int batchSize = BUFFER_SIZE - - while (isKeepIndexing || updateDocs.size() > 0) { - if (updateDocs.size() > 0) { - log.info "Starting indexing of ${updateDocs.size()} docs" - try { - // batch index docs - List batchDocs = [] - int end = (batchSize < updateDocs.size()) ? batchSize : updateDocs.size() - - (1..end).each { - if (updateDocs.peek()) { - batchDocs.add(updateDocs.poll()) + guids << taxonID + updates[taxonID] = update + + if (guids.size() >= COUNT_SIZE) { + def cts = biocacheService.counts(guids, occurrenceCountFilter) + guids.each { guid -> + def val = cts[guid] + def upd = updates[guid] + if (val && upd) + upd.occurrenceCount = [set: val] + buffer << upd } + guids = [] + updates = [:] } - - indexService.indexBatch(batchDocs) // index - } catch (Exception ex) { - log.warn "Error batch indexing: ${ex.message}", ex - log.warn "updateDocs = ${updateDocs}" - log("ERROR batch indexing: ${ex.message}
${ex.stackTrace}") + if (buffer.size() >= BUFFER_SIZE) { + indexService.indexBatch(buffer, online) + buffer = [] + } + processed++ } - } else { - sleep(500) - } - } - - log("Indexing thread is done: ${msg}") - } - - /** - * Extract a list of GUIDs from input list of docs and do paginated/batch search of occurrence records, - * updating index with occurrence status info (could be presence or record counts, etc) - * - * @param docs - * @return - */ - def searchOccurrencesWithGuids(List docs, Queue commitQueue) { - int batchSize = 20 // even with POST SOLR throws 400 code is batchSize is more than 100 - List guids = docs.collect { it.guid } - int totalPages = ((guids.size() + batchSize - 1) / batchSize) - 1 - log.debug "total = ${guids.size()} || batchSize = ${batchSize} || totalPages = ${totalPages}" - List docsWithRecs = [] // docs to index - //log("Getting occurrence data for ${docs.size()} docs") - - (0..totalPages).each { index -> - try { - int start = index * batchSize - int end = (start + batchSize < guids.size()) ? start + batchSize - 1 : guids.size() - log.debug "paging biocache search - ${start} to ${end}" - def guidSubset = guids.subList(start, end) - def counts = biocacheService.counts(guidSubset, occurrenceCountFilter) - counts?.each { guid, count -> - def docWithRecs = docs.find { it.guid == guid } - if (docWithRecs) { - docWithRecs["occurrenceCount"] = count - docsWithRecs.add(docWithRecs) + if (guids.size() > 0) { + def cts = biocacheService.counts(guids, occurrenceCountFilter) + guids.each { guid -> + def val = cts[guid] + def update = updates[guid] + if (val && update) + update.occurrenceCount = [set: val] + buffer.addAll(updates.values()) } } - } catch (Exception ex) { - log.warn "Error calling biocache SOLR: ${ex.message}", ex - log("ERROR calling biocache SOLR: ${ex.message}") + if (!buffer.isEmpty()) + indexService.indexBatch(buffer, online) + def percentage = total ? Math.round(processed * 100 / total) : 100 + def speed = total ? Math.round((pageSize * 1000) / (System.currentTimeMillis() - startTime)) : 0 + log("Processed ${processed} taxa (${percentage}%) speed ${speed} records per second") + if (total > 0) { + updateProgressBar(total, processed) + } + prevCursor = cursor + cursor = response.nextCursorMark } + log("Finished scan") + } catch (Exception ex) { + log.error("Unable to perform occurrence scan", ex) + log("Error during scan: " + ex.getMessage()) } - if (docsWithRecs.size() > 0) { - log.debug "docsWithRecs size = ${docsWithRecs.size()} vs docs size = ${docs.size()}" - updateTaxaWithLocationInfo(docsWithRecs, commitQueue) - } - } /** From bcea2d493a9197a279d4cf3a60157edd0f53053b Mon Sep 17 00:00:00 2001 From: pal155 Date: Tue, 16 Jun 2020 10:32:08 +1000 Subject: [PATCH 04/15] Improve config access --- grails-app/services/au/org/ala/bie/DownloadService.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grails-app/services/au/org/ala/bie/DownloadService.groovy b/grails-app/services/au/org/ala/bie/DownloadService.groovy index 7232414d..64c819ed 100644 --- a/grails-app/services/au/org/ala/bie/DownloadService.groovy +++ b/grails-app/services/au/org/ala/bie/DownloadService.groovy @@ -31,7 +31,7 @@ class DownloadService { def download(params, OutputStream outputStream, Locale locale){ def q = Encoder.escapeQuery(params.q ?: "*:*") def fq = new ArrayList<>(params.list('fq')) // Make modifiable - def fields = params.fields ?: grailsApplication.config.defaultDownloadFields + def fields = params.fields ?: grailsApplication.getRequiredProperty('defaultDownloadFields') def fqs = '' grailsApplication.config.solr.search.fq.each { fq << it } From 9c93445e36fdd9aef2eb3f55ce6ade7db95690a7 Mon Sep 17 00:00:00 2001 From: pal155 Date: Tue, 12 Jan 2021 10:38:57 +1100 Subject: [PATCH 05/15] Improve config access --- grails-app/services/au/org/ala/bie/DownloadService.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grails-app/services/au/org/ala/bie/DownloadService.groovy b/grails-app/services/au/org/ala/bie/DownloadService.groovy index 64c819ed..cbf7cc52 100644 --- a/grails-app/services/au/org/ala/bie/DownloadService.groovy +++ b/grails-app/services/au/org/ala/bie/DownloadService.groovy @@ -31,7 +31,7 @@ class DownloadService { def download(params, OutputStream outputStream, Locale locale){ def q = Encoder.escapeQuery(params.q ?: "*:*") def fq = new ArrayList<>(params.list('fq')) // Make modifiable - def fields = params.fields ?: grailsApplication.getRequiredProperty('defaultDownloadFields') + def fields = params.fields ?: grailsApplication.config.getRequiredProperty('defaultDownloadFields') def fqs = '' grailsApplication.config.solr.search.fq.each { fq << it } From 599fe42c47466e2d72948c1dc00536049296b9b9 Mon Sep 17 00:00:00 2001 From: pal155 Date: Tue, 12 Jan 2021 12:32:40 +1100 Subject: [PATCH 06/15] Docker image for BIE solr --- docker/README.md | 22 + docker/solr6/Dockerfile | 15 + docker/solr6/bie-offline.core.properties | 3 + docker/solr6/bie.core.properties | 3 + docker/solr6/conf/bie_stopwords.txt | 0 docker/solr6/conf/elevate.xml | 36 + docker/solr6/conf/mapping-ISOLatin1Accent.txt | 246 +++ docker/solr6/conf/protwords.txt | 21 + docker/solr6/conf/schema.xml | 534 +++++ docker/solr6/conf/solrconfig.xml | 1814 +++++++++++++++++ docker/solr6/conf/stopwords.txt | 58 + docker/solr6/conf/synonyms.txt | 31 + docker/solr6/solr.in.sh | 119 ++ docker/solr6/solr.xml | 53 + docker/solr6/zoo.cfg | 31 + 15 files changed, 2986 insertions(+) create mode 100644 docker/README.md create mode 100644 docker/solr6/Dockerfile create mode 100644 docker/solr6/bie-offline.core.properties create mode 100644 docker/solr6/bie.core.properties create mode 100644 docker/solr6/conf/bie_stopwords.txt create mode 100644 docker/solr6/conf/elevate.xml create mode 100644 docker/solr6/conf/mapping-ISOLatin1Accent.txt create mode 100644 docker/solr6/conf/protwords.txt create mode 100644 docker/solr6/conf/schema.xml create mode 100644 docker/solr6/conf/solrconfig.xml create mode 100644 docker/solr6/conf/stopwords.txt create mode 100644 docker/solr6/conf/synonyms.txt create mode 100644 docker/solr6/solr.in.sh create mode 100644 docker/solr6/solr.xml create mode 100644 docker/solr6/zoo.cfg diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..976a29eb --- /dev/null +++ b/docker/README.md @@ -0,0 +1,22 @@ +# Docker Images + +Subdirectories contain docker descriptions that can be used for testing and +debugging + +## Solr + +The solr image contains a solr server with pre-configred `bie` and `bie-offline` cores. +To build a solr6 image for use with the bie-index run, from this directory + +```shell +docker build solr6 -t bie-solr:v1 +``` + +To run the resulting image in a fresh container, use + +```shell +docker run -p 8983:8983 bie-solr:v1 +``` + +Set the solr connections in the configuration to +`http://localhost:8983/solr/bie` and `http://localhost:8983/solr/bie-offline` \ No newline at end of file diff --git a/docker/solr6/Dockerfile b/docker/solr6/Dockerfile new file mode 100644 index 00000000..70322760 --- /dev/null +++ b/docker/solr6/Dockerfile @@ -0,0 +1,15 @@ +FROM solr:6.6 +WORKDIR /opt/solr/server/solr +COPY --chown=solr:solr solr.xml solr.xml +COPY --chown=solr:solr zoo.cfg zoo.cfg +#COPY --chown=solr:solr lib /opt/solr/server/solr-webapp/webapp/WEB-INF/lib/ +RUN mkdir -p bie bie/conf bie/data bie/suggest-infix +COPY --chown=solr:solr conf bie/conf +COPY --chown=solr:solr bie.core.properties bie/core.properties +RUN chown -R solr.solr bie +RUN mkdir -p bie-offline bie-offline/conf bie-offline/data bie-offline/suggest-infix +COPY --chown=solr:solr conf bie-offline/conf +COPY --chown=solr:solr bie-offline.core.properties bie-offline/core.properties +RUN chown -R solr.solr bie-offline +COPY solr.in.sh /etc/default/solr.in.sh + diff --git a/docker/solr6/bie-offline.core.properties b/docker/solr6/bie-offline.core.properties new file mode 100644 index 00000000..422d6f8a --- /dev/null +++ b/docker/solr6/bie-offline.core.properties @@ -0,0 +1,3 @@ +config=solrconfig.xml +name=bie-offline +dataDir=data \ No newline at end of file diff --git a/docker/solr6/bie.core.properties b/docker/solr6/bie.core.properties new file mode 100644 index 00000000..d8624c39 --- /dev/null +++ b/docker/solr6/bie.core.properties @@ -0,0 +1,3 @@ +config=solrconfig.xml +name=bie +dataDir=data \ No newline at end of file diff --git a/docker/solr6/conf/bie_stopwords.txt b/docker/solr6/conf/bie_stopwords.txt new file mode 100644 index 00000000..e69de29b diff --git a/docker/solr6/conf/elevate.xml b/docker/solr6/conf/elevate.xml new file mode 100644 index 00000000..7630ebe2 --- /dev/null +++ b/docker/solr6/conf/elevate.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + diff --git a/docker/solr6/conf/mapping-ISOLatin1Accent.txt b/docker/solr6/conf/mapping-ISOLatin1Accent.txt new file mode 100644 index 00000000..ede77425 --- /dev/null +++ b/docker/solr6/conf/mapping-ISOLatin1Accent.txt @@ -0,0 +1,246 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Syntax: +# "source" => "target" +# "source".length() > 0 (source cannot be empty.) +# "target".length() >= 0 (target can be empty.) + +# example: +# "À" => "A" +# "\u00C0" => "A" +# "\u00C0" => "\u0041" +# "ß" => "ss" +# "\t" => " " +# "\n" => "" + +# À => A +"\u00C0" => "A" + +# Á => A +"\u00C1" => "A" + +#  => A +"\u00C2" => "A" + +# à => A +"\u00C3" => "A" + +# Ä => A +"\u00C4" => "A" + +# Å => A +"\u00C5" => "A" + +# Æ => AE +"\u00C6" => "AE" + +# Ç => C +"\u00C7" => "C" + +# È => E +"\u00C8" => "E" + +# É => E +"\u00C9" => "E" + +# Ê => E +"\u00CA" => "E" + +# Ë => E +"\u00CB" => "E" + +# Ì => I +"\u00CC" => "I" + +# Í => I +"\u00CD" => "I" + +# Î => I +"\u00CE" => "I" + +# Ï => I +"\u00CF" => "I" + +# IJ => IJ +"\u0132" => "IJ" + +# Ð => D +"\u00D0" => "D" + +# Ñ => N +"\u00D1" => "N" + +# Ò => O +"\u00D2" => "O" + +# Ó => O +"\u00D3" => "O" + +# Ô => O +"\u00D4" => "O" + +# Õ => O +"\u00D5" => "O" + +# Ö => O +"\u00D6" => "O" + +# Ø => O +"\u00D8" => "O" + +# Œ => OE +"\u0152" => "OE" + +# Þ +"\u00DE" => "TH" + +# Ù => U +"\u00D9" => "U" + +# Ú => U +"\u00DA" => "U" + +# Û => U +"\u00DB" => "U" + +# Ü => U +"\u00DC" => "U" + +# Ý => Y +"\u00DD" => "Y" + +# Ÿ => Y +"\u0178" => "Y" + +# à => a +"\u00E0" => "a" + +# á => a +"\u00E1" => "a" + +# â => a +"\u00E2" => "a" + +# ã => a +"\u00E3" => "a" + +# ä => a +"\u00E4" => "a" + +# å => a +"\u00E5" => "a" + +# æ => ae +"\u00E6" => "ae" + +# ç => c +"\u00E7" => "c" + +# è => e +"\u00E8" => "e" + +# é => e +"\u00E9" => "e" + +# ê => e +"\u00EA" => "e" + +# ë => e +"\u00EB" => "e" + +# ì => i +"\u00EC" => "i" + +# í => i +"\u00ED" => "i" + +# î => i +"\u00EE" => "i" + +# ï => i +"\u00EF" => "i" + +# ij => ij +"\u0133" => "ij" + +# ð => d +"\u00F0" => "d" + +# ñ => n +"\u00F1" => "n" + +# ò => o +"\u00F2" => "o" + +# ó => o +"\u00F3" => "o" + +# ô => o +"\u00F4" => "o" + +# õ => o +"\u00F5" => "o" + +# ö => o +"\u00F6" => "o" + +# ø => o +"\u00F8" => "o" + +# œ => oe +"\u0153" => "oe" + +# ß => ss +"\u00DF" => "ss" + +# þ => th +"\u00FE" => "th" + +# ù => u +"\u00F9" => "u" + +# ú => u +"\u00FA" => "u" + +# û => u +"\u00FB" => "u" + +# ü => u +"\u00FC" => "u" + +# ý => y +"\u00FD" => "y" + +# ÿ => y +"\u00FF" => "y" + +# ff => ff +"\uFB00" => "ff" + +# fi => fi +"\uFB01" => "fi" + +# fl => fl +"\uFB02" => "fl" + +# ffi => ffi +"\uFB03" => "ffi" + +# ffl => ffl +"\uFB04" => "ffl" + +# ſt => ft +"\uFB05" => "ft" + +# st => st +"\uFB06" => "st" diff --git a/docker/solr6/conf/protwords.txt b/docker/solr6/conf/protwords.txt new file mode 100644 index 00000000..1dfc0abe --- /dev/null +++ b/docker/solr6/conf/protwords.txt @@ -0,0 +1,21 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# Use a protected word file to protect against the stemmer reducing two +# unrelated words to the same base word. + +# Some non-words that normally won't be encountered, +# just to test that they won't be stemmed. +dontstems +zwhacky + diff --git a/docker/solr6/conf/schema.xml b/docker/solr6/conf/schema.xml new file mode 100644 index 00000000..85c6bdb1 --- /dev/null +++ b/docker/solr6/conf/schema.xml @@ -0,0 +1,534 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docker/solr6/conf/solrconfig.xml b/docker/solr6/conf/solrconfig.xml new file mode 100644 index 00000000..b34351cb --- /dev/null +++ b/docker/solr6/conf/solrconfig.xml @@ -0,0 +1,1814 @@ + + + + + + + + + 6.6.5 + + + + + + + + + + + + + + + + + + + + + + + + ${solr.data.dir:} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.ulog.dir:} + + + + + 15000 + false + + + + + + + + + + + + + + + + + + + + + 102400 + + + + + + + + + + + + + + + + + + + + + + true + + + + + + 20 + + + 200 + + + + + + + + + + + + static firstSearcher warming in solrconfig.xml + + + + + + false + + + 2 + + + + + + + + + + + + + + + + + + + + + + + explicit + 10 + text + + + + + + + + + + + + + + explicit + json + true + text + + + + + + + + true + json + true + + + + + + + + explicit + + + velocity + browse + layout + Solritas + + + edismax + + text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4 + title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0 + + text + 100% + *:* + 10 + *,score + + + text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4 + title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0 + + text,features,name,sku,id,manu,cat,title,description,keywords,author,resourcename + 3 + + + on + cat + manu_exact + content_type + author_s + ipod + GB + 1 + cat,inStock + after + price + 0 + 600 + 50 + popularity + 0 + 10 + 3 + manufacturedate_dt + NOW/YEAR-10YEARS + NOW + +1YEAR + before + after + + + on + content features title name + html + <b> + </b> + 0 + title + 0 + name + 3 + 200 + content + 750 + + + on + false + 5 + 2 + 5 + true + true + 5 + 3 + + + + + spellcheck + + + + + + + + + + + + + + + + + true + ignored_ + + + true + links + ignored_ + + + + + + + + + + + + + + + + + + + + + + + + solrpingquery + + + all + + + + + + + + + explicit + true + + + + + + + + + + + + + infixSuggester + true + 10 + + + suggest + + + + + + + + infixSuggester + BlendedInfixLookupFactory + DocumentDictionaryFactory + auto_text + suggestWeight + guid + idxtype --> + true + false + + text_suggest + false + + + + + + + textSpell + + + + + + default + auto_text_edge + solr.DirectSolrSpellChecker + + internal + + 0.5 + + 2 + + 1 + + 5 + + 4 + + 0.01 + + + + + + wordbreak + solr.WordBreakSolrSpellChecker + auto_text_edge + true + true + 10 + + + + + + + + + + + + + + + + text + + default + wordbreak + on + true + 10 + 5 + 5 + true + true + 10 + 5 + + + spellcheck + + + + + + + + + + text + true + + + tvComponent + + + + + + + + + default + + + org.carrot2.clustering.lingo.LingoClusteringAlgorithm + + + 20 + + + clustering/carrot2 + + + ENGLISH + + + stc + org.carrot2.clustering.stc.STCClusteringAlgorithm + + + + + + + true + default + true + + name + id + + features + + true + + + + false + + edismax + + text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4 + + *:* + 10 + *,score + + + clustering + + + + + + + + + + true + false + + + terms + + + + + + + + string + elevate.xml + + + + + + explicit + text + + + elevator + + + + + + + + + + + 100 + + + + + + + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? + + + + + + + WORD + + + en + US + + + + + + + + + + + + + + + + + + + + + + text/plain; charset=UTF-8 + + + + + + + + + 5 + + + + + + + + + + + + + + + + + + *:* + + + diff --git a/docker/solr6/conf/stopwords.txt b/docker/solr6/conf/stopwords.txt new file mode 100644 index 00000000..b5824da3 --- /dev/null +++ b/docker/solr6/conf/stopwords.txt @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# a couple of test stopwords to test that the words are really being +# configured from this file: +stopworda +stopwordb + +#Standard english stop words taken from Lucene's StopAnalyzer +a +an +and +are +as +at +be +but +by +for +if +in +into +is +it +no +not +of +on +or +s +such +t +that +the +their +then +there +these +they +this +to +was +will +with + diff --git a/docker/solr6/conf/synonyms.txt b/docker/solr6/conf/synonyms.txt new file mode 100644 index 00000000..b0e31cb7 --- /dev/null +++ b/docker/solr6/conf/synonyms.txt @@ -0,0 +1,31 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +#some test synonym mappings unlikely to appear in real input text +aaa => aaaa +bbb => bbbb1 bbbb2 +ccc => cccc1,cccc2 +a\=>a => b\=>b +a\,a => b\,b +fooaaa,baraaa,bazaaa + +# Some synonym groups specific to this example +GB,gib,gigabyte,gigabytes +MB,mib,megabyte,megabytes +Television, Televisions, TV, TVs +#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming +#after us won't split it into two words. + +# Synonym mappings can be used for spelling correction too +pixima => pixma + diff --git a/docker/solr6/solr.in.sh b/docker/solr6/solr.in.sh new file mode 100644 index 00000000..fd2f449f --- /dev/null +++ b/docker/solr6/solr.in.sh @@ -0,0 +1,119 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# By default the script will use JAVA_HOME to determine which java +# to use, but you can set a specific path for Solr to use without +# affecting other Java applications on your server/workstation. +#SOLR_JAVA_HOME="" + +# Increase Java Heap as needed to support your indexing / query needs +SOLR_HEAP="1g" + +# Expert: If you want finer control over memory options, specify them directly +# Comment out SOLR_HEAP if you are using this though, that takes precedence +#SOLR_JAVA_MEM="-Xms512m -Xmx512m" + +# Enable verbose GC logging +GC_LOG_OPTS="" + +# These GC settings have shown to work well for a number of common Solr workloads +GC_TUNE="-XX:NewRatio=3 \ +-XX:SurvivorRatio=4 \ +-XX:TargetSurvivorRatio=90 \ +-XX:MaxTenuringThreshold=8 \ +-XX:+UseConcMarkSweepGC \ +-XX:ConcGCThreads=4 -XX:ParallelGCThreads=4 \ +-XX:+CMSScavengeBeforeRemark \ +-XX:PretenureSizeThreshold=64m \ +-XX:+UseCMSInitiatingOccupancyOnly \ +-XX:CMSInitiatingOccupancyFraction=50 \ +-XX:CMSMaxAbortablePrecleanTime=6000 \ +-XX:+CMSParallelRemarkEnabled \ +-XX:+ParallelRefProcEnabled" + +# Set the ZooKeeper connection string if using an external ZooKeeper ensemble +# e.g. host1:2181,host2:2181/chroot +# Leave empty if not using SolrCloud +ZK_HOST="" + +# Set the ZooKeeper client timeout (for SolrCloud mode) +ZK_CLIENT_TIMEOUT="15000" + +# By default the start script uses "localhost"; override the hostname here +# for production SolrCloud environments to control the hostname exposed to cluster state +SOLR_HOST="localhost" + +# By default the start script uses UTC; override the timezone if needed +#SOLR_TIMEZONE="UTC" + +# Set to true to activate the JMX RMI connector to allow remote JMX client applications +# to monitor the JVM hosting Solr; set to "false" to disable that behavior +# (false is recommended in production environments) +ENABLE_REMOTE_JMX_OPTS="false" + +# The script will use SOLR_PORT+10000 for the RMI_PORT or you can set it here +# RMI_PORT=18983 + +# Set the thread stack size +SOLR_OPTS="$SOLR_OPTS -Xss256k" + +# Anything you add to the SOLR_OPTS variable will be included in the java +# start command line as-is, in ADDITION to other options. If you specify the +# -a option on start script, those options will be appended as well. Examples: +#SOLR_OPTS="$SOLR_OPTS -Dsolr.autoSoftCommit.maxTime=3000" +#SOLR_OPTS="$SOLR_OPTS -Dsolr.autoCommit.maxTime=60000" +#SOLR_OPTS="$SOLR_OPTS -Dsolr.clustering.enabled=true" + +# Location where the bin/solr script will save PID files for running instances +# If not set, the script will create PID files in $SOLR_TIP/bin +#SOLR_PID_DIR= + +# Path to a directory for Solr to store cores and their data. By default, Solr will use server/solr +# If solr.xml is not stored in ZooKeeper, this directory needs to contain solr.xml +#SOLR_HOME=/data/solr/data + +# Solr provides a default Log4J configuration properties file in server/resources +# however, you may want to customize the log settings and file appender location +# so you can point the script to use a different log4j.properties file +#LOG4J_PROPS=/data/solr/log4j.properties + +# Location where Solr should write logs to; should agree with the file appender +# settings in server/resources/log4j.properties +#SOLR_LOGS_DIR=/data/solr/logs + +# Sets the port Solr binds to, default is 8983 +SOLR_PORT=8983 + +# Uncomment to set SSL-related system properties +# Be sure to update the paths to the correct keystore for your environment +#SOLR_SSL_KEY_STORE=etc/solr-ssl.keystore.jks +#SOLR_SSL_KEY_STORE_PASSWORD=secret +#SOLR_SSL_TRUST_STORE=etc/solr-ssl.keystore.jks +#SOLR_SSL_TRUST_STORE_PASSWORD=secret +#SOLR_SSL_NEED_CLIENT_AUTH=false +#SOLR_SSL_WANT_CLIENT_AUTH=false + +# Uncomment if you want to override previously defined SSL values for HTTP client +# otherwise keep them commented and the above values will automatically be set for HTTP clients +#SOLR_SSL_CLIENT_KEY_STORE= +#SOLR_SSL_CLIENT_KEY_STORE_PASSWORD= +#SOLR_SSL_CLIENT_TRUST_STORE= +#SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD= + +# Settings for authentication +#SOLR_AUTHENTICATION_CLIENT_CONFIGURER= +#SOLR_AUTHENTICATION_OPTS= + +SOLR_MODE="" diff --git a/docker/solr6/solr.xml b/docker/solr6/solr.xml new file mode 100644 index 00000000..68b15ba0 --- /dev/null +++ b/docker/solr6/solr.xml @@ -0,0 +1,53 @@ + + + + + + + + + + ${host:} + ${jetty.port:8983} + ${hostContext:solr} + + ${genericCoreNodeNames:true} + + ${zkClientTimeout:30000} + ${distribUpdateSoTimeout:600000} + ${distribUpdateConnTimeout:60000} + ${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider} + ${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider} + + + + + ${socketTimeout:600000} + ${connTimeout:60000} + + + diff --git a/docker/solr6/zoo.cfg b/docker/solr6/zoo.cfg new file mode 100644 index 00000000..7e42d8c8 --- /dev/null +++ b/docker/solr6/zoo.cfg @@ -0,0 +1,31 @@ +# The number of milliseconds of each tick +tickTime=2000 +# The number of ticks that the initial +# synchronization phase can take +initLimit=10 +# The number of ticks that can pass between +# sending a request and getting an acknowledgement +syncLimit=5 + +# the directory where the snapshot is stored. +# dataDir=/opt/zookeeper/data +# NOTE: Solr defaults the dataDir to /zoo_data + +# the port at which the clients will connect +# clientPort=2181 +# NOTE: Solr sets this based on zkRun / zkHost params + +# the maximum number of client connections. +# increase this if you need to handle more clients +#maxClientCnxns=60 +# +# Be sure to read the maintenance section of the +# administrator guide before turning on autopurge. +# +# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance +# +# The number of snapshots to retain in dataDir +#autopurge.snapRetainCount=3 +# Purge task interval in hours +# Set to "0" to disable auto purge feature +#autopurge.purgeInterval=1 From 90a6ac041b9acc6bf6c11e748d9a5a73b33312c4 Mon Sep 17 00:00:00 2001 From: pal155 Date: Tue, 12 Jan 2021 14:59:46 +1100 Subject: [PATCH 07/15] Improve contents send to the bie frontend --- .../au/org/ala/bie/SearchService.groovy | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/grails-app/services/au/org/ala/bie/SearchService.groovy b/grails-app/services/au/org/ala/bie/SearchService.groovy index 9a503dc7..c6446878 100644 --- a/grails-app/services/au/org/ala/bie/SearchService.groovy +++ b/grails-app/services/au/org/ala/bie/SearchService.groovy @@ -990,19 +990,19 @@ class SearchService { "infoSourceName" : it.datasetName, "infoSourceURL" : "${grailsApplication.config.collectory.base}/public/show/${it.datasetID}" ] - } else if (it.idxtype == IndexDocType.COMMON.name()){ + } else if (it.idxtype == IndexDocType.COMMON.name()) { doc = [ - "id" : it.id, // needed for highlighting - "guid" : it.guid, - "taxonGuid" : it.taxonGuid, - "linkIdentifier" : it.linkIdentifier, - "idxtype": it.idxtype, - "name" : it.name, - "language" : it.language, + "id" : it.id, // needed for highlighting + "guid" : it.guid, + "taxonGuid" : it.taxonGuid, + "linkIdentifier" : it.linkIdentifier, + "idxtype" : it.idxtype, + "name" : it.name, + "language" : it.language, "acceptedConceptName": it.acceptedConceptName, - "favourite": it.favourite, - "infoSourceName" : it.datasetName, - "infoSourceURL" : "${grailsApplication.config.collectory.base}/public/show/${it.datasetID}" + "favourite" : it.favourite, + "infoSourceName" : it.datasetName, + "infoSourceURL" : "${grailsApplication.config.collectory.base}/public/show/${it.datasetID}" ] if (it.image) { doc.put("image", it.image) @@ -1029,6 +1029,9 @@ class SearchService { if(it.favourite){ doc.put("favourite", it.favourite) } + if (it.content){ + doc.put("content", it.content) + } } if (doc) { if(getAdditionalResultFields()){ From fde3a931240d5fa3ff6e9700627d9daa5f3b4d7f Mon Sep 17 00:00:00 2001 From: pal155 Date: Thu, 28 Jan 2021 11:48:39 +1100 Subject: [PATCH 08/15] Issue 300 Allow KB and site articles to be refreshed online --- .../au/org/ala/bie/ImportController.groovy | 6 ++-- .../au/org/ala/bie/ImportService.groovy | 32 +++++++++---------- .../au/org/ala/bie/IndexService.groovy | 7 ++-- grails-app/views/import/knowledgebase.gsp | 11 ++++++- grails-app/views/import/wordpress.gsp | 11 ++++++- src/main/resources/public/openapi.yml | 31 +++++++++++++++++- 6 files changed, 74 insertions(+), 24 deletions(-) diff --git a/grails-app/controllers/au/org/ala/bie/ImportController.groovy b/grails-app/controllers/au/org/ala/bie/ImportController.groovy index e7585c2b..049172f5 100755 --- a/grails-app/controllers/au/org/ala/bie/ImportController.groovy +++ b/grails-app/controllers/au/org/ala/bie/ImportController.groovy @@ -190,7 +190,8 @@ class ImportController { */ // Documented in openapi.yml def importWordPress(){ - def job = execute("importWordPressPages", "admin.button.loadwordpress", { importService.importWordPressPages() }) + boolean online = params.getBoolean('online', false) + def job = execute("importWordPressPages", "admin.button.loadwordpress", { importService.importWordPressPages(online) }) asJson(job.status()) } @@ -200,7 +201,8 @@ class ImportController { * @return */ def importKnowledgeBase(){ - def job = execute("importKnowledgeBasePages", "admin.button.loadknowledgebase", { importService.importKnowledgeBasePages() }) + boolean online = params.getBoolean('online', false) + def job = execute("importKnowledgeBasePages", "admin.button.loadknowledgebase", { importService.importKnowledgeBasePages(online) }) asJson(job.status()) } diff --git a/grails-app/services/au/org/ala/bie/ImportService.groovy b/grails-app/services/au/org/ala/bie/ImportService.groovy index e56759ef..6c7de417 100644 --- a/grails-app/services/au/org/ala/bie/ImportService.groovy +++ b/grails-app/services/au/org/ala/bie/ImportService.groovy @@ -505,22 +505,16 @@ class ImportService implements GrailsConfigurationAware { /** * Index WordPress pages */ - def importWordPressPages() throws Exception { + def importWordPressPages(boolean online) throws Exception { log "Starting wordpress import" - // clear the existing WP index - indexService.deleteFromIndex(IndexDocType.WORDPRESS) - if (!wordPressSitemap) { - return - } - - // get List of WordPress document URLs (each page's URL) - def pages = wordpressService.resources() + def buffer = [] + def pages = wordPressSitemap ? wordpressService.resources() : [] def documentCount = 0 def totalDocs = pages.size() - def buffer = [] log("WordPress pages found: ${totalDocs}") // update user via socket // slurp and build each SOLR doc (add to buffer) + // We're expecting less than 1000 documents here so we can delete and update in one batch pages.each { pageUrl -> log "indexing url: ${pageUrl}" try { @@ -567,7 +561,11 @@ class ImportService implements GrailsConfigurationAware { } } log("Committing to ${buffer.size()} documents to SOLR...") - indexService.indexBatch(buffer) + if (online) { + log "Search for wordpress pages may be temporarily unavailable" + } + indexService.deleteFromIndex(IndexDocType.WORDPRESS, online) + indexService.indexBatch(buffer, online) updateProgressBar(100, 100) // complete progress bar log "Finished wordpress import" } @@ -575,10 +573,8 @@ class ImportService implements GrailsConfigurationAware { /** * Index Knowledge Base pages. */ - def importKnowledgeBasePages() throws Exception { - log "Starting knowledge base import" - // clear the existing WP index - indexService.deleteFromIndex(IndexDocType.KNOWLEDGEBASE) + def importKnowledgeBasePages(boolean online) throws Exception { + log "Starting knowledge base import." // get List of Knowledge Base document URLs (each page's URL) def pages = knowledgeBaseService.resources() @@ -620,7 +616,11 @@ class ImportService implements GrailsConfigurationAware { } } log("Committing to ${buffer.size()} documents to SOLR...") - indexService.indexBatch(buffer) + if (online) { + log "Search for wordpress pages may be temporarily unavailable" + } + indexService.deleteFromIndex(IndexDocType.KNOWLEDGEBASE, online) + indexService.indexBatch(buffer, online) updateProgressBar(100, 100) // complete progress bar log "Finished knowledge base import" } diff --git a/grails-app/services/au/org/ala/bie/IndexService.groovy b/grails-app/services/au/org/ala/bie/IndexService.groovy index ca597ef1..fda37c73 100644 --- a/grails-app/services/au/org/ala/bie/IndexService.groovy +++ b/grails-app/services/au/org/ala/bie/IndexService.groovy @@ -63,9 +63,10 @@ class IndexService implements GrailsConfigurationAware { * @param docType * @return */ - def deleteFromIndex(IndexDocType docType){ + def deleteFromIndex(IndexDocType docType, boolean online = false) { log.info("Deleting from index: " + docType.name() + "....") - offlineSolrClient.deleteByQuery("idxtype:" + docType.name()) + def client = online ? updatingLiveSolrClient : offlineSolrClient + client.deleteByQuery("idxtype:" + docType.name()) log.info("Deleted from index: " + docType.name()) } @@ -75,7 +76,7 @@ class IndexService implements GrailsConfigurationAware { * @param docsToIndex * @param offline Use the offline index (defaults to true) */ - def indexBatch(List docsToIndex, online = false) throws Exception { + def indexBatch(List docsToIndex, boolean online = false) throws Exception { def client = online ? updatingLiveSolrClient : offlineSolrClient def buffer = [] diff --git a/grails-app/views/import/knowledgebase.gsp b/grails-app/views/import/knowledgebase.gsp index e029d187..e9479e15 100644 --- a/grails-app/views/import/knowledgebase.gsp +++ b/grails-app/views/import/knowledgebase.gsp @@ -35,11 +35,20 @@
-
+
+ +
+ + + function importKnowledgeBase(){ + loadInfo("${createLink(controller:'import', action:'importKnowledgeBase')}?online=" + $('#use-online').is(':checked')); + } + \ No newline at end of file diff --git a/grails-app/views/import/wordpress.gsp b/grails-app/views/import/wordpress.gsp index e6e227cf..d14e6516 100644 --- a/grails-app/views/import/wordpress.gsp +++ b/grails-app/views/import/wordpress.gsp @@ -35,10 +35,19 @@
- + +
+
+
+ + + function importWordPress(){ + loadInfo("${createLink(controller:'import', action:'importWordPress')}?online=" + $('#use-online').is(':checked')); + } + \ No newline at end of file diff --git a/src/main/resources/public/openapi.yml b/src/main/resources/public/openapi.yml index c6d88044..37fff6b8 100644 --- a/src/main/resources/public/openapi.yml +++ b/src/main/resources/public/openapi.yml @@ -912,9 +912,16 @@ paths: get: summary: Import workpress pages description: > - Imports wordpress pages into the offline index + Imports wordpress pages into the offline (defauult) or online index tags: - admin + parameters: + - name: online + in: query + description: Build using the online, rather than the offline, index + schema: + type: boolean + default: false security: - cas: [] responses: @@ -923,6 +930,28 @@ paths: application/json: schema: $ref: '#/components/schemas/JobStatus' + /admin/import/importKnowledgebase: + get: + summary: Import knowledgebase pages + description: > + Imports knowledgebase pages into the offline (default) or online index + tags: + - admin + parameters: + - name: online + in: query + description: Build using the online, rather than the offline, index + schema: + type: boolean + default: false + security: + - cas: [] + responses: + 200: + content: + application/json: + schema: + $ref: '#/components/schemas/JobStatus' /admin/import/importOccurrences: get: summary: Scan for occurrence counts From ce4e5b35289712c9e603be94e6b978b56d875a2d Mon Sep 17 00:00:00 2001 From: pal155 Date: Thu, 28 Jan 2021 14:58:38 +1100 Subject: [PATCH 09/15] Issue 300 Allow collectory and spatial information to be refreshed online --- .../au/org/ala/bie/ImportController.groovy | 12 ++- .../au/org/ala/bie/ImportService.groovy | 44 +++++++---- grails-app/views/import/collectory.gsp | 12 ++- grails-app/views/import/layers.gsp | 11 ++- grails-app/views/import/localities.gsp | 11 ++- grails-app/views/import/regions.gsp | 11 ++- src/main/resources/public/openapi.yml | 79 ++++++------------- 7 files changed, 102 insertions(+), 78 deletions(-) diff --git a/grails-app/controllers/au/org/ala/bie/ImportController.groovy b/grails-app/controllers/au/org/ala/bie/ImportController.groovy index 049172f5..787a2f34 100755 --- a/grails-app/controllers/au/org/ala/bie/ImportController.groovy +++ b/grails-app/controllers/au/org/ala/bie/ImportController.groovy @@ -119,8 +119,9 @@ class ImportController { */ // Documented in openapi.yml def importCollectory(){ + boolean online = params.getBoolean('online', false) if(grailsApplication.config.collectory.service){ - def job = execute("importCollectory", "admin.button.importcollectory", { importService.importCollectory() }) + def job = execute("importCollectory", "admin.button.importcollectory", { importService.importCollectory(online) }) asJson(job.status()) } else { asJson([success: false, message: 'collectoryServicesUrl not configured']) @@ -134,8 +135,9 @@ class ImportController { */ // Documented in openapi.yml def importLayers(){ + boolean online = params.getBoolean('online', false) if(grailsApplication.config.layers.service){ - def job = execute("importLayers", "admin.button.importlayer", { importService.importLayers() }) + def job = execute("importLayers", "admin.button.importlayer", { importService.importLayers(online) }) asJson(job.status()) } else { asJson([success: false, message: 'layers.service not configured']) @@ -149,8 +151,9 @@ class ImportController { */ // Documented in openapi.yml def importLocalities(){ + boolean online = params.getBoolean('online', false) if(grailsApplication.config.layers.service && grailsApplication.config.layers.gazetteerId){ - def job = execute("importLocalities", "admin.button.importlocalities", { importService.importLocalities() }) + def job = execute("importLocalities", "admin.button.importlocalities", { importService.importLocalities(online) }) asJson(job.status()) } else { asJson([success: false, message: 'layers.services not configured or layers.gazetteerId not configured']) @@ -164,8 +167,9 @@ class ImportController { */ // Documented in openapi.yml def importRegions(){ + boolean online = params.getBoolean('online', false) if(grailsApplication.config.layers.service){ - def job = execute("importRegions", "admin.button.importregions", { importService.importRegions() }) + def job = execute("importRegions", "admin.button.importregions", { importService.importRegions(online) }) asJson(job.status()) } else { asJson([success: false, message: 'layers.service not configured']) diff --git a/grails-app/services/au/org/ala/bie/ImportService.groovy b/grails-app/services/au/org/ala/bie/ImportService.groovy index 6c7de417..56bb87f0 100644 --- a/grails-app/services/au/org/ala/bie/ImportService.groovy +++ b/grails-app/services/au/org/ala/bie/ImportService.groovy @@ -292,11 +292,14 @@ class ImportService implements GrailsConfigurationAware { * * @return */ - def importLayers() { + def importLayers(boolean online) { log "Starting layer import" + if (online) { + log "Layers may be temporarily unavailable" + } def layers = layerService.layers() def batch = [] - indexService.deleteFromIndex(IndexDocType.LAYER) + indexService.deleteFromIndex(IndexDocType.LAYER, online) layers.each { layer -> def doc = [:] doc["id"] = layer.name @@ -308,20 +311,23 @@ class ImportService implements GrailsConfigurationAware { log "Importing layer ${layer.displayname}" batch << doc } - indexService.indexBatch(batch) + indexService.indexBatch(batch, online) log "Finished indexing ${layers.size()} layers" log "Finsihed layer import" } - def importLocalities() { + def importLocalities(boolean online) { log "Starting localities import" - indexService.deleteFromIndex(IndexDocType.LOCALITY) + if (online) { + log "Localities may be temporarily unavailable" + } + indexService.deleteFromIndex(IndexDocType.LOCALITY, online) if (gazetteerId) { log("Starting indexing ${gazetteerId}") log("Getting metadata for layer: ${gazetteerId}") def layer = layerService.get(gazetteerId) log("Starting indexing ${layer.id} - ${layer.name} gazetteer layer") - importLayer(layer) + importLayer(layer, online) log("Finished indexing ${layer.id} - ${layer.name} gazetteer layer") } else { log("Skipping localities, no gazetteer layer ID configured") @@ -329,13 +335,16 @@ class ImportService implements GrailsConfigurationAware { log "Finished localities import" } - def importRegions() { + def importRegions(boolean online) { log "Starting regions import" - indexService.deleteFromIndex(IndexDocType.REGION) + if (online) { + log "Regions may be temporarily unavailable" + } + indexService.deleteFromIndex(IndexDocType.REGION, online) def layers = layerService.layers() layers.each { layer -> if (layer.type == "Contextual") { - importLayer(layer) + importLayer(layer, online) } } log "Finished indexing ${layers.size()} region layers" @@ -349,7 +358,7 @@ class ImportService implements GrailsConfigurationAware { * @param layer * @return */ - private def importLayer(layer) { + private def importLayer(layer, boolean online) { log("Loading regions from layer " + layer.name) def file = layerService.getRegions(layer.id) @@ -402,13 +411,13 @@ class ImportService implements GrailsConfigurationAware { batch << doc if (batch.size() >= BATCH_SIZE) { - indexService.indexBatch(batch) + indexService.indexBatch(batch, online) batch.clear() } } } if (batch) { - indexService.indexBatch(batch) + indexService.indexBatch(batch, online) batch.clear() } } @@ -454,7 +463,7 @@ class ImportService implements GrailsConfigurationAware { * * @return */ - def importCollectory() { + def importCollectory(online) { log "Starting collectory import" [ "dataResource": IndexDocType.DATARESOURCE, @@ -466,7 +475,10 @@ class ImportService implements GrailsConfigurationAware { def drLists = collectoryService.resources(entityType) log("About to import ${drLists.size()} ${entityType}") log("Clearing existing: ${entityType}") - indexService.deleteFromIndex(indexDocType) + if (online) { + log "Search may be temporarily unavailable for ${entityType}" + } + indexService.deleteFromIndex(indexDocType, online) log("Cleared") drLists.each { @@ -490,12 +502,12 @@ class ImportService implements GrailsConfigurationAware { entities << doc if (entities.size() > BUFFER_SIZE) { - indexService.indexBatch(entities) + indexService.indexBatch(entities, online) entities.clear() } } if (entities) { - indexService.indexBatch(entities) + indexService.indexBatch(entities, online) } log("Finished indexing ${drLists.size()} ${entityType}") } diff --git a/grails-app/views/import/collectory.gsp b/grails-app/views/import/collectory.gsp index 1b7afa46..87186a3a 100644 --- a/grails-app/views/import/collectory.gsp +++ b/grails-app/views/import/collectory.gsp @@ -17,10 +17,20 @@
- + +
+
+
+ + + function importCollectory(){ + loadInfo("${createLink(controller:'import', action:'importCollectory')}?online=" + $('#use-online').is(':checked')); + } + + \ No newline at end of file diff --git a/grails-app/views/import/layers.gsp b/grails-app/views/import/layers.gsp index 6e4abbec..70b74f5a 100644 --- a/grails-app/views/import/layers.gsp +++ b/grails-app/views/import/layers.gsp @@ -15,9 +15,18 @@

- + +
+
+
+ + + function importLayers(){ + loadInfo("${createLink(controller:'import', action:'importLayers')}?online=" + $('#use-online').is(':checked')); + } + \ No newline at end of file diff --git a/grails-app/views/import/localities.gsp b/grails-app/views/import/localities.gsp index 07919191..b2c66b6d 100644 --- a/grails-app/views/import/localities.gsp +++ b/grails-app/views/import/localities.gsp @@ -16,10 +16,19 @@
- + +
+
+
+ + + function importLocalities(){ + loadInfo("${createLink(controller:'import', action:'importLocalities')}?online=" + $('#use-online').is(':checked')); + } + \ No newline at end of file diff --git a/grails-app/views/import/regions.gsp b/grails-app/views/import/regions.gsp index b7c6de36..ba983d9c 100644 --- a/grails-app/views/import/regions.gsp +++ b/grails-app/views/import/regions.gsp @@ -22,10 +22,19 @@
- + +
+
+
+ + + function importRegions(){ + loadInfo("${createLink(controller:'import', action:'importRegions')}?online=" + $('#use-online').is(':checked')); + } + \ No newline at end of file diff --git a/src/main/resources/public/openapi.yml b/src/main/resources/public/openapi.yml index 37fff6b8..143b1bda 100644 --- a/src/main/resources/public/openapi.yml +++ b/src/main/resources/public/openapi.yml @@ -840,6 +840,8 @@ paths: Import data resources, collections etc. from the collectory into the offline index tags: - admin + parameters: + - $ref: '#/components/parameters/onlineParam' security: - cas: [] responses: @@ -855,6 +857,8 @@ paths: Imports all spatial layer descriptions from the spatial service into the offline index tags: - admin + parameters: + - $ref: '#/components/parameters/onlineParam' security: - cas: [] responses: @@ -870,6 +874,8 @@ paths: Imports all locality descriptions from the spatial service and gazetteer layer into the offline index tags: - admin + parameters: + - $ref: '#/components/parameters/onlineParam' security: - cas: [] responses: @@ -885,6 +891,8 @@ paths: Imports all region descriptions from the spatial service into the offline index tags: - admin + parameters: + - $ref: '#/components/parameters/onlineParam' security: - cas: [] responses: @@ -916,12 +924,7 @@ paths: tags: - admin parameters: - - name: online - in: query - description: Build using the online, rather than the offline, index - schema: - type: boolean - default: false + - $ref: '#/components/parameters/onlineParam' security: - cas: [] responses: @@ -938,12 +941,7 @@ paths: tags: - admin parameters: - - name: online - in: query - description: Build using the online, rather than the offline, index - schema: - type: boolean - default: false + - $ref: '#/components/parameters/onlineParam' security: - cas: [] responses: @@ -1014,12 +1012,7 @@ paths: tags: - admin parameters: - - name: online - in: query - description: Build using the online, rather than the offline, index - schema: - type: boolean - default: false + - $ref: '#/components/parameters/onlineParam' security: - cas: [] responses: @@ -1036,12 +1029,7 @@ paths: tags: - admin parameters: - - name: online - in: query - description: Scan the online, rather than the offline, index - schema: - type: boolean - default: false + - $ref: '#/components/parameters/onlineParam' security: - cas: [] responses: @@ -1058,12 +1046,7 @@ paths: tags: - admin parameters: - - name: online - in: query - description: Build using the online, rather than the offline, index - schema: - type: boolean - default: false + - $ref: '#/components/parameters/onlineParam' security: - cas: [] responses: @@ -1081,12 +1064,7 @@ paths: tags: - admin parameters: - - name: online - in: query - description: Build using the online, rather than the offline, index - schema: - type: boolean - default: false + - $ref: '#/components/parameters/onlineParam' security: - cas: [] responses: @@ -1105,12 +1083,7 @@ paths: tags: - admin parameters: - - name: online - in: query - description: Scan the online, rather than the offline, index - schema: - type: boolean - default: false + - $ref: '#/components/parameters/onlineParam' security: - cas: [] responses: @@ -1127,12 +1100,7 @@ paths: tags: - admin parameters: - - name: online - in: query - description: Scan the online, rather than the offline, index - schema: - type: boolean - default: false + - $ref: '#/components/parameters/onlineParam' security: - cas: [] responses: @@ -1149,12 +1117,7 @@ paths: tags: - admin parameters: - - name: online - in: query - description: Scan the online, rather than the offline, index - schema: - type: boolean - default: false + - $ref: '#/components/parameters/onlineParam' security: - cas: [] responses: @@ -2015,6 +1978,14 @@ components: type: integer default: 10 required: false + onlineParam: + name: online + description: Build using the online, rather than the offline, index + in: query + schema: + type: boolean + default: false + required: false tags: - name: search description: Search the BIE From fccd5c6d66a78719d46e1f487eba687edd7eb079 Mon Sep 17 00:00:00 2001 From: pal155 Date: Mon, 1 Feb 2021 15:26:27 +1100 Subject: [PATCH 10/15] Fix typo --- src/main/resources/public/openapi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/public/openapi.yml b/src/main/resources/public/openapi.yml index 143b1bda..5ae11f78 100644 --- a/src/main/resources/public/openapi.yml +++ b/src/main/resources/public/openapi.yml @@ -920,7 +920,7 @@ paths: get: summary: Import workpress pages description: > - Imports wordpress pages into the offline (defauult) or online index + Imports wordpress pages into the offline (default) or online index tags: - admin parameters: From 2a5a8213bdeef172a673dcb901b77d82dce8fbe2 Mon Sep 17 00:00:00 2001 From: pal155 Date: Thu, 4 Feb 2021 09:09:58 +1100 Subject: [PATCH 11/15] Non-null online for full import --- .../au/org/ala/bie/ImportService.groovy | 24 +++++++++---------- .../au/org/ala/bie/IndexService.groovy | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/grails-app/services/au/org/ala/bie/ImportService.groovy b/grails-app/services/au/org/ala/bie/ImportService.groovy index 56bb87f0..072515fb 100644 --- a/grails-app/services/au/org/ala/bie/ImportService.groovy +++ b/grails-app/services/au/org/ala/bie/ImportService.groovy @@ -216,7 +216,7 @@ class ImportService implements GrailsConfigurationAware { try { switch (step) { case 'collectory': - importCollectory() + importCollectory(false) break case 'conservation-lists': importConservationSpeciesLists() @@ -232,19 +232,19 @@ class ImportService implements GrailsConfigurationAware { loadImages(false) break case 'layers': - importLayers() + importLayers(false) break case 'link-identifiers': buildLinkIdentifiers(false) break case 'localities': - importLocalities() + importLocalities(false) break case 'occurrences': importOccurrenceData(false) break case 'regions': - importRegions() + importRegions(false) break case 'suggest-index': buildSuggestIndex(false) @@ -259,10 +259,10 @@ class ImportService implements GrailsConfigurationAware { buildWeights(false) break case 'wordpress': - importWordPressPages() + importWordPressPages(false) break case 'knowledgebase': - importKnowledgeBasePages() + importKnowledgeBasePages(false) break default: log("Unknown step ${step}") @@ -426,7 +426,7 @@ class ImportService implements GrailsConfigurationAware { def importHabitats() { def batch = [] - indexService.deleteFromIndex(IndexDocType.HABITAT) + indexService.deleteFromIndex(IndexDocType.HABITAT, false) //read the DwC metadata Archive archive = ArchiveFactory.openArchive(new File("/data/habitat/")); @@ -463,7 +463,7 @@ class ImportService implements GrailsConfigurationAware { * * @return */ - def importCollectory(online) { + def importCollectory(boolean online) { log "Starting collectory import" [ "dataResource": IndexDocType.DATARESOURCE, @@ -1030,10 +1030,10 @@ class ImportService implements GrailsConfigurationAware { def clearTaxaIndex() { log("Deleting existing taxon entries in index...") - indexService.deleteFromIndex(IndexDocType.TAXON) - indexService.deleteFromIndex(IndexDocType.COMMON) - indexService.deleteFromIndex(IndexDocType.IDENTIFIER) - indexService.deleteFromIndex(IndexDocType.TAXONVARIANT) + indexService.deleteFromIndex(IndexDocType.TAXON, false) + indexService.deleteFromIndex(IndexDocType.COMMON, false) + indexService.deleteFromIndex(IndexDocType.IDENTIFIER, false) + indexService.deleteFromIndex(IndexDocType.TAXONVARIANT, false) log("Cleared.") } diff --git a/grails-app/services/au/org/ala/bie/IndexService.groovy b/grails-app/services/au/org/ala/bie/IndexService.groovy index fda37c73..72e932e2 100644 --- a/grails-app/services/au/org/ala/bie/IndexService.groovy +++ b/grails-app/services/au/org/ala/bie/IndexService.groovy @@ -63,7 +63,7 @@ class IndexService implements GrailsConfigurationAware { * @param docType * @return */ - def deleteFromIndex(IndexDocType docType, boolean online = false) { + def deleteFromIndex(IndexDocType docType, boolean online) { log.info("Deleting from index: " + docType.name() + "....") def client = online ? updatingLiveSolrClient : offlineSolrClient client.deleteByQuery("idxtype:" + docType.name()) From 8910e5092734613c70039f75c81c21a6e4403c5f Mon Sep 17 00:00:00 2001 From: pal155 Date: Thu, 11 Feb 2021 16:06:48 +1100 Subject: [PATCH 12/15] Deal with extra parameters in childConcepts API call. Improve swagger doco --- .../au/org/ala/bie/SearchController.groovy | 15 +++++++----- src/main/resources/public/openapi.yml | 23 ++++++++----------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/grails-app/controllers/au/org/ala/bie/SearchController.groovy b/grails-app/controllers/au/org/ala/bie/SearchController.groovy index 0f639a62..808511df 100755 --- a/grails-app/controllers/au/org/ala/bie/SearchController.groovy +++ b/grails-app/controllers/au/org/ala/bie/SearchController.groovy @@ -63,10 +63,11 @@ class SearchController implements GrailsConfigurationAware { */ // Documented in openapi.yml def imageLinkSearch() { - def showNoImage = params.containsKey("showNoImage") ? params.boolean("showNoImage") : true + def showNoImage = params.boolean('showNoImage', true) def guid = regularise(params.id) def locales = [request.locale, defaultLocale] - def url = searchService.imageLinkSearch(guid, params.imageType, params.qc, locales) + def imageType = params.imageType + def url = searchService.imageLinkSearch(guid, imageType, params.qc, locales) if (!url && showNoImage) { url = resource(dir: "images", file: "noImage85.jpg", absolute: true) @@ -86,13 +87,15 @@ class SearchController implements GrailsConfigurationAware { // Documented in openapi.yml def childConcepts(){ def taxonID = params.id - if(!taxonID){ + if(!taxonID) { response.sendError(400, "Please provide a GUID") return null } - def within = params.within && params.within.isInteger() ? params.within as Integer : 2000 - def unranked = params.unranked ? params.unranked.toBoolean() : true - render (searchService.getChildConcepts(regularise(taxonID), request.queryString, within, unranked) as JSON) + def within = params.int('within', 2000) + def unranked = params.boolean('unranked', true) + ['within', 'unranked', 'controller', 'action', 'id'].each {params.remove(it) } + def extra = params.toQueryString().replaceFirst('^\\?', '') + render (searchService.getChildConcepts(regularise(taxonID), extra, within, unranked) as JSON) } // Documented in openapi.yml diff --git a/src/main/resources/public/openapi.yml b/src/main/resources/public/openapi.yml index 5ae11f78..535fd28d 100644 --- a/src/main/resources/public/openapi.yml +++ b/src/main/resources/public/openapi.yml @@ -356,7 +356,7 @@ paths: /species/image/{imageType}/{id}: get: summary: Get a suitably sized representative image for a taxon - description: Return a URL to the representative image + description: Return a redirect to a URL with representative image. Note that testing this using the swagger UI will, most likey cause a CORS error. Copying and pasting the request URL should succeeed. tags: - lookup parameters: @@ -387,16 +387,13 @@ paths: responses: 404: description: If there is no image for the supplied id and showNoImage=false - 200: - content: - application/json: + 302: + headers: + Location: + description: The location of the image schema: - type: object - properties: - url: - type: string - format: url - description: The URL of the requested image + type: string + format: url /species/shortProfile/{id}: get: summary: Get a short description of a taxon @@ -1942,10 +1939,10 @@ components: in: path description: > The guid of a specific taxon, data resource, layer etc. - Since guids can be URLs but can also be interpolated into paths, a http:// prefix may be converted into http:/ by + Since guids can be URLs but can also be interpolated into paths, a http:// or https:// prefix may be converted into http:/ or https:/ by the tomcat container. - A supplied guid of, eg. http:/id.biodiversity.org.au/node/apni/50587232 will be converted into - http://id.biodiversity.org.au/node/apni/50587232 + A supplied guid of, eg. https:/id.biodiversity.org.au/node/apni/50587232 will be converted into + https://id.biodiversity.org.au/node/apni/50587232 required: true schema: $ref: '#/components/schemas/GUID' From 71cb5840b65be51f4b57d38a013ec675bd16607d Mon Sep 17 00:00:00 2001 From: pal155 Date: Wed, 17 Feb 2021 09:34:23 +1100 Subject: [PATCH 13/15] Bump development version --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 55c06bce..21c96f52 100644 --- a/build.gradle +++ b/build.gradle @@ -11,7 +11,7 @@ buildscript { } } -version "1.4.11-SNAPSHOT" +version "1.4.12-SNAPSHOT" group "au.org.ala" apply plugin:"eclipse" From da3e8cc636ffd770f4c21080e44f269435a5651f Mon Sep 17 00:00:00 2001 From: pal155 Date: Wed, 17 Feb 2021 14:39:18 +1100 Subject: [PATCH 14/15] Allow ranks to be received without authentication --- .../controllers/au/org/ala/bie/ImportController.groovy | 6 ------ grails-app/controllers/au/org/ala/bie/MiscController.groovy | 5 +++++ grails-app/controllers/bie/index/UrlMappings.groovy | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/grails-app/controllers/au/org/ala/bie/ImportController.groovy b/grails-app/controllers/au/org/ala/bie/ImportController.groovy index 787a2f34..497595bd 100755 --- a/grails-app/controllers/au/org/ala/bie/ImportController.groovy +++ b/grails-app/controllers/au/org/ala/bie/ImportController.groovy @@ -306,12 +306,6 @@ class ImportController { asJson (job.status()) } - // Documented in openapi.yml - def ranks() { - asJson(importService.ranks()) - } - - private def asJson = { model -> response.setContentType("application/json;charset=UTF-8") render (model as JSON) diff --git a/grails-app/controllers/au/org/ala/bie/MiscController.groovy b/grails-app/controllers/au/org/ala/bie/MiscController.groovy index 827ee27e..44bb6d79 100644 --- a/grails-app/controllers/au/org/ala/bie/MiscController.groovy +++ b/grails-app/controllers/au/org/ala/bie/MiscController.groovy @@ -28,6 +28,11 @@ class MiscController { return } + // Documented in openapi.yml + def ranks() { + render importService.ranks() as JSON + } + def indexFields() { def fields = indexService.getIndexFieldDetails(null) diff --git a/grails-app/controllers/bie/index/UrlMappings.groovy b/grails-app/controllers/bie/index/UrlMappings.groovy index 198c426a..f66e648d 100755 --- a/grails-app/controllers/bie/index/UrlMappings.groovy +++ b/grails-app/controllers/bie/index/UrlMappings.groovy @@ -35,7 +35,7 @@ class UrlMappings { "/habitat/ids/$guid**"(controller: "search", action: "getHabitatIDs") "/habitat/$guid**"(controller: "search", action: "getHabitat") - "/ranks"(controller: "import", action: "ranks") + "/ranks"(controller: "misc", action: "ranks") "/admin"(controller: "admin") "/admin/"(controller: "admin") From f18d8542a285bd9d19e20ddbadc78ce702020316 Mon Sep 17 00:00:00 2001 From: pal155 Date: Fri, 19 Feb 2021 10:42:25 +1100 Subject: [PATCH 15/15] Update travis link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8ef1d3c6..0272a7af 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# bie-index [![Build Status](https://travis-ci.org/AtlasOfLivingAustralia/bie-index.svg?branch=master)](https://travis-ci.org/AtlasOfLivingAustralia/bie-index) +# bie-index [![Build Status](https://travis-ci.com/AtlasOfLivingAustralia/bie-index.svg?branch=master)](https://travis-ci.com/AtlasOfLivingAustralia/bie-index) bie-index is a grails web application that indexes taxonomic content in DwC-A and provides search web services for this content. This includes: