Prepare release 1.9.11 (#186)

* Set next snapshot version * Skip resources with empty eml * Fail on validation error saving resources * #172 cors * #172 cors, remove cors setting * #173 migrate to Ubuntu 20 * Increase maxSize in subcollections. Fix for #181 (#182) * #184 Update usage statistics call to logger-service * GenomicDNA and EnvironmentalDNA content types (#178) Related to AtlasOfLivingAustralia/la-pipelines#397 * MIssing comma in list * Data repatriation tools (#185) * Work for #183 * Response to review feedback * missing dependency runtime 'org.apache.httpcomponents:httpclient:jar:4.4.1' * dependency fix Co-authored-by: Patricia Koh <[email protected]> Co-authored-by: vjrj <[email protected]> Co-authored-by: Rita <[email protected]> Co-authored-by: vjrj <[email protected]>
AtlasOfLivingAustralia · Jul 29, 2021 · 2da0df1 · 2da0df1
1 parent 3e5eff2
commit 2da0df1
Show file tree

Hide file tree

Showing 95 changed files with 1,221 additions and 549 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,5 @@
 *.iml
 target/*
 .idea/*
-.slcache/
+.slcache/
+classes/*
diff --git a/CollectoryGrailsPlugin.groovy b/CollectoryGrailsPlugin.groovy
@@ -9,7 +9,7 @@ class CollectoryGrailsPlugin {
     def authenticateService
 
     // the plugin version
-    def version = "1.9.10"
+    def version = "1.9.11-SNAPSHOT"
     // the version or versions of Grails the plugin is designed for
     def grailsVersion = "2.5 > *"
     // resources that are excluded from plugin packaging

diff --git a/grails-app/conf/BuildConfig.groovy b/grails-app/conf/BuildConfig.groovy
@@ -25,11 +25,10 @@ grails.project.dependency.resolution = {
     }
 
     dependencies {
-        //runtime 'mysql:mysql-connector-java:5.1.5' // Needed if you have older versions of mysql installed (5.6)
-        runtime 'mysql:mysql-connector-java:5.1.42'
+        runtime 'mysql:mysql-connector-java:8.0.25'
         runtime 'net.sf.opencsv:opencsv:2.3'
 	    runtime 'org.apache.ant:ant:1.10.1'
-        runtime 'commons-httpclient:commons-httpclient:3.1'
+        runtime 'org.apache.httpcomponents:httpclient:jar:4.4.1'
         runtime 'org.aspectj:aspectjweaver:1.6.6'
         compile "com.fasterxml.jackson.core:jackson-databind:2.7.0"
 	compile( "commons-validator:commons-validator:1.4.1" ) {
@@ -50,7 +49,6 @@ grails.project.dependency.resolution = {
         runtime ":audit-logging:1.1.1"
         runtime ":cache-headers:1.1.7"
         runtime ":rest:0.8"
-        //runtime ":richui:0.8"
         runtime ":tiny-mce:3.4.9"
         runtime ":cors:1.1.8"
         runtime ":ala-charts-plugin:1.3.3"

diff --git a/grails-app/controllers/au/org/ala/collectory/DataController.groovy b/grails-app/controllers/au/org/ala/collectory/DataController.groovy
@@ -35,7 +35,7 @@ class DataController {
                 }
             } else {
 
-                if(params.entity){
+                if (params.entity){
                     params.pg = ProviderGroup._get(params.uid, params.entity)
                 }
 
@@ -178,8 +178,10 @@ class DataController {
 
     def checkApiKey = {
         def apiKey = {
-            if(params.api_key){
+            if (params.api_key) {
                 params.api_key
+            } else if (params.apiKey){
+                params.apiKey
             } else {
                 request.getHeader("Authorization")
             }
@@ -357,7 +359,6 @@ class DataController {
                 def detail = params.summary ? summary : brief
                 def summaries = list.collect(detail)
                 def eTag = summaries.toString().encodeAsMD5()
-//                response.setCharacterEncoding("UTF-8")
                 response.setContentType("application/json")
                 renderAsJson summaries, last, eTag
             }

diff --git a/grails-app/controllers/au/org/ala/collectory/GbifController.groovy b/grails-app/controllers/au/org/ala/collectory/GbifController.groovy
@@ -1,12 +1,18 @@
 package au.org.ala.collectory
 
+import au.org.ala.collectory.resources.gbif.GbifRepatDataSourceAdapter
+import grails.converters.JSON
 import groovy.json.JsonSlurper
 
 class GbifController {
+    static final API_KEY_COOKIE = "ALA-API-Key"
 
+    def collectoryAuthService
     def gbifRegistryService
+    def gbifService
     def authService
     def grailsApplication
+    def externalDataService
 
     def healthCheck() {
         gbifRegistryService.generateSyncBreakdown()
@@ -115,11 +121,78 @@ class GbifController {
         def results = [:]
         def errorMessage = ""
 
-        if(authService.userInRole(grailsApplication.config.gbifRegistrationRole)){
+        if (authService.userInRole(grailsApplication.config.gbifRegistrationRole)){
             results = gbifRegistryService.syncAllResources()
         } else {
             errorMessage = "User does not have sufficient privileges to perform this."
         }
         [results:results, errorMessage:errorMessage]
     }
+
+    def scan(){
+
+        def apiKey = request.cookies.find { cookie -> cookie.name == API_KEY_COOKIE }
+        if (!apiKey){
+            // look in the standard place - http apiKey param
+            apiKey = params.apiKey
+        }
+        def keyCheck =  collectoryAuthService.checkApiKey(apiKey.value)
+
+        if (!keyCheck || !keyCheck.valid){
+            response.sendError(401)
+            return
+        }
+
+        DataProvider dataProvider = DataProvider.findByUid(params.uid)
+        if (!dataProvider){
+            response.sendError(404)
+            return
+        }
+
+        def resources = dataProvider.resources
+        def output = []
+        resources.each { DataResource resource ->
+            Date lastUpdated = gbifService.getGbifDatasetLastUpdated(resource.guid)
+            //get last updated data
+            output << [uid:resource.uid,
+                       name: resource.name,
+                       lastUpdated: resource.lastUpdated,
+                       guid: resource.guid,
+                       country: resource.repatriationCountry,
+                       pubDate: lastUpdated,
+                       inSync:  !(lastUpdated > resource.lastUpdated)
+            ]
+        }
+
+        DataSourceConfiguration configuration = new DataSourceConfiguration()
+        configuration.adaptorClass = GbifRepatDataSourceAdapter.class
+        configuration.endpoint = new URL(grailsApplication.config.gbifApiUrl)
+        configuration.username = grailsApplication.config.gbifApiUser
+        configuration.password = grailsApplication.config.gbifApiPassword
+
+        def externalResourceBeans = []
+
+        output.each { res ->
+            if (!res.inSync && res.guid){
+                res.status = "RELOADING"
+                externalResourceBeans << new ExternalResourceBean(
+                        uid: res.uid, guid: res.guid, name: res.name, country: res.country, updateMetadata:true, updateConnection:true)
+            } else {
+                res.status = "IN_SYNC"
+            }
+        }
+        configuration.resources = externalResourceBeans
+
+        def loadGuid = UUID.randomUUID().toString()
+
+        log.info("Reloading process ID " + loadGuid)
+        externalDataService.updateFromExternalSources(configuration, loadGuid)
+
+        def fullOutput =
+                [loadGuid: loadGuid,
+                 trackingUrl: createLink(controller:"manage", action:"externalLoadStatus", params: [loadGuid: loadGuid]),
+                 resources: output
+                ]
+        render(fullOutput as JSON)
+    }
 }
diff --git a/grails-app/controllers/au/org/ala/collectory/IptController.groovy b/grails-app/controllers/au/org/ala/collectory/IptController.groovy
@@ -42,6 +42,10 @@ class IptController {
         def isShareableWithGBIF = params.isShareableWithGBIF ? params.isShareableWithGBIF.toBoolean(): true
         def provider = ProviderGroup._get(params.uid)
         def apiKey = request.cookies.find { cookie -> cookie.name == API_KEY_COOKIE }
+        if (!apiKey){
+            // look in the standard place - http apiKey param
+            apiKey = params.apiKey
+        }
         def keyCheck = apiKey ? collectoryAuthService.checkApiKey(apiKey.value) : null
         def username = keyCheck?.userEmail ?: collectoryAuthService.username()
         def admin = keyCheck?.valid || collectoryAuthService.userInRole(ProviderGroup.ROLE_ADMIN)

diff --git a/grails-app/controllers/au/org/ala/collectory/ManageController.groovy b/grails-app/controllers/au/org/ala/collectory/ManageController.groovy
@@ -3,6 +3,7 @@ package au.org.ala.collectory
 import au.org.ala.audit.AuditLogEvent
 import au.org.ala.collectory.resources.DataSourceLoad
 import au.org.ala.collectory.resources.gbif.GbifDataSourceAdapter
+import au.org.ala.collectory.resources.gbif.GbifRepatDataSourceAdapter
 
 class ManageController {
 
@@ -26,6 +27,39 @@ class ManageController {
         }
     }
 
+    /**
+     * Renders the view that allows a user to load all the gbif resources for a country
+     */
+    def repatriate = {
+        DataSourceConfiguration configuration = new DataSourceConfiguration(
+                guid: UUID.randomUUID().toString(),
+                name: '',
+                description: '',
+                adaptorClass: GbifRepatDataSourceAdapter.class,
+                endpoint: new URL(grailsApplication.config.gbifApiUrl),
+                uniqueKeyTerm: grailsApplication.config.gbifUniqueKeyTerm,
+                username: '',
+                password: '',
+                country: Locale.default.getCountry(),
+                recordType: 'OCCURRENCE',
+                defaultDatasetValues: [:],
+                keyTerms: [],
+                resources: [],
+                countries: gbifService.getCountryMap().keySet()
+        )
+        def adaptor = configuration.createAdaptor()
+        render(view: "repatriate",
+                model: [
+                        repatriate: true,
+                        configuration: configuration,
+                        countryMap: gbifService.getCountryMap(),
+                        datasetTypeMap: adaptor.datasetTypeMap,
+                        adaptors: externalDataService.REPAT_ADAPTORMAP,
+                        dataProviders: DataProvider.all.sort { it.name }
+                ]
+        )
+    }
+
     /**
      * Renders the view that allows a user to load all the gbif resources for a country
      */
@@ -35,7 +69,8 @@ class ManageController {
                 name: '',
                 description: '',
                 adaptorClass: GbifDataSourceAdapter.class,
-                endpoint: new URL(grailsApplication.config.gbifApiUrl + '/'),
+                endpoint: new URL(grailsApplication.config.gbifApiUrl),
+                uniqueKeyTerm: grailsApplication.config.gbifUniqueKeyTerm,
                 username: '',
                 password: '',
                 country: Locale.default.getCountry(),
@@ -80,7 +115,30 @@ class ManageController {
     }
 
     /**
-     * Update from an externbal source
+     * Search for resources that may be loaded from an external source
+     */
+    def searchForRepatResources() {
+        log.debug "Searching for resources from external source: ${params}"
+        DataSourceConfiguration configuration = new DataSourceConfiguration(params)
+        def dataResources = DataResource.all.findAll({ dr -> dr.resourceType == 'records' }).sort({ it.name })
+        def resources = externalDataService.searchForDatasets(configuration)
+        configuration.resources = resources
+        def dataProvider = null
+        if (configuration.dataProviderUid){
+            dataProvider = DataProvider.findByUid(configuration.dataProviderUid)
+        }
+        render(view: 'repatriateReview',
+                model: [
+                        loadGuid: UUID.randomUUID().toString(),
+                        dataResources: dataResources,
+                        dataProvider: dataProvider,
+                        configuration: configuration
+                ]
+        )
+    }
+
+    /**
+     * Update from an external source
      * <p>
      * The web pade
      */
@@ -96,12 +154,12 @@ class ManageController {
      * @return
      */
     def loadDataset() {
+
         log.debug("Loading resources from GBIF: " + params)
-        if (params.guid && params.gbifUsername && params.gbifPassword) {
-            gbifService.getGbifDataset(
+        if (params.guid) {
+            gbifService.downloadGbifDataset(
                     params.guid,
-                    params.gbifUsername,
-                    params.gbifPassword)
+                    params.repatriationCountry)
             redirect(action: 'gbifDatasetLoadStatus', model: ['datasetKey': params.guid], params: ['datasetKey': params.guid])
         }
     }
@@ -113,7 +171,7 @@ class ManageController {
      * @return
      */
     def gbifDatasetLoadStatus(){
-        log.debug('key->'+params.datasetKey)
+        log.debug('key->' + params.datasetKey)
         def gbifSummary = gbifService.getDatasetKeyStatusInfoFor(params.datasetKey)
         log.debug(gbifSummary)
         [gbifSummary:gbifSummary,'datasetKey':params.datasetKey]
@@ -126,7 +184,7 @@ class ManageController {
     def gbifDatasetDownload() {
         log.debug('Dataset id ' + params.id)
         def dr = DataResource.findByUid(params.id)
-        render(view: "gbifDatasetDownload", model: ['uid': dr.uid, 'guid' : dr.guid])
+        render(view: "gbifDatasetDownload", model: ['uid': dr.uid, 'guid' : dr.guid, 'dr' : dr])
     }
 
     /**

diff --git a/grails-app/controllers/au/org/ala/collectory/PublicController.groovy b/grails-app/controllers/au/org/ala/collectory/PublicController.groovy
@@ -526,7 +526,7 @@ class PublicController {
 
     def resources = {
         cache shared:true, validFor: 3600*24
-        def drs = DataResource.findAllByStatusNotEqual('declined',[sort:'name']).collect {
+        def drs = DataResource.findAllByStatusNotEqual('declined', [sort:'name']).collect {
         //def drs = DataResource.list([sort:'name']).collect {
             def pdesc = it.pubDescription ? cl.formattedText(dummy:'1',limit(it.pubDescription,1000)) : ""
             def tdesc = it.techDescription ? cl.formattedText(dummy:'1',limit(it.techDescription,1000)) : ""
@@ -541,27 +541,23 @@ class PublicController {
         render drs as JSON
     }
 
+    def condensed = {
+        cache shared:true, validFor: 3600*24
+        def drs = DataResource.findAllByStatusNotEqual('declined',  [sort:'name'] ).collect {
+            //def drs = DataResource.list([sort:'name']).collect {
+            def inst = it.institution
+            def instName = (inst && inst.name.size() > 36 && inst.acronym) ? inst.acronym : inst?.name
+            [name: it.name, resourceType: it.resourceType, licenseType: it.licenseType,
+             uid: it.uid, status: it.status, websiteUrl: it.websiteUrl, contentTypes: it.contentTypes,
+             institution: instName]
+        }
+        render drs as JSON
+    }
+
     def downloadDataSets = {
         def filters = params.filters ? JSON.parse(params.filters) : [];
         def uids = params.uids.tokenize(',')
         def drs = DataResource.list([sort:'name'])
-        /*if (filters) {
-            drs = drs.findAll { dr ->
-                // check each filter
-                for (filter in filters) {
-                    if (filter.value == 'noValue') {
-                        if (dr[filter.name]) {
-                            return false
-                        }
-                    }
-                    // TODO: handle filters for properties with multiple values
-                    else if (dr[filter.name] != filter.value) {
-                        return false
-                    }
-                }
-                return true
-            }
-        }*/
         drs = drs.findAll { dr ->
             if (dr.status == 'declined') {
                 return false
@@ -703,17 +699,6 @@ class PublicController {
         return str
     }
 
-    private boolean matchNetwork(pg, filterString) {
-        def filters = filterString.tokenize(",")
-        for (int i = 0; i < filters.size(); i++) {
-            //println "Checking filter ${filters[i]} against network membership ${pg?.networkMembership}"
-            if (pg?.isMemberOf(filters[i])) {
-                return true;
-            }
-        }
-        return false
-    }
-
     private findCollection(id) {
         if (!id) { return null }
         // try lsid

diff --git a/grails-app/domain/au/org/ala/collectory/Collection.groovy b/grails-app/domain/au/org/ala/collectory/Collection.groovy
@@ -130,7 +130,7 @@ class Collection extends ProviderGroup implements Serializable {
                 return ok
             })
         scientificNames(nullable:true, maxSize:2048)
-        subCollections(nullable:true, maxSize:4096)
+        subCollections(nullable:true, maxSize:8192)
         providerMap(nullable:true)
         institution(nullable:true)
     }