Merge pull request #3154 from CSCfi/plugins

Plugins

CHANGELOG.md

@@ -11,6 +11,7 @@ Changes since v2.32
 ### Additions
 - PDF output can be customized a little. This enables changing the font, which should fix missing diacritics. (#3158)
 - Workflows can now disable commands dynamically by application state, user role, or both. Rules can be set in workflow administration page under disable commands field. (#3131)
+- Plugins can be written to extend REMS functionality in select extension points. So far there is support for plugins written in Clojure embedded in Markdown files. See `docs/plugins.md` for details. (#3133)
 
 ### Fixes
 - Label and header fields have URLs made into links. (#3155)

dev-config.edn

@@ -17,7 +17,8 @@
  ;; some attributes that google oauth returns:
  :oidc-extra-attributes [{:attribute "nickname" :name {:en "Nickname" :fi "Lempinimi"}}
                          {:attribute "picture"}
-                         {:attribute "organizations"}]
+                         {:attribute "organizations"}
+                         {:attribute "groups" :name {:en "Groups" :fi "Ryhmät" :sv "Grupper"}}]
  :languages [:en :fi :sv]
  :public-url "http://localhost:3000/"
  :extra-pages [{:id "about"
@@ -83,4 +84,26 @@
  :enable-duo true
  :enable-catalogue-tree true
  :enable-save-compaction true
- :enable-autosave true}
+ :enable-autosave true
+
+ ;; let's use plugins in dev
+ :oidc-require-name false
+ :oidc-require-email false
+ :plugins [{:id :plugin/AARC-G069-group-split
+            :filename "resources/plugins/AARC-G069-group-split.md"
+            :attribute-name "eduperson_entitlement"
+            :trusted-authorities ["perun.aai.lifescience-ri.eu"]}
+
+           {:id :plugin/validate-attributes
+            :filename "resources/plugins/validate-attributes.md"
+            :required-attributes [{:attribute-name "name" :error-key :t.login.errors/name}
+                                  {:attribute-name "email" :error-key :t.login.errors/email}]}
+
+           {:id :plugin/validate-group-membership ; NB: not in use at the moment
+            :filename "resources/plugins/validate-group-membership.md"
+            :attribute-name "groups"
+            :valid-groups ["VO1"]
+            :error-key :t.login.errors/group}]
+
+ :extension-points {:extension-point/transform-user-data [:plugin/AARC-G069-group-split]
+                    :extension-point/validate-user-data [:plugin/validate-attributes]}}

docs/hooks.md

@@ -1,5 +1,7 @@
 # Hooks
 
+See also [plugins](./plugins.md).
+
 ## Extra scripts
 
 You can configure REMS to include extra script files by defining configuration such as:

docs/plugins.md

@@ -0,0 +1,87 @@
+# Plugins
+
+REMS can be extended with plugins in certain extension points.
+
+The plugins are loaded dynamically from external code found in the specified file.
+
+There should be a function defined in the plugin, that is called at the right time. The name of the function
+depends on the type of the extension point.
+
+All the functions receive `config` where the plugin's configuration is (from the config file). Also context
+specific `data` will be passed.
+
+Certain types of libraries have been exposed to plugins. Please post an issue if you want more added.
+
+Examples of plugins can be found in the `resources/plugins` directory.
+
+## Types of plugins
+
+### Transform
+
+Take `data` and do any kind of transformations to it and return the new `data`.
+
+Return the original data if nothing should be done.
+
+```clj
+(defn transform [config data]
+  ...)
+```
+
+### Process
+
+Processes the passed `data` for side-effects, such as integration to another server using HTTP requests.
+
+Returns the errors so an empty sequence (`nil` or `[]` for example) should be returned if everything was good.
+Any errors will prevent the possible next process from running.
+
+In the case of a failure in processing of the same `data`, the process will be retried again, so the implementation should be idempotent. A retry can also happen for a successful process, if a processing plugin configured after this plugin fails.
+
+```clj
+(defn process [config data]
+  ...)
+```
+
+### Validate
+
+Validates the passed `data`.
+
+Returns the errors so an empty sequence (`nil` or `[]` for example) should be returned if everything was good.
+Any errors will prevent the possible next validation from running, and generally the action from happening (e.g. logging in).
+
+```clj
+(defn validate [config data]
+  ...)
+```
+
+## Extension points
+
+Next are all the current extension points and the function they expect to find in the plugin.
+
+### `:extension-point/transform-user-data`
+
+After logging in, after opening the OIDC token and potentially fetching the user info, allow transforming that data further. For example, a complex field can be parsed and the result stored in new fields.
+
+Expects `transform` function.
+
+See [AARC-G069-group-split.md](../resources/plugins/AARC-G069-group-split.md)
+
+### `:extension-point/validate-user-data`
+
+After logging in, after the user data is finalized, allow validating it to prevent invalid users from logging in.
+
+Expects `validate` function.
+
+The first returned error will be shown to the user on an error page. It can have the keys:
+- `:key` The translation key of the error message such as `:t.login.errors/invalid-user` (possibly from extra translations provided).
+- `:args` Additional arguments for the message translation (`%1`,`%2`, ...) if any. These must be translation keys too.
+
+See [validate-attributes.md](../resources/plugins/validate-attributes.md)
+
+### `:extension-point/process-entitlements`
+
+After entitlements have been updated, the new entitlements can be processed, and for example updated to
+another system.
+
+Expects `process` function.
+
+See [LS-AAI-GA4GH-push.md](../resources/plugins/LS-AAI-GA4GH-push.md)

example-theme/extra-translations/en.edn

@@ -2,4 +2,5 @@
  {:footer "CSC – Overridden in Extra Translations"
   ;; the extra %1 and the :unused-key are for rems.test-locales
   :create-license {:license-text "Text %1"}
-  :unused-key "Unused"}}
+  :unused-key "Unused"
+  :login {:errors {:group "group"}}}}

project.clj

@@ -61,7 +61,9 @@
                  [ring/ring-defaults "0.3.4"]
                  [ring/ring-devel "1.9.6"]
                  [ring/ring-servlet "1.9.6"]
-                 [nano-id "1.0.0"]]
+                 [nano-id "1.0.0"]
+                 [org.babashka/sci "0.7.39"]
+                 [com.nextjournal/beholder "1.0.2"]]
 
   :min-lein-version "2.9.8"
 
@@ -83,6 +85,7 @@
   :antq {}
 
   :cljfmt {:paths ["project.clj" "src/clj" "src/cljc" "src/cljs" "test/clj" "test/cljc" "test/cljs"] ; need explicit paths to include cljs
+           :indents {delay [[:inner 0]]}
            :remove-consecutive-blank-lines? false} ; too many changes for now, probably not desirable
 
   :clean-targets ["target"]

resources/config-defaults.edn

@@ -446,4 +446,23 @@
  ;; :pdf-metadata {:font {:encoding :unicode
  ;;                       :ttf-name "custom-font-file.ttf"}}
  ;;
- :pdf-metadata {}}
+ :pdf-metadata {}
+
+ ;; REMS can be extended using plugins:
+ ;;
+ ;; Define the plugins with `:plugins` and then
+ ;; attach them to the extension points using `:extension-points`.
+ ;;
+ ;; Example:
+ ;; :plugins [{:id :plugin/group-split
+ ;;            :filename "plugins/group-split.md"
+ ;;            :attribute-name "user_groups"
+ ;;            :split-with ";"}
+ ;;
+ ;; :extension-points {:extension-point/transform-user-data [:plugin/group-split] ; should match the previous id
+ ;;                    :extension-point/validate-user-data [...
+ ;;                                                         ]}
+ ;;
+ ;; See docs/plugins.md for more information.
+ :plugins nil ; by default no plugins
+ :extension-points nil} ; by default no plugins

resources/plugins/AARC-G069-group-split.md

@@ -0,0 +1,72 @@
+# AARC-G069 group parse
+
+A plugin to parse group attributes according to AARC-G069.
+
+The format comes from AARC-G069 as so:
+
+    <NAMESPACE>:group:<GROUP>[:<SUBGROUP>*][:role=<ROLE>][#<AUTHORITY>]
+
+The OIDC attribute name that is parsed is configured in the `:attribute-name` config. With these example attibutes:
+
+    urn:geant:lifescience-ri.eu:group:example-vo.lifescience-ri.eu#aai.lifescience-ri.eu
+    urn:geant:elixir-europe.org:group:elixir:ELIXIR%20AAI:staff#perun.elixir-czech.cz
+
+The <NAMESPACE> is:
+
+    urn:geant:lifescience-ri.eu
+    urn:geant:elixir-europe.org
+
+The <AUTHORITY> comes after `#`:
+
+    aai.lifescience-ri.eu
+    perun.elixir-czech.cz
+
+The part `:group:` is just a splitter like the `#` is.
+
+The middle part are the <GROUP> (and <SUBGROUP>s) of the person:
+
+    example-vo.lifescience-ri.eu (only group)
+    elixir:ELIXIR%20AAI:staff (group "elixir", subgroup "ELIXIR AAI", subgroup "staff")
+
+The <ROLE> is not present in the example but it is currently handled as just another subgroup. So
+a `role=R` would literally be just an additional `role=R` group.
+
+For a group to be valid the authority must declared in config `:trusted-authorities`.
+
+```clj
+(require '[clojure.string :as str])
+(require '[clojure.tools.logging :as log])
+(require '[rems.config :refer [env]])
+(require '[rems.common.util :refer [getx]])
+
+(defn transform [config data]
+  (when (:log-authentication-details env)
+    (log/info "Data" data))
+
+  (let [attribute-name (getx config :attribute-name)]
+    (if-some [attribute-value (get data (keyword attribute-name))]
+      (let [group-index (str/index-of attribute-value ":group:")
+            authority-index (str/index-of attribute-value "#")]
+
+        (when (:log-authentication-details env)
+          (log/info "Indices" group-index authority-index))
+
+        (if (and group-index
+                 authority-index
+                 (< -1 group-index authority-index))
+          (let [namespace (subs attribute-value 0 group-index)
+                groups-and-role (subs attribute-value (+ group-index (count ":group:")) authority-index)
+                authority (subs attribute-value (+ authority-index (count "#")))
+
+                groups (str/split groups-and-role #":")]
+
+            (when (:log-authentication-details env)
+              (log/info "Parts" namespace groups authority))
+
+            ;; TODO: check trusted authority
+
+            (assoc data :groups groups))
+          data))
+      data)))
+
+```

resources/plugins/LS-AAI-GA4GH-push.md

@@ -0,0 +1,11 @@
+# LifeScience AAI GA4GH push
+
+A plugin to push REMS entitlements to LifeScience AAI using GA4GH visas.
+
+See [GA4GH](../../docs/ga4gh-visas.md).
+
+```clj
+(defn process [config data]
+  ;; TODO implement
+  )
+```

resources/plugins/validate-attributes.md

@@ -0,0 +1,22 @@
+# Validate attributes
+
+A plugin to validate that specified attributes are present and not empty strings.
+
+```clj
+(require '[clojure.string :as str])
+(require '[clojure.tools.logging :as log])
+
+(defn empty-attributes [user attributes]
+  (for [{:keys [attribute-name error-key]} attributes
+        :let [attribute-value (get user (keyword attribute-name))
+              error? (str/blank? attribute-value)]
+        :when error?]
+    error-key))
+
+(defn validate [config data]
+  (when-some [invalid-attributes (seq (empty-attributes data (get config :required-attributes)))]
+    (log/info invalid-attributes)
+    [{:key :t.login.errors/invalid-user
+      :args invalid-attributes}]))
+
+```

resources/plugins/validate-group-membership.md

@@ -0,0 +1,22 @@
+# Validate group membership
+
+Check that the specified group attribute contains at least
+one of the valid groups.
+
+```clj
+(require '[rems.config :refer [env]])
+(require '[clojure.string :as str])
+(require '[clojure.tools.logging :as log])
+
+(defn validate [config data]
+  (let [{:keys [attribute-name valid-groups error-key]} config
+        groups (get data (keyword attribute-name))]
+
+    (when (:log-authentication-details env)
+      (log/info "Groups" groups))
+
+    (when (or (empty? groups)
+              (empty? (clojure.set/intersection (set groups) valid-groups)))
+      [{:key error-key}])))
+
+```

src/clj/rems/auth/fake_login.clj

@@ -6,6 +6,7 @@
             [rems.common.util :refer [escape-element-id]]
             [rems.config :refer [env]]
             [rems.db.test-data-users :as test-data-users]
+            [rems.plugins :as plugins]
             [ring.util.response :refer [redirect]]))
 
 (defn get-fake-user-descriptions []
@@ -44,6 +45,7 @@
   (let [id-data (get-fake-id-data username)
         user-info (get-fake-user-info username)
         user-data (merge id-data user-info)
+        user-data (plugins/transform :extension-point/transform-user-data user-data)
         user (oidc/find-or-create-user! user-data)]
     (-> (redirect "/redirect")
         (assoc :session session)
@@ -95,3 +97,4 @@
 (defroutes routes
   (GET (login-url) req (fake-login-screen req))
   (GET (logout-url) req (fake-logout req)))
+

src/clj/rems/auth/oidc.clj

@@ -11,6 +11,7 @@
             [rems.ga4gh :as ga4gh]
             [rems.json :as json]
             [rems.jwt :as jwt]
+            [rems.plugins :as plugins]
             [rems.util :refer [getx]]
             [ring.util.response :refer [redirect]])
   (:import [java.time Instant]))
@@ -91,6 +92,7 @@
     user))
 
 (defn- get-user-attributes [user-data]
+  ;; XXX: consider using a plugin for the renaming
   ;; TODO all attributes could support :rename
   (let [userid (or (find-user user-data) (get-new-userid user-data))
         _ (assert userid (when (:log-authentication-details env) {:user-data user-data}))
@@ -104,12 +106,19 @@
 ;; XXX: consider joining with rems.db.users/invalid-user?
 (defn- validate-user! [user]
   ;; userid already checked
+  ;; XXX: we can migrate this old way to the new plugin system
   (when-let [errors (seq (remove nil?
                                  [(when (and (:oidc-require-name env) (str/blank? (:name user))) :t.login.errors/name)
                                   (when (and (:oidc-require-email env) (str/blank? (:email user))) :t.login.errors/email)]))]
     (throw (ex-info "Invalid user"
                     {:key :t.login.errors/invalid-user
                      :args errors
+                     :user user})))
+
+  (when-let [errors (seq (remove nil? (plugins/validate :extension-point/validate-user-data user)))]
+    (throw (ex-info "Invalid user"
+                    {:key (or (:key (first errors)) :t.login.errors/invalid-user)
+                     :args (:args (first errors))
                      :user user}))))
 
 (defn find-or-create-user! [user-data]
@@ -165,9 +174,12 @@
                                 json/parse-string))
                 researcher-status (ga4gh/passport->researcher-status-by user-info)
                 user-data (merge id-data user-info researcher-status)
+                user-data (plugins/transform :extension-point/transform-user-data user-data)
                 user (find-or-create-user! user-data)]
+
             (when (:log-authentication-details env)
               (log/info "logged in" user-data user))
+
             (-> (redirect "/redirect")
                 (assoc :session (:session request))
                 (assoc-in [:session :access-token] access-token)