-
Notifications
You must be signed in to change notification settings - Fork 119
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
14 validate against skohub shacl (#16)
* Use skohub shape to validate data * Revert "Use skohub shape to validate data" This reverts commit 2cec370. * Use skohub shape to validate data #14 * Remove branches from github action #14 Action should run on every push not depending on branch * Update changed files action version #14 * Run on changes of github action file (e.g. main.yml) #15 * Separate changed file action to find ttl and yml files #15 * WIP validate files with skohub shape #14 * WIP fix github environment variable * WIP cat output to see why riot cries #14 * WIP disable workflow #14 * WIP Use pwd for providing full path in mount #14 * WIP Invalidate file for test purposes #14 * WIP cat out testfile #14 * WIP add riot validation step before validating with shacl #14 * WIP clean up a bit #14 * WIP reenable cleanup #14 * WIP GitHub Action not finding testfile. VMs is teared down anyway so its ok * WIP use correct SPARQL query to query result. We are checking for warning and violations * WIP Adjust path to query #14 * WIP Check only for violations. Warning are also present in the SkoHub Vocabs output * WIP make skos file valid turtle, but not valid skos for test purposes * WIP test new matrix preparation * WIP check for yml and ttl files in prepare matrix step #14 * WIP parse json output with jq #14 * WIP Testing: Just change a ttl file * WIP Test: Just change README * Cleanup #14 * Remove validation step #14 SkoHub Vocabs will already validate the vocabularys.
- Loading branch information
Showing
6 changed files
with
202 additions
and
90 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,138 @@ | ||
| #!/bin/bash | ||
| set -euo pipefail | ||
|
|
||
| scripts=$(realpath $(dirname -- "$0")) | ||
| shape=$(realpath "$scripts/../skos.shacl.ttl") | ||
| severity=all | ||
| report= | ||
|
|
||
| usage() { | ||
| echo "$0 [OPTION]... FILE" | ||
| echo "Validate SKOS file (Turtle syntax). No return message means everything is fine." | ||
| echo | ||
| echo "Options:" | ||
| echo " -s FILE shape file (default: $shape)" | ||
| echo " -l LEVEL severity violation|warning|all (default: $severity)" | ||
| echo " -o FILE keep full validation report in this file" | ||
| echo " -r show raw validation report and exit" | ||
| echo " -h show this help message" | ||
| exit $1 | ||
| } | ||
|
|
||
| die() { | ||
| echo "$*" >&2 | ||
| exit 1 | ||
| } | ||
|
|
||
| cleanup() { | ||
| echo "Cleaning up" | ||
| docker container stop validate-skos-fuseki > /dev/null | ||
| } | ||
|
|
||
| trap cleanup 0 2 3 15 | ||
|
|
||
| while getopts s:l:o:rh flag | ||
| do | ||
| case "${flag}" in | ||
| s) shape=${OPTARG};; | ||
| l) severity=${OPTARG};; | ||
| o) result=${OPTARG};; | ||
| r) report=1;; | ||
| h) usage 0;; | ||
| *) usage 1;; | ||
| esac | ||
| done | ||
| shift $(($OPTIND - 1)) | ||
|
|
||
| [ -z "${1:-}" ] && usage 1 | ||
|
|
||
| file=$(realpath "$1") | ||
| [ -f "$file" ] || die "File not found: $file" | ||
| # create temporary testfile and make sure it gets deleted | ||
| testfile=$(mktemp /tmp/validate-script.XXXXXX) | ||
|
|
||
|
|
||
| shape=$(realpath "$shape") | ||
| [ -f "$shape" ] || die "File not found: $shape" | ||
| # add the skos definitions to the file if the shape is "skos.shacl.ttl" | ||
| if [ "$(basename $shape)" = "skos.shacl.ttl" ]; then | ||
| cat $file $(realpath skosClassAndPropertyDefinitions.ttl) > $testfile | ||
| else | ||
| cat $file > $testfile | ||
| fi | ||
|
|
||
| grep -vE '^\s*(#.*)?$' "$file" >/dev/null || die "File contains no RDF statements: $testfile" | ||
|
|
||
| if [[ $severity == "warning" ]]; then | ||
| SEVERITY_FILE="./scripts/checkForWarning.rq" | ||
| elif [[ $severity == "all" ]]; then | ||
| SEVERITY_FILE="./scripts/checkForBoth.rq" | ||
| elif [[ $severity == "violation" ]]; then | ||
| SEVERITY_FILE="./scripts/checkForViolation.rq" | ||
| else | ||
| die "Unknown severity: $severity" | ||
| fi | ||
|
|
||
| # create temporary file (will be deleted in cleanup function) | ||
| if [[ -z "${result:-}" ]]; then | ||
| result=$(mktemp /tmp/validate-script.XXXXXX) | ||
| else | ||
| result=$(realpath "$result") | ||
| fi | ||
|
|
||
| # Check if the container is running | ||
| if docker ps | grep -q "validate-skos-fuseki"; then | ||
| docker stop validate-skos-fuseki | ||
| sleep 1 | ||
| fi | ||
|
|
||
| # wait till fuseki is up | ||
| max_attempts=5 | ||
| delay=3 | ||
| attempt=1 | ||
|
|
||
| echo "Starting validation container" | ||
|
|
||
| while [ $attempt -le $max_attempts ]; do | ||
| # start fuseki | ||
| docker run -d --rm --name validate-skos-fuseki -p 0:3030 -v $(pwd)/fuseki/config_inference.ttl:/fuseki/config_inference.ttl skohub/jena-fuseki:latest /jena-fuseki/fuseki-server --config /fuseki/config_inference.ttl > /dev/null | ||
| port=$(docker port validate-skos-fuseki 3030/tcp | head -1 | awk -F: '{print $2}') | ||
| sleep $delay | ||
| curl "http://localhost:$port/$/ping" > /dev/null && break | ||
| attempt=$((attempt + 1)) | ||
| done | ||
|
|
||
| if [ $attempt -gt $max_attempts ]; then | ||
| echo "The command has failed after $max_attempts attempts." | ||
| exit 1 | ||
| fi | ||
|
|
||
| # validate ttl file | ||
| riotResult="$(docker run --rm -v $testfile:/rdf/testfile.ttl skohub/jena:4.6.1 riot --validate /rdf/testfile.ttl)" | ||
| echo $? | ||
|
|
||
| # upload file | ||
| curl --request POST \ | ||
| --url "http://localhost:$port/dataset/data?graph=default" \ | ||
| --header 'Content-Type: text/turtle' \ | ||
| --data-binary @$testfile > /dev/null | ||
|
|
||
| # validate w/ shacl | ||
| curl --request POST \ | ||
| --url "http://localhost:$port/dataset/shacl?graph=default" \ | ||
| --header 'Content-Type: text/turtle' \ | ||
| --data-binary @$shape > "$result" | ||
|
|
||
| echo "Checking validation result" | ||
|
|
||
| if [[ "$report" -eq 1 ]]; then | ||
| cat "$result" | ||
| else | ||
| validationResult="$(docker run --rm -v $(pwd)/scripts/checkForViolation.rq:/rdf/checkForViolation.rq -v $result:/rdf/result.ttl skohub/jena:4.6.1 arq --data /rdf/result.ttl --query /rdf/checkForViolation.rq)" | ||
|
|
||
| lines=$(echo "$validationResult" | wc -l ) | ||
|
|
||
| # Correct validation has 4 lines of output | ||
| [[ ${lines} -eq 4 ]] || die "$validationResult" | ||
|
|
||
| fi |
This file was deleted.
Oops, something went wrong.