diff --git a/data/software-tools/apptainer.json b/data/software-tools/apptainer.json index 964905e8..073abd53 100644 --- a/data/software-tools/apptainer.json +++ b/data/software-tools/apptainer.json @@ -11,11 +11,33 @@ }, "isAccessibleForFree": true, "hasQualityDimension": [ - { "@id": "dim:compatibility", "@type": "@id" }, - { "@id": "dim:flexibility", "@type": "@id" }, - { "@id": "dim:reliability", "@type": "@id" }, - { "@id": "dim:sustainability", "@type": "@id" } + { + "@id": "dim:compatibility", + "@type": "@id" + }, + { + "@id": "dim:flexibility", + "@type": "@id" + }, + { + "@id": "dim:reliability", + "@type": "@id" + }, + { + "@id": "dim:sustainability", + "@type": "@id" + } ], "howToUse": ["command-line"], - "license": "https://opensource.org/license/bsd-3-clause" + "license": "https://opensource.org/license/bsd-3-clause", + "improvesQualityIndicator": [ + { + "@id": "https://w3id.org/everse/i/indicators/dependency_management", + "@type": "@id" + }, + { + "@id": "https://w3id.org/everse/i/indicators/requirements_specified", + "@type": "@id" + } + ] } diff --git a/data/software-tools/bearer.json b/data/software-tools/bearer.json index e98a2032..f6df2940 100644 --- a/data/software-tools/bearer.json +++ b/data/software-tools/bearer.json @@ -2,7 +2,10 @@ "@context": "https://w3id.org/everse/rs#", "@id": "https://w3id.org/everse/tools/bearer", "@type": "SoftwareApplication", - "applicationCategory": { "@id": "rs:AnalysisCode", "@type": "@id" }, + "applicationCategory": { + "@id": "rs:AnalysisCode", + "@type": "@id" + }, "appliesToProgrammingLanguage": [ "JavaScript", "TypeScript", @@ -13,10 +16,27 @@ "Python" ], "description": "Bearer CLI is a static application security testing (SAST) tool that scans your source code and analyzes your data flows to discover, filter and prioritize security and privacy risks.", - "hasQualityDimension": { "@id": "dim:security", "@type": "@id" }, + "hasQualityDimension": { + "@id": "dim:security", + "@type": "@id" + }, "howToUse": ["online-service"], "isAccessibleForFree": true, "license": "https://spdx.org/licenses/Elastic-2.0", "name": "bearer", - "url": "https://github.com/bearer/bearer" + "url": "https://github.com/bearer/bearer", + "measuresQualityIndicator": [ + { + "@id": "https://w3id.org/everse/i/indicators/no_critical_vulnerability", + "@type": "@id" + }, + { + "@id": "https://w3id.org/everse/i/indicators/no_leaked_credentials", + "@type": "@id" + }, + { + "@id": "https://w3id.org/everse/i/indicators/static_analysis_common_vulnerabilities", + "@type": "@id" + } + ] } diff --git a/data/software-tools/creative-commons-license-chooser.json b/data/software-tools/creative-commons-license-chooser.json index d3520494..a423bb31 100644 --- a/data/software-tools/creative-commons-license-chooser.json +++ b/data/software-tools/creative-commons-license-chooser.json @@ -7,10 +7,19 @@ "@type": "@id" }, "description": "A web-based tool to help users select an appropriate Creative Commons license for their work.", - "hasQualityDimension": { "@id": "dim:fairness", "@type": "@id" }, + "hasQualityDimension": { + "@id": "dim:fairness", + "@type": "@id" + }, "howToUse": ["command-line"], "isAccessibleForFree": true, "license": "https://creativecommons.org/publicdomain/zero/1.0", "name": "Creative Commons License Chooser", - "url": "https://chooser-beta.creativecommons.org/" + "url": "https://chooser-beta.creativecommons.org/", + "improvesQualityIndicator": [ + { + "@id": "https://w3id.org/everse/i/indicators/software_has_license", + "@type": "@id" + } + ] } diff --git a/data/software-tools/doxygen.json b/data/software-tools/doxygen.json index 4d51d961..0208f85a 100644 --- a/data/software-tools/doxygen.json +++ b/data/software-tools/doxygen.json @@ -22,7 +22,7 @@ "howToUse": ["CI/CD", "command-line"], "improvesQualityIndicator": [ { - "@id": "https://w3id.org/everse/i/indicators/software_documentation", + "@id": "https://w3id.org/everse/i/indicators/software_has_documentation", "@type": "@id" } ], diff --git a/data/software-tools/gitlab-cicd.json b/data/software-tools/gitlab-cicd.json index a1dcb634..ecdd3aa1 100644 --- a/data/software-tools/gitlab-cicd.json +++ b/data/software-tools/gitlab-cicd.json @@ -8,12 +8,46 @@ }, "description": "Integrated continuous integration and deployment platform that automates research software testing, building, and deployment workflows, improving software reliability and sustainability through comprehensive DevOps practices.", "hasQualityDimension": [ - { "@id": "dim:sustainability", "@type": "@id" }, - { "@id": "dim:reliability", "@type": "@id" } + { + "@id": "dim:sustainability", + "@type": "@id" + }, + { + "@id": "dim:reliability", + "@type": "@id" + } + ], + "improvesQualityIndicator": [ + { + "@id": "https://w3id.org/everse/i/indicators/repository_workflows", + "@type": "@id" + }, + { + "@id": "https://w3id.org/everse/i/indicators/software_has_tests", + "@type": "@id" + }, + { + "@id": "https://w3id.org/everse/i/indicators/has_ci-tests", + "@type": "@id" + } ], "howToUse": ["CI/CD", "command-line", "online-service"], "isAccessibleForFree": false, "license": "https://spdx.org/licenses/MIT", "name": "GitLab CICD", - "url": "https://docs.gitlab.com/ci/" + "url": "https://docs.gitlab.com/ci/", + "improvesQualityIndicator": [ + { + "@id": "https://w3id.org/everse/i/indicators/repository_workflows", + "@type": "@id" + }, + { + "@id": "https://w3id.org/everse/i/indicators/software_has_tests", + "@type": "@id" + }, + { + "@id": "https://w3id.org/everse/i/indicators/has_ci-tests", + "@type": "@id" + } + ] } diff --git a/data/software-tools/gitleaks.json b/data/software-tools/gitleaks.json index 90716fb0..81af1301 100644 --- a/data/software-tools/gitleaks.json +++ b/data/software-tools/gitleaks.json @@ -8,13 +8,34 @@ "isAccessibleForFree": true, "license": "https://spdx.org/licenses/MIT", "applicationCategory": [ - { "@id": "rs:PrototypeTool", "@type": "@id" }, - { "@id": "rs:ResearchInfrastructureSoftware", "@type": "@id" }, - { "@id": "rs:AnalysisCode", "@type": "@id" } + { + "@id": "rs:PrototypeTool", + "@type": "@id" + }, + { + "@id": "rs:ResearchInfrastructureSoftware", + "@type": "@id" + }, + { + "@id": "rs:AnalysisCode", + "@type": "@id" + } ], "hasQualityDimension": [ - { "@id": "dim:security", "@type": "@id" }, - { "@id": "dim:sustainability", "@type": "@id" } + { + "@id": "dim:security", + "@type": "@id" + }, + { + "@id": "dim:sustainability", + "@type": "@id" + } ], - "howToUse": ["CI/CD", "command-line"] + "howToUse": ["CI/CD", "command-line"], + "measuresQualityIndicator": [ + { + "@id": "https://w3id.org/everse/i/indicators/no_leaked_credentials", + "@type": "@id" + } + ] } diff --git a/data/software-tools/howfairis.json b/data/software-tools/howfairis.json index 9fb4de60..f0b2ff37 100644 --- a/data/software-tools/howfairis.json +++ b/data/software-tools/howfairis.json @@ -14,15 +14,15 @@ "@type": "@id" }, { - "@id": "https://w3id.org/everse/i/indicators/listed_in_a_registry", + "@id": "https://w3id.org/everse/i/indicators/listed_in_registry", "@type": "@id" }, { - "@id": "https://w3id.org/everse/i/indicators/uses_citation", + "@id": "https://w3id.org/everse/i/indicators/software_has_citation", "@type": "@id" }, { - "@id": "https://w3id.org/everse/i/indicators/has_documentation", + "@id": "https://w3id.org/everse/i/indicators/software_has_citation", "@type": "@id" }, { diff --git a/data/software-tools/resqui.json b/data/software-tools/resqui.json index 2c7d8d44..45614883 100644 --- a/data/software-tools/resqui.json +++ b/data/software-tools/resqui.json @@ -34,19 +34,19 @@ "@type": "@id" }, { - "@id": "https://w3id.org/everse/i/indicators/software_has_ci_tests", + "@id": "https://w3id.org/everse/i/indicators/has_ci-tests", "@type": "@id" }, { - "@id": "https://w3id.org/everse/i/indicators/software_human_code_review_requirement", + "@id": "https://w3id.org/everse/i/indicators/human_code_review_requirement", "@type": "@id" }, { - "@id": "https://w3id.org/everse/i/indicators/software_has_published_package", + "@id": "https://w3id.org/everse/i/indicators/has_published_package", "@type": "@id" }, { - "@id": "https://w3id.org/everse/i/indicators/software_has_no_security_leak", + "@id": "https://w3id.org/everse/i/indicators/no_leaked_credentials", "@type": "@id" } ], diff --git a/data/software-tools/somef.json b/data/software-tools/somef.json index fed32d83..5fa1ca99 100644 --- a/data/software-tools/somef.json +++ b/data/software-tools/somef.json @@ -14,11 +14,11 @@ "@type": "@id" }, { - "@id": "https://w3id.org/everse/i/indicators/listed_in_a_registry", + "@id": "https://w3id.org/everse/i/indicators/listed_in_registry", "@type": "@id" }, { - "@id": "https://w3id.org/everse/i/indicators/uses_citation", + "@id": "https://w3id.org/everse/i/indicators/software_has_citation", "@type": "@id" }, { @@ -26,7 +26,7 @@ "@type": "@id" }, { - "@id": "https://w3id.org/everse/i/indicators/software_documentation", + "@id": "https://w3id.org/everse/i/indicators/software_has_documentation", "@type": "@id" }, { diff --git a/data/software-tools/sonarqube.json b/data/software-tools/sonarqube.json index 7ed38222..2aaa3fd0 100644 --- a/data/software-tools/sonarqube.json +++ b/data/software-tools/sonarqube.json @@ -8,14 +8,37 @@ }, "description": "Continuous code quality platform that automatically detects bugs, vulnerabilities, and code smells across multiple programming languages, providing comprehensive static analysis to improve software maintainability, security, and reliability.", "hasQualityDimension": [ - { "@id": "dim:maintainability", "@type": "@id" }, - { "@id": "dim:security", "@type": "@id" }, - { "@id": "dim:reliability", "@type": "@id" } + { + "@id": "dim:maintainability", + "@type": "@id" + }, + { + "@id": "dim:security", + "@type": "@id" + }, + { + "@id": "dim:reliability", + "@type": "@id" + } ], "howToUse": ["CI/CD", "online-service"], "isAccessibleForFree": true, "license": "https://spdx.org/licenses/GPL-3.0-only", "name": "SonarQube", "appliesToProgrammingLanguage": ["Java", "Python", "JavaScript", "C#", "C++"], - "url": "https://www.sonarqube.org/" + "url": "https://www.sonarqube.org/", + "improvesQualityIndicator": [ + { + "@id": "https://w3id.org/everse/i/indicators/no_critical_vulnerability", + "@type": "@id" + }, + { + "@id": "https://w3id.org/everse/i/indicators/no_leaked_credentials", + "@type": "@id" + }, + { + "@id": "https://w3id.org/everse/i/indicators/static_analysis_common_vulnerabilities", + "@type": "@id" + } + ] } diff --git a/data/software-tools/sqaaas.json b/data/software-tools/sqaaas.json index 96d3d5c5..409a2fc1 100644 --- a/data/software-tools/sqaaas.json +++ b/data/software-tools/sqaaas.json @@ -2,12 +2,52 @@ "@context": "https://w3id.org/everse/rs#", "@id": "https://w3id.org/everse/tools/sqaaas", "@type": "SoftwareApplication", - "applicationCategory": { "@id": "rs:AnalysisCode", "@type": "@id" }, + "applicationCategory": { + "@id": "rs:AnalysisCode", + "@type": "@id" + }, "description": "Software Quality Assessment as a Service platform that analyzes completeness against FAIR4RS reference criteria for research software releases, providing comprehensive quality assessment and awarding capabilities for research software projects.", - "hasQualityDimension": { "@id": "dim:fairness", "@type": "@id" }, + "hasQualityDimension": { + "@id": "dim:fairness", + "@type": "@id" + }, "howToUse": ["online-service", "CI/CD"], "isAccessibleForFree": true, "license": "https://spdx.org/licenses/AGPL-3.0-or-later", "name": "SQAaas", - "url": "https://sqaaas.eosc-synergy.eu/" + "url": "https://sqaaas.eosc-synergy.eu/", + "measuresQualityIndicator": [ + { + "@id": "https://w3id.org/everse/i/indicators/listed_in_registry", + "@type": "@id" + }, + { + "@id": "https://w3id.org/everse/i/indicators/requirements_specified", + "@type": "@id" + }, + { + "@id": "https://w3id.org/everse/i/indicators/software_has_documentation", + "@type": "@id" + }, + { + "@id": "https://w3id.org/everse/i/indicators/software_has_license", + "@type": "@id" + }, + { + "@id": "https://w3id.org/everse/i/indicators/descriptive_metadata", + "@type": "@id" + }, + { + "@id": "https://w3id.org/everse/i/indicators/no_leaked_credentials", + "@type": "@id" + }, + { + "@id": "https://w3id.org/everse/i/indicators/software_has_tests", + "@type": "@id" + }, + { + "@id": "https://w3id.org/everse/i/indicators/has_no_linting_issues", + "@type": "@id" + } + ] } diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000..b11875a3 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,19 @@ +# Tests + +Validation tests for EVERSE TechRadar tool and service JSON files. + +## Running Tests + +```bash +# Run all tests +python -m pytest tests/ + +# Run specific test +python -m pytest tests/test_tools.py -v +``` + +## Dependencies + +```bash +pip install -r tests/requirements.txt +``` diff --git a/tests/helpers.py b/tests/helpers.py index df9dd28d..8ff0853f 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,6 +1,14 @@ """ -copy of the original file: -https://github.com/EVERSE-ResearchSoftware/indicators/blob/main/tests/helpers.py +Helper functions for validating JSON files against schemas. + +This module provides utilities for loading JSON schemas and validating JSON files. +It includes dynamic fetching of quality dimensions and indicators from the EVERSE +indicators API to ensure validation uses the most up-to-date lists. + +Dimensions: Always validated (with fallback if API unavailable) +Indicators: Validated only if API is available (skipped otherwise) + +Adapted from: https://github.com/EVERSE-ResearchSoftware/indicators/blob/main/tests/helpers.py """ import json @@ -8,18 +16,153 @@ import pytest from jsonschema import validate, ValidationError import os +import urllib.request +import urllib.error + + +# Fallback list of dimensions if API is not accessible +FALLBACK_DIMENSIONS = [ + "dim:compatibility", + "dim:fairness", + "dim:flexibility", + "dim:functional_suitability", + "dim:interaction_capability", + "dim:maintainability", + "dim:performance_efficiency", + "dim:reliability", + "dim:safety", + "dim:security", + "dim:sustainability", +] + + +def fetch_quality_indicators(): + """ + Fetch quality indicators from the EVERSE indicators API. + + Attempts to fetch the current list of quality indicators from the EVERSE + indicators API. If the API is unavailable, raises a RuntimeError. + + API Endpoint: https://everse.software/indicators/api/indicators.json + + Returns: + list: Indicator URIs if successful. + Example: ["https://w3id.org/everse/i/no_leaked_credentials", ...] + """ + api_url = "https://everse.software/indicators/api/indicators.json" + + try: + with urllib.request.urlopen(api_url, timeout=5) as response: + data = json.loads(response.read().decode("utf-8")) + + indicators_identifiers = [] + + indicators_list = data.get("indicators", []) + for indicator in indicators_list: + if isinstance(indicator, dict) and "identifier" in indicator: + indicators_identifiers.append(indicator["identifier"]["@id"]) + + if indicators_identifiers: + print(f"Successfully fetched {len(indicators_identifiers)} indicators from {api_url}") + return indicators_identifiers + else: + print(f"No indicators found in API response from {api_url}") + return None + + except ( + urllib.error.URLError, + urllib.error.HTTPError, + json.JSONDecodeError, + TimeoutError, + ) as e: + raise RuntimeError(f"Could not fetch indicators from {api_url}: {e}") + + +def fetch_quality_dimensions(): + """ + Fetch quality dimensions from the EVERSE indicators API with offline fallback. + + Attempts to fetch the current list of quality dimensions from the EVERSE + indicators API. If the API is unavailable (no internet connection or API error), + falls back to a hardcoded list of dimensions. + + API Endpoint: https://everse.software/indicators/api/dimensions.json + + Returns: + list: Dimension identifiers in the format "dim:". + Example: ["dim:compatibility", "dim:fairness", ...] + """ + api_url = "https://everse.software/indicators/api/dimensions.json" + + try: + with urllib.request.urlopen(api_url, timeout=5) as response: + data = json.loads(response.read().decode('utf-8')) + + # Extract dimension identifiers from the API response + dimensions_identifiers = [] + + dimensions = data.get('dimensions', []) + for item in dimensions: + if isinstance(item, dict) and 'identifier' in item: + dim_id = item['identifier'] + if '/' in dim_id: + dim_name = dim_id.split('/')[-1] + dimensions_identifiers.append(f"dim:{dim_name}") + else: + dimensions_identifiers.append(dim_id if dim_id.startswith('dim:') else f"dim:{dim_id}") + + if dimensions_identifiers: + print(f"Successfully fetched {len(dimensions_identifiers)} dimensions from {api_url}") + return dimensions_identifiers + else: + print(f"No dimensions found in API response from {api_url}") + + except (urllib.error.URLError, urllib.error.HTTPError, json.JSONDecodeError, TimeoutError) as e: + print(f"Could not fetch dimensions from {api_url}: {e}") + + print(f"Using fallback dimension list:\n{FALLBACK_DIMENSIONS}") + return FALLBACK_DIMENSIONS def load_local_schema(schema_path): - """Loads the JSON schema from a local file.""" + """ + Load JSON schema from file and update it with current quality dimensions and indicators. + + Loads the JSON schema file and dynamically updates: + - Quality dimension enum with latest values from EVERSE API (or fallback) + - Quality indicator enum with latest values from EVERSE API (or makes it optional if unavailable) + + This ensures validation always uses current dimension and indicator definitions. + + Args: + schema_path (str): Absolute path to the JSON schema file + + Returns: + dict: The loaded and updated JSON schema + + Raises: + pytest.Failed: If schema file not found or invalid JSON + """ print(f"Attempting to load local schema from: {schema_path}") if not os.path.exists(schema_path): pytest.fail(f"Schema file not found at {schema_path}", pytrace=False) + try: with open(schema_path, "r") as f: schema_data = json.load(f) - print("Successfully loaded local schema.") + + # Update the quality dimension enums dynamically + dimensions = fetch_quality_dimensions() + schema_data['definitions']['qualityDimensionObject']['properties']['@id']['enum'] = dimensions + print(f"Updated schema with {len(dimensions)} quality dimensions") + + # Update the quality indicator enums dynamically + indicators = fetch_quality_indicators() + schema_data['definitions']['qualityIndicatorObject']['properties']['@id']['enum'] = indicators + print(f"Updated schema with {len(indicators)} quality indicators") + return schema_data + except json.JSONDecodeError as e: pytest.fail( f"Failed to decode JSON from schema file {schema_path}: {e}", pytrace=False @@ -33,8 +176,19 @@ def load_local_schema(schema_path): def validate_json_files_using_schema(schema_file_path, json_file_path): """ - Validates all JSON files in the 'json_file_path/' directory against the - local JSON Schema file (schema_file_path). + Validate all JSON files in a directory against a JSON schema. + + Loads the schema (with dynamic dimension updates), then validates each JSON file + in the specified directory. Collects all validation errors and fails the test + if any files are invalid. + + Args: + schema_file_path (str): Path to the JSON schema file + json_file_path (str): Directory containing JSON files to validate + + Raises: + pytest.skip: If no JSON files found in the directory + AssertionError: If any JSON files fail validation """ json_files = glob.glob(f"{json_file_path}/*.json") diff --git a/tests/tools_validation_schema.json b/tests/tools_validation_schema.json index c4ea30d4..484b6389 100644 --- a/tests/tools_validation_schema.json +++ b/tests/tools_validation_schema.json @@ -1,7 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "title": "EVERSE Research Software (RS) Tools Validation Schema", - "description": "Schema for validating software tool entries in the EVERSE Research Software context", + "description": "Schema for validating software tool entries in the EVERSE Research Software context.", "type": "object", "definitions": { "applicationCategoryObject": { @@ -24,23 +24,12 @@ "properties": { "@id": { "type": "string", - "description": "Identifier for the quality dimension", - "enum": [ - "dim:compatibility", - "dim:fairness", - "dim:flexibility", - "dim:functional_suitability", - "dim:interaction_capability", - "dim:maintainability", - "dim:performance_efficiency", - "dim:reliability", - "dim:safety", - "dim:security", - "dim:sustainability" - ] + "description": "Identifier for the quality dimension. Valid values are dynamically fetched from the EVERSE indicators API at validation time.", + "pattern": "^dim:[a-z_]+$" } }, - "required": [ "@id" ] + "required": [ "@id" ], + "comment": "The enum of valid dimension values is populated dynamically by helpers.py EVERSE api" }, "qualityIndicatorObject": { @@ -48,11 +37,12 @@ "properties": { "@id": { "type": "string", - "description": "Identifier for the quality indicator", + "description": "Identifier for the quality indicator. Valid values are dynamically fetched from the EVERSE indicators API at validation time.", "format": "uri" } }, - "required": ["@id"] + "required": ["@id"], + "comment": "The enum of valid indicator URIs is populated dynamically by helpers.py. If unavailable, validation is skipped." } }, "properties": {