QA Spec

Makefile

@@ -1,10 +1,10 @@
-json: \
-	bagit-profile.json \
-	ocrd_tool.schema.json \
-	openapi.json
-
-deps:
-	pip install yaml click
+json: $(shell find -name '*.json')
 
 %.json: %.yml
-	python3 scripts/yaml-to-json.py $< $@
+	python3 scripts/yaml-to-json.py --indent 0 $< $@
+
+validate: json
+	jsonschema --output pretty --validator Draft201909Validator --instance ocrd_eval.sample.json ocrd_eval.schema.json
+
+deps:
+	pip install yaml click jsonschema

bagit-profile.json

@@ -1 +1 @@
-{"BagIt-Profile-Info":{"BagIt-Profile-Identifier":"https://ocr-d.de/en/spec/bagit-profile.json","BagIt-Profile-Version":"1.2.0","Source-Organization":"OCR-D","External-Description":"BagIt profile for OCR data","Contact-Name":"Konstantin Baierer","Contact-Email":"[email protected]","Version":0.1},"Bag-Info":{"Bagging-Date":{"required":false},"Source-Organization":{"required":false},"Ocrd-Mets":{"required":false,"default":"mets.xml"},"Ocrd-Identifier":{"required":true},"Ocrd-Checksum":{"required":false,"default":"cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e"}},"Manifests-Required":["sha512"],"Tag-Manifests-Required":[],"Tag-Files-Required":[],"Tag-Files-Allowed":["README.md","Makefile","build.sh","sources.csv","metadata/*.xml","metadata/*.txt"],"Allow-Fetch.txt":false,"Serialization":"required","Accept-Serialization":"application/zip","Accept-BagIt-Version":["1.0"]}
+{"BagIt-Profile-Info": {"BagIt-Profile-Identifier": "https://ocr-d.de/en/spec/bagit-profile.json", "BagIt-Profile-Version": "1.2.0", "Source-Organization": "OCR-D", "External-Description": "BagIt profile for OCR data", "Contact-Name": "Konstantin Baierer", "Contact-Email": "[email protected]", "Version": 0.1}, "Bag-Info": {"Bagging-Date": {"required": false}, "Source-Organization": {"required": false}, "Ocrd-Mets": {"required": false, "default": "mets.xml"}, "Ocrd-Identifier": {"required": true}, "Ocrd-Checksum": {"required": false, "default": "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e"}}, "Manifests-Required": ["sha512"], "Tag-Manifests-Required": [], "Tag-Files-Required": [], "Tag-Files-Allowed": ["README.md", "Makefile", "build.sh", "sources.csv", "metadata/*.xml", "metadata/*.txt"], "Allow-Fetch.txt": false, "Serialization": "required", "Accept-Serialization": "application/zip", "Accept-BagIt-Version": ["1.0"]}

ocrd_eval.sample.json

@@ -0,0 +1 @@
+[{"@id": "https://github.com/OCR-D/quiver/tree/data/evaluations/wf1-data345-eval1.json", "label": "OCR workflow 1 on workspace 345", "metadata": {"ocr_workflow": {"@id": "https://github.com/OCR-D/quiver/tree/data/workflows/1.nf", "label": "OCR Workflow 1"}, "eval_workflow": {"@id": "https://github.com/OCR-D/quiver/tree/data/workflows/eval1.nf", "label": "Evaluation Workflow 1"}, "gt_workspace": {"@id": "https://gt.ocr-d.de/workspace/789", "label": "GT workspace 789 (19th century fraktur)"}, "ocr_workspace": {"@id": "https://github.com/OCR-D/quiver/tree/data/workspaces/3000.ocrd.zip", "label": "OCR result workspace 3000"}, "eval_workspace": {"@id": "https://github.com/OCR-D/quiver/tree/data/workspaces/345.ocrd.zip", "label": "Evaluation Workspace 345"}, "workflow_steps": {"0": "Processor A", "1": "Processor B"}, "workflow_model": "Fraktur_GT4HistOCR", "document_metadata": {"fonts": ["antiqua", "fraktur"], "publication_century": "1800-1900", "publication_decade": "1850-1860", "publication_year": 1855, "number_of_pages": 100, "layout": "simple"}}, "evaluation": {"document_wide": {"wall_time": 1234, "cer": 0.57, "cer_min_max": [0.2, 0.57]}, "by_page": [{"page_id": "PHYS_0001", "cer": 0.8, "processing_time": 2.1}]}}, {"@id": "https://github.com/OCR-D/quiver/tree/data/evaluations/wf2-data345-eval1.json", "label": "OCR Workflow 2 on Data 345", "metadata": {"ocr_workflow": {"@id": "https://github.com/OCR-D/quiver/tree/data/workflows/2.nf", "label": "OCR Workflow 2"}, "eval_workflow": {"@id": "https://github.com/OCR-D/quiver/tree/data/workflows/eval1.nf", "label": "Evaluation Workflow 1"}, "gt_workspace": {"@id": "https://gt.ocr-d.de/workspace/789", "label": "GT workspace 789 (19th century fraktur)"}, "ocr_workspace": {"@id": "https://github.com/OCR-D/quiver/tree/data/workspaces/3000.ocrd.zip", "label": "OCR result workspace 3000"}, "eval_workspace": {"@id": "https://github.com/OCR-D/quiver/tree/data/workspaces/345.ocrd.zip", "label": "Evaluation Workspace 345"}, "workflow_steps": {"0": "Processor A", "1": "Processor B"}, "workflow_model": "Fraktur_GT4HistOCR", "document_metadata": {"fonts": ["antiqua", "fraktur"], "publication_century": "1800-1900", "publication_decade": "1850-1860", "publication_year": 1855, "number_of_pages": 100, "layout": "simple"}}, "evaluation": {"document_wide": {"wall_time": 4567, "cer": 0.9, "cer_min_max": [0.2, 0.99]}, "by_page": [{"page_id": "PHYS_0001", "cer": 0.9, "processing_time": 2.1}]}}]

ocrd_eval.sample.yml

@@ -0,0 +1,85 @@
+- '@id': https://github.com/OCR-D/quiver/tree/data/evaluations/wf1-data345-eval1.json
+  label: OCR workflow 1 on workspace 345
+  metadata:
+    ocr_workflow:
+      '@id': https://github.com/OCR-D/quiver/tree/data/workflows/1.nf
+      label: OCR Workflow 1
+    eval_workflow:
+      '@id': https://github.com/OCR-D/quiver/tree/data/workflows/eval1.nf
+      label: Evaluation Workflow 1
+    gt_workspace:
+      '@id': https://gt.ocr-d.de/workspace/789
+      label: GT workspace 789 (19th century fraktur)
+    ocr_workspace:
+      '@id': https://github.com/OCR-D/quiver/tree/data/workspaces/3000.ocrd.zip
+      label: OCR result workspace 3000
+    eval_workspace:
+      '@id': https://github.com/OCR-D/quiver/tree/data/workspaces/345.ocrd.zip
+      label: Evaluation Workspace 345
+    workflow_steps:
+      '0': Processor A
+      '1': Processor B
+    workflow_model: Fraktur_GT4HistOCR
+    document_metadata:
+      fonts:
+      - antiqua
+      - fraktur
+      publication_century: 1800-1900
+      publication_decade: 1850-1860
+      publication_year: 1855
+      number_of_pages: 100
+      layout: simple
+  evaluation:
+    document_wide:
+      wall_time: 1234
+      cer: 0.57
+      cer_min_max:
+      - 0.2
+      - 0.57
+    by_page:
+    - page_id: PHYS_0001
+      cer: 0.8
+      processing_time: 2.1
+
+- '@id': https://github.com/OCR-D/quiver/tree/data/evaluations/wf2-data345-eval1.json
+  label: OCR Workflow 2 on Data 345
+  metadata:
+    ocr_workflow:
+      '@id': https://github.com/OCR-D/quiver/tree/data/workflows/2.nf
+      label: OCR Workflow 2
+    eval_workflow:
+      '@id': https://github.com/OCR-D/quiver/tree/data/workflows/eval1.nf
+      label: Evaluation Workflow 1
+    gt_workspace:
+      '@id': https://gt.ocr-d.de/workspace/789
+      label: GT workspace 789 (19th century fraktur)
+    ocr_workspace:
+      '@id': https://github.com/OCR-D/quiver/tree/data/workspaces/3000.ocrd.zip
+      label: OCR result workspace 3000
+    eval_workspace:
+      '@id': https://github.com/OCR-D/quiver/tree/data/workspaces/345.ocrd.zip
+      label: Evaluation Workspace 345
+    workflow_steps:
+      '0': Processor A
+      '1': Processor B
+    workflow_model: Fraktur_GT4HistOCR
+    document_metadata:
+      fonts:
+      - antiqua
+      - fraktur
+      publication_century: 1800-1900
+      publication_decade: 1850-1860
+      publication_year: 1855
+      number_of_pages: 100
+      layout: simple
+  evaluation:
+    document_wide:
+      wall_time: 4567
+      cer: 0.9
+      cer_min_max:
+      - 0.2
+      - 0.99
+    by_page:
+    - page_id: PHYS_0001
+      cer: 0.9
+      processing_time: 2.1

ocrd_eval.schema.json

@@ -0,0 +1 @@
+{"$schema": "https://json-schema.org/draft/2019-09/schema", "$id": "https://ocr-d.de/en/spec/ocrd_eval.schema.json", "title": "A list of evaluations for OCR-D", "description": "- All references to URL are JSON-LD-like objects with at least an `@id`\n  property referencing the URL and `label` for a human-readable label to be\n  used in the UI\n", "type": "array", "items": {"required": ["@id", "label", "metadata", "evaluation"], "unevaluatedProperties": false, "allOf": [{"$ref": "#/$defs/LabeledUrl"}, {"properties": {"metadata": {"$ref": "#/$defs/EvaluationMetadata"}, "evaluation": {"$ref": "#/$defs/EvaluationReport"}}}]}, "$defs": {"LabeledUrl": {"type": "object", "required": ["@id"], "properties": {"@id": {"type": "string", "format": "uri", "description": "URL of the thing"}, "label": {"type": "string", "description": "Description of the thing for UI purposes"}}}, "EvaluationMetadata": {"type": "object", "title": "Metadata about one evaluation", "additionalProperties": false, "description": "EvaluationMetadata contains all the info on how an EvaluationReport came to be.\nThere are two OCR-D *workflows* involved:\n  - ocr_workflow: The workflow which produced the OCR results to evaluate\n  - eval_workflow: The workflow run to evaluate OCR and GT\n\nThere are three OCR-D *workspaces* involved:\n  - gt_workspace: The workspace containing the GT\n  - ocr_workspace: The workspace containing the OCR results from ocr_workflow\n  - eval_workspace: The workspace on which the eval_workflow was run\n", "required": ["ocr_workflow", "ocr_workspace", "eval_workflow", "eval_workspace", "gt_workspace", "document_metadata"], "properties": {"ocr_workflow": {"allOf": [{"$ref": "#/$defs/LabeledUrl"}], "description": "The OCR-D workflow that produced the ocr_workspace"}, "ocr_workspace": {"allOf": [{"$ref": "#/$defs/LabeledUrl"}], "description": "The workspace containing the OCR"}, "eval_workflow": {"allOf": [{"$ref": "#/$defs/LabeledUrl"}], "description": "The OCR-D workflow that produced the eval_workspace"}, "eval_workspace": {"allOf": [{"$ref": "#/$defs/LabeledUrl"}], "description": "The workspace containing the evaluation results"}, "gt_workspace": {"allOf": [{"$ref": "#/$defs/LabeledUrl"}], "description": "The workspace containing the GT"}, "workflow_steps": {"type": "object", "description": "Human readable description of the individual steps in the workflow (for UI)", "patternProperties": {"^[0-9]+$": {"type": "string", "description": "Description of this workflow step"}}}, "workflow_model": {"type": "string", "description": "Human readable name of the main model used for recognition in the OCR workflow (for UI)"}, "eval_tool": {"type": "string", "description": "Human readable name and version of evaluation tool used (for UI)"}, "document_metadata": {"type": "object", "title": "Bibliographical and typographical metadata about the work to be evaluated", "properties": {"publication_year": {"type": "number", "description": "Year the document was originally published"}, "publication_century": {"type": "string", "description": "Century the document was originally published", "pattern": "[12][0-9]{3}-[12][0-9]{3}"}, "publication_decade": {"type": "string", "description": "Decade the document was originally published", "pattern": "[12][0-9]{3}-[12][0-9]{3}"}, "number_of_pages": {"type": "number", "description": "Number of pages in this work (i.e. the number of images in the gt_workspace)"}, "layout": {"type": "string", "enum": ["simple", "complex"]}, "fonts": {"type": "array", "items": {"type": "string", "enum": ["antiqua", "fraktur", "ancient_greek", "hebrew"]}}}}, "provenance": {"type": "object", "description": "Information on which tools in which version were used in determining metrics", "properties": {"parameters": {"type": "object", "description": "Parameters passed to the evaluation processor"}}}}}, "EvaluationReport": {"type": "object", "additionalProperties": false, "description": "The metrics measured for this document", "properties": {"document_wide": {"type": "object", "description": "Document-wide metrics", "properties": {"$ref": "#$defs/EvaluationMetrics"}}, "by_page": {"type": "array", "description": "Metrics page-by-page", "items": {"type": "object", "allOf": [{"properties": {"page_id": {"type": "string", "description": "PAGE ID"}}}, {"properties": {"$ref": "#$defs/EvaluationMetrics"}}]}}}}, "EvaluationMetrics": {"cer": {"description": "CER calculated over the text of a whole page (in by_page) or combined text of all pages (in document_wide)"}, "cer_mean": {"description": "Arithmetic mean of the page-wise CER (in document_wide) or regions on a page (in by_page)"}, "cer_median": {"description": "Median of the page-wise CER (in document_wide) or regions on a page (in by_page)"}, "cer_range": {"type": "array", "minItems": 2, "maxItems": 2, "items": {"type": "number", "description": "Minimum and maximum of CER calculated over the text of a whole page (in by_page) or combined text of all pages (in document_wide)"}}, "cer_standard_deviation": {"description": "Standard deviation the page-wise CER (in document_wide) or regions on a page (in by_page)"}, "wer": {"description": "CER calculated over the text of a whole page (in by_page) or combined text of all pages (in document_wide)"}, "wall_time": {"description": "Actual time needed for processing workflow"}, "cpu_time": {"description": "Cumulative CPU time used for processing workflow"}, "pages_per_minute": {"description": "Number of pages processed per minute"}}}}

ocrd_eval.schema.yml

@@ -0,0 +1,194 @@
+$schema: https://json-schema.org/draft/2019-09/schema
+$id: https://ocr-d.de/en/spec/ocrd_eval.schema.json
+
+title: A list of evaluations for OCR-D
+description: >
+  - All references to URL are JSON-LD-like objects with at least an `@id`
+    property referencing the URL and `label` for a human-readable label to be
+    used in the UI
+type: array
+items:
+  required: ['@id', 'label', 'metadata', 'evaluation']
+  unevaluatedProperties: false
+  allOf:
+    - { '$ref': '#/$defs/LabeledUrl' }
+    - properties:
+        metadata: { '$ref': '#/$defs/EvaluationMetadata' }
+        evaluation: { '$ref': '#/$defs/EvaluationReport' }
+
+# Reusable definitions
+$defs:
+
+  LabeledUrl:
+    type: object
+    required: ['@id']
+    properties:
+      '@id':
+        type: string
+        format: uri
+        description: URL of the thing
+      label:
+        type: string
+        description: Description of the thing for UI purposes
+
+  EvaluationMetadata:
+    type: object
+    title: Metadata about one evaluation
+    additionalProperties: false
+    description: >
+      EvaluationMetadata contains all the info on how an EvaluationReport came to be.
+
+      There are two OCR-D *workflows* involved:
+        - ocr_workflow: The workflow which produced the OCR results to evaluate
+        - eval_workflow: The workflow run to evaluate OCR and GT
+
+      There are three OCR-D *workspaces* involved:
+        - gt_workspace: The workspace containing the GT
+        - ocr_workspace: The workspace containing the OCR results from ocr_workflow
+        - eval_workspace: The workspace on which the eval_workflow was run
+
+    required:
+      - ocr_workflow
+      - ocr_workspace
+      - eval_workflow
+      - eval_workspace
+      - gt_workspace
+      - document_metadata
+
+    properties:
+
+      ocr_workflow:
+        allOf:  [{ '$ref': '#/$defs/LabeledUrl' }]
+        description: The OCR-D workflow that produced the ocr_workspace
+
+      ocr_workspace:
+        allOf:  [{ '$ref': '#/$defs/LabeledUrl' }]
+        description: The workspace containing the OCR
+
+      eval_workflow:
+        allOf:  [{ '$ref': '#/$defs/LabeledUrl' }]
+        description: The OCR-D workflow that produced the eval_workspace
+
+      eval_workspace:
+        allOf:  [{ '$ref': '#/$defs/LabeledUrl' }]
+        description: The workspace containing the evaluation results
+
+      gt_workspace:
+        allOf:  [{ '$ref': '#/$defs/LabeledUrl' }]
+        description: The workspace containing the GT
+
+      workflow_steps:
+        type: object
+        description: Human readable description of the individual steps in the workflow (for UI)
+        patternProperties:
+          '^[0-9]+$':
+            type: string
+            description: Description of this workflow step
+
+      workflow_model:
+        type: string
+        description: Human readable name of the main model used for recognition in the OCR workflow (for UI)
+
+      eval_tool:
+        type: string
+        description: Human readable name and version of evaluation tool used (for UI)
+
+      document_metadata:
+        type: object
+        title: Bibliographical and typographical metadata about the work to be evaluated
+        properties:
+
+          publication_year:
+            type: number
+            description: Year the document was originally published
+
+          publication_century:
+            type: string
+            description: Century the document was originally published
+            pattern: '[12][0-9]{3}-[12][0-9]{3}'
+
+          publication_decade:
+            type: string
+            description: Decade the document was originally published
+            pattern: '[12][0-9]{3}-[12][0-9]{3}'
+
+          number_of_pages:
+            type: number
+            description: Number of pages in this work (i.e. the number of images in the gt_workspace)
+
+          layout:
+            type: string
+            enum: ['simple', 'complex']
+
+          fonts:
+            type: array
+            items:
+              type: string
+              enum: ['antiqua', 'fraktur', 'ancient_greek', 'hebrew']
+
+      provenance:
+        type: object
+        description: Information on which tools in which version were used in determining metrics
+        properties:
+          parameters:
+            type: object
+            description: Parameters passed to the evaluation processor
+
+  EvaluationReport:
+    type: object
+    additionalProperties: false
+    description: The metrics measured for this document
+    properties:
+      document_wide:
+        type: object
+        description: Document-wide metrics
+        properties: { $ref: '#$defs/EvaluationMetrics' }
+      by_page:
+        type: array
+        description: Metrics page-by-page
+        items:
+          type: object
+          allOf:
+            - properties:
+                page_id:
+                  type: string
+                  description: PAGE ID
+            - properties: { $ref: '#$defs/EvaluationMetrics' }
+
+  EvaluationMetrics:
+
+    cer:
+      description: CER calculated over the text of a whole page (in by_page) or combined text of all pages (in document_wide)
+
+#  To be implemented in the future.
+#    cer_mean:
+#      description: Arithmetic mean of the page-wise CER (in document_wide) or regions on a page (in by_page)
+#
+#    cer_median:
+#      description: Median of the page-wise CER (in document_wide) or regions on a page (in by_page)
+
+    cer_range:
+      type: array
+      minItems: 2
+      maxItems: 2
+      items:
+        type: number
+        description: Minimum and maximum of CER calculated over the text of a whole page (in by_page) or combined text of all pages (in document_wide)
+
+#  To be implemented in the future.
+#    cer_standard_deviation:
+#      description: Standard deviation the page-wise CER (in document_wide) or regions on a page (in by_page)
+
+    wer:
+      description: CER calculated over the text of a whole page (in by_page) or combined text of all pages (in document_wide)
+
+    wall_time:
+      description: Actual time needed for processing workflow
+
+    cpu_time:
+      description: Cumulative CPU time used for processing workflow
+
+    pages_per_minute:
+      description: Number of pages processed per minute
+
+

ocrd_tool.md

@@ -113,6 +113,7 @@ properties:
           parameters:
             description: Object describing the parameters of a tool. Keys are parameter names, values sub-schemas.
             type: object
+            default: {}
             patternProperties:
               ".*":
                 type: object