[ai-form-recognizer] Selection mark and other new output fields in 2.1-preview.1 (Azure#12540)

* [ai-form-recognizer] Add support for selection mark and other new fields

* Added simple tests for selection marks

* Recorded new tests

* Modified samples to show selection marks in pages

* Removed it.only that I left in

* Made array concat reduce more explicit

* Addressed feedback

* Made flattening composed results simpler

* Added a test for selection mark values, updated test-resources.json

* Addressed more feedback

* Fixed typo in test-resources

* Updated test-resources to use new container

Author: witemple-msft

sdk/formrecognizer/ai-form-recognizer/CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## 3.1.0-beta.1 (Unreleased)
 
+- Added a `formTypeConfidence` property to `RecognizedForm` indicating the model's confidence in determining the correct form type (and therefore the correct model to use) during recognition.
+- Added a `properties` field to `CustomFormModelInfo` that may optionally contain extra properties. Currently, the only property is `isComposedModel` which will indicate whether the model is a composed model or a single trained model.
+- Added a `modelId` field to the `CustomFormSubmodel`, `TrainingDocumentInfo`, and `RecognizedForm` types containing the ID of the exact model that they are associated with (for example, in the context of a composed model, the `modelId` field can determine which specific component model is associated with the submodel, training document, or recognized form).
+- Added support for selection marks in form fields. In addition to the previously-existing variants of `FormField`, custom models can now return fields with `valueType: "selectionMark"` and their `value` will be the state of the selection mark.
+- Added a new page element `FormSelectionMark` that represents marks on a page that can be selected (such as checkboxes and radio buttons). The `selectionMarks` field of `FormPage` contains the selection marks that were recognized in the page. A selection mark has a state value that is either "checked" or "unchecked."
 - Made optimizations to the long-running operation infrastructure that should result in faster and more memory-efficient polling for results of custom form recognition, receipt recognition, and business card recognition.
 - Added an option for specifying the locale of a document to receipt and business card methods through the `locale` property of the options bag.
 - Added support for Business Card recognition through the `beginRecognizeBusinessCards` and `beginRecognizeBusinessCardsFromUrl` methods, which mirror their receipt counterparts.

sdk/formrecognizer/ai-form-recognizer/karma.conf.js

@@ -60,6 +60,7 @@ module.exports = function(config) {
       "FORM_RECOGNIZER_API_KEY",
       "FORM_RECOGNIZER_TRAINING_CONTAINER_SAS_URL",
       "FORM_RECOGNIZER_TESTING_CONTAINER_SAS_URL",
+      "FORM_RECOGNIZER_SELECTION_MARK_STORAGE_CONTAINER_SAS_URL",
       "FORM_RECOGNIZER_TARGET_RESOURCE_ID",
       "FORM_RECOGNIZER_TARGET_RESOURCE_REGION",
       "AZURE_CLIENT_ID",

sdk/formrecognizer/ai-form-recognizer/recordings/node/aad_formrecognizerclient_nodejs_only_content_analysis/recording_with_selection_marks.js

@@ -111,16 +111,23 @@ export interface CustomFormModelField {
 export interface CustomFormModelInfo {
     modelId: string;
     modelName?: string;
+    properties?: CustomFormModelProperties;
     status: ModelStatus;
     trainingCompletedOn: Date;
     trainingStartedOn: Date;
 }
 
+// @public
+export interface CustomFormModelProperties {
+    isComposedModel?: boolean;
+}
+
 // @public
 export interface CustomFormSubmodel {
     accuracy?: number;
     fields: Record<string, CustomFormModelField>;
     formType: string;
+    modelId?: string;
 }
 
 // @public
@@ -138,13 +145,13 @@ export interface FieldData {
 export type FormContentType = "application/pdf" | "image/jpeg" | "image/png" | "image/tiff";
 
 // @public
-export type FormElement = FormWord | FormLine;
+export type FormElement = FormWord | FormLine | FormSelectionMark;
 
 // @public
 export interface FormElementCommon {
     boundingBox: Point2D[];
     pageNumber: number;
-    text: string;
+    text?: string;
 }
 
 // @public
@@ -177,6 +184,9 @@ export type FormField = {
 } | {
     value?: Record<string, FormField>;
     valueType?: "object";
+} | {
+    value?: SelectionMarkState;
+    valueType?: "selectionMark";
 });
 
 // @public
@@ -188,6 +198,7 @@ export interface FormFieldsReport {
 // @public
 export interface FormLine extends FormElementCommon {
     kind: "line";
+    text: string;
     words: FormWord[];
 }
 
@@ -204,6 +215,7 @@ export interface FormPage {
     height: number;
     lines?: FormLine[];
     pageNumber: number;
+    selectionMarks?: FormSelectionMark[];
     tables?: FormTable[];
     textAngle: number;
     unit: LengthUnit;
@@ -254,6 +266,13 @@ export interface FormRecognizerOperationOptions extends OperationOptions {
 // @public
 export type FormRecognizerRequestBody = Blob | ArrayBuffer | ArrayBufferView | NodeJS.ReadableStream;
 
+// @public
+export interface FormSelectionMark extends FormElementCommon {
+    confidence?: number;
+    kind: "selectionMark";
+    state: SelectionMarkState;
+}
+
 // @public
 export interface FormTable {
     cells: FormTableCell[];
@@ -295,6 +314,7 @@ export class FormTrainingClient {
 export interface FormWord extends FormElementCommon {
     confidence?: number;
     kind: "word";
+    text: string;
 }
 
 // @public
@@ -401,6 +421,8 @@ export type RecognizeContentOptions = FormRecognizerOperationOptions;
 export interface RecognizedForm {
     fields: Record<string, FormField>;
     formType: string;
+    formTypeConfidence?: number;
+    modelId?: string;
     pageRange: FormPageRange;
     pages: FormPage[];
 }
@@ -424,9 +446,13 @@ export type RecognizeFormsOptions = FormRecognizerOperationOptions & {
 
 export { RestResponse }
 
+// @public
+export type SelectionMarkState = "selected" | "unselected" | string;
+
 // @public
 export interface TrainingDocumentInfo {
     errors: FormRecognizerError[];
+    modelId?: string;
     name: string;
     pageCount: number;
     status: TrainingStatus;

sdk/formrecognizer/ai-form-recognizer/recordings/node/aad_formrecognizerclient_nodejs_only_custom_forms/recording_with_selection_marks.js

@@ -14,11 +14,23 @@ const path = require("path");
 const dotenv = require("dotenv");
 dotenv.config();
 
+/**
+ * Make a string representing a bounding box.
+ */
+function boundingBoxToString(box) {
+  let out = "[";
+  for (const { x, y } of box) {
+    out += `(${x}, ${y}),`;
+  }
+  // Remove the last comma and add the closing bracket
+  return out.slice(0, -1) + "]";
+}
+
 async function main() {
   // You will need to set these environment variables or edit the following values
   const endpoint = process.env["FORM_RECOGNIZER_ENDPOINT"] || "<cognitive services endpoint>";
   const apiKey = process.env["FORM_RECOGNIZER_API_KEY"] || "<api key>";
-  const fileName = path.join(__dirname, "./assets/Form_1.jpg");
+  const fileName = path.join(__dirname, "./assets/selection_mark_form.pdf");
 
   if (!fs.existsSync(fileName)) {
     throw new Error(`Expecting file ${fileName} exists`);
@@ -35,14 +47,19 @@ async function main() {
   }
 
   for (const page of pages) {
-    console.log(
-      `Page ${page.pageNumber}: width ${page.width} and height ${page.height} with unit ${page.unit}`
-    );
-    for (const table of page.tables) {
+    console.log(`- Page number: ${page.pageNumber}`);
+    console.log("  Tables:");
+    for (const table of page.tables || []) {
+      console.log(`  - Table (${table.rowCount} x ${table.columnCount})`);
       for (const cell of table.cells) {
-        console.log(`cell [${cell.rowIndex},${cell.columnIndex}] has text ${cell.text}`);
+        console.log(`    cell (${cell.rowIndex},${cell.columnIndex}) ${cell.text}`);
       }
     }
+    console.log("  Selection Marks:");
+    for (const mark of page.selectionMarks || []) {
+      const box = boundingBoxToString(mark.boundingBox);
+      console.log(`  - ${mark.state} @${box} (${mark.confidence} confidence)`);
+    }
   }
 }
 

sdk/formrecognizer/ai-form-recognizer/recordings/node/api_key_formrecognizerclient_nodejs_only_content_analysis/recording_with_selection_marks.js

@@ -17,6 +17,18 @@ const path = require("path");
 const dotenv = require("dotenv");
 dotenv.config();
 
+/**
+ * Make a string representing a bounding box.
+ */
+function boundingBoxToString(box) {
+  let out = "[";
+  for (const { x, y } of box) {
+    out += `(${x}, ${y}),`;
+  }
+  // Remove the last comma and add the closing bracket
+  return out.slice(0, -1) + "]";
+}
+
 async function main() {
   // You will need to set these environment variables or edit the following values
   const endpoint = process.env["FORM_RECOGNIZER_ENDPOINT"] || "<cognitive services endpoint>";
@@ -42,23 +54,29 @@ async function main() {
 
   console.log("Forms:");
   for (const form of forms || []) {
-    console.log(`${form.formType}, page range: ${form.pageRange}`);
-    console.log("Pages:");
+    console.log(`- ${form.formType}, page range: ${form.pageRange}`);
+    console.log("  Pages:");
     for (const page of form.pages || []) {
-      console.log(`Page number: ${page.pageNumber}`);
-      console.log("Tables");
+      console.log(`  - Page number: ${page.pageNumber}`);
+      console.log("    Tables:");
       for (const table of page.tables || []) {
+        console.log(`    - (${table.rowCount} x ${table.columnCount}`);
         for (const cell of table.cells) {
-          console.log(`cell (${cell.rowIndex},${cell.columnIndex}) ${cell.text}`);
+          console.log(`      cell (${cell.rowIndex},${cell.columnIndex}) ${cell.text}`);
         }
       }
+      console.log("    Selection Marks:");
+      for (const mark of page.selectionMarks || []) {
+        const box = boundingBoxToString(mark.boundingBox);
+        console.log(`    - ${mark.state} @(${box}) (${mark.confidence} confidence)`);
+      }
     }
 
-    console.log("Fields:");
+    console.log("  Fields:");
     for (const [fieldName, field] of Object.entries(form.fields)) {
       // each field is of type FormField
       console.log(
-        `Field '${fieldName}' has value '${field.value}' with a confidence score of ${field.confidence}`
+        `  - Field '${fieldName}' has value '${field.value}' with a confidence score of ${field.confidence}`
       );
     }
   }