Merge pull request #6745 from saffronsnail/accessibility-tests

Add accessibility sniffs to pageslayout tests

.circleci/config.yml

@@ -169,6 +169,13 @@ jobs:
             DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" \
               make validate-test-html || true
 
+      - run:
+          name: Validate accessibility (informational)
+          command: |
+            fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}')
+            DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" \
+              make accessibility-summary || true
+
       - store_test_results:
           path: ~/project/test-results
 

Makefile

@@ -282,6 +282,12 @@ validate-test-html:
 	@$(DEVSHELL) html5validator tests/functional/pageslayout/html
 	@echo
 
+.PHONY: accessibility-summary
+accessibility-summary:
+	@echo "███ Processing accessibility results..."
+	@$(DEVSHELL) $(SDBIN)/summarize-accessibility-info
+	cat securedrop/tests/functional/pageslayout/accessibility-info/summary.txt
+
 .PHONY: docker-vnc
 docker-vnc:  ## Open a VNC connection to a running Docker instance.
 	@echo "███ Opening VNC connection to dev container..."

devops/docker/CIDockerfile

@@ -7,8 +7,9 @@ ENV DOCKER_SHA256_x86_64 340e0b5a009ba70e1b644136b94d13824db0aeb52e09071410f35a9
 
 RUN apt-get update && \
     apt-get install -y flake8 make virtualenv ccontrol libpython2.7-dev \
-            libffi-dev libssl-dev libyaml-dev python-pip curl git &&\
-    apt-get clean
+            libffi-dev libssl-dev libyaml-dev python-pip curl git npm &&\
+    apt-get clean && \
+    npm --global install [email protected]
 
 RUN curl -L -o /tmp/docker-${DOCKER_VER}.tgz https://get.docker.com/builds/Linux/x86_64/docker-${DOCKER_VER}.tgz; \
 	echo "${DOCKER_SHA256_x86_64} /tmp/docker-${DOCKER_VER}.tgz" | sha256sum -c -; \

securedrop/bin/summarize-accessibility-info

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -euo pipefail
+
+export PATH="/opt/venvs/securedrop-app-code/bin:$PATH"
+
+cd ..
+export REPOROOT="${REPOROOT:-$PWD}"
+git config --global --add safe.directory "$REPOROOT"
+cd "${REPOROOT}/securedrop"
+
+PYTHONPATH=".:${PYTHONPATH:-}"
+export PYTHONPATH
+
+python3 -c 'from tests.functional.pageslayout.accessibility import summarize_accessibility_results; summarize_accessibility_results()'

securedrop/dockerfiles/focal/python3/Dockerfile

@@ -7,7 +7,8 @@ RUN apt-get update  && apt-get install -y \
     # cached along with everything else too.
     default-jdk \
     libasound2 libdbus-glib-1-2 libgtk2.0-0 libfontconfig1 libxrender1 \
-    libcairo-gobject2 libgtk-3-0 libstartup-notification0
+    libcairo-gobject2 libgtk-3-0 libstartup-notification0 npm && \
+    npm install --global [email protected]
 
 # Current versions of the test browser software. Tor Browser is based
 # on a specific version of Firefox, noted in Help > About Tor Browser.

securedrop/tests/functional/pageslayout/.gitignore

@@ -1,2 +1,3 @@
+accessibility-info
 html
 screenshots

securedrop/tests/functional/pageslayout/accessibility.py

@@ -0,0 +1,191 @@
+import os
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+from typing import Dict, List, Union
+
+from selenium.webdriver.firefox.webdriver import WebDriver
+
+_ACCESSIBILITY_DIR = (Path(__file__).parent / "accessibility-info").absolute()
+
+_HTMLCS_RUNNER_CODE = """
+var all_messages = [];
+console.log = msg => {
+    all_messages.push(msg);
+};
+
+HTMLCS_RUNNER.run('WCAG2AAA');
+return all_messages;
+"""
+
+
+class MessageType(Enum):
+    ERROR = 1
+    WARNING = 2
+    NOTICE = 3
+
+
+@dataclass
+class Message:
+    """Contains all of the information in a message emitted by HTML CodeSniffer."""
+
+    principle_id: str
+    message: str
+    message_type: MessageType
+    responsible_html: str
+    selector: str
+    element_type: str
+
+    @staticmethod
+    def from_output(output: str) -> "Message":
+        """Parses the output of htmlcs and returns an instance containing all data.
+
+        No processing is performed for flexibility.
+
+        Example message, post-split (note: contents of index 4 contains no newlines, but I had to
+        split it to keep the linter happy):
+
+        0: [HTMLCS] Error
+        1: WCAG2AAA.Principle1.Guideline1_3.1_3_1_AAA.G141
+        2: h2
+        3: #security-level-heading
+        4: The heading structure is not logically nested. This h2 element appears to be the
+           primary document heading, so should be an h1 element.
+        5: <h2 id="security-level-heading" hidden="">...</h2>
+        """
+
+        fields = output.split("|")
+
+        if "Error" in fields[0]:
+            message_type = MessageType.ERROR
+        elif "Warning" in fields[0]:
+            message_type = MessageType.WARNING
+        elif "Notice" in fields[0]:
+            message_type = MessageType.NOTICE
+
+        return Message(
+            message_type=message_type,
+            principle_id=fields[1],
+            element_type=fields[2],
+            selector=fields[3],
+            message=fields[4],
+            responsible_html=fields[5],
+        )
+
+    def __format__(self, _spec: str) -> str:
+        newline = "\n"
+        return f"""
+{self.message_type}: {self.principle_id}
+    {self.message}
+
+    html:
+        {self.responsible_html.replace(newline, f"{newline}        ")}
+        """
+
+
+def sniff_accessibility_issues(driver: WebDriver, locale: str, test_name: str) -> None:
+    """Runs accessibility sniffs on the driver's current page.
+
+    This function is responsible for injecting HTML CodeSniffer into the current page and writing
+    the results to a file. This way, test functions can focus on the setup required to navigate to
+    a particular URL (for example, logging in to get to the messages page).
+    """
+
+    # 1. Retrieve/compute required data
+    with open(f"/usr/local/lib/node_modules/html_codesniffer/build/HTMLCS.js") as htmlcs:
+        html_codesniffer = htmlcs.read()
+
+    errors_dir = _ACCESSIBILITY_DIR / locale / "errors"
+    errors_dir.mkdir(parents=True, exist_ok=True)
+
+    reviews_dir = _ACCESSIBILITY_DIR / locale / "reviews"
+    reviews_dir.mkdir(parents=True, exist_ok=True)
+
+    # 2. Do the thing
+    raw_messages = driver.execute_script(html_codesniffer + _HTMLCS_RUNNER_CODE)
+
+    # 3. Organize the data
+    messages: Dict[str, List[Message]] = {
+        "machine-verified": [],
+        "human-reviewed": [],
+    }
+
+    for message in map(Message.from_output, raw_messages[:-1]):  # last message is effectievly EOF
+        if message.message_type == MessageType.ERROR:
+            messages["machine-verified"].append(message)
+        else:
+            messages["human-reviewed"].append(message)
+
+    # 4. Report the data
+    # Note: it is useful to create empty files when there are no results to simplify the logic for
+    #       summarizing the results, implemented in `summarize_accessibility_results`.
+    with open(errors_dir / f"{test_name}.txt", "w") as error_file:
+        for message in messages["machine-verified"]:
+            error_file.write(f"{message}")
+
+    with open(reviews_dir / f"{test_name}.txt", "w") as review_file:
+        for message in messages["human-reviewed"]:
+            review_file.write(f"{message}")
+
+
+def summarize_accessibility_results() -> None:
+    """Creates a file containing summary information about the result of accessiblity sniffing
+
+    Note: This does not automatically run as part of the test suite, use
+          `make accessibility-summary` instead.
+    """
+
+    try:
+        summary: Dict[str, Dict[str, Dict[str, Union[int, bool]]]] = {}
+
+        # since `sniff_accessibility_issues` creates empty files, all locale/type combinations will
+        # contain the same set of files; getting filenames from en_US/reviews instead of, say,
+        # ar/errors is arbitrary and sufficient
+        for out_filename in os.listdir(_ACCESSIBILITY_DIR / "en_US" / "reviews"):
+            summary[out_filename] = {
+                "reviews": {"count": 0, "locale_differs": False},
+                "errors": {"count": 0, "locale_differs": False},
+            }
+
+            # collect all of the relevant data
+            for message_type in ["reviews", "errors"]:
+                outputs: Dict[str, Dict[str, List[str]]] = {}
+
+                for locale in ["en_US", "ar"]:
+                    outputs[locale] = {}
+                    with open(
+                        _ACCESSIBILITY_DIR / locale / message_type / out_filename
+                    ) as out_file:
+                        # Only look at lines specifying the message (including the exact WCAG error
+                        # code); this is exactly correct for the count, and approximately correct
+                        # for comparing locales. If the order of the errors differs, or if there
+                        # are a different number of any kind of error, this approximation will catch
+                        # it.
+                        outputs[locale][message_type] = [
+                            line for line in out_file.readlines() if "MessageType." in line
+                        ]
+
+                summary[out_filename][message_type]["count"] = len(outputs["en_US"][message_type])
+                summary[out_filename][message_type]["locale_differs"] = (
+                    outputs["en_US"][message_type] != outputs["ar"][message_type]
+                )
+
+        # save the data to a convenient file
+        with open(_ACCESSIBILITY_DIR / "summary.txt", "w") as summary_file:
+            for name in sorted(summary.keys()):
+                summary_file.write(name + ":\n")
+                for message_type in ["errors", "reviews"]:
+                    summary_file.write(
+                        f"\t{message_type}: {summary[name][message_type]['count']}\n"
+                    )
+                    if summary[name][message_type]["locale_differs"]:
+                        summary_file.write(f"\t        NOTE: {message_type} differ by locale\n")
+
+                summary_file.write("\n")
+
+    # this should only happen if the pageslayout tests have not created the raw output files
+    except FileNotFoundError:
+        print(
+            f"ERROR: Run `make test TESTFILES={os.path.dirname(_ACCESSIBILITY_DIR)}` before "
+            "running `make accessibility-summary`"
+        )

securedrop/tests/functional/pageslayout/test_journalist.py

@@ -17,7 +17,7 @@
 #
 import pytest
 from tests.functional.app_navigators.journalist_app_nav import JournalistAppNavigator
-from tests.functional.pageslayout.utils import list_locales, save_screenshot_and_html
+from tests.functional.pageslayout.utils import list_locales, save_static_data
 
 
 @pytest.mark.parametrize("locale", list_locales())
@@ -33,7 +33,7 @@ def test_login_index_and_edit(self, locale, sd_servers, firefox_web_driver):
             accept_languages=locale_with_commas,
         )
         journ_app_nav.driver.get(f"{sd_servers.journalist_app_base_url}/login")
-        save_screenshot_and_html(journ_app_nav.driver, locale, "journalist-login")
+        save_static_data(journ_app_nav.driver, locale, "journalist-login")
 
         # And they log into the app and are an admin
         assert sd_servers.journalist_is_admin
@@ -42,19 +42,17 @@ def test_login_index_and_edit(self, locale, sd_servers, firefox_web_driver):
             password=sd_servers.journalist_password,
             otp_secret=sd_servers.journalist_otp_secret,
         )
-        save_screenshot_and_html(journ_app_nav.driver, locale, "journalist-index_no_documents")
+        save_static_data(journ_app_nav.driver, locale, "journalist-index_no_documents")
         # The documentation uses an identical screenshot with a different name:
         # https://github.com/freedomofpress/securedrop-docs/blob/main/docs/images/manual
         # /screenshots/journalist-admin_index_no_documents.png
         # So we take the same screenshot again here
         # TODO(AD): Update the documentation to use a single screenshot
-        save_screenshot_and_html(
-            journ_app_nav.driver, locale, "journalist-admin_index_no_documents"
-        )
+        save_static_data(journ_app_nav.driver, locale, "journalist-admin_index_no_documents")
 
         # Take a screenshot of the edit account page
         journ_app_nav.journalist_visits_edit_account()
-        save_screenshot_and_html(journ_app_nav.driver, locale, "journalist-edit_account_user")
+        save_static_data(journ_app_nav.driver, locale, "journalist-edit_account_user")
 
     def test_index_entered_text(self, locale, sd_servers, firefox_web_driver):
         # Given an SD server
@@ -73,7 +71,7 @@ def test_index_entered_text(self, locale, sd_servers, firefox_web_driver):
             otp_secret="2HGGVF5VPHWMCAYQ",
             should_submit_login_form=False,
         )
-        save_screenshot_and_html(journ_app_nav.driver, locale, "journalist-index_with_text")
+        save_static_data(journ_app_nav.driver, locale, "journalist-index_with_text")
 
     def test_index_with_submission_and_select_documents(
         self, locale, sd_servers_with_submitted_file, firefox_web_driver
@@ -93,20 +91,20 @@ def test_index_with_submission_and_select_documents(
             password=sd_servers_with_submitted_file.journalist_password,
             otp_secret=sd_servers_with_submitted_file.journalist_otp_secret,
         )
-        save_screenshot_and_html(journ_app_nav.driver, locale, "journalist-index")
+        save_static_data(journ_app_nav.driver, locale, "journalist-index")
         # The documentation uses an identical screenshot with a different name:
         # https://github.com/freedomofpress/securedrop-docs/blob/main/docs/images/manual
         # /screenshots/journalist-index_javascript.png
         # So we take the same screenshot again here
         # TODO(AD): Update the documentation to use a single screenshot
-        save_screenshot_and_html(journ_app_nav.driver, locale, "journalist-index_javascript")
+        save_static_data(journ_app_nav.driver, locale, "journalist-index_javascript")
 
         # Take a screenshot of the source's page
         journ_app_nav.journalist_selects_the_first_source()
         checkboxes = journ_app_nav.get_submission_checkboxes_on_current_page()
         for checkbox in checkboxes:
             checkbox.click()
-        save_screenshot_and_html(
+        save_static_data(
             journ_app_nav.driver, locale, "journalist-clicks_on_source_and_selects_documents"
         )
 
@@ -115,7 +113,7 @@ def test_index_with_submission_and_select_documents(
             reply_content="Thanks for the documents."
             " Can you submit more information about the main program?"
         )
-        save_screenshot_and_html(journ_app_nav.driver, locale, "journalist-composes_reply")
+        save_static_data(journ_app_nav.driver, locale, "journalist-composes_reply")
 
     def test_fail_to_visit_admin(self, locale, sd_servers, firefox_web_driver):
         # Given an SD server
@@ -128,9 +126,7 @@ def test_fail_to_visit_admin(self, locale, sd_servers, firefox_web_driver):
         )
         # Take a screenshot of them trying to force-browse to the admin interface
         journ_app_nav.driver.get(f"{sd_servers.journalist_app_base_url}/admin")
-        save_screenshot_and_html(
-            journ_app_nav.driver, locale, "journalist-code-fail_to_visit_admin"
-        )
+        save_static_data(journ_app_nav.driver, locale, "journalist-code-fail_to_visit_admin")
 
     def test_fail_login(self, locale, sd_servers, firefox_web_driver):
         # Given an SD server
@@ -149,10 +145,10 @@ def test_fail_login(self, locale, sd_servers, firefox_web_driver):
             otp_secret="2HGGVF5VPHWMCAYQ",
             should_submit_login_form=True,
         )
-        save_screenshot_and_html(journ_app_nav.driver, locale, "journalist-code-fail_login")
+        save_static_data(journ_app_nav.driver, locale, "journalist-code-fail_login")
         # The documentation uses an identical screenshot with a different name:
         # https://github.com/freedomofpress/securedrop-docs/blob/main/docs/images/manual
         # /screenshots/journalist-code-fail_login_many.png
         # So we take the same screenshot again here
         # TODO(AD): Update the documentation to use a single screenshot
-        save_screenshot_and_html(journ_app_nav.driver, locale, "journalist-code-fail_login_many")
+        save_static_data(journ_app_nav.driver, locale, "journalist-code-fail_login_many")