ABDM changes for consuming APIs in the mobile app.

CONTRIBUTING.rst

@@ -1,8 +1,8 @@
-==========================
-Contributing to CommCareHQ
-==========================
+===========================
+Contributing to CommCare HQ
+===========================
 
-CommCareHQ is primarily developed by `Dimagi`_, but we welcome contributions.
+CommCare HQ is primarily developed by `Dimagi`_, but we welcome contributions.
 
 Code Contributions
 ------------------

corehq/apps/README.rst

@@ -1,5 +1,5 @@
-Django Apps in CommCareHQ
-#########################
+Django Apps in CommCare HQ
+##########################
 
 Most CommCare HQ functionality is contained in a django app.
 A few areas are contained in ``corehq`` or ``corehq/ex-submodules``,

corehq/apps/accounting/forms.py

@@ -579,23 +579,18 @@ def __init__(self, subscription, account_id, web_user, *args, **kwargs):
             self.fields['account'].initial = subscription.account.id
             account_field = hqcrispy.B3TextField(
                 'account',
-                '<a href="%(account_url)s">%(account_name)s</a>' % {
-                    'account_url': reverse(ManageBillingAccountView.urlname,
-                                           args=[subscription.account.id]),
-                    'account_name': subscription.account.name,
-                }
+                format_html('<a href="{}">{}</a>',
+                    reverse(ManageBillingAccountView.urlname, args=[subscription.account.id]),
+                    subscription.account.name)
             )
 
             self.fields['plan_version'].initial = subscription.plan_version.id
             plan_version_field = hqcrispy.B3TextField(
                 'plan_version',
-                '<a href="%(plan_version_url)s">%(plan_name)s</a>' % {
-                    'plan_version_url': reverse(
-                        SoftwarePlanVersionView.urlname,
-                        args=[subscription.plan_version.plan.id, subscription.plan_version_id]
-                    ),
-                    'plan_name': subscription.plan_version,
-                },
+                format_html('<a href="{}">{}</a>',
+                    reverse(SoftwarePlanVersionView.urlname,
+                        args=[subscription.plan_version.plan.id, subscription.plan_version_id]),
+                    subscription.plan_version)
             )
             self.fields['plan_edition'].initial = subscription.plan_version.plan.edition
             plan_edition_field = hqcrispy.B3TextField(
@@ -610,11 +605,9 @@ def __init__(self, subscription, account_id, web_user, *args, **kwargs):
 
             domain_field = hqcrispy.B3TextField(
                 'domain',
-                '<a href="%(project_url)s">%(project_name)s</a>' % {
-                    'project_url': reverse(DefaultProjectSettingsView.urlname,
-                                           args=[subscription.subscriber.domain]),
-                    'project_name': subscription.subscriber.domain,
-                },
+                format_html('<a href="{}">{}</a>',
+                    reverse(DefaultProjectSettingsView.urlname, args=[subscription.subscriber.domain]),
+                    subscription.subscriber.domain)
             )
 
             self.fields['start_date'].initial = subscription.date_start.isoformat()

corehq/apps/accounting/templates/accounting/partials/confirm_plan_summary.html

@@ -113,7 +113,7 @@ <h4 class="text-center">
       {{ message.message }}
       <ul>
         {% for detail in message.details %}
-          <li>{{ detail|safe }}</li>
+          <li>{{ detail }}</li>
         {% endfor %}
       </ul>
     </div>

corehq/apps/accounting/templates/accounting/partials/invoice_table.html

@@ -10,7 +10,7 @@
 <div style="overflow: auto">
   <table class="table table-striped datatable">
     <thead>
-    {{ headers.render_html|safe }}
+    {{ headers.render_html }}
     </thead>
     <tbody>
     {% block tabular-body %}

corehq/apps/api/resources/messaging_event/filters.py

@@ -165,7 +165,7 @@ def _consumer(key, value):
 
 COMPOUND_FILTERS = [
     _get_date_filter_consumer("date"),
-    _get_date_filter_consumer("date_last_activity", "date_last_activity_computed"),
+    _get_date_filter_consumer("date_last_activity"),
 ]
 
 SIMPLE_FILTERS = {

corehq/apps/api/resources/messaging_event/serializers.py

@@ -10,7 +10,7 @@ def serialize_event(event):
         "id": event.id,
         "content_type": MessagingEvent.CONTENT_TYPE_SLUGS.get(event.content_type, "unknown"),
         "date": event.date.isoformat(),
-        "date_last_activity": event.date_last_activity_computed.isoformat(),
+        "date_last_activity": event.date_last_activity.isoformat(),
         "case_id": event.case_id,
         "domain": event.parent.domain,
         "error": _serialize_event_error(event),

corehq/apps/api/resources/messaging_event/utils.py

@@ -44,11 +44,7 @@ def get_order_by_field(request_params):
     if order_by not in ("date", "-date", "date_last_activity", "-date_last_activity"):
         raise BadRequest(_("No matching '{field_name}' field for ordering").format(field_name=order_by))
 
-    # rename to query field
-    return {
-        "date_last_activity": "date_last_activity_computed",
-        "-date_last_activity": "-date_last_activity_computed"
-    }.get(order_by, order_by)
+    return order_by
 
 
 @attrs

corehq/apps/api/resources/messaging_event/view.py

@@ -56,27 +56,4 @@ def _get_list(request):
 
 
 def _get_base_query(domain):
-    """The base query includes a 'date_last_activity_computed' field. This field
-    is calculated as:
-      Max(
-        event.date,
-        xform_session.modified_time,  # if it exists
-        Max(sms.date_modified),  # max for the current event
-        Max(email.date_modified)  # max for the current event
-      )
-  """
-    query = MessagingSubEvent.objects.select_related("parent").filter(parent__domain=domain)
-    newest_sms = (
-        SMS.objects.filter(messaging_subevent=OuterRef('pk'))
-        .order_by('-date_modified')
-        .values('date_modified')[:1]
-    )
-    newest_email = (
-       Email.objects.filter(messaging_subevent=OuterRef('pk'))
-       .order_by('-date_modified')
-       .values('date_modified')[:1]
-    )
-    query = query.annotate(date_last_activity_computed=Greatest(
-        'date', 'xforms_session__modified_time', Subquery(newest_sms), Subquery(newest_email)
-    ))
-    return query
+    return MessagingSubEvent.objects.select_related("parent").filter(parent__domain=domain)

corehq/apps/api/tests/test_es.py

@@ -1,39 +1,31 @@
 from django.test import SimpleTestCase
 
 from corehq.apps.es.tests.utils import es_test
-from corehq.apps.es.client import ElasticManageAdapter
-from corehq.apps.es.cases import ElasticCase
+from corehq.apps.es.client import manager
+from corehq.apps.es.cases import ElasticCase, case_adapter
 from corehq.util.es.elasticsearch import TransportError
 
 from ..es import CaseESView
 
 
 class ElasticCase2(ElasticCase):
-
-    type = "type2"
-    mapping = ElasticCase().mapping  # TODO: remove with transient_util
+    mapping = case_adapter.mapping
 
 
 @es_test
 class TestESView(SimpleTestCase):
 
     def setUp(self):
         super().setUp()
-        self.manager = ElasticManageAdapter()
-        self.cases = ElasticCase()
-        self._purge_indices()
-        self.manager.index_create(self.cases.index_name)
-        self.manager.index_put_mapping(self.cases.index_name, self.cases.type,
-                                       self.cases.mapping)
-        self.manager.index_put_alias(self.cases.index_name, CaseESView.es_alias)
-
-    def tearDown(self):
-        self._purge_indices()
-        super().tearDown()
+        self.cases = case_adapter
+        manager.index_create(self.cases.index_name)
+        self.addCleanup(self._purge_indices)
+        manager.index_put_mapping(self.cases.index_name, self.cases.type, self.cases.mapping)
+        manager.index_put_alias(self.cases.index_name, CaseESView.es_alias)
 
     def _purge_indices(self):
         try:
-            self.manager.index_delete(self.cases.index_name)
+            manager.index_delete(self.cases.index_name)
         except TransportError:
             # TransportError(404, 'index_not_found_exception', 'no such index')
             pass
@@ -63,8 +55,8 @@ def mk_doc(doc_id, location_id):
         self.assertEqual(doc_ny, to_dict(view.get_document(doc_id)))
 
         # index a doc with a new _type ('type2')
-        cases_type2 = ElasticCase2()
-        self.manager.index_put_mapping(cases_type2.index_name, cases_type2.type,
+        cases_type2 = ElasticCase2(case_adapter.index_name, "type2")
+        manager.index_put_mapping(cases_type2.index_name, cases_type2.type,
                                        cases_type2.mapping)
         doc_dc = mk_doc(doc_id, "DC")
         cases_type2.index(doc_dc, refresh=True)

corehq/apps/api/tests/test_messaging_event_api.py

@@ -1,6 +1,7 @@
 import json
 import urllib.parse
 from datetime import datetime, timedelta
+from operator import attrgetter
 from urllib.parse import urlencode
 
 from django.urls import reverse
@@ -81,15 +82,15 @@ def test_date_ordering(self):
 
     def test_date_last_activity_ordering(self):
         sms = _create_sms_messages(self.domain, 5, randomize=True)
-        sms[0].save()  # this should move this one to the end of the list
+        sms[0].messaging_subevent.save()  # this should move this one to the end of the list
 
         response = self._auth_get_resource(f'{self.list_endpoint}?order_by=date_last_activity')
         self.assertEqual(response.status_code, 200, response.content)
         ordered_data = json.loads(response.content)['objects']
         self.assertEqual(5, len(ordered_data))
         dates = [r['date_last_activity'] for r in ordered_data]
         expected_order = sms[1:] + [sms[0]]  # modification order
-        self.assertEqual(dates, [s.date_modified.isoformat() for s in expected_order])
+        self.assertEqual(dates, [s.messaging_subevent.date_last_activity.isoformat() for s in expected_order])
 
         response = self._auth_get_resource(f'{self.list_endpoint}?order_by=-date_last_activity')
         self.assertEqual(response.status_code, 200, response.content)
@@ -211,7 +212,7 @@ def test_case_rule(self):
             "case_id": None,
             "content_type": "sms",
             "date": "2016-01-01T12:00:00",
-            "date_last_activity": sms.date_modified.isoformat(),
+            "date_last_activity": event.date_last_activity.isoformat(),
             "domain": "qwerty",
             "error": None,
             "form": None,
@@ -258,11 +259,13 @@ def test_survey_sms(self):
         self.addCleanup(event.delete)  # cascades to subevent
         self.addCleanup(sms.delete)
 
+        subevent = event.subevents[0]
+
         expected = {
             "case_id": None,
             "content_type": "ivr-survey",
             "date": "2016-01-01T12:00:00",
-            "date_last_activity": sms.date_modified.isoformat(),
+            "date_last_activity": subevent.date_last_activity.isoformat(),
             "domain": "qwerty",
             "error": None,
             "form": {
@@ -315,7 +318,7 @@ def test_email(self):
             "case_id": None,
             "content_type": "email",
             "date": event.date.isoformat(),
-            "date_last_activity": email.date_modified.isoformat(),
+            "date_last_activity": event.date_last_activity.isoformat(),
             "domain": "qwerty",
             "error": None,
             "form": None,
@@ -367,7 +370,7 @@ def test_email_null_date_modified(self):
         data = json.loads(response.content)['objects']
         self.assertEqual(1, len(data))
         for result in data:
-            self.assertEqual(result["date_last_activity"], event.date.isoformat())
+            self.assertEqual(result["date_last_activity"], event.date_last_activity.isoformat())
             self.assertIsNone(result["messages"][0]["date_modified"])
 
     def test_cursor(self):
@@ -532,21 +535,18 @@ def test_email_date_last_activity_filtering(self):
             user = CommCareUser.create(self.domain.name, f"bob{i}", "123", None, None, email=f"bob{i}@email.com")
             self.addCleanup(user.delete, self.domain.name, deleted_by=None)
             user_ids.append(user.get_id)
-        events = make_email_event_for_test(self.domain.name, "test broadcast", user_ids)
-
-        emails = [
-            Email.objects.get(messaging_subevent=events[user_id]) for user_id in user_ids
-        ]
-        emails[0].save()  # update date_modified
-        expected_order = emails[1:] + [emails[0]]  # modification order
-        dates = [email.date_modified for email in expected_order]
+        events_by_user = make_email_event_for_test(self.domain.name, "test broadcast", user_ids)
+        events = list(sorted(events_by_user.values(), key=attrgetter('date_last_activity')))
+        events[0].save()  # update date_last_activity
+        expected_order = events[1:] + [events[0]]  # modification order
+        dates = [event.date_last_activity for event in expected_order]
 
         self._check_date_filtering_response({
             f"{self.field}.lt": dates[3].isoformat()
         }, [d.isoformat() for d in dates[:3]])
 
     def test_survey_date_last_activity_filter(self):
-        sessions = []
+        events = []
         messages = []
         for i in range(5):
             rule, xforms_session, event, sms = make_survey_sms_for_test(
@@ -556,26 +556,25 @@ def test_survey_date_last_activity_filter(self):
             self.addCleanup(xforms_session.delete)
             self.addCleanup(event.delete)  # cascades to subevent
             self.addCleanup(sms.delete)
-            sessions.append(xforms_session)
+            events.append(event.subevents[0])
             messages.append(sms)
 
         # update modified time for 1st session to move it to the end of the list
-        sessions[0].modified_time = datetime.utcnow()
-        sessions[0].save()
+        events[0].save()
 
-        # last_activity_date will be the max date so use sms dates except for the session that was updated
-        dates = [sms.date_modified for sms in messages[1:]] + [sessions[0].modified_time]  # modification order
+        expected_order = events[1:] + [events[0]]  # modification order
+        dates = [event.date_last_activity for event in expected_order]
 
         self._check_date_filtering_response({
             f"{self.field}.lt": dates[3].isoformat()
         }, [d.isoformat() for d in dates[:3]])
 
     def _setup_for_date_filter_test(self):
         results = _create_sms_messages(self.domain, 5, randomize=True)
-        results[0].save()  # update modification date
+        results[0].messaging_subevent.save()  # update modification date
         expected_order = results[1:] + [results[0]]  # modification order
         return list(
-            sms.date_modified for sms in expected_order
+            sms.messaging_subevent.date_last_activity for sms in expected_order
         )
 
 
@@ -592,7 +591,7 @@ def _serialized_messaging_event(sms):
         "id": sms.messaging_subevent_id,
         "content_type": "sms",
         "date": "2016-01-01T12:00:00",
-        "date_last_activity": sms.date_modified.isoformat() if sms.date_modified else "2016-01-01T12:00:00",
+        "date_last_activity": sms.messaging_subevent.date_last_activity.isoformat(),
         "case_id": None,
         "domain": "qwerty",
         "error": None,

corehq/apps/app_manager/fields.py

@@ -6,6 +6,7 @@
 from django import forms
 from django.http import Http404
 from django.urls import reverse
+from django.utils.safestring import mark_safe
 from django.utils.translation import gettext as _
 
 from memoized import memoized
@@ -98,7 +99,8 @@ def __init__(self, enable_raw=False, enable_registry=False, registry_permission_
                                                    widget=forms.Select(choices=source_choices))
 
         self.source_field = forms.ChoiceField(label=_('Data Source'), widget=forms.Select())
-        self.source_field.label = '<span data-bind="text: labelMap[sourceType()]"></span>'
+        self.source_field.label = mark_safe(  # nosec: no user input
+            '<span data-bind=\'text: labelMap[sourceType()]\'></span>')
 
         self.registry_slug_field = forms.ChoiceField(label=_('Data Registry'), widget=forms.HiddenInput,
                                                      required=False)

corehq/apps/app_manager/static/app_manager/js/details/case_claim.js

@@ -148,7 +148,10 @@ hqDefine("app_manager/js/details/case_claim", function () {
             var itemLists = get('js_options').item_lists;
             return _.map(
                 _.filter(itemLists, function (p) {
-                    return p.fixture_type === self.appearance();
+                    return (
+                        p.fixture_type === self.appearance()
+                        || (p.fixture_type === 'lookup_table_fixture' && self.appearance() === 'checkbox')
+                    );
                 }),
                 function (p) {
                     return {
@@ -300,15 +303,14 @@ hqDefine("app_manager/js/details/case_claim", function () {
             var uri = searchProperty.itemset.instance_uri;
             if (uri !== null && uri.includes("commcare-reports")) {
                 appearance = "report_fixture";
-            }
-            else {
+            } else {
                 appearance = "lookup_table_fixture";
             }
         }
         if (searchProperty.appearance === "address") {
             appearance = "address";
         }
-        if (["date", "daterange"].indexOf(searchProperty.input_) !== -1) {
+        if (["date", "daterange", "checkbox"].indexOf(searchProperty.input_) !== -1) {
             appearance = searchProperty.input_;
         }
         return appearance;