Merge branch 'master' into cs/SC-4053-add-metric-to-datadog

.prettierrc

@@ -0,0 +1,14 @@
+{
+    "plugins": ["prettier-plugin-jinja-template"],
+    "overrides": [
+        {
+            "files": [
+                "*.html"
+            ],
+            "options": {
+                "parser": "jinja-template"
+            }
+        }
+    ],
+    "tabWidth": 2
+}

corehq/apps/api/odata/tests/utils.py

@@ -150,6 +150,5 @@ def get_instance(cls, domain_name):
 
 def generate_api_key_from_web_user(web_user):
     api_key = HQApiKey.objects.get_or_create(user=web_user.get_django_user())[0]
-    api_key.key = api_key.generate_key()
     api_key.save()
     return api_key

corehq/apps/api/tests/core_api.py

@@ -473,7 +473,7 @@ def test_get_user(self):
         endpoint = "%s?%s" % (self.single_endpoint(self.user._id),
                               urlencode({
                                   "username": self.user.username,
-                                  "api_key": self.api_key.key
+                                  "api_key": self.api_key.plaintext_key
                               }))
         response = self.client.get(endpoint)
         self.assertEqual(response.status_code, 200)
@@ -501,7 +501,7 @@ def test_wrong_user_api_key(self):
         endpoint = "%s?%s" % (self.single_endpoint(self.user._id),
                               urlencode({
                                   "username": self.user.username,
-                                  "api_key": other_api_key.key
+                                  "api_key": other_api_key.plaintext_key
                               }))
         response = self.client.get(endpoint)
         self.assertEqual(response.status_code, 401)
@@ -710,7 +710,7 @@ def setUp(self):
         super().setUp()
         self.endpoint = "%s?%s" % (self.single_endpoint(self.user._id), urlencode({
             "username": self.user.username,
-            "api_key": self.api_key.key
+            "api_key": self.api_key.plaintext_key
         }))
 
     def test_throttle_allowlist(self):

corehq/apps/api/tests/test_auth.py

@@ -47,7 +47,7 @@ def _get_request_with_basic_auth(self, domain=None):
         )
 
     def _construct_api_auth_header(self, username, api_key):
-        return f'ApiKey {username}:{api_key.key}'
+        return f'ApiKey {username}:{api_key.plaintext_key}'
 
     def _construct_basic_auth_header(self, username, password):
         # https://stackoverflow.com/q/5495452/8207

corehq/apps/api/tests/utils.py

@@ -124,11 +124,11 @@ def _get_api_key_auth_headers(self, headers=None, username=None):
             return {}
 
         username = username or self.username
-        api_key = self.api_key.key
+        api_key = self.api_key.plaintext_key
         if username != self.username:
             web_user = WebUser.get_by_username(username)
             api_key, _ = HQApiKey.objects.get_or_create(user=WebUser.get_django_user(web_user))
-            api_key = api_key.key
+            api_key = api_key.plaintext_key
 
         return {
             'HTTP_AUTHORIZATION': f'apikey {username}:{api_key}'

corehq/apps/app_manager/tests/test_views.py

@@ -776,7 +776,6 @@ def setUpClass(cls):
         cls.web_user_api_key = HQApiKey.objects.get_or_create(
             user=cls.web_user.get_django_user()
         )[0]
-        cls.web_user_api_key.key = cls.web_user_api_key.generate_key()
         cls.web_user_api_key.save()
 
         # The URL that tests in this class will use.
@@ -803,7 +802,7 @@ def test_correct_api_key(self):
         """Sending a correct API key returns a response with the case summary file."""
         response = self.client.get(
             self.url,
-            HTTP_AUTHORIZATION=f"ApiKey {self.web_user.username}:{self.web_user_api_key.key}",
+            HTTP_AUTHORIZATION=f"ApiKey {self.web_user.username}:{self.web_user_api_key.plaintext_key}",
         )
 
         self.assertEqual(response.status_code, 200)
@@ -819,7 +818,7 @@ def test_incorrect_api_key(self):
 
         with self.subTest("Missing username"):
             response = self.client.get(
-                self.url, HTTP_AUTHORIZATION=f"ApiKey :{self.web_user_api_key.key}"
+                self.url, HTTP_AUTHORIZATION=f"ApiKey :{self.web_user_api_key.plaintext_key}"
             )
             self.assertEqual(response.status_code, 401)
 

corehq/apps/data_analytics/migrations/0011_domainmetrics.py

@@ -0,0 +1,73 @@
+# Generated by Django 4.2.16 on 2024-12-06 01:12
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data_analytics', '0010_maltrow_last_run_date'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='DomainMetrics',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('domain', models.TextField(db_index=True, unique=True)),
+                ('date_created', models.DateTimeField(auto_now_add=True)),
+                ('last_modified', models.DateTimeField(auto_now=True)),
+                ('has_project_icon', models.BooleanField()),
+                ('has_security_settings', models.BooleanField()),
+                ('is_active', models.BooleanField()),
+                ('is_first_domain_for_creating_user', models.BooleanField()),
+                ('lookup_tables', models.IntegerField()),
+                ('repeaters', models.IntegerField()),
+                ('ucrs', models.IntegerField()),
+                ('apps', models.IntegerField()),
+                ('apps_with_icon', models.IntegerField()),
+                ('apps_with_multiple_languages', models.IntegerField()),
+                ('mobile_workers', models.IntegerField()),
+                ('web_users', models.IntegerField()),
+                ('active_mobile_workers', models.IntegerField()),
+                ('active_mobile_workers_in_last_365_days', models.IntegerField()),
+                ('case_sharing_groups', models.IntegerField()),
+                ('case_sharing_locations', models.IntegerField()),
+                ('has_custom_roles', models.BooleanField()),
+                ('has_locations', models.BooleanField()),
+                ('location_restricted_users', models.IntegerField()),
+                ('users_with_submission', models.IntegerField()),
+                ('cases', models.IntegerField()),
+                ('active_cases', models.IntegerField()),
+                ('cases_modified_in_last_30_days', models.IntegerField()),
+                ('cases_modified_in_last_60_days', models.IntegerField()),
+                ('cases_modified_in_last_90_days', models.IntegerField()),
+                ('inactive_cases', models.IntegerField()),
+                ('usercases_modified_in_last_30_days', models.IntegerField()),
+                ('forms', models.IntegerField()),
+                ('forms_submitted_in_last_30_days', models.IntegerField()),
+                ('forms_submitted_in_last_60_days', models.IntegerField()),
+                ('forms_submitted_in_last_90_days', models.IntegerField()),
+                ('first_form_submission', models.DateTimeField(null=True)),
+                ('most_recent_form_submission', models.DateTimeField(null=True)),
+                ('three_hundredth_form_submission', models.DateTimeField(null=True)),
+                ('total_sms', models.IntegerField()),
+                ('sms_in_last_30_days', models.IntegerField()),
+                ('sms_in_last_60_days', models.IntegerField()),
+                ('sms_in_last_90_days', models.IntegerField()),
+                ('incoming_sms', models.IntegerField()),
+                ('incoming_sms_in_last_30_days', models.IntegerField()),
+                ('incoming_sms_in_last_60_days', models.IntegerField()),
+                ('incoming_sms_in_last_90_days', models.IntegerField()),
+                ('outgoing_sms', models.IntegerField()),
+                ('outgoing_sms_in_last_30_days', models.IntegerField()),
+                ('outgoing_sms_in_last_60_days', models.IntegerField()),
+                ('outgoing_sms_in_last_90_days', models.IntegerField()),
+                ('telerivet_backends', models.IntegerField()),
+                ('case_exports', models.IntegerField()),
+                ('custom_exports', models.IntegerField()),
+                ('deid_exports', models.IntegerField()),
+                ('saved_exports', models.IntegerField()),
+            ],
+        ),
+    ]

corehq/apps/data_analytics/models.py

@@ -157,3 +157,94 @@ def export_row(self, past_months):
             experienced_threshold=self.experienced_threshold or DEFAULT_EXPERIENCED_THRESHOLD,
             performance_threshold=self.performance_threshold or DEFAULT_PERFORMANCE_THRESHOLD,
         )
+
+
+class DomainMetrics(models.Model):
+
+    domain = models.TextField(unique=True, db_index=True)
+    date_created = models.DateTimeField(auto_now_add=True)
+    last_modified = models.DateTimeField(auto_now=True)
+
+    has_project_icon = models.BooleanField()
+    has_security_settings = models.BooleanField()
+    is_active = models.BooleanField()
+    is_first_domain_for_creating_user = models.BooleanField()
+
+    lookup_tables = models.IntegerField()
+    repeaters = models.IntegerField()
+    ucrs = models.IntegerField()
+
+    # App Metrics
+    apps = models.IntegerField()
+    apps_with_icon = models.IntegerField()
+    apps_with_multiple_languages = models.IntegerField()
+
+    # User Metrics
+    mobile_workers = models.IntegerField()
+    web_users = models.IntegerField()
+
+    active_mobile_workers = models.IntegerField()
+    active_mobile_workers_in_last_365_days = models.IntegerField()
+    case_sharing_groups = models.IntegerField()
+    case_sharing_locations = models.IntegerField()
+    has_custom_roles = models.BooleanField()
+    has_locations = models.BooleanField()
+    location_restricted_users = models.IntegerField()
+    users_with_submission = models.IntegerField()
+
+    # Case Metrics
+    cases = models.IntegerField()
+
+    active_cases = models.IntegerField()
+    cases_modified_in_last_30_days = models.IntegerField()
+    cases_modified_in_last_60_days = models.IntegerField()
+    cases_modified_in_last_90_days = models.IntegerField()
+    inactive_cases = models.IntegerField()
+    usercases_modified_in_last_30_days = models.IntegerField()
+
+    # Form Metrics
+    forms = models.IntegerField()
+    forms_submitted_in_last_30_days = models.IntegerField()
+    forms_submitted_in_last_60_days = models.IntegerField()
+    forms_submitted_in_last_90_days = models.IntegerField()
+
+    first_form_submission = models.DateTimeField(null=True)
+    most_recent_form_submission = models.DateTimeField(null=True)
+    three_hundredth_form_submission = models.DateTimeField(null=True)
+
+    # SMS Metrics
+    total_sms = models.IntegerField()
+    sms_in_last_30_days = models.IntegerField()
+    sms_in_last_60_days = models.IntegerField()
+    sms_in_last_90_days = models.IntegerField()
+
+    incoming_sms = models.IntegerField()
+    incoming_sms_in_last_30_days = models.IntegerField()
+    incoming_sms_in_last_60_days = models.IntegerField()
+    incoming_sms_in_last_90_days = models.IntegerField()
+
+    outgoing_sms = models.IntegerField()
+    outgoing_sms_in_last_30_days = models.IntegerField()
+    outgoing_sms_in_last_60_days = models.IntegerField()
+    outgoing_sms_in_last_90_days = models.IntegerField()
+
+    telerivet_backends = models.IntegerField()
+
+    # Export Metrics
+    case_exports = models.IntegerField()
+    custom_exports = models.IntegerField()
+    deid_exports = models.IntegerField()
+    saved_exports = models.IntegerField()
+
+    # Calculated properties
+    @property
+    def has_app(self):
+        return bool(self.apps)
+
+    @property
+    def has_used_sms(self):
+        return bool(self.total_sms)
+
+    @property
+    def has_used_sms_in_last_30_days(self):
+        return bool(self.sms_in_last_30_days)

corehq/apps/domain/deletion.py

@@ -366,6 +366,7 @@ def _delete_demo_user_restores(domain_name):
     ModelDeletion('custom_data_fields', 'CustomDataFieldsDefinition', 'domain', [
         'CustomDataFieldsProfile', 'Field',
     ]),
+    ModelDeletion('data_analytics', 'DomainMetrics', 'domain'),
     ModelDeletion('data_analytics', 'GIRRow', 'domain_name'),
     ModelDeletion('data_analytics', 'MALTRow', 'domain_name'),
     ModelDeletion('data_dictionary', 'CaseType', 'domain', [

corehq/apps/domain/tests/test_api_key_auth.py

@@ -51,7 +51,7 @@ def setUpClass(cls):
         domain_obj = create_domain(name=cls.domain)
         cls.addClassCleanup(domain_obj.delete)
         cls.user = WebUser.create(cls.domain, USERNAME, 'password', None, None)
-        cls.api_key = HQApiKey.objects.create(user=cls.user.get_django_user()).key
+        cls.api_key = HQApiKey.objects.create(user=cls.user.get_django_user()).plaintext_key
 
     def call_api(self, request, allow_creds_in_data):
 

corehq/apps/dump_reload/tests/test_dump_models.py

@@ -52,6 +52,7 @@
     "cleanup.DeletedCouchDoc",
     "cleanup.DeletedSQLDoc",
     "contenttypes.ContentType",
+    "data_analytics.DomainMetrics",
     "data_analytics.GIRRow",
     "data_analytics.MALTRow",
     "django_celery_results.ChordCounter",

corehq/apps/email/models.py

@@ -1,7 +1,11 @@
 from django.db import models
 
-from corehq.motech.const import PASSWORD_PLACEHOLDER, ALGO_AES
-from corehq.motech.utils import b64_aes_decrypt, b64_aes_encrypt
+from corehq.motech.const import PASSWORD_PLACEHOLDER, ALGO_AES, ALGO_AES_CBC
+from corehq.motech.utils import (
+    b64_aes_cbc_decrypt,
+    b64_aes_cbc_encrypt,
+    b64_aes_decrypt,
+)
 
 
 class EmailSettings(models.Model):
@@ -24,13 +28,16 @@ def __str__(self):
 
     @property
     def plaintext_password(self):
-        if self.password.startswith(f'${ALGO_AES}$'):
+        if self.password.startswith(f'${ALGO_AES_CBC}$'):
+            ciphertext = self.password.split('$', 2)[2]
+            return b64_aes_cbc_decrypt(ciphertext)
+        if self.password.startswith(f'${ALGO_AES}$'):  # This will be deleted after migration to cbc is done
             ciphertext = self.password.split('$', 2)[2]
             return b64_aes_decrypt(ciphertext)
         return self.password
 
     @plaintext_password.setter
     def plaintext_password(self, plaintext):
         if plaintext != PASSWORD_PLACEHOLDER:
-            ciphertext = b64_aes_encrypt(plaintext)
-            self.password = f'${ALGO_AES}${ciphertext}'
+            ciphertext = b64_aes_cbc_encrypt(plaintext)
+            self.password = f'${ALGO_AES_CBC}${ciphertext}'

corehq/apps/enterprise/tests/test_permissions.py

@@ -127,6 +127,6 @@ def test_api_call(self):
             'resource_name': InternalFixtureResource._meta.resource_name,
         })
         username = self.web_user_non_admin.username
-        api_params = urlencode({'username': username, 'api_key': self.api_key.key})
+        api_params = urlencode({'username': username, 'api_key': self.api_key.plaintext_key})
         response = self.client.get(f"{url}?{api_params}")
         self.assertEqual(response.status_code, 200)

corehq/apps/export/static/export/js/download_export.js

@@ -367,7 +367,7 @@ hqDefine('export/js/download_export', [
         };
 
         self._dealWithErrors = function (data) {
-            if (self._numErrors > 3) {
+            if (self._numErrors > 3 || data.retry === false) {
                 if (data && data.error) {
                     self.downloadError(data.error);
                 } else {

corehq/apps/export/views/download.py

@@ -390,14 +390,22 @@ def poll_custom_export_download(request, domain):
     permissions = ExportsPermissionsManager(form_or_case, domain, request.couch_user)
     permissions.access_download_export_or_404()
     download_id = request.GET.get('download_id')
+
+    if not download_id:
+        return JsonResponse({
+            'error': _('Could not find download. Please refresh page and try again.'),
+            'retry': False,
+        })
+
     try:
         context = get_download_context(download_id)
     except TaskFailedError as e:
         if e.exception_name == 'XlsLengthException':
             return JsonResponse({
                 'error': _(
                     'This file has more than 256 columns, which is not supported by xls. '
-                    'Please change the output type to csv or xlsx to export this file.')
+                    'Please change the output type to csv or xlsx to export this file.'),
+                'retry': False,
             })
         else:
             notify_exception(

corehq/apps/linked_domain/tests/test_linked_case_claim.py

@@ -62,7 +62,7 @@ def setUpClass(cls):
         cls.couch_user = WebUser.create(cls.domain, "test", "foobar", None, None)
         cls.django_user = cls.couch_user.get_django_user()
         cls.api_key, _ = HQApiKey.objects.get_or_create(user=cls.django_user)
-        cls.auth_headers = {'HTTP_AUTHORIZATION': 'apikey test:%s' % cls.api_key.key}
+        cls.auth_headers = {'HTTP_AUTHORIZATION': 'apikey test:%s' % cls.api_key.plaintext_key}
         cls.domain_link.save()
 
     @classmethod

corehq/apps/linked_domain/tests/test_remote_auth.py

@@ -24,14 +24,14 @@ def test_returns_401_if_wrong_api_key(self):
         self.assertEqual(resp.status_code, 401)
 
     def test_returns_403_if_valid_api_key_but_no_linked_domain_access(self):
-        headers = {'HTTP_AUTHORIZATION': f'apikey test:{self.api_key.key}'}
+        headers = {'HTTP_AUTHORIZATION': f'apikey test:{self.api_key.plaintext_key}'}
         with patch.object(decorators, 'can_user_access_linked_domains', return_value=False):
             resp = self.client.get(self.url, **headers)
 
         self.assertEqual(resp.status_code, 403)
 
     def test_returns_200_if_valid_api_key_and_linked_domain_access(self):
-        headers = {'HTTP_AUTHORIZATION': f'apikey test:{self.api_key.key}'}
+        headers = {'HTTP_AUTHORIZATION': f'apikey test:{self.api_key.plaintext_key}'}
 
         with patch.object(decorators, 'can_user_access_linked_domains', return_value=True):
             resp = self.client.get(self.url, **headers)