Add Enterprise CommCare Client Version Compliance Report

corehq/apps/builds/tests/test_utils.py

@@ -0,0 +1,28 @@
+from django.test import SimpleTestCase
+from corehq.apps.builds.utils import is_out_of_date
+
+
+class TestVersionUtils(SimpleTestCase):
+
+    def test_is_out_of_date(self):
+        test_cases = [
+            # (version_in_use, latest_version, expected_result)
+            ('2.53.0', '2.53.1', True),     # Normal case - out of date
+            ('2.53.1', '2.53.1', False),    # Same version - not out of date
+            ('2.53.2', '2.53.1', False),    # Higher version - not out of date
+            (None, '2.53.1', False),        # None version_in_use
+            ('2.53.1', None, False),        # None latest_version
+            ('invalid', '2.53.1', False),   # Invalid version string
+            ('2.53.1', 'invalid', False),   # Invalid latest version
+            ('6', '7', True),               # Normal case - app version is integer
+            (None, None, False),            # None version_in_use and latest_version
+        ]
+
+        for version_in_use, latest_version, expected in test_cases:
+            with self.subTest(version_in_use=version_in_use, latest_version=latest_version):
+                result = is_out_of_date(version_in_use, latest_version)
+                self.assertEqual(
+                    result,
+                    expected,
+                    f"Expected is_out_of_date('{version_in_use}', '{latest_version}') to be {expected}"
+                )

corehq/apps/builds/utils.py

@@ -1,6 +1,10 @@
 import re
+from datetime import datetime
 
-from .models import CommCareBuild, CommCareBuildConfig
+from dimagi.utils.parsing import ISO_DATETIME_FORMAT
+
+from corehq.apps.builds.models import CommCareBuild, CommCareBuildConfig
+from corehq.util.quickcache import quickcache
 
 
 def get_all_versions(versions):
@@ -40,3 +44,60 @@ def extract_build_info_from_filename(content_disposition):
     else:
         raise ValueError('Could not find filename like {!r} in {!r}'.format(
             pattern, content_disposition))
+
+
+@quickcache(['config', 'target_time'], timeout=100 * 60, memoize_timeout=100 * 60)
+def get_latest_version_at_time(config, target_time):
+    """
+    Get the latest CommCare version that was available at a given time.
+    Excludes superuser-only versions.
+    Menu items are already in chronological order (newest last).
+    If no target time is provided, return the latest version available now.
+
+    Args:
+        config: CommCareBuildConfig instance
+        target_time: datetime or string in ISO format, or None for latest version
+    """
+    if not target_time:
+        return config.get_default().version
+
+    if isinstance(target_time, str):
+        target_time = datetime.strptime(target_time, ISO_DATETIME_FORMAT)
+
+    # Iterate through menu items in reverse (newest to oldest)
+    for item in reversed(config.menu):
+        if item.superuser_only:
+            continue
+        build_time = get_build_time(item.build.version)
+        if build_time and build_time <= target_time:
+            return item.build.version
+
+    return None
+
+
+@quickcache(['version'], timeout=100 * 60, memoize_timeout=100 * 60)
+def get_build_time(version):
+    try:
+        build = CommCareBuild.get_build(version, latest=True)
+        if build and build.time:
+            return build.time
+    except KeyError:
+        return None
+
+
+def is_out_of_date(version_in_use, latest_version):
+    version_in_use_tuple = _parse_version(version_in_use)
+    latest_version_tuple = _parse_version(latest_version)
+    if not version_in_use_tuple or not latest_version_tuple:
+        return False
+    return version_in_use_tuple < latest_version_tuple
+
+
+def _parse_version(version_str):
+    """Convert version string to comparable tuple"""
+    if version_str:
+        try:
+            return tuple(int(n) for n in version_str.split('.'))
+        except (ValueError, AttributeError):
+            return None
+    return None

corehq/apps/enterprise/api/api.py

@@ -1,6 +1,7 @@
 from tastypie.api import Api
 
 from corehq.apps.enterprise.api.resources import (
+    CommCareVersionComplianceResource,
     DomainResource,
     FormSubmissionResource,
     MobileUserResource,
@@ -15,4 +16,5 @@
 v1_api.register(MobileUserResource())
 v1_api.register(FormSubmissionResource())
 v1_api.register(ODataFeedResource())
+v1_api.register(CommCareVersionComplianceResource())
 v1_api.register(SMSResource())

corehq/apps/enterprise/api/resources.py

@@ -194,7 +194,12 @@ def get_object_list(self, request):
             response=http.HttpTooManyRequests(headers={'Retry-After': self.RETRY_IN_PROGRESS_DELAY}))
 
     def get_report_task(self, request):
-        raise NotImplementedError()
+        account = BillingAccount.get_account_by_domain(request.domain)
+        return generate_enterprise_report.s(
+            self.REPORT_SLUG,
+            account.id,
+            request.couch_user.username,
+        )
 
     def _add_query_id_to_request(self, request, query_id):
         if 'report' not in request.GET:
@@ -214,14 +219,6 @@ class DomainResource(ODataEnterpriseReportResource):
 
     REPORT_SLUG = EnterpriseReport.DOMAINS
 
-    def get_report_task(self, request):
-        account = BillingAccount.get_account_by_domain(request.domain)
-        return generate_enterprise_report.s(
-            self.REPORT_SLUG,
-            account.id,
-            request.couch_user.username,
-        )
-
     def dehydrate(self, bundle):
         bundle.data['domain'] = bundle.obj[6]
         bundle.data['created_on'] = self.convert_datetime(bundle.obj[0])
@@ -248,14 +245,6 @@ class WebUserResource(ODataEnterpriseReportResource):
 
     REPORT_SLUG = EnterpriseReport.WEB_USERS
 
-    def get_report_task(self, request):
-        account = BillingAccount.get_account_by_domain(request.domain)
-        return generate_enterprise_report.s(
-            self.REPORT_SLUG,
-            account.id,
-            request.couch_user.username,
-        )
-
     def dehydrate(self, bundle):
         bundle.data['email'] = bundle.obj[0]
         bundle.data['name'] = self.convert_not_available(bundle.obj[1])
@@ -289,14 +278,6 @@ class MobileUserResource(ODataEnterpriseReportResource):
 
     REPORT_SLUG = EnterpriseReport.MOBILE_USERS
 
-    def get_report_task(self, request):
-        account = BillingAccount.get_account_by_domain(request.domain)
-        return generate_enterprise_report.s(
-            self.REPORT_SLUG,
-            account.id,
-            request.couch_user.username,
-        )
-
     def dehydrate(self, bundle):
         bundle.data['username'] = bundle.obj[0]
         bundle.data['name'] = bundle.obj[1]
@@ -373,14 +354,6 @@ class ODataFeedResource(ODataEnterpriseReportResource):
 
     REPORT_SLUG = EnterpriseReport.ODATA_FEEDS
 
-    def get_report_task(self, request):
-        account = BillingAccount.get_account_by_domain(request.domain)
-        return generate_enterprise_report.s(
-            self.REPORT_SLUG,
-            account.id,
-            request.couch_user.username,
-        )
-
     def dehydrate(self, bundle):
         bundle.data['num_feeds_used'] = bundle.obj[0]
         bundle.data['num_feeds_available'] = bundle.obj[1]
@@ -433,3 +406,22 @@ def dehydrate(self, bundle):
 
     def get_primary_keys(self):
         return ('form_id', 'submitted',)
+
+
+class CommCareVersionComplianceResource(ODataEnterpriseReportResource):
+    mobile_worker = fields.CharField()
+    project_space = fields.CharField()
+    latest_version_available_at_submission = fields.CharField()
+    version_in_use = fields.CharField()
+
+    REPORT_SLUG = EnterpriseReport.COMMCARE_VERSION_COMPLIANCE
+
+    def dehydrate(self, bundle):
+        bundle.data['mobile_worker'] = bundle.obj[0]
+        bundle.data['project_space'] = bundle.obj[1]
+        bundle.data['latest_version_available_at_submission'] = bundle.obj[2]
+        bundle.data['version_in_use'] = bundle.obj[3]
+        return bundle
+
+    def get_primary_keys(self):
+        return ('mobile_worker', 'project_space',)

corehq/apps/enterprise/enterprise.py

@@ -2,26 +2,38 @@
 import re
 from django.db.models import Count
 from datetime import datetime, timedelta
+from functools import partial
+from gevent.pool import Pool
 
+from django.conf import settings
+from dimagi.utils.chunked import chunked
+from dimagi.utils.parsing import ISO_DATETIME_FORMAT
 from django.utils.translation import gettext as _
 from django.utils.translation import gettext_lazy
-from django.conf import settings
 
 from memoized import memoized
 
 from couchforms.analytics import get_last_form_submission_received
 from dimagi.utils.dates import DateSpan
 
-from corehq.apps.enterprise.exceptions import EnterpriseReportError, TooMuchRequestedDataError
-from corehq.apps.enterprise.iterators import raise_after_max_elements
 from corehq.apps.accounting.models import BillingAccount
 from corehq.apps.accounting.utils import get_default_domain_url
 from corehq.apps.app_manager.dbaccessors import get_brief_apps_in_domain
+from corehq.apps.builds.utils import get_latest_version_at_time, is_out_of_date
+from corehq.apps.builds.models import CommCareBuildConfig
 from corehq.apps.domain.calculations import sms_in_last
 from corehq.apps.domain.models import Domain
+from corehq.apps.enterprise.exceptions import (
+    EnterpriseReportError,
+    TooMuchRequestedDataError,
+)
+from corehq.apps.enterprise.iterators import raise_after_max_elements
 from corehq.apps.es import forms as form_es
 from corehq.apps.es.users import UserES
 from corehq.apps.export.dbaccessors import ODataExportFetcher
+from corehq.apps.reports.util import (
+    get_commcare_version_and_date_from_last_usage,
+)
 from corehq.apps.sms.models import SMS, OUTGOING, INCOMING
 from corehq.apps.users.dbaccessors import (
     get_all_user_rows,
@@ -30,13 +42,16 @@
 )
 from corehq.apps.users.models import CouchUser, Invitation
 
+ES_CLIENT_CONNECTION_POOL_LIMIT = 10  # Matches ES client connection pool limit
+
 
 class EnterpriseReport(ABC):
     DOMAINS = 'domains'
     WEB_USERS = 'web_users'
     MOBILE_USERS = 'mobile_users'
     FORM_SUBMISSIONS = 'form_submissions'
     ODATA_FEEDS = 'odata_feeds'
+    COMMCARE_VERSION_COMPLIANCE = 'commcare_version_compliance'
     SMS = 'sms'
 
     DATE_ROW_FORMAT = '%Y/%m/%d %H:%M:%S'
@@ -81,6 +96,8 @@ def create(cls, slug, account_id, couch_user, **kwargs):
             report = EnterpriseFormReport(account, couch_user, **kwargs)
         elif slug == cls.ODATA_FEEDS:
             report = EnterpriseODataReport(account, couch_user, **kwargs)
+        elif slug == cls.COMMCARE_VERSION_COMPLIANCE:
+            report = EnterpriseCommCareVersionReport(account, couch_user, **kwargs)
         elif slug == cls.SMS:
             report = EnterpriseSMSReport(account, couch_user, **kwargs)
 
@@ -405,6 +422,94 @@ def _get_individual_export_rows(self, exports, export_line_counts):
         return rows
 
 
+class EnterpriseCommCareVersionReport(EnterpriseReport):
+    title = gettext_lazy('CommCare Version Compliance')
+    total_description = gettext_lazy('% of Mobile Workers on the Latest CommCare Version')
+
+    @property
+    def headers(self):
+        return [
+            _('Mobile Worker'),
+            _('Project Space'),
+            _('Latest Version Available at Submission'),
+            _('Version in Use'),
+        ]
+
+    @property
+    def rows(self):
+        config = CommCareBuildConfig.fetch()
+        partial_func = partial(self.rows_for_domain, config)
+        return _process_domains_in_parallel(partial_func, self.account.get_domains())
+
+    @property
+    def total(self):
+        total_mobile_workers = 0
+        total_up_to_date = 0
+        config = CommCareBuildConfig.fetch()
+
+        def total_for_domain(domain):
+            mobile_workers = get_mobile_user_count(domain, include_inactive=False)
+            outdated_users = len(self.rows_for_domain(domain, config))
+            return mobile_workers, outdated_users
+
+        results = _process_domains_in_parallel(total_for_domain, self.account.get_domains())
+
+        for domain_mobile_workers, outdated_users in results:
+            if domain_mobile_workers:
+                total_mobile_workers += domain_mobile_workers
+                total_up_to_date += domain_mobile_workers - outdated_users
+
+        return _format_percentage_for_enterprise_tile(total_up_to_date, total_mobile_workers)
+
+    def rows_for_domain(self, domain, config):
+        rows = []
+
+        user_query = (UserES()
+            .domain(domain)
+            .mobile_users()
+            .source([
+                'username',
+                'reporting_metadata.last_submission_for_user.commcare_version',
+                'reporting_metadata.last_submission_for_user.submission_date',
+                'last_device.commcare_version',
+                'last_device.last_used'
+            ]))
+
+        for user in user_query.run().hits:
+            last_submission = user.get('reporting_metadata', {}).get('last_submission_for_user', {})
+            last_device = user.get('last_device', {})
+
+            version_in_use, date_of_use = get_commcare_version_and_date_from_last_usage(last_submission,
+                                                                                        last_device)
+
+            if not version_in_use:
+                continue
+
+            # Remove seconds and microseconds to reduce the number of unique timestamps
+            # This helps with performance because get_latest_version_at_time is memoized
+            if isinstance(date_of_use, str):
+                date_of_use = datetime.strptime(date_of_use, ISO_DATETIME_FORMAT)
+            date_of_use_minute_precision = date_of_use.replace(second=0, microsecond=0)
+
+            latest_version_at_time_of_use = get_latest_version_at_time(config, date_of_use_minute_precision)
+
+            if is_out_of_date(version_in_use, latest_version_at_time_of_use):
+                rows.append([
+                    user['username'],
+                    domain,
+                    latest_version_at_time_of_use,
+                    version_in_use,
+                ])
+
+        return rows
+
+
+def _format_percentage_for_enterprise_tile(dividend, divisor):
+    if not divisor:
+        return '--'
+    return f"{dividend / divisor * 100:.1f}%"
+
+
 class EnterpriseSMSReport(EnterpriseReport):
     title = gettext_lazy('SMS Usage')
     total_description = gettext_lazy('# of SMS Sent')
@@ -464,3 +569,18 @@ def rows_for_domain(self, domain_obj):
         num_errors = sum([result['direction_count'] for result in results if result['error']])
 
         return [(domain_obj.name, num_sent, num_received, num_errors), ]
+
+
+def _process_domains_in_parallel(process_func, domains):
+    """
+    Helper method to process domains in parallel using gevent pooling.
+    """
+    results = []
+
+    for chunk in chunked(domains, ES_CLIENT_CONNECTION_POOL_LIMIT):
+        pool = Pool(size=ES_CLIENT_CONNECTION_POOL_LIMIT)
+        chunk_results = pool.map(process_func, chunk)
+        pool.join()
+        results.extend(chunk_results)
+
+    return results