From b71b73b651d4740070138f95f9e394a3051acb1d Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Tue, 24 Dec 2024 16:53:16 -0500 Subject: [PATCH 01/34] Add App Version Compliance Report --- corehq/apps/enterprise/enterprise.py | 92 +++++++++++++++++++++++++++- corehq/apps/enterprise/views.py | 1 + 2 files changed, 91 insertions(+), 2 deletions(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index f06c22d73651..7578f542fd8a 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -16,6 +16,7 @@ from corehq.apps.accounting.models import BillingAccount from corehq.apps.accounting.utils import get_default_domain_url from corehq.apps.app_manager.dbaccessors import get_brief_apps_in_domain +from corehq.apps.app_manager.models import Application, SavedAppBuild from corehq.apps.builds.utils import get_latest_version_at_time, is_out_of_date from corehq.apps.builds.models import CommCareBuildConfig from corehq.apps.domain.calculations import sms_in_last @@ -25,7 +26,7 @@ TooMuchRequestedDataError, ) from corehq.apps.enterprise.iterators import raise_after_max_elements -from corehq.apps.es import forms as form_es +from corehq.apps.es import AppES, forms as form_es from corehq.apps.es.users import UserES from corehq.apps.export.dbaccessors import ODataExportFetcher from corehq.apps.reports.util import ( @@ -37,7 +38,7 @@ get_mobile_user_count, get_web_user_count, ) -from corehq.apps.users.models import CouchUser, HQApiKey, Invitation, WebUser +from corehq.apps.users.models import CouchUser, HQApiKey, Invitation, LastBuild, WebUser class EnterpriseReport(ABC): @@ -50,6 +51,7 @@ class EnterpriseReport(ABC): SMS = 'sms' API_USAGE = 'api_usage' TWO_FACTOR_AUTH = '2fa' + APP_VERSION_COMPLIANCE = 'app_version_compliance' DATE_ROW_FORMAT = '%Y/%m/%d %H:%M:%S' @@ -101,6 +103,8 @@ def create(cls, slug, account_id, couch_user, **kwargs): report = EnterpriseAPIReport(account, couch_user, **kwargs) elif slug == cls.TWO_FACTOR_AUTH: report = Enterprise2FAReport(account, couch_user, **kwargs) + elif slug == cls.APP_VERSION_COMPLIANCE: + report = EnterpriseAppVersionComplianceReport(account, couch_user, **kwargs) if report: report.slug = slug @@ -635,3 +639,87 @@ def rows_for_domain(self, domain_obj): if domain_obj.two_factor_auth: return [] return [(domain_obj.name,)] + + +class EnterpriseAppVersionComplianceReport(EnterpriseReport): + title = gettext_lazy('Application Version Compliance') + total_description = gettext_lazy('% of Applications Up-to-Date Across All Mobile Workers') + + def __init__(self, account, couch_user): + super().__init__(account, couch_user) + self.builds_by_app_id = {} + self.wrapped_builds_cache = {} + + def get_app_builds(self, domain, app_id): + if app_id not in self.builds_by_app_id: + app_es = ( + AppES() + .domain(domain) + .is_build() + .app_id(app_id) + .sort('version', desc=True) + .is_released() + ) + self.builds_by_app_id[app_id] = app_es.run().hits + + return self.builds_by_app_id[app_id] + + @property + def headers(self): + return [_('Mobile Worker'), _('Project Space'), _('Application'), + _('Latest Version Available at Submission'), + _('Version in Use'), _('Last Used [UTC]')] + + def rows_for_domain(self, domain_obj): + rows = [] + + user_query = (UserES() + .domain(domain_obj.name) + .mobile_users() + .source([ + 'username', + 'reporting_metadata.last_builds', + ])) + for user in user_query.run().hits: + last_builds = user.get('reporting_metadata', {}).get('last_builds', []) + for build in last_builds: + build = LastBuild.wrap(build) + if build.build_version: + all_builds = self.get_app_builds(domain_obj.name, build.app_id) + latest_version = self.get_latest_build_version_at_time(all_builds, + build.build_version_date) + if is_out_of_date(build.build_version, latest_version): + rows.append([ + user['username'], + domain_obj.name, + Application.get(build.app_id).name, + latest_version, + build.build_version, + self.format_date(build.build_version_date), + ]) + + return rows + + def total_for_domain(self, domain_obj): + return 0 + + def get_latest_build_version_at_time(self, all_builds, time): + """ + Get the latest build version at the time + + :param all_builds: List of raw build documents sorted by version in descending order + :param time: The date of the build version to compare against + :return: The latest build version available at the given date + """ + + for build_doc in all_builds: + build_id = build_doc['_id'] + if build_id in self.wrapped_builds_cache: + build = self.wrapped_builds_cache[build_id] + else: + build = SavedAppBuild.wrap(build_doc) + self.wrapped_builds_cache[build_id] = build + + if build.last_released <= time: + return build.version + return None diff --git a/corehq/apps/enterprise/views.py b/corehq/apps/enterprise/views.py index e2cf427ba1d5..56ab28d88514 100644 --- a/corehq/apps/enterprise/views.py +++ b/corehq/apps/enterprise/views.py @@ -92,6 +92,7 @@ def platform_overview(request, domain): EnterpriseReport.ODATA_FEEDS, EnterpriseReport.COMMCARE_VERSION_COMPLIANCE, EnterpriseReport.SMS, + EnterpriseReport.APP_VERSION_COMPLIANCE, )], 'uses_date_range': [EnterpriseReport.FORM_SUBMISSIONS, EnterpriseReport.SMS], 'metric_type': 'Platform Overview', From 0fcd2729917d2b1686ca00559f9f204a7598370b Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Wed, 25 Dec 2024 12:39:44 -0500 Subject: [PATCH 02/34] app_id can be None or empty string --- corehq/apps/enterprise/enterprise.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index 7578f542fd8a..89b956c14592 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -684,7 +684,7 @@ def rows_for_domain(self, domain_obj): last_builds = user.get('reporting_metadata', {}).get('last_builds', []) for build in last_builds: build = LastBuild.wrap(build) - if build.build_version: + if build.build_version and build.app_id: all_builds = self.get_app_builds(domain_obj.name, build.app_id) latest_version = self.get_latest_build_version_at_time(all_builds, build.build_version_date) From 1e27613b384e23dd96f23f81572ed93df4817747 Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Wed, 25 Dec 2024 12:42:27 -0500 Subject: [PATCH 03/34] is_out_of_date expect string --- corehq/apps/builds/utils.py | 8 ++++++++ corehq/apps/enterprise/enterprise.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/corehq/apps/builds/utils.py b/corehq/apps/builds/utils.py index bb946ccde9bc..1407e40dac33 100644 --- a/corehq/apps/builds/utils.py +++ b/corehq/apps/builds/utils.py @@ -90,6 +90,14 @@ def get_build_time(version, cache=None): def is_out_of_date(version_in_use, latest_version): + """ + Check if the version in use is out of date compared to the latest version. + params: + version_in_use: str, the version in use + latest_version: str, the latest version + returns: + bool, True if the version in use is out of date, False otherwise + """ version_in_use_tuple = _parse_version(version_in_use) latest_version_tuple = _parse_version(latest_version) if not version_in_use_tuple or not latest_version_tuple: diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index 89b956c14592..bd8301e457eb 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -688,7 +688,7 @@ def rows_for_domain(self, domain_obj): all_builds = self.get_app_builds(domain_obj.name, build.app_id) latest_version = self.get_latest_build_version_at_time(all_builds, build.build_version_date) - if is_out_of_date(build.build_version, latest_version): + if is_out_of_date(str(build.build_version), str(latest_version)): rows.append([ user['username'], domain_obj.name, From 64ccd2616c5dc01bb730e9cf067af298fb17f9a8 Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Wed, 25 Dec 2024 14:00:06 -0500 Subject: [PATCH 04/34] Exclude apps that has been deleted --- corehq/apps/enterprise/enterprise.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index bd8301e457eb..ce201ac825cb 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -15,7 +15,7 @@ from corehq.apps.accounting.models import BillingAccount from corehq.apps.accounting.utils import get_default_domain_url -from corehq.apps.app_manager.dbaccessors import get_brief_apps_in_domain +from corehq.apps.app_manager.dbaccessors import get_app_ids_in_domain, get_brief_apps_in_domain from corehq.apps.app_manager.models import Application, SavedAppBuild from corehq.apps.builds.utils import get_latest_version_at_time, is_out_of_date from corehq.apps.builds.models import CommCareBuildConfig @@ -672,6 +672,7 @@ def headers(self): def rows_for_domain(self, domain_obj): rows = [] + apps = get_app_ids_in_domain(domain_obj.name) user_query = (UserES() .domain(domain_obj.name) @@ -684,7 +685,7 @@ def rows_for_domain(self, domain_obj): last_builds = user.get('reporting_metadata', {}).get('last_builds', []) for build in last_builds: build = LastBuild.wrap(build) - if build.build_version and build.app_id: + if build.build_version and build.app_id in apps: all_builds = self.get_app_builds(domain_obj.name, build.app_id) latest_version = self.get_latest_build_version_at_time(all_builds, build.build_version_date) From 9c8d16a5c9b145e1ff1beefa7aa3eb1fb0450582 Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Thu, 26 Dec 2024 13:40:06 -0500 Subject: [PATCH 05/34] Exclude deleted app before wrap --- corehq/apps/enterprise/enterprise.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index ce201ac825cb..0bba7e49fb5f 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -684,20 +684,21 @@ def rows_for_domain(self, domain_obj): for user in user_query.run().hits: last_builds = user.get('reporting_metadata', {}).get('last_builds', []) for build in last_builds: + if build.get('app_id') not in apps or not build.get('build_version'): + continue build = LastBuild.wrap(build) - if build.build_version and build.app_id in apps: - all_builds = self.get_app_builds(domain_obj.name, build.app_id) - latest_version = self.get_latest_build_version_at_time(all_builds, - build.build_version_date) - if is_out_of_date(str(build.build_version), str(latest_version)): - rows.append([ - user['username'], - domain_obj.name, - Application.get(build.app_id).name, - latest_version, - build.build_version, - self.format_date(build.build_version_date), - ]) + all_builds = self.get_app_builds(domain_obj.name, build.app_id) + latest_version = self.get_latest_build_version_at_time(all_builds, + build.build_version_date) + if is_out_of_date(str(build.build_version), str(latest_version)): + rows.append([ + user['username'], + domain_obj.name, + Application.get(build.app_id).name, + latest_version, + build.build_version, + self.format_date(build.build_version_date), + ]) return rows From f94b4b31014a3b7da4c709e1361746aa534fc7e3 Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Thu, 26 Dec 2024 13:50:28 -0500 Subject: [PATCH 06/34] Extract necessary attributes directly instead of wrapping the raw document This commit improved performance by 92% --- corehq/apps/enterprise/enterprise.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index 0bba7e49fb5f..b075e9e6b3e4 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -5,6 +5,7 @@ from django.conf import settings from django.contrib.auth.models import User from django.db.models import Count, Subquery, Q +from dimagi.ext.jsonobject import DateTimeProperty from django.utils.translation import gettext as _ from django.utils.translation import gettext_lazy @@ -16,7 +17,7 @@ from corehq.apps.accounting.models import BillingAccount from corehq.apps.accounting.utils import get_default_domain_url from corehq.apps.app_manager.dbaccessors import get_app_ids_in_domain, get_brief_apps_in_domain -from corehq.apps.app_manager.models import Application, SavedAppBuild +from corehq.apps.app_manager.models import Application from corehq.apps.builds.utils import get_latest_version_at_time, is_out_of_date from corehq.apps.builds.models import CommCareBuildConfig from corehq.apps.domain.calculations import sms_in_last @@ -659,6 +660,7 @@ def get_app_builds(self, domain, app_id): .app_id(app_id) .sort('version', desc=True) .is_released() + .source(['_id', 'version', 'last_released']) ) self.builds_by_app_id[app_id] = app_es.run().hits @@ -717,11 +719,14 @@ def get_latest_build_version_at_time(self, all_builds, time): for build_doc in all_builds: build_id = build_doc['_id'] if build_id in self.wrapped_builds_cache: - build = self.wrapped_builds_cache[build_id] + build_info = self.wrapped_builds_cache[build_id] else: - build = SavedAppBuild.wrap(build_doc) - self.wrapped_builds_cache[build_id] = build - - if build.last_released <= time: - return build.version + build_info = { + 'version': build_doc['version'], + 'last_released': DateTimeProperty.deserialize(build_doc['last_released']) + } + self.wrapped_builds_cache[build_id] = build_info + + if build_info['last_released'] <= time: + return build_info['version'] return None From 553d90152b53bfcc858f5db1c7061af4a25836ae Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Thu, 26 Dec 2024 14:58:14 -0500 Subject: [PATCH 07/34] Rename wrapped_builds_cache to build_by_build_id since we're no longer wrapping build doc --- corehq/apps/enterprise/enterprise.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index b075e9e6b3e4..6a26f3b963fa 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -649,7 +649,7 @@ class EnterpriseAppVersionComplianceReport(EnterpriseReport): def __init__(self, account, couch_user): super().__init__(account, couch_user) self.builds_by_app_id = {} - self.wrapped_builds_cache = {} + self.build_by_build_id = {} def get_app_builds(self, domain, app_id): if app_id not in self.builds_by_app_id: @@ -718,14 +718,14 @@ def get_latest_build_version_at_time(self, all_builds, time): for build_doc in all_builds: build_id = build_doc['_id'] - if build_id in self.wrapped_builds_cache: - build_info = self.wrapped_builds_cache[build_id] + if build_id in self.build_by_build_id: + build_info = self.build_by_build_id[build_id] else: build_info = { 'version': build_doc['version'], 'last_released': DateTimeProperty.deserialize(build_doc['last_released']) } - self.wrapped_builds_cache[build_id] = build_info + self.build_by_build_id[build_id] = build_info if build_info['last_released'] <= time: return build_info['version'] From d414a5c60077a50880df414adebe6d24a7bde2dc Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Thu, 26 Dec 2024 14:55:54 -0500 Subject: [PATCH 08/34] Use built_on when no last_released --- corehq/apps/enterprise/enterprise.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index 6a26f3b963fa..d35b05e5e545 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -660,7 +660,7 @@ def get_app_builds(self, domain, app_id): .app_id(app_id) .sort('version', desc=True) .is_released() - .source(['_id', 'version', 'last_released']) + .source(['_id', 'version', 'last_released', 'built_on']) ) self.builds_by_app_id[app_id] = app_es.run().hits @@ -721,9 +721,13 @@ def get_latest_build_version_at_time(self, all_builds, time): if build_id in self.build_by_build_id: build_info = self.build_by_build_id[build_id] else: + # last_released is added in 2019, build before 2019 don't have this field + # TODO: have a migration to populate last_released from built_on + # Then this code can be modified to use last_released only + released_date = build_doc['last_released'] or build_doc['built_on'] build_info = { 'version': build_doc['version'], - 'last_released': DateTimeProperty.deserialize(build_doc['last_released']) + 'last_released': DateTimeProperty.deserialize(released_date) } self.build_by_build_id[build_id] = build_info From 1d64d78e89121a0d921e93e041f9643f5377324b Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Thu, 26 Dec 2024 15:11:10 -0500 Subject: [PATCH 09/34] Cache application name by id This commit improved performance by 92% --- corehq/apps/enterprise/enterprise.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index d35b05e5e545..88c05a071028 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -674,6 +674,7 @@ def headers(self): def rows_for_domain(self, domain_obj): rows = [] + app_name_by_id = {} apps = get_app_ids_in_domain(domain_obj.name) user_query = (UserES() @@ -693,10 +694,12 @@ def rows_for_domain(self, domain_obj): latest_version = self.get_latest_build_version_at_time(all_builds, build.build_version_date) if is_out_of_date(str(build.build_version), str(latest_version)): + if build.app_id not in app_name_by_id: + app_name_by_id[build.app_id] = Application.get(build.app_id).name rows.append([ user['username'], domain_obj.name, - Application.get(build.app_id).name, + app_name_by_id[build.app_id], latest_version, build.build_version, self.format_date(build.build_version_date), From cbf48319bf89a10c7ea18d192b148d90f00d4f70 Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Fri, 27 Dec 2024 10:30:40 -0500 Subject: [PATCH 10/34] Don't query all domains object --- corehq/apps/enterprise/enterprise.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index 88c05a071028..7cbcef6b7412 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -672,13 +672,20 @@ def headers(self): _('Latest Version Available at Submission'), _('Version in Use'), _('Last Used [UTC]')] - def rows_for_domain(self, domain_obj): + @property + def rows(self): + rows = [] + for domain in self.account.get_domains(): + rows.extend(self.rows_for_domain(domain)) + return rows + + def rows_for_domain(self, domain): rows = [] app_name_by_id = {} - apps = get_app_ids_in_domain(domain_obj.name) + apps = get_app_ids_in_domain(domain) user_query = (UserES() - .domain(domain_obj.name) + .domain(domain) .mobile_users() .source([ 'username', @@ -690,7 +697,7 @@ def rows_for_domain(self, domain_obj): if build.get('app_id') not in apps or not build.get('build_version'): continue build = LastBuild.wrap(build) - all_builds = self.get_app_builds(domain_obj.name, build.app_id) + all_builds = self.get_app_builds(domain, build.app_id) latest_version = self.get_latest_build_version_at_time(all_builds, build.build_version_date) if is_out_of_date(str(build.build_version), str(latest_version)): @@ -698,7 +705,7 @@ def rows_for_domain(self, domain_obj): app_name_by_id[build.app_id] = Application.get(build.app_id).name rows.append([ user['username'], - domain_obj.name, + domain, app_name_by_id[build.app_id], latest_version, build.build_version, @@ -707,7 +714,7 @@ def rows_for_domain(self, domain_obj): return rows - def total_for_domain(self, domain_obj): + def total_for_domain(self, domain): return 0 def get_latest_build_version_at_time(self, all_builds, time): From 5175df2104370844e5e5373fdba594bfe96c246c Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Fri, 27 Dec 2024 10:33:30 -0500 Subject: [PATCH 11/34] Refactor: reorder funcion --- corehq/apps/enterprise/enterprise.py | 42 ++++++++++++++-------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index 7cbcef6b7412..2b852bf9001f 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -508,12 +508,6 @@ def rows_for_domain(self, domain, config, cache): return rows -def _format_percentage_for_enterprise_tile(dividend, divisor): - if not divisor: - return '--' - return f"{dividend / divisor * 100:.1f}%" - - class EnterpriseSMSReport(EnterpriseReport): title = gettext_lazy('SMS Usage') total_description = gettext_lazy('# of SMS Sent') @@ -651,21 +645,6 @@ def __init__(self, account, couch_user): self.builds_by_app_id = {} self.build_by_build_id = {} - def get_app_builds(self, domain, app_id): - if app_id not in self.builds_by_app_id: - app_es = ( - AppES() - .domain(domain) - .is_build() - .app_id(app_id) - .sort('version', desc=True) - .is_released() - .source(['_id', 'version', 'last_released', 'built_on']) - ) - self.builds_by_app_id[app_id] = app_es.run().hits - - return self.builds_by_app_id[app_id] - @property def headers(self): return [_('Mobile Worker'), _('Project Space'), _('Application'), @@ -717,6 +696,21 @@ def rows_for_domain(self, domain): def total_for_domain(self, domain): return 0 + def get_app_builds(self, domain, app_id): + if app_id not in self.builds_by_app_id: + app_es = ( + AppES() + .domain(domain) + .is_build() + .app_id(app_id) + .sort('version', desc=True) + .is_released() + .source(['_id', 'version', 'last_released', 'built_on']) + ) + self.builds_by_app_id[app_id] = app_es.run().hits + + return self.builds_by_app_id[app_id] + def get_latest_build_version_at_time(self, all_builds, time): """ Get the latest build version at the time @@ -744,3 +738,9 @@ def get_latest_build_version_at_time(self, all_builds, time): if build_info['last_released'] <= time: return build_info['version'] return None + + +def _format_percentage_for_enterprise_tile(dividend, divisor): + if not divisor: + return '--' + return f"{dividend / divisor * 100:.1f}%" From 106d01dff4b37a48e311866a3cf35fea6c93e89d Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Fri, 27 Dec 2024 12:03:19 -0500 Subject: [PATCH 12/34] Avoid wraping Application object Improve performance by 20% --- corehq/apps/enterprise/enterprise.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index 2b852bf9001f..7778f0d83ba2 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -681,7 +681,7 @@ def rows_for_domain(self, domain): build.build_version_date) if is_out_of_date(str(build.build_version), str(latest_version)): if build.app_id not in app_name_by_id: - app_name_by_id[build.app_id] = Application.get(build.app_id).name + app_name_by_id[build.app_id] = Application.get_db().get(build.app_id).get('name') rows.append([ user['username'], domain, From 1859605aa0f265c91ba8b11553dcf4c154bdf655 Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Fri, 27 Dec 2024 12:08:12 -0500 Subject: [PATCH 13/34] Avoid wrap LastBuild Improved performance by 30% --- corehq/apps/enterprise/enterprise.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index 7778f0d83ba2..d7b388dadb48 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -39,7 +39,7 @@ get_mobile_user_count, get_web_user_count, ) -from corehq.apps.users.models import CouchUser, HQApiKey, Invitation, LastBuild, WebUser +from corehq.apps.users.models import CouchUser, HQApiKey, Invitation, WebUser class EnterpriseReport(ABC): @@ -673,22 +673,24 @@ def rows_for_domain(self, domain): for user in user_query.run().hits: last_builds = user.get('reporting_metadata', {}).get('last_builds', []) for build in last_builds: - if build.get('app_id') not in apps or not build.get('build_version'): + app_id = build.get('app_id') + build_version = build.get('build_version') + if app_id not in apps or not build_version: continue - build = LastBuild.wrap(build) - all_builds = self.get_app_builds(domain, build.app_id) + build_version_date = DateTimeProperty.deserialize(build.get('build_version_date')) + all_builds = self.get_app_builds(domain, app_id) latest_version = self.get_latest_build_version_at_time(all_builds, - build.build_version_date) - if is_out_of_date(str(build.build_version), str(latest_version)): - if build.app_id not in app_name_by_id: - app_name_by_id[build.app_id] = Application.get_db().get(build.app_id).get('name') + build_version_date) + if is_out_of_date(str(build_version), str(latest_version)): + if app_id not in app_name_by_id: + app_name_by_id[app_id] = Application.get_db().get(app_id).get('name') rows.append([ user['username'], domain, - app_name_by_id[build.app_id], + app_name_by_id[app_id], latest_version, - build.build_version, - self.format_date(build.build_version_date), + build_version, + self.format_date(build_version_date), ]) return rows From 0e7f9bb3567352cfe35e6af8aed104f999ce4c56 Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Mon, 30 Dec 2024 15:32:54 -0500 Subject: [PATCH 14/34] Only fetch all builds if necessary It increases num of calls to `get_app_builds` by 10%, but still saves a lot of time because most `get_app_builds` now only need to return first 10 builds --- corehq/apps/enterprise/enterprise.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index d7b388dadb48..8654dd4eabc9 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -639,6 +639,7 @@ def rows_for_domain(self, domain_obj): class EnterpriseAppVersionComplianceReport(EnterpriseReport): title = gettext_lazy('Application Version Compliance') total_description = gettext_lazy('% of Applications Up-to-Date Across All Mobile Workers') + INITIAL_QUERY_LIMIT = 10 def __init__(self, account, couch_user): super().__init__(account, couch_user) @@ -678,9 +679,12 @@ def rows_for_domain(self, domain): if app_id not in apps or not build_version: continue build_version_date = DateTimeProperty.deserialize(build.get('build_version_date')) - all_builds = self.get_app_builds(domain, app_id) - latest_version = self.get_latest_build_version_at_time(all_builds, - build_version_date) + all_builds = self.get_app_builds(domain, app_id, limit=self.INITIAL_QUERY_LIMIT) + latest_version = self.get_latest_build_version_at_time(all_builds, build_version_date) + # If the latest version is not found within the limit, all builds are fetched as a fallback. + if not latest_version: + all_builds = self.get_app_builds(domain, app_id) + latest_version = self.get_latest_build_version_at_time(all_builds, build_version_date) if is_out_of_date(str(build_version), str(latest_version)): if app_id not in app_name_by_id: app_name_by_id[app_id] = Application.get_db().get(app_id).get('name') @@ -698,7 +702,7 @@ def rows_for_domain(self, domain): def total_for_domain(self, domain): return 0 - def get_app_builds(self, domain, app_id): + def get_app_builds(self, domain, app_id, limit=None): if app_id not in self.builds_by_app_id: app_es = ( AppES() @@ -709,6 +713,8 @@ def get_app_builds(self, domain, app_id): .is_released() .source(['_id', 'version', 'last_released', 'built_on']) ) + if limit: + app_es = app_es.size(limit) self.builds_by_app_id[app_id] = app_es.run().hits return self.builds_by_app_id[app_id] From ea9809a706ac32467e1fecb253ab1f3282f22dbf Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Mon, 30 Dec 2024 15:49:52 -0500 Subject: [PATCH 15/34] Implement total calculation --- corehq/apps/enterprise/enterprise.py | 41 +++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index 8654dd4eabc9..0cb16a643e07 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -659,6 +659,18 @@ def rows(self): rows.extend(self.rows_for_domain(domain)) return rows + @property + def total(self): + total_rows = 0 + total_last_builds = 0 + + for domain in self.account.get_domains(): + domain_rows, domain_last_builds = self.total_for_domain(domain) + total_rows += domain_rows + total_last_builds += domain_last_builds + + return _format_percentage_for_enterprise_tile(total_rows, total_last_builds) + def rows_for_domain(self, domain): rows = [] app_name_by_id = {} @@ -700,7 +712,34 @@ def rows_for_domain(self, domain): return rows def total_for_domain(self, domain): - return 0 + apps = get_app_ids_in_domain(domain) + total_last_builds = 0 + total_out_of_date = 0 + + user_query = (UserES() + .domain(domain) + .mobile_users() + .source([ + 'reporting_metadata.last_builds', + ])) + for user in user_query.run().hits: + last_builds = user.get('reporting_metadata', {}).get('last_builds', []) + for build in last_builds: + app_id = build.get('app_id') + build_version = build.get('build_version') + if app_id not in apps or not build_version: + continue + total_last_builds += 1 + build_version_date = DateTimeProperty.deserialize(build.get('build_version_date')) + all_builds = self.get_app_builds(domain, app_id, limit=self.INITIAL_QUERY_LIMIT) + latest_version = self.get_latest_build_version_at_time(all_builds, build_version_date) + if not latest_version: + all_builds = self.get_app_builds(domain, app_id) + latest_version = self.get_latest_build_version_at_time(all_builds, build_version_date) + if is_out_of_date(str(build_version), str(latest_version)): + total_out_of_date += 1 + + return total_out_of_date, total_last_builds def get_app_builds(self, domain, app_id, limit=None): if app_id not in self.builds_by_app_id: From a0f29a25c1d5dcd251cc0b0ed6b558ca929972c6 Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Mon, 30 Dec 2024 16:33:05 -0500 Subject: [PATCH 16/34] Refactor: DRY --- corehq/apps/enterprise/enterprise.py | 60 +++++++++++----------------- 1 file changed, 23 insertions(+), 37 deletions(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index 0cb16a643e07..28ffd23de4e3 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -676,38 +676,19 @@ def rows_for_domain(self, domain): app_name_by_id = {} apps = get_app_ids_in_domain(domain) - user_query = (UserES() - .domain(domain) - .mobile_users() - .source([ - 'username', - 'reporting_metadata.last_builds', - ])) - for user in user_query.run().hits: - last_builds = user.get('reporting_metadata', {}).get('last_builds', []) - for build in last_builds: - app_id = build.get('app_id') - build_version = build.get('build_version') - if app_id not in apps or not build_version: - continue - build_version_date = DateTimeProperty.deserialize(build.get('build_version_date')) - all_builds = self.get_app_builds(domain, app_id, limit=self.INITIAL_QUERY_LIMIT) - latest_version = self.get_latest_build_version_at_time(all_builds, build_version_date) - # If the latest version is not found within the limit, all builds are fetched as a fallback. - if not latest_version: - all_builds = self.get_app_builds(domain, app_id) - latest_version = self.get_latest_build_version_at_time(all_builds, build_version_date) - if is_out_of_date(str(build_version), str(latest_version)): - if app_id not in app_name_by_id: - app_name_by_id[app_id] = Application.get_db().get(app_id).get('name') - rows.append([ - user['username'], - domain, - app_name_by_id[app_id], - latest_version, - build_version, - self.format_date(build_version_date), - ]) + for user, build, latest_version in self._get_user_builds(domain, apps): + if is_out_of_date(str(build['build_version']), str(latest_version)): + app_id = build['app_id'] + if app_id not in app_name_by_id: + app_name_by_id[app_id] = Application.get_db().get(app_id).get('name') + rows.append([ + user['username'], + domain, + app_name_by_id[app_id], + latest_version, + build['build_version'], + self.format_date(DateTimeProperty.deserialize(build['build_version_date'])), + ]) return rows @@ -716,10 +697,19 @@ def total_for_domain(self, domain): total_last_builds = 0 total_out_of_date = 0 + for user, build, latest_version in self._get_user_builds(domain, apps): + total_last_builds += 1 + if is_out_of_date(str(build['build_version']), str(latest_version)): + total_out_of_date += 1 + + return total_out_of_date, total_last_builds + + def _get_user_builds(self, domain, apps): user_query = (UserES() .domain(domain) .mobile_users() .source([ + 'username', 'reporting_metadata.last_builds', ])) for user in user_query.run().hits: @@ -729,17 +719,13 @@ def total_for_domain(self, domain): build_version = build.get('build_version') if app_id not in apps or not build_version: continue - total_last_builds += 1 build_version_date = DateTimeProperty.deserialize(build.get('build_version_date')) all_builds = self.get_app_builds(domain, app_id, limit=self.INITIAL_QUERY_LIMIT) latest_version = self.get_latest_build_version_at_time(all_builds, build_version_date) if not latest_version: all_builds = self.get_app_builds(domain, app_id) latest_version = self.get_latest_build_version_at_time(all_builds, build_version_date) - if is_out_of_date(str(build_version), str(latest_version)): - total_out_of_date += 1 - - return total_out_of_date, total_last_builds + yield user, build, latest_version def get_app_builds(self, domain, app_id, limit=None): if app_id not in self.builds_by_app_id: From 848cf09907dadc7919d3727a602d50fb4c5c5b50 Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Mon, 30 Dec 2024 16:40:01 -0500 Subject: [PATCH 17/34] Rename variables and show up-to-date percentage instead of out-of-date percentage --- corehq/apps/enterprise/enterprise.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index 28ffd23de4e3..4c954623ba71 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -661,15 +661,17 @@ def rows(self): @property def total(self): - total_rows = 0 + total_out_of_date_rows = 0 total_last_builds = 0 for domain in self.account.get_domains(): - domain_rows, domain_last_builds = self.total_for_domain(domain) - total_rows += domain_rows + domain_out_of_date_rows, domain_last_builds = self.total_for_domain(domain) + total_out_of_date_rows += domain_out_of_date_rows total_last_builds += domain_last_builds - return _format_percentage_for_enterprise_tile(total_rows, total_last_builds) + total_up_to_date = total_last_builds - total_out_of_date_rows + + return _format_percentage_for_enterprise_tile(total_up_to_date, total_last_builds) def rows_for_domain(self, domain): rows = [] From b024957ecb43d0261689e096d0e669e7b6566c21 Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Wed, 1 Jan 2025 13:24:05 -0500 Subject: [PATCH 18/34] text update and reformat --- corehq/apps/enterprise/enterprise.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index 4c954623ba71..0ced41e69488 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -638,7 +638,7 @@ def rows_for_domain(self, domain_obj): class EnterpriseAppVersionComplianceReport(EnterpriseReport): title = gettext_lazy('Application Version Compliance') - total_description = gettext_lazy('% of Applications Up-to-Date Across All Mobile Workers') + total_description = gettext_lazy('% of Applications Up to Date Across All Mobile Workers') INITIAL_QUERY_LIMIT = 10 def __init__(self, account, couch_user): @@ -648,9 +648,14 @@ def __init__(self, account, couch_user): @property def headers(self): - return [_('Mobile Worker'), _('Project Space'), _('Application'), - _('Latest Version Available at Submission'), - _('Version in Use'), _('Last Used [UTC]')] + return [ + _('Mobile Worker'), + _('Project Space'), + _('Application'), + _('Latest Version Available at Submission'), + _('Version in Use'), + _('Last Used [UTC]'), + ] @property def rows(self): From 716a387656dd0cbf4471713edc3ec9a7c5869725 Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Wed, 1 Jan 2025 13:42:52 -0500 Subject: [PATCH 19/34] Rename column Because this tile based on `LastBuild`, build will be updated by either a form submission, or a sync, or mobile heartbeat. --- corehq/apps/enterprise/enterprise.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index 0ced41e69488..5b84bfd75648 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -652,7 +652,7 @@ def headers(self): _('Mobile Worker'), _('Project Space'), _('Application'), - _('Latest Version Available at Submission'), + _('Latest Version Available when Last Used'), _('Version in Use'), _('Last Used [UTC]'), ] From 990c12a94fd54f0a0775bc0f5965090b824bd581 Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Wed, 1 Jan 2025 13:46:49 -0500 Subject: [PATCH 20/34] Add ApplicationVersionCompliance api --- corehq/apps/enterprise/api/api.py | 2 ++ corehq/apps/enterprise/api/resources.py | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/corehq/apps/enterprise/api/api.py b/corehq/apps/enterprise/api/api.py index b04111b11017..9ceb2862be9a 100644 --- a/corehq/apps/enterprise/api/api.py +++ b/corehq/apps/enterprise/api/api.py @@ -10,6 +10,7 @@ SMSResource, APIUsageResource, TwoFactorAuthResource, + ApplicationVersionComplianceResource, ) v1_api = Api(api_name='v1') @@ -22,3 +23,4 @@ v1_api.register(SMSResource()) v1_api.register(APIUsageResource()) v1_api.register(TwoFactorAuthResource()) +v1_api.register(ApplicationVersionComplianceResource()) diff --git a/corehq/apps/enterprise/api/resources.py b/corehq/apps/enterprise/api/resources.py index 8d8656c52558..7f475ec1bece 100644 --- a/corehq/apps/enterprise/api/resources.py +++ b/corehq/apps/enterprise/api/resources.py @@ -461,3 +461,26 @@ def dehydrate(self, bundle): def get_primary_keys(self): return ('web_user', 'api_key_name',) + + +class ApplicationVersionComplianceResource(ODataEnterpriseReportResource): + mobile_worker = fields.CharField() + domain = fields.CharField() + application = fields.CharField() + latest_version_available_when_last_used = fields.CharField() + version_in_use = fields.CharField() + last_used = fields.DateTimeField() + + REPORT_SLUG = EnterpriseReport.APP_VERSION_COMPLIANCE + + def dehydrate(self, bundle): + bundle.data['mobile_worker'] = bundle.obj[0] + bundle.data['domain'] = bundle.obj[1] + bundle.data['application'] = bundle.obj[2] + bundle.data['latest_version_available_when_last_used'] = bundle.obj[3] + bundle.data['version_in_use'] = bundle.obj[4] + bundle.data['last_used'] = self.convert_datetime(bundle.obj[5]) + return bundle + + def get_primary_keys(self): + return ('mobile_worker', 'application',) From 25e5e521275eec772dda66abc24667216cda82c6 Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Thu, 2 Jan 2025 17:37:36 -0500 Subject: [PATCH 21/34] Refactor: Move get_app_builds inside get_latest_build_version_at_time --- corehq/apps/enterprise/enterprise.py | 33 ++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index 5b84bfd75648..bdf715ebc928 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -727,14 +727,28 @@ def _get_user_builds(self, domain, apps): if app_id not in apps or not build_version: continue build_version_date = DateTimeProperty.deserialize(build.get('build_version_date')) - all_builds = self.get_app_builds(domain, app_id, limit=self.INITIAL_QUERY_LIMIT) - latest_version = self.get_latest_build_version_at_time(all_builds, build_version_date) - if not latest_version: - all_builds = self.get_app_builds(domain, app_id) - latest_version = self.get_latest_build_version_at_time(all_builds, build_version_date) + latest_version = self.get_latest_build_version_at_time(domain, app_id, build_version_date) yield user, build, latest_version - def get_app_builds(self, domain, app_id, limit=None): + def get_latest_build_version_at_time(self, domain, app_id, time): + """ + Get the latest build version available at the given time. + + :param domain: The domain of the app + :param app_id: The application id + :param time: A datetime object representing the date of the build version to compare against + :return: The latest build version available at the given date + """ + builds = self.get_app_builds(domain, app_id, limit=self.INITIAL_QUERY_LIMIT) + latest_build = self._find_latest_build_version_from_builds(builds, time) + + if latest_build is None: + builds = self.get_app_builds(domain, app_id, start=self.INITIAL_QUERY_LIMIT) + latest_build = self._find_latest_build_version_from_builds(builds, time) + + return latest_build + + def get_app_builds(self, domain, app_id, limit=None, start=0): if app_id not in self.builds_by_app_id: app_es = ( AppES() @@ -744,6 +758,7 @@ def get_app_builds(self, domain, app_id, limit=None): .sort('version', desc=True) .is_released() .source(['_id', 'version', 'last_released', 'built_on']) + .start(start) ) if limit: app_es = app_es.size(limit) @@ -751,13 +766,13 @@ def get_app_builds(self, domain, app_id, limit=None): return self.builds_by_app_id[app_id] - def get_latest_build_version_at_time(self, all_builds, time): + def _find_latest_build_version_from_builds(self, all_builds, time): """ Get the latest build version at the time :param all_builds: List of raw build documents sorted by version in descending order - :param time: The date of the build version to compare against - :return: The latest build version available at the given date + :param time: A datetime object representing the date of the build version to compare against + :return: The latest build version available at the given date. """ for build_doc in all_builds: From c446ad1ebfcd3440147dc3172f9a330afa07936d Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Fri, 3 Jan 2025 11:32:21 -0500 Subject: [PATCH 22/34] Refactor: rename app_id to app_ids --- corehq/apps/enterprise/enterprise.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index bdf715ebc928..8127d90e1d30 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -681,9 +681,9 @@ def total(self): def rows_for_domain(self, domain): rows = [] app_name_by_id = {} - apps = get_app_ids_in_domain(domain) + app_ids = get_app_ids_in_domain(domain) - for user, build, latest_version in self._get_user_builds(domain, apps): + for user, build, latest_version in self._get_user_builds(domain, app_ids): if is_out_of_date(str(build['build_version']), str(latest_version)): app_id = build['app_id'] if app_id not in app_name_by_id: @@ -700,18 +700,18 @@ def rows_for_domain(self, domain): return rows def total_for_domain(self, domain): - apps = get_app_ids_in_domain(domain) + app_ids = get_app_ids_in_domain(domain) total_last_builds = 0 total_out_of_date = 0 - for user, build, latest_version in self._get_user_builds(domain, apps): + for user, build, latest_version in self._get_user_builds(domain, app_ids): total_last_builds += 1 if is_out_of_date(str(build['build_version']), str(latest_version)): total_out_of_date += 1 return total_out_of_date, total_last_builds - def _get_user_builds(self, domain, apps): + def _get_user_builds(self, domain, app_ids): user_query = (UserES() .domain(domain) .mobile_users() @@ -724,7 +724,7 @@ def _get_user_builds(self, domain, apps): for build in last_builds: app_id = build.get('app_id') build_version = build.get('build_version') - if app_id not in apps or not build_version: + if app_id not in app_ids or not build_version: continue build_version_date = DateTimeProperty.deserialize(build.get('build_version_date')) latest_version = self.get_latest_build_version_at_time(domain, app_id, build_version_date) From 980b616e77d674942bab2acb6b0d1e2ae5d75688 Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Mon, 6 Jan 2025 16:24:06 -0500 Subject: [PATCH 23/34] Clear the cache for initial 10 builds --- corehq/apps/enterprise/enterprise.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index 8127d90e1d30..3b5c3ff2e656 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -743,12 +743,13 @@ def get_latest_build_version_at_time(self, domain, app_id, time): latest_build = self._find_latest_build_version_from_builds(builds, time) if latest_build is None: - builds = self.get_app_builds(domain, app_id, start=self.INITIAL_QUERY_LIMIT) + del self.builds_by_app_id[app_id] + builds = self.get_app_builds(domain, app_id) latest_build = self._find_latest_build_version_from_builds(builds, time) return latest_build - def get_app_builds(self, domain, app_id, limit=None, start=0): + def get_app_builds(self, domain, app_id, limit=None): if app_id not in self.builds_by_app_id: app_es = ( AppES() @@ -758,7 +759,6 @@ def get_app_builds(self, domain, app_id, limit=None, start=0): .sort('version', desc=True) .is_released() .source(['_id', 'version', 'last_released', 'built_on']) - .start(start) ) if limit: app_es = app_es.size(limit) From 3a271f18862c3ad16f7705c6bc36c1e6b4585195 Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Tue, 7 Jan 2025 10:46:28 -0500 Subject: [PATCH 24/34] Refactor: renaming variable and text update --- corehq/apps/enterprise/enterprise.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index 3b5c3ff2e656..9860351d76e4 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -652,7 +652,7 @@ def headers(self): _('Mobile Worker'), _('Project Space'), _('Application'), - _('Latest Version Available when Last Used'), + _('Latest Version Available When Last Used'), _('Version in Use'), _('Last Used [UTC]'), ] @@ -666,17 +666,17 @@ def rows(self): @property def total(self): - total_out_of_date_rows = 0 - total_last_builds = 0 + total_out_of_date_builds = 0 + total_builds = 0 for domain in self.account.get_domains(): - domain_out_of_date_rows, domain_last_builds = self.total_for_domain(domain) - total_out_of_date_rows += domain_out_of_date_rows - total_last_builds += domain_last_builds + domain_out_of_date_builds, domain_builds = self.total_for_domain(domain) + total_out_of_date_builds += domain_out_of_date_builds + total_builds += domain_builds - total_up_to_date = total_last_builds - total_out_of_date_rows + total_up_to_date_builds = total_builds - total_out_of_date_builds - return _format_percentage_for_enterprise_tile(total_up_to_date, total_last_builds) + return _format_percentage_for_enterprise_tile(total_up_to_date_builds, total_builds) def rows_for_domain(self, domain): rows = [] From 08443df3c5703913c93a4a71b0f007bc9f919881 Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Tue, 7 Jan 2025 15:29:58 -0500 Subject: [PATCH 25/34] Remove Initial Query Limit since it makes the logic complicated It will worsen the performance by 20% --- corehq/apps/enterprise/enterprise.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index 9860351d76e4..9eb31d0a642c 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -639,7 +639,6 @@ def rows_for_domain(self, domain_obj): class EnterpriseAppVersionComplianceReport(EnterpriseReport): title = gettext_lazy('Application Version Compliance') total_description = gettext_lazy('% of Applications Up to Date Across All Mobile Workers') - INITIAL_QUERY_LIMIT = 10 def __init__(self, account, couch_user): super().__init__(account, couch_user) @@ -739,17 +738,12 @@ def get_latest_build_version_at_time(self, domain, app_id, time): :param time: A datetime object representing the date of the build version to compare against :return: The latest build version available at the given date """ - builds = self.get_app_builds(domain, app_id, limit=self.INITIAL_QUERY_LIMIT) + builds = self.get_app_builds(domain, app_id) latest_build = self._find_latest_build_version_from_builds(builds, time) - if latest_build is None: - del self.builds_by_app_id[app_id] - builds = self.get_app_builds(domain, app_id) - latest_build = self._find_latest_build_version_from_builds(builds, time) - return latest_build - def get_app_builds(self, domain, app_id, limit=None): + def get_app_builds(self, domain, app_id): if app_id not in self.builds_by_app_id: app_es = ( AppES() @@ -760,10 +754,7 @@ def get_app_builds(self, domain, app_id, limit=None): .is_released() .source(['_id', 'version', 'last_released', 'built_on']) ) - if limit: - app_es = app_es.size(limit) self.builds_by_app_id[app_id] = app_es.run().hits - return self.builds_by_app_id[app_id] def _find_latest_build_version_from_builds(self, all_builds, time): From 4d009d5829e1b6fea28c7ac85b334011f509145a Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Wed, 8 Jan 2025 11:10:17 -0500 Subject: [PATCH 26/34] Return a dictionary in _get_user_builds --- corehq/apps/enterprise/enterprise.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index 9eb31d0a642c..2f7130c20f13 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -682,18 +682,20 @@ def rows_for_domain(self, domain): app_name_by_id = {} app_ids = get_app_ids_in_domain(domain) - for user, build, latest_version in self._get_user_builds(domain, app_ids): - if is_out_of_date(str(build['build_version']), str(latest_version)): - app_id = build['app_id'] + for build_info in self._get_user_builds(domain, app_ids): + version_in_use = str(build_info['build']['build_version']) + latest_version = str(build_info['latest_version']) + if is_out_of_date(version_in_use, latest_version): + app_id = build_info['build']['app_id'] if app_id not in app_name_by_id: app_name_by_id[app_id] = Application.get_db().get(app_id).get('name') rows.append([ - user['username'], + build_info['username'], domain, app_name_by_id[app_id], latest_version, - build['build_version'], - self.format_date(DateTimeProperty.deserialize(build['build_version_date'])), + version_in_use, + self.format_date(DateTimeProperty.deserialize(build_info['build']['build_version_date'])), ]) return rows @@ -703,9 +705,9 @@ def total_for_domain(self, domain): total_last_builds = 0 total_out_of_date = 0 - for user, build, latest_version in self._get_user_builds(domain, app_ids): + for build_info in self._get_user_builds(domain, app_ids): total_last_builds += 1 - if is_out_of_date(str(build['build_version']), str(latest_version)): + if is_out_of_date(str(build_info['build']['build_version']), str(build_info['latest_version'])): total_out_of_date += 1 return total_out_of_date, total_last_builds @@ -727,7 +729,11 @@ def _get_user_builds(self, domain, app_ids): continue build_version_date = DateTimeProperty.deserialize(build.get('build_version_date')) latest_version = self.get_latest_build_version_at_time(domain, app_id, build_version_date) - yield user, build, latest_version + yield { + 'username': user['username'], + 'build': build, + 'latest_version': latest_version, + } def get_latest_build_version_at_time(self, domain, app_id, time): """ From 12a4ff624b06a8581092cf70b8c11f085340e6b4 Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Thu, 9 Jan 2025 09:02:29 -0500 Subject: [PATCH 27/34] Always show '--' when display metric --- corehq/apps/enterprise/enterprise.py | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index 2f7130c20f13..4d86218b1ac5 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -665,17 +665,7 @@ def rows(self): @property def total(self): - total_out_of_date_builds = 0 - total_builds = 0 - - for domain in self.account.get_domains(): - domain_out_of_date_builds, domain_builds = self.total_for_domain(domain) - total_out_of_date_builds += domain_out_of_date_builds - total_builds += domain_builds - - total_up_to_date_builds = total_builds - total_out_of_date_builds - - return _format_percentage_for_enterprise_tile(total_up_to_date_builds, total_builds) + return '--' def rows_for_domain(self, domain): rows = [] @@ -700,18 +690,6 @@ def rows_for_domain(self, domain): return rows - def total_for_domain(self, domain): - app_ids = get_app_ids_in_domain(domain) - total_last_builds = 0 - total_out_of_date = 0 - - for build_info in self._get_user_builds(domain, app_ids): - total_last_builds += 1 - if is_out_of_date(str(build_info['build']['build_version']), str(build_info['latest_version'])): - total_out_of_date += 1 - - return total_out_of_date, total_last_builds - def _get_user_builds(self, domain, app_ids): user_query = (UserES() .domain(domain) From 5bb7b5a94a024e004b718c4399b9da7892252b93 Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Thu, 9 Jan 2025 09:07:00 -0500 Subject: [PATCH 28/34] Refactor: renaming parameter so the function is self-explanatory and docstring can be removed --- corehq/apps/enterprise/enterprise.py | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index 4d86218b1ac5..9c418a11796c 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -706,24 +706,16 @@ def _get_user_builds(self, domain, app_ids): if app_id not in app_ids or not build_version: continue build_version_date = DateTimeProperty.deserialize(build.get('build_version_date')) - latest_version = self.get_latest_build_version_at_time(domain, app_id, build_version_date) + latest_version = self.get_latest_build_version(domain, app_id, build_version_date) yield { 'username': user['username'], 'build': build, 'latest_version': latest_version, } - def get_latest_build_version_at_time(self, domain, app_id, time): - """ - Get the latest build version available at the given time. - - :param domain: The domain of the app - :param app_id: The application id - :param time: A datetime object representing the date of the build version to compare against - :return: The latest build version available at the given date - """ + def get_latest_build_version(self, domain, app_id, at_datetime): builds = self.get_app_builds(domain, app_id) - latest_build = self._find_latest_build_version_from_builds(builds, time) + latest_build = self._find_latest_build_version_from_builds(builds, at_datetime) return latest_build @@ -741,15 +733,7 @@ def get_app_builds(self, domain, app_id): self.builds_by_app_id[app_id] = app_es.run().hits return self.builds_by_app_id[app_id] - def _find_latest_build_version_from_builds(self, all_builds, time): - """ - Get the latest build version at the time - - :param all_builds: List of raw build documents sorted by version in descending order - :param time: A datetime object representing the date of the build version to compare against - :return: The latest build version available at the given date. - """ - + def _find_latest_build_version_from_builds(self, all_builds, at_datetime): for build_doc in all_builds: build_id = build_doc['_id'] if build_id in self.build_by_build_id: @@ -765,7 +749,7 @@ def _find_latest_build_version_from_builds(self, all_builds, time): } self.build_by_build_id[build_id] = build_info - if build_info['last_released'] <= time: + if build_info['last_released'] <= at_datetime: return build_info['version'] return None From 936c43efb7c951a3262d4b8c1278cda2e35ea389 Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Thu, 9 Jan 2025 09:32:52 -0500 Subject: [PATCH 29/34] Refactor: rename variable to make it more self-explanatory --- corehq/apps/enterprise/enterprise.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index 9c418a11796c..7c8318eb4747 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -643,7 +643,7 @@ class EnterpriseAppVersionComplianceReport(EnterpriseReport): def __init__(self, account, couch_user): super().__init__(account, couch_user) self.builds_by_app_id = {} - self.build_by_build_id = {} + self.build_info_cache = {} @property def headers(self): @@ -672,20 +672,20 @@ def rows_for_domain(self, domain): app_name_by_id = {} app_ids = get_app_ids_in_domain(domain) - for build_info in self._get_user_builds(domain, app_ids): - version_in_use = str(build_info['build']['build_version']) - latest_version = str(build_info['latest_version']) + for row_data in self._get_user_builds(domain, app_ids): + version_in_use = str(row_data['build']['build_version']) + latest_version = str(row_data['latest_version']) if is_out_of_date(version_in_use, latest_version): - app_id = build_info['build']['app_id'] + app_id = row_data['build']['app_id'] if app_id not in app_name_by_id: app_name_by_id[app_id] = Application.get_db().get(app_id).get('name') rows.append([ - build_info['username'], + row_data['username'], domain, app_name_by_id[app_id], latest_version, version_in_use, - self.format_date(DateTimeProperty.deserialize(build_info['build']['build_version_date'])), + self.format_date(DateTimeProperty.deserialize(row_data['build']['build_version_date'])), ]) return rows @@ -736,9 +736,8 @@ def get_app_builds(self, domain, app_id): def _find_latest_build_version_from_builds(self, all_builds, at_datetime): for build_doc in all_builds: build_id = build_doc['_id'] - if build_id in self.build_by_build_id: - build_info = self.build_by_build_id[build_id] - else: + build_info = self.build_info_cache.get(build_id) + if not build_info: # last_released is added in 2019, build before 2019 don't have this field # TODO: have a migration to populate last_released from built_on # Then this code can be modified to use last_released only @@ -747,7 +746,7 @@ def _find_latest_build_version_from_builds(self, all_builds, at_datetime): 'version': build_doc['version'], 'last_released': DateTimeProperty.deserialize(released_date) } - self.build_by_build_id[build_id] = build_info + self.build_info_cache[build_id] = build_info if build_info['last_released'] <= at_datetime: return build_info['version'] From 8a4accd4b46b7b5cd4469ad4ea4eaea26bfa9bd0 Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Thu, 9 Jan 2025 09:38:12 -0500 Subject: [PATCH 30/34] Refactor: extract the logic in _find_latest_build_version_from_builds to a function --- corehq/apps/enterprise/enterprise.py | 29 +++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index 7c8318eb4747..d5b557f21e1c 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -735,23 +735,26 @@ def get_app_builds(self, domain, app_id): def _find_latest_build_version_from_builds(self, all_builds, at_datetime): for build_doc in all_builds: - build_id = build_doc['_id'] - build_info = self.build_info_cache.get(build_id) - if not build_info: - # last_released is added in 2019, build before 2019 don't have this field - # TODO: have a migration to populate last_released from built_on - # Then this code can be modified to use last_released only - released_date = build_doc['last_released'] or build_doc['built_on'] - build_info = { - 'version': build_doc['version'], - 'last_released': DateTimeProperty.deserialize(released_date) - } - self.build_info_cache[build_id] = build_info - + build_info = self._get_build_info(build_doc) if build_info['last_released'] <= at_datetime: return build_info['version'] return None + def _get_build_info(self, build_doc): + build_id = build_doc['_id'] + build_info = self.build_info_cache.get(build_id) + if not build_info: + # last_released is added in 2019, build before 2019 don't have this field + # TODO: have a migration to populate last_released from built_on + # Then this code can be modified to use last_released only + released_date = build_doc.get('last_released') or build_doc['built_on'] + build_info = { + 'version': build_doc['version'], + 'last_released': DateTimeProperty.deserialize(released_date) + } + self.build_info_cache[build_id] = build_info + return build_info + def _format_percentage_for_enterprise_tile(dividend, divisor): if not divisor: From c79f198120fb8001c67bc05d5274c5f2a35deace Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Thu, 9 Jan 2025 09:40:38 -0500 Subject: [PATCH 31/34] Refactor: check the cache first and return immediately --- corehq/apps/enterprise/enterprise.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index d5b557f21e1c..39b05357bbda 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -720,17 +720,19 @@ def get_latest_build_version(self, domain, app_id, at_datetime): return latest_build def get_app_builds(self, domain, app_id): - if app_id not in self.builds_by_app_id: - app_es = ( - AppES() - .domain(domain) - .is_build() - .app_id(app_id) - .sort('version', desc=True) - .is_released() - .source(['_id', 'version', 'last_released', 'built_on']) - ) - self.builds_by_app_id[app_id] = app_es.run().hits + if app_id in self.builds_by_app_id: + return self.builds_by_app_id[app_id] + + app_es = ( + AppES() + .domain(domain) + .is_build() + .app_id(app_id) + .sort('version', desc=True) + .is_released() + .source(['_id', 'version', 'last_released', 'built_on']) + ) + self.builds_by_app_id[app_id] = app_es.run().hits return self.builds_by_app_id[app_id] def _find_latest_build_version_from_builds(self, all_builds, at_datetime): From 3a0985a9bbb38580f6bf7dd01267f0902ddd6478 Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Tue, 14 Jan 2025 22:43:21 -0500 Subject: [PATCH 32/34] Add comment --- corehq/apps/enterprise/enterprise.py | 1 + 1 file changed, 1 insertion(+) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index 39b05357bbda..c292a1d8ddd5 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -665,6 +665,7 @@ def rows(self): @property def total(self): + # Skip the stat for this report due to performance issue return '--' def rows_for_domain(self, domain): From df144d43d8404bb056f1f5370cf4176ca312f11a Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Wed, 22 Jan 2025 14:00:56 -0500 Subject: [PATCH 33/34] Refactor: rename to improve readability --- corehq/apps/enterprise/enterprise.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index 6ae0a7a8818c..0eeadc06b1e5 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -785,25 +785,29 @@ def rows_for_domain(self, domain): app_name_by_id = {} app_ids = get_app_ids_in_domain(domain) - for row_data in self._get_user_builds(domain, app_ids): - version_in_use = str(row_data['build']['build_version']) - latest_version = str(row_data['latest_version']) + for build_and_latest_version in self.all_last_builds_with_latest_version(domain, app_ids): + version_in_use = str(build_and_latest_version['build']['build_version']) + latest_version = str(build_and_latest_version['latest_version']) if is_out_of_date(version_in_use, latest_version): - app_id = row_data['build']['app_id'] + app_id = build_and_latest_version['build']['app_id'] if app_id not in app_name_by_id: app_name_by_id[app_id] = Application.get_db().get(app_id).get('name') rows.append([ - row_data['username'], + build_and_latest_version['username'], domain, app_name_by_id[app_id], latest_version, version_in_use, - self.format_date(DateTimeProperty.deserialize(row_data['build']['build_version_date'])), + self.format_date( + DateTimeProperty.deserialize( + build_and_latest_version['build']['build_version_date'] + ) + ), ]) return rows - def _get_user_builds(self, domain, app_ids): + def all_last_builds_with_latest_version(self, domain, app_ids): user_query = (UserES() .domain(domain) .mobile_users() From 95fb0d81367333835738b8e55c9f08c7f9408b92 Mon Sep 17 00:00:00 2001 From: Jing Cheng Date: Wed, 22 Jan 2025 14:38:14 -0500 Subject: [PATCH 34/34] Update the descriptor --- corehq/apps/enterprise/enterprise.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corehq/apps/enterprise/enterprise.py b/corehq/apps/enterprise/enterprise.py index 0eeadc06b1e5..af733ead8e47 100644 --- a/corehq/apps/enterprise/enterprise.py +++ b/corehq/apps/enterprise/enterprise.py @@ -750,7 +750,7 @@ def rows_for_domain(self, domain_obj): class EnterpriseAppVersionComplianceReport(EnterpriseReport): title = gettext_lazy('Application Version Compliance') - total_description = gettext_lazy('% of Applications Up to Date Across All Mobile Workers') + total_description = gettext_lazy('The statistic of this tile is not currently supported') def __init__(self, account, couch_user): super().__init__(account, couch_user)