diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 40d4befb5..5ae3457c3 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -3,7 +3,14 @@ import copy from typing import Optional from django import forms -from django.db.models import Value, CharField, Q +from django.db.models import ( + Case, + CharField, + F, + Q, + Value, + When, +) from django.db.models.functions import Concat, Coalesce from django.http import HttpResponseRedirect from registrar.models.federal_agency import FederalAgency @@ -1467,21 +1474,57 @@ class Meta: class DomainInformationAdmin(ListHeaderAdmin, ImportExportModelAdmin): """Customize domain information admin class.""" + class GenericOrgFilter(admin.SimpleListFilter): + """Custom Generic Organization filter that accomodates portfolio feature. + If we have a portfolio, use the portfolio's organization. If not, use the + organization in the Domain Information object.""" + + title = "generic organization" + parameter_name = "converted_generic_orgs" + + def lookups(self, request, model_admin): + converted_generic_orgs = set() + + # Populate the set with tuples of (value, display value) + for domain_info in DomainInformation.objects.all(): + converted_generic_org = domain_info.converted_generic_org_type # Actual value + converted_generic_org_display = domain_info.converted_generic_org_type_display # Display value + + if converted_generic_org: + converted_generic_orgs.add((converted_generic_org, converted_generic_org_display)) # Value, Display + + # Sort the set by display value + return sorted(converted_generic_orgs, key=lambda x: x[1]) # x[1] is the display value + + # Filter queryset + def queryset(self, request, queryset): + if self.value(): # Check if a generic org is selected in the filter + return queryset.filter( + Q(portfolio__organization_type=self.value()) + | Q(portfolio__isnull=True, generic_org_type=self.value()) + ) + return queryset + resource_classes = [DomainInformationResource] form = DomainInformationAdminForm + # Customize column header text + @admin.display(description=_("Generic Org Type")) + def converted_generic_org_type(self, obj): + return obj.converted_generic_org_type_display + # Columns list_display = [ "domain", - "generic_org_type", + "converted_generic_org_type", "created_at", ] orderable_fk_fields = [("domain", "name")] # Filters - list_filter = ["generic_org_type"] + list_filter = [GenericOrgFilter] # Search search_fields = [ @@ -1661,24 +1704,23 @@ class GenericOrgFilter(admin.SimpleListFilter): def lookups(self, request, model_admin): converted_generic_orgs = set() + # Populate the set with tuples of (value, display value) for domain_request in DomainRequest.objects.all(): - converted_generic_org = domain_request.converted_generic_org_type + converted_generic_org = domain_request.converted_generic_org_type # Actual value + converted_generic_org_display = domain_request.converted_generic_org_type_display # Display value + if converted_generic_org: - converted_generic_orgs.add(converted_generic_org) + converted_generic_orgs.add((converted_generic_org, converted_generic_org_display)) # Value, Display - return sorted((org, org) for org in converted_generic_orgs) + # Sort the set by display value + return sorted(converted_generic_orgs, key=lambda x: x[1]) # x[1] is the display value # Filter queryset def queryset(self, request, queryset): if self.value(): # Check if a generic org is selected in the filter return queryset.filter( - # Filter based on the generic org value returned by converted_generic_org_type - id__in=[ - domain_request.id - for domain_request in queryset - if domain_request.converted_generic_org_type - and domain_request.converted_generic_org_type == self.value() - ] + Q(portfolio__organization_type=self.value()) + | Q(portfolio__isnull=True, generic_org_type=self.value()) ) return queryset @@ -1693,24 +1735,25 @@ class FederalTypeFilter(admin.SimpleListFilter): def lookups(self, request, model_admin): converted_federal_types = set() + # Populate the set with tuples of (value, display value) for domain_request in DomainRequest.objects.all(): - converted_federal_type = domain_request.converted_federal_type + converted_federal_type = domain_request.converted_federal_type # Actual value + converted_federal_type_display = domain_request.converted_federal_type_display # Display value + if converted_federal_type: - converted_federal_types.add(converted_federal_type) + converted_federal_types.add( + (converted_federal_type, converted_federal_type_display) # Value, Display + ) - return sorted((type, type) for type in converted_federal_types) + # Sort the set by display value + return sorted(converted_federal_types, key=lambda x: x[1]) # x[1] is the display value # Filter queryset def queryset(self, request, queryset): - if self.value(): # Check if federal Type is selected in the filter + if self.value(): # Check if a federal type is selected in the filter return queryset.filter( - # Filter based on the federal type returned by converted_federal_type - id__in=[ - domain_request.id - for domain_request in queryset - if domain_request.converted_federal_type - and domain_request.converted_federal_type == self.value() - ] + Q(portfolio__federal_agency__federal_type=self.value()) + | Q(portfolio__isnull=True, federal_type=self.value()) ) return queryset @@ -1776,7 +1819,7 @@ def queryset(self, request, queryset): @admin.display(description=_("Generic Org Type")) def converted_generic_org_type(self, obj): - return obj.converted_generic_org_type + return obj.converted_generic_org_type_display @admin.display(description=_("Organization Name")) def converted_organization_name(self, obj): @@ -1788,7 +1831,7 @@ def converted_federal_agency(self, obj): @admin.display(description=_("Federal Type")) def converted_federal_type(self, obj): - return obj.converted_federal_type + return obj.converted_federal_type_display @admin.display(description=_("City")) def converted_city(self, obj): @@ -2679,6 +2722,7 @@ class DomainAdmin(ListHeaderAdmin, ImportExportModelAdmin): resource_classes = [DomainResource] + # ------- FILTERS class ElectionOfficeFilter(admin.SimpleListFilter): """Define a custom filter for is_election_board""" @@ -2697,18 +2741,135 @@ def queryset(self, request, queryset): if self.value() == "0": return queryset.filter(Q(domain_info__is_election_board=False) | Q(domain_info__is_election_board=None)) + class GenericOrgFilter(admin.SimpleListFilter): + """Custom Generic Organization filter that accomodates portfolio feature. + If we have a portfolio, use the portfolio's organization. If not, use the + organization in the Domain Information object.""" + + title = "generic organization" + parameter_name = "converted_generic_orgs" + + def lookups(self, request, model_admin): + converted_generic_orgs = set() + + # Populate the set with tuples of (value, display value) + for domain_info in DomainInformation.objects.all(): + converted_generic_org = domain_info.converted_generic_org_type # Actual value + converted_generic_org_display = domain_info.converted_generic_org_type_display # Display value + + if converted_generic_org: + converted_generic_orgs.add((converted_generic_org, converted_generic_org_display)) # Value, Display + + # Sort the set by display value + return sorted(converted_generic_orgs, key=lambda x: x[1]) # x[1] is the display value + + # Filter queryset + def queryset(self, request, queryset): + if self.value(): # Check if a generic org is selected in the filter + return queryset.filter( + Q(domain_info__portfolio__organization_type=self.value()) + | Q(domain_info__portfolio__isnull=True, domain_info__generic_org_type=self.value()) + ) + + return queryset + + class FederalTypeFilter(admin.SimpleListFilter): + """Custom Federal Type filter that accomodates portfolio feature. + If we have a portfolio, use the portfolio's federal type. If not, use the + federal type in the Domain Information object.""" + + title = "federal type" + parameter_name = "converted_federal_types" + + def lookups(self, request, model_admin): + converted_federal_types = set() + + # Populate the set with tuples of (value, display value) + for domain_info in DomainInformation.objects.all(): + converted_federal_type = domain_info.converted_federal_type # Actual value + converted_federal_type_display = domain_info.converted_federal_type_display # Display value + + if converted_federal_type: + converted_federal_types.add( + (converted_federal_type, converted_federal_type_display) # Value, Display + ) + + # Sort the set by display value + return sorted(converted_federal_types, key=lambda x: x[1]) # x[1] is the display value + + # Filter queryset + def queryset(self, request, queryset): + if self.value(): # Check if a federal type is selected in the filter + return queryset.filter( + Q(domain_info__portfolio__federal_agency__federal_type=self.value()) + | Q(domain_info__portfolio__isnull=True, domain_info__federal_agency__federal_type=self.value()) + ) + return queryset + + def get_annotated_queryset(self, queryset): + return queryset.annotate( + converted_generic_org_type=Case( + # When portfolio is present, use its value instead + When(domain_info__portfolio__isnull=False, then=F("domain_info__portfolio__organization_type")), + # Otherwise, return the natively assigned value + default=F("domain_info__generic_org_type"), + ), + converted_federal_agency=Case( + # When portfolio is present, use its value instead + When( + Q(domain_info__portfolio__isnull=False) & Q(domain_info__portfolio__federal_agency__isnull=False), + then=F("domain_info__portfolio__federal_agency__agency"), + ), + # Otherwise, return the natively assigned value + default=F("domain_info__federal_agency__agency"), + ), + converted_federal_type=Case( + # When portfolio is present, use its value instead + When( + Q(domain_info__portfolio__isnull=False) & Q(domain_info__portfolio__federal_agency__isnull=False), + then=F("domain_info__portfolio__federal_agency__federal_type"), + ), + # Otherwise, return the natively assigned value + default=F("domain_info__federal_agency__federal_type"), + ), + converted_organization_name=Case( + # When portfolio is present, use its value instead + When(domain_info__portfolio__isnull=False, then=F("domain_info__portfolio__organization_name")), + # Otherwise, return the natively assigned value + default=F("domain_info__organization_name"), + ), + converted_city=Case( + # When portfolio is present, use its value instead + When(domain_info__portfolio__isnull=False, then=F("domain_info__portfolio__city")), + # Otherwise, return the natively assigned value + default=F("domain_info__city"), + ), + converted_state_territory=Case( + # When portfolio is present, use its value instead + When(domain_info__portfolio__isnull=False, then=F("domain_info__portfolio__state_territory")), + # Otherwise, return the natively assigned value + default=F("domain_info__state_territory"), + ), + ) + + # Filters + list_filter = [GenericOrgFilter, FederalTypeFilter, ElectionOfficeFilter, "state"] + + # ------- END FILTERS + + # Inlines inlines = [DomainInformationInline] # Columns list_display = [ "name", - "generic_org_type", - "federal_type", - "federal_agency", - "organization_name", + "converted_generic_org_type", + "converted_federal_type", + "converted_federal_agency", + "converted_organization_name", "custom_election_board", - "city", - "state_territory", + "converted_city", + "converted_state_territory", "state", "expiration_date", "created_at", @@ -2723,28 +2884,81 @@ def queryset(self, request, queryset): ), ) + # ------- Domain Information Fields + + # --- Generic Org Type + # Use converted value in the table + @admin.display(description=_("Generic Org Type")) + def converted_generic_org_type(self, obj): + return obj.domain_info.converted_generic_org_type_display + + converted_generic_org_type.admin_order_field = "converted_generic_org_type" # type: ignore + + # Use native value for the change form def generic_org_type(self, obj): return obj.domain_info.get_generic_org_type_display() - generic_org_type.admin_order_field = "domain_info__generic_org_type" # type: ignore + # --- Federal Agency + @admin.display(description=_("Federal Agency")) + def converted_federal_agency(self, obj): + return obj.domain_info.converted_federal_agency + converted_federal_agency.admin_order_field = "converted_federal_agency" # type: ignore + + # Use native value for the change form def federal_agency(self, obj): if obj.domain_info: return obj.domain_info.federal_agency else: return None - federal_agency.admin_order_field = "domain_info__federal_agency" # type: ignore + # --- Federal Type + # Use converted value in the table + @admin.display(description=_("Federal Type")) + def converted_federal_type(self, obj): + return obj.domain_info.converted_federal_type_display + + converted_federal_type.admin_order_field = "converted_federal_type" # type: ignore + # Use native value for the change form def federal_type(self, obj): return obj.domain_info.federal_type if obj.domain_info else None - federal_type.admin_order_field = "domain_info__federal_type" # type: ignore + # --- Organization Name + # Use converted value in the table + @admin.display(description=_("Organization Name")) + def converted_organization_name(self, obj): + return obj.domain_info.converted_organization_name + + converted_organization_name.admin_order_field = "converted_organization_name" # type: ignore + # Use native value for the change form def organization_name(self, obj): return obj.domain_info.organization_name if obj.domain_info else None - organization_name.admin_order_field = "domain_info__organization_name" # type: ignore + # --- City + # Use converted value in the table + @admin.display(description=_("City")) + def converted_city(self, obj): + return obj.domain_info.converted_city + + converted_city.admin_order_field = "converted_city" # type: ignore + + # Use native value for the change form + def city(self, obj): + return obj.domain_info.city if obj.domain_info else None + + # --- State + # Use converted value in the table + @admin.display(description=_("State / territory")) + def converted_state_territory(self, obj): + return obj.domain_info.converted_state_territory + + converted_state_territory.admin_order_field = "converted_state_territory" # type: ignore + + # Use native value for the change form + def state_territory(self, obj): + return obj.domain_info.state_territory if obj.domain_info else None def dnssecdata(self, obj): return "Yes" if obj.dnssecdata else "No" @@ -2777,23 +2991,14 @@ def custom_election_board(self, obj): custom_election_board.admin_order_field = "domain_info__is_election_board" # type: ignore custom_election_board.short_description = "Election office" # type: ignore - def city(self, obj): - return obj.domain_info.city if obj.domain_info else None - - city.admin_order_field = "domain_info__city" # type: ignore - - @admin.display(description=_("State / territory")) - def state_territory(self, obj): - return obj.domain_info.state_territory if obj.domain_info else None - - state_territory.admin_order_field = "domain_info__state_territory" # type: ignore - - # Filters - list_filter = ["domain_info__generic_org_type", "domain_info__federal_type", ElectionOfficeFilter, "state"] - + # Search search_fields = ["name"] search_help_text = "Search by domain name." + + # Change Form change_form_template = "django/admin/domain_change_form.html" + + # Readonly Fields readonly_fields = ( "state", "expiration_date", @@ -3058,7 +3263,8 @@ def has_change_permission(self, request, obj=None): def get_queryset(self, request): """Custom get_queryset to filter by portfolio if portfolio is in the request params.""" - qs = super().get_queryset(request) + initial_qs = super().get_queryset(request) + qs = self.get_annotated_queryset(initial_qs) # Check if a 'portfolio' parameter is passed in the request portfolio_id = request.GET.get("portfolio") if portfolio_id: diff --git a/src/registrar/config/urls.py b/src/registrar/config/urls.py index 66a8a9b74..277db5b66 100644 --- a/src/registrar/config/urls.py +++ b/src/registrar/config/urls.py @@ -255,11 +255,6 @@ ExportDataTypeRequests.as_view(), name="export_data_type_requests", ), - path( - "reports/export_data_type_requests/", - ExportDataTypeRequests.as_view(), - name="export_data_type_requests", - ), path( "domain-request//edit/", views.DomainRequestWizard.as_view(), diff --git a/src/registrar/models/domain_information.py b/src/registrar/models/domain_information.py index 7dadf26ac..378d59137 100644 --- a/src/registrar/models/domain_information.py +++ b/src/registrar/models/domain_information.py @@ -426,13 +426,14 @@ def get_state_display_of_domain(self): else: return None + # ----- Portfolio Properties ----- + @property def converted_organization_name(self): if self.portfolio: return self.portfolio.organization_name return self.organization_name - # ----- Portfolio Properties ----- @property def converted_generic_org_type(self): if self.portfolio: @@ -442,8 +443,8 @@ def converted_generic_org_type(self): @property def converted_federal_agency(self): if self.portfolio: - return self.portfolio.federal_agency - return self.federal_agency + return self.portfolio.federal_agency.agency + return self.federal_agency.agency @property def converted_federal_type(self): @@ -454,20 +455,20 @@ def converted_federal_type(self): @property def converted_senior_official(self): if self.portfolio: - return self.portfolio.senior_official - return self.senior_official + return self.portfolio.display_senior_official + return self.display_senior_official @property def converted_address_line1(self): if self.portfolio: - return self.portfolio.address_line1 - return self.address_line1 + return self.portfolio.display_address_line1 + return self.display_address_line1 @property def converted_address_line2(self): if self.portfolio: - return self.portfolio.address_line2 - return self.address_line2 + return self.portfolio.display_address_line2 + return self.display_address_line2 @property def converted_city(self): @@ -478,17 +479,30 @@ def converted_city(self): @property def converted_state_territory(self): if self.portfolio: - return self.portfolio.state_territory - return self.state_territory + return self.portfolio.get_state_territory_display() + return self.get_state_territory_display() @property def converted_zipcode(self): if self.portfolio: - return self.portfolio.zipcode - return self.zipcode + return self.portfolio.display_zipcode + return self.display_zipcode @property def converted_urbanization(self): if self.portfolio: - return self.portfolio.urbanization - return self.urbanization + return self.portfolio.display_urbanization + return self.display_urbanization + + # ----- Portfolio Properties (display values)----- + @property + def converted_generic_org_type_display(self): + if self.portfolio: + return self.portfolio.get_organization_type_display() + return self.get_generic_org_type_display() + + @property + def converted_federal_type_display(self): + if self.portfolio: + return self.portfolio.federal_agency.get_federal_type_display() + return self.get_federal_type_display() diff --git a/src/registrar/models/domain_request.py b/src/registrar/models/domain_request.py index 0d8bbd5cf..3fa889a3b 100644 --- a/src/registrar/models/domain_request.py +++ b/src/registrar/models/domain_request.py @@ -1437,6 +1437,18 @@ def converted_federal_type(self): return self.portfolio.federal_type return self.federal_type + @property + def converted_address_line1(self): + if self.portfolio: + return self.portfolio.address_line1 + return self.address_line1 + + @property + def converted_address_line2(self): + if self.portfolio: + return self.portfolio.address_line2 + return self.address_line2 + @property def converted_city(self): if self.portfolio: @@ -1449,8 +1461,33 @@ def converted_state_territory(self): return self.portfolio.state_territory return self.state_territory + @property + def converted_urbanization(self): + if self.portfolio: + return self.portfolio.urbanization + return self.urbanization + + @property + def converted_zipcode(self): + if self.portfolio: + return self.portfolio.zipcode + return self.zipcode + @property def converted_senior_official(self): if self.portfolio: return self.portfolio.senior_official return self.senior_official + + # ----- Portfolio Properties (display values)----- + @property + def converted_generic_org_type_display(self): + if self.portfolio: + return self.portfolio.get_organization_type_display() + return self.get_generic_org_type_display() + + @property + def converted_federal_type_display(self): + if self.portfolio: + return self.portfolio.federal_agency.get_federal_type_display() + return self.get_federal_type_display() diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index 6a5bbdd78..e1f4f5a27 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -563,9 +563,12 @@ def sharedSetUp(cls): cls.federal_agency_1, _ = FederalAgency.objects.get_or_create(agency="World War I Centennial Commission") cls.federal_agency_2, _ = FederalAgency.objects.get_or_create(agency="Armed Forces Retirement Home") + cls.federal_agency_3, _ = FederalAgency.objects.get_or_create( + agency="Portfolio 1 Federal Agency", federal_type="executive" + ) cls.portfolio_1, _ = Portfolio.objects.get_or_create( - creator=cls.custom_superuser, federal_agency=cls.federal_agency_1 + creator=cls.custom_superuser, federal_agency=cls.federal_agency_3, organization_type="federal" ) current_date = get_time_aware_date(datetime(2024, 4, 2)) diff --git a/src/registrar/tests/test_admin_domain.py b/src/registrar/tests/test_admin_domain.py index f02b59a91..0a2af50db 100644 --- a/src/registrar/tests/test_admin_domain.py +++ b/src/registrar/tests/test_admin_domain.py @@ -728,9 +728,9 @@ def test_short_org_name_in_domains_list(self): response = self.client.get("/admin/registrar/domain/") # There are 4 template references to Federal (4) plus four references in the table # for our actual domain_request - self.assertContains(response, "Federal", count=56) + self.assertContains(response, "Federal", count=57) # This may be a bit more robust - self.assertContains(response, 'Federal', count=1) + self.assertContains(response, 'Federal', count=1) # Now let's make sure the long description does not exist self.assertNotContains(response, "Federal: an agency of the U.S. government") diff --git a/src/registrar/tests/test_admin_request.py b/src/registrar/tests/test_admin_request.py index a903188f3..d2e4c1c1b 100644 --- a/src/registrar/tests/test_admin_request.py +++ b/src/registrar/tests/test_admin_request.py @@ -576,9 +576,9 @@ def test_short_org_name_in_domain_requests_list(self): response = self.client.get("/admin/registrar/domainrequest/?generic_org_type__exact=federal") # There are 2 template references to Federal (4) and two in the results data # of the request - self.assertContains(response, "Federal", count=51) + self.assertContains(response, "Federal", count=55) # This may be a bit more robust - self.assertContains(response, 'federal', count=1) + self.assertContains(response, 'Federal', count=1) # Now let's make sure the long description does not exist self.assertNotContains(response, "Federal: an agency of the U.S. government") @@ -1693,7 +1693,6 @@ def test_readonly_when_restricted_creator(self): "notes", "alternative_domains", ] - self.maxDiff = None self.assertEqual(readonly_fields, expected_fields) def test_readonly_fields_for_analyst(self): @@ -1702,7 +1701,6 @@ def test_readonly_fields_for_analyst(self): request.user = self.staffuser readonly_fields = self.admin.get_readonly_fields(request) - self.maxDiff = None expected_fields = [ "portfolio_senior_official", "portfolio_organization_type", diff --git a/src/registrar/tests/test_migrations.py b/src/registrar/tests/test_migrations.py index eaaae8727..11daca870 100644 --- a/src/registrar/tests/test_migrations.py +++ b/src/registrar/tests/test_migrations.py @@ -63,7 +63,6 @@ def test_groups_created(self): # Get the codenames of actual permissions associated with the group actual_permissions = [p.codename for p in cisa_analysts_group.permissions.all()] - self.maxDiff = None # Assert that the actual permissions match the expected permissions self.assertListEqual(actual_permissions, expected_permissions) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 377216aa4..d801ce76a 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -71,8 +71,8 @@ def test_generate_federal_report(self): fake_open = mock_open() expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), + call("cdomain1.gov,Federal - Executive,Portfolio 1 Federal Agency,,,,(blank)\r\n"), call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), - call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), call("adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), ] @@ -93,8 +93,8 @@ def test_generate_full_report(self): fake_open = mock_open() expected_file_content = [ call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), + call("cdomain1.gov,Federal - Executive,Portfolio 1 Federal Agency,,,,(blank)\r\n"), call("cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), - call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\r\n"), call("adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,,(blank)\r\n"), call("zdomain12.gov,Interstate,,,,,(blank)\r\n"), @@ -251,32 +251,35 @@ def test_domain_data_type(self): # We expect READY domains, # sorted alphabetially by domain name expected_content = ( - "Domain name,Status,First ready on,Expiration date,Domain type,Agency,Organization name,City,State,SO," - "SO email,Security contact email,Domain managers,Invited domain managers\n" - "cdomain11.gov,Ready,2024-04-02,(blank),Federal - Executive,World War I Centennial Commission,,,,(blank),,," + "Domain name,Status,First ready on,Expiration date,Domain type,Agency," + "Organization name,City,State,SO,SO email," + "Security contact email,Domain managers,Invited domain managers\n" + "adomain2.gov,Dns needed,(blank),(blank),Federal - Executive," + "Portfolio 1 Federal Agency,,,, ,,(blank)," + "meoward@rocks.com,squeaker@rocks.com\n" + "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive," + "Portfolio 1 Federal Agency,,,, ,,(blank)," + '"big_lebowski@dude.co, info@example.com, meoward@rocks.com",woofwardthethird@rocks.com\n' + "cdomain11.gov,Ready,2024-04-02,(blank),Federal - Executive," + "World War I Centennial Commission,,,, ,,(blank)," "meoward@rocks.com,\n" - "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive,World War I Centennial Commission,,," - ',,,(blank),"big_lebowski@dude.co, info@example.com, meoward@rocks.com",' - "woofwardthethird@rocks.com\n" - "adomain10.gov,Ready,2024-04-03,(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,," + "adomain10.gov,Ready,2024-04-03,(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),," "squeaker@rocks.com\n" - "bdomain4.gov,Unknown,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" - "bdomain5.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" - "bdomain6.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" - "ddomain3.gov,On hold,(blank),2023-11-15,Federal,Armed Forces Retirement Home,,,,,," - "security@mail.gov,,\n" - "sdomain8.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" - "xdomain7.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" - "zdomain9.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,,(blank),,,,\n" - "adomain2.gov,Dns needed,(blank),(blank),Interstate,,,,,(blank),,," - "meoward@rocks.com,squeaker@rocks.com\n" - "zdomain12.gov,Ready,2024-04-02,(blank),Interstate,,,,,(blank),,,meoward@rocks.com,\n" + "bdomain4.gov,Unknown,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" + "bdomain5.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" + "bdomain6.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" + "ddomain3.gov,On hold,(blank),2023-11-15,Federal," + "Armed Forces Retirement Home,,,, ,,security@mail.gov,,\n" + "sdomain8.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" + "xdomain7.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" + "zdomain9.gov,Deleted,(blank),(blank),Federal,Armed Forces Retirement Home,,,, ,,(blank),,\n" + "zdomain12.gov,Ready,2024-04-02,(blank),Interstate,,,,, ,,(blank),meoward@rocks.com,\n" ) + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -312,20 +315,17 @@ def test_domain_data_type_user(self): # We expect only domains associated with the user expected_content = ( "Domain name,Status,First ready on,Expiration date,Domain type,Agency,Organization name," - "City,State,SO,SO email," - "Security contact email,Domain managers,Invited domain managers\n" - "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive,World War I Centennial Commission,,,, ,," - '(blank),"big_lebowski@dude.co, info@example.com, meoward@rocks.com",' - "woofwardthethird@rocks.com\n" - "adomain2.gov,Dns needed,(blank),(blank),Interstate,,,,, ,,(blank)," + "City,State,SO,SO email,Security contact email,Domain managers,Invited domain managers\n" + "adomain2.gov,Dns needed,(blank),(blank),Federal - Executive,Portfolio 1 Federal Agency,,,, ,,(blank)," '"info@example.com, meoward@rocks.com",squeaker@rocks.com\n' + "defaultsecurity.gov,Ready,2023-11-01,(blank),Federal - Executive,Portfolio 1 Federal Agency,,,, ,,(blank)," + '"big_lebowski@dude.co, info@example.com, meoward@rocks.com",woofwardthethird@rocks.com\n' ) # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -493,17 +493,17 @@ def test_domain_data_full(self): # sorted alphabetially by domain name expected_content = ( "Domain name,Domain type,Agency,Organization name,City,State,Security contact email\n" - "cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" - "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" - "adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\n" - "ddomain3.gov,Federal,Armed Forces Retirement Home,,,,security@mail.gov\n" + "defaultsecurity.gov,Federal - Executive,Portfolio1FederalAgency,,,,(blank)\n" + "cdomain11.gov,Federal - Executive,WorldWarICentennialCommission,,,,(blank)\n" + "adomain10.gov,Federal,ArmedForcesRetirementHome,,,,(blank)\n" + "ddomain3.gov,Federal,ArmedForcesRetirementHome,,,,security@mail.gov\n" "zdomain12.gov,Interstate,,,,,(blank)\n" ) + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -533,16 +533,16 @@ def test_domain_data_federal(self): # sorted alphabetially by domain name expected_content = ( "Domain name,Domain type,Agency,Organization name,City,State,Security contact email\n" - "cdomain11.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" - "defaultsecurity.gov,Federal - Executive,World War I Centennial Commission,,,,(blank)\n" - "adomain10.gov,Federal,Armed Forces Retirement Home,,,,(blank)\n" - "ddomain3.gov,Federal,Armed Forces Retirement Home,,,,security@mail.gov\n" + "defaultsecurity.gov,Federal - Executive,Portfolio1FederalAgency,,,,(blank)\n" + "cdomain11.gov,Federal - Executive,WorldWarICentennialCommission,,,,(blank)\n" + "adomain10.gov,Federal,ArmedForcesRetirementHome,,,,(blank)\n" + "ddomain3.gov,Federal,ArmedForcesRetirementHome,,,,security@mail.gov\n" ) + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -587,13 +587,13 @@ def test_domain_growth(self): expected_content = ( "Domain name,Domain type,Agency,Organization name,City," "State,Status,Expiration date, Deleted\n" - "cdomain1.gov,Federal-Executive,World War I Centennial Commission,,,,Ready,(blank)\n" - "adomain10.gov,Federal,Armed Forces Retirement Home,,,,Ready,(blank)\n" - "cdomain11.govFederal-ExecutiveWorldWarICentennialCommissionReady(blank)\n" - "zdomain12.govInterstateReady(blank)\n" + "cdomain1.gov,Federal-Executive,Portfolio1FederalAgency,Ready,(blank)\n" + "adomain10.gov,Federal,ArmedForcesRetirementHome,Ready,(blank)\n" + "cdomain11.gov,Federal-Executive,WorldWarICentennialCommission,Ready,(blank)\n" + "zdomain12.gov,Interstate,Ready,(blank)\n" "zdomain9.gov,Federal,ArmedForcesRetirementHome,Deleted,(blank),2024-04-01\n" - "sdomain8.gov,Federal,Armed Forces Retirement Home,,,,Deleted,(blank),2024-04-02\n" - "xdomain7.gov,FederalArmedForcesRetirementHome,Deleted,(blank),2024-04-02\n" + "sdomain8.gov,Federal,ArmedForcesRetirementHome,Deleted,(blank),2024-04-02\n" + "xdomain7.gov,Federal,ArmedForcesRetirementHome,Deleted,(blank),2024-04-02\n" ) # Normalize line endings and remove commas, # spaces and leading/trailing whitespace @@ -611,7 +611,6 @@ def test_domain_managed(self): squeaker@rocks.com is invited to domain2 (DNS_NEEDED) and domain10 (No managers). She should show twice in this report but not in test_DomainManaged.""" - self.maxDiff = None # Create a CSV file in memory csv_file = StringIO() # Call the export functions @@ -646,7 +645,6 @@ def test_domain_managed(self): # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.maxDiff = None self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -683,7 +681,6 @@ def test_domain_unmanaged(self): # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.assertEqual(csv_content, expected_content) @less_console_noise_decorator @@ -721,10 +718,9 @@ def test_domain_request_growth(self): # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() - self.assertEqual(csv_content, expected_content) - @less_console_noise_decorator + # @less_console_noise_decorator def test_domain_request_data_full(self): """Tests the full domain request report.""" # Remove "Submitted at" because we can't guess this immutable, dynamically generated test data @@ -766,35 +762,34 @@ def test_domain_request_data_full(self): csv_file.seek(0) # Read the content into a variable csv_content = csv_file.read() + expected_content = ( # Header - "Domain request,Status,Domain type,Federal type," - "Federal agency,Organization name,Election office,City,State/territory," - "Region,Creator first name,Creator last name,Creator email,Creator approved domains count," - "Creator active requests count,Alternative domains,SO first name,SO last name,SO email," - "SO title/role,Request purpose,Request additional details,Other contacts," + "Domain request,Status,Domain type,Federal type,Federal agency,Organization name,Election office," + "City,State/territory,Region,Creator first name,Creator last name,Creator email," + "Creator approved domains count,Creator active requests count,Alternative domains,SO first name," + "SO last name,SO email,SO title/role,Request purpose,Request additional details,Other contacts," "CISA regional representative,Current websites,Investigator\n" # Content - "city5.gov,,Approved,Federal,Executive,,Testorg,N/A,,NY,2,,,,1,0,city1.gov,Testy,Tester,testy@town.com," - "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" - "city2.gov,,In review,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester," - "testy@town.com," + "city5.gov,Approved,Federal,Executive,,Testorg,N/A,,NY,2,,,,1,0,city1.gov,Testy,Tester,testy@town.com," "Chief Tester,Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" - 'city3.gov,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,"cheeseville.gov, city1.gov,' - 'igorville.gov",Testy,Tester,testy@town.com,Chief Tester,Purpose of the site,CISA-first-name ' - "CISA-last-name " - '| There is more,"Meow Tester24 te2@town.com, Testy1232 Tester24 te2@town.com, Testy Tester ' - 'testy2@town.com"' - ',test@igorville.com,"city.com, https://www.example2.com, https://www.example.com",\n' - "city4.gov,Submitted,City,Executive,,Testorg,Yes,,NY,2,,,,0,1,city1.gov,Testy,Tester,testy@town.com," - "Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester " - "testy2@town.com" - ",cisaRep@igorville.gov,city.com,\n" - "city6.gov,Submitted,Federal,Executive,,Testorg,N/A,,NY,2,,,,0,1,city1.gov,Testy,Tester,testy@town.com," - "Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester " - "testy2@town.com," + "city2.gov,In review,Federal,Executive,Portfolio 1 Federal Agency,,N/A,,,2,,,,0,1,city1.gov,,,,," + "Purpose of the site,There is more,Testy Tester testy2@town.com,,city.com,\n" + "city3.gov,Submitted,Federal,Executive,Portfolio 1 Federal Agency,,N/A,,,2,,,,0,1," + '"cheeseville.gov, city1.gov, igorville.gov",,,,,Purpose of the site,CISA-first-name CISA-last-name | ' + 'There is more,"Meow Tester24 te2@town.com, Testy1232 Tester24 te2@town.com, ' + 'Testy Tester testy2@town.com",' + 'test@igorville.com,"city.com, https://www.example2.com, https://www.example.com",\n' + "city4.gov,Submitted,City,Executive,,Testorg,Yes,,NY,2,,,,0,1,city1.gov,Testy," + "Tester,testy@town.com," + "Chief Tester,Purpose of the site,CISA-first-name CISA-last-name | There is more," + "Testy Tester testy2@town.com," + "cisaRep@igorville.gov,city.com,\n" + "city6.gov,Submitted,Federal,Executive,Portfolio 1 Federal Agency,,N/A,,,2,,,,0,1,city1.gov,,,,," + "Purpose of the site,CISA-first-name CISA-last-name | There is more,Testy Tester testy2@town.com," "cisaRep@igorville.gov,city.com,\n" ) + # Normalize line endings and remove commas, # spaces and leading/trailing whitespace csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() @@ -862,7 +857,6 @@ def test_member_export(self): # Create a request and add the user to the request request = self.factory.get("/") request.user = self.user - self.maxDiff = None # Add portfolio to session request = GenericTestHelper._mock_user_request_for_factory(request) request.session["portfolio"] = self.portfolio_1 diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index a03e51de5..2758375b1 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -525,6 +525,115 @@ def model(cls): # Return the model class that this export handles return DomainInformation + @classmethod + def get_computed_fields(cls, **kwargs): + """ + Get a dict of computed fields. + """ + # NOTE: These computed fields imitate @Property functions in the Domain model and Portfolio model where needed. + # This is for performance purposes. Since we are working with dictionary values and not + # model objects as we export data, trying to reinstate model objects in order to grab @property + # values negatively impacts performance. Therefore, we will follow best practice and use annotations + return { + "converted_generic_org_type": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__organization_type")), + # Otherwise, return the natively assigned value + default=F("generic_org_type"), + output_field=CharField(), + ), + "converted_federal_agency": Case( + # When portfolio is present, use its value instead + When( + Q(portfolio__isnull=False) & Q(portfolio__federal_agency__isnull=False), + then=F("portfolio__federal_agency__agency"), + ), + # Otherwise, return the natively assigned value + default=F("federal_agency__agency"), + output_field=CharField(), + ), + "converted_federal_type": Case( + # When portfolio is present, use its value instead + # NOTE: this is an @Property funciton in portfolio. + When( + Q(portfolio__isnull=False) & Q(portfolio__federal_agency__isnull=False), + then=F("portfolio__federal_agency__federal_type"), + ), + # Otherwise, return the natively assigned value + default=F("federal_type"), + output_field=CharField(), + ), + "converted_organization_name": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__organization_name")), + # Otherwise, return the natively assigned value + default=F("organization_name"), + output_field=CharField(), + ), + "converted_city": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__city")), + # Otherwise, return the natively assigned value + default=F("city"), + output_field=CharField(), + ), + "converted_state_territory": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__state_territory")), + # Otherwise, return the natively assigned value + default=F("state_territory"), + output_field=CharField(), + ), + "converted_so_email": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__senior_official__email")), + # Otherwise, return the natively assigned senior official + default=F("senior_official__email"), + output_field=CharField(), + ), + "converted_senior_official_last_name": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__senior_official__last_name")), + # Otherwise, return the natively assigned senior official + default=F("senior_official__last_name"), + output_field=CharField(), + ), + "converted_senior_official_first_name": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__senior_official__first_name")), + # Otherwise, return the natively assigned senior official + default=F("senior_official__first_name"), + output_field=CharField(), + ), + "converted_senior_official_title": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__senior_official__title")), + # Otherwise, return the natively assigned senior official + default=F("senior_official__title"), + output_field=CharField(), + ), + "converted_so_name": Case( + # When portfolio is present, use that senior official instead + When( + Q(portfolio__isnull=False) & Q(portfolio__senior_official__isnull=False), + then=Concat( + Coalesce(F("portfolio__senior_official__first_name"), Value("")), + Value(" "), + Coalesce(F("portfolio__senior_official__last_name"), Value("")), + output_field=CharField(), + ), + ), + # Otherwise, return the natively assigned senior official + default=Concat( + Coalesce(F("senior_official__first_name"), Value("")), + Value(" "), + Coalesce(F("senior_official__last_name"), Value("")), + output_field=CharField(), + ), + output_field=CharField(), + ), + } + @classmethod def update_queryset(cls, queryset, **kwargs): """ @@ -614,10 +723,10 @@ def parse_row(cls, columns, model): if first_ready_on is None: first_ready_on = "(blank)" - # organization_type has generic_org_type AND is_election - domain_org_type = model.get("organization_type") + # organization_type has organization_type AND is_election + domain_org_type = model.get("converted_generic_org_type") human_readable_domain_org_type = DomainRequest.OrgChoicesElectionOffice.get_org_label(domain_org_type) - domain_federal_type = model.get("federal_type") + domain_federal_type = model.get("converted_federal_type") human_readable_domain_federal_type = BranchChoices.get_branch_label(domain_federal_type) domain_type = human_readable_domain_org_type if domain_federal_type and domain_org_type == DomainRequest.OrgChoicesElectionOffice.FEDERAL: @@ -640,12 +749,12 @@ def parse_row(cls, columns, model): "First ready on": first_ready_on, "Expiration date": expiration_date, "Domain type": domain_type, - "Agency": model.get("federal_agency__agency"), - "Organization name": model.get("organization_name"), - "City": model.get("city"), - "State": model.get("state_territory"), - "SO": model.get("so_name"), - "SO email": model.get("senior_official__email"), + "Agency": model.get("converted_federal_agency"), + "Organization name": model.get("converted_organization_name"), + "City": model.get("converted_city"), + "State": model.get("converted_state_territory"), + "SO": model.get("converted_so_name"), + "SO email": model.get("converted_so_email"), "Security contact email": security_contact_email, "Created at": model.get("domain__created_at"), "Deleted": model.get("domain__deleted"), @@ -654,8 +763,23 @@ def parse_row(cls, columns, model): } row = [FIELDS.get(column, "") for column in columns] + return row + def get_filtered_domain_infos_by_org(domain_infos_to_filter, org_to_filter_by): + """Returns a list of Domain Requests that has been filtered by the given organization value.""" + + annotated_queryset = domain_infos_to_filter.annotate( + converted_generic_org_type=Case( + # Recreate the logic of the converted_generic_org_type property + # here in annotations + When(portfolio__isnull=False, then=F("portfolio__organization_type")), + default=F("generic_org_type"), + output_field=CharField(), + ) + ) + return annotated_queryset.filter(converted_generic_org_type=org_to_filter_by) + @classmethod def get_sliced_domains(cls, filter_condition): """Get filtered domains counts sliced by org type and election office. @@ -663,23 +787,51 @@ def get_sliced_domains(cls, filter_condition): when a domain has more that one manager. """ - domains = DomainInformation.objects.all().filter(**filter_condition).distinct() - domains_count = domains.count() - federal = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count() - interstate = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.INTERSTATE).count() + domain_informations = DomainInformation.objects.all().filter(**filter_condition).distinct() + domains_count = domain_informations.count() + federal = ( + cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.FEDERAL) + .distinct() + .count() + ) + interstate = cls.get_filtered_domain_infos_by_org( + domain_informations, DomainRequest.OrganizationChoices.INTERSTATE + ).count() state_or_territory = ( - domains.filter(generic_org_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() + cls.get_filtered_domain_infos_by_org( + domain_informations, DomainRequest.OrganizationChoices.STATE_OR_TERRITORY + ) + .distinct() + .count() + ) + tribal = ( + cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.TRIBAL) + .distinct() + .count() + ) + county = ( + cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.COUNTY) + .distinct() + .count() + ) + city = ( + cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.CITY) + .distinct() + .count() ) - tribal = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() - county = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() - city = domains.filter(generic_org_type=DomainRequest.OrganizationChoices.CITY).distinct().count() special_district = ( - domains.filter(generic_org_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + cls.get_filtered_domain_infos_by_org( + domain_informations, DomainRequest.OrganizationChoices.SPECIAL_DISTRICT + ) + .distinct() + .count() ) school_district = ( - domains.filter(generic_org_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + cls.get_filtered_domain_infos_by_org(domain_informations, DomainRequest.OrganizationChoices.SCHOOL_DISTRICT) + .distinct() + .count() ) - election_board = domains.filter(is_election_board=True).distinct().count() + election_board = domain_informations.filter(is_election_board=True).distinct().count() return [ domains_count, @@ -706,6 +858,7 @@ def get_columns(cls): """ Overrides the columns for CSV export specific to DomainExport. """ + return [ "Domain name", "Status", @@ -723,6 +876,13 @@ def get_columns(cls): "Invited domain managers", ] + @classmethod + def get_annotations_for_sort(cls): + """ + Get a dict of annotations to make available for sorting. + """ + return cls.get_computed_fields() + @classmethod def get_sort_fields(cls): """ @@ -730,9 +890,9 @@ def get_sort_fields(cls): """ # Coalesce is used to replace federal_type of None with ZZZZZ return [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", + "converted_generic_org_type", + Coalesce("converted_federal_type", Value("ZZZZZ")), + "converted_federal_agency", "domain__name", ] @@ -773,20 +933,6 @@ def get_prefetch_related(cls): """ return ["domain__permissions"] - @classmethod - def get_computed_fields(cls, delimiter=", ", **kwargs): - """ - Get a dict of computed fields. - """ - return { - "so_name": Concat( - Coalesce(F("senior_official__first_name"), Value("")), - Value(" "), - Coalesce(F("senior_official__last_name"), Value("")), - output_field=CharField(), - ), - } - @classmethod def get_related_table_fields(cls): """ @@ -892,7 +1038,7 @@ def exporting_dr_data_to_csv(cls, response, request=None): cls.safe_get(getattr(request, "region_field", None)), request.status, cls.safe_get(getattr(request, "election_office", None)), - request.federal_type, + request.converted_federal_type, cls.safe_get(getattr(request, "domain_type", None)), cls.safe_get(getattr(request, "additional_details", None)), cls.safe_get(getattr(request, "creator_approved_domains_count", None)), @@ -943,6 +1089,13 @@ def get_columns(cls): "Security contact email", ] + @classmethod + def get_annotations_for_sort(cls, delimiter=", "): + """ + Get a dict of annotations to make available for sorting. + """ + return cls.get_computed_fields() + @classmethod def get_sort_fields(cls): """ @@ -950,9 +1103,9 @@ def get_sort_fields(cls): """ # Coalesce is used to replace federal_type of None with ZZZZZ return [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", + "converted_generic_org_type", + Coalesce("converted_federal_type", Value("ZZZZZ")), + "converted_federal_agency", "domain__name", ] @@ -990,20 +1143,6 @@ def get_filter_conditions(cls, **kwargs): ], ) - @classmethod - def get_computed_fields(cls, delimiter=", ", **kwargs): - """ - Get a dict of computed fields. - """ - return { - "so_name": Concat( - Coalesce(F("senior_official__first_name"), Value("")), - Value(" "), - Coalesce(F("senior_official__last_name"), Value("")), - output_field=CharField(), - ), - } - @classmethod def get_related_table_fields(cls): """ @@ -1037,6 +1176,13 @@ def get_columns(cls): "Security contact email", ] + @classmethod + def get_annotations_for_sort(cls, delimiter=", "): + """ + Get a dict of annotations to make available for sorting. + """ + return cls.get_computed_fields() + @classmethod def get_sort_fields(cls): """ @@ -1044,9 +1190,9 @@ def get_sort_fields(cls): """ # Coalesce is used to replace federal_type of None with ZZZZZ return [ - "organization_type", - Coalesce("federal_type", Value("ZZZZZ")), - "federal_agency", + "converted_generic_org_type", + Coalesce("converted_federal_type", Value("ZZZZZ")), + "converted_federal_agency", "domain__name", ] @@ -1085,20 +1231,6 @@ def get_filter_conditions(cls, **kwargs): ], ) - @classmethod - def get_computed_fields(cls, delimiter=", ", **kwargs): - """ - Get a dict of computed fields. - """ - return { - "so_name": Concat( - Coalesce(F("senior_official__first_name"), Value("")), - Value(" "), - Coalesce(F("senior_official__last_name"), Value("")), - output_field=CharField(), - ), - } - @classmethod def get_related_table_fields(cls): """ @@ -1476,24 +1608,180 @@ def model(cls): # Return the model class that this export handles return DomainRequest + def get_filtered_domain_requests_by_org(domain_requests_to_filter, org_to_filter_by): + """Returns a list of Domain Requests that has been filtered by the given organization value""" + annotated_queryset = domain_requests_to_filter.annotate( + converted_generic_org_type=Case( + # Recreate the logic of the converted_generic_org_type property + # here in annotations + When(portfolio__isnull=False, then=F("portfolio__organization_type")), + default=F("generic_org_type"), + output_field=CharField(), + ) + ) + return annotated_queryset.filter(converted_generic_org_type=org_to_filter_by) + + # return domain_requests_to_filter.filter( + # # Filter based on the generic org value returned by converted_generic_org_type + # id__in=[ + # domainRequest.id + # for domainRequest in domain_requests_to_filter + # if domainRequest.converted_generic_org_type + # and domainRequest.converted_generic_org_type == org_to_filter_by + # ] + # ) + + @classmethod + def get_computed_fields(cls, delimiter=", ", **kwargs): + """ + Get a dict of computed fields. + """ + # NOTE: These computed fields imitate @Property functions in the Domain model and Portfolio model where needed. + # This is for performance purposes. Since we are working with dictionary values and not + # model objects as we export data, trying to reinstate model objects in order to grab @property + # values negatively impacts performance. Therefore, we will follow best practice and use annotations + return { + "converted_generic_org_type": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__organization_type")), + # Otherwise, return the natively assigned value + default=F("generic_org_type"), + output_field=CharField(), + ), + "converted_federal_agency": Case( + # When portfolio is present, use its value instead + When( + Q(portfolio__isnull=False) & Q(portfolio__federal_agency__isnull=False), + then=F("portfolio__federal_agency__agency"), + ), + # Otherwise, return the natively assigned value + default=F("federal_agency__agency"), + output_field=CharField(), + ), + "converted_federal_type": Case( + # When portfolio is present, use its value instead + # NOTE: this is an @Property funciton in portfolio. + When( + Q(portfolio__isnull=False) & Q(portfolio__federal_agency__isnull=False), + then=F("portfolio__federal_agency__federal_type"), + ), + # Otherwise, return the natively assigned value + default=F("federal_type"), + output_field=CharField(), + ), + "converted_organization_name": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__organization_name")), + # Otherwise, return the natively assigned value + default=F("organization_name"), + output_field=CharField(), + ), + "converted_city": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__city")), + # Otherwise, return the natively assigned value + default=F("city"), + output_field=CharField(), + ), + "converted_state_territory": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__state_territory")), + # Otherwise, return the natively assigned value + default=F("state_territory"), + output_field=CharField(), + ), + "converted_so_email": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__senior_official__email")), + # Otherwise, return the natively assigned senior official + default=F("senior_official__email"), + output_field=CharField(), + ), + "converted_senior_official_last_name": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__senior_official__last_name")), + # Otherwise, return the natively assigned senior official + default=F("senior_official__last_name"), + output_field=CharField(), + ), + "converted_senior_official_first_name": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__senior_official__first_name")), + # Otherwise, return the natively assigned senior official + default=F("senior_official__first_name"), + output_field=CharField(), + ), + "converted_senior_official_title": Case( + # When portfolio is present, use its value instead + When(portfolio__isnull=False, then=F("portfolio__senior_official__title")), + # Otherwise, return the natively assigned senior official + default=F("senior_official__title"), + output_field=CharField(), + ), + "converted_so_name": Case( + # When portfolio is present, use that senior official instead + When( + Q(portfolio__isnull=False) & Q(portfolio__senior_official__isnull=False), + then=Concat( + Coalesce(F("portfolio__senior_official__first_name"), Value("")), + Value(" "), + Coalesce(F("portfolio__senior_official__last_name"), Value("")), + output_field=CharField(), + ), + ), + # Otherwise, return the natively assigned senior official + default=Concat( + Coalesce(F("senior_official__first_name"), Value("")), + Value(" "), + Coalesce(F("senior_official__last_name"), Value("")), + output_field=CharField(), + ), + output_field=CharField(), + ), + } + @classmethod def get_sliced_requests(cls, filter_condition): """Get filtered requests counts sliced by org type and election office.""" requests = DomainRequest.objects.all().filter(**filter_condition).distinct() requests_count = requests.count() - federal = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.FEDERAL).distinct().count() - interstate = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.INTERSTATE).distinct().count() + federal = ( + cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.FEDERAL) + .distinct() + .count() + ) + interstate = ( + cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.INTERSTATE) + .distinct() + .count() + ) state_or_territory = ( - requests.filter(generic_org_type=DomainRequest.OrganizationChoices.STATE_OR_TERRITORY).distinct().count() + cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.STATE_OR_TERRITORY) + .distinct() + .count() + ) + tribal = ( + cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.TRIBAL) + .distinct() + .count() + ) + county = ( + cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.COUNTY) + .distinct() + .count() + ) + city = ( + cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.CITY).distinct().count() ) - tribal = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.TRIBAL).distinct().count() - county = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.COUNTY).distinct().count() - city = requests.filter(generic_org_type=DomainRequest.OrganizationChoices.CITY).distinct().count() special_district = ( - requests.filter(generic_org_type=DomainRequest.OrganizationChoices.SPECIAL_DISTRICT).distinct().count() + cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.SPECIAL_DISTRICT) + .distinct() + .count() ) school_district = ( - requests.filter(generic_org_type=DomainRequest.OrganizationChoices.SCHOOL_DISTRICT).distinct().count() + cls.get_filtered_domain_requests_by_org(requests, DomainRequest.OrganizationChoices.SCHOOL_DISTRICT) + .distinct() + .count() ) election_board = requests.filter(is_election_board=True).distinct().count() @@ -1517,11 +1805,11 @@ def parse_row(cls, columns, model): """ # Handle the federal_type field. Defaults to the wrong format. - federal_type = model.get("federal_type") + federal_type = model.get("converted_federal_type") human_readable_federal_type = BranchChoices.get_branch_label(federal_type) if federal_type else None # Handle the org_type field - org_type = model.get("generic_org_type") or model.get("organization_type") + org_type = model.get("converted_generic_org_type") human_readable_org_type = DomainRequest.OrganizationChoices.get_org_label(org_type) if org_type else None # Handle the status field. Defaults to the wrong format. @@ -1569,19 +1857,19 @@ def parse_row(cls, columns, model): "Other contacts": model.get("all_other_contacts"), "Current websites": model.get("all_current_websites"), # Untouched FK fields - passed into the request dict. - "Federal agency": model.get("federal_agency__agency"), - "SO first name": model.get("senior_official__first_name"), - "SO last name": model.get("senior_official__last_name"), - "SO email": model.get("senior_official__email"), - "SO title/role": model.get("senior_official__title"), + "Federal agency": model.get("converted_federal_agency"), + "SO first name": model.get("converted_senior_official_first_name"), + "SO last name": model.get("converted_senior_official_last_name"), + "SO email": model.get("converted_so_email"), + "SO title/role": model.get("converted_senior_official_title"), "Creator first name": model.get("creator__first_name"), "Creator last name": model.get("creator__last_name"), "Creator email": model.get("creator__email"), "Investigator": model.get("investigator__email"), # Untouched fields - "Organization name": model.get("organization_name"), - "City": model.get("city"), - "State/territory": model.get("state_territory"), + "Organization name": model.get("converted_organization_name"), + "City": model.get("converted_city"), + "State/territory": model.get("converted_state_territory"), "Request purpose": model.get("purpose"), "CISA regional representative": model.get("cisa_representative_email"), "Last submitted date": model.get("last_submitted_date"), @@ -1724,24 +2012,34 @@ def get_computed_fields(cls, delimiter=", ", **kwargs): """ Get a dict of computed fields. """ - return { - "creator_approved_domains_count": cls.get_creator_approved_domains_count_query(), - "creator_active_requests_count": cls.get_creator_active_requests_count_query(), - "all_current_websites": StringAgg("current_websites__website", delimiter=delimiter, distinct=True), - "all_alternative_domains": StringAgg("alternative_domains__website", delimiter=delimiter, distinct=True), - # Coerce the other contacts object to "{first_name} {last_name} {email}" - "all_other_contacts": StringAgg( - Concat( - "other_contacts__first_name", - Value(" "), - "other_contacts__last_name", - Value(" "), - "other_contacts__email", + # Get computed fields from the parent class + computed_fields = super().get_computed_fields() + + # Add additional computed fields + computed_fields.update( + { + "creator_approved_domains_count": cls.get_creator_approved_domains_count_query(), + "creator_active_requests_count": cls.get_creator_active_requests_count_query(), + "all_current_websites": StringAgg("current_websites__website", delimiter=delimiter, distinct=True), + "all_alternative_domains": StringAgg( + "alternative_domains__website", delimiter=delimiter, distinct=True ), - delimiter=delimiter, - distinct=True, - ), - } + # Coerce the other contacts object to "{first_name} {last_name} {email}" + "all_other_contacts": StringAgg( + Concat( + "other_contacts__first_name", + Value(" "), + "other_contacts__last_name", + Value(" "), + "other_contacts__email", + ), + delimiter=delimiter, + distinct=True, + ), + } + ) + + return computed_fields @classmethod def get_related_table_fields(cls): diff --git a/src/registrar/views/portfolios.py b/src/registrar/views/portfolios.py index 89fae0862..880472509 100644 --- a/src/registrar/views/portfolios.py +++ b/src/registrar/views/portfolios.py @@ -626,34 +626,3 @@ def submit_new_member(self, form): if permission_exists: messages.warning(self.request, "User is already a member of this portfolio.") return redirect(self.get_success_url()) - - # look up a user with that email - try: - requested_user = User.objects.get(email=requested_email) - except User.DoesNotExist: - # no matching user, go make an invitation - return self._make_invitation(requested_email, requestor) - else: - # If user already exists, check to see if they are part of the portfolio already - # If they are already part of the portfolio, raise an error. Otherwise, send an invite. - existing_user = UserPortfolioPermission.objects.get(user=requested_user, portfolio=self.object) - if existing_user: - messages.warning(self.request, "User is already a member of this portfolio.") - else: - try: - self._send_portfolio_invitation_email(requested_email, requestor, add_success=False) - except EmailSendingError: - logger.warn( - "Could not send email invitation (EmailSendingError)", - self.object, - exc_info=True, - ) - messages.warning(self.request, "Could not send email invitation.") - except Exception: - logger.warn( - "Could not send email invitation (Other Exception)", - self.object, - exc_info=True, - ) - messages.warning(self.request, "Could not send email invitation.") - return redirect(self.get_success_url())