Merge pull request #3651 from teovin/bulk-org-user-creation

Bulk organization user creation/addition feature

perma_web/perma/forms.py

@@ -1,5 +1,7 @@
 import logging
 import secrets
+import csv
+from io import TextIOWrapper
 import string
 from typing import Any, Mapping
 
@@ -8,6 +10,7 @@
 from django.conf import settings
 from django.contrib.auth.forms import SetPasswordForm
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
+from django.core.validators import EmailValidator
 from django.db.models.fields import BLANK_CHOICE_DASH
 from django.forms import Form, ModelForm
 from django.http import HttpRequest, HttpResponseRedirect
@@ -437,6 +440,169 @@ def save(self, commit=True):
         return instance
 
 
+class MultipleUsersFormWithOrganization(ModelForm):
+    """
+    Create multiple organization users via CSV file
+    """
+    organizations = forms.ModelChoiceField(label='Organization', queryset=Organization.objects.order_by('name'))
+    indefinite_affiliation = forms.BooleanField(
+        label="Permanent affiliation",
+        required=False,
+        initial=True
+    )
+    expires_at = forms.DateTimeField(
+        label="Affiliation expiration date",
+        widget=forms.DateTimeInput(attrs={"type": "date"}),
+        required=False
+    )
+    csv_file = forms.FileField(label='* User information',
+                               help_text=mark_safe("<br>* When creating your CSV, please include the following fields: first_name, last_name, email. "
+                                                   "First and last name columns may be left blank.<br><br>"
+                                                   "If there is already a Perma.cc account associated with an "
+                                                   "email, we will add an Organization affiliation. If there is not, "
+                                                   "an account will be created and automatically affiliated with this "
+                                                   "Organization."))
+
+    def __init__(self, request, data=None, files=None, **kwargs):
+        super(MultipleUsersFormWithOrganization, self).__init__(data, files, **kwargs)
+        self.request = request
+        self.user_data = {}
+        self.created_users = {}
+        self.updated_users = {}
+        self.ineligible_users = {}
+
+        # Filter available organizations based on the current user
+        query = self.fields['organizations'].queryset
+        if request.user.is_registrar_user():
+            query = query.filter(registrar_id=request.user.registrar_id)
+        elif request.user.is_organization_user:
+            query = query.filter(users=request.user.pk)
+        self.fields['organizations'].queryset = query
+
+    class Meta:
+        model = LinkUser
+        fields = ["organizations", "indefinite_affiliation", "expires_at", "csv_file"]
+
+    def clean_csv_file(self):
+        file = self.cleaned_data['csv_file']
+
+        # check if file is CSV
+        if not file.name.endswith('.csv'):
+            raise forms.ValidationError("The file must be a CSV.")
+
+        file = TextIOWrapper(file, encoding='utf-8')
+        reader = csv.DictReader(file)
+
+        # validate the headers
+        headers = reader.fieldnames
+        if not all(item in headers for item in ['first_name', 'last_name', 'email']):
+            raise forms.ValidationError("CSV file must contain a header row with first_name, last_name and email columns.")
+
+        # validate the rows
+        seen = set()
+        row_count = 0
+
+        for row in reader:
+            row_count += 1
+            email = row.get('email')
+            email = email.strip() if email else None
+
+            if not email:
+                raise forms.ValidationError("Each row in the CSV file must contain email.")
+
+            email_validator = EmailValidator()
+            try:
+                email_validator(email)
+            except ValidationError:
+                raise forms.ValidationError(f"CSV file contains invalid email address: {email}")
+
+            if email in seen:
+                raise forms.ValidationError(f"CSV file cannot contain duplicate users: {email}")
+            else:
+                seen.add(email)
+                self.user_data[email] = {
+                    'first_name': row.get('first_name', '').strip(),
+                    'last_name': row.get('last_name', '').strip()
+                }
+
+        if row_count == 0:
+            raise forms.ValidationError("CSV file must contain at least one user.")
+
+        file.seek(0)
+        self.cleaned_data['csv_file'] = file
+        return file
+
+    def save(self, commit=True):
+        expires_at = self.cleaned_data['expires_at']
+        organization = self.cleaned_data['organizations']
+
+        raw_emails = set(self.user_data.keys())
+        # lower casing the emails to feed into the filter query in order to prevent duplicate user creation
+        lower_case_emails = {email.lower() for email in self.user_data.keys()}
+        existing_users = LinkUser.objects.filter(email__in=lower_case_emails)
+        updated_user_affiliations = []
+
+        for user in existing_users:
+            if commit:
+                if user.is_staff or user.is_registrar_user():
+                    self.ineligible_users[user.email] = user
+                else:
+                    updated_user_affiliations.append(user)
+                    self.updated_users[user.email] = user
+
+        new_user_emails = lower_case_emails - set(self.ineligible_users.keys()) - set(self.updated_users.keys())
+
+        created_user_affiliations = []
+
+        if new_user_emails and commit:
+            for email in new_user_emails:
+                raw_email = next((raw_email for raw_email in raw_emails if raw_email.lower() == email.lower()), None)
+                new_user = LinkUser(
+                        email=raw_email,
+                        first_name=self.user_data[raw_email]['first_name'],
+                        last_name=self.user_data[raw_email]['last_name']
+                )
+                new_user.save()
+                self.created_users[email] = new_user
+
+                created_user_affiliations.append(
+                    UserOrganizationAffiliation(
+                        user=new_user,
+                        organization=organization,
+                        expires_at=expires_at
+                    )
+                )
+
+        if commit:
+            # create the affiliations for new users
+            UserOrganizationAffiliation.objects.bulk_create(created_user_affiliations)
+
+            # create or update the affiliations of existing users
+            # affiliations that already exist
+            preexisting_affiliations = (UserOrganizationAffiliation.objects.filter(user__in=updated_user_affiliations,
+                                                                                   organization=organization))
+
+            preexisting_affiliations_set = set(affiliation.user for affiliation in preexisting_affiliations)
+            all_user_affiliations = set(updated_user_affiliations)
+            # new affiliations
+            new_affiliations = all_user_affiliations - preexisting_affiliations_set
+            new_affiliation_objs = []
+
+            for item in new_affiliations:
+                new_affiliation_objs.append(UserOrganizationAffiliation(
+                    user=item,
+                    organization=organization,
+                    expires_at=expires_at
+                ))
+
+            if preexisting_affiliations:
+                preexisting_affiliations.update(expires_at=expires_at)
+            if new_affiliation_objs:
+                UserOrganizationAffiliation.objects.bulk_create(new_affiliation_objs)
+
+        return self
+
+
 ### USER EDIT FORMS ###
 
 class UserAddRegistrarForm(UserFormWithRegistrar):

perma_web/perma/migrations/0057_auto_20241120_1837.py

@@ -0,0 +1,29 @@
+# Generated by Django 4.2.16 on 2024-11-20 18:37
+
+from django.db import migrations
+
+FLAG_NAME = "bulk-organization-user-creation"
+
+def create_bulk_organization_user_creation_feature_flag(apps, schema_editor):
+    Flag = apps.get_model("waffle", "Flag")
+    flag = Flag(
+        name=FLAG_NAME,
+        testing=True
+    )
+    flag.save()
+
+def delete_bulk_organization_user_creation_feature_flag(apps, schema_editor):
+    Flag = apps.get_model("waffle", "Flag")
+    flags = Flag.objects.filter(name=FLAG_NAME)
+    flags.delete()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('perma', '0056_delete_historicallink'),
+        ('waffle', '0004_update_everyone_nullbooleanfield'),
+    ]
+
+    operations = [
+        migrations.RunPython(create_bulk_organization_user_creation_feature_flag, delete_bulk_organization_user_creation_feature_flag),
+    ]

perma_web/perma/templates/email/new_user_added_to_org_by_other.txt

@@ -1,5 +1,5 @@
 TITLE: A Perma.cc account has been created for you
-{% with form.cleaned_data.organizations.0 as org %}
+{% with form.cleaned_data.organizations as org %}
 A Perma.cc account has been created for you by {{ request.user.get_full_name }} on behalf of {{ org }}.{% endwith %}
 
 {% include 'email/includes/activation.txt' %}

perma_web/perma/templates/includes/fieldset.html

@@ -23,7 +23,8 @@
           {% for error in field.errors %}
             <span class="field-error">{{ error }}</span>
           {% endfor %}
-        {% elif field.help_text %}
+        {% endif %}
+        {% if field.help_text %}
           <span class="help-inline">{{ field.help_text }}</span>
         {% endif %}
       </div>

perma_web/perma/templates/user_management/add_multiple_users_to_org.html

@@ -0,0 +1,35 @@
+{% extends "admin-layout.html" %}
+
+{% block title %} | Add users to organization{% endblock %}
+
+{% block adminContent %}
+  <h3 class="body-bh">Add multiple users to organization via CSV</h3>
+
+  {% if not form %}
+    <p>{{ error_message }}</p>
+  {% else %}
+
+    <form class="add-user" method="post" enctype="multipart/form-data">
+      {% csrf_token %}
+      {% include "includes/fieldset.html" %}
+        <button type="submit" class="btn">Add organization users</button>
+      <a href="{% url 'user_management_manage_organization_user' %}" class="btn cancel">Cancel</a>
+    </form>
+
+    <script>
+      const checkbox = document.getElementById("id_a-indefinite_affiliation");
+      const datetimeField = document.getElementById("id_a-expires_at");
+      const datetimeFieldLabel = document.querySelector('label[for="id_a-expires_at"]');
+
+      const toggleExpirationDateField = () => {
+        const displayStyle = checkbox.checked ? "none" : "block";
+        datetimeField.style.display = displayStyle;
+        datetimeFieldLabel.style.display = displayStyle;
+      };
+
+      document.addEventListener("DOMContentLoaded", toggleExpirationDateField);
+      checkbox.addEventListener("change", toggleExpirationDateField);
+    </script>
+
+  {% endif %}
+{% endblock %}

perma_web/perma/templates/user_management/manage_users.html

@@ -8,6 +8,9 @@ <h2 class="body-ah">{{ pretty_group_name_plural|title }}</h2>
 
   <a class="action-heading" data-toggle="collapse" href="#add-member" aria-expanded="false" aria-controls="#add-member"><i class="icon-plus-sign" aria-hidden="true"></i> add<span class="_verbose"> {{ pretty_group_name|lower }}</span></a>
 
+  {% if group_name == 'organization_user' and bulk_org_user_creation_feature_flag %}
+    <p class="bulk-users">or <a href="{% url 'user_management_organization_user_add_multiple_users' %}">add multiple users</a></p>
+  {% endif %}
   {% if messages %}
     {% for message in messages %}
       <div class="alert-{{ message.level_tag }} alert-block" >{% if 'safe' in message.tags %}{{ message|safe }}{% else %}{{ message }}{% endif %}</div>

perma_web/perma/tests/test_permissions.py

@@ -94,6 +94,7 @@ def test_permissions(client, admin_user, registrar_user, org_user, link_user_fac
                 ['user_management_manage_organization_user_export_user_list'],
                 ['user_management_manage_organization'],
                 ['user_management_organization_user_add_user'],
+                ['user_management_organization_user_add_multiple_users']
             ],
             'allowed': {admin_user, registrar_user, org_user_registrar_user, sponsored_user_registrar_user, org_user},
         },

perma_web/perma/tests/test_views_user_management.py

@@ -10,12 +10,15 @@
 from django.http import HttpResponse, JsonResponse
 from django.urls import reverse
 from django.core import mail
+from django.core.files.uploadedfile import SimpleUploadedFile
 from django.conf import settings
 from django.db import IntegrityError
 from django.test import override_settings
+from django.test.client import RequestFactory
 
 from perma.models import LinkUser, Organization, Registrar, Sponsorship, UserOrganizationAffiliation
 from perma.tests.utils import PermaTestCase
+from perma.forms import MultipleUsersFormWithOrganization
 
 
 class UserManagementViewsTestCase(PermaTestCase):
@@ -689,6 +692,80 @@ def add_org_user(self):
                              organizations=self.organization
                          ).exists())
 
+    def test_add_multiple_org_users_via_csv(self):
+        def create_csv_file(filename, content):
+            return SimpleUploadedFile(filename, content.encode('utf-8'), content_type='text/csv')
+
+        def initialize_form(csv_file, data=None):
+            data = {'organizations': selected_organization.pk, 'indefinite_affiliation': True}
+            return MultipleUsersFormWithOrganization(request=request, data=data, files={'csv_file': csv_file})
+
+        # --- initialize data ---
+        csv_data = 'first_name,last_name,email\nJohn,Doe,[email protected]\nJane,Smith,[email protected]'
+        another_csv_data = 'first_name,last_name,email\nJohn2,Doe,[email protected]\nJane2,Smith,[email protected]'
+        invalid_csv_data = 'name\nJohn Doe'
+        another_invalid_csv_data = 'first_name,last_name,email\nJohn,Doe,\nJane,Smith,[email protected]'
+
+        valid_csv_file = create_csv_file('users.csv', csv_data)
+        another_valid_csv_file = create_csv_file('another_valid_users.csv', another_csv_data)
+        one_more_valid_csv_file = create_csv_file('one_more_valid_users.csv', csv_data)
+        invalid_csv_file = create_csv_file('invalid_users.csv', invalid_csv_data)
+        another_invalid_csv_file = create_csv_file('another_invalid_users.csv', another_invalid_csv_data)
+
+        request = RequestFactory().get('/')
+        request.user = self.registrar_user
+        selected_organization = self.another_organization
+
+        # --- test form initialization ---
+        form = MultipleUsersFormWithOrganization(request=request)
+        # the registrar user has 3 organizations tied to it as verified in the users.json sample data
+        self.assertEqual(form.fields['organizations'].queryset.count(), 3)
+        # confirm that the first item in organization selection field matches the first organization of the registrar
+        self.assertEqual(form.fields['organizations'].queryset.first(), request.user.registrar.organizations
+                         .order_by('name').first())
+
+        # --- test csv validation ---
+        # valid csv
+        form1 = initialize_form(valid_csv_file)
+        self.assertTrue(form1.is_valid())
+
+        # invalid csv - missing headers
+        form2 = initialize_form(invalid_csv_file)
+        self.assertFalse(form2.is_valid())
+        self.assertTrue("CSV file must contain a header row with first_name, last_name and email columns."
+                        in form2.errors['csv_file'])
+
+        # invalid csv - missing email field
+        form3 = initialize_form(another_invalid_csv_file)
+        self.assertFalse(form3.is_valid())
+        self.assertTrue("Each row in the CSV file must contain email."
+                        in form3.errors['csv_file'])
+
+        # --- test user creation ---
+        self.assertTrue(form1.is_valid())
+        form1.save(commit=True)
+        created_user_ids = [user.id for user in form1.created_users.values()]
+        self.assertEqual(len(created_user_ids), 2)
+        self.assertEqual(UserOrganizationAffiliation.objects.filter(user_id__in=created_user_ids).count(), 2)
+
+        # --- test user update ---
+        existing_user = LinkUser.objects.create(email="[email protected]", first_name="John2", last_name="Doe")
+        form4 = initialize_form(another_valid_csv_file)
+        self.assertTrue(form4.is_valid())
+        form4.save(commit=True)
+        self.assertEqual(len(form4.updated_users), 1)
+        self.assertTrue(existing_user in form4.updated_users.values())
+        self.assertEqual(len(form4.created_users), 1)
+        self.assertEqual(next(iter(form4.updated_users)), "[email protected]")
+
+        # --- test validation errors ---
+        LinkUser.objects.filter(raw_email="[email protected]").update(is_staff=True)
+        form5 = initialize_form(one_more_valid_csv_file)
+        self.assertTrue(form5.is_valid())
+        form5.save(commit=True)
+        self.assertEqual(len(form5.ineligible_users), 1)
+        self.assertEqual("[email protected]", next(iter(form5.ineligible_users)))
+
     def test_admin_user_can_add_new_user_to_org(self):
         self.log_in_user(self.admin_user)
         self.add_org_user()