Add import manual validation csv

Author: Nora Schneider

signbank/dictionary/csv_import.py

@@ -26,7 +26,7 @@
 
 from .forms import CSVFileOnlyUpload, CSVUploadForm
 from .models import (Dataset, FieldChoice, Gloss, GlossTranslations, Language,
-                     ShareValidationAggregation, ValidationRecord)
+                     ManualValidationAggregation, ShareValidationAggregation, ValidationRecord)
 from .tasks import retrieve_videos_for_glosses
 
 User = get_user_model()
@@ -721,3 +721,157 @@ def confirm_import_qualtrics_csv(request):
             "missing_gloss_question_pairs": missing_gloss_pk_question_pairs
         }
     )
+
+
+@login_required
+@permission_required("dictionary.import_csv")
+def import_manual_validation(request):
+    """
+    Import ManualValidationAggregations from a CSV file
+    """
+    # Make sure that the session variables are flushed before using this view.
+    request.session.pop("group_row_map", None)
+    request.session.pop("glosses", None)
+
+    if not request.method == "POST":
+        # If request type is not POST, return to the original form.
+        csv_form = CSVFileOnlyUpload()
+        return render(request, "dictionary/import_manual_validation_csv.html",
+                      {"import_csv_form": csv_form}, )
+
+    form = CSVFileOnlyUpload(request.POST, request.FILES)
+
+    if not form.is_valid():
+        # If form is not valid, set a error message and return to the original form.
+        messages.add_message(request, messages.ERROR,
+                             _("The provided CSV-file does not meet the requirements "
+                               "or there is some other problem."))
+        return render(request, "dictionary/import_manual_validation_csv.html",
+                      {"import_csv_form": form}, )
+
+    group_row_map = {}
+    group_gloss_count = {}
+    glosses = []
+    required_headers = [
+        "group",
+        "idgloss",
+        "yes",
+        "no",
+        "abstain or not sure",
+        "comments"
+    ]
+    try:
+        validation_record_reader = csv.DictReader(
+            codecs.iterdecode(form.cleaned_data["file"], "utf-8"),
+            delimiter=",",
+            quotechar='"'
+        )
+        for header in required_headers:
+            if not header in validation_record_reader.fieldnames:
+                request.session.pop("group_row_map", None)
+                request.session.pop("glosses", None)
+                # Set a message to be shown so that the user knows what is going on.
+                messages.add_message(request, messages.ERROR,
+                                     _(f"CSV is missing required column: {header}"))
+                return render(request,
+                              "dictionary/import_manual_validation_csv.html",
+                              {"import_csv_form": CSVFileOnlyUpload()}, )
+
+        for row in validation_record_reader:
+            if validation_record_reader.line_num == 1:
+                continue
+            try:
+                group_row_map[row["group"]].append(row)
+                group_gloss_count[row["group"]] += 1
+            except KeyError:
+                group_row_map[row["group"]] = [row]
+                group_gloss_count[row["group"]] = 1
+            glosses.append(row["idgloss"].split(":")[1])
+
+    except csv.Error as e:
+        # Can't open file, remove session variables
+        request.session.pop("group_row_map", None)
+        request.session.pop("glosses", None)
+        # Set a message to be shown so that the user knows what is going on.
+        messages.add_message(request, messages.ERROR, _("Cannot open the file:" + str(e)))
+        return render(request, "dictionary/import_manual_validation_csv.html",
+                      {"import_csv_form": CSVFileOnlyUpload()}, )
+    except UnicodeDecodeError as e:
+        # File is not UTF-8 encoded.
+        messages.add_message(request, messages.ERROR, _("File must be UTF-8 encoded!"))
+        return render(request, "dictionary/import_manual_validation_csv.html",
+                      {"import_csv_form": CSVFileOnlyUpload()}, )
+
+    # Store dataset's id and the list of glosses to be added in session.
+    request.session["group_row_map"] = group_row_map
+    request.session["glosses"] = list(set(glosses))
+
+    return render(request, "dictionary/import_manual_validation_csv_confirmation.html",
+                  {"group_row_map": group_row_map, "group_gloss_count": group_gloss_count})
+
+
+@login_required
+@permission_required("dictionary.import_csv")
+@transaction.atomic()
+def confirm_import_manual_validation(request):
+    """This view adds the data to database if the user confirms the action"""
+    if not request.method == "POST":
+        # If request method is not POST, redirect to the import form
+        return HttpResponseRedirect(reverse("dictionary:import_manual_validation_csv"))
+
+    if "cancel" in request.POST:
+        # If user cancels adding data, flush session variables
+        request.session.pop("group_row_map", None)
+        request.session.pop("glosses", None)
+        # Set a message to be shown so that the user knows what is going on.
+        messages.add_message(request, messages.WARNING, _("Cancelled adding CSV data."))
+        return HttpResponseRedirect(reverse("dictionary:import_manual_validation_csv"))
+
+    if not "confirm" in request.POST:
+        return HttpResponseRedirect(reverse("dictionary:import_manual_validation_csv"))
+
+    manual_validation_aggregations = []
+    missing_glosses = []
+
+    if "group_row_map" and "glosses" in request.session:
+        gloss_pk_list = request.session["glosses"]
+        gloss_dict = Gloss.objects.in_bulk(gloss_pk_list)
+
+        gloss_row_map = request.session["group_row_map"]
+
+        # Go through csv data
+        for group, rows in gloss_row_map.items():
+            for row in rows:
+                gloss = gloss_dict.get(int(row["idgloss"].split(":")[1]))
+                if not gloss:
+                    missing_glosses.append((group, row["idgloss"]))
+                    continue
+                sign_seen_yes = row["yes"]
+                sign_seen_no = row["no"]
+                sign_seen_not_sure = row["abstain or not sure"]
+                comments = row["comments"]
+                manual_validation_aggregations.append(ManualValidationAggregation(
+                    gloss=gloss,
+                    group=group,
+                    sign_seen_yes=int(sign_seen_yes) if sign_seen_yes else 0,
+                    sign_seen_no=int(sign_seen_no) if sign_seen_no else 0,
+                    sign_seen_not_sure=int(sign_seen_not_sure) if sign_seen_not_sure else 0,
+                    comments=comments
+                ))
+
+        ManualValidationAggregation.objects.bulk_create(manual_validation_aggregations)
+
+        del request.session["group_row_map"]
+        del request.session["glosses"]
+
+        # Set a message to be shown so that the user knows what is going on.
+        messages.add_message(request, messages.SUCCESS,
+                             _("ValidationRecords were added succesfully."))
+    return render(
+        request, "dictionary/import_manual_validation_csv_confirmation.html",
+        {
+            "manual_validation_aggregations": manual_validation_aggregations,
+            "manual_validation_aggregations_count": len(manual_validation_aggregations),
+            "missing_glosses": missing_glosses
+        }
+    )

signbank/dictionary/templates/dictionary/import_manual_validation_csv.html

@@ -0,0 +1,40 @@
+{% extends 'baselayout.html' %}
+{% load stylesheet %}
+{% load bootstrap3 %}
+{% load i18n %}
+{% block bootstrap3_title %}{% blocktrans %}Import Manual Validation CSV{% endblocktrans %} | {% endblock %}
+
+{% block content %}
+    {% if perms.dictionary.import_csv %}
+    <div id="import-csv">
+        {# Translators: #}
+        <h3>{% blocktrans %}Import Manual Validation CSV{% endblocktrans %}:</h3>
+
+        <ul>
+            <li>{% blocktrans %}CSV-file should contain the following columns:{% endblocktrans %}</li>
+            <ul>
+              <li>{% blocktrans %}group{% endblocktrans %}</li>
+              <li>{% blocktrans %}idgloss{% endblocktrans %}</li>
+              <li>{% blocktrans %}yes{% endblocktrans %}</li>
+              <li>{% blocktrans %}no{% endblocktrans %}</li>
+              <li>{% blocktrans %}abstain or not sure{% endblocktrans %}</li>
+              <li>{% blocktrans %}comments{% endblocktrans %}</li>
+            </ul>
+            <li>{% blocktrans %}Column headers should be all lower case{% endblocktrans %}</li>
+            <li>{% blocktrans %}Any further columns will be ignored during import{% endblocktrans %}</li>
+            <li>{% blocktrans %}The CSV-file's character set needs to be UTF-8. When exporting the CSV-file,
+                make sure that you select UTF-8 charset.{% endblocktrans %}</li>
+        </ul>
+
+        <form enctype="multipart/form-data" action='{% url "dictionary:import_manual_validation_csv" %}' method='post'>
+            {% csrf_token %}
+            {% bootstrap_field import_csv_form.file %}
+            <br>
+            <input class='btn btn-primary' type='submit' value='{% blocktrans %}Import manual validation aggregations from CSV{% endblocktrans %}'>
+        </form>
+    </div>
+    {% else %}
+    {# Translators: Message that appears if user doesn't have proper user rights to view this page. #}
+    <p>{% blocktrans %}You do not have sufficient user rights to view this page.{% endblocktrans %}</p>
+    {% endif %}
+{% endblock %}

signbank/dictionary/templates/dictionary/import_manual_validation_csv_confirmation.html

@@ -0,0 +1,40 @@
+{% extends 'baselayout.html' %}
+{% load stylesheet %}
+{% load bootstrap3 %}
+{% load i18n %}
+{% block bootstrap3_title %}{% blocktrans %}Confirm Import Manual Validation CSV{% endblocktrans %} | {% endblock %}
+
+{% block content %}
+{% if perms.dictionary.import_csv %}
+    <div>
+      {% if group_row_map %}
+        <h3>{% blocktrans %}ManualValidationAggregations to be added{% endblocktrans %}</h3>
+        {% for group, gloss_count in group_gloss_count.items %}
+          <p>{% blocktrans %}Group: {{ group }} - {{ gloss_count }} glosses{% endblocktrans %}</p>
+        {% endfor %}
+        <form action='{% url "dictionary:confirm_import_manual_validation_csv" %}' method='post'>
+          {% csrf_token %}
+          <input class='btn btn-primary' name='confirm' type='submit' value='{% blocktrans %}Confirm{% endblocktrans %}'>
+          <input class='btn btn-primary' name='cancel' type='submit' value='{% blocktrans %}Cancel{% endblocktrans %}'>
+        </form>
+      {% endif %}
+    </div>
+    <div>
+      {% if manual_validation_aggregations %}
+        <div>
+          <h3>{% blocktrans %}Manual validation aggregations were successfully added{% endblocktrans %}</h3>
+          <p>{% blocktrans %}Added total of {{ manual_validation_aggregations_count }} validation aggregations.{% endblocktrans %}</p>
+        </div>
+        {% if missing_glosses %}
+          <div>
+            <h4>{% blocktrans %}Could not find some glosses{% endblocktrans %}</h4>
+            {% for group, idgloss in missing_glosses %}
+              <p>{% blocktrans %}Group: {{ group }} - Gloss: {{ idgloss }}{% endblocktrans %}</p>
+            {% endfor %}
+          </div>
+        {% endif %}
+        <p><a href="{% url "dictionary:import_manual_validation_csv" %}">{% blocktrans %}Return to the form{% endblocktrans %}</a></p>
+      {% endif %}
+    </div>
+{% endif %}
+{% endblock %}