[GRDM-52662]Feature export csv for 証跡管理情報、NIIストレージのクオータ情報

.gitignore

@@ -206,3 +206,4 @@ ssl/
 
 # pyenv
 .python-version
+.github/copilot-instructions.md

admin/institutions/urls.py

@@ -23,6 +23,7 @@
     url(r'^(?P<institution_id>[0-9]+)/tsvexport/$', views.ExportFileTSV.as_view(), name='tsvexport'),
     url(r'^user_list_by_institution_id/(?P<institution_id>[0-9]+)/$', views.UserListByInstitutionID.as_view(), name='institution_user_list'),
     url(r'^statistical_status_default_storage/$', views.StatisticalStatusDefaultStorage.as_view(), name='statistical_status_default_storage'),
+    url(r'^statistical_status_default_storage/tsvexport/$', views.StatisticalExportFileTSV.as_view(), name='statistical_tsvexport'),
     url(r'^recalculate_quota/$', views.RecalculateQuota.as_view(), name='recalculate_quota'),
     url(r'^(?P<institution_id>[0-9]+)/recalculate_quota/$',
         views.RecalculateQuota.as_view(), name='recalculate_quota_of_users_in_institution'),

admin/institutions/views.py

@@ -1,7 +1,8 @@
 from __future__ import unicode_literals
 
+from datetime import datetime
+from http import HTTPStatus
 import json
-import logging
 from operator import itemgetter
 
 from django.db import connection, transaction, IntegrityError
@@ -28,8 +29,6 @@
 from api.base import settings as api_settings
 import csv
 
-logger = logging.getLogger(__name__)
-
 
 class InstitutionList(PermissionRequiredMixin, ListView):
     paginate_by = 25
@@ -371,9 +370,18 @@ class ExportFileTSV(PermissionRequiredMixin, QuotaUserList):
 
     def get(self, request, **kwargs):
         institution_id = self.kwargs.get('institution_id')
-        if not Institution.objects.filter(id=institution_id, is_deleted=False).exists():
+        # Get institution by institution_id
+        institution = Institution.objects.filter(id=institution_id, is_deleted=False).first()
+        if not institution:
+            # If institution is not found, redirect to HTTP 404 page
             raise Http404(f'Institution with id "{institution_id}" not found. Please double check.')
 
+        # Get user quota type for institution if using NII Storage
+        user_quota_type = institution.get_user_quota_type_for_nii_storage()
+        if not user_quota_type:
+            # Institution is not using NII storage, redirect to HTTP 404 page
+            raise Http404(f'Institution with id "{institution_id}" not using NII storage. Please double check.')
+
         response = HttpResponse(content_type='text/tsv')
         writer = csv.writer(response, delimiter='\t')
         writer.writerow(['GUID', 'Username', 'Fullname', 'Ratio (%)', 'Usage (Byte)', 'Remaining (Byte)', 'Quota (Byte)'])
@@ -566,6 +574,112 @@ def get_institution(self):
         return self.request.user.affiliated_institutions.filter(is_deleted=False).first()
 
 
+class StatisticalExportFileTSV(RdmPermissionMixin, UserPassesTestMixin, QuotaUserList):
+    permission_required = 'osf.view_institution'
+    raise_exception = True
+
+    def test_func(self):
+        """ Check user permissions """
+        if not self.is_authenticated:
+            # If user is not authenticated then redirect to login page
+            self.raise_exception = False
+            return False
+        return not self.is_super_admin and self.is_admin \
+            and self.request.user.affiliated_institutions.exists()
+
+    def handle_no_permission(self):
+        """ Handle user has no permission """
+        if not self.raise_exception:
+            # If user is not authenticated then return HTTP 401
+            return JsonResponse(
+                {'error_message': 'Authentication credentials were not provided.'},
+                status=HTTPStatus.UNAUTHORIZED
+            )
+        return super(StatisticalExportFileTSV, self).handle_no_permission()
+
+    def get(self, request, **kwargs):
+        institution = self.get_institution()
+        if not institution:
+            # If institution is not found, redirect to HTTP 404 page
+            raise Http404
+
+        # Get user quota type for institution if using NII Storage
+        user_quota_type = institution.get_user_quota_type_for_nii_storage()
+        if not user_quota_type:
+            # Institution is not using NII storage, redirect to 404 page
+            raise Http404
+        response = HttpResponse(content_type='text/tsv')
+        writer = csv.writer(response, delimiter='\t')
+        writer.writerow(
+            ['GUID', 'eduPersonPrincipleName', 'Username', 'Fullname', 'Ratio (%)', 'Usage (Byte)', 'Remaining (Byte)', 'Quota (Byte)'])
+
+        for user in self.get_userlist(institution, user_quota_type):
+            if user.has_quota:
+                max_quota = user.quota_max
+                used_quota = user.quota_used
+            else:
+                max_quota = api_settings.DEFAULT_MAX_QUOTA
+                used_quota = quota.used_quota(user._id, user_quota_type)
+            max_quota_bytes = max_quota * api_settings.SIZE_UNIT_GB
+            remaining_quota = max_quota_bytes - used_quota
+            if max_quota == 0:
+                writer.writerow([user.guid_id, user.eppn, user.username,
+                                 user.fullname,
+                                 round(100, 1),
+                                 round(used_quota, 0),
+                                 round(remaining_quota, 0),
+                                 round(max_quota_bytes, 0)])
+            else:
+                writer.writerow([user.guid_id, user.eppn, user.username,
+                                 user.fullname,
+                                 round(float(used_quota) / max_quota_bytes * 100, 1),
+                                 round(used_quota, 0),
+                                 round(remaining_quota, 0),
+                                 round(max_quota_bytes, 0)])
+        time_now = datetime.today().strftime('%Y%m%d%H%M%S')
+        query = 'attachment; filename= export_users_{}.tsv'.format(time_now)
+        response['Content-Disposition'] = query
+        return response
+
+    def get_userlist(self, institution, user_quota_type):
+        """Get list of users' quota info using efficient subqueries"""
+        from django.db.models import (
+            F, Subquery, OuterRef, Value,
+            Case, When, IntegerField, BooleanField
+        )
+
+        # Subquery to get the first quota record for each user
+        first_quota = UserQuota.objects.filter(
+            user=OuterRef('pk'),
+            storage_type=user_quota_type
+        ).order_by('id')
+
+        user_list = OSFUser.objects.filter(
+            affiliated_institutions=institution.id
+        ).annotate(
+            guid_id=F('guids___id'),
+            # Specify output_field for the subqueries
+            quota_max=Subquery(
+                first_quota.values('max_quota')[:1],
+                output_field=IntegerField()
+            ),
+            quota_used=Subquery(
+                first_quota.values('used')[:1],
+                output_field=IntegerField()
+            ),
+            has_quota=Case(
+                When(userquota__storage_type=user_quota_type, then=Value(True)),
+                default=Value(False),
+                output_field=BooleanField()
+            )
+        )
+        return user_list
+
+    def get_institution(self):
+        """ Get logged in user's first affiliated institution """
+        return self.request.user.affiliated_institutions.filter(is_deleted=False).first()
+
+
 class RecalculateQuota(RdmPermissionMixin, UserPassesTestMixin, View):
     """
     Recalculate used quota for institutions that is using NII Storage for integrated administrators.

admin/rdm_timestampadd/urls.py

@@ -5,6 +5,7 @@
 urlpatterns = [
     url(r'^$', views.InstitutionList.as_view(), name='institutions'),
     url(r'^(?P<institution_id>[0-9]+)/nodes/$', views.InstitutionNodeList.as_view(), name='nodes'),
+    url(r'^(?P<institution_id>[0-9]+)/nodes/csvexport/$', views.InstitutionNodeListExportCsv.as_view(), name='csvexport'),
     url(r'^(?P<institution_id>[0-9]+)/nodes/(?P<guid>[a-z0-9]+)/$',
         views.TimeStampAddList.as_view(), name='timestamp_add'),
     url(r'^(?P<institution_id>[0-9]+)/nodes/(?P<guid>[a-z0-9]+)/verify/$',

admin/rdm_timestampadd/views.py

@@ -1,16 +1,24 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import unicode_literals
+
+from datetime import datetime
+from http import HTTPStatus
+
+from django.contrib.postgres.aggregates import StringAgg
+from django.db.models import F
+
 from admin.base import settings
 from admin.rdm.utils import RdmPermissionMixin, get_dummy_institution
 from django.contrib.auth.mixins import UserPassesTestMixin
 from django.core.urlresolvers import reverse
-from django.http import HttpResponse
+from django.http import HttpResponse, JsonResponse
 from django.shortcuts import redirect
 from django.views.generic import ListView, View, TemplateView
 from osf.models import Institution, Node, AbstractNode, TimestampTask
 from website.util import timestamp
 import json
+import csv
 
 
 class InstitutionList(RdmPermissionMixin, UserPassesTestMixin, ListView):
@@ -81,6 +89,60 @@ def get_context_data(self, **kwargs):
         kwargs.setdefault('logohost', settings.OSF_URL)
         return super(InstitutionNodeList, self).get_context_data(**kwargs)
 
+
+class InstitutionNodeListExportCsv(RdmPermissionMixin, UserPassesTestMixin, ListView):
+    ordering = '-modified'
+    raise_exception = True
+
+    def test_func(self):
+        """validate user permissions"""
+        if not self.is_authenticated:
+            # If user is not authenticated throw 401 error
+            self.raise_exception = False
+            return False
+        institution_id = int(self.kwargs.get('institution_id'))
+        return self.has_auth(institution_id)
+
+    def handle_no_permission(self):
+        """ Handle user has no permission """
+        if not self.raise_exception:
+            # If user is not authenticated then return HTTP 401
+            return JsonResponse(
+                {'error_message': 'Authentication credentials were not provided.'},
+                status=HTTPStatus.UNAUTHORIZED
+            )
+        return super(InstitutionNodeListExportCsv, self).handle_no_permission()
+
+    def get_queryset(self):
+        inst = self.kwargs['institution_id']
+        return Node.objects.filter(
+            affiliated_institutions=inst).annotate(
+            parent_title=F('_parents__parent__title'),
+            root_title=F('root__title'),
+            contributor_names=StringAgg('_contributors__username', delimiter=', ')
+        ).order_by(self.ordering)
+
+    def get(self, request, **kwargs):
+        node_list = self.get_queryset().all()
+        response = HttpResponse(content_type='text/csv')
+        writer = csv.writer(response, delimiter=',')
+        writer.writerow(['Node id', 'GUID', 'Title', 'Parent', 'Root', 'Date created', 'Public', 'Withdrawn', 'Embargo',
+                         'Contributors'])
+        for node in node_list:
+            parent = getattr(node, 'parent_title', None)
+            root = getattr(node, 'root_title', None)
+            public = getattr(node, 'is_public', None)
+            created = getattr(node, 'created', None).strftime('%Y-%m-%d') if getattr(node, 'created', None) else None
+            contributors = getattr(node, 'contributor_names', None)
+            writer.writerow(
+                [node.id, node.guid, node.title, parent, root, created, public, node.retraction, node.embargo,
+                 contributors])
+        time_now = datetime.today().strftime('%Y%m%d%H%M%S')
+        query = 'attachment; filename= export_nodes_{}.csv'.format(time_now)
+        response['Content-Disposition'] = query
+        return response
+
+
 class TimeStampAddList(RdmPermissionMixin, UserPassesTestMixin, TemplateView):
     template_name = 'rdm_timestampadd/timestampadd.html'
     ordering = 'provider'

admin/templates/institutions/statistical_status_default_storage_user.html

@@ -1,10 +1,78 @@
 {% load i18n %}
 <h2 id="title">{% trans "Statistical Status of Institutional Storage" %}</h2>
 <h3 id="sub-title">{% blocktrans %}Institutional Storage > {{ institution_name }}{% endblocktrans %}</h3>
+<style>
+    /* Base Utilities */
+    .nowrap {
+        white-space: nowrap;
+    }
 
-<div class="flex-item" style="flex-grow: 0;">
-    {% include "util/pagination.html" with items=page status=direction order=order_by %}
+    /* Form Elements */
+    .form-group.flex-item {
+        max-width: 20em !important;
+    }
+
+    /* Flex Container System */
+    .flex-container {
+        display: flex;
+        align-items: center;
+        margin-bottom: 1rem;
+    }
+
+    .flex-container .form-group.flex-item {
+        display: flex;
+        align-items: center;
+    }
+
+    /* Grid System */
+    .row.flex-container {
+        padding-inline: 15px;
+    }
+
+    /* Flex Items */
+    .flex-container .flex-item {
+        flex: 1 1 0;
+        white-space: nowrap;
+        margin: 0;
+    }
+
+    /* Nested Elements */
+    .flex-item .flex-container,
+    .flex-item .flex-container .flex-item {
+        margin-bottom: 0;
+    }
+
+    /* Pagination Component */
+    .pagination-extra-div {
+        & .pagination {
+            margin: 0;
+        }
+
+        & > .flex-item {
+            margin-bottom: 15px;
+        }
+    }
+
+    /* Spacing Utilities */
+    .flex-item + .flex-item {
+        margin-inline-start: 2px;
+    }
+</style>
+
+<div class="row flex-container pagination-extra-div">
+    <div class="flex-item" style="flex-grow: 0; margin-left: 2em;">
+        {% include "util/pagination.html" with items=page status=direction order=order_by %}
+    </div>
+    <div class="flex-item" style="flex-grow: 0; margin-left: auto">
+        <div>
+            <a class="btn btn-primary" style="padding-left: 6px; padding-right: 6px;"
+               href="{% url 'institutions:statistical_tsvexport' %}">
+               {% trans "Export All Users Statistics (TSV)" %}
+            </a>
+        </div>
+    </div>
 </div>
+
 <div class="table-responsive">
 <table class="table table-striped table-hover table-responsive">
     <thead>