Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add capture errors panel to admin stats page. #3427

Merged
merged 1 commit into from
Nov 2, 2023
Merged

Add capture errors panel to admin stats page. #3427

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 88 additions & 0 deletions perma_web/perma/templates/user_management/stats.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
<li class="active"><a href="#celery_data" data-toggle="tab">Celery Queues</a></li>
<li><a href="#rate-limits-pane" data-toggle="tab">Internet Archive</a></li>
<li><a href="#capture-job-pane" data-toggle="tab">Capture Jobs</a></li>
<li><a href="#capture-error-pane" data-toggle="tab">Capture Errors</a></li>
<li><a href="#days-pane" data-toggle="tab">This Month's Links</a></li>
<li><a href="#random-pane" data-toggle="tab">Random</a></li>
<li><a href="#emails-pane" data-toggle="tab">Users by Domain</a></li>
Expand Down Expand Up @@ -42,6 +43,13 @@ <h3 class="body-ah">Capture jobs</h3>
<div id="job_queue">Loading...</div>
</div>

<div class="tab-pane" id="capture-error-pane">
<h3 class="body-ah">Capture errors</h3>
<button type="button" id="refresh-capture-errors">Refresh</button>&nbsp;<button type="button" id="auto-refresh-capture-errors">Start Auto-Refresh (every 15s)</button><span id="capture-errors-status"></span>
<br><br>
<div id="capture_errors">Loading...</div>
</div>


<div class="tab-pane" id="days-pane">
<h3 class="body-ah">Links from the past month</h3>
Expand Down Expand Up @@ -339,5 +347,85 @@ <h5>Finished task count:</h5>
{{/each}}
</div>
</script>

<script id="capture_errors-template" type="text/x-handlebars-template">
<div class="row">

<div class="col-sm-3">
<h4>Last hour</h4>
<dl class="dl-horizontal">
<dt>Completed:</dt><dd>{{ this.last_hour.completed }} ({{ this.last_hour.completed_percent}}%)</dd>
<dt>Failed:</dt><dd>{{ this.last_hour.failed }} ({{ this.last_hour.failed_percent}}%)</dd>
<dt>Killed by Celery:</dt><dd>{{ this.last_hour.celery_timeout }} ({{ this.last_hour.celery_timeout_percent}}%)</dd>
<dt>Mystery error:</dt><dd>{{ this.last_hour.mystery_error }} ({{ this.last_hour.mystery_error_percent}}%)</dd>
<dt>Scoop timeout:</dt><dd>{{ this.last_hour.timeout }} ({{ this.last_hour.timeout_percent}}%)</dd>
<dt>URL didn't load:</dt><dd>{{ this.last_hour.didnt_load }} ({{ this.last_hour.didnt_load_percent}}%)</dd>
</dl>
</div>

<div class="col-sm-3">
<h4>Last 3 hours</h4>
<dl class="dl-horizontal">
<dt>Completed:</dt><dd>{{ this.last_3_hrs.completed }} ({{ this.last_3_hrs.completed_percent}}%)</dd>
<dt>Failed:</dt><dd>{{ this.last_3_hrs.failed }} ({{ this.last_3_hrs.failed_percent}}%)</dd>
<dt>Killed by Celery:</dt><dd>{{ this.last_3_hrs.celery_timeout }} ({{ this.last_3_hrs.celery_timeout_percent}}%)</dd>
<dt>Mystery error:</dt><dd>{{ this.last_3_hrs.mystery_error }} ({{ this.last_3_hrs.mystery_error_percent}}%)</dd>
<dt>Scoop timeout:</dt><dd>{{ this.last_3_hrs.timeout }} ({{ this.last_3_hrs.timeout_percent}}%)</dd>
<dt>URL didn't load:</dt><dd>{{ this.last_3_hrs.didnt_load }} ({{ this.last_3_hrs.didnt_load_percent}}%)</dd>
</dl>
</div>

<div class="col-sm-3">
<h4>Last 24 hrs</h4>
<dl class="dl-horizontal">
<dt>Completed:</dt><dd>{{ this.last_24_hrs.completed }} ({{ this.last_24_hrs.completed_percent}}%)</dd>
<dt>Failed:</dt><dd>{{ this.last_24_hrs.failed }} ({{ this.last_24_hrs.failed_percent}}%)</dd>
<dt>Killed by Celery:</dt><dd>{{ this.last_24_hrs.celery_timeout }} ({{ this.last_24_hrs.celery_timeout_percent}}%)</dd>
<dt>Mystery error:</dt><dd>{{ this.last_24_hrs.mystery_error }} ({{ this.last_24_hrs.mystery_error_percent}}%)</dd>
<dt>Scoop timeout:</dt><dd>{{ this.last_24_hrs.timeout }} ({{ this.last_24_hrs.timeout_percent}}%)</dd>
<dt>URL didn't load:</dt><dd>{{ this.last_24_hrs.didnt_load }} ({{ this.last_24_hrs.didnt_load_percent}}%)</dd>
</dl>
</div>
</div>

<div class="row">

<div class="col-sm-3">
<h4>Last hour, previous day</h4>
<dl class="dl-horizontal">
<dt>Completed:</dt><dd>{{ this.last_hour_on_previous_day.completed }} ({{ this.last_hour_on_previous_day.completed_percent}}%)</dd>
<dt>Failed:</dt><dd>{{ this.last_hour_on_previous_day.failed }} ({{ this.last_hour_on_previous_day.failed_percent}}%)</dd>
<dt>Killed by Celery:</dt><dd>{{ this.last_hour_on_previous_day.celery_timeout }} ({{ this.last_hour_on_previous_day.celery_timeout_percent}}%)</dd>
<dt>Mystery error:</dt><dd>{{ this.last_hour_on_previous_day.mystery_error }} ({{ this.last_hour_on_previous_day.mystery_error_percent}}%)</dd>
<dt>Scoop timeout:</dt><dd>{{ this.last_hour_on_previous_day.timeout }} ({{ this.last_hour_on_previous_day.timeout_percent}}%)</dd>
<dt>URL didn't load:</dt><dd>{{ this.last_hour_on_previous_day.didnt_load }} ({{ this.last_hour_on_previous_day.didnt_load_percent}}%)</dd>
</dl>
</div>

<div class="col-sm-3">
<h4>Last 3 hours, previous day</h4>
<dl class="dl-horizontal">
<dt>Completed:</dt><dd>{{ this.last_3_hrs_on_previous_day.completed }} ({{ this.last_3_hrs_on_previous_day.completed_percent}}%)</dd>
<dt>Failed:</dt><dd>{{ this.last_3_hrs_on_previous_day.failed }} ({{ this.last_3_hrs_on_previous_day.failed_percent}}%)</dd>
<dt>Killed by Celery:</dt><dd>{{ this.last_3_hrs_on_previous_day.celery_timeout }} ({{ this.last_3_hrs_on_previous_day.celery_timeout_percent}}%)</dd>
<dt>Mystery error:</dt><dd>{{ this.last_3_hrs_on_previous_day.mystery_error }} ({{ this.last_3_hrs_on_previous_day.mystery_error_percent}}%)</dd>
<dt>Scoop timeout:</dt><dd>{{ this.last_3_hrs_on_previous_day.timeout }} ({{ this.last_3_hrs_on_previous_day.timeout_percent}}%)</dd>
<dt>URL didn't load:</dt><dd>{{ this.last_3_hrs_on_previous_day.didnt_load }} ({{ this.last_3_hrs_on_previous_day.didnt_load_percent}}%)</dd>
</dl>
</div>


<div class="col-sm-3">
<h4>Previous 24 hrs</h4>
<dl class="dl-horizontal">
<dt>Completed:</dt><dd>{{ this.previous_24_hrs.completed }} ({{ this.previous_24_hrs.completed_percent}}%)</dd>
<dt>Failed:</dt><dd>{{ this.previous_24_hrs.failed }} ({{ this.previous_24_hrs.failed_percent}}%)</dd>
<dt>Killed by Celery:</dt><dd>{{ this.previous_24_hrs.celery_timeout }} ({{ this.previous_24_hrs.celery_timeout_percent}}%)</dd>
<dt>Mystery error:</dt><dd>{{ this.previous_24_hrs.mystery_error }} ({{ this.previous_24_hrs.mystery_error_percent}}%)</dd>
<dt>Scoop timeout:</dt><dd>{{ this.previous_24_hrs.timeout }} ({{ this.previous_24_hrs.timeout_percent}}%)</dd>
<dt>URL didn't load:</dt><dd>{{ this.previous_24_hrs.didnt_load }} ({{ this.previous_24_hrs.didnt_load_percent}}%)</dd>
</dl>
</div>
</script>
{% endverbatim %}
{% endblock %}
116 changes: 116 additions & 0 deletions perma_web/perma/views/user_management.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,122 @@ def stats(request, stat_type=None):
out['total_ia_queue'] = r.llen('ia')
out['total_ia_readonly_queue'] = r.llen('ia-readonly')

elif stat_type == 'capture_errors':

# Set up time ranges
now = timezone.now()
last_24_hrs = (
now - timedelta(days=1),
now
)
previous_24_hrs = (
now - timedelta(days=2),
now - timedelta(days=1)
)
last_hour = (
now - timedelta(hours=1),
now
)
last_hour_on_previous_day = (
now - timedelta(days=1) - timedelta(hours=1),
now - timedelta(days=1)
)
last_3_hrs = (
now - timedelta(hours=3),
now
)
last_3_hrs_on_previous_day = (
now - timedelta(days=1) - timedelta(hours=3),
now - timedelta(hours=3)
)
ranges = {
"last_24_hrs": last_24_hrs,
"previous_24_hrs": previous_24_hrs,
"last_hour": last_hour,
"last_hour_on_previous_day": last_hour_on_previous_day,
"last_3_hrs": last_3_hrs,
"last_3_hrs_on_previous_day": last_3_hrs_on_previous_day
}

out = {}

for range_name, range_tuple in ranges.items():

#
# Get Totals
#

completed = CaptureJob.objects.filter(
engine='scoop-api',
status='completed',
capture_start_time__range=range_tuple
).count()

failed = CaptureJob.objects.filter(
engine='scoop-api',
status='failed',
capture_start_time__range=range_tuple
).count()

denominator = completed + failed

#
# Get By Error Type
#
if denominator:

celery_timeout = Link.objects.all_with_deleted().filter(
tags__name__in=['timeout-failure'],
creation_timestamp__range=range_tuple
).count()

mystery_error = Link.objects.all_with_deleted().filter(
tags__name__in=['scoop-stopped-failure'],
creation_timestamp__range=range_tuple
).count()

timeout = Link.objects.all_with_deleted().filter(
tags__name__in=['scoop-silent-failure'],
creation_timestamp__range=range_tuple
).count()

didnt_load = Link.objects.all_with_deleted().filter(
tags__name__in=['scoop-load-failure'],
creation_timestamp__range=range_tuple
).count()

out[range_name] = {
"completed": completed,
"completed_percent": completed/denominator * 100,
"failed": failed,
"failed_percent": failed/denominator * 100,
"celery_timeout": celery_timeout,
"celery_timeout_percent": celery_timeout/denominator * 100,
"mystery_error": mystery_error,
"mystery_error_percent": mystery_error/denominator * 100,
"timeout": timeout,
"timeout_percent": timeout/denominator * 100,
"didnt_load": didnt_load,
"didnt_load_percent": didnt_load/denominator * 100,
}

else:

out[range_name] = {
"completed": 0,
"completed_percent": 0,
"failed": 0,
"failed_percent": 0,
"celery_timeout": 0,
"celery_timeout_percent": 0,
"mystery_error": 0,
"mystery_error_percent": 0,
"timeout": 0,
"timeout_percent": 0,
"didnt_load": 0,
"didnt_load_percent": 0,
}

if out:
return JsonResponse(out)

Expand Down
36 changes: 36 additions & 0 deletions perma_web/static/bundles/admin-stats.js
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion perma_web/static/bundles/admin-stats.js.map
View file
Open in desktop

Large diffs are not rendered by default.

32 changes: 32 additions & 0 deletions perma_web/static/js/admin-stats.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ function fillSection(name, callback){
});
}

fillSection("capture_errors");
fillSection("celery_queues");
fillSection("celery");
fillSection("rate_limits");
Expand Down Expand Up @@ -89,6 +90,37 @@ document.getElementById('toggle-capture-jobs-auto-refresh').addEventListener('cl
})


// Refresh the capture errors once, on button press
// or, start auto-refreshing every 15s, when the other button is pressed
function refresh_capture_errors(){
let status = document.getElementById('capture-errors-status')
status.innerText = 'Refreshing...';
fillSection("capture_errors", () => {
status.innerText = 'Refreshed!';
setTimeout(()=> status.innerText = '', 2000);
});
}
function auto_refresh_capture_errors(){
return setInterval(function(){ refresh_capture_errors()}, 15000);
}
document.getElementById('refresh-capture-errors').addEventListener('click', (e) => {
refresh_capture_errors()
})
let capture_errors_refresh = null;
document.getElementById('auto-refresh-capture-errors').addEventListener('click', (e) => {
if (capture_errors_refresh){
clearInterval(capture_errors_refresh);
capture_errors_refresh = null;
e.target.innerText = 'Start Auto-Refresh (every 15s)';
} else {
e.target.innerText = 'Stop Auto-Refresh';
refresh_capture_errors()
capture_errors_refresh = auto_refresh_capture_errors();
}
})



// Select the tab specified in the hash, if present on page load
if (window.location.hash) {
let tabNav = document.querySelector(`a[href="${window.location.hash}"]`);
Expand Down