Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Logview threadpool #861

Merged
merged 11 commits into from
Jan 30, 2024
Merged

Logview threadpool #861

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
2eb4936
Update dispatchQuery to use min_cores
DiegoTavares Sep 23, 2020
dd3172a
Merge remote-tracking branch 'upstream/master'
DiegoTavares Dec 10, 2020
eaa4951
Revert "Update dispatchQuery to use min_cores"
DiegoTavares Dec 10, 2020
e1ae37e
Add threadpool logic for handling incoming log update requests
FermiPerumal Oct 30, 2020
a10c80e
Suppress known ui warning messages
FermiPerumal Oct 30, 2020
07a10b0
Fix logview unit tests to work around the threading logic
FermiPerumal Dec 18, 2020
821d38a
Merge branch 'master' into logview_threadpool
DiegoTavares Jan 29, 2024
652ccfd
FIx unit tests
DiegoTavares Jan 29, 2024
2aa77f0
Fix python lint
DiegoTavares Jan 29, 2024
9394fb2
Fix python lint
DiegoTavares Jan 29, 2024
54c7067
Fix lint
DiegoTavares Jan 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion cuegui/cuegui/Main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,6 @@ def startup(app_name, app_version, argv):
app.aboutToQuit.connect(closingTime) # pylint: disable=no-member
app.exec_()


def closingTime():
"""Window close callback."""
logger.info("Closing all threads...")
Expand Down
141 changes: 97 additions & 44 deletions cuegui/cuegui/plugins/LogViewPlugin.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,9 @@
import os
import re
import string
import sys
import time
import traceback

from qtpy import QtGui
from qtpy import QtCore
Expand Down Expand Up @@ -292,10 +294,40 @@ def line_number_area_paint_event(self, event):
bottom = top + self.blockBoundingRect(block).height()
block_number += 1

class LogLoadSignals(QtCore.QObject):
"""Signals for the LoadLog action"""
SIG_LOG_LOAD_ERROR = QtCore.Signal(tuple)
SIG_LOG_LOAD_RESULT = QtCore.Signal(str, str)
SIG_LOG_LOAD_FINISHED = QtCore.Signal()

class LogLoader(QtCore.QRunnable):
"""A thread to load logs"""
def __init__(self, fn, *args, **kwargs):
super(LogLoader, self).__init__()
self.fn = fn
self.args = args
self.kwargs = kwargs
self.signals = LogLoadSignals()

@QtCore.Slot()
def run(self):
# pylint: disable=bare-except
try:
content, log_mtime = self.fn(*self.args, **self.kwargs)
except:
exctype, value = sys.exc_info()[:2]
self.signals.SIG_LOG_LOAD_ERROR.emit(
(exctype, value, traceback.format_exc()))
else:
self.signals.SIG_LOG_LOAD_RESULT.emit(content, log_mtime)
finally:
self.signals.SIG_LOG_LOAD_FINISHED.emit()

class LogViewWidget(QtWidgets.QWidget):
"""Displays the log file for the selected frame."""

"""
Displays the log file for the selected frame
"""
SIG_CONTENT_UPDATED = QtCore.Signal(str, str)
def __init__(self, parent=None):
"""
Create the UI elements
Expand Down Expand Up @@ -453,6 +485,9 @@ def __init__(self, parent=None):
self._current_match = 0
self._content_box.mousePressedSignal.connect(self._on_mouse_pressed)

self.SIG_CONTENT_UPDATED.connect(self._update_log_content)
self.log_thread_pool = QtCore.QThreadPool()

def _on_mouse_pressed(self, pos):
"""
Mouse press event, to be called when the user scrolls by hand or moves
Expand Down Expand Up @@ -788,12 +823,56 @@ def _display_log_content(self):
"""

try:
self._update_log()
self._new_log = False
if not os.path.exists(self._log_file):
self._log_file_exists = False
content = 'Log file does not exist: %s' % self._log_file
self._content_timestamp = time.time()
self._update_log_content(content, self._log_mtime)
else:
# Creating the load logs process as qrunnables so
# that they don't block the ui while loading
log_loader = LogLoader(self._load_log, self._log_file,
self._new_log, self._log_mtime)
log_loader.signals.SIG_LOG_LOAD_RESULT.connect(
self._receive_log_results)
log_loader.setAutoDelete(True)
self.log_thread_pool.start(log_loader)
self.log_thread_pool.waitForDone()
self._new_log = False
finally:
QtCore.QTimer.singleShot(5000, self._display_log_content)

def _update_log(self):
# pylint: disable=no-self-use
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: If there's no self use, can this be a static method? If not, add a comment explaining why.

@QtCore.Slot()
def _load_log(self, log_file, new_log, curr_log_mtime):
content = None
log_size = int(os.stat(log_file).st_size)
if log_size > 1 * 1e6:
content = ('Log file size (%0.1f MB) exceeds the size '
'threshold (1.0 MB).'
% float(log_size / (1024 * 1024)))
elif not new_log and os.path.exists(log_file):
log_mtime = os.path.getmtime(log_file)
if log_mtime > curr_log_mtime:
curr_log_mtime = log_mtime # no new updates
content = ''

if content is None:
content = ''
try:
with open(log_file, 'r') as f:
content = f.read()
except IOError:
content = 'Can not access log file: %s' % log_file

return content, curr_log_mtime

@QtCore.Slot()
def _receive_log_results(self, content, log_mtime):
self.SIG_CONTENT_UPDATED.emit(content, log_mtime)

@QtCore.Slot(str, str)
def _update_log_content(self, content, log_mtime):
"""
Updates the content of the content box with the content of the log
file, if necessary. The full path to the log file will be populated in
Expand All @@ -813,49 +892,23 @@ def _update_log(self):
(if necessary)
"""

# Get the content of the log file
if not self._log_file:
return # There's no log file, nothing to do here!
self._path.setText(self._log_file)
content = None
if not os.path.exists(self._log_file):
self._log_file_exists = False
content = 'Log file does not exist: %s' % self._log_file
self._content_timestamp = time.time()
else:
log_size = int(os.stat(self._log_file).st_size)
if log_size > 5 * 1e6:
content = ('Log file size (%0.1f MB) exceeds the size '
'threshold (5.0 MB).'
% float(log_size / (1024 * 1024)))
elif not self._new_log and os.path.exists(self._log_file):
log_mtime = os.path.getmtime(self._log_file)
if log_mtime > self._log_mtime:
self._log_mtime = log_mtime # no new updates
content = ''

if content is None:
content = ''
try:
with open(self._log_file, 'r') as f:
content = f.read()
except IOError:
content = 'Can not access log file: %s' % self._log_file
self._log_mtime = log_mtime

# Do we need to scroll to the end?
scroll_to_end = (self._scrollbar_max == self._scrollbar_value
or self._new_log)
self.app.processEvents()

# Update the content in the gui (if necessary)
current_text = (self._content_box.toPlainText() or '')
new_text = content.lstrip(str(current_text))
if new_text:
if self._new_log:
self._content_box.setPlainText(content)
else:
if self._new_log:
self._content_box.setPlainText(content)
else:
current_text = (self._content_box.toPlainText() or '')
new_text = content.lstrip(str(current_text))
if new_text:
self._content_box.appendPlainText(new_text)
self._content_timestamp = time.time()
self.app.processEvents()
self._content_timestamp = time.time()
self._path.setText(self._log_file)

scroll_to_end = (self._scrollbar_max == self._scrollbar_value
or self._new_log)

# Adjust scrollbar value (if necessary)
self._scrollbar_max = self._log_scrollbar.maximum()
Expand Down
17 changes: 8 additions & 9 deletions cuegui/tests/plugins/LogViewPlugin_tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,21 +71,21 @@ def setUp(self):

def test_shouldDisplayFirstLogFile(self):
cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2])

self.logViewPlugin.logview_widget._receive_log_results(_LOG_TEXT_1, 0)
self.assertEqual(_LOG_TEXT_1, self.logViewPlugin.logview_widget._content_box.toPlainText())

def test_shouldUpdateLogFile(self):
cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2])
self.logViewPlugin.logview_widget._receive_log_results(_LOG_TEXT_1, 0)
new_contents = _LOG_TEXT_1 + '\nanother line at the end'
self.log1.set_contents(new_contents)
cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2])

self.logViewPlugin.logview_widget._receive_log_results(new_contents, 0)
self.assertEqual(new_contents, self.logViewPlugin.logview_widget._content_box.toPlainText())

def test_shouldHighlightAllSearchResults(self):
cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2])
self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState(
qtpy.QtCore.Qt.CheckState.Unchecked)
self.logViewPlugin.logview_widget._receive_log_results(_LOG_TEXT_1, 0)
self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState(qtpy.QtCore.Qt.CheckState.Unchecked)
DiegoTavares marked this conversation as resolved.
Show resolved Hide resolved

self.logViewPlugin.logview_widget._search_box.setText('lorem')
self.logViewPlugin.logview_widget._search_button.click()
Expand All @@ -100,7 +100,7 @@ def test_shouldHighlightAllSearchResults(self):
self.logViewPlugin.logview_widget._content_box, matches[1][0], matches[1][1]))

def test_shouldMoveCursorToSecondSearchResult(self):
cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2])
self.logViewPlugin.logview_widget._receive_log_results(_LOG_TEXT_1, 0)
self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState(
qtpy.QtCore.Qt.CheckState.Unchecked)

Expand All @@ -114,7 +114,7 @@ def test_shouldMoveCursorToSecondSearchResult(self):
self.assertEqual(132, self.logViewPlugin.logview_widget._cursor.position())

def test_shouldMoveCursorLastSearchResult(self):
cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2])
self.logViewPlugin.logview_widget._receive_log_results(_LOG_TEXT_1, 0)
self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState(
qtpy.QtCore.Qt.CheckState.Unchecked)

Expand All @@ -128,10 +128,9 @@ def test_shouldMoveCursorLastSearchResult(self):
self.assertEqual(132, self.logViewPlugin.logview_widget._cursor.position())

def test_shouldPerformCaseInsensitiveSearch(self):
cuegui.app().display_log_file_content.emit([self.logPath1, self.logPath2])
self.logViewPlugin.logview_widget._receive_log_results(_LOG_TEXT_1, 0)
self.logViewPlugin.logview_widget._case_stv_checkbox.setCheckState(
qtpy.QtCore.Qt.CheckState.Checked)

self.logViewPlugin.logview_widget._search_box.setText('lorem')
self.logViewPlugin.logview_widget._search_button.click()
matches = self.logViewPlugin.logview_widget._matches
Expand Down
Loading