Optimize lazy log rendering and tail following

Signed-off-by: Loren Eteval <loren.eteval@proton.me>
This commit is contained in:
Loren Eteval
2026-08-17 19:21:47 +08:00
parent 2ead6120d9
commit 44cfd599b1
4 changed files with 659 additions and 38 deletions
+8
View File
@@ -374,6 +374,14 @@ class DraculaTextBrowser(Mixins.ThemeAware, AppQTextBrowser):
)
)
def setSyntaxHighlightingEnabled(self, enabled: bool):
"""Enable or suspend automatic syntax highlighting for document edits."""
self._syntaxHighlighter.setHighlightingEnabled(enabled)
def rehighlightBlock(self, block):
"""Rehighlight one document block without rebuilding the document."""
self._syntaxHighlighter.rehighlightBlock(block)
def themeChangedCallback(self, theme: str):
"""Refresh browser chrome after an application theme change."""
self.setStyleSheet(
+12
View File
@@ -123,11 +123,20 @@ class AppQSyntaxHighlighter(Mixins.ThemeAware, QSyntaxHighlighter):
def __init__(self, *args, **kwargs):
"""Initialize the AppQSyntaxHighlighter."""
self._theme = AppStyleSheet.normalizeTheme(kwargs.pop('theme', _currentTheme()))
self._highlightingEnabled = True
super().__init__(*args, **kwargs)
self.highlightRules = list()
def setHighlightingEnabled(self, enabled: bool):
"""Enable or suspend formatting of blocks changed by the document.
Suspending is useful for a large bulk replacement whose blocks will be
rehighlighted incrementally by its persistent presentation owner.
"""
self._highlightingEnabled = bool(enabled)
def buildHighlightRules(self, palette):
"""Return highlight rules for *palette*."""
return list()
@@ -145,6 +154,9 @@ class AppQSyntaxHighlighter(Mixins.ThemeAware, QSyntaxHighlighter):
def highlightBlock(self, text):
"""Handle highlight block for the app q syntax highlighter."""
if not self._highlightingEnabled:
return
for highlightRule in self.highlightRules:
iterator = highlightRule.regex.globalMatch(text)
+342 -33
View File
@@ -31,8 +31,11 @@ from Furious.Service.LogManager import (
)
from PySide6 import QtCore
from PySide6.QtGui import QTextCursor
from PySide6.QtWidgets import *
from bisect import bisect_left
__all__ = ['LogPage']
registerAppSettings('LogViewerWidgetPointSize')
@@ -113,6 +116,10 @@ def saveAsFile(content: str):
class LogPage(Mixins.QTranslatable, QMainWindow):
"""Present and filter the unified application log stream as a page."""
UpdateDelay = 25
HighlightBatchSize = 250
TailTolerance = 10
def __init__(self, *args, **kwargs):
"""Initialize the logging page."""
manager, fontFamily = (
@@ -143,10 +150,34 @@ class LogPage(Mixins.QTranslatable, QMainWindow):
pointSizeSettingsName='LogViewerWidgetPointSize',
)
self.textBrowser.setLineWrapMode(DraculaTextBrowser.LineWrapMode.NoWrap)
self.textBrowser.setUndoRedoEnabled(False)
self.textBrowser.document().setMaximumBlockCount(manager.maximumEntries)
self._entriesDirty = True
self._representationInvalid = True
self._renderedSequence = 0
self._renderedCategoryId = None
self._renderedEntrySequences = tuple()
self._followTail = True
self._documentMutation = False
self._highlightNextBlock = None
self._pendingScrollRatio = None
self._updateTimer = QtCore.QTimer(self)
self._updateTimer.setSingleShot(True)
self._updateTimer.timeout.connect(self._renderPendingEntries)
self._highlightTimer = QtCore.QTimer(self)
self._highlightTimer.setSingleShot(True)
self._highlightTimer.timeout.connect(self._highlightNextBatch)
self._scrollTimer = QtCore.QTimer(self)
self._scrollTimer.setSingleShot(True)
self._scrollTimer.timeout.connect(self._restoreScrollPosition)
self._followStateTimer = QtCore.QTimer(self)
self._followStateTimer.setSingleShot(True)
self._followStateTimer.timeout.connect(self._updateFollowTailFromScrollbar)
filterLayout = QHBoxLayout()
filterLayout.setContentsMargins(0, 0, 0, 0)
@@ -257,6 +288,10 @@ class LogPage(Mixins.QTranslatable, QMainWindow):
self.manager.entryAdded.connect(self._entryAdded)
self.manager.entriesCleared.connect(self._entriesCleared)
scrollbar = self.textBrowser.verticalScrollBar()
scrollbar.actionTriggered.connect(self._scrollActionTriggered)
scrollbar.sliderReleased.connect(self._scrollActionTriggered)
self.retranslate()
def _registerMenuShortcuts(self, menu):
@@ -301,27 +336,300 @@ class LogPage(Mixins.QTranslatable, QMainWindow):
finally:
self.filterComboBox.blockSignals(False)
def _refreshEntries(self):
"""Render the current immutable filtered-entry snapshot."""
selectedCategoryId = self.filterComboBox.currentData()
sequence, entries = self.manager.snapshot(selectedCategoryId)
def _pageCanRender(self) -> bool:
"""Return whether document work can currently reach the screen."""
window = self.window()
return self.isVisible() and not (
hasattr(window, 'isMinimized') and window.isMinimized()
)
def _requestRefresh(self, *, invalidate=False, immediate=False):
"""Mark the presentation stale and coalesce visible catch-up work."""
self._entriesDirty = True
if invalidate:
self._representationInvalid = True
if not self._pageCanRender():
return
if immediate:
self._updateTimer.stop()
self._updateTimer.start(0)
elif not self._updateTimer.isActive():
self._updateTimer.start(self.UpdateDelay)
@staticmethod
def _incrementalPlan(oldSequences, currentSequences):
"""Return leading removals and the first new snapshot entry.
Retention pruning can only remove a prefix, while ordinary collection
can only append a suffix. Any other relationship requires one clean
rebuild because a clear/filter change invalidated the representation.
"""
if not oldSequences:
return 0, 0
if not currentSequences:
return len(oldSequences), 0
oldStart = bisect_left(oldSequences, currentSequences[0])
survivors = oldSequences[oldStart:]
if len(currentSequences) < len(survivors):
return None
if tuple(currentSequences[: len(survivors)]) != survivors:
return None
return oldStart, len(survivors)
def _scrollRatio(self) -> float:
"""Return the current vertical position as a bounded range ratio."""
scrollbar = self.textBrowser.verticalScrollBar()
if scrollbar.maximum() <= 0:
return 1.0
return min(1.0, max(0.0, scrollbar.value() / scrollbar.maximum()))
def _removeLeadingBlocks(self, count: int):
"""Remove *count* rendered entries in one document edit."""
if count <= 0:
return
if count >= len(self._renderedEntrySequences):
self.textBrowser.clear()
return
document = self.textBrowser.document()
firstRetainedBlock = document.findBlockByNumber(count)
if not firstRetainedBlock.isValid():
self.textBrowser.clear()
return
cursor = QTextCursor(document)
cursor.setPosition(0)
cursor.setPosition(
firstRetainedBlock.position(),
QTextCursor.MoveMode.KeepAnchor,
)
cursor.removeSelectedText()
def _appendEntries(self, entries, *, hasExistingEntries: bool):
"""Append one formatted batch and return its first block number."""
if not entries:
return None
content = '\n'.join(formatLogEntry(entry) for entry in entries)
if not hasExistingEntries:
self.textBrowser.setPlainText(content)
return 0
document = self.textBrowser.document()
firstChangedBlock = max(0, document.blockCount() - 1)
cursor = QTextCursor(document)
cursor.movePosition(QTextCursor.MoveOperation.End)
cursor.insertBlock()
cursor.insertText(content)
# Inserting the paragraph separator also invalidates the previous last
# block, so progressive highlighting must resume there rather than at
# the first newly inserted block.
return firstChangedBlock
def _replaceEntries(self, entries):
"""Replace the text in one cheap unhighlighted document operation."""
self._highlightTimer.stop()
self._highlightNextBlock = None
content = '\n'.join(formatLogEntry(entry) for entry in entries)
self.textBrowser.setPlainText(content)
return 0 if entries else None
def _synchronizeDocument(self, entries, categoryId: str):
"""Apply one snapshot with the smallest safe document mutation."""
currentSequences = tuple(entry.sequence for entry in entries)
fullRebuild = (
self._representationInvalid or self._renderedCategoryId != categoryId
)
plan = None
if not fullRebuild:
plan = self._incrementalPlan(
self._renderedEntrySequences,
currentSequences,
)
fullRebuild = plan is None
oldRatio = self._scrollRatio()
firstHighlightBlock = None
restoreManualPosition = fullRebuild
leadingBoundaryChanged = False
self._documentMutation = True
self.textBrowser.setUpdatesEnabled(False)
self.textBrowser.setSyntaxHighlightingEnabled(False)
try:
if fullRebuild:
firstHighlightBlock = self._replaceEntries(entries)
else:
dropCount, firstNewEntry = plan
if self._highlightNextBlock is not None and dropCount:
self._highlightNextBlock = max(
0,
self._highlightNextBlock - dropCount,
)
self._removeLeadingBlocks(dropCount)
leadingBoundaryChanged = bool(dropCount)
newEntries = entries[firstNewEntry:]
firstHighlightBlock = self._appendEntries(
newEntries,
hasExistingEntries=bool(currentSequences[:firstNewEntry]),
)
restoreManualPosition = bool(dropCount)
finally:
self.textBrowser.setSyntaxHighlightingEnabled(True)
if leadingBoundaryChanged:
firstBlock = self.textBrowser.document().firstBlock()
if firstBlock.isValid():
# Removing a retained prefix invalidates the new leading
# paragraph independently from the appended tail range.
self.textBrowser.rehighlightBlock(firstBlock)
self.textBrowser.setUpdatesEnabled(True)
self._documentMutation = False
self.textBrowser.viewport().update()
self._renderedCategoryId = categoryId
self._renderedEntrySequences = currentSequences
self._representationInvalid = False
if firstHighlightBlock is not None:
self._scheduleHighlight(firstHighlightBlock)
if self._followTail:
self._scheduleScrollRestore()
elif restoreManualPosition:
self._scheduleScrollRestore(oldRatio)
@QtCore.Slot()
def _renderPendingEntries(self):
"""Catch the visible document up to one immutable manager snapshot."""
if not self._entriesDirty or not self._pageCanRender():
return
selectedCategoryId = self.filterComboBox.currentData()
if not isinstance(selectedCategoryId, str):
selectedCategoryId = ALL_LOGS_FILTER
sequence, entries = self.manager.snapshot(selectedCategoryId)
self._synchronizeDocument(entries, selectedCategoryId)
self._renderedSequence = sequence
scrollbar = self.textBrowser.verticalScrollBar()
scrollbar.setValue(scrollbar.maximum())
self._entriesDirty = False
def _refreshEntriesIfVisible(self):
"""Render pending entries only while this page is actually visible."""
if self.isVisible():
self._refreshEntries()
def _scheduleHighlight(self, firstBlock: int):
"""Coalesce incremental highlighting from the earliest changed block."""
if self._highlightNextBlock is None:
self._highlightNextBlock = firstBlock
else:
self._entriesDirty = True
self._highlightNextBlock = min(self._highlightNextBlock, firstBlock)
if self._pageCanRender() and not self._highlightTimer.isActive():
self._highlightTimer.start(0)
@QtCore.Slot()
def _highlightNextBatch(self):
"""Format a bounded block batch and yield back to the event loop."""
if self._highlightNextBlock is None or not self._pageCanRender():
return
document = self.textBrowser.document()
start = self._highlightNextBlock
end = min(document.blockCount(), start + self.HighlightBatchSize)
for blockNumber in range(start, end):
block = document.findBlockByNumber(blockNumber)
if block.isValid():
self.textBrowser.rehighlightBlock(block)
if end < document.blockCount():
self._highlightNextBlock = end
self._highlightTimer.start(0)
else:
self._highlightNextBlock = None
if self._followTail:
self._scheduleScrollRestore()
def _scheduleScrollRestore(self, ratio=None):
"""Restore tail or reading position after document layout settles."""
self._pendingScrollRatio = ratio
if self._pageCanRender():
self._scrollTimer.start(0)
@QtCore.Slot()
def _restoreScrollPosition(self):
"""Apply a coalesced post-layout scroll position without changing intent."""
if not self._pageCanRender():
return
scrollbar = self.textBrowser.verticalScrollBar()
self._documentMutation = True
try:
if self._followTail:
scrollbar.setValue(scrollbar.maximum())
self.textBrowser.horizontalScrollBar().setValue(0)
elif self._pendingScrollRatio is not None:
scrollbar.setValue(
round(scrollbar.maximum() * self._pendingScrollRatio)
)
finally:
self._documentMutation = False
self._pendingScrollRatio = None
@QtCore.Slot()
def _updateFollowTailFromScrollbar(self):
"""Capture whether the user's viewport is following the newest entry."""
if self._documentMutation:
return
scrollbar = self.textBrowser.verticalScrollBar()
self._followTail = scrollbar.maximum() - scrollbar.value() <= self.TailTolerance
def _scrollActionTriggered(self, *_args):
"""Defer follow-state capture until the user scroll action is applied."""
if self._documentMutation:
return
self._scrollTimer.stop()
self._pendingScrollRatio = None
self._followStateTimer.start(0)
@QtCore.Slot(int)
def _filterChanged(self, _index: int):
@@ -335,7 +643,7 @@ class LogPage(Mixins.QTranslatable, QMainWindow):
AppSettings.set('LogViewerSelectedCategory', categoryId)
self._refreshEntriesIfVisible()
self._requestRefresh(invalidate=True, immediate=True)
@QtCore.Slot(object)
def _categoryRegistered(self, category):
@@ -350,37 +658,38 @@ class LogPage(Mixins.QTranslatable, QMainWindow):
@QtCore.Slot(object)
def _entryAdded(self, entry):
"""Append an entry when it matches the active filter."""
if not self.isVisible():
self._entriesDirty = True
return
"""Coalesce newly collected entries into the visible presentation."""
if entry.sequence <= self._renderedSequence:
return
if entry.sequence != self._renderedSequence + 1:
self._refreshEntries()
return
self._renderedSequence = entry.sequence
categoryId = self.filterComboBox.currentData()
if categoryId in (ALL_LOGS_FILTER, entry.categoryId):
self.textBrowser.appendLine(formatLogEntry(entry))
self._requestRefresh()
@QtCore.Slot(object)
def _entriesCleared(self, _categoryIds):
"""Refresh the presentation after the underlying collection changes."""
self._refreshEntriesIfVisible()
self._requestRefresh(invalidate=True, immediate=True)
def showEvent(self, event):
"""Render entries accumulated while the page was hidden."""
super().showEvent(event)
if self._entriesDirty:
self._refreshEntries()
self._requestRefresh(immediate=True)
elif self._highlightNextBlock is not None:
self._highlightTimer.start(0)
if self._followTail:
self._scheduleScrollRestore()
def hideEvent(self, event):
"""Pause presentation work while preserving collection and scroll intent."""
self._updateFollowTailFromScrollbar()
self._updateTimer.stop()
self._highlightTimer.stop()
self._scrollTimer.stop()
self._followStateTimer.stop()
super().hideEvent(event)
def plainText(self) -> str:
"""Return the plain text currently shown by the selected filter."""
@@ -398,4 +707,4 @@ class LogPage(Mixins.QTranslatable, QMainWindow):
self.pageTitleLabel.setText(_('Log'))
self.filterLabel.setText(_('Log Type'))
self._populateFilters(selectedCategoryId)
self._refreshEntriesIfVisible()
self._requestRefresh(immediate=True)
+297 -5
View File
@@ -49,10 +49,12 @@ from tests.support import (
collectAtBoundary,
isolatedSettings,
processQtEvents,
waitFor,
)
import copy
import unittest
import weakref
from unittest import mock
@@ -166,6 +168,23 @@ class UnifiedLogPageTest(unittest.TestCase):
"""Create the process-wide headless QApplication."""
application()
def tearDown(self):
"""Finish deferred document and widget cleanup between tests."""
collectAtBoundary()
def assertRendered(self, page, *, highlighting=True):
"""Wait for the coalesced snapshot and optional highlighting batches."""
self.assertTrue(waitFor(lambda: not page._entriesDirty))
if highlighting:
self.assertTrue(waitFor(lambda: page._highlightNextBlock is None))
@staticmethod
def disposePage(page):
"""Release one persistent page and its owned timers."""
page.close()
page.deleteLater()
def testHiddenPageRendersOneOrderedSnapshotWhenShown(self):
"""Do not mutate the document while hidden; catch up exactly once."""
with isolatedSettings():
@@ -182,7 +201,7 @@ class UnifiedLogPageTest(unittest.TestCase):
page.show()
processQtEvents()
self.assertRendered(page)
self.assertEqual(
page.textBrowser.toPlainText().splitlines(),
@@ -198,7 +217,7 @@ class UnifiedLogPageTest(unittest.TestCase):
page.show()
processQtEvents()
self.assertRendered(page)
self.assertEqual(
page.textBrowser.toPlainText().splitlines(),
@@ -208,15 +227,288 @@ class UnifiedLogPageTest(unittest.TestCase):
coreIndex = page.filterComboBox.findData(CORE_LOG_CATEGORY)
page.filterComboBox.setCurrentIndex(coreIndex)
processQtEvents()
self.assertRendered(page)
self.assertEqual(
page.textBrowser.toPlainText().splitlines(),
['core one', 'core two'],
)
page.close()
page.deleteLater()
self.disposePage(page)
def testLargeHiddenBacklogDefersOneBulkDocumentCatchUp(self):
"""Return from show before rendering and avoid per-entry replay."""
with isolatedSettings():
entryCount = 5_000
manager = LogManager(maximumEntries=entryCount)
page = LogPage(manager=manager)
page.resize(900, 420)
for index in range(entryCount):
manager.append(f'bulk entry {index:05d}')
self.assertEqual(page.textBrowser.toPlainText(), '')
page.show()
# showEvent only schedules the catch-up; it does not synchronously
# build and highlight thousands of QTextDocument blocks.
self.assertTrue(page._entriesDirty)
self.assertEqual(page.textBrowser.toPlainText(), '')
self.assertRendered(page)
lines = page.textBrowser.toPlainText().splitlines()
self.assertEqual(len(lines), entryCount)
self.assertEqual(lines[0], 'bulk entry 00000')
self.assertEqual(lines[-1], 'bulk entry 04999')
self.assertEqual(page.textBrowser.document().blockCount(), entryCount)
self.disposePage(page)
def testFollowingTailSurvivesHideAndCatchUp(self):
"""Restore the newest entry after collecting while the page is hidden."""
with isolatedSettings():
manager = LogManager(maximumEntries=500)
page = LogPage(manager=manager)
page.resize(800, 320)
for index in range(200):
manager.append(f'before hide {index:03d}')
page.show()
self.assertRendered(page)
scrollbar = page.textBrowser.verticalScrollBar()
self.assertGreater(scrollbar.maximum(), 0)
self.assertEqual(scrollbar.value(), scrollbar.maximum())
self.assertTrue(page._followTail)
page.hide()
for index in range(25):
manager.append(f'while hidden {index:03d}')
page.show()
self.assertRendered(page)
self.assertTrue(waitFor(lambda: scrollbar.value() == scrollbar.maximum()))
self.assertTrue(page._followTail)
self.assertTrue(page.plainText().endswith('while hidden 024'))
self.disposePage(page)
def testManualHistoryReadingDisablesAndThenResumesTail(self):
"""Do not yank an upward-scrolled viewport until it returns to bottom."""
with isolatedSettings():
manager = LogManager(maximumEntries=500)
page = LogPage(manager=manager)
page.resize(800, 320)
for index in range(200):
manager.append(f'history {index:03d}')
page.show()
self.assertRendered(page)
scrollbar = page.textBrowser.verticalScrollBar()
scrollbar.setValue(0)
page._updateFollowTailFromScrollbar()
self.assertFalse(page._followTail)
manager.append('arrived while reading')
self.assertRendered(page)
self.assertEqual(scrollbar.value(), 0)
self.assertLess(scrollbar.value(), scrollbar.maximum())
scrollbar.setValue(scrollbar.maximum())
page._updateFollowTailFromScrollbar()
self.assertTrue(page._followTail)
manager.append('tail resumed')
self.assertRendered(page)
self.assertTrue(waitFor(lambda: scrollbar.value() == scrollbar.maximum()))
self.assertTrue(page.plainText().endswith('tail resumed'))
self.disposePage(page)
def testFilteredTailCatchUpAndPruningRemainExact(self):
"""Keep filtered tail semantics and discard pruned filtered entries."""
with isolatedSettings():
manager = LogManager(maximumEntries=40)
page = LogPage(manager=manager)
page.resize(800, 260)
for index in range(30):
manager.append(f'application {index:03d}', APPLICATION_LOG_CATEGORY)
manager.append(f'core {index:03d}', CORE_LOG_CATEGORY)
coreIndex = page.filterComboBox.findData(CORE_LOG_CATEGORY)
page.filterComboBox.setCurrentIndex(coreIndex)
page.show()
self.assertRendered(page)
scrollbar = page.textBrowser.verticalScrollBar()
self.assertTrue(page.plainText().endswith('core 029'))
self.assertEqual(scrollbar.value(), scrollbar.maximum())
page.hide()
for index in range(45):
manager.append(
f'new application {index:03d}',
APPLICATION_LOG_CATEGORY,
)
manager.append('new core tail', CORE_LOG_CATEGORY)
page.show()
self.assertRendered(page)
self.assertTrue(waitFor(lambda: scrollbar.value() == scrollbar.maximum()))
self.assertEqual(page.plainText(), 'new core tail')
self.assertTrue(page._followTail)
self.disposePage(page)
def testIncrementalBatchesRehighlightChangedBoundaryBlocks(self):
"""Color paragraph boundaries invalidated by append and retention edits."""
with isolatedSettings():
manager = LogManager(maximumEntries=24)
page = LogPage(manager=manager)
def appendEntries(start, count):
"""Append deterministic lines matched by several log rules."""
for index in range(start, start + count):
manager.append(
'2026/08/17 18:45:'
f'{index % 60:02d}.000000 from '
f'127.0.0.1:{5000 + index} accepted '
'//example.com:443 [http >> proxy]'
)
def missingFormats():
"""Return blocks that fell back to the default text format."""
document = page.textBrowser.document()
return tuple(
blockNumber
for blockNumber in range(document.blockCount())
if not document.findBlockByNumber(blockNumber).layout().formats()
)
appendEntries(0, 20)
page.show()
self.assertRendered(page)
appendEntries(20, 3)
self.assertRendered(page)
self.assertEqual(missingFormats(), ())
# This batch prunes four old entries as it appends five new ones,
# exercising both the new first and previous last block boundaries.
appendEntries(23, 5)
self.assertRendered(page)
self.assertEqual(page.textBrowser.document().blockCount(), 24)
self.assertEqual(missingFormats(), ())
self.disposePage(page)
def testRepeatedVisibilityCyclesReuseTimersWithoutDuplicateEntries(self):
"""Keep one owned render pipeline stable through repeated page visits."""
with isolatedSettings():
manager = LogManager(maximumEntries=500)
page = LogPage(manager=manager)
timers = (
page._updateTimer,
page._highlightTimer,
page._scrollTimer,
page._followStateTimer,
)
page.show()
self.assertRendered(page)
expected = []
for index in range(30):
page.hide()
line = f'visibility cycle {index:02d}'
expected.append(line)
manager.append(line)
page.show()
self.assertRendered(page, highlighting=False)
self.assertEqual(page.plainText().splitlines(), expected)
self.assertEqual(
(
page._updateTimer,
page._highlightTimer,
page._scrollTimer,
page._followStateTimer,
),
timers,
)
self.disposePage(page)
def testPageDestructionReleasesOwnedRenderTimers(self):
"""Do not retain the page or its persistent timers after destruction."""
with isolatedSettings():
manager = LogManager(maximumEntries=20)
page = LogPage(manager=manager)
references = tuple(
weakref.ref(value)
for value in (
page,
page._updateTimer,
page._highlightTimer,
page._scrollTimer,
page._followStateTimer,
)
)
page.show()
self.assertRendered(page)
self.disposePage(page)
del page
collectAtBoundary()
self.assertTrue(all(reference() is None for reference in references))
class DialogBehaviorTest(unittest.TestCase):