From 2be63cae26e3c42f8f20c0adfed5e4529ec25399 Mon Sep 17 00:00:00 2001 From: Loren Eteval Date: Sat, 19 Sep 2026 19:05:51 +0800 Subject: [PATCH] Make tests independent of Git history and update suite documentation Signed-off-by: Loren Eteval --- tests/AGENTS.md | 6 +- tests/README.md | 530 +++++++++++++--------- tests/benchmarks/benchmark_log_manager.py | 214 +++++++++ tests/test_log_manager_benchmark.py | 159 +++++++ tests/test_log_manager_generation.py | 98 ---- 5 files changed, 684 insertions(+), 323 deletions(-) create mode 100644 tests/benchmarks/benchmark_log_manager.py create mode 100644 tests/test_log_manager_benchmark.py diff --git a/tests/AGENTS.md b/tests/AGENTS.md index 1bdd8a4..d983a25 100644 --- a/tests/AGENTS.md +++ b/tests/AGENTS.md @@ -48,9 +48,9 @@ and test-tier selection; test convenience never weakens a production invariant. tests -v` for full source-suite discovery (opt-in tests still skip). The runner is unittest, not pytest. Run the narrow module first, then the affected tier documented in `tests/README.md`. The release-confidence tier is explicitly opt-in with `FURIOUS_VERY_HEAVY_TESTS=1`; packaged/manual smoke work uses disposable environments. - The historical log comparison additionally requires `FURIOUS_LOG_MANAGER_BASELINE` naming a trusted, compatible - local Git revision; it loads that revision's implementation. Report this opt-in and any skips separately rather - than assuming the very-heavy switch alone executes every discovered case. + Discovered tests must be self-contained in the checkout: do not load or execute source from Git history. + The standalone log benchmark compares saved JSON timing reports; keep that optional measurement outside + unittest and report opt-ins, platform skips, and standalone measurements separately. - Source-only tests and an offscreen platform do not prove a packaged Qt runtime. Compiler-sensitive changes need native lifecycle tests and the compiled fixture documented in `tests/README.md`, including accept/reject/close and owner-first teardown. An unavailable private Nuitka counter is unknown, not measured zero; combine toolchain diff --git a/tests/README.md b/tests/README.md index cb09662..ddeab31 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,275 +1,361 @@ # Furious test suite -The suite uses Python's built-in `unittest` runner. In the main test process, -Qt tests use one deliberately small `QApplication` and route `QSettings` to a -unique temporary directory. Focused lifecycle, public-import, and display-scale -regressions may start a hermetic child Python process, and external-runtime tests use that child interpreter as -a harmless stand-in for a core. Tests do not initialize Furious's singleton IPC -server, production repositories, system proxy, TUN, routing, update network -clients, or real proxy cores. +The suite uses Python's built-in `unittest` runner. Run it from the repository +root with the project dependencies installed in an existing virtual environment. +Tests cover source behavior, real Qt interactions, and resource ownership; they +do not establish that a packaged build or every supported Python/Qt version works. -## Testing philosophy +## Setup and quick start -Choose the smallest layer that can prove the contract under test. Do not replace -Qt behavior with direct slot calls when focus, selection, model mapping, shortcut -scope, event delivery, or destruction is part of that contract. +Activate an existing root `.venv*` or `venv*` environment before using the commands +below. To reproduce the dependency selection in +[the source CI workflow](../.github/workflows/source-tests.yml), use: -1. **Pure logic tests** exercise models, repositories, codecs, controllers, and - services without constructing widgets. Inject only the runtime, network, or - host-operation boundary that would otherwise cause an external side effect. -2. **Qt integration tests** construct the real widget/model/proxy composition and - use `QTest` keyboard or mouse input plus the real event loop. Assertions target - semantic state such as profile identity, current selection, focus owner, - signal count, and wrapper destruction—not incidental row numbers or pixels. -3. **Small workflow tests** compose a few real UI surfaces around one shared - controller or repository. They keep Qt signals and widgets real while mocking - only unavailable platform privileges, network/process launch, plugin discovery, - or user-facing host dialogs. +```text +python -m pip install -r requirements.txt "PySide6-Essentials==6.8.3" "PySide6-Addons==6.8.3" +``` -`tests.support` is the canonical Qt harness: `application()` owns the one -process-wide test application, `isolatedSettings()` contains persistence, -`processQtEvents()` drains deferred deletion, and `waitFor()` handles bounded -event-loop convergence. Add a helper there only when multiple modules need the -same lifecycle primitive; do not create a second application or event-pumping -strategy in an individual test. +CI uses Python 3.13. Linux CI also installs `libegl1` and `libopengl0` for Qt. +Backend Python packages in `requirements.txt` are import dependencies; tests do +not require separately installed proxy-core executables or a working proxy. -## Coverage map +Select the offscreen platform **before starting Python**. Some test modules +import Qt-backed Furious modules before `tests.support` can set its defensive +default. Use a fresh Python process for each invocation and do not use `python -O`: +several child scripts and the standalone lifetime probe use plain `assert`. -| Area | Principal tests | -| --- | --- | -| Configuration, profiles, migration, repositories | `test_models_and_services.py`, `test_repository_contracts.py` | -| Generation log invariants, model fuzzing, concurrency, reclamation, complexity, and opt-in soak/latency probes | `test_log_manager_generation.py` | -| Low-level application, runtime, lease/thread handoff, editor, and storage contracts | `test_interface.py`, `test_runtime_lifecycle.py` | -| Application composition, asynchronous readiness/TUN startup, rollback, connection ownership, entry-point and crash boundaries | `test_architecture_refactors.py`, `test_connection_startup_async.py`, `test_application_process.py` | -| Plugin registration, capability dispatch, factories, rollback, and Hysteria1 ownership | `test_plugin_architecture.py`, `test_hysteria1_protocol.py` | -| Controller state and error transitions with injected runtimes | `test_controllers.py` | -| SOCKS and SIP002 Shadowsocks codecs and generated round trips | `test_socks_uri.py`, `test_shadowsocks_uri.py` | -| Subscription workflow, worker responsiveness, stale requests, reconciliation, and deterministic scale | `test_subscription_manager.py`, `test_subscription_sync.py`, `test_subscription_scalability.py` | -| Service-first profile-test identity, explicit results, endpoint deduplication, adaptive Tcping, reusable Stop All cancellation, shutdown admission, late callbacks, and worker/thread lifetime | `test_profile_test_jobs.py` | -| External process launch, output, shutdown, threads, TUN metadata | `test_external_core.py` | -| Backend structured-editor observational load and unknown-value preservation | `test_backend_editor_contract.py` | -| Xray asset checksum validation and atomic replacement | `test_xray_asset_download.py` | -| Xray/Hysteria2 native-TUN ownership and proxy-only stripping | `test_native_tun_semantics.py` | -| Rolling metrics, stable buckets, lazy rendering, and hover | `test_metrics_behavior.py` | -| Proxy-only endpoint discovery, caching, and presentation | `test_endpoint_info.py` | -| Bounded service work, update validation, plugin UI, worker lifetime, and reentrant HTTP completion/cancellation | `test_service_runtime.py` | -| Frozenlib state helpers and mocked platform-operation boundaries | `test_frozenlib.py` | -| Settings sandbox, navigation overlay behavior, public exports, and scale/theme isolation | `test_isolation_and_navigation.py`, `test_public_api.py`, `test_layout_matrix.py` | -| Shared Fluent visual states, zero table padding and intact rounded list corners, rounded menu/combo popup transparency and interaction, and native line-edit clear-button alignment and theme changes | `test_stylesheet_states.py` | -| Theme cross-fade activation, interruption, multi-window cleanup, and animation policy | `test_theme_transition.py` | -| AppQMainWindow lifecycle, subclass policies, geometry restoration, and migration | `test_main_window_geometry.py` | -| AppQDialog first-presentation geometry, native show paths, centering, and async lifetime | `test_dialog_geometry.py` | -| Editor mappings, lazy log rendering, routing-rule internal-move drag restrictions, Move menu/shortcuts and persisted order, message-box/connection UI | `test_ui_behavior.py` | -| Bounded, incremental, cancellable QR export and snapshot/lifetime safety | `test_qr_export_scalability.py` | -| Real keyboard/mouse/focus, proxy mapping, shared Home/Settings state, Home empty/filter recovery, shared menus and Tests-popup selection highlighting, direct small profile operations and batched imports/deletions with throttled progress and stable cancellation targets, and transient editor input | `test_qt_interactions.py` | -| Direct Qt ownership and destruction across independent UI families, message-box button detachment/reuse, and callback-driven owner destruction | `test_qt_lifetime.py` | -| Batched real/probe Qt object, QR rendering/window lifecycle, handle, Python allocation, and RSS trends | `test_qt_stress.py` | -| Repeated harmless subprocess, pipe, thread, handle, and RSS trends | `test_process_stress.py` | -| Opt-in release-confidence counts (100 app children, 100 external cores, 100k metrics, 40k logs, 20k navigation, 5k plugins, 1k dialogs, 1k real QR tabs) | `test_very_heavy.py` | - -The lifecycle tests classify `AppQTransientDialog`, protocol/plugin editors, -routing dialogs, subscription editors, message boxes, QR windows, and TUN -settings dialogs as transient. `TextEditorWindow` is intentionally reusable: it -must survive normal close/show cycles without multiplying actions, and is then -explicitly destroyed by its owner. Main pages/controllers are application -lifetime objects and are tested through isolated service/UI boundaries rather -than by starting the production application runtime. - -## Commands - -Run commands from the repository root with the project and its runtime -dependencies installed in the active environment. If the root contains a -`.venv*` or `venv*` environment, activate it first; the examples intentionally -use its `python` command rather than a machine-specific interpreter path. - -Select the offscreen Qt platform **before Python starts**. Some test modules -import Qt-backed Furious modules before `tests.support` can apply its defensive -default, so setting it only after test discovery begins is too late. - -Windows PowerShell: +PowerShell: ```powershell $env:QT_QPA_PLATFORM = 'offscreen' +$env:PYTHONUTF8 = '1' +python -m unittest discover -s tests -v ``` -Windows Command Prompt: +Command Prompt: ```cmd set QT_QPA_PLATFORM=offscreen +set PYTHONUTF8=1 +python -m unittest discover -s tests -v ``` -Linux, macOS, and other Unix-compatible shells: +POSIX shell: ```sh export QT_QPA_PLATFORM=offscreen -``` - -Then run the desired test tier. - -```text -# Complete suite, including repeated lifetime/process stress tests -python -m unittest discover -s tests -v - -# Regular logic, persistence, plugin, controller, codec, and UI regressions -python -m unittest tests.test_interface tests.test_models_and_services tests.test_repository_contracts tests.test_architecture_refactors tests.test_connection_startup_async tests.test_plugin_architecture tests.test_hysteria1_protocol tests.test_hysteria2_compatibility tests.test_controllers tests.test_subscription_manager tests.test_subscription_sync tests.test_subscription_scalability tests.test_profile_test_jobs tests.test_socks_uri tests.test_shadowsocks_uri tests.test_backend_editor_contract tests.test_xray_asset_download tests.test_native_tun_semantics tests.test_metrics_behavior tests.test_endpoint_info tests.test_service_runtime tests.test_frozenlib tests.test_isolation_and_navigation tests.test_main_window_geometry tests.test_dialog_geometry tests.test_ui_behavior tests.test_qr_export_scalability tests.test_qt_interactions tests.test_stylesheet_states tests.test_theme_transition tests.test_public_api -v - -# Direct Qt/process integration and destruction/lifetime checks -python -m unittest tests.test_application_process tests.test_external_core tests.test_layout_matrix tests.test_qt_lifetime -v - -# Explicit repeated stress tier -python -m unittest tests.test_qt_stress tests.test_process_stress -v - -# Explicit release-confidence tier (skipped unless opted in) -# PowerShell: $env:FURIOUS_VERY_HEAVY_TESTS = '1' -# POSIX shell: export FURIOUS_VERY_HEAVY_TESTS=1 -python -m unittest tests.test_very_heavy -v - -# Generation-specific release-confidence campaign (same opt-in switch) -python -m unittest tests.test_log_manager_generation.VeryHeavyGenerationLogManagerTest -v - -# Shared-state order-independence spot check -python -m unittest tests.test_public_api tests.test_theme_transition tests.test_stylesheet_states tests.test_qt_interactions tests.test_qr_export_scalability tests.test_ui_behavior tests.test_dialog_geometry tests.test_main_window_geometry tests.test_isolation_and_navigation tests.test_frozenlib tests.test_service_runtime tests.test_endpoint_info tests.test_metrics_behavior tests.test_native_tun_semantics tests.test_xray_asset_download tests.test_backend_editor_contract tests.test_shadowsocks_uri tests.test_socks_uri tests.test_profile_test_jobs tests.test_subscription_sync tests.test_subscription_manager tests.test_controllers tests.test_hysteria2_compatibility tests.test_hysteria1_protocol tests.test_plugin_architecture tests.test_connection_startup_async tests.test_architecture_refactors tests.test_repository_contracts tests.test_models_and_services tests.test_interface -v +export PYTHONUTF8=1 python -m unittest discover -s tests -v ``` -To run one module, class, or method while developing, pass its dotted test name -to the same runner, for example: +Default discovery includes the regular Qt/process stress tests. Very-heavy cases +are discovered but skipped unless explicitly enabled. An inherited opt-in +variable also affects focused module runs; clear it when returning to the default +tier. Standalone benchmark and fixture entrypoints are not run by discovery. + +For a focused module, class, or method: ```text python -m unittest tests.test_endpoint_info -v python -m unittest tests.test_endpoint_info.EndpointInfoServiceTest -v +python -m unittest tests.test_controllers.RoutingControllerTest.testDisabledCustomRoutingStaysOnFallbackAfterReenable -v ``` -No external network access or separately installed Xray/Hysteria executable is -required. Process-boundary tests create only temporary scripts and launch the -active Python interpreter. Two regressions cover Windows-specific behavior and -are skipped on other platforms; native handle/RSS trend checks use the safe -platform counters available on the current host. +## Coverage map -Several negative-path tests intentionally emit warning or error log messages. -The opt-in release-confidence tier uses high operation counts but samples native -resources only at batch boundaries, so its assertions remain ownership-oriented. -Neither stress tier is enabled by a normal focused module run. Treat the runner's -final status and exit code as authoritative; expected diagnostic output still ends in -`OK`. +The tables list every `test_*.py` module. Areas overlap: a service module may also +contain a small UI workflow, and a Qt test may exercise persistence or a real +worker. Choose tests by the changed contract rather than by filename alone. -`tests/benchmarks/benchmark_subscription_updates.py` is a development-only 1/3/8-group benchmark. It generates 1,500 profiles -per group by default and reports decode/parse CPU time, reconciliation preparation, worker wall time, and GUI-thread -commit time. Pass `--url` explicitly to benchmark a live subscription; normal tests never use the network. +### Models, persistence, plugins, and protocols -`tests/benchmarks/benchmark_qr_export.py` measures the real Segno-to-`QImage` renderer and the complete synchronous or -incremental QR-window pipeline. The 5,000-item UI workloads are intentionally benchmarks rather than correctness tests; -run each mode in a fresh process so allocator and Qt-widget state do not affect the next measurement: +| Module | Coverage | +| --- | --- | +| [test_interface.py](test_interface.py) | Stable exit/runtime contracts, configuration serialization and diagnostics, editor binding, live storage ownership. | +| [test_models_and_services.py](test_models_and_services.py) | Profile identity/copies/fingerprints, legacy metadata, repository round trips, settings migration, bounded logs and metrics, translation extraction. | +| [test_repository_contracts.py](test_repository_contracts.py) | Routing/TUN/subscription persistence, stable filtered moves, remote ownership detachment, atomic hydration, malformed-storage preservation, private-data diagnostics. | +| [test_plugin_architecture.py](test_plugin_architecture.py) | Registry creation, API-v3 capabilities and prepared runtimes, atomic registration/rollback, failure isolation, shutdown, transient editor ownership. | +| [test_public_api.py](test_public_api.py) | Curated package exports, compatibility imports, and lazy bundled backend discovery in isolated child interpreters. | +| [test_hysteria1_protocol.py](test_hysteria1_protocol.py) | URI/mapping compatibility, plugin/editor discovery, independent download-test configuration. | +| [test_hysteria2_compatibility.py](test_hysteria2_compatibility.py) | Compact editor mappings and defaults, unknown/JSON-only fields, ECH/Gecko/Realm/Mimic URI and configuration handling, full runtime JSON, editor destruction. | +| [test_backend_editor_contract.py](test_backend_editor_contract.py) | Observational editor loading, unsupported Hysteria1/Xray values, Xray transport alias normalization. | +| [test_shadowsocks_uri.py](test_shadowsocks_uri.py) | SIP002 encoding, IPv6/Unicode/plugin fields, deterministic generated round trips, malformed-input rejection. | +| [test_socks_uri.py](test_socks_uri.py) | Canonical and legacy/Base64 SOCKS forms, strict decoding, IPv6, aliases, Xray/plugin/subscription integration. | + +### Application, controllers, and services + +| Module | Coverage | +| --- | --- | +| [test_application_process.py](test_application_process.py) | Exact application-child ownership, shared crash flag, exception/signal handling, temporary crash logs, command-line dispatch. | +| [test_architecture_refactors.py](test_architecture_refactors.py) | Startup acquisition/rollback, singleton election and real isolated IPC race, tray/exit policy, host integration ownership, bounded core-log transport, connection transactions, stylesheet composition. | +| [test_connection_startup_async.py](test_connection_startup_async.py) | Real local-listener readiness, timeout/cancel/replacement, semantic exits, DNS reply lifetime, mocked platform-specific TUN sequencing. | +| [test_controllers.py](test_controllers.py) | Connection state/error/reconnect transitions, startup restoration, shared settings, routing fallback persistence and tray/selector agreement after custom-routing disable/re-enable. | +| [test_runtime_lifecycle.py](test_runtime_lifecycle.py) | Qt-thread exit dispatch, commit/exit races, duplicate and late exits, idempotent release, spawn failure, queue/timer disposal on preparation failure. | +| [test_external_core.py](test_external_core.py) | Harmless real process launch/output/shutdown, partial thread-start rollback, readiness/TUN metadata, Windows paths with spaces, subscription rejection of executable profiles, bounded DNS references. | +| [test_frozenlib.py](test_frozenlib.py) | Nested state guards, cleanup isolation, bounded caches/throttling, dual-stack probe selection, mocked proxy/DNS/routes/startup/session boundaries and failure handling. | +| [test_native_tun_semantics.py](test_native_tun_semantics.py) | Xray/Hysteria2 runtime-copy TUN preservation/replacement, managed-TUN failures, download-test stripping, prevention of a second tun2socks owner. | +| [test_subscription_sync.py](test_subscription_sync.py) | Group-local preparation/commit, stable duplicate identity, atomic failure, preservation of newer local metadata, rejection of changed source state. | +| [test_subscription_manager.py](test_subscription_manager.py) | Stable request generations, provider metadata, batch/partial failures, timer policy, targeted updates, real Stop Updates input, worker responsiveness, cancellation and synchronous shutdown ownership. | +| [test_subscription_scalability.py](test_subscription_scalability.py) | Deterministic 1/3/8-group preparation and commit with 1,500 profiles per group and bounded workers; uses the offline benchmark helper. | +| [test_profile_test_jobs.py](test_profile_test_jobs.py) | Stable profile/fingerprint jobs, stale results, endpoint deduplication, bounded fan-out, adaptive Tcping, download scheduling, reusable Stop All, real pool/thread teardown and retry, narrow cell repaint. | +| [test_service_runtime.py](test_service_runtime.py) | Update-response validation, HTTP timeouts/context release/reentrant destruction, duplicate completion, plugin-page registration, bounded connectivity requests, blocked statistics-worker callback lifetime. | +| [test_xray_asset_download.py](test_xray_asset_download.py) | Checksum validation, atomic asset replacement, real pool delivery, early owner destruction, callback release, plugin shutdown. | +| [test_endpoint_info.py](test_endpoint_info.py) | Opt-in proxy-only discovery using fake HTTP responses, fallback/cache/session invalidation, privacy/presentation controls, local map styling and persistent-scene contracts. | +| [test_metrics_behavior.py](test_metrics_behavior.py) | Immutable time-series history, pruning/buckets, usage versus speed clearing, hidden-page rendering, graph hover lookup. | +| [test_log_manager_generation.py](test_log_manager_generation.py) | Reference-model state transitions, concurrency, live/retired ownership, bounded FIFO cleanup, structural complexity, incremental/multiline LogPage rendering; also contains an opt-in generation campaign. | +| [test_log_manager_benchmark.py](test_log_manager_benchmark.py) | Data-only report validation, workload compatibility, timing ratios, and real standalone CLI JSON round trips; no performance gates or historical imports. | + +### Qt behavior, layout, and lifetime + +| Module | Coverage | +| --- | --- | +| [test_isolation_and_navigation.py](test_isolation_and_navigation.py) | Nested/canonical QSettings sandbox checks, outside-path rejection, navigation overlay geometry, dismissal, page/object reuse. | +| [test_main_window_geometry.py](test_main_window_geometry.py) | First-show lifecycle, restored/default/legacy geometry, main/routing window reuse, session-only navigation state, protection of never-shown windows' saved geometry. | +| [test_dialog_geometry.py](test_dialog_geometry.py) | Dialog show/open/exec preparation, sizing/centering, failure cleanup, specialized message-box geometry and transient destruction. | +| [test_layout_matrix.py](test_layout_matrix.py) | Navigation and message-box layouts in fresh processes at scale factors 1, 1.25, 1.5, and 2, under both themes. | +| [test_ui_behavior.py](test_ui_behavior.py) | Translation and editor mappings, settings organization, stable server/routing moves, real QR rendering/decoding, incremental log filtering/tail behavior, connected-routing change notices, message boxes, shared Home/tray state. | +| [test_qt_interactions.py](test_qt_interactions.py) | Real keyboard/mouse/focus and proxy mapping, scoped shortcuts, sorting/selection, stable subscription deletion confirmations, debounced Home search, Tests-menu selection color, shared settings, batched profile mutation and cancellation. | +| [test_qr_export_scalability.py](test_qr_export_scalability.py) | Production capture cap, immediate single export, incremental yielding, failure/cancel/close paths, immutable snapshots, window-owned state destruction. | +| [test_stylesheet_states.py](test_stylesheet_states.py) | Targeted rendering/alpha/geometry assertions for table/list insets, popup corners, clear buttons, focus/disabled states, and stylesheet composition. | +| [test_theme_transition.py](test_theme_transition.py) | Real cross-fades, immediate theme activation, interruption, per-window resize/destruction, coordinator teardown, animation policy. | +| [test_qt_lifetime.py](test_qt_lifetime.py) | Native destruction and weak-wrapper/registry evidence across dialogs, menus, actions, timers, signals, message-box buttons/masks, owner-first confirmations, reusable editors, simulated compiled-method retention. | + +### Repeated stress and release confidence + +| Module | Coverage | +| --- | --- | +| [test_qt_stress.py](test_qt_stress.py) | Warmed-up dialog batches, real editor families, native/wrapper/registry counts, Python/RSS trends, repeated real QR exports at the production cap. | +| [test_process_stress.py](test_process_stress.py) | Repeated real Python-backed external-core start/stop, exact child reaping, pipes/readers/watchers, thread/handle checks and RSS reporting. | +| [test_very_heavy.py](test_very_heavy.py) | Explicitly enabled high-count plugin/application-child/external-core lifecycles, metrics/logs/navigation, transient dialogs/message boxes, and 1,000 real QR tabs. | + +## Selecting tiers + +Full discovery is the most reliable way to include all source modules. For +focused work, these smaller groups collectively cover the suite; they are +convenience commands, not separate runners or markers. The generation module's +regular cases run by default while its heavy class remains gated. ```text -python tests/benchmarks/benchmark_qr_export.py --mode images --count 5000 -python tests/benchmarks/benchmark_qr_export.py --mode synchronous --count 5000 -python tests/benchmarks/benchmark_qr_export.py --mode asynchronous --count 5000 +# Models, storage, plugin API, protocol/editor compatibility +python -m unittest tests.test_interface tests.test_models_and_services tests.test_repository_contracts tests.test_plugin_architecture tests.test_public_api tests.test_hysteria1_protocol tests.test_hysteria2_compatibility tests.test_backend_editor_contract tests.test_shadowsocks_uri tests.test_socks_uri -v + +# Application, connection, runtime, and mocked host boundaries +python -m unittest tests.test_application_process tests.test_architecture_refactors tests.test_connection_startup_async tests.test_controllers tests.test_runtime_lifecycle tests.test_external_core tests.test_frozenlib tests.test_native_tun_semantics -v + +# Subscription, profile tests, HTTP/assets, endpoint information, metrics, and logs +python -m unittest tests.test_subscription_sync tests.test_subscription_manager tests.test_subscription_scalability tests.test_profile_test_jobs tests.test_service_runtime tests.test_xray_asset_download tests.test_endpoint_info tests.test_metrics_behavior tests.test_log_manager_generation tests.test_log_manager_benchmark -v + +# Real Qt input, presentation, rendering, geometry, and lifetime +python -m unittest tests.test_isolation_and_navigation tests.test_main_window_geometry tests.test_dialog_geometry tests.test_layout_matrix tests.test_ui_behavior tests.test_qt_interactions tests.test_qr_export_scalability tests.test_stylesheet_states tests.test_theme_transition tests.test_qt_lifetime -v + +# Regular repeated stress: no opt-in required; also runs in default discovery +python -m unittest tests.test_qt_stress tests.test_process_stress -v + +# Release-confidence classes: set the opt-in below before starting Python +python -m unittest tests.test_very_heavy tests.test_log_manager_generation.VeryHeavyGenerationLogManagerTest -v ``` -The regular Qt stress tier renders repeated real QR batches at the production cap. The opt-in release-confidence tier -temporarily raises the cap to 1,000 and verifies real generation, event-loop yielding, result-window presentation, -exact tab completion, and window-owned timer/state destruction. These tests intentionally assert behavior and cleanup -rather than machine-dependent elapsed-time thresholds. +To check suspected order dependence, pass the affected module names in the +opposite order in one invocation, then run default discovery in a fresh process. +Passing modules in separate processes cannot expose leaked shared Qt/repository +state between those modules. -Subscription shutdown coverage in `test_subscription_manager.py` checks cooperative cancellation, queued-work -removal, admission closure, and late-result rejection. A gated worker verifies that the slow-shutdown diagnostic -preserves synchronous ownership until work actually finishes; the warning threshold is not an exit deadline. +### Very-heavy opt-in -## Packaged-build smoke procedure +Set `FURIOUS_VERY_HEAVY_TESTS` to exactly `1` **before discovery/import**. It enables +both `VeryHeavyContractTest` and `VeryHeavyGenerationLogManagerTest`. -Packaged/Nuitka behavior is outside the source-level `unittest` fixtures. For an -optional release smoke check, use an otherwise disposable test OS account or -VM, redirect all Furious application-data/settings locations to a temporary -directory, and keep system proxy and TUN disabled. Open and close each transient -editor family 50 times, verify one reusable `TextEditorWindow` does not -duplicate actions, and compare live-object diagnostics from an instrumented -build before/after the loop. Do not run this procedure against a production -profile or rely on process-name cleanup; close only the exact packaged process -started for the smoke test. +PowerShell: -## Isolation rules +```powershell +$env:FURIOUS_VERY_HEAVY_TESTS = '1' +python -m unittest discover -s tests -v +Remove-Item Env:FURIOUS_VERY_HEAVY_TESTS +``` -- Tests clean up only exact subprocess handles/PIDs and threads they create. -- Child-process lifecycle tests disable single-instance discovery and host - integration before running a real Qt event loop. -- Tests never search for, signal, or terminate another Furious/core process. -- Persistence tests use temporary INI-backed `QSettings` namespaces. -- Controller tests inject fake runtime managers and patch host-mutation APIs. -- Qt tests use normal close/deferred-delete paths and collect Python cycles only - at diagnostic batch boundaries, never once per UI operation. -- A lifetime failure must be investigated as an ownership defect; increasing - thresholds or forcing production garbage collection is not an acceptable fix. +POSIX shell: -### Focused lifetime audit regressions +```sh +FURIOUS_VERY_HEAVY_TESTS=1 python -m unittest discover -s tests -v +``` -`test_qt_lifetime.py` checks that independent signal endpoints do not accumulate -cleanup hooks when senders die first and releases message-box masks on native deletion. It also -checks reusable-dialog reopen generations, owner-first routing confirmation teardown, and static -routing documentation callbacks under simulated compiled-method protection. -`test_profile_test_jobs.py` gates a real pool worker to verify that a missed Ping shutdown deadline -still reaps the independent TCPing thread, cancels downloads, and permits a later shutdown retry. -`test_theme_transition.py` covers both target-window and coordinator destruction during a fade. `test_connection_startup_async.py` covers DNS -cancellation/timeout with already-deleted recursive replies. `test_service_runtime.py` -rejects deleted plugin-page wrappers. `test_xray_asset_download.py` exercises real -pool-thread delivery, early manager destruction, callback release, and plugin shutdown -of pending replies and hashes. +Command Prompt users can set `FURIOUS_VERY_HEAVY_TESTS=1` with `set`, and clear it +with `set FURIOUS_VERY_HEAVY_TESTS=` afterward. -Run `python -m tests.fixtures.editor_lifetime_probe --iterations 100 --pattern representative --close-method close` -natively and compile that fixture with Nuitka's PySide6 plugin for a separate standalone -check. It checks reopen generations, routing confirmation owner destruction, both independent -signal endpoint destruction orders, and seven transient editor families. Repeat with `--close-method accept` and `--close-method reject`. -A null protected-list count means Nuitka does not expose that diagnostic; inspect its -installed package configuration and require zero live wrappers and registry entries -instead. +All discovered tests use the checked-out implementation. No Git history or +historical source execution is required. The optional log timing comparison is +a standalone benchmark using saved JSON reports, described below. -The same standalone fixture also checks both HTTP clients across normal completion, early reply deletion, and -manager-first deletion, retaining invalid wrappers deliberately to prove request context is released. It also -checks action-owned versus explicitly widget-owned menus and representative asset delete/overwrite confirmations -whose views are destroyed while their containing widget remains valid. Keep this harness self-contained so its -compiled import graph does not pull in the full unittest modules. -`test_qt_lifetime.py` covers 20 cycles for each of five confirmation workflows under each mocked Windows/Linux/macOS -branch, plus 30 action/menu cycles for each ownership model. `test_service_runtime.py` additionally rejects duplicate -HTTP completion and checks 30 early-deletion cycles per client without per-cycle garbage collection. +The generation campaign mixes exact structural/backlog assertions with relative +scaling gates and reported latency distributions. Its long soak reports RSS +samples; that report is not itself an RSS-plateau assertion. Record interpreter, +Qt, OS, workload, and opt-ins when interpreting timings. Benchmarks and packaged +smoke checks remain separate when the very-heavy tier is enabled. -### Evolution regressions and CI +## Isolation and interpreting results -`test_repository_contracts.py` verifies all-or-nothing hydration and preservation of -original stored bytes after malformed records or plugin parsing failures. -`test_runtime_lifecycle.py` verifies real Qt timer destruction and queue disposal when -launch preparation fails before a service acquires the runtime. -`test_subscription_manager.py` exercises Stop Updates with a real button click, -partial batch success, stale-result rejection, unchanged automatic schedules, and restart. +[tests/support.py](support.py) supplies the shared harness: -The reusable `source-tests.yml` workflow runs full unittest discovery on Windows, -Linux, and macOS with Python 3.13 and PySide6 6.8.3. The publication workflow requires -that job before PyPI publication. Opt-in tests and packaged/manual smoke checks remain -separate; local Windows results do not establish the other CI targets. +- `application()` creates one small test `QApplication`; it does not compose the + normal application runtime, singleton, tray, or host integration. +- `isolatedSettings()` uses unique temporary INI identities and restores the + caller's identity. `assertIsolatedSettings()` compares canonical paths, including + macOS temporary-directory aliases and Windows short names. In-memory `Storage` + collections still need their own fixture isolation/restoration. +- `processQtEvents()` drains regular/deferred-delete events; `waitFor()` pumps + until a bounded deadline and returns a boolean that callers must check. + `collectAtBoundary()` is a diagnostic collection boundary, not a production + cleanup technique or a replacement for native destruction evidence. +- `runPythonChild()` captures one exact interpreter child's output with a timeout. + `childEnvironment()` copies the caller environment and forces offscreen Qt; + the child fixture must still disable or replace host effects and production + persistence before exercising application paths. +- `currentRSS()` and `currentNativeHandleCount()` use Windows APIs or Linux + `/proc` counters. They currently return `None` on macOS; unavailable counters + mean unmeasured, not zero. Object, registry, thread, and exit assertions remain + useful independently of those counters. -The settings harness compares canonical paths on every access so macOS temporary-directory -aliases and Windows short names do not look like sandbox escapes. Its isolation regressions -cover an aliased root and rejection of paths outside that root. The QR responsiveness test -queues unrelated work after generation starts and requires delivery before completion; -it does not assume a platform-specific order between zero-delay Qt timers. +Normal tests do not need external network access and must not change real proxy, +TUN, DNS, routing, startup registration, production settings, or unrelated +processes. HTTP/provider responses and privileged host boundaries are faked or +mocked. Some tests deliberately use **real isolated local IPC, loopback listeners, +threads, and child processes**: the singleton race has a unique socket name, and +readiness tests own their local listener. Cleanup targets only resources created +by that test. Mocked platform branches prove call/ownership contracts, not actual +OS integration. -On the CI Qt 6.8.3 runtime, the shared offscreen application uses -`fixtures/offscreen.json` to provide a 1920×1080 virtual desktop. Geometry fixtures -account for translated controls' Qt minimum sizes instead of assuming one host's -font metrics. Menu tests explicitly reactivate the owning test window after popup -dismissal: macOS offscreen does not supply the desktop's reactivation event. They -still assert the retained child focus and focus-scoped shortcuts. Selection-color -checks sample an empty cell in logical coordinates, avoiding text glyphs and -high-DPI image-coordinate assumptions. Log catch-up checks wait for the separate -queued scroll update as well as document rendering. +Use real Qt input/event delivery for focus, selection, shortcuts, proxy mapping, +and destruction. Targeted pixel/alpha assertions are appropriate when rendering +is the contract; arbitrary whole-window snapshots or fixed timing guesses are +not substitutes for state and ownership assertions. -The singleton race uses a short unique socket name that fits under macOS's long -temporary paths and retains child stderr in failure diagnostics. The Windows -executable-path test copies a Python interpreter and its DLLs into a disposable -path with spaces, so it also works when Python and the checkout are on different -volumes; it waits for child output before asserting shutdown. +On CI's Qt 6.8.3 runtime, [fixtures/offscreen.json](fixtures/offscreen.json) gives +the shared application a 1920×1080 virtual desktop. Geometry tests account for +translated controls' minimum sizes. Popup tests explicitly restore window +activation where offscreen macOS does not, selection-color checks avoid glyphs +and respect image scaling, and log catch-up waits for queued scrolling as well +as document updates. QR yielding checks do not rely on the relative ordering of +zero-delay timers. -The Home/Log workflow regressions exercise debounced typing, explicit submit/clear, navigation catch-up, focus-scoped -Find/edit shortcuts, and live log filtering. These are real Qt tests in `test_qt_interactions.py` and -`test_ui_behavior.py`; no production log source or profile network is started. The two optional pause regressions -(selection across eviction/clear and frozen filtered export/retranslation) are retained as comments alongside -LogPage's disabled Pause/Resume Updates implementation; re-enable them together when restoring that feature. +Two cases are Windows-only: the native-session finalization regression in +`test_architecture_refactors.py` and executable-path-with-spaces coverage in +`test_external_core.py`. Other platforms report those as skips. The latter uses +a temporary copy of the base interpreter and its DLLs, including across volumes. + +Expected negative-path warnings/errors can accompany a passing run. Capture +expected errors with `assertLogs` when their text is part of the contract; the +malformed TUN-settings test does this to prevent an expected traceback becoming +a misleading GitHub problem-matcher annotation. Check the runner's final result, +exit code, and skip reasons. An exception inside a Qt-dispatched callback can +reach `sys.excepthook` without failing unittest, so reentrant/destruction tests +must capture and assert callback errors as well. + +Cancellation, logical completion, and resource release are different assertions. +For example, subscription Stop Updates rejects late work but running preparation +remains owned until it returns. Its slow-shutdown warning is not an exit deadline. +The profile-test shutdown regression checks independent Tcping/download cleanup +and a later retry while a Ping worker is gated; it does not prove arbitrary +third-party work can always be forcibly stopped. + +The two paused-log examples in `test_ui_behavior.py` are commented out alongside +the disabled Pause/Resume Updates feature. They are neither discovered tests nor +reported skips. Re-enable them together with the feature. + +## Standalone benchmarks and fixtures + +[benchmarks/benchmark_log_manager.py](benchmarks/benchmark_log_manager.py) measures +steady append, core rollover, category/global snapshots, and retention-heavy +append using the current checkout. Save a baseline before a change, then compare +with another report using the same workload and machine/toolchain: + +```text +python -m tests.benchmarks.benchmark_log_manager --output baseline.json +python -m tests.benchmarks.benchmark_log_manager --baseline baseline.json --output current.json +``` + +`--entries` defaults to 50,000 and must match the saved baseline. Reports include +the workload schema, entry count, Python/Qt/platform metadata, and timings. Ratios +are current/baseline (below 1 means faster); they are informational, not test +thresholds. Only compatible JSON data is read: the benchmark never looks up Git +revisions or imports historical source. Run measurements in fresh processes and +keep both reports outside tracked source. Timing noise and environment changes +still require interpretation; older ad-hoc comparison output is not this report +format. The benchmark has no unittest opt-in variable. + +[benchmarks/benchmark_subscription_updates.py](benchmarks/benchmark_subscription_updates.py) +generates offline data by default and measures decode/parse, reconciliation +preparation, worker wall time, and sequential commit time. The `decode_cpu_s` +and `reconcile_cpu_s` fields sum elapsed worker durations measured with +`perf_counter`; they are not process CPU-time measurements. `gui_commit_s` +measures caller-thread commit work, not a rendered GUI workflow. + +```text +python -m tests.benchmarks.benchmark_subscription_updates --profiles 1500 --groups 1,3,8 +``` + +Its optional `--url` downloads a live subscription and is an explicit networked +benchmark mode, not part of normal tests. The discovered scalability test calls +the offline helper and checks counts/worker bounds without elapsed-time gates. + +[benchmarks/benchmark_qr_export.py](benchmarks/benchmark_qr_export.py) measures +real Segno-to-`QImage` rendering and synchronous/incremental QR windows. Run each +mode in a fresh process to avoid sharing allocator/widget state: + +```text +python -m tests.benchmarks.benchmark_qr_export --mode images --count 5000 +python -m tests.benchmarks.benchmark_qr_export --mode synchronous --count 5000 +python -m tests.benchmarks.benchmark_qr_export --mode asynchronous --count 5000 --timeout 900 +``` + +These large UI workloads are benchmarks, distinct from correctness tests. Regular +QR stress uses the production cap; the very-heavy QR case temporarily raises it +to 1,000 and verifies yielding, visible presentation, completion, and destruction. + +[fixtures/editor_lifetime_probe.py](fixtures/editor_lifetime_probe.py) is a +self-contained source/compiled lifecycle probe using the shared harness, without +importing the unittest modules. Run the representative editor sequence through +each completion path: + +```text +python -m tests.fixtures.editor_lifetime_probe --iterations 100 --pattern representative --close-method close +python -m tests.fixtures.editor_lifetime_probe --iterations 100 --pattern representative --close-method accept +python -m tests.fixtures.editor_lifetime_probe --iterations 100 --pattern representative --close-method reject +``` + +Patterns also include `alternating`, `reverse`, `hysteria2`, and `vless`. The +representative pattern cycles through Hysteria2, VLESS, VMess, Trojan, SOCKS, +Hysteria1, and External Core. Each invocation additionally probes HTTP completion +and owner/reply-first destruction, button ownership, signal endpoints, masks, +reopen generations, animations, menus, and view-owned confirmations. It records +JSON diagnostics and asserts captured Qt callback exceptions are absent. + +For compiler-sensitive work, compile this fixture separately with Nuitka's +PySide6 plugin, its imported support code, and required data, then repeat the +close/accept/reject checks. Record the Python/PySide6/Nuitka versions, target, and +build flags. A missing private protected-method counter is unknown, not zero; +combine available diagnostics with native destruction, weak-wrapper, and registry +evidence. A diagnostic build passing does not establish that an ordinary release +build has the same behavior. + +## CI and packaged validation + +[Source regression CI](../.github/workflows/source-tests.yml) runs default +discovery on `ubuntu-22.04`, `windows-2025`, and `macos-14`, with Python 3.13, +PySide6 Essentials/Addons 6.8.3, offscreen Qt, and a 30-minute job timeout. +[The publication workflow](../.github/workflows/deploy-pypi.yml) requires this job +before PyPI publication. CI does not enable the very-heavy opt-in, +run standalone benchmarks, or compile the lifetime fixture. This matrix does +not validate every Python/Qt floor declared by the package. + +For a packaged smoke check, use a disposable OS account or VM, redirect settings +and application data to temporary locations, and keep system proxy/TUN disabled. +Exercise the ordinary release artifact as well as any diagnostic build. Open and +close representative transient editors repeatedly, verify reusable +`TextEditorWindow` close/show cycles do not duplicate actions, and inspect +ownership/destruction evidence. Close only the exact process started for the +check. Source-suite success and release import checks do not replace packaged +behavior or real desktop integration testing. + +When reporting verification, separate source execution, mocked platform evidence, +opt-in stress/benchmark output, and packaged/manual results. Include skipped or +unavailable cases and targets rather than presenting discovery alone as execution. diff --git a/tests/benchmarks/benchmark_log_manager.py b/tests/benchmarks/benchmark_log_manager.py new file mode 100644 index 0000000..3a66705 --- /dev/null +++ b/tests/benchmarks/benchmark_log_manager.py @@ -0,0 +1,214 @@ +# Copyright (C) 2024–present Loren Eteval & contributors +# +# This file is part of Furious. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Measure the checked-out log manager and compare saved JSON timing reports.""" + +from __future__ import annotations + +from Furious.Service.LogManager import ( + APPLICATION_LOG_CATEGORY, + CORE_LOG_CATEGORY, + TUN2SOCKS_LOG_CATEGORY, + LogManager, +) + +from PySide6 import QtCore + +from tests.support import application, processQtEvents + +from contextlib import contextmanager +from pathlib import Path + +import argparse +import json +import math +import platform +import time + +MEASUREMENTS = ( + 'steady_append_ms', + 'core_rollover_us', + 'category_snapshot_us', + 'global_snapshot_us', + 'retention_heavy_ms', +) + + +def validateReport(report): + """Reject incomparable workloads and invalid timing values before division.""" + if ( + not isinstance(report, dict) + or type(report.get('schema')) is not int + or report['schema'] != 1 + ): + raise ValueError('expected a log-manager benchmark report with schema 1') + + entries = report.get('entries') + + if type(entries) is not int or entries < 10: + raise ValueError('report entries must be an integer of at least 10') + + measurements = report.get('measurements') + + if not isinstance(measurements, dict) or set(measurements) != set(MEASUREMENTS): + raise ValueError('report measurement names do not match this workload') + + for name, value in measurements.items(): + if type(value) not in (int, float) or value <= 0 or not math.isfinite(value): + raise ValueError(f'{name} must be a positive finite timing') + + return report + + +def compareReports(current, baseline): + """Return informational current/baseline ratios for identical workloads.""" + validateReport(current) + validateReport(baseline) + + if current['entries'] != baseline['entries']: + raise ValueError('baseline entries must match --entries') + + return { + name: current['measurements'][name] / baseline['measurements'][name] + for name in MEASUREMENTS + } + + +@contextmanager +def _manager(**options): + """Release the exact benchmark QObject and queued cleanup between workloads.""" + manager = LogManager(**options) + + try: + yield manager + finally: + manager.deleteLater() + processQtEvents() + + +def run(entries=50_000): + """Measure five fixed workloads using only the current implementation.""" + if type(entries) is not int or entries < 10: + raise ValueError('entries must be an integer of at least 10') + + application() + measurements = {} + + with _manager(maximumEntries=entries, autoClearEnabled=False) as manager: + started = time.perf_counter_ns() + + for index in range(entries): + manager.append(str(index), APPLICATION_LOG_CATEGORY) + + measurements['steady_append_ms'] = (time.perf_counter_ns() - started) / 1e6 + + threshold = entries * 2 // 5 + + with _manager(maximumEntries=entries, autoClearMaximumEntries=threshold) as manager: + for index in range(threshold): + manager.append(str(index), CORE_LOG_CATEGORY) + for index in range(threshold): + manager.append(str(index), TUN2SOCKS_LOG_CATEGORY) + + started = time.perf_counter_ns() + manager.append('trigger', CORE_LOG_CATEGORY) + measurements['core_rollover_us'] = (time.perf_counter_ns() - started) / 1e3 + + with _manager(maximumEntries=entries, autoClearEnabled=False) as manager: + for index in range(entries * 3 // 5): + manager.append( + str(index), + CORE_LOG_CATEGORY if index % 5 == 0 else APPLICATION_LOG_CATEGORY, + ) + + started = time.perf_counter_ns() + manager.snapshot(CORE_LOG_CATEGORY) + measurements['category_snapshot_us'] = (time.perf_counter_ns() - started) / 1e3 + + started = time.perf_counter_ns() + manager.snapshot() + measurements['global_snapshot_us'] = (time.perf_counter_ns() - started) / 1e3 + + with _manager( + maximumEntries=max(1, entries // 50), autoClearEnabled=False + ) as manager: + started = time.perf_counter_ns() + + for index in range(entries): + manager.append(str(index), APPLICATION_LOG_CATEGORY) + + measurements['retention_heavy_ms'] = (time.perf_counter_ns() - started) / 1e6 + + return validateReport( + { + 'schema': 1, + 'entries': entries, + 'environment': { + 'python': platform.python_version(), + 'platform': platform.platform(), + 'qt': QtCore.qVersion(), + }, + 'measurements': measurements, + } + ) + + +def main(argv=None): + """Write a current report, optionally comparing a data-only baseline file.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--entries', type=int, default=50_000) + parser.add_argument('--output', type=Path) + parser.add_argument('--baseline', type=Path) + arguments = parser.parse_args(argv) + + if arguments.entries < 10: + parser.error('--entries must be at least 10') + + baseline = None + + if arguments.baseline is not None: + try: + baseline = validateReport( + json.loads(arguments.baseline.read_text(encoding='utf-8')) + ) + if baseline['entries'] != arguments.entries: + raise ValueError('baseline entries must match --entries') + except (OSError, ValueError) as error: + parser.error(str(error)) + + report = run(arguments.entries) + + if baseline is not None: + report['comparison'] = { + 'baseline': { + key: baseline[key] + for key in ('schema', 'entries', 'environment', 'measurements') + if key in baseline + }, + 'ratios': compareReports(report, baseline), + } + + serialized = json.dumps(report, indent=2, sort_keys=True, allow_nan=False) + + if arguments.output is not None: + arguments.output.write_text(serialized + '\n', encoding='utf-8') + + print(serialized) + + +if __name__ == '__main__': + main() diff --git a/tests/test_log_manager_benchmark.py b/tests/test_log_manager_benchmark.py new file mode 100644 index 0000000..83bb661 --- /dev/null +++ b/tests/test_log_manager_benchmark.py @@ -0,0 +1,159 @@ +# Copyright (C) 2024–present Loren Eteval & contributors +# +# This file is part of Furious. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Protect data-only log benchmark reports and standalone CLI comparison.""" + +from __future__ import annotations + +from tests.benchmarks import benchmark_log_manager as benchmark +from tests.support import assertChildSucceeded, runPythonChild + +from pathlib import Path +from unittest import mock + +import copy +import io +import json +import tempfile +import unittest + + +class LogManagerBenchmarkTest(unittest.TestCase): + """Keep saved measurements comparable without importing historical code.""" + + @staticmethod + def report(): + return { + 'schema': 1, + 'entries': 50, + 'measurements': { + name: float(index + 1) + for index, name in enumerate(benchmark.MEASUREMENTS) + }, + } + + def testComparisonUsesMeasuredRatiosWithoutChangingReports(self): + baseline = self.report() + current = copy.deepcopy(baseline) + current['measurements'] = { + name: value * 2 for name, value in baseline['measurements'].items() + } + before = copy.deepcopy((current, baseline)) + + self.assertEqual( + benchmark.compareReports(current, baseline), + {name: 2.0 for name in benchmark.MEASUREMENTS}, + ) + self.assertEqual((current, baseline), before) + + def testInvalidOrIncompatibleReportsAreRejected(self): + for value in (None, [], {}, {'schema': True}, {'schema': 2}): + with self.subTest(report=value), self.assertRaises(ValueError): + benchmark.validateReport(value) + + for field, value in ( + ('entries', True), + ('entries', 9), + ('entries', 50.0), + ('measurements', {}), + ('measurements', []), + ): + report = self.report() + report[field] = value + + with self.subTest(field=field, value=value), self.assertRaises(ValueError): + benchmark.validateReport(report) + + current = self.report() + baseline = self.report() + baseline['entries'] = 100 + + with self.assertRaisesRegex(ValueError, 'entries must match'): + benchmark.compareReports(current, baseline) + + def testInvalidTimingsCannotProduceMisleadingRatios(self): + for value in (0, -1, True, '1.0', None, float('nan'), float('inf')): + report = self.report() + report['measurements']['steady_append_ms'] = value + + with self.subTest(value=value), self.assertRaises(ValueError): + benchmark.validateReport(report) + + def testCommandLineMeasuresAndComparesSavedJsonInFreshProcesses(self): + with tempfile.TemporaryDirectory() as directory: + baseline = Path(directory) / 'baseline.json' + current = Path(directory) / 'current.json' + + for output, extra in ( + (baseline, []), + (current, ['--baseline', str(baseline)]), + ): + arguments = ['--entries', '50', '--output', str(output), *extra] + result = runPythonChild( + 'from tests.benchmarks.benchmark_log_manager import main\n' + f'main({arguments!r})' + ) + assertChildSucceeded(self, result, 'log benchmark CLI') + self.assertEqual( + json.loads(result.stdout), + json.loads(output.read_text(encoding='utf-8')), + ) + + baselineReport = json.loads(baseline.read_text(encoding='utf-8')) + currentReport = json.loads(current.read_text(encoding='utf-8')) + + self.assertNotIn('comparison', baselineReport) + self.assertEqual(currentReport['comparison']['baseline'], baselineReport) + self.assertEqual( + currentReport['comparison']['ratios'], + benchmark.compareReports(currentReport, baselineReport), + ) + self.assertIn('qt', currentReport['environment']) + + def testInvalidBaselineFailsBeforeWorkOrOutput(self): + with tempfile.TemporaryDirectory() as directory: + baseline = Path(directory) / 'baseline.json' + output = Path(directory) / 'current.json' + + for contents in ('not JSON', '{"schema": 99}', json.dumps(self.report())): + baseline.write_text(contents, encoding='utf-8') + + with self.subTest(contents=contents), mock.patch.object( + benchmark, 'run' + ) as run, mock.patch( + 'sys.stderr', new_callable=io.StringIO + ), self.assertRaises( + SystemExit + ) as exited: + benchmark.main( + [ + '--entries', + '100', + '--baseline', + str(baseline), + '--output', + str(output), + ] + ) + + self.assertEqual(exited.exception.code, 2) + run.assert_not_called() + self.assertFalse(output.exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_log_manager_generation.py b/tests/test_log_manager_generation.py index 9c3cd00..acdc342 100644 --- a/tests/test_log_manager_generation.py +++ b/tests/test_log_manager_generation.py @@ -42,13 +42,9 @@ from statistics import median import gc import json -import os import random -import subprocess -import sys import threading import time -import types import unittest import weakref @@ -2000,100 +1996,6 @@ class VeryHeavyGenerationLogManagerTest(unittest.TestCase): synchronous_traversals=0, ) - @unittest.skipUnless( - os.environ.get('FURIOUS_LOG_MANAGER_BASELINE'), - 'set FURIOUS_LOG_MANAGER_BASELINE to a Git revision for comparison', - ) - def testIdenticalWorkloadAgainstPreviousImplementation(self): - """Compare the same steady, rollover, snapshot, and retention workloads.""" - revision = os.environ['FURIOUS_LOG_MANAGER_BASELINE'] - source = subprocess.run( - ['git', 'show', f'{revision}:Furious/Service/LogManager.py'], - cwd=os.getcwd(), - check=True, - capture_output=True, - text=True, - ).stdout - - baselineModule = types.ModuleType('tests._baseline_log_manager') - previousModule = sys.modules.get(baselineModule.__name__) - sys.modules[baselineModule.__name__] = baselineModule - - try: - exec( - compile(source, f'', 'exec'), - baselineModule.__dict__, - ) - finally: - if previousModule is None: - sys.modules.pop(baselineModule.__name__, None) - else: - sys.modules[baselineModule.__name__] = previousModule - - def benchmark(managerClass): - results = {} - - manager = managerClass(maximumEntries=50_000, autoClearEnabled=False) - - started = time.perf_counter_ns() - for index in range(50_000): - manager.append(str(index), APPLICATION_LOG_CATEGORY) - - results['steady_append_ms'] = (time.perf_counter_ns() - started) / 1e6 - - manager = managerClass( - maximumEntries=50_000, - autoClearMaximumEntries=20_000, - ) - - for index in range(20_000): - manager.append(str(index), CORE_LOG_CATEGORY) - for index in range(20_000): - manager.append(str(index), TUN2SOCKS_LOG_CATEGORY) - - started = time.perf_counter_ns() - manager.append('trigger', CORE_LOG_CATEGORY) - results['core_rollover_us'] = (time.perf_counter_ns() - started) / 1e3 - - manager = managerClass(maximumEntries=50_000, autoClearEnabled=False) - - for index in range(30_000): - manager.append( - str(index), - CORE_LOG_CATEGORY if index % 5 == 0 else APPLICATION_LOG_CATEGORY, - ) - - started = time.perf_counter_ns() - manager.snapshot(CORE_LOG_CATEGORY) - results['category_snapshot_us'] = (time.perf_counter_ns() - started) / 1e3 - - started = time.perf_counter_ns() - manager.snapshot() - results['global_snapshot_us'] = (time.perf_counter_ns() - started) / 1e3 - - manager = managerClass(maximumEntries=1_000, autoClearEnabled=False) - - started = time.perf_counter_ns() - for index in range(50_000): - manager.append(str(index), APPLICATION_LOG_CATEGORY) - - results['retention_heavy_ms'] = (time.perf_counter_ns() - started) / 1e6 - - return results - - current = benchmark(LogManager) - baseline = benchmark(baselineModule.LogManager) - - self.assertEqual(LogManager(maximumEntries=1).entries(), tuple()) - - self.report( - 'baseline-comparison', - baseline_revision=revision, - current=current, - baseline=baseline, - ratios={key: current[key] / max(baseline[key], 0.001) for key in current}, - ) - if __name__ == '__main__': unittest.main()