Scale subscription update processing

Signed-off-by: Loren Eteval <loren.eteval@proton.me>
This commit is contained in:
Loren Eteval
2026-08-31 10:07:17 +08:00
parent 80ac887255
commit a2deb73839
17 changed files with 874 additions and 27 deletions
@@ -47,6 +47,7 @@ _TRANSLATABLE = (
class ExternalCoreProtocolHandler(ProtocolHandler):
"""Own local External Core mappings while declining URI import/export."""
workerSafe = True
descriptor = ProtocolDescriptor(
id=EXTERNAL_CORE_TYPE,
displayName='External Core',
+1
View File
@@ -51,6 +51,7 @@ _TRANSLATABLE = (
class Hysteria1ProtocolHandler(ProtocolHandler):
"""Own Hysteria 1 URI, mapping, validation, and export behavior."""
workerSafe = True
descriptor = ProtocolDescriptor(
id='hysteria1',
displayName='Hysteria1',
+1
View File
@@ -51,6 +51,7 @@ _TRANSLATABLE = (
class Hysteria2ProtocolHandler(ProtocolHandler):
"""Own Hysteria 2 URI, mapping, validation, and export behavior."""
workerSafe = True
descriptor = ProtocolDescriptor(
id='hysteria2',
displayName='Hysteria2',
+2
View File
@@ -48,6 +48,8 @@ __all__ = ['XRAY_PROTOCOL_HANDLERS']
class XrayProtocolHandler(ProtocolHandler):
"""Adapt one Xray outbound protocol to the host protocol contract."""
workerSafe = True
def __init__(
self,
descriptor,
@@ -50,6 +50,7 @@ def _shareLinks(text: str):
class PlainShareLinkDecoder(SubscriptionDecoder):
"""Decode newline-delimited plain-text share links."""
workerSafe = True
decoderId = 'plain-share-links'
displayName = 'Share Links (plain text)'
priority = 100
@@ -76,6 +77,7 @@ class PlainShareLinkDecoder(SubscriptionDecoder):
class Base64ShareLinkDecoder(SubscriptionDecoder):
"""Decode a Base64 envelope containing plain share links."""
workerSafe = True
decoderId = 'base64-share-links'
displayName = 'Share Links (Base64)'
priority = 90
+3
View File
@@ -25,6 +25,9 @@
entry-point plugins; do not give bundled code hidden repository/UI side channels.
- Evolve contracts additively when practical. Before a breaking change, inspect external discovery, compatibility
exports, every bundled implementation, tests, and compiled inclusion; do not infer compatibility from built-ins alone.
- Capability instances default to GUI-thread-only for background subscription preparation. A decoder or protocol
handler opts into worker execution only after its parsing, validation, caches, globals, and Qt usage are audited as
safe for concurrent copied inputs; keep unclassified third-party capability execution on the GUI thread.
## Verification
+3
View File
@@ -91,6 +91,9 @@ class PluginCapability:
"""Define one independently queryable plugin capability."""
capabilityKind = CapabilityKind.Utility
# Capability instances may be process-lifetime and externally supplied.
# Worker execution therefore requires an explicit opt-in after auditing.
workerSafe = False
@property
def capabilityId(self) -> str:
+31
View File
@@ -576,6 +576,37 @@ class PluginRegistry:
)
)
def subscriptionDecoderWorkerSafe(self, decoderId=None) -> bool:
"""Return whether the selected subscription decoder path is worker-safe."""
if decoderId:
entry = self._decoders.get(_normalizeIdentifier(decoderId))
decoders = (entry[1],) if entry is not None else tuple()
else:
decoders = self.subscriptionDecoders()
return bool(decoders) and all(
bool(getattr(decoder, 'workerSafe', False)) for decoder in decoders
)
def subscriptionItemWorkerSafe(self, item: SubscriptionItem) -> bool:
"""Classify only the protocol capability needed by one decoded item."""
if item.uri is not None:
entry = self._schemes.get(_schemeFromURI(item.uri))
return entry is None or bool(getattr(entry[1], 'workerSafe', False))
protocol = item.configuration.get('type', '')
entry = self._protocols.get(_normalizeIdentifier(protocol))
if entry is not None:
return bool(getattr(entry[1], 'workerSafe', False))
# Undiscriminated mappings are probed against every handler.
return all(
bool(getattr(handler, 'workerSafe', False))
for _plugin, handler in self._protocolEntries
)
def trafficStatsProviders(self):
"""Return registered runtime traffic-statistics providers."""
return self.capabilities(CapabilityKind.TrafficStats)
+10
View File
@@ -123,6 +123,16 @@ class Storage:
"""Persist one subscription group through the shared repository."""
Storage._UserSubsStorage().upsertGroup(group)
@staticmethod
def upsertSubscriptionGroups(groups):
"""Mutate a validated subscription-group batch in one commit."""
Storage._UserSubsStorage().upsertGroups(groups)
@staticmethod
def persistSubscriptionGroups():
"""Serialize the current subscription repository once."""
Storage._UserSubsStorage().sync()
@staticmethod
def removeSubscriptionGroup(unique: str) -> SubscriptionGroup | None:
"""Remove one subscription group through the shared repository."""
+15 -3
View File
@@ -221,10 +221,22 @@ class UserSubs(Mixins.CleanupOnExit, StorageBackend):
def upsertGroup(self, group: SubscriptionGroup):
"""Insert or replace one group without changing its identity."""
if not group.id:
raise ValueError('subscription group ID must not be empty')
self.upsertGroups((group,))
self._data[group.id] = group.toMapping()
def upsertGroups(self, groups):
"""Atomically insert or replace a batch of validated groups."""
staged = {}
for group in groups:
if not isinstance(group, SubscriptionGroup):
raise TypeError('subscription group must be a SubscriptionGroup')
if not group.id:
raise ValueError('subscription group ID must not be empty')
staged[group.id] = group.toMapping()
self._data.update(staged)
def removeGroup(self, unique: str) -> SubscriptionGroup | None:
"""Remove and return one group definition."""
+3
View File
@@ -22,6 +22,9 @@
connectivity, endpoint, subscription, and asset requests own their exact reply and reject stale generations.
- Subscription stages remain separate: decoders return neutral items; import constructs profiles/metadata;
synchronization prepares one group reconciliation; the manager owns request/schedule generations and commits it.
Large payload import and reconciliation preparation run in the manager's bounded pool over copied payload/profile
data. Workers never read live repositories or Qt models; the GUI thread verifies the full source signature and group
revision, commits while preserving live profile identity/local metadata, then publishes coalesced status/structure.
Post-commit reconnect/test invalidation failure is reported without undoing the committed profiles.
- Log transport, traffic collection, and metric history remain bounded and independent of page visibility. Rendering may
be lazy; collection/draining ownership is not.
+31 -1
View File
@@ -30,9 +30,14 @@ __all__ = [
'SubscriptionImportResult',
'SubscriptionImportService',
'SubscriptionSource',
'SubscriptionWorkerUnsafe',
]
class SubscriptionWorkerUnsafe(RuntimeError):
"""Signal that one plugin capability must remain on the GUI thread."""
@dataclass(frozen=True)
class SubscriptionSource:
"""Describe where a subscription payload came from."""
@@ -62,8 +67,25 @@ class SubscriptionImportService:
"""Use the supplied capability registry or the application registry."""
self.registry = registry or getPluginRegistry()
def importPayload(self, data: bytes, source: SubscriptionSource):
def importPayload(
self,
data: bytes,
source: SubscriptionSource,
*,
requireWorkerSafe: bool = False,
isCancelled=None,
):
"""Decode *data* and construct supported profiles for *source*."""
if requireWorkerSafe and not self.registry.subscriptionDecoderWorkerSafe(
source.decoderId
):
raise SubscriptionWorkerUnsafe(
'subscription capabilities did not opt in to worker execution'
)
if callable(isCancelled) and isCancelled():
return None
result = self.registry.decodeSubscription(data, source.decoderId)
if result is None:
@@ -74,6 +96,14 @@ class SubscriptionImportService:
identityOccurrences = {}
for item in result.items:
if callable(isCancelled) and isCancelled():
return None
if requireWorkerSafe and not self.registry.subscriptionItemWorkerSafe(item):
raise SubscriptionWorkerUnsafe(
'subscription protocol capability did not opt in to worker execution'
)
value = item.configuration if item.configuration is not None else item.uri
metadata = {
**dict(item.metadata),
+508 -17
View File
@@ -31,6 +31,11 @@ from Furious.Repository import Storage
from Furious.Service.SubscriptionImporter import (
SubscriptionImportService,
SubscriptionSource,
SubscriptionWorkerUnsafe,
)
from Furious.Service.SubscriptionPreparation import (
SubscriptionPreparationJob,
SubscriptionPreparationRelay,
)
from Furious.Service.SubscriptionSync import SubscriptionSynchronizer
@@ -40,6 +45,8 @@ from PySide6.QtNetwork import QNetworkRequest
from dataclasses import dataclass
import re
import os
import time
import logging
import datetime
@@ -99,6 +106,17 @@ class SubscriptionUpdateBatch:
showMessageBox: bool
@dataclass
class _SubscriptionBatchState:
"""Track logical completion independently from network reply completion."""
pending: set
showMessageBox: bool
successful: list
failed: list
structural: bool = False
class SubscriptionManager(HttpGetManager):
"""Own subscription networking, decoding, reconciliation, and persistence."""
@@ -124,6 +142,23 @@ class SubscriptionManager(HttpGetManager):
self._requestVersions = {}
self._activeReplies = {}
self._replySubscriptions = {}
self._batches = {}
self._preparationJobs = {}
self._preparationPayloads = {}
self._nextBatchId = 0
self._nextPreparationJobId = 0
self._shuttingDown = False
self._preparationRelay = SubscriptionPreparationRelay(self)
self._preparationRelay.completed.connect(
self._handlePreparationOutcome,
QtCore.Qt.ConnectionType.QueuedConnection,
)
self._preparationPool = QtCore.QThreadPool(self)
self._preparationPool.setMaxThreadCount(
max(1, min((os.cpu_count() or 1) // 2, 4))
)
self.refreshAutoUpdates()
@@ -157,9 +192,25 @@ class SubscriptionManager(HttpGetManager):
subscription = Storage.UserSubs().get(unique)
return bool(
subscription
not self._shuttingDown
and subscription
and self._requestVersions.get(unique) == version
and subscription.get('webURL') == kwargs.get('webURL')
and (
'requestSignature' not in kwargs
or self._requestSignature(subscription) == kwargs['requestSignature']
)
)
@staticmethod
def _requestSignature(subscription):
"""Capture every source option whose edit invalidates prepared data."""
return (
subscription.get('webURL', ''),
subscription.get('enabled', True),
subscription.get('userAgent', ''),
subscription.get('filter', ''),
subscription.get('lastDecoderId', ''),
)
@QtCore.Slot(object)
@@ -296,7 +347,7 @@ class SubscriptionManager(HttpGetManager):
self._pruneRequestVersion(unique)
def cancelUpdates(self, unique: str | None = None):
"""Cancel exact active replies and invalidate their eventual completions."""
"""Cancel network and preparation work and invalidate eventual completions."""
if unique is None:
subscriptions = {
*self._requestVersions,
@@ -312,6 +363,332 @@ class SubscriptionManager(HttpGetManager):
if unique is None or self._replySubscriptions.get(reply) == unique:
reply.abort()
for job in tuple(self._preparationJobs.values()):
if unique is None or job.context.get('unique') == unique:
job.cancel()
def shutdown(self):
"""Boundedly stop every manager-owned request, timer, and preparation job."""
if self._shuttingDown:
return
self._shuttingDown = True
for timer in self._autoUpdateTimers.values():
timer.stop()
self.cancelUpdates()
self._preparationPool.clear()
for job in self._preparationJobs.values():
job.cancel()
# Running third-party Python code may not be interruptible. Keep the
# relay/pool alive until those exact jobs finish rather than allowing a
# worker to publish through a destroyed QObject during application exit.
self._preparationPool.waitForDone()
self._preparationJobs.clear()
self._preparationPayloads.clear()
self._batches.clear()
@staticmethod
def _filterImportResult(result, profileFilter: str, remark: str):
"""Apply a copied regex filter as part of subscription preparation."""
profileFilter = str(profileFilter).strip()
if result is None or not profileFilter:
return result
try:
pattern = re.compile(profileFilter, re.IGNORECASE)
except re.error as ex:
logger.error(
f'invalid subscription filter for {remark!r}: {ex}. '
f'Importing all profiles'
)
return result
return type(result)(
result.decoderId,
tuple(
profile
for profile in result.profiles
if pattern.search(str(getattr(profile, 'itemRemark', '')))
),
result.rejectedItems,
)
def _startPreparationJob(self, stage: str, context: dict, work):
"""Dispatch one copied operation to the bounded preparation pool."""
if not self._isCurrentRequest(context):
self._finishOperation(context)
return None
self._nextPreparationJobId += 1
jobId = self._nextPreparationJobId
job = SubscriptionPreparationJob(
jobId,
stage,
context,
work,
self._preparationRelay,
)
self._preparationJobs[jobId] = job
self._preparationPool.start(job)
return jobId
def _startImportPreparation(self, data: bytes, context: dict):
"""Decode, parse, normalize, and validate a copied payload off-thread."""
source = SubscriptionSource(
context.get('unique', ''),
context.get('webURL', ''),
context.get('remark', ''),
context.get('decoderId'),
)
importer = self.importer
filterResult = type(self)._filterImportResult
profileFilter = str(context.get('filter', ''))
remark = str(context.get('remark', ''))
def work(isCancelled):
"""Operate only on captured bytes, values, and plugin data capabilities."""
result = importer.importPayload(
data,
source,
requireWorkerSafe=True,
isCancelled=isCancelled,
)
return filterResult(result, profileFilter, remark)
jobId = self._startPreparationJob('import', context, work)
if jobId is not None:
self._preparationPayloads[jobId] = data
def _runGuiThreadImport(self, data: bytes, context: dict):
"""Isolate non-opted-in third-party parsing on its required GUI thread."""
source = SubscriptionSource(
context.get('unique', ''),
context.get('webURL', ''),
context.get('remark', ''),
context.get('decoderId'),
)
try:
result = self.importer.importPayload(data, source)
result = self._filterImportResult(
result,
context.get('filter', ''),
context.get('remark', ''),
)
except Exception as ex:
# Any non-exit exceptions
logger.exception(
f'failed to prepare subscription {context.get("unique", "")!r}'
)
self._failOperation(context, str(ex) or type(ex).__name__)
return
self._handleImportedResult(context, result, 0.0)
def _handleImportedResult(self, context: dict, result, duration: float):
"""Capture current group data and dispatch copied reconciliation work."""
if not self._isCurrentRequest(context):
self._finishOperation(context)
return
if result is None or not result.profiles:
self._failOperation(context, 'UnsupportedSubscriptionFormat')
return
context = {**context, 'decoderId': result.decoderId}
logger.info(
f'prepared subscription ({context.get("remark", "")}, '
f'{context.get("unique", "")!r}) with {len(result.profiles)} profiles '
f'from {result.decoderId!r}; rejected {result.rejectedItems}; '
f'decode/parse {duration:.3f}s'
)
try:
snapshot = self.synchronizer.snapshot(
Storage.UserServers(),
context.get('unique', ''),
)
except Exception as ex:
# Any non-exit exceptions
self._failOperation(context, str(ex) or type(ex).__name__)
return
synchronizer = self.synchronizer
incoming = result.profiles
def work(isCancelled):
"""Compare copied existing and incoming profiles without live state."""
if isCancelled():
return None
return synchronizer.prepare(snapshot, incoming)
self._startPreparationJob('reconcile', context, work)
@QtCore.Slot(object)
def _handlePreparationOutcome(self, outcome):
"""Validate one queued worker result and commit only on this Qt thread."""
jobId = getattr(outcome, 'jobId', -1)
self._preparationJobs.pop(jobId, None)
fallbackPayload = self._preparationPayloads.pop(jobId, None)
context = getattr(outcome, 'context', {})
if self._shuttingDown:
return
if getattr(outcome, 'cancelled', False) or not self._isCurrentRequest(context):
logger.debug(
f'discard stale/cancelled subscription preparation for '
f'{context.get("unique", "")!r}'
)
self._finishOperation(context)
return
if getattr(outcome, 'errorType', ''):
if (
outcome.errorType == SubscriptionWorkerUnsafe.__name__
and fallbackPayload is not None
):
self._runGuiThreadImport(fallbackPayload, context)
else:
logger.error(
f'subscription {outcome.stage} preparation failed for '
f'{context.get("unique", "")!r} ({outcome.errorType})'
)
self._failOperation(context, outcome.error)
return
if outcome.stage == 'import':
self._handleImportedResult(context, outcome.value, outcome.duration)
return
if outcome.stage != 'reconcile' or outcome.value is None:
self._finishOperation(context)
return
started = time.perf_counter()
try:
result = self._synchronizePreparedProfiles(context['unique'], outcome.value)
except Exception as ex:
# Any non-exit exceptions
error = str(ex) or type(ex).__name__
if not self._isCurrentRequest(context):
self._finishOperation(context)
else:
logger.exception(
f'failed to commit subscription {context.get("unique", "")!r}'
)
self._failOperation(context, error)
return
committed = {**context, 'syncResult': result}
self.subscriptionCommitted.emit(context['unique'])
self._recordGroupSuccess(committed, result)
self.subscriptionStateChanged.emit((context['unique'],))
self._finishOperation(committed, successful=committed, structural=True)
logger.info(
f'committed subscription {context["unique"]!r}; reconciliation '
f'{outcome.duration:.3f}s, GUI commit {time.perf_counter() - started:.3f}s'
)
def _failOperation(self, context: dict, error: str):
"""Finalize one current logical operation as an isolated failure."""
failed = {**context, 'error': error}
if self._isCurrentRequest(context):
self._recordGroupFailure(failed)
self.subscriptionStateChanged.emit((context.get('unique', ''),))
self._finishOperation(context, failed=failed)
def _finishOperation(
self,
context: dict,
*,
successful=None,
failed=None,
structural: bool = False,
):
"""Complete one batch member and publish one coalesced batch outcome."""
batchId = context.get('batchId')
state = self._batches.get(batchId)
if state is None or self._shuttingDown:
return
token = (context.get('unique', ''), context.get('requestVersion'))
if token not in state.pending:
return
state.pending.remove(token)
if successful is not None:
state.successful.append(successful)
if failed is not None and self._isCurrentRequest(context):
state.failed.append(failed)
state.structural = state.structural or structural
if state.pending:
return
self._batches.pop(batchId, None)
try:
Storage.persistSubscriptionGroups()
except Exception:
# Any non-exit exceptions
logger.exception('failed to persist completed subscription batch')
if state.structural:
self.subscriptionsChanged.emit()
if state.successful or state.failed:
self.updateCompleted.emit(
SubscriptionUpdateBatch(
tuple(state.successful),
tuple(state.failed),
state.showMessageBox,
)
)
def _synchronizeProfiles(self, unique: str, profiles):
"""Reconcile one group and apply connection effects without UI ownership."""
servers = Storage.UserServers()
@@ -371,6 +748,59 @@ class SubscriptionManager(HttpGetManager):
return result
def _synchronizePreparedProfiles(self, unique: str, plan):
"""Commit one accepted worker plan and apply GUI-owned connection effects."""
servers = Storage.UserServers()
activatedIndex = Storage.UserActivatedItemIndex()
activeProfileId = ''
activeWasManagedByGroup = False
if 0 <= activatedIndex < len(servers):
active = servers[activatedIndex]
activeProfileId = active.metadata.profileId
activeWasManagedByGroup = (
active.itemSubscription == unique and active.itemSubscriptionManaged
)
controller = AppConnectionController()
wasConnected = controller is not None and controller.isConnected()
result = self.synchronizer.commit(servers, plan)
newActivatedIndex = next(
(
index
for index, profile in enumerate(servers)
if profile.metadata.profileId == activeProfileId
),
-1,
)
try:
AppSettings.set('ActivatedItemIndex', str(newActivatedIndex))
except Exception:
# Any non-exit exceptions
logger.exception(
'failed to persist the active profile index after '
f'synchronizing subscription {unique!r}'
)
try:
if wasConnected and activeProfileId:
if newActivatedIndex < 0 and activeWasManagedByGroup:
controller.startDisconnection()
elif activeProfileId in result.changedProfileIds:
controller.startReconnection()
except Exception:
# Any non-exit exceptions
logger.exception(
f'failed to apply connection effects after synchronizing '
f'subscription {unique!r}'
)
return result
@staticmethod
def _recordGroupFailure(param):
"""Best-effort persist one current request's terminal failure state."""
@@ -507,6 +937,9 @@ class SubscriptionManager(HttpGetManager):
def completionCallback(self, **kwargs):
"""Complete a batch after its final reply finishes."""
if 'batchId' in kwargs:
return
depthMap = kwargs.get('depthMap', {})
depthMap['depth'] -= 1
@@ -516,6 +949,20 @@ class SubscriptionManager(HttpGetManager):
def successCallback(self, networkReply, **kwargs):
"""Decode one successful subscription response."""
if not self._isCurrentRequest(kwargs):
self._finishOperation(kwargs)
return
if 'batchId' in kwargs:
data = bytes(networkReply.readAll().data())
decoderId = kwargs.get('decoderId') or kwargs.get('lastDecoderId')
context = {**kwargs, 'decoderId': decoderId}
if self.importer.registry.subscriptionDecoderWorkerSafe(decoderId):
self._startImportPreparation(data, context)
else:
self._runGuiThreadImport(data, context)
return
unique = kwargs.get('unique', '')
@@ -573,6 +1020,8 @@ class SubscriptionManager(HttpGetManager):
def failureCallback(self, networkReply, **kwargs):
"""Record one failed subscription response."""
if not self._isCurrentRequest(kwargs):
self._finishOperation(kwargs)
return
unique = kwargs.get('unique', '')
@@ -583,6 +1032,11 @@ class SubscriptionManager(HttpGetManager):
logger.error(f'update subscription ({remark}, {unique!r}) failed: {error}')
if 'batchId' in kwargs:
self._failOperation(kwargs, error)
return
failureArgs.append({'error': error, **kwargs})
def updateSubsByWebGET(self, **kwargs):
@@ -631,8 +1085,14 @@ class SubscriptionManager(HttpGetManager):
return
changedSubscriptions = []
groups = []
operations = []
for unique, _subscription in batch:
self._nextBatchId += 1
batchId = self._nextBatchId
for unique, subscription in batch:
group = Storage.SubscriptionGroup(unique)
if group is None:
@@ -641,26 +1101,57 @@ class SubscriptionManager(HttpGetManager):
group.lastSyncStatus = 'syncing'
group.lastSyncError = ''
Storage.upsertSubscriptionGroup(group)
self.cancelUpdates(unique)
version = self._requestVersions[unique]
context = {
'unique': unique,
'remark': subscription.get('remark', ''),
'webURL': subscription.get('webURL', ''),
'userAgent': subscription.get('userAgent', ''),
'filter': subscription.get('filter', ''),
'decoderId': subscription.get('lastDecoderId') or None,
'lastDecoderId': subscription.get('lastDecoderId', ''),
'batchId': batchId,
'requestVersion': version,
'requestSignature': self._requestSignature(subscription),
}
groups.append(group)
operations.append(context)
changedSubscriptions.append(unique)
if not operations:
return
Storage.upsertSubscriptionGroups(groups)
Storage.persistSubscriptionGroups()
self._batches[batchId] = _SubscriptionBatchState(
{(context['unique'], context['requestVersion']) for context in operations},
bool(kwargs.get('showMessageBox', True)),
[],
[],
)
if changedSubscriptions:
self.subscriptionStateChanged.emit(tuple(changedSubscriptions))
depthMap = {'depth': len(batch)}
successArgs = []
failureArgs = []
for context in operations:
try:
self.updateSubsByWebGET(
**context,
logActionMessage=bool(kwargs.get('logActionMessage', False)),
)
except Exception as ex:
# Any non-exit exceptions
for unique, subscription in batch:
self.updateSubsByWebGET(
unique=unique,
**subscription,
**kwargs,
depthMap=depthMap,
successArgs=successArgs,
failureArgs=failureArgs,
requestVersion=self._nextRequestVersion(unique),
)
logger.exception(
f'failed to start subscription request for '
f'{context.get("unique", "")!r}'
)
self._failOperation(context, str(ex) or type(ex).__name__)
def updateSubsByUnique(self, unique: str, **kwargs):
"""Update one enabled subscription through the canonical batch path."""
+109
View File
@@ -0,0 +1,109 @@
# Copyright (C) 2024present Loren Eteval & contributors <loren.eteval@proton.me>
#
# This file is part of Furious.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
"""Run copied subscription preparation work in a bounded Qt thread pool."""
from __future__ import annotations
from PySide6 import QtCore
from dataclasses import dataclass
import threading
import time
__all__ = [
'SubscriptionPreparationJob',
'SubscriptionPreparationOutcome',
'SubscriptionPreparationRelay',
]
@dataclass(frozen=True)
class SubscriptionPreparationOutcome:
"""Return plain worker data to the subscription manager's Qt thread."""
jobId: int
stage: str
context: dict
value: object = None
errorType: str = ''
error: str = ''
cancelled: bool = False
duration: float = 0.0
workerThreadId: int = 0
class SubscriptionPreparationRelay(QtCore.QObject):
"""Provide one process-lifetime queued result path for all pool jobs."""
completed = QtCore.Signal(object)
class SubscriptionPreparationJob(QtCore.QRunnable):
"""Own one cancellable plain-data preparation callable."""
def __init__(self, jobId: int, stage: str, context: dict, work, relay):
"""Capture copied operation context and a non-Qt work callable."""
super().__init__()
self.jobId = jobId
self.stage = stage
self.context = dict(context)
self.work = work
self.relay = relay
self.cancelled = threading.Event()
self.setAutoDelete(True)
def cancel(self):
"""Make queued/running work stop cooperatively and reject its result."""
self.cancelled.set()
def run(self):
"""Execute without touching QObject, widget, model, or live repository state."""
started = time.perf_counter()
value = None
errorType = ''
error = ''
try:
if not self.cancelled.is_set():
value = self.work(self.cancelled.is_set)
except Exception as ex:
# Any non-exit exceptions
errorType = type(ex).__name__
error = str(ex) or errorType
try:
self.relay.completed.emit(
SubscriptionPreparationOutcome(
self.jobId,
self.stage,
self.context,
value,
errorType,
error,
self.cancelled.is_set(),
time.perf_counter() - started,
threading.get_ident(),
)
)
except RuntimeError:
# The owning manager may already be gone after an abnormal teardown.
pass
+131 -1
View File
@@ -25,7 +25,12 @@ import copy
from dataclasses import dataclass
__all__ = ['SubscriptionSyncResult', 'SubscriptionSynchronizer']
__all__ = [
'SubscriptionSyncPlan',
'SubscriptionSyncResult',
'SubscriptionSyncSnapshot',
'SubscriptionSynchronizer',
]
@dataclass(frozen=True)
@@ -41,6 +46,25 @@ class SubscriptionSyncResult:
changedProfileIds: tuple[str, ...]
@dataclass(frozen=True)
class SubscriptionSyncSnapshot:
"""Hold copied group profiles and the live source revision they represent."""
groupId: str
profiles: tuple[ServerProfile, ...]
sourceRevision: tuple[tuple[str, str, str], ...]
@dataclass(frozen=True)
class SubscriptionSyncPlan:
"""Describe a worker-prepared replacement for one subscription group."""
groupId: str
sourceRevision: tuple[tuple[str, str, str], ...]
profiles: tuple[ServerProfile, ...]
result: SubscriptionSyncResult
class SubscriptionSynchronizer:
"""Own stable, group-scoped profile reconciliation semantics."""
@@ -53,6 +77,112 @@ class SubscriptionSynchronizer:
'tags',
)
@staticmethod
def _sourceRevision(profiles, groupId: str):
"""Fingerprint the relevant live group state without using row identity."""
return tuple(
(
profile.metadata.profileId,
profile.metadata.subscriptionProfileKey,
profileConnectionFingerprint(profile),
)
for profile in profiles
if profile.itemSubscription == groupId and profile.itemSubscriptionManaged
)
def snapshot(self, profiles, groupId: str) -> SubscriptionSyncSnapshot:
"""Copy only the live profiles owned by *groupId* for worker preparation."""
if not groupId:
raise ValueError('subscription group ID must not be empty')
owned = tuple(
copy.deepcopy(profile)
for profile in profiles
if profile.itemSubscription == groupId and profile.itemSubscriptionManaged
)
return SubscriptionSyncSnapshot(
groupId,
owned,
self._sourceRevision(profiles, groupId),
)
def prepare(
self,
snapshot: SubscriptionSyncSnapshot,
incomingProfiles,
) -> SubscriptionSyncPlan:
"""Build a reconciliation plan using copied profiles only."""
working = list(snapshot.profiles)
result = self.reconcile(working, incomingProfiles, snapshot.groupId)
return SubscriptionSyncPlan(
snapshot.groupId,
snapshot.sourceRevision,
tuple(working),
result,
)
def commit(self, profiles, plan: SubscriptionSyncPlan) -> SubscriptionSyncResult:
"""Atomically apply a current worker plan while preserving live identities."""
groupId = plan.groupId
if self._sourceRevision(profiles, groupId) != plan.sourceRevision:
raise RuntimeError('subscription profile source changed during preparation')
managedIndexes = [
index
for index, profile in enumerate(profiles)
if profile.itemSubscription == groupId and profile.itemSubscriptionManaged
]
insertionIndex = min(managedIndexes) if managedIndexes else len(profiles)
existingById = {
profiles[index].metadata.profileId: profiles[index]
for index in managedIndexes
}
synchronized = []
for prepared in plan.profiles:
existing = existingById.pop(prepared.metadata.profileId, None)
if existing is None:
synchronized.append(prepared)
continue
metadata = copy.deepcopy(prepared.metadata)
for fieldName in self.LocalMetadataFields:
setattr(metadata, fieldName, getattr(existing.metadata, fieldName))
existing.connection = prepared.connection
existing.metadata = metadata
synchronized.append(existing)
for removed in existingById.values():
removed.deleted = True
unmanagedOrOther = [
profile
for profile in profiles
if not (
profile.itemSubscription == groupId and profile.itemSubscriptionManaged
)
]
finalProfiles = (
unmanagedOrOther[:insertionIndex]
+ synchronized
+ unmanagedOrOther[insertionIndex:]
)
profiles[:] = finalProfiles
for index, profile in enumerate(finalProfiles):
profile.index = index
profile.deleted = False
return plan.result
@staticmethod
def _keyAssignments(profiles: list[ServerProfile], groupId: str):
"""Plan deterministic keys for legacy profiles without mutating them."""
+1
View File
@@ -1917,6 +1917,7 @@ class ServerTableView(
def cleanup(self):
"""Release resources owned by the user servers Qt table view."""
self.subsManager.shutdown()
self.profileTestManager.shutdown()
self._clearSubscriptionActions()
+22 -5
View File
@@ -728,18 +728,32 @@ class SubscriptionTableView(Mixins.QTranslatable, AppQTableView):
self.ItemKey.index('profiles'),
)
for unique in dict.fromkeys(uniques):
row = rows.get(unique)
changedRows = sorted(
row
for unique in dict.fromkeys(uniques)
if (row := rows.get(unique)) is not None
)
if not changedRows:
return
firstRow = previousRow = changedRows[0]
for row in (*changedRows[1:], None):
if row is not None and row == previousRow + 1:
previousRow = row
if row is None:
continue
self.sourceModel.dataChanged.emit(
self.sourceModel.index(row, firstColumn),
self.sourceModel.index(row, lastColumn),
self.sourceModel.index(firstRow, firstColumn),
self.sourceModel.index(previousRow, lastColumn),
[],
)
if row is not None:
firstRow = previousRow = row
def flushItem(self, row, column, item):
"""Refresh item."""
if row < 0 or row >= self.sourceModel.rowCount():
@@ -806,6 +820,9 @@ class SubscriptionTableView(Mixins.QTranslatable, AppQTableView):
subsob = {unique: group.toMapping()}
if unique in Storage.UserSubs():
if self.subsManager is not None:
self.subsManager.cancelUpdates(unique)
row = list(Storage.UserSubs().keys()).index(unique)
Storage.upsertSubscriptionGroup(group)