Fix application shutdown

Signed-off-by: Loren Eteval <loren.eteval@proton.me>

Furious/Utility/AppMainProcess.py

@@ -26,6 +26,7 @@ from typing import Callable
 
 import os
 import sys
+import time
 import signal
 import logging
 import datetime
@@ -47,19 +48,58 @@ else:
 class AppMainProcess(ProcessContext.Process):
     """Represent app main process."""
 
+    ShutdownTimeout = 10.0
+    ShutdownPollInterval = 0.1
+    KillJoinTimeout = 3.0
+
     def __init__(self, func: Callable[[], ApplicationRunner], **kwargs):
         """Initialize the AppMainProcess."""
         super().__init__(**kwargs)
 
         self.startupTime = str(datetime.datetime.now()).replace(':', '')
         self.logFileName = f'{self.startupTime}.log'
-        # One synchronized scalar is sufficient for the parent/child result.
-        # A multiprocessing.Manager would create a second child process whose
-        # lifecycle is unrelated to the exact application process we own.
-        self.fileWritten = ProcessContext.Value('b', False)
+        # Shared scalars need neither a manager process nor a response from the
+        # child's potentially blocked event loop during shutdown.
+        self.fileWritten = ProcessContext.Value('b', False, lock=False)
+        self._shutdownDeadline = ProcessContext.Value('d', 0.0, lock=False)
+        self._shutdownExitCode = ProcessContext.Value('i', 0, lock=False)
 
         self.func = func
         self.application = None
+        self.forcedExit = False
+
+    def beginShutdown(self, exitcode=None):
+        """Publish the semantic result and arm one deadline before cleanup starts."""
+        if exitcode is not None:
+            self._shutdownExitCode.value = int(exitcode)
+
+        if not self._shutdownDeadline.value:
+            self._shutdownDeadline.value = time.monotonic() + self.ShutdownTimeout
+
+    def waitForExit(self):
+        """Join this child, forcing exit once its shutdown grace period expires."""
+        while self.is_alive():
+            deadline = self._shutdownDeadline.value
+
+            if deadline and time.monotonic() >= deadline:
+                exitcode = self._shutdownExitCode.value
+                self.forcedExit = True
+                logger.warning('application shutdown timed out; forcing process exit')
+
+                # terminate() is SIGTERM on POSIX, which this child handles by
+                # requesting graceful shutdown again. kill() cannot be caught.
+                self.kill()
+                self.join(self.KillJoinTimeout)
+
+                if self.is_alive():
+                    raise RuntimeError('application process survived forced shutdown')
+
+                return exitcode
+
+            self.join(self.ShutdownPollInterval)
+
+        self.join(0)
+        return self.exitcode
 
     def exceptHook(self, exceptionType, exceptionValue, tb, *, raiseSystemExit=False):
         """Handle except hook for the app main process."""
@@ -71,6 +111,7 @@ class AppMainProcess(ProcessContext.Process):
         else:
             exitcode = ApplicationRunner.ExitCode.UnknownException.value
 
+        self.beginShutdown(exitcode)
         logger.error(f'stopped with exitcode {exitcode}')
 
         self.saveCrashLog(exceptionType, exceptionValue, tb)
@@ -118,6 +159,7 @@ class AppMainProcess(ProcessContext.Process):
 
     def handler(self, signum, frame):
         """Handle r."""
+        self.beginShutdown()
         logger.info(f'received signal {signal.Signals(signum).name}. Exit application')
 
         if self.application is None:
@@ -132,10 +174,22 @@ class AppMainProcess(ProcessContext.Process):
         try:
             self.application = self.func()
 
+            shutdownRequested = getattr(self.application, 'shutdownRequested', None)
+
+            if shutdownRequested is not None:
+                shutdownRequested.connect(self.beginShutdown)
+
             for sig in [signal.SIGTERM, signal.SIGINT]:
                 signal.signal(sig, self.handler)
 
-            sys.exit(self.application.run())
+            exitcode = self.application.run()
+            self.beginShutdown(exitcode)
+            sys.exit(exitcode)
+        except SystemExit as ex:
+            self.beginShutdown(
+                ex.code if isinstance(ex.code, int) else (0 if ex.code is None else 1)
+            )
+            raise
         except Exception:
             # Any non-exit exceptions