feat: harden isolated Tornado test workflow

src/MBN_STOCK_WEBVIEW.Playout/TornadoPlayoutEngine.cs

@@ -9,6 +9,9 @@ namespace MBN_STOCK_WEBVIEW.Playout;
 
 internal sealed class TornadoPlayoutEngine : IPlayoutEngine
 {
+    private const string ProcessChangedMessage =
+        "Tornado 프로세스가 변경되어 안전한 재연결이 필요합니다.";
+
     private readonly ValidatedPlayoutOptions _options;
     private readonly IK3dSessionFactory _sessionFactory;
     private readonly IK3dRegistrationProbe _registrationProbe;
@@ -22,6 +25,7 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
     private readonly Task _monitorTask;
 
     private IK3dSession? _session;
+    private TornadoProcessSnapshot? _connectedProcessSnapshot;
     private PlayoutCue? _preparedCue;
     private string? _onAirSceneName;
     private TornadoProcessSnapshot _processSnapshot = new(0, 0, 0);
@@ -30,8 +34,9 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
     private int _reconnectAttempts;
     private long _sequence;
     private bool _connectionRequested;
-    private bool _outcomeUnknown;
-    private bool _disposed;
+    private volatile bool _outcomeUnknown;
+    private volatile bool _disposed;
+    private int _disposeStarted;
 
     internal TornadoPlayoutEngine(
         ValidatedPlayoutOptions options,
@@ -110,9 +115,41 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
             cancellationToken,
             token => TakeOutCoreAsync(scope, token));
 
+    public async ValueTask QuarantineAsync()
+    {
+        await _gate.WaitAsync(CancellationToken.None).ConfigureAwait(false);
+        try
+        {
+            if (_disposed)
+            {
+                return;
+            }
+
+            _outcomeUnknown = true;
+            _connectionRequested = false;
+            _connectedProcessSnapshot = null;
+            _preparedCue = null;
+            _onAirSceneName = null;
+            _connectionState = PlayoutConnectionState.OutcomeUnknown;
+
+            var session = _session;
+            _session = null;
+            if (session is not null)
+            {
+                await AbandonSessionOnStaAsync(session).ConfigureAwait(false);
+            }
+
+            PublishStatus("송출 결과를 확인할 수 없어 연결을 격리했습니다. 출력 상태를 직접 확인하세요.");
+        }
+        finally
+        {
+            _gate.Release();
+        }
+    }
+
     public async ValueTask DisposeAsync()
     {
-        if (_disposed)
+        if (Interlocked.CompareExchange(ref _disposeStarted, 1, 0) != 0)
         {
             return;
         }
@@ -133,7 +170,15 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
         {
             if (_session is not null && _dispatcher is not null && !_dispatcher.IsQuarantined)
             {
-                await ReleaseSessionAsync(CancellationToken.None).ConfigureAwait(false);
+                if (_onAirSceneName is not null || _outcomeUnknown)
+                {
+                    _outcomeUnknown = true;
+                    await AbandonCurrentSessionAsync().ConfigureAwait(false);
+                }
+                else
+                {
+                    await ReleaseSessionAsync(CancellationToken.None).ConfigureAwait(false);
+                }
             }
 
             _preparedCue = null;
@@ -152,7 +197,6 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
         }
 
         _monitorCancellation.Dispose();
-        _gate.Dispose();
     }
 
     internal async Task PollProcessOnceAsync(CancellationToken cancellationToken = default)
@@ -193,6 +237,13 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
             if (!eligible)
             {
                 _reconnectAttempts = 0;
+                if (_onAirSceneName is not null)
+                {
+                    await AbandonCurrentSessionAsync().ConfigureAwait(false);
+                    MarkOutcomeUnknown(PlayoutOperation.Disconnect);
+                    return;
+                }
+
                 if (_session is not null && !_outcomeUnknown)
                 {
                     if (!await ReleaseSessionAsync(cancellationToken).ConfigureAwait(false))
@@ -212,6 +263,30 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
                 return;
             }
 
+            if (HasConnectedProcessGenerationChanged(_processSnapshot))
+            {
+                _reconnectAttempts = 0;
+                if (_onAirSceneName is not null)
+                {
+                    await AbandonCurrentSessionAsync().ConfigureAwait(false);
+                    MarkOutcomeUnknown(PlayoutOperation.Disconnect);
+                    return;
+                }
+
+                if (!await ReleaseSessionAsync(cancellationToken).ConfigureAwait(false))
+                {
+                    MarkOutcomeUnknown(PlayoutOperation.Disconnect);
+                    return;
+                }
+
+                _preparedCue = null;
+                _onAirSceneName = null;
+                ScheduleReconnect();
+                _connectionState = _connectionRequested
+                    ? PlayoutConnectionState.Reconnecting
+                    : PlayoutConnectionState.Disconnected;
+            }
+
             if (_session is not null || !_connectionRequested || !_options.ReconnectEnabled ||
                 _outcomeUnknown || _reconnectAttempts >= _options.MaximumReconnectAttempts ||
                 _timeProvider.GetUtcNow() < _nextReconnectAt)
@@ -245,14 +320,41 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
         }
         catch (OperationCanceledException)
         {
-            return Result(operation, PlayoutResultCode.Cancelled, "송출 요청이 취소되었습니다.");
+            // A caller can be cancelled while another serialized command is still
+            // deciding whether its COM outcome is known. Wait for that decision so an
+            // existing OutcomeUnknown latch can never be downgraded to retryable Cancelled.
+            await _gate.WaitAsync(CancellationToken.None).ConfigureAwait(false);
+            try
+            {
+                if (_disposed)
+                {
+                    return _outcomeUnknown
+                        ? ExistingOutcomeUnknown(operation)
+                        : Result(operation, PlayoutResultCode.Unavailable, "송출 어댑터가 종료되었습니다.");
+                }
+
+                return _outcomeUnknown
+                    ? ExistingOutcomeUnknown(operation)
+                    : Result(operation, PlayoutResultCode.Cancelled, "송출 요청이 취소되었습니다.");
+            }
+            finally
+            {
+                _gate.Release();
+            }
         }
 
         try
         {
             if (_disposed)
             {
-                return Result(operation, PlayoutResultCode.Unavailable, "송출 어댑터가 종료되었습니다.");
+                return _outcomeUnknown
+                    ? ExistingOutcomeUnknown(operation)
+                    : Result(operation, PlayoutResultCode.Unavailable, "송출 어댑터가 종료되었습니다.");
+            }
+
+            if (_outcomeUnknown)
+            {
+                return ExistingOutcomeUnknown(operation);
             }
 
             return await callback(cancellationToken).ConfigureAwait(false);
@@ -308,16 +410,26 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
             return MarkOutcomeUnknown(PlayoutOperation.Connect);
         }
 
-        if (_session is not null)
-        {
-            _connectionState = PlayoutConnectionState.Connected;
-            PublishStatus("Tornado 테스트 송출에 연결되었습니다.");
-            return Result(PlayoutOperation.Connect, PlayoutResultCode.Success, "송출 연결이 준비되었습니다.");
-        }
-
         _processSnapshot = CaptureProcessSnapshot();
         if (!IsProcessEligible(_processSnapshot))
         {
+            if (_onAirSceneName is not null)
+            {
+                await AbandonCurrentSessionAsync().ConfigureAwait(false);
+                return MarkOutcomeUnknown(PlayoutOperation.Connect);
+            }
+
+            if (_session is not null)
+            {
+                if (!await ReleaseSessionAsync(cancellationToken).ConfigureAwait(false))
+                {
+                    return MarkOutcomeUnknown(PlayoutOperation.Disconnect);
+                }
+
+                _preparedCue = null;
+                _onAirSceneName = null;
+            }
+
             _connectionState = reconnecting
                 ? PlayoutConnectionState.Reconnecting
                 : PlayoutConnectionState.Disconnected;
@@ -328,6 +440,32 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
                 ProcessUnavailableMessage(_processSnapshot));
         }
 
+        if (_session is not null && !HasConnectedProcessGenerationChanged(_processSnapshot))
+        {
+            _connectionState = PlayoutConnectionState.Connected;
+            PublishStatus("Tornado 테스트 송출에 연결되었습니다.");
+            return Result(PlayoutOperation.Connect, PlayoutResultCode.Success, "송출 연결이 준비되었습니다.");
+        }
+
+        if (_session is not null)
+        {
+            if (_onAirSceneName is not null)
+            {
+                await AbandonCurrentSessionAsync().ConfigureAwait(false);
+                return MarkOutcomeUnknown(PlayoutOperation.Connect);
+            }
+
+            if (!await ReleaseSessionAsync(cancellationToken).ConfigureAwait(false))
+            {
+                return MarkOutcomeUnknown(PlayoutOperation.Disconnect);
+            }
+
+            _preparedCue = null;
+            _onAirSceneName = null;
+        }
+
+        var processSnapshotBeforeConnect = _processSnapshot;
+
         K3dRegistrationReport registration;
         try
         {
@@ -379,6 +517,30 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
                 static lateSession => lateSession.Dispose(),
                 () => pendingSession?.Dispose()).ConfigureAwait(false);
 
+            _processSnapshot = CaptureProcessSnapshot();
+            if (!IsProcessEligible(_processSnapshot) ||
+                !processSnapshotBeforeConnect.HasSameProcessGeneration(_processSnapshot))
+            {
+                if (!await ReleaseSessionAsync(CancellationToken.None).ConfigureAwait(false))
+                {
+                    return MarkOutcomeUnknown(PlayoutOperation.Disconnect);
+                }
+
+                ScheduleReconnect();
+                _connectionState = reconnecting
+                    ? PlayoutConnectionState.Reconnecting
+                    : PlayoutConnectionState.Disconnected;
+                var processMessage = IsProcessEligible(_processSnapshot)
+                    ? ProcessChangedMessage
+                    : ProcessUnavailableMessage(_processSnapshot);
+                PublishStatus(processMessage);
+                return Result(
+                    PlayoutOperation.Connect,
+                    PlayoutResultCode.Unavailable,
+                    processMessage);
+            }
+
+            _connectedProcessSnapshot = _processSnapshot;
             _connectionState = PlayoutConnectionState.Connected;
             _reconnectAttempts = 0;
             _preparedCue = null;
@@ -435,6 +597,12 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
             return MarkOutcomeUnknown(PlayoutOperation.Disconnect);
         }
 
+        if (_onAirSceneName is not null)
+        {
+            await AbandonCurrentSessionAsync().ConfigureAwait(false);
+            return MarkOutcomeUnknown(PlayoutOperation.Disconnect);
+        }
+
         _connectionState = PlayoutConnectionState.Disconnecting;
         PublishStatus("Tornado 송출 연결을 해제하는 중입니다.");
         if (!await ReleaseSessionAsync(cancellationToken).ConfigureAwait(false))
@@ -569,6 +737,14 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
 
     private async Task<PlayoutResult> NextCoreAsync(PlayoutCue cue, CancellationToken cancellationToken)
     {
+        if (_onAirSceneName is null)
+        {
+            return Result(
+                PlayoutOperation.Next,
+                PlayoutResultCode.Rejected,
+                "NEXT는 송출 중인 장면이 있을 때만 사용할 수 있습니다.");
+        }
+
         if (_options.Mode == PlayoutMode.Disabled)
         {
             return Result(PlayoutOperation.Next, PlayoutResultCode.Rejected, "송출 기능이 비활성화되어 있습니다.");
@@ -689,6 +865,12 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
         _processSnapshot = CaptureProcessSnapshot();
         if (!IsProcessEligible(_processSnapshot))
         {
+            if (_onAirSceneName is not null)
+            {
+                await AbandonCurrentSessionAsync().ConfigureAwait(false);
+                return MarkOutcomeUnknown(operation);
+            }
+
             if (_session is not null)
             {
                 if (!await ReleaseSessionAsync(cancellationToken).ConfigureAwait(false))
@@ -704,6 +886,32 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
             return Result(operation, PlayoutResultCode.Unavailable, ProcessUnavailableMessage(_processSnapshot));
         }
 
+        if (HasConnectedProcessGenerationChanged(_processSnapshot))
+        {
+            if (_onAirSceneName is not null)
+            {
+                await AbandonCurrentSessionAsync().ConfigureAwait(false);
+                return MarkOutcomeUnknown(operation);
+            }
+
+            if (!await ReleaseSessionAsync(cancellationToken).ConfigureAwait(false))
+            {
+                return MarkOutcomeUnknown(operation);
+            }
+
+            _preparedCue = null;
+            _onAirSceneName = null;
+            ScheduleReconnect();
+            _connectionState = _connectionRequested
+                ? PlayoutConnectionState.Reconnecting
+                : PlayoutConnectionState.Disconnected;
+            PublishStatus(ProcessChangedMessage);
+            return Result(
+                operation,
+                PlayoutResultCode.Unavailable,
+                ProcessChangedMessage);
+        }
+
         if (_session is null)
         {
             return Result(operation, PlayoutResultCode.Unavailable, "Tornado 송출 연결이 필요합니다.");
@@ -719,11 +927,26 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
         CancellationToken cancellationToken)
     {
         var session = _session!;
+        var callbackStarted = 0;
+        var preserveActiveOutput = partialOutcomeUnknown || _onAirSceneName is not null;
+        void CleanupAbandonedOperation()
+        {
+            if (preserveActiveOutput)
+            {
+                session.Abandon();
+            }
+            else
+            {
+                session.Dispose();
+            }
+        }
+
         try
         {
             await _dispatcher!.InvokeAsync(
                 () =>
                 {
+                    Volatile.Write(ref callbackStarted, 1);
                     try
                     {
                         callback(session);
@@ -732,7 +955,7 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
                     {
                         if (_dispatcher.IsQuarantined)
                         {
-                            session.Dispose();
+                            CleanupAbandonedOperation();
                         }
                     }
 
@@ -740,12 +963,20 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
                 },
                 _options.OperationTimeout,
                 cancellationToken,
-                _ => session.Dispose(),
-                () => session.Dispose()).ConfigureAwait(false);
+                _ => CleanupAbandonedOperation(),
+                CleanupAbandonedOperation).ConfigureAwait(false);
             return Result(operation, PlayoutResultCode.Success, SuccessMessage(operation));
         }
         catch (OperationCanceledException)
         {
+            if (preserveActiveOutput && Volatile.Read(ref callbackStarted) != 0)
+            {
+                _session = null;
+                _connectedProcessSnapshot = null;
+                await AbandonSessionOnStaAsync(session).ConfigureAwait(false);
+                return MarkOutcomeUnknown(operation);
+            }
+
             return Result(operation, PlayoutResultCode.Cancelled, "송출 요청이 취소되었습니다.");
         }
         catch (StaOperationTimedOutException)
@@ -758,14 +989,20 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
         }
         catch (Exception)
         {
-            _session = session;
-            if (partialOutcomeUnknown)
+            if (preserveActiveOutput)
+            {
+                _session = null;
+                _connectedProcessSnapshot = null;
+                await AbandonSessionOnStaAsync(session).ConfigureAwait(false);
+                return MarkOutcomeUnknown(operation);
+            }
+
+            _session = session;
+            if (!await RecycleFailedSessionAsync().ConfigureAwait(false))
             {
-                await ReleaseSessionAsync(CancellationToken.None).ConfigureAwait(false);
                 return MarkOutcomeUnknown(operation);
             }
 
-            await RecycleFailedSessionAsync().ConfigureAwait(false);
             return Result(
                 operation,
                 PlayoutResultCode.Failed,
@@ -773,15 +1010,14 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
         }
     }
 
-    private async Task RecycleFailedSessionAsync()
+    private async Task<bool> RecycleFailedSessionAsync()
     {
         _preparedCue = null;
         _onAirSceneName = null;
         var released = await ReleaseSessionAsync(CancellationToken.None).ConfigureAwait(false);
         if (!released)
         {
-            MarkOutcomeUnknown(PlayoutOperation.Disconnect);
-            return;
+            return false;
         }
 
         _connectionState = _connectionRequested
@@ -789,12 +1025,20 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
             : PlayoutConnectionState.Faulted;
         ScheduleReconnect();
         PublishStatus("Tornado 송출 세션을 다시 연결해야 합니다.");
+        return true;
     }
 
     private async Task<bool> ReleaseSessionAsync(CancellationToken cancellationToken)
     {
+        if (_onAirSceneName is not null)
+        {
+            await AbandonCurrentSessionAsync().ConfigureAwait(false);
+            return false;
+        }
+
         var session = _session;
         _session = null;
+        _connectedProcessSnapshot = null;
         if (session is null)
         {
             return true;
@@ -833,6 +1077,45 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
         }
     }
 
+    private async Task<bool> AbandonSessionOnStaAsync(IK3dSession session)
+    {
+        if (_dispatcher is null || _dispatcher.IsQuarantined)
+        {
+            return false;
+        }
+
+        try
+        {
+            await _dispatcher.InvokeAsync(
+                () =>
+                {
+                    session.Abandon();
+                    return true;
+                },
+                _options.DisconnectTimeout,
+                CancellationToken.None,
+                _ => session.Abandon(),
+                () => session.Abandon()).ConfigureAwait(false);
+            return true;
+        }
+        catch
+        {
+            // Never fall back to Disconnect when the physical output state is unknown.
+            return false;
+        }
+    }
+
+    private async Task AbandonCurrentSessionAsync()
+    {
+        var session = _session;
+        _session = null;
+        _connectedProcessSnapshot = null;
+        if (session is not null)
+        {
+            await AbandonSessionOnStaAsync(session).ConfigureAwait(false);
+        }
+    }
+
     private Task<bool> CleanupSessionOnStaAsync(
         IK3dSession session,
         CancellationToken cancellationToken) =>
@@ -895,20 +1178,29 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
         }
         catch (Exception)
         {
-            return new TornadoProcessSnapshot(0, 0, 0);
+            return new TornadoProcessSnapshot(0, 0, 0)
+            {
+                IdentityInspectionSucceeded = false
+            };
         }
     }
 
     private bool IsProcessEligible(TornadoProcessSnapshot snapshot) => _options.Mode switch
     {
         PlayoutMode.Test =>
+            snapshot.IdentityInspectionSucceeded &&
             snapshot.TotalProcessCount == 1 &&
             snapshot.EligibleProcessCount == 1 &&
             snapshot.ProgramProcessCount == 0,
-        PlayoutMode.Live => snapshot.AnyTornadoProcess,
+        PlayoutMode.Live => snapshot.IdentityInspectionSucceeded && snapshot.AnyTornadoProcess,
         _ => false
     };
 
+    private bool HasConnectedProcessGenerationChanged(TornadoProcessSnapshot snapshot) =>
+        _session is not null &&
+        (_connectedProcessSnapshot is null ||
+         !_connectedProcessSnapshot.HasSameProcessGeneration(snapshot));
+
     private bool CanTakeIn() => _options.Mode switch
     {
         PlayoutMode.Test => true,
@@ -925,15 +1217,19 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
         _outcomeUnknown = true;
         _connectionRequested = false;
         _session = null;
+        _connectedProcessSnapshot = null;
         _preparedCue = null;
         _onAirSceneName = null;
         _connectionState = PlayoutConnectionState.OutcomeUnknown;
         PublishStatus("송출 결과를 확인할 수 없습니다. 앱을 다시 시작하고 출력 상태를 확인하세요.");
-        return Result(
+        return ExistingOutcomeUnknown(operation);
+    }
+
+    private PlayoutResult ExistingOutcomeUnknown(PlayoutOperation operation) =>
+        Result(
             operation,
             PlayoutResultCode.OutcomeUnknown,
             "송출 결과를 확인할 수 없습니다. 앱을 다시 시작하고 출력 상태를 확인하세요.");
-    }
 
     private void ScheduleReconnect() =>
         _nextReconnectAt = _timeProvider.GetUtcNow() + _options.ReconnectDelay;
@@ -987,7 +1283,10 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
             _onAirSceneName,
             message,
             _timeProvider.GetUtcNow(),
-            Interlocked.Increment(ref _sequence));
+            Interlocked.Increment(ref _sequence))
+        {
+            OperationTimeoutMilliseconds = (int)_options.OperationTimeout.TotalMilliseconds
+        };
     }
 
     private PlayoutResult Result(