feat: verify Tornado PGM cut sequence

This commit is contained in:
2026-07-10 16:05:06 +09:00
parent 930424b752
commit d12a5d8095
15 changed files with 4344 additions and 74 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -29,6 +29,19 @@ internal interface IK3dSession : IDisposable
void Abandon();
}
internal interface IK3dGuardedConnectSession : IK3dSession
{
void ConnectGuarded(
ValidatedPlayoutOptions options,
Func<bool> finalSafetyCheck,
IK3dSdkDispatchLatch dispatchLatch);
}
internal interface IK3dSdkDispatchLatch
{
IDisposable? TryClaim();
}
internal interface IK3dSessionFactory
{
IK3dSession Create();
@@ -102,7 +115,7 @@ internal sealed class RuntimeComObjectReleaser : ILateBoundComObjectReleaser
}
}
internal sealed class DynamicK3dSession : IK3dSession
internal sealed class DynamicK3dSession : IK3dGuardedConnectSession
{
private readonly ILateBoundComActivator _activator;
private readonly ILateBoundComObjectReleaser _releaser;
@@ -164,6 +177,27 @@ internal sealed class DynamicK3dSession : IK3dSession
}
public void Connect(ValidatedPlayoutOptions options)
=> ConnectCore(options, finalSafetyCheck: null, disconnectOnFailure: true);
public void ConnectGuarded(
ValidatedPlayoutOptions options,
Func<bool> finalSafetyCheck,
IK3dSdkDispatchLatch dispatchLatch)
{
ArgumentNullException.ThrowIfNull(finalSafetyCheck);
ArgumentNullException.ThrowIfNull(dispatchLatch);
ConnectCore(
options,
finalSafetyCheck,
disconnectOnFailure: false,
dispatchLatch: dispatchLatch);
}
private void ConnectCore(
ValidatedPlayoutOptions options,
Func<bool>? finalSafetyCheck,
bool disconnectOnFailure,
IK3dSdkDispatchLatch? dispatchLatch = null)
{
ArgumentNullException.ThrowIfNull(options);
EnsureThread();
@@ -178,22 +212,37 @@ internal sealed class DynamicK3dSession : IK3dSession
{
_eventHandler = _activator.Create(K3dComConstants.KaEventHandlerClassGuid);
_engine = _activator.Create(K3dComConstants.KaEngineClassGuid);
if (Interlocked.CompareExchange(ref _ktapDispatchGate, 1, 0) != 0)
if (finalSafetyCheck is not null && !finalSafetyCheck())
{
throw new InvalidOperationException(
"The K3D connection was abandoned before KTAP dispatch.");
throw new K3dConnectSafetyException();
}
object? result;
using (var dispatchClaim = dispatchLatch?.TryClaim())
{
if (dispatchLatch is not null && dispatchClaim is null)
{
throw new K3dConnectSafetyException();
}
if (Interlocked.CompareExchange(ref _ktapDispatchGate, 1, 0) != 0)
{
throw new InvalidOperationException(
"The K3D connection was abandoned before KTAP dispatch.");
}
ktapConnectAttempted = true;
SetKtapConnectState(PlayoutKtapConnectState.Attempted);
result = Invoke(
_engine,
"KTAPConnect",
options.TcpMode,
options.Host,
options.Port,
options.ClientPort,
_eventHandler);
}
ktapConnectAttempted = true;
SetKtapConnectState(PlayoutKtapConnectState.Attempted);
var result = Invoke(
_engine,
"KTAPConnect",
options.TcpMode,
options.Host,
options.Port,
options.ClientPort,
_eventHandler);
var hresult = result is null
? 0
: Convert.ToInt32(result, CultureInfo.InvariantCulture);
@@ -206,9 +255,23 @@ internal sealed class DynamicK3dSession : IK3dSession
SetKtapConnectState(PlayoutKtapConnectState.AcceptedUnconfirmed);
var outputChannel = options.OutputChannel;
_player = outputChannel is { } channel
? InvokeRequired(_engine, "GetScenePlayerOnChannel", channel)
: InvokeRequired(_engine, "GetScenePlayer");
if (finalSafetyCheck is not null && !finalSafetyCheck())
{
throw new K3dConnectSafetyException();
}
using (var dispatchClaim = dispatchLatch?.TryClaim())
{
if (dispatchLatch is not null && dispatchClaim is null)
{
throw new K3dConnectSafetyException();
}
_player = outputChannel is { } channel
? InvokeRequired(_engine, "GetScenePlayerOnChannel", channel)
: InvokeRequired(_engine, "GetScenePlayer");
}
_outputChannel = outputChannel;
}
catch (Exception exception)
@@ -219,7 +282,7 @@ internal sealed class DynamicK3dSession : IK3dSession
}
var originalFailure = ExceptionDispatchInfo.Capture(exception);
if (ktapConnectAttempted && _engine is not null)
if (disconnectOnFailure && ktapConnectAttempted && _engine is not null)
{
try
{
@@ -553,3 +616,5 @@ internal sealed class DynamicK3dSession : IK3dSession
}
}
}
internal sealed class K3dConnectSafetyException : Exception;

View File

@@ -14,7 +14,8 @@ internal interface IStaDispatcher : IAsyncDisposable
TimeSpan timeout,
CancellationToken cancellationToken,
Action<T>? abandonedResultCleanup = null,
Action? abandonedFailureCleanup = null);
Action? abandonedFailureCleanup = null,
Action? preventFutureDispatch = null);
}
internal interface IStaDispatcherFactory
@@ -75,7 +76,8 @@ internal sealed class StaDispatcher : IStaDispatcher
TimeSpan timeout,
CancellationToken cancellationToken,
Action<T>? abandonedResultCleanup = null,
Action? abandonedFailureCleanup = null)
Action? abandonedFailureCleanup = null,
Action? preventFutureDispatch = null)
{
ArgumentNullException.ThrowIfNull(callback);
if (timeout <= TimeSpan.Zero)
@@ -110,7 +112,7 @@ internal sealed class StaDispatcher : IStaDispatcher
return await completion.ConfigureAwait(false);
}
if (item.TryTimeout())
if (item.TryTimeout(preventFutureDispatch))
{
Quarantine();
}
@@ -295,7 +297,7 @@ internal sealed class StaDispatcher : IStaDispatcher
}
}
public bool TryTimeout()
public bool TryTimeout(Action? preventFutureDispatch)
{
while (true)
{
@@ -307,6 +309,15 @@ internal sealed class StaDispatcher : IStaDispatcher
if (Interlocked.CompareExchange(ref _executionState, 2, state) == state)
{
try
{
preventFutureDispatch?.Invoke();
}
catch
{
// A timeout must still complete and quarantine the dispatcher.
}
TimeoutCore();
return true;
}

View File

@@ -19,6 +19,9 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
private readonly IStaDispatcher? _dispatcher;
private readonly ILiveAuthorization _liveAuthorization;
private readonly TimeProvider _timeProvider;
private readonly Func<bool>? _finalConnectSafetyCheck;
private readonly bool _abandonOnTargetMismatch;
private readonly Action? _beforeSdkDispatchClaim;
private readonly SemaphoreSlim _gate = new(1, 1);
private readonly CancellationTokenSource _monitorCancellation = new();
private readonly object _statusSync = new();
@@ -36,6 +39,7 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
private int _lastKtapConnectState;
private bool _connectionRequested;
private volatile bool _outcomeUnknown;
private volatile bool _quarantineIncomplete;
private volatile bool _disposed;
private int _disposeStarted;
@@ -47,7 +51,10 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
IStaDispatcher? dispatcher,
ILiveAuthorization liveAuthorization,
TimeProvider timeProvider,
bool startMonitor = true)
bool startMonitor = true,
Func<bool>? finalConnectSafetyCheck = null,
bool abandonOnTargetMismatch = false,
Action? beforeSdkDispatchClaim = null)
{
_options = options;
_sessionFactory = sessionFactory;
@@ -56,6 +63,9 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
_dispatcher = dispatcher;
_liveAuthorization = liveAuthorization;
_timeProvider = timeProvider;
_finalConnectSafetyCheck = finalConnectSafetyCheck;
_abandonOnTargetMismatch = abandonOnTargetMismatch;
_beforeSdkDispatchClaim = beforeSdkDispatchClaim;
_connectionState = options.Mode switch
{
PlayoutMode.Disabled => PlayoutConnectionState.Disabled,
@@ -135,9 +145,18 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
var session = _session;
_session = null;
var quarantineCompleted = !_quarantineIncomplete;
if (session is not null)
{
await AbandonSessionOnStaAsync(session).ConfigureAwait(false);
quarantineCompleted =
await AbandonSessionOnStaAsync(session).ConfigureAwait(false) &&
quarantineCompleted;
}
if (!quarantineCompleted || _quarantineIncomplete)
{
throw new InvalidOperationException(
"The playout session could not be quarantined.");
}
PublishStatus("송출 결과를 확인할 수 없어 연결을 격리했습니다. 출력 상태를 직접 확인하세요.");
@@ -496,6 +515,7 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
PublishStatus(reconnecting ? "Tornado 테스트 송출에 다시 연결하는 중입니다." : "Tornado 송출에 연결하는 중입니다.");
IK3dSession? pendingSession = null;
var dispatchLatch = new SdkDispatchLatch(_beforeSdkDispatchClaim);
try
{
var session = _sessionFactory.Create();
@@ -505,19 +525,53 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
{
try
{
session.Connect(_options);
if (_finalConnectSafetyCheck is null)
{
session.Connect(_options);
}
else if (session is IK3dGuardedConnectSession guardedSession)
{
guardedSession.ConnectGuarded(
_options,
IsExactTargetSafeForSdkDispatch,
dispatchLatch);
}
else
{
throw new K3dConnectSafetyException();
}
return session;
}
catch
{
session.Dispose();
if (_abandonOnTargetMismatch)
{
AbandonSessionInline(session);
}
else
{
session.Dispose();
}
throw;
}
},
_options.ConnectTimeout,
cancellationToken,
static lateSession => lateSession.Dispose(),
() => Volatile.Read(ref pendingSession)?.TryPreventKtapConnect()).ConfigureAwait(false);
lateSession =>
{
if (_abandonOnTargetMismatch)
{
AbandonSessionInline(lateSession);
}
else
{
lateSession.Dispose();
}
},
() => Volatile.Read(ref pendingSession)?.TryPreventKtapConnect(),
dispatchLatch.PreventFutureDispatch).ConfigureAwait(false);
CaptureKtapEvidence(_session);
@@ -525,6 +579,12 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
if (!IsProcessEligible(_processSnapshot) ||
!processSnapshotBeforeConnect.HasSameProcessGeneration(_processSnapshot))
{
if (_abandonOnTargetMismatch)
{
await AbandonCurrentSessionAsync().ConfigureAwait(false);
return MarkOutcomeUnknown(PlayoutOperation.Connect);
}
if (!await ReleaseSessionAsync(CancellationToken.None).ConfigureAwait(false))
{
return MarkOutcomeUnknown(PlayoutOperation.Disconnect);
@@ -557,25 +617,52 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
}
catch (OperationCanceledException)
{
PreventLateKtapDispatchAndCapture(Volatile.Read(ref pendingSession));
var cancelledSession = Volatile.Read(ref pendingSession);
PreventLateKtapDispatchAndCapture(cancelledSession);
if (_abandonOnTargetMismatch &&
cancelledSession is not null &&
cancelledSession.LastKtapConnectState != PlayoutKtapConnectState.NotAttempted)
{
_session = null;
return MarkOutcomeUnknown(PlayoutOperation.Connect);
}
_connectionState = PlayoutConnectionState.Disconnected;
PublishStatus("송출 연결 요청이 취소되었습니다.");
return Result(PlayoutOperation.Connect, PlayoutResultCode.Cancelled, "송출 요청이 취소되었습니다.");
}
catch (StaOperationTimedOutException)
{
if (_abandonOnTargetMismatch)
{
_quarantineIncomplete = true;
}
PreventLateKtapDispatchAndCapture(Volatile.Read(ref pendingSession));
return MarkOutcomeUnknown(PlayoutOperation.Connect);
}
catch (StaDispatcherQuarantinedException)
{
if (_abandonOnTargetMismatch)
{
_quarantineIncomplete = true;
}
PreventLateKtapDispatchAndCapture(Volatile.Read(ref pendingSession));
return MarkOutcomeUnknown(PlayoutOperation.Connect);
}
catch (Exception)
{
CaptureKtapEvidence(Volatile.Read(ref pendingSession));
var failedSession = Volatile.Read(ref pendingSession);
CaptureKtapEvidence(failedSession);
_session = null;
if (_abandonOnTargetMismatch &&
failedSession is not null &&
failedSession.LastKtapConnectState != PlayoutKtapConnectState.NotAttempted)
{
return MarkOutcomeUnknown(PlayoutOperation.Connect);
}
ScheduleReconnect();
_connectionState = reconnecting
? PlayoutConnectionState.Reconnecting
@@ -683,7 +770,9 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
var result = await RunSessionOperationAsync(
PlayoutOperation.Prepare,
session => session.Prepare(cue, _options.LayoutIndex),
(session, dispatchLatch) => ExecuteSdkDispatch(
dispatchLatch,
() => session.Prepare(cue, _options.LayoutIndex)),
partialOutcomeUnknown: false,
cancellationToken).ConfigureAwait(false);
if (result.IsSuccess)
@@ -736,7 +825,9 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
var prepared = _preparedCue;
var result = await RunSessionOperationAsync(
PlayoutOperation.TakeIn,
session => session.Play(_options.LayoutIndex),
(session, dispatchLatch) => ExecuteSdkDispatch(
dispatchLatch,
() => session.Play(_options.LayoutIndex)),
partialOutcomeUnknown: true,
cancellationToken).ConfigureAwait(false);
if (result.IsSuccess)
@@ -802,10 +893,14 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
var result = await RunSessionOperationAsync(
PlayoutOperation.Next,
session =>
(session, dispatchLatch) =>
{
session.Prepare(resolved, _options.LayoutIndex);
session.Play(_options.LayoutIndex);
ExecuteSdkDispatch(
dispatchLatch,
() => session.Prepare(resolved, _options.LayoutIndex));
ExecuteSdkDispatch(
dispatchLatch,
() => session.Play(_options.LayoutIndex));
},
partialOutcomeUnknown: true,
cancellationToken).ConfigureAwait(false);
@@ -856,7 +951,9 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
var result = await RunSessionOperationAsync(
PlayoutOperation.TakeOut,
session => session.TakeOut(_options.LayoutIndex, scope),
(session, dispatchLatch) => ExecuteSdkDispatch(
dispatchLatch,
() => session.TakeOut(_options.LayoutIndex, scope)),
partialOutcomeUnknown: true,
cancellationToken).ConfigureAwait(false);
if (result.IsSuccess)
@@ -881,6 +978,12 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
_processSnapshot = CaptureProcessSnapshot();
if (!IsProcessEligible(_processSnapshot))
{
if (_abandonOnTargetMismatch && _session is not null)
{
await AbandonCurrentSessionAsync().ConfigureAwait(false);
return MarkOutcomeUnknown(operation);
}
if (_onAirSceneName is not null)
{
await AbandonCurrentSessionAsync().ConfigureAwait(false);
@@ -904,6 +1007,12 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
if (HasConnectedProcessGenerationChanged(_processSnapshot))
{
if (_abandonOnTargetMismatch)
{
await AbandonCurrentSessionAsync().ConfigureAwait(false);
return MarkOutcomeUnknown(operation);
}
if (_onAirSceneName is not null)
{
await AbandonCurrentSessionAsync().ConfigureAwait(false);
@@ -938,18 +1047,22 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
private async Task<PlayoutResult> RunSessionOperationAsync(
PlayoutOperation operation,
Action<IK3dSession> callback,
Action<IK3dSession, SdkDispatchLatch> callback,
bool partialOutcomeUnknown,
CancellationToken cancellationToken)
{
var session = _session!;
var dispatchLatch = new SdkDispatchLatch(_beforeSdkDispatchClaim);
var callbackStarted = 0;
var preserveActiveOutput = partialOutcomeUnknown || _onAirSceneName is not null;
var preserveActiveOutput =
_abandonOnTargetMismatch ||
partialOutcomeUnknown ||
_onAirSceneName is not null;
void CleanupAbandonedOperation()
{
if (preserveActiveOutput)
{
session.Abandon();
AbandonSessionInline(session);
}
else
{
@@ -965,7 +1078,7 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
Volatile.Write(ref callbackStarted, 1);
try
{
callback(session);
callback(session, dispatchLatch);
}
finally
{
@@ -980,7 +1093,8 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
_options.OperationTimeout,
cancellationToken,
_ => CleanupAbandonedOperation(),
CleanupAbandonedOperation).ConfigureAwait(false);
CleanupAbandonedOperation,
dispatchLatch.PreventFutureDispatch).ConfigureAwait(false);
return Result(operation, PlayoutResultCode.Success, SuccessMessage(operation));
}
catch (OperationCanceledException)
@@ -997,10 +1111,20 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
}
catch (StaOperationTimedOutException)
{
if (preserveActiveOutput)
{
_quarantineIncomplete = true;
}
return MarkOutcomeUnknown(operation);
}
catch (StaDispatcherQuarantinedException)
{
if (preserveActiveOutput)
{
_quarantineIncomplete = true;
}
return MarkOutcomeUnknown(operation);
}
catch (Exception)
@@ -1062,6 +1186,11 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
if (_dispatcher is null || _dispatcher.IsQuarantined)
{
if (_abandonOnTargetMismatch)
{
_quarantineIncomplete = true;
}
return false;
}
@@ -1074,6 +1203,11 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
{
if (_dispatcher.IsQuarantined)
{
if (_abandonOnTargetMismatch)
{
_quarantineIncomplete = true;
}
return false;
}
@@ -1084,11 +1218,21 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
}
catch (Exception)
{
if (_abandonOnTargetMismatch)
{
_quarantineIncomplete = true;
}
return false;
}
}
catch (Exception)
{
if (_abandonOnTargetMismatch)
{
_quarantineIncomplete = true;
}
return false;
}
}
@@ -1097,6 +1241,7 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
{
if (_dispatcher is null || _dispatcher.IsQuarantined)
{
_quarantineIncomplete = true;
return false;
}
@@ -1105,18 +1250,19 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
await _dispatcher.InvokeAsync(
() =>
{
session.Abandon();
AbandonSessionInline(session);
return true;
},
_options.DisconnectTimeout,
CancellationToken.None,
_ => session.Abandon(),
() => session.Abandon()).ConfigureAwait(false);
_ => AbandonSessionInline(session),
() => AbandonSessionInline(session)).ConfigureAwait(false);
return true;
}
catch
{
// Never fall back to Disconnect when the physical output state is unknown.
_quarantineIncomplete = true;
return false;
}
}
@@ -1134,13 +1280,39 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
private Task<bool> CleanupSessionOnStaAsync(
IK3dSession session,
CancellationToken cancellationToken) =>
_dispatcher!.InvokeAsync(
CancellationToken cancellationToken)
{
var dispatchLatch = new SdkDispatchLatch(_beforeSdkDispatchClaim);
return _dispatcher!.InvokeAsync(
() =>
{
if (_abandonOnTargetMismatch)
{
var disconnected = false;
try
{
ExecuteSdkDispatch(dispatchLatch, session.Disconnect);
disconnected = true;
}
catch
{
AbandonSessionInline(session);
throw;
}
finally
{
if (disconnected)
{
session.Dispose();
}
}
return true;
}
try
{
session.Disconnect();
ExecuteSdkDispatch(dispatchLatch, session.Disconnect);
}
finally
{
@@ -1150,7 +1322,130 @@ internal sealed class TornadoPlayoutEngine : IPlayoutEngine
return true;
},
_options.DisconnectTimeout,
cancellationToken);
cancellationToken,
preventFutureDispatch: dispatchLatch.PreventFutureDispatch);
}
private void EnsureExactTarget()
{
if (_finalConnectSafetyCheck is not null &&
!IsExactTargetSafeForSdkDispatch())
{
throw new K3dConnectSafetyException();
}
}
private void ExecuteSdkDispatch(
SdkDispatchLatch dispatchLatch,
Action sdkCall)
{
EnsureExactTarget();
using var dispatchClaim = dispatchLatch.TryClaim();
if (dispatchClaim is null)
{
throw new K3dConnectSafetyException();
}
sdkCall();
EnsureExactTarget();
}
private bool IsExactTargetSafeForSdkDispatch()
{
var dispatcher = _dispatcher;
if (dispatcher is null || dispatcher.IsQuarantined)
{
return false;
}
try
{
return _finalConnectSafetyCheck is not null &&
_finalConnectSafetyCheck() &&
!dispatcher.IsQuarantined;
}
catch
{
return false;
}
}
private void AbandonSessionInline(IK3dSession session)
{
try
{
session.Abandon();
}
catch
{
_quarantineIncomplete = true;
throw;
}
}
private sealed class SdkDispatchLatch(Action? beforeClaim) : IK3dSdkDispatchLatch
{
private const int Open = 0;
private const int Claimed = 1;
private const int Prevented = 2;
private const int ClaimedAndPrevented = 3;
private int _state;
public IDisposable? TryClaim()
{
beforeClaim?.Invoke();
return Interlocked.CompareExchange(ref _state, Claimed, Open) == Open
? new DispatchClaim(this)
: null;
}
public void PreventFutureDispatch()
{
while (true)
{
var state = Volatile.Read(ref _state);
var next = state switch
{
Open => Prevented,
Claimed => ClaimedAndPrevented,
_ => state
};
if (next == state ||
Interlocked.CompareExchange(ref _state, next, state) == state)
{
return;
}
}
}
private void Release()
{
while (true)
{
var state = Volatile.Read(ref _state);
var next = state switch
{
Claimed => Open,
ClaimedAndPrevented => Prevented,
_ => state
};
if (next == state ||
Interlocked.CompareExchange(ref _state, next, state) == state)
{
return;
}
}
}
private sealed class DispatchClaim(SdkDispatchLatch owner) : IDisposable
{
private SdkDispatchLatch? _owner = owner;
public void Dispose() =>
Interlocked.Exchange(ref _owner, null)?.Release();
}
}
private async Task MonitorLoopAsync()
{