feat: harden isolated Tornado test workflow
This commit is contained in:
491
tools/MBN_STOCK_WEBVIEW.PlayoutSmoke/IsolatedTestCommand.cs
Normal file
491
tools/MBN_STOCK_WEBVIEW.PlayoutSmoke/IsolatedTestCommand.cs
Normal file
@@ -0,0 +1,491 @@
|
||||
using System.Collections;
|
||||
using MBN_STOCK_WEBVIEW.Core.Playout;
|
||||
using MBN_STOCK_WEBVIEW.Playout;
|
||||
using MBN_STOCK_WEBVIEW.Playout.Configuration;
|
||||
|
||||
namespace MBN_STOCK_WEBVIEW.PlayoutSmoke;
|
||||
|
||||
internal sealed record IsolatedTestCommandPlan(
|
||||
SmokeCommandKind Kind,
|
||||
PlayoutOptions Options,
|
||||
PlayoutCue? PrepareCue,
|
||||
PlayoutCue? NextCue,
|
||||
TimeSpan ObservationDuration);
|
||||
|
||||
internal sealed class IsolatedTestPreflightException : Exception
|
||||
{
|
||||
public IsolatedTestPreflightException(string errorCode)
|
||||
: base("The isolated Test command was rejected before engine creation.")
|
||||
{
|
||||
ErrorCode = errorCode;
|
||||
}
|
||||
|
||||
public string ErrorCode { get; }
|
||||
}
|
||||
|
||||
internal static class IsolatedTestCommandPreflight
|
||||
{
|
||||
internal const string EnvironmentVariablePrefix = "MBN_STOCK_PLAYOUT_";
|
||||
|
||||
public static IsolatedTestCommandPlan Create(SmokeCommandInvocation invocation)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(invocation);
|
||||
if (invocation.Kind is not (
|
||||
SmokeCommandKind.TestPlan or
|
||||
SmokeCommandKind.TestConnect or
|
||||
SmokeCommandKind.TestSequence))
|
||||
{
|
||||
throw Rejected("invalid-test-command");
|
||||
}
|
||||
|
||||
if (HasPlayoutEnvironmentOverride(Environment.GetEnvironmentVariables()))
|
||||
{
|
||||
throw Rejected("environment-override-present");
|
||||
}
|
||||
|
||||
var configPath = ValidateConfigPath(invocation.ConfigPath);
|
||||
PlayoutOptions options;
|
||||
try
|
||||
{
|
||||
options = PlayoutOptionsLoader.LoadFileOnly(configPath);
|
||||
}
|
||||
catch (Exception exception) when (
|
||||
exception is PlayoutConfigurationException or
|
||||
ArgumentException or
|
||||
IOException or
|
||||
UnauthorizedAccessException)
|
||||
{
|
||||
throw Rejected("configuration-rejected");
|
||||
}
|
||||
|
||||
if (options.Mode != PlayoutMode.Test || options.TrustedLiveOutputEnabled)
|
||||
{
|
||||
throw Rejected("test-mode-required");
|
||||
}
|
||||
|
||||
// This one-shot diagnostic never reconnects or replays a command. The runtime
|
||||
// still checks process eligibility before every operation.
|
||||
options.ReconnectEnabled = false;
|
||||
options.MaximumReconnectAttempts = 0;
|
||||
|
||||
PlayoutCue? prepareCue = null;
|
||||
PlayoutCue? nextCue = null;
|
||||
if (invocation.Kind is SmokeCommandKind.TestPlan or SmokeCommandKind.TestSequence)
|
||||
{
|
||||
if (invocation.PrepareSceneCode is null || invocation.NextSceneCode is null ||
|
||||
invocation.ObservationMilliseconds is not (>= 1_000 and <= 30_000) ||
|
||||
!SmokeCommandLine.IsSafeSceneCode(invocation.PrepareSceneCode) ||
|
||||
!SmokeCommandLine.IsSafeSceneCode(invocation.NextSceneCode))
|
||||
{
|
||||
throw Rejected("scene-code-rejected");
|
||||
}
|
||||
|
||||
prepareCue = CreateCue(invocation.PrepareSceneCode);
|
||||
nextCue = CreateCue(invocation.NextSceneCode);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
PlayoutEngineFactory.ValidateIsolatedTestCommand(
|
||||
options,
|
||||
prepareCue is null ? [] : [prepareCue, nextCue!]);
|
||||
}
|
||||
catch (Exception exception) when (
|
||||
exception is PlayoutConfigurationException or ArgumentException)
|
||||
{
|
||||
throw Rejected("safety-gate-rejected");
|
||||
}
|
||||
|
||||
return new IsolatedTestCommandPlan(
|
||||
invocation.Kind,
|
||||
options,
|
||||
prepareCue,
|
||||
nextCue,
|
||||
TimeSpan.FromMilliseconds(invocation.ObservationMilliseconds ?? 0));
|
||||
}
|
||||
|
||||
internal static bool HasPlayoutEnvironmentOverride(IDictionary environment)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(environment);
|
||||
foreach (var key in environment.Keys)
|
||||
{
|
||||
if (key is string name &&
|
||||
name.StartsWith(EnvironmentVariablePrefix, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static string ValidateConfigPath(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value) || !Path.IsPathFullyQualified(value))
|
||||
{
|
||||
throw Rejected("config-file-rejected");
|
||||
}
|
||||
|
||||
string fullPath;
|
||||
FileAttributes attributes;
|
||||
try
|
||||
{
|
||||
fullPath = Path.GetFullPath(value);
|
||||
var root = Path.GetPathRoot(fullPath);
|
||||
if (string.IsNullOrEmpty(root) ||
|
||||
fullPath.StartsWith("\\\\", StringComparison.Ordinal) ||
|
||||
new DriveInfo(root).DriveType != DriveType.Fixed)
|
||||
{
|
||||
throw Rejected("config-file-rejected");
|
||||
}
|
||||
|
||||
if (!File.Exists(fullPath))
|
||||
{
|
||||
throw Rejected("config-file-rejected");
|
||||
}
|
||||
|
||||
attributes = File.GetAttributes(fullPath);
|
||||
}
|
||||
catch (IsolatedTestPreflightException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception) when (
|
||||
exception is ArgumentException or
|
||||
IOException or
|
||||
NotSupportedException or
|
||||
System.Security.SecurityException or
|
||||
UnauthorizedAccessException)
|
||||
{
|
||||
throw Rejected("config-file-rejected");
|
||||
}
|
||||
|
||||
if ((attributes & (FileAttributes.Directory | FileAttributes.ReparsePoint)) != 0)
|
||||
{
|
||||
throw Rejected("config-file-rejected");
|
||||
}
|
||||
|
||||
return fullPath;
|
||||
}
|
||||
|
||||
private static PlayoutCue CreateCue(string sceneCode) =>
|
||||
new($"{sceneCode}.t2s", sceneCode);
|
||||
|
||||
private static IsolatedTestPreflightException Rejected(string errorCode) => new(errorCode);
|
||||
}
|
||||
|
||||
internal sealed record SafeCommandStep(
|
||||
string Phase,
|
||||
string Operation,
|
||||
string Code);
|
||||
|
||||
internal sealed record SafeCleanupReport(
|
||||
bool DisconnectAttempted,
|
||||
string? DisconnectCode,
|
||||
bool QuarantineAttempted,
|
||||
bool QuarantineCompleted,
|
||||
bool AdapterDisposed);
|
||||
|
||||
internal sealed record SafeTestCommandReport(
|
||||
string Command,
|
||||
string Mode,
|
||||
bool ConnectRequestIssued,
|
||||
bool? ComActivationAttempted,
|
||||
bool Completed,
|
||||
bool OutcomeUnknown,
|
||||
bool OutputMayBeActive,
|
||||
string? StoppedAfter,
|
||||
IReadOnlyList<SafeCommandStep> Steps,
|
||||
SafeCleanupReport Cleanup);
|
||||
|
||||
internal static class IsolatedTestCommandExecutor
|
||||
{
|
||||
public static Task<SafeTestCommandReport> ExecuteAsync(
|
||||
IsolatedTestCommandPlan plan,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
ExecuteAsync(
|
||||
plan,
|
||||
PlayoutEngineFactory.Create,
|
||||
static (delay, token) => Task.Delay(delay, token),
|
||||
cancellationToken);
|
||||
|
||||
internal static async Task<SafeTestCommandReport> ExecuteAsync(
|
||||
IsolatedTestCommandPlan plan,
|
||||
Func<PlayoutOptions, IPlayoutEngine> engineFactory,
|
||||
Func<TimeSpan, CancellationToken, Task> observationDelay,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(plan);
|
||||
ArgumentNullException.ThrowIfNull(engineFactory);
|
||||
ArgumentNullException.ThrowIfNull(observationDelay);
|
||||
if (plan.Kind is not (SmokeCommandKind.TestConnect or SmokeCommandKind.TestSequence))
|
||||
{
|
||||
throw new ArgumentException("The plan is not executable.", nameof(plan));
|
||||
}
|
||||
|
||||
var steps = new List<SafeCommandStep>();
|
||||
IPlayoutEngine? engine = null;
|
||||
var connectRequestIssued = false;
|
||||
var connectResultReceived = false;
|
||||
bool? comActivationAttempted = false;
|
||||
var completed = false;
|
||||
var outcomeUnknown = false;
|
||||
string? stoppedAfter = null;
|
||||
var disconnectAttempted = false;
|
||||
string? disconnectCode = null;
|
||||
var adapterDisposed = false;
|
||||
var quarantineAttempted = false;
|
||||
var quarantineCompleted = false;
|
||||
var normalDisconnectCompleted = false;
|
||||
var outputMayBeActive = false;
|
||||
var takeOutAttempted = false;
|
||||
|
||||
try
|
||||
{
|
||||
engine = engineFactory(plan.Options);
|
||||
do
|
||||
{
|
||||
connectRequestIssued = true;
|
||||
var connect = await engine.ConnectAsync(cancellationToken).ConfigureAwait(false);
|
||||
connectResultReceived = true;
|
||||
AddStep(steps, "command", connect);
|
||||
comActivationAttempted = InferComActivationAttempt(connect.Code);
|
||||
if (!connect.IsSuccess)
|
||||
{
|
||||
stoppedAfter = connect.Operation.ToString();
|
||||
outcomeUnknown = IsOutcomeUnknown(connect);
|
||||
break;
|
||||
}
|
||||
|
||||
if (plan.Kind == SmokeCommandKind.TestSequence)
|
||||
{
|
||||
var prepare = await engine.PrepareAsync(plan.PrepareCue!, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
AddStep(steps, "command", prepare);
|
||||
if (!prepare.IsSuccess)
|
||||
{
|
||||
stoppedAfter = prepare.Operation.ToString();
|
||||
outcomeUnknown = IsOutcomeUnknown(prepare);
|
||||
break;
|
||||
}
|
||||
|
||||
// Once TAKE IN is issued, an exception or ambiguous result can mean
|
||||
// the Test output became active even though no success was observed.
|
||||
outputMayBeActive = true;
|
||||
var takeIn = await engine.TakeInAsync(cancellationToken).ConfigureAwait(false);
|
||||
AddStep(steps, "command", takeIn);
|
||||
if (!takeIn.IsSuccess)
|
||||
{
|
||||
stoppedAfter = takeIn.Operation.ToString();
|
||||
outcomeUnknown = IsOutcomeUnknown(takeIn);
|
||||
if (!outcomeUnknown)
|
||||
{
|
||||
outputMayBeActive = false;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
if (!await ObserveAsync().ConfigureAwait(false))
|
||||
{
|
||||
stoppedAfter = "Observation";
|
||||
break;
|
||||
}
|
||||
|
||||
var next = await engine.NextAsync(plan.NextCue!, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
AddStep(steps, "command", next);
|
||||
if (!next.IsSuccess)
|
||||
{
|
||||
stoppedAfter = next.Operation.ToString();
|
||||
outcomeUnknown = IsOutcomeUnknown(next);
|
||||
break;
|
||||
}
|
||||
|
||||
if (!await ObserveAsync().ConfigureAwait(false))
|
||||
{
|
||||
stoppedAfter = "Observation";
|
||||
break;
|
||||
}
|
||||
|
||||
takeOutAttempted = true;
|
||||
var takeOut = await engine.TakeOutAsync(
|
||||
PlayoutTakeOutScope.All,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
AddStep(steps, "command", takeOut);
|
||||
if (!takeOut.IsSuccess)
|
||||
{
|
||||
stoppedAfter = takeOut.Operation.ToString();
|
||||
// TAKE IN succeeded, so every non-successful TAKE OUT leaves the
|
||||
// physical Test output state unsafe to infer from its result code.
|
||||
outcomeUnknown = true;
|
||||
break;
|
||||
}
|
||||
|
||||
outputMayBeActive = false;
|
||||
}
|
||||
|
||||
disconnectAttempted = true;
|
||||
var disconnect = await engine.DisconnectAsync(cancellationToken).ConfigureAwait(false);
|
||||
disconnectCode = disconnect.Code.ToString();
|
||||
AddStep(steps, "command", disconnect);
|
||||
normalDisconnectCompleted = true;
|
||||
if (!disconnect.IsSuccess)
|
||||
{
|
||||
stoppedAfter = disconnect.Operation.ToString();
|
||||
outcomeUnknown = IsOutcomeUnknown(disconnect);
|
||||
break;
|
||||
}
|
||||
|
||||
completed = true;
|
||||
}
|
||||
while (false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
stoppedAfter ??= connectRequestIssued ? "Internal" : "EngineCreation";
|
||||
if (!connectResultReceived)
|
||||
{
|
||||
comActivationAttempted = connectRequestIssued ? null : false;
|
||||
}
|
||||
|
||||
outcomeUnknown = connectRequestIssued;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (engine is not null)
|
||||
{
|
||||
if (outputMayBeActive && !takeOutAttempted && !outcomeUnknown)
|
||||
{
|
||||
takeOutAttempted = true;
|
||||
try
|
||||
{
|
||||
var cleanupTakeOut = await engine.TakeOutAsync(
|
||||
PlayoutTakeOutScope.All,
|
||||
CancellationToken.None).ConfigureAwait(false);
|
||||
AddStep(steps, "cleanup", cleanupTakeOut);
|
||||
if (cleanupTakeOut.IsSuccess)
|
||||
{
|
||||
outputMayBeActive = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
outcomeUnknown = true;
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
outcomeUnknown = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!normalDisconnectCompleted && !outcomeUnknown)
|
||||
{
|
||||
disconnectAttempted = true;
|
||||
try
|
||||
{
|
||||
var cleanup = await engine.DisconnectAsync(CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
disconnectCode = cleanup.Code.ToString();
|
||||
AddStep(steps, "cleanup", cleanup);
|
||||
outcomeUnknown |= IsOutcomeUnknown(cleanup);
|
||||
}
|
||||
catch
|
||||
{
|
||||
disconnectCode = "Failed";
|
||||
outcomeUnknown = true;
|
||||
}
|
||||
}
|
||||
|
||||
var safeToDispose = true;
|
||||
if (outcomeUnknown)
|
||||
{
|
||||
quarantineAttempted = true;
|
||||
try
|
||||
{
|
||||
await engine.QuarantineAsync().ConfigureAwait(false);
|
||||
quarantineCompleted = true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// A normal Dispose may issue SDK Disconnect. If quarantine could
|
||||
// not first detach the session, leaking locally is safer than an
|
||||
// unrequested control call against an unknown physical output.
|
||||
safeToDispose = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (safeToDispose)
|
||||
{
|
||||
try
|
||||
{
|
||||
await engine.DisposeAsync().ConfigureAwait(false);
|
||||
adapterDisposed = true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
outcomeUnknown = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return BuildReport();
|
||||
|
||||
async Task<bool> ObserveAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
await observationDelay(plan.ObservationDuration, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return true;
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Observation is local timing only. Its failure does not make a COM
|
||||
// command ambiguous, so retain the bounded TAKE OUT cleanup path.
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
SafeTestCommandReport BuildReport() => new(
|
||||
plan.Kind == SmokeCommandKind.TestConnect ? "test-connect" : "test-sequence",
|
||||
"Test",
|
||||
connectRequestIssued,
|
||||
comActivationAttempted,
|
||||
completed,
|
||||
outcomeUnknown,
|
||||
outputMayBeActive,
|
||||
stoppedAfter,
|
||||
steps.AsReadOnly(),
|
||||
new SafeCleanupReport(
|
||||
disconnectAttempted,
|
||||
disconnectCode,
|
||||
quarantineAttempted,
|
||||
quarantineCompleted,
|
||||
adapterDisposed));
|
||||
}
|
||||
|
||||
private static void AddStep(
|
||||
ICollection<SafeCommandStep> steps,
|
||||
string phase,
|
||||
PlayoutResult result) =>
|
||||
steps.Add(new SafeCommandStep(
|
||||
phase,
|
||||
result.Operation.ToString(),
|
||||
result.Code.ToString()));
|
||||
|
||||
private static bool IsOutcomeUnknown(PlayoutResult result) =>
|
||||
result.Code is PlayoutResultCode.OutcomeUnknown or PlayoutResultCode.TimedOut;
|
||||
|
||||
private static bool? InferComActivationAttempt(PlayoutResultCode code) => code switch
|
||||
{
|
||||
PlayoutResultCode.Success or PlayoutResultCode.Failed => true,
|
||||
PlayoutResultCode.Rejected => false,
|
||||
_ => null
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user