Skip to content

Commit

Permalink
Send updates to dashboard when resources fail to start
Browse files Browse the repository at this point in the history
  • Loading branch information
@davidfowl
davidfowl committed Mar 2, 2024
1 parent 76f939d commit cf4fce3
Show file tree
Hide file tree
Showing 3 changed files with 83 additions and 85 deletions.
120 changes: 41 additions & 79 deletions src/Aspire.Hosting/Dashboard/DcpDataSource.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System.Collections.Concurrent;
using System.Collections.Immutable;
using System.Text.Json;
using Aspire.Dashboard.Model;
using Aspire.Hosting.ApplicationModel;
using Aspire.Hosting.Dcp;
using Aspire.Hosting.Dcp.Model;
Expand Down Expand Up @@ -116,120 +117,81 @@ bool IsFilteredResource<T>(T resource) where T : CustomResource

private async Task ProcessInitialResourceAsync(IResource resource, CancellationToken cancellationToken)
{
if (resource.TryGetLastAnnotation<ContainerImageAnnotation>(out var containerImageAnnotation))
// The initial snapshots are all generic resources until we get the real state from DCP (for projects, containers and executables).

if (resource.IsContainer())
{
var snapshot = new ContainerSnapshot
var snapshot = CreateResourceSnapshot(resource, DateTime.UtcNow, new CustomResourceSnapshot
{
Name = resource.Name,
DisplayName = resource.Name,
Uid = resource.Name,
CreationTimeStamp = DateTime.UtcNow,
Image = containerImageAnnotation.Image,
State = "Starting",
ContainerId = null,
ExitCode = null,
ExpectedEndpointsCount = null,
Environment = [],
Endpoints = [],
Services = [],
Command = null,
Args = [],
Ports = []
};
ResourceType = KnownResourceTypes.Container,
Properties = [],
State = "Starting"
});

_placeHolderResources.TryAdd(resource.Name, snapshot);

await _onResourceChanged(snapshot, ResourceSnapshotChangeType.Upsert).ConfigureAwait(false);
}
else if (resource is ProjectResource p)
else if (resource is ProjectResource)
{
var snapshot = new ProjectSnapshot
var snapshot = CreateResourceSnapshot(resource, DateTime.UtcNow, new CustomResourceSnapshot
{
Name = p.Name,
DisplayName = p.Name,
Uid = p.Name,
CreationTimeStamp = DateTime.UtcNow,
ProjectPath = p.GetProjectMetadata().ProjectPath,
State = "Starting",
ExpectedEndpointsCount = null,
Environment = [],
Endpoints = [],
Services = [],
ExecutablePath = null,
ExitCode = null,
Arguments = null,
ProcessId = null,
StdErrFile = null,
StdOutFile = null,
WorkingDirectory = null
};
ResourceType = KnownResourceTypes.Project,
Properties = [],
State = "Starting"
});

_placeHolderResources.TryAdd(resource.Name, snapshot);

await _onResourceChanged(snapshot, ResourceSnapshotChangeType.Upsert).ConfigureAwait(false);
}
else if (resource is ExecutableResource exe)
else if (resource is ExecutableResource)
{
var snapshot = new ExecutableSnapshot
var snapshot = CreateResourceSnapshot(resource, DateTime.UtcNow, new CustomResourceSnapshot
{
Name = exe.Name,
DisplayName = exe.Name,
Uid = exe.Name,
CreationTimeStamp = DateTime.UtcNow,
ExecutablePath = exe.Command,
WorkingDirectory = exe.WorkingDirectory,
Arguments = null,
State = "Starting",
ExitCode = null,
StdOutFile = null,
StdErrFile = null,
ProcessId = null,
ExpectedEndpointsCount = null,
Environment = [],
Endpoints = [],
Services = []
};
ResourceType = KnownResourceTypes.Executable,
Properties = [],
State = "Starting"
});

_placeHolderResources.TryAdd(resource.Name, snapshot);

await _onResourceChanged(snapshot, ResourceSnapshotChangeType.Upsert).ConfigureAwait(false);
}
else
{
var creationTimestamp = DateTime.UtcNow;

var creationTimestamp = DateTime.UtcNow;

_ = Task.Run(async () =>
_ = Task.Run(async () =>
{
await foreach (var state in _notificationService.WatchAsync(resource).WithCancellation(cancellationToken))
{
await foreach (var state in _notificationService.WatchAsync(resource).WithCancellation(cancellationToken))
try
{
try
{
var snapshot = CreateResourceSnapshot(resource, creationTimestamp, state);
var snapshot = CreateResourceSnapshot(resource, creationTimestamp, state);
await _onResourceChanged(snapshot, ResourceSnapshotChangeType.Upsert).ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
_logger.LogError(ex, "Error updating resource snapshot for {Name}", resource.Name);
}
await _onResourceChanged(snapshot, ResourceSnapshotChangeType.Upsert).ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
_logger.LogError(ex, "Error updating resource snapshot for {Name}", resource.Name);
}
}
}, cancellationToken);
}
}, cancellationToken);
}

private static GenericResourceSnapshot CreateResourceSnapshot(IResource resource, DateTime creationTimestamp, CustomResourceSnapshot dashboardState)
private static GenericResourceSnapshot CreateResourceSnapshot(IResource resource, DateTime creationTimestamp, CustomResourceSnapshot snapshot)
{
ImmutableArray<EnvironmentVariableSnapshot> environmentVariables = [..
dashboardState.EnvironmentVariables.Select(e => new EnvironmentVariableSnapshot(e.Name, e.Value, false))];
snapshot.EnvironmentVariables.Select(e => new EnvironmentVariableSnapshot(e.Name, e.Value, false))];

ImmutableArray<ResourceServiceSnapshot> services = [..
dashboardState.Urls.Select(u => new ResourceServiceSnapshot(u.Name, u.Url, null))];
snapshot.Urls.Select(u => new ResourceServiceSnapshot(u.Name, u.Url, null))];

ImmutableArray<EndpointSnapshot> endpoints = [..
dashboardState.Urls.Select(u => new EndpointSnapshot(u.Url, u.Url))];
snapshot.Urls.Select(u => new EndpointSnapshot(u.Url, u.Url))];

return new GenericResourceSnapshot(dashboardState)
return new GenericResourceSnapshot(snapshot)
{
Uid = resource.Name,
CreationTimeStamp = creationTimestamp,
Expand All @@ -240,7 +202,7 @@ private static GenericResourceSnapshot CreateResourceSnapshot(IResource resource
ExitCode = null,
ExpectedEndpointsCount = endpoints.Length,
Services = services,
State = dashboardState.State ?? "Running"
State = snapshot.State ?? "Running"
};
}

Expand Down
44 changes: 39 additions & 5 deletions src/Aspire.Hosting/Dcp/ApplicationExecutor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,9 @@ internal sealed class ApplicationExecutor(ILogger<ApplicationExecutor> logger,
IOptions<DcpOptions> options,
IDashboardEndpointProvider dashboardEndpointProvider,
IDashboardAvailability dashboardAvailability,
DistributedApplicationExecutionContext executionContext)
DistributedApplicationExecutionContext executionContext,
ResourceNotificationService notificationService,
ResourceLoggerService loggerService)
{
private const string DebugSessionPortVar = "DEBUG_SESSION_PORT";

Expand Down Expand Up @@ -515,10 +517,26 @@ private Task CreateExecutablesAsync(IEnumerable<AppResource> executableResources
sortedExecutableResources.Insert(0, dashboardAppResource);
}

async Task CreateExecutableAsyncCore(AppResource cr, CancellationToken cancellationToken)
{
var logger = loggerService.GetLogger(cr.ModelResource);

try
{
await CreateExecutableAsync(cr, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to create resource {ResourceName}", cr.ModelResource.Name);

await notificationService.PublishUpdateAsync(cr.ModelResource, s => s with { State = "FailedToStart" }).ConfigureAwait(false);
}
}

var tasks = new List<Task>();
foreach (var er in sortedExecutableResources)
{
tasks.Add(CreateExecutableAsync(distributedApplicationOptions, kubernetesService, configuration, er, cancellationToken));
tasks.Add(CreateExecutableAsyncCore(er, cancellationToken));
}

return Task.WhenAll(tasks);
Expand All @@ -529,7 +547,7 @@ private Task CreateExecutablesAsync(IEnumerable<AppResource> executableResources
}
}

private async Task CreateExecutableAsync(DistributedApplicationOptions distributedApplicationOptions, IKubernetesService kubernetesService, IConfiguration configuration, AppResource er, CancellationToken cancellationToken)
private async Task CreateExecutableAsync(AppResource er, CancellationToken cancellationToken)
{
ExecutableSpec spec;
Func<Task<CustomResource>> createResource;
Expand Down Expand Up @@ -751,10 +769,26 @@ private Task CreateContainersAsync(IEnumerable<AppResource> containerResources,
{
AspireEventSource.Instance.DcpContainersCreateStart();

async Task CreateContainerAsyncCore(AppResource cr, CancellationToken cancellationToken)
{
var logger = loggerService.GetLogger(cr.ModelResource);

try
{
await CreateContainerAsync(cr, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to create container resource {ResourceName}", cr.ModelResource.Name);

await notificationService.PublishUpdateAsync(cr.ModelResource, s => s with { State = "FailedToStart" }).ConfigureAwait(false);
}
}

var tasks = new List<Task>();
foreach (var cr in containerResources)
{
tasks.Add(CreateContainerAsync(kubernetesService, cr, cancellationToken));
tasks.Add(CreateContainerAsyncCore(cr, cancellationToken));
}

return Task.WhenAll(tasks);
Expand All @@ -765,7 +799,7 @@ private Task CreateContainersAsync(IEnumerable<AppResource> containerResources,
}
}

private async Task CreateContainerAsync(IKubernetesService kubernetesService, AppResource cr, CancellationToken cancellationToken)
private async Task CreateContainerAsync(AppResource cr, CancellationToken cancellationToken)
{
var dcpContainerResource = (Container)cr.DcpResource;
var modelContainerResource = cr.ModelResource;
Expand Down
4 changes: 3 additions & 1 deletion tests/Aspire.Hosting.Tests/Dcp/ApplicationExecutorTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,9 @@ private static ApplicationExecutor CreateAppExecutor(
}),
new MockDashboardEndpointProvider(),
new MockDashboardAvailability(),
new DistributedApplicationExecutionContext(DistributedApplicationOperation.Run)
new DistributedApplicationExecutionContext(DistributedApplicationOperation.Run),
new ResourceNotificationService(),
new ResourceLoggerService()
);
}
}

0 comments on commit cf4fce3

Please sign in to comment.