From 8b0ea477a13cf19842712a84fa569c3f2d937765 Mon Sep 17 00:00:00 2001 From: Lee Wright <258036@NTTDATA.COM> Date: Tue, 9 Jul 2024 17:56:28 -0700 Subject: [PATCH 1/5] Change in flow --- .../Controllers/NotificationContoller.cs | 2 + .../DIAMCornetServiceConfiguration.cs | 9 + .../IncomingDisclosureNotificationHandler.cs | 48 ++--- .../Infrastructure/AccessTokenClient.cs | 2 + .../AuthorizationConfiguration.cs | 84 +++++++++ .../Infrastructure/HttpClientConfiguration.cs | 20 +- .../Infrastructure/ServiceConfiguration.cs | 2 +- ...erModel.cs => ParticipantResponseModel.cs} | 4 +- backend/DIAMCornetService/Program.cs | 2 + .../Services/CornetORDSClient.cs | 82 ++++++++- .../Services/CornetService.cs | 174 +++++++++--------- .../Services/ICornetORDSClient.cs | 7 +- .../Services/ICornetService.cs | 35 +++- backend/common/Authorization/KeycloakUrls.cs | 18 ++ backend/common/Common.csproj | 3 - .../Helpers/Extensions/StringExtensions.cs | 18 ++ 16 files changed, 385 insertions(+), 125 deletions(-) create mode 100644 backend/DIAMCornetService/Infrastructure/AuthorizationConfiguration.cs rename backend/DIAMCornetService/Models/{ParticipantCSNumberModel.cs => ParticipantResponseModel.cs} (78%) create mode 100644 backend/common/Authorization/KeycloakUrls.cs create mode 100644 backend/common/Helpers/Extensions/StringExtensions.cs diff --git a/backend/DIAMCornetService/Controllers/NotificationContoller.cs b/backend/DIAMCornetService/Controllers/NotificationContoller.cs index 0f6d80d5..666f88ed 100644 --- a/backend/DIAMCornetService/Controllers/NotificationContoller.cs +++ b/backend/DIAMCornetService/Controllers/NotificationContoller.cs @@ -1,9 +1,11 @@ namespace DIAMCornetService.Controllers; +using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; [ApiController] [Route("[controller]")] +[Authorize] public class NotificationContoller : ControllerBase { private readonly ILogger _logger; diff --git a/backend/DIAMCornetService/DIAMCornetServiceConfiguration.cs b/backend/DIAMCornetService/DIAMCornetServiceConfiguration.cs index 292d346f..c2c22429 100644 --- a/backend/DIAMCornetService/DIAMCornetServiceConfiguration.cs +++ b/backend/DIAMCornetService/DIAMCornetServiceConfiguration.cs @@ -1,5 +1,7 @@ namespace DIAMCornetService; +using Common.Authorization; + public class DIAMCornetServiceConfiguration { @@ -9,6 +11,7 @@ public class DIAMCornetServiceConfiguration public KafkaClusterConfiguration KafkaCluster { get; set; } = new(); public ConnectionStringConfiguration ConnectionStrings { get; set; } = new(); public CornetConfiguration CornetService { get; set; } = new(); + public KeycloakConfiguration Keycloak { get; set; } = new(); public class KafkaClusterConfiguration { @@ -30,6 +33,12 @@ public class KafkaClusterConfiguration } + public class KeycloakConfiguration + { + public string RealmUrl { get; set; } = string.Empty; + public string WellKnownConfig => KeycloakUrls.WellKnownConfig(this.RealmUrl); + } + public class ConnectionStringConfiguration { public string DIAMCornetDatabase { get; set; } = string.Empty; diff --git a/backend/DIAMCornetService/Features/IncomingDisclosureNotifications/IncomingDisclosureNotificationHandler.cs b/backend/DIAMCornetService/Features/IncomingDisclosureNotifications/IncomingDisclosureNotificationHandler.cs index ae0daee4..3d542d35 100644 --- a/backend/DIAMCornetService/Features/IncomingDisclosureNotifications/IncomingDisclosureNotificationHandler.cs +++ b/backend/DIAMCornetService/Features/IncomingDisclosureNotifications/IncomingDisclosureNotificationHandler.cs @@ -3,7 +3,6 @@ namespace DIAMCornetService.Features.MessageConsumer; using System.Threading.Tasks; using Common.Kafka; using DIAMCornetService.Data; -using DIAMCornetService.Exceptions; using DIAMCornetService.Services; using global::DIAMCornetService.Models; @@ -46,37 +45,40 @@ public async Task HandleAsync(string consumerName, string key, IncomingDis logger.LogInformation("Message received on {0} with key {1}", consumerName, key); // this is where we'll produce a response - var response = await this.cornetService.PublishCSNumberResponseAsync(value.ParticipantId); - incomingMessage.CSNumber = response["CSNumber"]; - incomingMessage.ProcessResponseId = response["id"]; + var response = await this.cornetService.LookupCSNumberForParticipant(value.ParticipantId); - // assuming no error on CS Number lookup - // change to if error = true - if (false) + if (response.ErrorType != null) { - incomingMessage.ErrorMessage = response["ErrorMessage"]; - - } - // submit notification to users - var notificationResponse = await this.cornetService.SubmitNotificationToEServices(response["CSNumber"], value.MessageText); - - // if publish returned an error - if (false) - { - // log error - - return Task.FromException(new CornetException("Failed to publish notification")); + // error on getting the CS Number + response = await this.cornetService.PublishErrorsToDIAMAsync(response); } else { - //add to tell message has been processed by consumer - await this.context.AddIdempotentConsumer(messageId: key, consumer: consumerName); + incomingMessage.CSNumber = response.CSNumber; - incomingMessage.CompletedTimestamp = DateTime.UtcNow; + // submit notification to users + response = await this.cornetService.SubmitNotificationToEServices(response, value.MessageText); + + // if submission was good we'll notify DIAM to provision the account + if (response.ErrorType == null) + { + response = await this.cornetService.PublishNotificationToDIAMAsync(response); + } + // otherwise we'll notify the business of the errors + else + { + response = await this.cornetService.PublishErrorsToDIAMAsync(response); + } - return Task.CompletedTask; } + //add to tell message has been processed by consumer + await this.context.AddIdempotentConsumer(messageId: key, consumer: consumerName); + + incomingMessage.CompletedTimestamp = DateTime.UtcNow; + + return Task.CompletedTask; + } catch (Exception ex) { diff --git a/backend/DIAMCornetService/Infrastructure/AccessTokenClient.cs b/backend/DIAMCornetService/Infrastructure/AccessTokenClient.cs index 3ed5743d..cb94637e 100644 --- a/backend/DIAMCornetService/Infrastructure/AccessTokenClient.cs +++ b/backend/DIAMCornetService/Infrastructure/AccessTokenClient.cs @@ -13,4 +13,6 @@ public async Task GetAccessTokenAsync(ClientCredentialsTokenRequest requ var response = await this.client.RequestClientCredentialsTokenAsync(request); return response.AccessToken; } + + } diff --git a/backend/DIAMCornetService/Infrastructure/AuthorizationConfiguration.cs b/backend/DIAMCornetService/Infrastructure/AuthorizationConfiguration.cs new file mode 100644 index 00000000..0b7bd20d --- /dev/null +++ b/backend/DIAMCornetService/Infrastructure/AuthorizationConfiguration.cs @@ -0,0 +1,84 @@ +namespace DIAMCornetService.Infrastructure; + +using Microsoft.AspNetCore.Authentication.JwtBearer; +using Microsoft.IdentityModel.Tokens; +using Newtonsoft.Json; + +public static class AuthorizationConfiguration +{ + public static IServiceCollection ConfigureAuthorization(this IServiceCollection services, DIAMCornetServiceConfiguration config) + { + services.AddAuthentication(option => + { + option.DefaultAuthenticateScheme = JwtBearerDefaults.AuthenticationScheme; + option.DefaultChallengeScheme = JwtBearerDefaults.AuthenticationScheme; + }) + .AddJwtBearer(options => + { + options.Authority = config.Keycloak.RealmUrl; + options.RequireHttpsMetadata = false; + options.Audience = "DIAM-INTERNAL"; + options.MetadataAddress = config.Keycloak.WellKnownConfig; + options.TokenValidationParameters = new TokenValidationParameters() + { + ValidateIssuerSigningKey = true, + ValidateIssuer = false, + ValidateAudience = false, + ValidAlgorithms = new List() { "RS256" } + }; + options.Events = new JwtBearerEvents + { + OnTokenValidated = context => + { + return Task.CompletedTask; + }, + OnAuthenticationFailed = context => + { + context.Response.OnStarting(async () => + { + context.NoResult(); + context.Response.StatusCode = StatusCodes.Status401Unauthorized; + context.Response.ContentType = "application/json"; + var response = + JsonConvert.SerializeObject("The access token provided is not valid."); + if (context.Exception.GetType() == typeof(SecurityTokenExpiredException)) + { + context.Response.Headers.Add("Token-Expired", "true"); + response = + JsonConvert.SerializeObject("The access token provided has expired."); + } + await context.Response.WriteAsync(response); + }); + + //context.HandleResponse(); + //context.Response.WriteAsync(response).Wait(); + return Task.CompletedTask; + + }, + OnForbidden = context => + { + return Task.CompletedTask; + }, + OnChallenge = context => + { + context.HandleResponse(); + context.Response.StatusCode = StatusCodes.Status401Unauthorized; + context.Response.ContentType = "application/json"; + + if (string.IsNullOrEmpty(context.Error)) + context.Error = "invalid_token"; + if (string.IsNullOrEmpty(context.ErrorDescription)) + context.ErrorDescription = "This request requires a valid JWT access token to be provided"; + + return context.Response.WriteAsync(JsonConvert.SerializeObject(new + { + error = context.Error, + error_description = context.ErrorDescription + })); + } + }; + }); + + return services; + } +} diff --git a/backend/DIAMCornetService/Infrastructure/HttpClientConfiguration.cs b/backend/DIAMCornetService/Infrastructure/HttpClientConfiguration.cs index fbefc963..7025d98b 100644 --- a/backend/DIAMCornetService/Infrastructure/HttpClientConfiguration.cs +++ b/backend/DIAMCornetService/Infrastructure/HttpClientConfiguration.cs @@ -2,20 +2,28 @@ namespace DIAMCornetService.Infrastructure; using System.Net.Http.Headers; using System.Text; +using Common.Helpers.Extensions; using DIAMCornetService.Services; +using Serilog; public static class HttpClientConfiguration { public static IServiceCollection AddHttpClientServices(this IServiceCollection services, DIAMCornetServiceConfiguration config) { - // service with basic auth - services.AddHttpClient(client => + // use basic auth if username/password set + if (!string.IsNullOrEmpty(config.CornetService.Username) && !string.IsNullOrEmpty(config.CornetService.Password)) { - client.BaseAddress = new Uri(config.CornetService.BaseAddress); - client.DefaultRequestHeaders.Add("Accept", "application/json"); - client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes($"{config.CornetService.Username}:{config.CornetService.Password}"))); - }); + Log.Logger.Information("Using CORNET endpoint {0}", config.CornetService.BaseAddress); + + services.AddHttpClientWithBaseAddress(config.CornetService.BaseAddress) + .ConfigureHttpClient(c => c.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes($"{config.CornetService.Username}:{config.CornetService.Password}")))); + } return services; } + + public static IHttpClientBuilder AddHttpClientWithBaseAddress(this IServiceCollection services, string baseAddress) + where TClient : class + where TImplementation : class, TClient + => services.AddHttpClient(client => client.BaseAddress = new Uri(baseAddress.EnsureTrailingSlash())); } diff --git a/backend/DIAMCornetService/Infrastructure/ServiceConfiguration.cs b/backend/DIAMCornetService/Infrastructure/ServiceConfiguration.cs index 238403ab..30f0994a 100644 --- a/backend/DIAMCornetService/Infrastructure/ServiceConfiguration.cs +++ b/backend/DIAMCornetService/Infrastructure/ServiceConfiguration.cs @@ -20,7 +20,7 @@ public static IServiceCollection AddCornetServices(this IServiceCollection servi services.AddSingleton(SystemClock.Instance); services.AddScoped(); services.AddScoped(); - services.AddScoped(); + // services.AddScoped(); services.AddHealthChecks() .AddCheck("liveliness", () => HealthCheckResult.Healthy()) .AddNpgSql(config.ConnectionStrings.DIAMCornetDatabase, tags: Tags).ForwardToPrometheus(); diff --git a/backend/DIAMCornetService/Models/ParticipantCSNumberModel.cs b/backend/DIAMCornetService/Models/ParticipantResponseModel.cs similarity index 78% rename from backend/DIAMCornetService/Models/ParticipantCSNumberModel.cs rename to backend/DIAMCornetService/Models/ParticipantResponseModel.cs index 0cc6209c..359f22e5 100644 --- a/backend/DIAMCornetService/Models/ParticipantCSNumberModel.cs +++ b/backend/DIAMCornetService/Models/ParticipantResponseModel.cs @@ -1,10 +1,11 @@ namespace DIAMCornetService.Models; -public class ParticipantCSNumberModel +public class ParticipantResponseModel { public string? CSNumber { get; set; } public string? ParticipantId { get; set; } public string? ErrorMessage { get; set; } + public string? DIAMPublishId { get; set; } public CornetCSNumberErrorType? ErrorType { get; set; } } @@ -14,5 +15,6 @@ public enum CornetCSNumberErrorType noActiveBioMetrics, eDisclosureNotProvisioned, unknownResponseError, + participantNotFound, otherError } diff --git a/backend/DIAMCornetService/Program.cs b/backend/DIAMCornetService/Program.cs index 3676ff42..f537a2f0 100644 --- a/backend/DIAMCornetService/Program.cs +++ b/backend/DIAMCornetService/Program.cs @@ -25,7 +25,9 @@ public static void Main(string[] args) builder.Services.AddEndpointsApiExplorer(); builder.Services.AddSwaggerGen(); + builder.Services.ConfigureAuthorization(config); builder.Services.AddKafkaClients(config); + builder.Services.AddHttpClientServices(config); builder.Services.AddCornetServices(config); builder.Services.AddApiVersioning(options => diff --git a/backend/DIAMCornetService/Services/CornetORDSClient.cs b/backend/DIAMCornetService/Services/CornetORDSClient.cs index d8b5ecf5..06affa35 100644 --- a/backend/DIAMCornetService/Services/CornetORDSClient.cs +++ b/backend/DIAMCornetService/Services/CornetORDSClient.cs @@ -2,22 +2,98 @@ namespace DIAMCornetService.Services; using System.Threading.Tasks; using DIAMCornetService.Exceptions; +using DIAMCornetService.Models; +using DomainResults.Common; public class CornetORDSClient(HttpClient httpClient, ILogger logger) : BaseClient(httpClient, logger), ICornetORDSClient { - public async Task GetCSNumberForParticipant(string participantId) + + /// + /// + /// + /// + /// + /// + public async Task GetCSNumberForParticipantAsync(string participantId) { - var response = await GetAsync($"api/v1/Participant/{participantId}/CSNumber"); + var response = await this.GetAsync($"v1/participant/{participantId}/csnumber"); if (response.IsSuccess) { - return response.Value; + return new ParticipantResponseModel() + { + CSNumber = response.Value.Csnum, + ParticipantId = participantId + }; + } + else if (response.Status == DomainOperationStatus.NotFound) + { + return new ParticipantResponseModel() + { + ParticipantId = participantId, + ErrorMessage = "Participant Not Found", + ErrorType = CornetCSNumberErrorType.participantNotFound + }; } else { throw new CornetException($"Failed to get CS Number for participant {participantId}"); } } + + /// + /// Send a notification to eServices devices for a participant + /// Can return a 200 OK but a responseCode that indicates a failure + /// + /// + /// + /// + /// + public async Task SubmitParticipantNotificationAsync(ParticipantResponseModel model, string messageText) + { + var response = await this.PostAsync($"v1/participant/{model.ParticipantId}/disclosure-message", new CornetMessageInput() { MessageTxt = messageText }); + + var responseText = response.Value.ResponseCode; + + // todo determine possible response codes + if (response.IsSuccess && string.IsNullOrEmpty(responseText)) + { + + logger.LogInformation($"Successfully submitted notification for participant {model.ParticipantId} CS: {model.CSNumber}"); + return "ok"; + + } + else if (response.IsSuccess && !string.IsNullOrEmpty(responseText)) + { + logger.LogWarning($"Failed to submit notification for participant {model.ParticipantId} CS: {model.CSNumber} Reason: {responseText}"); + return responseText; + } + else + { + logger.LogError($"Failed to submit notification for participant {model.ParticipantId} CS: {model.CSNumber} Reason: {responseText}"); + + throw new CornetException($"Failed to submit notification for participant {model.ParticipantId} CS: {model.CSNumber}"); + } + } + + + + + private class CornetCSResponse + { + public string Csnum { get; set; } = string.Empty; + } + + public class CornetMessageInput + { + public string MessageTxt { get; set; } = string.Empty; + } + + private class CornetResponse + { + public string ResponseCode { get; set; } = string.Empty; + } } + diff --git a/backend/DIAMCornetService/Services/CornetService.cs b/backend/DIAMCornetService/Services/CornetService.cs index 00eb7770..974f5b80 100644 --- a/backend/DIAMCornetService/Services/CornetService.cs +++ b/backend/DIAMCornetService/Services/CornetService.cs @@ -6,115 +6,82 @@ namespace DIAMCornetService.Services; using DIAMCornetService.Exceptions; using DIAMCornetService.Models; -public class CornetService(IKafkaProducer producer, DIAMCornetServiceConfiguration cornetServiceConfiguration, ICornetORDSClient cornetORDSClient, ILogger logger) : ICornetService +public class CornetService(IKafkaProducer producer, DIAMCornetServiceConfiguration cornetServiceConfiguration, ICornetORDSClient cornetORDSClient, ILogger logger) : ICornetService { /// - /// To be implemented fully JAMAL + /// Publish message to DIAM topic /// - /// + /// /// - public async Task GetParticipantCSNumberAsync(string participantId) + /// + public async Task PublishNotificationToDIAMAsync(ParticipantResponseModel model) { - - // JAMAL Code here - if this cannot lookup the participant, it should throw an exception (CornetException) - logger.LogInformation($"*************************** ADD CS NUMBER LOOKUP CODE HERE ***************************"); - - //var response = cornetORDSClient.GetCSNumberForParticipant(participantId); - - var responseModel = new ParticipantCSNumberModel - { - ParticipantId = participantId, - CSNumber = GetRandomEightDigitNumber().ToString() - }; - - if (!string.IsNullOrEmpty(responseModel.ErrorMessage)) + try { - logger.LogWarning($"Error occurred getting CS Number for {participantId} {responseModel.ErrorMessage}"); - + var guid = Guid.NewGuid(); + var response = await producer.ProduceAsync(cornetServiceConfiguration.KafkaCluster.ParticipantCSNumberMappingTopic, guid.ToString(), model); - switch (responseModel.ErrorMessage) + if (response.Status != PersistenceStatus.Persisted) { - - case "MISC": // Missing CS Number - logger.LogWarning($"Participant {participantId} has no CS Number"); - responseModel.ErrorMessage = "Participant {participantId} has no CS Number"; - responseModel.ErrorType = CornetCSNumberErrorType.missingCSNumber; - break; - case "NABI": // Missing BioMetric Registration - logger.LogWarning($"Participant {participantId} has no BioMetric Registration"); - responseModel.ErrorMessage = "Participant {participantId} has no BioMetric Registration"; - responseModel.ErrorType = CornetCSNumberErrorType.noActiveBioMetrics; - break; - - case "EDNP": // eDisclosure not provisioned - logger.LogWarning($"Participant {participantId} eDisclosure not provisioned"); - responseModel.ErrorMessage = "Participant {participantId} eDisclosure not provisioned"; - responseModel.ErrorType = CornetCSNumberErrorType.eDisclosureNotProvisioned; - break; - - case "OTHR": // Other errors - logger.LogWarning($"Participant {participantId} unknown error occurred"); - responseModel.ErrorMessage = "Participant {participantId} unknown error occurred"; - responseModel.ErrorType = CornetCSNumberErrorType.otherError; - - break; - - default: - logger.LogWarning($"Participant {participantId} unhandled response returned {responseModel.ErrorMessage}"); - responseModel.ErrorMessage = "Participant {participantId} unhandled response returned {response.ErrorMessage}"; - responseModel.ErrorType = CornetCSNumberErrorType.unknownResponseError; - break; - + logger.LogError($"Failed to publish CS number lookup to topic for {model.ParticipantId}"); + throw new DIAMKafkaException($"Failed to publish cs number mapping for {guid.ToString()} Part: {model.ParticipantId} CSNumber: {model.CSNumber}"); + } + else + { + model.DIAMPublishId = guid.ToString(); } } + catch (Exception ex) + { + logger.LogError($"Failed to publish CS number lookup to topic for {model.ParticipantId} [{ex.Message}]"); + throw; + } - - - return responseModel; - + return model; } - - /// - /// Publish a CS number to Participant ID response mapping to outbound topic - /// - /// - /// - /// - /// - - - public async Task> PublishCSNumberResponseAsync(string participantId) + public async Task PublishErrorsToDIAMAsync(ParticipantResponseModel model) { try { var guid = Guid.NewGuid(); - var csNumberLookupResponse = await this.GetParticipantCSNumberAsync(participantId); - var response = await producer.ProduceAsync(cornetServiceConfiguration.KafkaCluster.ParticipantCSNumberMappingTopic, guid.ToString(), csNumberLookupResponse); + var response = await producer.ProduceAsync(cornetServiceConfiguration.KafkaCluster.ProcessResponseTopic, guid.ToString(), model); if (response.Status != PersistenceStatus.Persisted) { - logger.LogError($"Failed to publish CS number lookup to topic for {participantId}"); - throw new DIAMKafkaException($"Failed to publish cs number mapping for {guid.ToString()} Part: {csNumberLookupResponse.ParticipantId} CSNumber: {csNumberLookupResponse.CSNumber}"); + logger.LogError($"Failed to publish Error response for {model.ParticipantId} to {cornetServiceConfiguration.KafkaCluster.ProcessResponseTopic}"); + throw new DIAMKafkaException($"Failed to publish Error response for {model.ParticipantId} to {cornetServiceConfiguration.KafkaCluster.ProcessResponseTopic}"); } else { - return new Dictionary - { - { "id", guid.ToString() }, - { "CSNumber", string.IsNullOrEmpty(csNumberLookupResponse.CSNumber) ? "NOT_FOUND" : csNumberLookupResponse.CSNumber }, - }; + model.DIAMPublishId = guid.ToString(); } } catch (Exception ex) { - logger.LogError($"Failed to publish CS number lookup to topic for {participantId} [{ex.Message}]"); + logger.LogError($"Failed to publish Error response for {model.ParticipantId} to {cornetServiceConfiguration.KafkaCluster.ProcessResponseTopic} [{ex.Message}]"); throw; } + + return model; + + } + + /// + /// Simple CS Number lookup + /// + /// + /// + public async Task LookupCSNumberForParticipant(string participantId) + { + var responseModel = await cornetORDSClient.GetCSNumberForParticipantAsync(participantId); + + return responseModel; + } @@ -124,17 +91,58 @@ public async Task> PublishCSNumberResponseAsync(strin /// /// /// - public async Task SubmitNotificationToEServices(string csNumber, string message) + public async Task SubmitNotificationToEServices(ParticipantResponseModel model, string message) { - logger.Log(LogLevel.Information, new EventId(), $"Publish notification for {csNumber} {message}", null, (state, ex) => state.ToString()); + logger.Log(LogLevel.Information, new EventId(), $"Publish notification for partId: {model.ParticipantId} CS: {model.CSNumber} {message}", null, (state, ex) => state.ToString()); - // await this call - cornetORDSClient.GetCSNumberToParticipant(csNumber, message); + var response = await cornetORDSClient.SubmitParticipantNotificationAsync(model, message); - // JAMAL add code here - logger.LogInformation($"*************************** ADD PUBLISH EVENT CODE HERE ***************************"); + switch (response) + { + case "ok": + { + logger.Log(LogLevel.Information, new EventId(), $"Successfully published notification for partId: {model.ParticipantId} CS: {model.CSNumber} {message}", null, (state, ex) => state.ToString()); + break; + } + case "NABI": + { + logger.LogInformation($"Failed to publish notification for partId: {model.ParticipantId} CS: {model.CSNumber} {message} Reason: {response}"); + model.ErrorMessage = "Participant has no active BioMetrics"; + model.ErrorType = CornetCSNumberErrorType.noActiveBioMetrics; + break; + } + case "ENDP": + { + logger.LogInformation($"Failed to publish notification for partId: {model.ParticipantId} CS: {model.CSNumber} {message} Reason: {response}"); + model.ErrorMessage = "eDisclosure not provisioned for user"; + model.ErrorType = CornetCSNumberErrorType.eDisclosureNotProvisioned; + break; + } + // shouldnt get to this one as CS number is looked up previously but added to cover all responses + case "MISC": + { + logger.LogInformation($"Failed to publish notification for partId: {model.ParticipantId} CS: {model.CSNumber} {message} Reason: {response}"); + model.ErrorMessage = "Missing CS Number"; + model.ErrorType = CornetCSNumberErrorType.missingCSNumber; + break; + } + case "OTHR": + { + logger.LogInformation($"Failed to publish notification for partId: {model.ParticipantId} CS: {model.CSNumber} {message} Reason: {response}"); + model.ErrorMessage = "Unknown CORNET error occurred"; + model.ErrorType = CornetCSNumberErrorType.otherError; + break; + } + default: + { + logger.LogInformation($"Failed to publish notification for partId: {model.ParticipantId} CS: {model.CSNumber} {message} Reason: {response}"); + model.ErrorMessage = $"Unknown CORNET response {response}"; + model.ErrorType = CornetCSNumberErrorType.unknownResponseError; + break; + } + } - // some response code to show notification worked - return 0; + return model; } diff --git a/backend/DIAMCornetService/Services/ICornetORDSClient.cs b/backend/DIAMCornetService/Services/ICornetORDSClient.cs index 95f85cae..8d29232c 100644 --- a/backend/DIAMCornetService/Services/ICornetORDSClient.cs +++ b/backend/DIAMCornetService/Services/ICornetORDSClient.cs @@ -1,7 +1,12 @@ namespace DIAMCornetService.Services; +using DIAMCornetService.Models; + public interface ICornetORDSClient { // get a cs number for a participant - Task GetCSNumberForParticipant(string participantId); + Task GetCSNumberForParticipantAsync(string participantId); + + // send a notification to a participant + Task SubmitParticipantNotificationAsync(ParticipantResponseModel model, string messageText); } diff --git a/backend/DIAMCornetService/Services/ICornetService.cs b/backend/DIAMCornetService/Services/ICornetService.cs index 7986a024..8bdf5c6e 100644 --- a/backend/DIAMCornetService/Services/ICornetService.cs +++ b/backend/DIAMCornetService/Services/ICornetService.cs @@ -1,10 +1,37 @@ namespace DIAMCornetService.Services; - using DIAMCornetService.Models; public interface ICornetService { - public Task GetParticipantCSNumberAsync(string participantId); - public Task SubmitNotificationToEServices(string csNumber, string message); - public Task> PublishCSNumberResponseAsync(string participantId); + /// + /// Get the CS number for a participant ID + /// + /// + /// + public Task LookupCSNumberForParticipant(string participantId); + + /// + /// Submit a message to eServices devices for Participant + /// + /// + /// + /// + /// + public Task SubmitNotificationToEServices(ParticipantResponseModel model, string message); + + /// + /// Publish the participant and CS Number to the DIAM topic + /// + /// + /// + /// + public Task PublishNotificationToDIAMAsync(ParticipantResponseModel model); + + /// + /// Publish and Error notifications to DIAM + /// + /// + /// + /// + public Task PublishErrorsToDIAMAsync(ParticipantResponseModel model); } diff --git a/backend/common/Authorization/KeycloakUrls.cs b/backend/common/Authorization/KeycloakUrls.cs new file mode 100644 index 00000000..61b581ef --- /dev/null +++ b/backend/common/Authorization/KeycloakUrls.cs @@ -0,0 +1,18 @@ +namespace Common.Authorization; + +using Flurl; + +public static class KeycloakUrls +{ + /// + /// Returns the URL for OAuth token issuance. + /// + /// URL of the keycloak instance up to the realm name; I.e. "[base url]/auth/realms/[realm name]" + public static string Token(string realmUrl) => Url.Combine(realmUrl, "protocol/openid-connect/token"); + + /// + /// Returns the URL for the OAuth well-known config. + /// + /// URL of the keycloak instance up to the realm name; I.e. "[base url]/auth/realms/[realm name]" + public static string WellKnownConfig(string realmUrl) => Url.Combine(realmUrl, ".well-known/openid-configuration"); +} diff --git a/backend/common/Common.csproj b/backend/common/Common.csproj index 9e6d7fb0..6cb8f2fd 100644 --- a/backend/common/Common.csproj +++ b/backend/common/Common.csproj @@ -77,8 +77,5 @@ - - - diff --git a/backend/common/Helpers/Extensions/StringExtensions.cs b/backend/common/Helpers/Extensions/StringExtensions.cs new file mode 100644 index 00000000..732a4361 --- /dev/null +++ b/backend/common/Helpers/Extensions/StringExtensions.cs @@ -0,0 +1,18 @@ +namespace Common.Helpers.Extensions; + +using System; + +public static class StringExtensions +{ + + public static string EnsureTrailingSlash(this string url) + { + if (!url.EndsWith("/", StringComparison.Ordinal)) + { + return url + "/"; + } + + return url; + } +} + From 91342ff257935a808349c65da664176dd5d4dedf Mon Sep 17 00:00:00 2001 From: Lee Wright <258036@NTTDATA.COM> Date: Thu, 11 Jul 2024 15:51:44 -0700 Subject: [PATCH 2/5] WIP --- .../Controllers/NotificationContoller.cs | 15 +-- .../DIAMCornetServiceConfiguration.cs | 6 ++ .../IncomingDisclosureNotificationConsumer.cs | 12 +-- .../IncomingDisclosureNotificationHandler.cs | 32 +++--- .../IncomingNotificationConsumer.cs | 21 ++-- .../Infrastructure/KafkaConfiguration.cs | 2 + .../Infrastructure/KafkaConsumer.cs | 99 +++++++++---------- ...eModel.cs => InCustodyParticipantModel.cs} | 6 +- backend/DIAMCornetService/Program.cs | 50 ++++++++-- .../DIAMCornetService/Services/BaseClient.cs | 12 +-- .../Services/CornetORDSClient.cs | 60 +++++++---- .../Services/CornetService.cs | 68 +++++++++---- .../Services/ICornetORDSClient.cs | 4 +- .../Services/ICornetService.cs | 8 +- backend/DIAMCornetService/appsettings.json | 4 + 15 files changed, 223 insertions(+), 176 deletions(-) rename backend/DIAMCornetService/Models/{ParticipantResponseModel.cs => InCustodyParticipantModel.cs} (73%) diff --git a/backend/DIAMCornetService/Controllers/NotificationContoller.cs b/backend/DIAMCornetService/Controllers/NotificationContoller.cs index 666f88ed..a74dc418 100644 --- a/backend/DIAMCornetService/Controllers/NotificationContoller.cs +++ b/backend/DIAMCornetService/Controllers/NotificationContoller.cs @@ -6,29 +6,20 @@ namespace DIAMCornetService.Controllers; [ApiController] [Route("[controller]")] [Authorize] -public class NotificationContoller : ControllerBase +public class NotificationContoller(Services.INotificationService notificationService) : ControllerBase { - private readonly ILogger _logger; - private readonly DIAMCornetService.Services.INotificationService _notificationService; - - public NotificationContoller(ILogger logger, DIAMCornetService.Services.INotificationService notificationService) - { - this._logger = logger; - this._notificationService = notificationService; - } [HttpGet(Name = "GenerateTestNotification")] public async Task> PublishTestNotification([FromQuery] string participantId, [FromQuery] string messageText) { try { - var publishedGUID = await this._notificationService.PublishTestNotificationAsync(participantId, messageText); + var publishedGUID = await notificationService.PublishTestNotificationAsync(participantId, messageText); return publishedGUID; } catch (Exception ex) { - this._logger.LogError(ex, "Failed to publish test notification"); - return StatusCode(500, $"Failed to publish test notification: {ex.Message}"); + return this.StatusCode(500, $"Failed to publish test notification: {ex.Message}"); } } } diff --git a/backend/DIAMCornetService/DIAMCornetServiceConfiguration.cs b/backend/DIAMCornetService/DIAMCornetServiceConfiguration.cs index c2c22429..512ff5eb 100644 --- a/backend/DIAMCornetService/DIAMCornetServiceConfiguration.cs +++ b/backend/DIAMCornetService/DIAMCornetServiceConfiguration.cs @@ -12,7 +12,13 @@ public class DIAMCornetServiceConfiguration public ConnectionStringConfiguration ConnectionStrings { get; set; } = new(); public CornetConfiguration CornetService { get; set; } = new(); public KeycloakConfiguration Keycloak { get; set; } = new(); + public SplunkConfiguration SplunkConfig { get; set; } = new SplunkConfiguration(); + public class SplunkConfiguration + { + public string Host { get; set; } = string.Empty; + public string CollectorToken { get; set; } = string.Empty; + } public class KafkaClusterConfiguration { public string Url { get; set; } = string.Empty; diff --git a/backend/DIAMCornetService/Features/IncomingDisclosureNotifications/IncomingDisclosureNotificationConsumer.cs b/backend/DIAMCornetService/Features/IncomingDisclosureNotifications/IncomingDisclosureNotificationConsumer.cs index a7d09304..40b0e6aa 100644 --- a/backend/DIAMCornetService/Features/IncomingDisclosureNotifications/IncomingDisclosureNotificationConsumer.cs +++ b/backend/DIAMCornetService/Features/IncomingDisclosureNotifications/IncomingDisclosureNotificationConsumer.cs @@ -5,20 +5,18 @@ namespace DIAMCornetService.Features.MessageConsumer; using System.Threading; using Confluent.Kafka; -public class IncomingDisclosureNotificationConsumer(ConsumerConfig config, IServiceScopeFactory serviceScopeFactory, DIAMCornetServiceConfiguration configuration) +public class IncomingDisclosureNotificationConsumer(ILogger logger, ConsumerConfig config, DIAMCornetServiceConfiguration configuration) { - private readonly IServiceScopeFactory serviceScopeFactory = serviceScopeFactory; - private readonly ConsumerConfig config = config; - private readonly DIAMCornetServiceConfiguration configuration = configuration; + public void StartConsuming(CancellationToken cancellationToken) { - using (var consumer = new ConsumerBuilder(this.config).Build()) + using (var consumer = new ConsumerBuilder(config).Build()) { // subscribe to this topic - consumer.Subscribe(this.configuration.KafkaCluster.ParticipantCSNumberMappingTopic); + consumer.Subscribe(configuration.KafkaCluster.ParticipantCSNumberMappingTopic); try { @@ -27,7 +25,7 @@ public void StartConsuming(CancellationToken cancellationToken) var consumeResult = consumer.Consume(cancellationToken); // Handle the message, e.g., process consumeResult.Message - Console.WriteLine($"Received message: {consumeResult.Message.Value}"); + logger.LogInformation($"Received message: {consumeResult.Message.Value}"); } } catch (OperationCanceledException) diff --git a/backend/DIAMCornetService/Features/IncomingDisclosureNotifications/IncomingDisclosureNotificationHandler.cs b/backend/DIAMCornetService/Features/IncomingDisclosureNotifications/IncomingDisclosureNotificationHandler.cs index 3d542d35..5cee31d8 100644 --- a/backend/DIAMCornetService/Features/IncomingDisclosureNotifications/IncomingDisclosureNotificationHandler.cs +++ b/backend/DIAMCornetService/Features/IncomingDisclosureNotifications/IncomingDisclosureNotificationHandler.cs @@ -3,21 +3,11 @@ namespace DIAMCornetService.Features.MessageConsumer; using System.Threading.Tasks; using Common.Kafka; using DIAMCornetService.Data; +using DIAMCornetService.Models; using DIAMCornetService.Services; -using global::DIAMCornetService.Models; -public class IncomingDisclosureNotificationHandler : IKafkaHandler +public class IncomingDisclosureNotificationHandler(ILogger logger, ICornetService cornetService, DIAMCornetDbContext context) : IKafkaHandler { - private readonly ILogger logger; - private ICornetService cornetService; - private DIAMCornetDbContext context; - - public IncomingDisclosureNotificationHandler(ILogger logger, ICornetService cornetService, DIAMCornetDbContext context) - { - this.logger = logger; - this.cornetService = cornetService; - this.context = context; - } public async Task HandleAsync(string consumerName, string key, IncomingDisclosureNotificationModel value) { @@ -27,7 +17,7 @@ public async Task HandleAsync(string consumerName, string key, IncomingDis try { // check we havent consumed this message before - var processedAlready = await this.context.HasMessageBeenProcessed(key, consumerName); + var processedAlready = await context.HasMessageBeenProcessed(key, consumerName); if (processedAlready) { logger.LogWarning($"Already processed message with key {key}"); @@ -40,17 +30,17 @@ public async Task HandleAsync(string consumerName, string key, IncomingDis incomingMessage.MessageTimestamp = DateTime.UtcNow; // we'll track message processing time and responses here - this.context.Add(incomingMessage); + context.Add(incomingMessage); logger.LogInformation("Message received on {0} with key {1}", consumerName, key); // this is where we'll produce a response - var response = await this.cornetService.LookupCSNumberForParticipant(value.ParticipantId); + var response = await cornetService.LookupCSNumberForParticipant(value.ParticipantId); if (response.ErrorType != null) { // error on getting the CS Number - response = await this.cornetService.PublishErrorsToDIAMAsync(response); + response = await cornetService.PublishErrorsToDIAMAsync(response); } else { @@ -58,22 +48,22 @@ public async Task HandleAsync(string consumerName, string key, IncomingDis incomingMessage.CSNumber = response.CSNumber; // submit notification to users - response = await this.cornetService.SubmitNotificationToEServices(response, value.MessageText); + response = await cornetService.SubmitNotificationToEServices(response, value.MessageText); // if submission was good we'll notify DIAM to provision the account if (response.ErrorType == null) { - response = await this.cornetService.PublishNotificationToDIAMAsync(response); + response = await cornetService.PublishNotificationToDIAMAsync(response); } // otherwise we'll notify the business of the errors else { - response = await this.cornetService.PublishErrorsToDIAMAsync(response); + response = await cornetService.PublishErrorsToDIAMAsync(response); } } //add to tell message has been processed by consumer - await this.context.AddIdempotentConsumer(messageId: key, consumer: consumerName); + await context.AddIdempotentConsumer(messageId: key, consumer: consumerName); incomingMessage.CompletedTimestamp = DateTime.UtcNow; @@ -88,7 +78,7 @@ public async Task HandleAsync(string consumerName, string key, IncomingDis } finally { - await this.context.SaveChangesAsync(); + await context.SaveChangesAsync(); } } } diff --git a/backend/DIAMCornetService/Features/IncomingDisclosureNotifications/IncomingNotificationConsumer.cs b/backend/DIAMCornetService/Features/IncomingDisclosureNotifications/IncomingNotificationConsumer.cs index 4d23170c..a1235d49 100644 --- a/backend/DIAMCornetService/Features/IncomingDisclosureNotifications/IncomingNotificationConsumer.cs +++ b/backend/DIAMCornetService/Features/IncomingDisclosureNotifications/IncomingNotificationConsumer.cs @@ -3,34 +3,29 @@ namespace DIAMCornetService.Features.IncomingDisclosureNotifications; using System.Net; using DIAMCornetService.Infrastructure; using DIAMCornetService.Models; -public class IncomingNotificationConsumer : BackgroundService + +public class IncomingNotificationConsumer(ILogger logger, IKafkaConsumer kafkaConsumer, DIAMCornetServiceConfiguration config) : BackgroundService { - private readonly IKafkaConsumer consumer; - private readonly DIAMCornetServiceConfiguration config; - public IncomingNotificationConsumer(IKafkaConsumer kafkaConsumer, DIAMCornetServiceConfiguration config) - { - this.consumer = kafkaConsumer; - this.config = config; - } + protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - Serilog.Log.Information("Starting consumer {0}", this.config.KafkaCluster.DisclosureNotificationTopic); + logger.LogInformation("Starting consumer {0}", config.KafkaCluster.DisclosureNotificationTopic); try { - await this.consumer.Consume(this.config.KafkaCluster.DisclosureNotificationTopic, stoppingToken); + await kafkaConsumer.Consume(config.KafkaCluster.DisclosureNotificationTopic, stoppingToken); } catch (Exception ex) { - Serilog.Log.Warning($"{(int)HttpStatusCode.InternalServerError} ConsumeFailedOnTopic - {this.config.KafkaCluster.DisclosureNotificationTopic}, {ex}"); + logger.LogWarning($"{(int)HttpStatusCode.InternalServerError} ConsumeFailedOnTopic - {config.KafkaCluster.DisclosureNotificationTopic}, {ex}"); } } public override void Dispose() { - this.consumer.Close(); - this.consumer.Dispose(); + kafkaConsumer.Close(); + kafkaConsumer.Dispose(); base.Dispose(); GC.SuppressFinalize(this); diff --git a/backend/DIAMCornetService/Infrastructure/KafkaConfiguration.cs b/backend/DIAMCornetService/Infrastructure/KafkaConfiguration.cs index 40871fec..38633cb3 100644 --- a/backend/DIAMCornetService/Infrastructure/KafkaConfiguration.cs +++ b/backend/DIAMCornetService/Infrastructure/KafkaConfiguration.cs @@ -66,11 +66,13 @@ public static IServiceCollection AddKafkaClients(this IServiceCollection service HeartbeatIntervalMs = 20000 }; + services.AddSingleton(consumerConfig); services.AddSingleton(producerConfig); services.AddSingleton(typeof(IKafkaProducer<,>), typeof(KafkaProducer<,>)); services.AddScoped, IncomingDisclosureNotificationHandler>(); services.AddSingleton(typeof(IKafkaConsumer<,>), typeof(KafkaConsumer<,>)); + services.AddHostedService(); return services; diff --git a/backend/DIAMCornetService/Infrastructure/KafkaConsumer.cs b/backend/DIAMCornetService/Infrastructure/KafkaConsumer.cs index 1378ac8c..294db2ff 100644 --- a/backend/DIAMCornetService/Infrastructure/KafkaConsumer.cs +++ b/backend/DIAMCornetService/Infrastructure/KafkaConsumer.cs @@ -5,79 +5,71 @@ namespace DIAMCornetService.Infrastructure; using Common.Kafka.Deserializer; using Confluent.Kafka; using IdentityModel.Client; -using Serilog; public class KafkaConsumer : IKafkaConsumer where TValue : class { - private const string EXPIRY_CLAIM = "exp"; - private const string SUBJECT_CLAIM = "sub"; - private readonly ConsumerConfig config; - private IKafkaHandler handler; - private IConsumer consumer; - private string topic; - - private readonly IServiceScopeFactory serviceScopeFactory; - - public KafkaConsumer(ConsumerConfig config, IServiceScopeFactory serviceScopeFactory) + private readonly ConsumerConfig _config; + private readonly IServiceScopeFactory _serviceScopeFactory; + private readonly ILogger> _logger; + private IKafkaHandler _handler; + private IConsumer _consumer; + private string _topic; + + public KafkaConsumer(ConsumerConfig config, IServiceScopeFactory serviceScopeFactory, ILogger> logger) { - this.serviceScopeFactory = serviceScopeFactory; - this.config = config; - + _config = config; + _serviceScopeFactory = serviceScopeFactory; + _logger = logger; } public async Task Consume(string topic, CancellationToken stoppingToken) { - using var scope = this.serviceScopeFactory.CreateScope(); + using var scope = _serviceScopeFactory.CreateScope(); - Log.Logger.Information("DIAM Cornet Starting consumer for topic {0}", topic); + _logger.LogInformation("DIAM Cornet Starting consumer for topic {0}", topic); - this.handler = scope.ServiceProvider.GetRequiredService>(); - this.consumer = new ConsumerBuilder(this.config) + _handler = scope.ServiceProvider.GetRequiredService>(); + _consumer = new ConsumerBuilder(_config) .SetLogHandler((consumer, log) => Console.WriteLine($"CON _______________________ {log}")) .SetErrorHandler((consumer, log) => Console.WriteLine($"CON ERR _______________________ {log}")) - .SetOAuthBearerTokenRefreshHandler(OauthTokenRefreshCallback).SetValueDeserializer(new DefaultKafkaDeserializer()).Build(); - this.topic = topic; + .SetOAuthBearerTokenRefreshHandler(OauthTokenRefreshCallback) + .SetValueDeserializer(new DefaultKafkaDeserializer()) + .Build(); + _topic = topic; - await Task.Run(() => this.StartConsumerLoop(stoppingToken), stoppingToken); + await Task.Run(() => StartConsumerLoop(stoppingToken), stoppingToken); } - - /// - /// Start consuming messages - /// - /// - /// private async Task StartConsumerLoop(CancellationToken cancellationToken) { - - Log.Logger.Information("Start consuming from {0}", this.topic); - this.consumer.Subscribe(this.topic); + _logger.LogInformation("Start consuming from {0}", _topic); + _consumer.Subscribe(_topic); while (!cancellationToken.IsCancellationRequested) { try { - var result = this.consumer.Consume(cancellationToken); + var result = _consumer.Consume(cancellationToken); if (result != null) { - var consumerResult = await this.handler.HandleAsync(this.consumer.Name, result.Message.Key, result.Message.Value); + var consumerResult = await _handler.HandleAsync(_consumer.Name, result.Message.Key, result.Message.Value); if (consumerResult.Status == TaskStatus.RanToCompletion && consumerResult.Exception == null) { - Log.Logger.Information($"Marking complete {result.Message.Key}"); + _logger.LogInformation($"Marking complete {result.Message.Key}"); - this.consumer.Commit(result); + _consumer.Commit(result); } else { - Log.Logger.Information($"Error processing message {result.Message.Key} {consumerResult.Exception}"); + _logger.LogInformation($"Error processing message {result.Message.Key} {consumerResult.Exception}"); } } else { - Log.Logger.Information("No messages received"); + _logger.LogInformation("No messages received"); continue; } } @@ -87,8 +79,7 @@ private async Task StartConsumerLoop(CancellationToken cancellationToken) } catch (ConsumeException e) { - // Consumer errors should generally be ignored (or logged) unless fatal. - Log.Logger.Information("Consumer error ontopic {0} [{1}]", this.topic, e.Message); + _logger.LogInformation("Consumer error on topic {0} [{1}]", _topic, e.Message); if (e.Error.IsFatal) { @@ -97,7 +88,7 @@ private async Task StartConsumerLoop(CancellationToken cancellationToken) } catch (Exception e) { - Log.Logger.Information("General consumer error on topic {0} [{1}]", this.topic, e.Message); + _logger.LogInformation("General consumer error on topic {0} [{1}]", _topic, e.Message); Console.WriteLine($"Unexpected error: {e}"); break; @@ -105,24 +96,26 @@ private async Task StartConsumerLoop(CancellationToken cancellationToken) } } - /// - /// Releases all resources used by the current instance of the consumer - /// - public void Dispose() => this.consumer.Dispose(); + public void Dispose() + { + _consumer.Dispose(); + } - public void Close() => this.consumer.Close(); + public void Close() + { + _consumer.Close(); + } private static async void OauthTokenRefreshCallback(IClient client, string config) { try { - var settingsFile = DIAMCornetServiceConfiguration.IsDevelopment() ? "appsettings.Development.json" : "appsettings.json"; - var clusterConfig = new ConfigurationBuilder() .SetBasePath(Directory.GetCurrentDirectory()) - .AddJsonFile(settingsFile).Build(); + .AddJsonFile(settingsFile) + .Build(); var tokenEndpoint = Environment.GetEnvironmentVariable("KafkaCluster__SaslOauthbearerTokenEndpointUrl"); var clientId = Environment.GetEnvironmentVariable("KafkaCluster__SaslOauthbearerConsumerClientId"); @@ -131,10 +124,8 @@ private static async void OauthTokenRefreshCallback(IClient client, string confi clientSecret ??= clusterConfig.GetValue("KafkaCluster:SaslOauthbearerConsumerClientSecret"); clientId ??= clusterConfig.GetValue("KafkaCluster:SaslOauthbearerConsumerClientId"); tokenEndpoint ??= clusterConfig.GetValue("KafkaCluster:SaslOauthbearerTokenEndpointUrl"); - Log.Logger.Debug("Pidp Kafka Consumer getting token {0} {1}", tokenEndpoint, clientId); var accessTokenClient = new HttpClient(); - var accessToken = await accessTokenClient.RequestClientCredentialsTokenAsync(new ClientCredentialsTokenRequest { Address = tokenEndpoint, @@ -142,27 +133,27 @@ private static async void OauthTokenRefreshCallback(IClient client, string confi ClientSecret = clientSecret, GrantType = "client_credentials" }); + var tokenTicks = GetTokenExpirationTime(accessToken.AccessToken); var subject = GetTokenSubject(accessToken.AccessToken); var tokenDate = DateTimeOffset.FromUnixTimeSeconds(tokenTicks); var timeSpan = new DateTime() - tokenDate; var ms = tokenDate.ToUnixTimeMilliseconds(); - Log.Logger.Debug("Consumer got token {0}", ms); client.OAuthBearerSetToken(accessToken.AccessToken, ms, subject); } catch (Exception ex) { - Log.Logger.Error(ex.Message); client.OAuthBearerSetTokenFailure(ex.ToString()); } } + private static long GetTokenExpirationTime(string token) { var handler = new System.IdentityModel.Tokens.Jwt.JwtSecurityTokenHandler(); var jwtSecurityToken = handler.ReadJwtToken(token); - var tokenExp = jwtSecurityToken.Claims.First(claim => claim.Type.Equals(KafkaConsumer.EXPIRY_CLAIM, StringComparison.Ordinal)).Value; + var tokenExp = jwtSecurityToken.Claims.First(claim => claim.Type.Equals("exp", StringComparison.Ordinal)).Value; var ticks = long.Parse(tokenExp, CultureInfo.InvariantCulture); return ticks; } @@ -171,8 +162,6 @@ private static string GetTokenSubject(string token) { var handler = new System.IdentityModel.Tokens.Jwt.JwtSecurityTokenHandler(); var jwtSecurityToken = handler.ReadJwtToken(token); - return jwtSecurityToken.Claims.First(claim => claim.Type.Equals(KafkaConsumer.SUBJECT_CLAIM, StringComparison.Ordinal)).Value; - + return jwtSecurityToken.Claims.First(claim => claim.Type.Equals("sub", StringComparison.Ordinal)).Value; } - } diff --git a/backend/DIAMCornetService/Models/ParticipantResponseModel.cs b/backend/DIAMCornetService/Models/InCustodyParticipantModel.cs similarity index 73% rename from backend/DIAMCornetService/Models/ParticipantResponseModel.cs rename to backend/DIAMCornetService/Models/InCustodyParticipantModel.cs index 359f22e5..b53740d3 100644 --- a/backend/DIAMCornetService/Models/ParticipantResponseModel.cs +++ b/backend/DIAMCornetService/Models/InCustodyParticipantModel.cs @@ -1,11 +1,11 @@ namespace DIAMCornetService.Models; - -public class ParticipantResponseModel +public class InCustodyParticipantModel { public string? CSNumber { get; set; } - public string? ParticipantId { get; set; } + public string ParticipantId { get; set; } = string.Empty; public string? ErrorMessage { get; set; } public string? DIAMPublishId { get; set; } + public DateTime? CreationDateUTC { get; set; } public CornetCSNumberErrorType? ErrorType { get; set; } } diff --git a/backend/DIAMCornetService/Program.cs b/backend/DIAMCornetService/Program.cs index f537a2f0..8dcfbd75 100644 --- a/backend/DIAMCornetService/Program.cs +++ b/backend/DIAMCornetService/Program.cs @@ -1,6 +1,8 @@ namespace DIAMCornetService; +using System; +using System.Reflection; using Asp.Versioning; using DIAMCornetService.Data; using DIAMCornetService.Infrastructure; @@ -8,6 +10,8 @@ namespace DIAMCornetService; using Microsoft.Extensions.DependencyInjection; using Prometheus; using Serilog; +using Serilog.Events; +using Serilog.Sinks.SystemConsole.Themes; public class Program { @@ -15,21 +19,21 @@ public static void Main(string[] args) { var builder = WebApplication.CreateBuilder(args); - // Load config var config = new DIAMCornetServiceConfiguration(); builder.Configuration.Bind(config); // Bind configuration + builder.Services.AddLogging(builder => builder.AddConsole()); builder.Services.AddSingleton(config); builder.Services.AddControllers(); builder.Services.AddEndpointsApiExplorer(); builder.Services.AddSwaggerGen(); - builder.Services.ConfigureAuthorization(config); - builder.Services.AddKafkaClients(config); + builder.Services.AddHttpClientServices(config); builder.Services.AddCornetServices(config); + builder.Services.AddKafkaClients(config); builder.Services.AddApiVersioning(options => { options.ReportApiVersions = true; @@ -37,7 +41,6 @@ public static void Main(string[] args) options.ApiVersionReader = new HeaderApiVersionReader("api-version"); }); - builder.Services.AddDbContext(options => options .UseNpgsql(config.ConnectionStrings.DIAMCornetDatabase, sql => sql.UseNodaTime()) .EnableSensitiveDataLogging(sensitiveDataLoggingEnabled: false)); @@ -57,6 +60,41 @@ public static void Main(string[] args) } } + var name = Assembly.GetExecutingAssembly().GetName(); + var outputTemplate = "[{Timestamp:HH:mm:ss} {Level:u3}] {Message:lj}{NewLine}{Exception}"; + + var loggerConfiguration = new LoggerConfiguration() + .MinimumLevel.Information() + .Filter.ByExcluding("RequestPath like '/health%'") + .Filter.ByExcluding("RequestPath like '/metrics%'") + .MinimumLevel.Override("Microsoft", LogEventLevel.Warning) + .MinimumLevel.Override("Microsoft.Hosting.Lifetime", LogEventLevel.Information) + .MinimumLevel.Override("System", LogEventLevel.Warning) + .Enrich.FromLogContext() + .Enrich.WithMachineName() + .Enrich.WithProperty("Assembly", $"{name.Name}") + .Enrich.WithProperty("Version", $"{name.Version}") + .WriteTo.Console( + outputTemplate: outputTemplate, + theme: AnsiConsoleTheme.Code); + + + if (!string.IsNullOrEmpty(config.SplunkConfig.Host)) + { + loggerConfiguration.WriteTo.EventCollector(config.SplunkConfig.Host, config.SplunkConfig.CollectorToken); + } + + Log.Logger = loggerConfiguration.CreateLogger(); + + if (string.IsNullOrEmpty(config.SplunkConfig.Host)) + { + Log.Warning("*** Splunk Host is not configured - check Splunk environment *** "); + } + else + { + Log.Information($"*** Splunk logging to {config.SplunkConfig.Host} ***"); + } + var app = builder.Build(); // Configure the HTTP request pipeline. @@ -66,18 +104,14 @@ public static void Main(string[] args) app.UseSwaggerUI(); } - var logger = app.Services.GetRequiredService>(); - app.UseHttpsRedirection(); app.UseCors(); app.UseAuthorization(); app.MapMetrics(); app.MapHealthChecks("/health/liveness").AllowAnonymous(); - app.MapControllers(); app.Run(); } - } diff --git a/backend/DIAMCornetService/Services/BaseClient.cs b/backend/DIAMCornetService/Services/BaseClient.cs index 31dedf2c..e7c6b3a5 100644 --- a/backend/DIAMCornetService/Services/BaseClient.cs +++ b/backend/DIAMCornetService/Services/BaseClient.cs @@ -4,6 +4,7 @@ namespace DIAMCornetService.Services; using System.Net.Mime; using System.Text; using System.Text.Json; +using Common.Logging; using DomainResults.Common; using Flurl; @@ -294,14 +295,3 @@ private async Task> SendCoreInternalAsync(HttpMethod method, } } -public static partial class BaseClientLoggingExtensions -{ - [LoggerMessage(1, LogLevel.Error, "Received non-success status code {statusCode} with message: {responseMessage}.")] - public static partial void LogNonSuccessStatusCode(this ILogger logger, HttpStatusCode statusCode, string responseMessage); - - [LoggerMessage(2, LogLevel.Error, "Response content was null.")] - public static partial void LogNullResponseContent(this ILogger logger); - - [LoggerMessage(3, LogLevel.Error, "Unhandled exception when calling the API.")] - public static partial void LogBaseClientException(this ILogger logger, Exception e); -} diff --git a/backend/DIAMCornetService/Services/CornetORDSClient.cs b/backend/DIAMCornetService/Services/CornetORDSClient.cs index 06affa35..2d0ddc4b 100644 --- a/backend/DIAMCornetService/Services/CornetORDSClient.cs +++ b/backend/DIAMCornetService/Services/CornetORDSClient.cs @@ -5,7 +5,7 @@ namespace DIAMCornetService.Services; using DIAMCornetService.Models; using DomainResults.Common; -public class CornetORDSClient(HttpClient httpClient, ILogger logger) : BaseClient(httpClient, logger), ICornetORDSClient +public class CornetORDSClient(ILogger logger, HttpClient httpClient) : BaseClient(httpClient, logger), ICornetORDSClient { /// @@ -14,7 +14,7 @@ public class CornetORDSClient(HttpClient httpClient, ILogger l /// /// /// - public async Task GetCSNumberForParticipantAsync(string participantId) + public async Task GetCSNumberForParticipantAsync(string participantId) { var response = await this.GetAsync($"v1/participant/{participantId}/csnumber"); @@ -22,7 +22,7 @@ public async Task GetCSNumberForParticipantAsync(strin if (response.IsSuccess) { - return new ParticipantResponseModel() + return new InCustodyParticipantModel() { CSNumber = response.Value.Csnum, ParticipantId = participantId @@ -30,7 +30,7 @@ public async Task GetCSNumberForParticipantAsync(strin } else if (response.Status == DomainOperationStatus.NotFound) { - return new ParticipantResponseModel() + return new InCustodyParticipantModel() { ParticipantId = participantId, ErrorMessage = "Participant Not Found", @@ -51,31 +51,44 @@ public async Task GetCSNumberForParticipantAsync(strin /// /// /// - public async Task SubmitParticipantNotificationAsync(ParticipantResponseModel model, string messageText) + public async Task SubmitParticipantNotificationAsync(InCustodyParticipantModel model, string messageText) { - var response = await this.PostAsync($"v1/participant/{model.ParticipantId}/disclosure-message", new CornetMessageInput() { MessageTxt = messageText }); - var responseText = response.Value.ResponseCode; - - // todo determine possible response codes - if (response.IsSuccess && string.IsNullOrEmpty(responseText)) + try { + var response = await this.PostAsync($"v1/participant/{model.ParticipantId}/disclosure-message", new CornetMessageInput() { MessageTxt = messageText }); + + + var responseText = response.Value.ResponseCode; + + // todo determine possible response codes + if (response.IsSuccess && !string.IsNullOrEmpty(responseText) && responseText.Equals("SUCC", StringComparison.Ordinal)) + { + logger.SubmissionSuccessful(model.ParticipantId, model.CSNumber); + return responseText; + + } + else if (response.IsSuccess && !string.IsNullOrEmpty(responseText)) + { + logger.LogWarning($"Failed to submit notification for participant {model.ParticipantId} CS: {model.CSNumber} Reason: {responseText}"); + return responseText; + } + else + { + logger.LogError($"Failed to submit notification for participant {model.ParticipantId} CS: {model.CSNumber} Reason: {responseText}"); - logger.LogInformation($"Successfully submitted notification for participant {model.ParticipantId} CS: {model.CSNumber}"); - return "ok"; + throw new CornetException($"Failed to submit notification for participant {model.ParticipantId} CS: {model.CSNumber}"); + } } - else if (response.IsSuccess && !string.IsNullOrEmpty(responseText)) + catch (Exception ex) { - logger.LogWarning($"Failed to submit notification for participant {model.ParticipantId} CS: {model.CSNumber} Reason: {responseText}"); - return responseText; + Logger.LogError(ex, $"Failed to submit notification for participant {model.ParticipantId} CS: {model.CSNumber} [{ex.Message}]"); + throw ex; } - else - { - logger.LogError($"Failed to submit notification for participant {model.ParticipantId} CS: {model.CSNumber} Reason: {responseText}"); - throw new CornetException($"Failed to submit notification for participant {model.ParticipantId} CS: {model.CSNumber}"); - } + + } @@ -97,3 +110,10 @@ private class CornetResponse } } +public static partial class CornetORDSClientLoggingExtensions +{ + [LoggerMessage(1, LogLevel.Information, "Successfully submitted notification for participant {participantId} CS: {CSNumber}")] + public static partial void SubmissionSuccessful(this ILogger logger, string participantId, string CSNumber); + [LoggerMessage(2, LogLevel.Error, "Failed to submit notification for participant {participantId} CS: {CSNumber} Reason: {reason}")] + public static partial void FailedToSubmitNotification(this ILogger logger, string participantId, string CSNumber, string reason); +} diff --git a/backend/DIAMCornetService/Services/CornetService.cs b/backend/DIAMCornetService/Services/CornetService.cs index 974f5b80..31659240 100644 --- a/backend/DIAMCornetService/Services/CornetService.cs +++ b/backend/DIAMCornetService/Services/CornetService.cs @@ -3,10 +3,12 @@ namespace DIAMCornetService.Services; using System.Threading.Tasks; using Common.Kafka; using Confluent.Kafka; +using DIAM.Common.Models; using DIAMCornetService.Exceptions; using DIAMCornetService.Models; +using NodaTime; -public class CornetService(IKafkaProducer producer, DIAMCornetServiceConfiguration cornetServiceConfiguration, ICornetORDSClient cornetORDSClient, ILogger logger) : ICornetService +public class CornetService(ILogger logger, IKafkaProducer producer, IKafkaProducer errorProducer, DIAMCornetServiceConfiguration cornetServiceConfiguration, ICornetORDSClient cornetORDSClient) : ICornetService { /// @@ -15,17 +17,19 @@ public class CornetService(IKafkaProducer prod /// /// /// - public async Task PublishNotificationToDIAMAsync(ParticipantResponseModel model) + public async Task PublishNotificationToDIAMAsync(InCustodyParticipantModel model) { try { var guid = Guid.NewGuid(); + model.CreationDateUTC = DateTime.UtcNow; + var response = await producer.ProduceAsync(cornetServiceConfiguration.KafkaCluster.ParticipantCSNumberMappingTopic, guid.ToString(), model); if (response.Status != PersistenceStatus.Persisted) { - logger.LogError($"Failed to publish CS number lookup to topic for {model.ParticipantId}"); + logger.DIAMResponseFailed(model.ParticipantId, cornetServiceConfiguration.KafkaCluster.ParticipantCSNumberMappingTopic, response.Status.ToString()); throw new DIAMKafkaException($"Failed to publish cs number mapping for {guid.ToString()} Part: {model.ParticipantId} CSNumber: {model.CSNumber}"); } else @@ -35,7 +39,7 @@ public async Task PublishNotificationToDIAMAsync(Parti } catch (Exception ex) { - logger.LogError($"Failed to publish CS number lookup to topic for {model.ParticipantId} [{ex.Message}]"); + logger.DIAMResponseFailed(model.ParticipantId, cornetServiceConfiguration.KafkaCluster.ParticipantCSNumberMappingTopic, ex.Message); throw; } @@ -43,17 +47,29 @@ public async Task PublishNotificationToDIAMAsync(Parti } - public async Task PublishErrorsToDIAMAsync(ParticipantResponseModel model) + public async Task PublishErrorsToDIAMAsync(InCustodyParticipantModel model) { try { var guid = Guid.NewGuid(); - var response = await producer.ProduceAsync(cornetServiceConfiguration.KafkaCluster.ProcessResponseTopic, guid.ToString(), model); + var responseModel = new GenericProcessStatusResponse() + { + DomainEvent = "digitalevidence-incustody-notify-failure", + ErrorList = [model.ErrorMessage], + EventTime = SystemClock.Instance.GetCurrentInstant(), + PartId = model.ParticipantId!, + Status = "Error" + }; + + model.CreationDateUTC = DateTime.UtcNow; + + var response = await errorProducer.ProduceAsync(cornetServiceConfiguration.KafkaCluster.ProcessResponseTopic, guid.ToString(), responseModel); if (response.Status != PersistenceStatus.Persisted) { - logger.LogError($"Failed to publish Error response for {model.ParticipantId} to {cornetServiceConfiguration.KafkaCluster.ProcessResponseTopic}"); + logger.DIAMErrorResponseFailed(model.ParticipantId, cornetServiceConfiguration.KafkaCluster.ProcessResponseTopic, response.Status.ToString()); + throw new DIAMKafkaException($"Failed to publish Error response for {model.ParticipantId} to {cornetServiceConfiguration.KafkaCluster.ProcessResponseTopic}"); } else @@ -63,7 +79,7 @@ public async Task PublishErrorsToDIAMAsync(Participant } catch (Exception ex) { - logger.LogError($"Failed to publish Error response for {model.ParticipantId} to {cornetServiceConfiguration.KafkaCluster.ProcessResponseTopic} [{ex.Message}]"); + logger.DIAMErrorResponseFailed(model.ParticipantId, cornetServiceConfiguration.KafkaCluster.ProcessResponseTopic, ex.Message); throw; } @@ -76,7 +92,7 @@ public async Task PublishErrorsToDIAMAsync(Participant /// /// /// - public async Task LookupCSNumberForParticipant(string participantId) + public async Task LookupCSNumberForParticipant(string participantId) { var responseModel = await cornetORDSClient.GetCSNumberForParticipantAsync(participantId); @@ -91,29 +107,30 @@ public async Task LookupCSNumberForParticipant(string /// /// /// - public async Task SubmitNotificationToEServices(ParticipantResponseModel model, string message) + public async Task SubmitNotificationToEServices(InCustodyParticipantModel model, string message) { - logger.Log(LogLevel.Information, new EventId(), $"Publish notification for partId: {model.ParticipantId} CS: {model.CSNumber} {message}", null, (state, ex) => state.ToString()); + + logger.PublishingToEServices(model.ParticipantId, model.CSNumber); var response = await cornetORDSClient.SubmitParticipantNotificationAsync(model, message); switch (response) { - case "ok": + case "SUCC": { - logger.Log(LogLevel.Information, new EventId(), $"Successfully published notification for partId: {model.ParticipantId} CS: {model.CSNumber} {message}", null, (state, ex) => state.ToString()); + logger.NotificationSubmissionSuccessful(model.ParticipantId, model.CSNumber); break; } case "NABI": { - logger.LogInformation($"Failed to publish notification for partId: {model.ParticipantId} CS: {model.CSNumber} {message} Reason: {response}"); + logger.NotificationSubmissionFailed(model.ParticipantId, model.CSNumber, response); model.ErrorMessage = "Participant has no active BioMetrics"; model.ErrorType = CornetCSNumberErrorType.noActiveBioMetrics; break; } case "ENDP": { - logger.LogInformation($"Failed to publish notification for partId: {model.ParticipantId} CS: {model.CSNumber} {message} Reason: {response}"); + logger.NotificationSubmissionFailed(model.ParticipantId, model.CSNumber, response); model.ErrorMessage = "eDisclosure not provisioned for user"; model.ErrorType = CornetCSNumberErrorType.eDisclosureNotProvisioned; break; @@ -121,21 +138,21 @@ public async Task SubmitNotificationToEServices(Partic // shouldnt get to this one as CS number is looked up previously but added to cover all responses case "MISC": { - logger.LogInformation($"Failed to publish notification for partId: {model.ParticipantId} CS: {model.CSNumber} {message} Reason: {response}"); + logger.NotificationSubmissionFailed(model.ParticipantId, model.CSNumber, response); model.ErrorMessage = "Missing CS Number"; model.ErrorType = CornetCSNumberErrorType.missingCSNumber; break; } case "OTHR": { - logger.LogInformation($"Failed to publish notification for partId: {model.ParticipantId} CS: {model.CSNumber} {message} Reason: {response}"); + logger.NotificationSubmissionFailed(model.ParticipantId, model.CSNumber, response); model.ErrorMessage = "Unknown CORNET error occurred"; model.ErrorType = CornetCSNumberErrorType.otherError; break; } default: { - logger.LogInformation($"Failed to publish notification for partId: {model.ParticipantId} CS: {model.CSNumber} {message} Reason: {response}"); + logger.NotificationSubmissionFailed(model.ParticipantId, model.CSNumber, response); model.ErrorMessage = $"Unknown CORNET response {response}"; model.ErrorType = CornetCSNumberErrorType.unknownResponseError; break; @@ -146,7 +163,18 @@ public async Task SubmitNotificationToEServices(Partic } - private static int GetRandomEightDigitNumber() => new Random().Next(10000000, 99999999); - } +public static partial class CornetServiceLoggingExtensions +{ + [LoggerMessage(1, LogLevel.Information, "Successfully published notification for partId: {participantId} CS: {CSNumber}")] + public static partial void NotificationSubmissionSuccessful(this ILogger logger, string? participantId, string? CSNumber); + [LoggerMessage(2, LogLevel.Error, "Failed to submit eServices notification for partId: {participantId} CS: {CSNumber} - Reason {reason}")] + public static partial void NotificationSubmissionFailed(this ILogger logger, string? participantId, string? CSNumber, string reason); + [LoggerMessage(3, LogLevel.Error, "Failed to publish Error response for {participantId} to {topic} [{message}]")] + public static partial void DIAMErrorResponseFailed(this ILogger logger, string? participantId, string? topic, string? message); + [LoggerMessage(4, LogLevel.Error, "Failed to publish CS Number response {participantId} to {topic} [{message}]")] + public static partial void DIAMResponseFailed(this ILogger logger, string? participantId, string? topic, string? message); + [LoggerMessage(5, LogLevel.Information, "Publishing eServices notification to {participantId} CS: {CSNumber}")] + public static partial void PublishingToEServices(this ILogger logger, string? participantId, string? CSNumber); +} diff --git a/backend/DIAMCornetService/Services/ICornetORDSClient.cs b/backend/DIAMCornetService/Services/ICornetORDSClient.cs index 8d29232c..04e433a6 100644 --- a/backend/DIAMCornetService/Services/ICornetORDSClient.cs +++ b/backend/DIAMCornetService/Services/ICornetORDSClient.cs @@ -5,8 +5,8 @@ namespace DIAMCornetService.Services; public interface ICornetORDSClient { // get a cs number for a participant - Task GetCSNumberForParticipantAsync(string participantId); + Task GetCSNumberForParticipantAsync(string participantId); // send a notification to a participant - Task SubmitParticipantNotificationAsync(ParticipantResponseModel model, string messageText); + Task SubmitParticipantNotificationAsync(InCustodyParticipantModel model, string messageText); } diff --git a/backend/DIAMCornetService/Services/ICornetService.cs b/backend/DIAMCornetService/Services/ICornetService.cs index 8bdf5c6e..1d156e0f 100644 --- a/backend/DIAMCornetService/Services/ICornetService.cs +++ b/backend/DIAMCornetService/Services/ICornetService.cs @@ -8,7 +8,7 @@ public interface ICornetService /// /// /// - public Task LookupCSNumberForParticipant(string participantId); + public Task LookupCSNumberForParticipant(string participantId); /// /// Submit a message to eServices devices for Participant @@ -17,7 +17,7 @@ public interface ICornetService /// /// /// - public Task SubmitNotificationToEServices(ParticipantResponseModel model, string message); + public Task SubmitNotificationToEServices(InCustodyParticipantModel model, string message); /// /// Publish the participant and CS Number to the DIAM topic @@ -25,7 +25,7 @@ public interface ICornetService /// /// /// - public Task PublishNotificationToDIAMAsync(ParticipantResponseModel model); + public Task PublishNotificationToDIAMAsync(InCustodyParticipantModel model); /// /// Publish and Error notifications to DIAM @@ -33,5 +33,5 @@ public interface ICornetService /// /// /// - public Task PublishErrorsToDIAMAsync(ParticipantResponseModel model); + public Task PublishErrorsToDIAMAsync(InCustodyParticipantModel model); } diff --git a/backend/DIAMCornetService/appsettings.json b/backend/DIAMCornetService/appsettings.json index 249c2440..0c517d13 100644 --- a/backend/DIAMCornetService/appsettings.json +++ b/backend/DIAMCornetService/appsettings.json @@ -15,6 +15,10 @@ "Username": "cornet", "Password": "cornet" }, + "SplunkConfig": { + "Host": "", + "CollectorToken": "" + }, "KafkaCluster": { "BootstrapServers": "", "DisclosureNotificationTopic": "digitalevidencedisclosure-notification-topic", From c9f50a4129d8c05228787254b64ed03ece4b8e55 Mon Sep 17 00:00:00 2001 From: Lee Wright <258036@NTTDATA.COM> Date: Mon, 15 Jul 2024 16:59:14 -0700 Subject: [PATCH 3/5] remove hard-coded realms. Start in-custody provisioning WIP --- .../common/Constants/Auth/RealmConstants.cs | 8 + .../common/Logging/CommonLoggingExtensions.cs | 30 +++ .../CORNET/InCustodyParticipantModel.cs | 20 ++ .../Keycloak/IKeycloakAdministrationClient.cs | 2 +- .../Keycloak/KeycloakAdministrationClient.cs | 5 +- .../Handler/CaseAccessRequestHandler.cs | 2 +- .../Keycloak/IKeycloakAdministrationClient.cs | 4 +- .../Keycloak/KeycloakAdministrationClient.cs | 42 ++-- .../ServiceEvents/KafkaProducer.cs | 2 +- backend/webapi/Data/PidpDbContext.cs | 12 + .../AccessRequests/DigitalEvidence.cs | 4 +- .../AccessRequests/DigitalEvidenceDefence.cs | 2 +- .../AccessRequests/DigitalEvidenceUpdate.cs | 26 +- .../AccessRequests/HcimAccountTransfer.cs | 8 +- .../Features/AccessRequests/SAEforms.cs | 2 +- backend/webapi/Features/AccessRequests/Uci.cs | 2 +- .../Features/AccessRequests/ValidateUser.cs | 2 +- .../IdentityProviderQuery.cs | 2 +- backend/webapi/Features/Admin/PartyDelete.cs | 4 +- .../webapi/Features/Admin/PartyDetailQuery.cs | 8 +- .../SubmittingAgencyQuery.cs | 6 +- .../webapi/Features/Parties/Demographics.cs | 4 +- .../Parties/ProfileUpdateServiceImpl.cs | 4 +- .../Auth/AuthenticationSetup.cs | 4 +- .../Infrastructure/HttpClients/BaseClient.cs | 17 ++ .../HttpClients/HttpClientSetup.cs | 5 +- .../Keycloak/IKeycloakAdministrationClient.cs | 94 ++++++-- .../Keycloak/KeycloakAdministrationClient.cs | 164 +++++++++---- .../Keycloak/KeycloakApiDefinitions.cs | 27 ++- .../IInCustodyService.cs | 10 + .../InCustodyProvisioning/InCustodyHandler.cs | 49 ++++ .../InCustodyMessageConsumer.cs | 31 +++ .../InCustodyProvisioning/InCustodyService.cs | 225 ++++++++++++++++++ .../Notifications/NotificationAckHandler.cs | 4 +- backend/webapi/PidpConfiguration.cs | 3 +- backend/webapi/Program.cs | 15 +- backend/webapi/Startup.cs | 4 +- backend/webapi/appsettings.json | 1 + 38 files changed, 701 insertions(+), 153 deletions(-) create mode 100644 backend/common/Constants/Auth/RealmConstants.cs create mode 100644 backend/common/Logging/CommonLoggingExtensions.cs create mode 100644 backend/common/Models/CORNET/InCustodyParticipantModel.cs create mode 100644 backend/webapi/Kafka/Consumer/InCustodyProvisioning/IInCustodyService.cs create mode 100644 backend/webapi/Kafka/Consumer/InCustodyProvisioning/InCustodyHandler.cs create mode 100644 backend/webapi/Kafka/Consumer/InCustodyProvisioning/InCustodyMessageConsumer.cs create mode 100644 backend/webapi/Kafka/Consumer/InCustodyProvisioning/InCustodyService.cs diff --git a/backend/common/Constants/Auth/RealmConstants.cs b/backend/common/Constants/Auth/RealmConstants.cs new file mode 100644 index 00000000..8bb30e2d --- /dev/null +++ b/backend/common/Constants/Auth/RealmConstants.cs @@ -0,0 +1,8 @@ +namespace Common.Constants.Auth; + +public static class RealmConstants +{ + public const string BCPSRealm = "BCPS"; + public const string CorrectionsRealm = "Corrections"; + +} diff --git a/backend/common/Logging/CommonLoggingExtensions.cs b/backend/common/Logging/CommonLoggingExtensions.cs new file mode 100644 index 00000000..165842a9 --- /dev/null +++ b/backend/common/Logging/CommonLoggingExtensions.cs @@ -0,0 +1,30 @@ +namespace Common.Logging; +using System; +using System.Net; +using Microsoft.Extensions.Logging; + +public static partial class CommonLoggingExtensions +{ + //-------------------------------------------------------------------------------- + // Http Logging + //-------------------------------------------------------------------------------- + [LoggerMessage(1, LogLevel.Error, "Received non-success status code {statusCode} with message: {responseMessage}.")] + public static partial void LogNonSuccessStatusCode(this ILogger logger, HttpStatusCode statusCode, string responseMessage); + + [LoggerMessage(2, LogLevel.Error, "Response content was null.")] + public static partial void LogNullResponseContent(this ILogger logger); + + [LoggerMessage(3, LogLevel.Error, "Unhandled exception when calling the API.")] + public static partial void LogBaseClientException(this ILogger logger, Exception e); + + + //-------------------------------------------------------------------------------- + // Kafka Logging + //-------------------------------------------------------------------------------- + [LoggerMessage(4, LogLevel.Information, "Message {msgId} sent to partition {partition}")] + public static partial void LogKafkaMsgSent(this ILogger logger, string msgId, int partition); + + + [LoggerMessage(5, LogLevel.Error, "Message {msgId} failed to send Status: {status}")] + public static partial void LogKafkaMsgSendFailure(this ILogger logger, string msgId, string status); +} diff --git a/backend/common/Models/CORNET/InCustodyParticipantModel.cs b/backend/common/Models/CORNET/InCustodyParticipantModel.cs new file mode 100644 index 00000000..8a83539f --- /dev/null +++ b/backend/common/Models/CORNET/InCustodyParticipantModel.cs @@ -0,0 +1,20 @@ +namespace Common.Models.CORNET; + +public class InCustodyParticipantModel +{ + public string? CSNumber { get; set; } + public string? ParticipantId { get; set; } + public string? ErrorMessage { get; set; } + public string? DIAMPublishId { get; set; } + public CornetCSNumberErrorType? ErrorType { get; set; } +} + +public enum CornetCSNumberErrorType +{ + missingCSNumber, + noActiveBioMetrics, + eDisclosureNotProvisioned, + unknownResponseError, + participantNotFound, + otherError +} diff --git a/backend/edt.casemanagement/HttpClients/Keycloak/IKeycloakAdministrationClient.cs b/backend/edt.casemanagement/HttpClients/Keycloak/IKeycloakAdministrationClient.cs index e9eae6a9..300ece1c 100644 --- a/backend/edt.casemanagement/HttpClients/Keycloak/IKeycloakAdministrationClient.cs +++ b/backend/edt.casemanagement/HttpClients/Keycloak/IKeycloakAdministrationClient.cs @@ -9,6 +9,6 @@ public interface IKeycloakAdministrationClient /// Returns null if unccessful. /// /// - Task GetUser(Guid userId); + Task GetUser(string realm, Guid userId); } diff --git a/backend/edt.casemanagement/HttpClients/Keycloak/KeycloakAdministrationClient.cs b/backend/edt.casemanagement/HttpClients/Keycloak/KeycloakAdministrationClient.cs index 52b1620c..6c5a094d 100644 --- a/backend/edt.casemanagement/HttpClients/Keycloak/KeycloakAdministrationClient.cs +++ b/backend/edt.casemanagement/HttpClients/Keycloak/KeycloakAdministrationClient.cs @@ -2,7 +2,6 @@ namespace edt.service.HttpClients.Keycloak; -using System.Net; using edt.casemanagement.HttpClients; using EdtService.HttpClients.Keycloak; @@ -17,9 +16,9 @@ public KeycloakAdministrationClient(HttpClient httpClient, ILogger /// /// IDPs - public async Task GetUser(Guid userId) + public async Task GetUser(string realm, Guid userId) { - var result = await this.GetAsync($"users/{userId}"); + var result = await this.GetAsync($"{realm}/users/{userId}"); if (!result.IsSuccess) { return null; diff --git a/backend/edt.casemanagement/ServiceEvents/CaseManagement/Handler/CaseAccessRequestHandler.cs b/backend/edt.casemanagement/ServiceEvents/CaseManagement/Handler/CaseAccessRequestHandler.cs index 712fdca7..6265d665 100644 --- a/backend/edt.casemanagement/ServiceEvents/CaseManagement/Handler/CaseAccessRequestHandler.cs +++ b/backend/edt.casemanagement/ServiceEvents/CaseManagement/Handler/CaseAccessRequestHandler.cs @@ -54,7 +54,7 @@ public async Task HandleAsync(string consumerName, string key, SubAgencyDo using (CaseRequestDuration.NewTimer()) { - var userInfo = await this.keycloakAdministrationClient.GetUser(caseEvent.UserId); + var userInfo = await this.keycloakAdministrationClient.GetUser("BCPS", caseEvent.UserId); if (userInfo == null) diff --git a/backend/jumwebapi/Infrastructure/HttpClients/Keycloak/IKeycloakAdministrationClient.cs b/backend/jumwebapi/Infrastructure/HttpClients/Keycloak/IKeycloakAdministrationClient.cs index a572b95a..77344399 100644 --- a/backend/jumwebapi/Infrastructure/HttpClients/Keycloak/IKeycloakAdministrationClient.cs +++ b/backend/jumwebapi/Infrastructure/HttpClients/Keycloak/IKeycloakAdministrationClient.cs @@ -1,4 +1,4 @@ -namespace jumwebapi.Infrastructure.HttpClients.Keycloak; +namespace jumwebapi.Infrastructure.HttpClients.Keycloak; public interface IKeycloakAdministrationClient { @@ -46,7 +46,7 @@ public interface IKeycloakAdministrationClient /// Returns null if unccessful. /// /// - Task GetUser(Guid userId); + Task GetUser(string realm, Guid userId); /// /// Updates the User with the given Keycloak User Representation. diff --git a/backend/jumwebapi/Infrastructure/HttpClients/Keycloak/KeycloakAdministrationClient.cs b/backend/jumwebapi/Infrastructure/HttpClients/Keycloak/KeycloakAdministrationClient.cs index 8c58df5a..27b2e5de 100644 --- a/backend/jumwebapi/Infrastructure/HttpClients/Keycloak/KeycloakAdministrationClient.cs +++ b/backend/jumwebapi/Infrastructure/HttpClients/Keycloak/KeycloakAdministrationClient.cs @@ -1,4 +1,4 @@ -using System.Net; +using System.Net; namespace jumwebapi.Infrastructure.HttpClients.Keycloak; @@ -7,17 +7,17 @@ public class KeycloakAdministrationClient : BaseClient, IKeycloakAdministrationC public KeycloakAdministrationClient(HttpClient httpClient, ILogger logger) : base(httpClient, logger) { } - public async Task AssignClientRole(Guid userId, string clientId, string roleName) + public async Task AssignClientRole(string realm, Guid userId, string clientId, string roleName) { // We need both the name and ID of the role to assign it. - var role = await this.GetClientRole(clientId, roleName); + var role = await this.GetClientRole(realm, clientId, roleName); if (role == null) { return false; } // Keycloak expects an array of roles. - var result = await this.PostAsync($"users/{userId}/role-mappings/clients/{role.ContainerId}", new[] { role }); + var result = await this.PostAsync($"{realm}/users/{userId}/role-mappings/clients/{role.ContainerId}", new[] { role }); if (result.IsSuccess) { this.Logger.LogClientRoleAssigned(userId, clientId, roleName); @@ -26,17 +26,17 @@ public async Task AssignClientRole(Guid userId, string clientId, string ro return result.IsSuccess; } - public async Task AssignRealmRole(Guid userId, string roleName) + public async Task AssignRealmRole(string realm, Guid userId, string roleName) { // We need both the name and ID of the role to assign it. - var role = await this.GetRealmRole(roleName); + var role = await this.GetRealmRole(realm, roleName); if (role == null) { return false; } // Keycloak expects an array of roles. - var response = await this.PostAsync($"users/{userId}/role-mappings/realm", new[] { role }); + var response = await this.PostAsync($"{realm}/users/{userId}/role-mappings/realm", new[] { role }); if (response.IsSuccess) { this.Logger.LogRealmRoleAssigned(userId, roleName); @@ -64,7 +64,7 @@ public async Task AssignRealmRole(Guid userId, string roleName) return client; } - public async Task GetClientRole(string clientId, string roleName) + public async Task GetClientRole(string realm, string clientId, string roleName) { // Need ID of Client (not the same as ClientId!) to fetch roles. var client = await this.GetClient(clientId); @@ -73,7 +73,7 @@ public async Task AssignRealmRole(Guid userId, string roleName) return null; } - var result = await this.GetAsync>($"clients/{client.Id}/roles"); + var result = await this.GetAsync>($"{realm}/clients/{client.Id}/roles"); if (!result.IsSuccess) { @@ -90,9 +90,9 @@ public async Task AssignRealmRole(Guid userId, string roleName) return role; } - public async Task GetRealmRole(string roleName) + public async Task GetRealmRole(string realm, string roleName) { - var result = await GetAsync($"roles/{WebUtility.UrlEncode(roleName)}"); + var result = await GetAsync($"{realm}/roles/{WebUtility.UrlEncode(roleName)}"); if (!result.IsSuccess) { @@ -106,11 +106,11 @@ public async Task AssignRealmRole(Guid userId, string roleName) /// /// /// IDPs - - public async Task GetUser(Guid userId) + + public async Task GetUser(string realm, Guid userId) { - var result = await this.GetAsync($"users/{userId}"); + var result = await this.GetAsync($"{realm}/users/{userId}"); if (!result.IsSuccess) { return null; @@ -119,22 +119,22 @@ public async Task AssignRealmRole(Guid userId, string roleName) return result.Value; } - public async Task> IdentityProviders() + public async Task> IdentityProviders(string realm) { - var result = await this.GetAsync>($"identity-provider/instances"); + var result = await this.GetAsync>($"{realm}/identity-provider/instances"); //GetAsync($"roles/{WebUtility.UrlEncode(roleName)}"); return result.Value; } - public async Task UpdateUser(Guid userId, UserRepresentation userRep) + public async Task UpdateUser(string realm, Guid userId, UserRepresentation userRep) { - var result = await this.PutAsync($"users/{userId}", userRep); + var result = await this.PutAsync($"{realm}/users/{userId}", userRep); return result.IsSuccess; } - public async Task UpdateUser(Guid userId, Action updateAction) + public async Task UpdateUser(string realm, Guid userId, Action updateAction) { - var user = await this.GetUser(userId); + var user = await this.GetUser(realm, userId); if (user == null) { return false; @@ -142,7 +142,7 @@ public async Task UpdateUser(Guid userId, Action updat updateAction(user); - return await this.UpdateUser(userId, user); + return await this.UpdateUser(realm, userId, user); } } public static partial class KeycloakAdministrationClientLoggingExtensions diff --git a/backend/service.edt/ServiceEvents/KafkaProducer.cs b/backend/service.edt/ServiceEvents/KafkaProducer.cs index 19520dfb..26d1fd39 100644 --- a/backend/service.edt/ServiceEvents/KafkaProducer.cs +++ b/backend/service.edt/ServiceEvents/KafkaProducer.cs @@ -81,7 +81,7 @@ private static async void OauthTokenRefreshCallback(IClient client, string confi clientId ??= clusterConfig.GetValue("KafkaCluster:SaslOauthbearerProducerClientId"); tokenEndpoint ??= clusterConfig.GetValue("KafkaCluster:SaslOauthbearerTokenEndpointUrl"); - Log.Logger.Information("Pidp Kafka Producer getting token {0} {1}", tokenEndpoint, clientId); + Log.Logger.Debug("Pidp Kafka Producer getting token {0} {1}", tokenEndpoint, clientId); var accessTokenClient = new HttpClient(); Log.Logger.Debug("Producer getting token {0}", tokenEndpoint); diff --git a/backend/webapi/Data/PidpDbContext.cs b/backend/webapi/Data/PidpDbContext.cs index d37976a5..30a8148a 100644 --- a/backend/webapi/Data/PidpDbContext.cs +++ b/backend/webapi/Data/PidpDbContext.cs @@ -157,4 +157,16 @@ protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) } } + public async Task AddIdempotentConsumer(string messageId, string consumer) + { + await this.IdempotentConsumers.AddAsync(new IdempotentConsumer + { + MessageId = messageId, + Consumer = consumer + }); + await this.SaveChangesAsync(); + } + + public async Task HasMessageBeenProcessed(string messageId, string consumer) => await this.IdempotentConsumers.AnyAsync(x => x.MessageId == messageId && x.Consumer == consumer); + } diff --git a/backend/webapi/Features/AccessRequests/DigitalEvidence.cs b/backend/webapi/Features/AccessRequests/DigitalEvidence.cs index 6d7914e2..48e6d0ba 100644 --- a/backend/webapi/Features/AccessRequests/DigitalEvidence.cs +++ b/backend/webapi/Features/AccessRequests/DigitalEvidence.cs @@ -268,7 +268,7 @@ private async Task> PublishAccessReq private async Task UpdateKeycloakUser(Guid userId, IEnumerable assignedGroup, string partId) { - if (!await this.keycloakClient.UpdateUser(userId, (user) => user.SetPartId(partId))) + if (!await this.keycloakClient.UpdateUser(Common.Constants.Auth.RealmConstants.BCPSRealm, userId, (user) => user.SetPartId(partId))) { Serilog.Log.Logger.Error("Failed to set user {0} partId in keycloak", partId); @@ -276,7 +276,7 @@ private async Task UpdateKeycloakUser(Guid userId, IEnumerable HandleAsync(Command command) } // get the user details from keycloak and check they are valid - otherwise will require an approval step - var keycloakUser = await this.keycloakClient.GetUser(dto.UserId); + var keycloakUser = await this.keycloakClient.GetUser(Common.Constants.Auth.RealmConstants.BCPSRealm, dto.UserId); if (keycloakUser == null) { diff --git a/backend/webapi/Features/AccessRequests/DigitalEvidenceUpdate.cs b/backend/webapi/Features/AccessRequests/DigitalEvidenceUpdate.cs index 0cb095b7..25c6428e 100644 --- a/backend/webapi/Features/AccessRequests/DigitalEvidenceUpdate.cs +++ b/backend/webapi/Features/AccessRequests/DigitalEvidenceUpdate.cs @@ -125,7 +125,7 @@ public async Task HandleAsync(Command command) else { // determine what has changed - var keycloakUserInfo = await this.keycloakClient.GetUser(party.UserId); + var keycloakUserInfo = await this.keycloakClient.GetUser(Common.Constants.Auth.RealmConstants.BCPSRealm, party.UserId); Serilog.Log.Information($"Keycloak user {keycloakUserInfo}"); @@ -175,7 +175,7 @@ public async Task HandleAsync(Command command) // deactivate the account keycloakUserInfo!.Enabled = false; - var deactivated = await this.UpdateKeycloakUser(party.UserId, keycloakUserInfo); + var deactivated = await this.UpdateKeycloakUser(Common.Constants.Auth.RealmConstants.BCPSRealm, party.UserId, keycloakUserInfo); if (deactivated) { @@ -234,7 +234,7 @@ public async Task HandleAsync(Command command) if (newRegions.Count > 0) { Serilog.Log.Information($"Adding [{string.Join(",", newRegions)}] regions for user {party.Id}"); - var removedGroupsOk = await this.AddKeycloakUserRegions(party.UserId, newRegions); + var removedGroupsOk = await this.AddKeycloakUserRegions(Common.Constants.Auth.RealmConstants.BCPSRealm, party.UserId, newRegions); if (regionChanges.From.Count() == 0) { // went from no groups to having groups - user is now active @@ -245,14 +245,14 @@ public async Task HandleAsync(Command command) if (removedRegions.Count > 0) { Serilog.Log.Information($"Removing [{string.Join(",", newRegions)}] regions for user {party.Id}"); - var removedGroupsOk = await this.RemoveKeycloakUserRegions(party.UserId, removedRegions); + var removedGroupsOk = await this.RemoveKeycloakUserRegions(Common.Constants.Auth.RealmConstants.BCPSRealm, party.UserId, removedRegions); } } } - var updated = await this.UpdateKeycloakUser(party.UserId, keycloakUserInfo); + var updated = await this.UpdateKeycloakUser(Common.Constants.Auth.RealmConstants.BCPSRealm, party.UserId, keycloakUserInfo); await this.context.SaveChangesAsync(); @@ -301,13 +301,13 @@ public async Task HandleAsync(Command command) } - private async Task AddKeycloakUserRegions(Guid userId, IEnumerable groups) + private async Task AddKeycloakUserRegions(string realm, Guid userId, IEnumerable groups) { foreach (var group in groups) { - if (!await this.keycloakClient.AddGrouptoUser(userId, group)) + if (!await this.keycloakClient.AddGrouptoUser(realm, userId, group)) { Serilog.Log.Logger.Error("Failed to add user {0} group {1} to keycloak", userId, group); return false; @@ -317,13 +317,13 @@ private async Task AddKeycloakUserRegions(Guid userId, IEnumerable return true; } - private async Task RemoveKeycloakUserRegions(Guid userId, IEnumerable groups) + private async Task RemoveKeycloakUserRegions(string realm, Guid userId, IEnumerable groups) { foreach (var group in groups) { - if (!await this.keycloakClient.RemoveUserFromGroup(userId, group)) + if (!await this.keycloakClient.RemoveUserFromGroup(realm, userId, group)) { Serilog.Log.Logger.Error("Failed to remove user {0} from keycloak group {1} ", userId, group); return false; @@ -334,11 +334,11 @@ private async Task RemoveKeycloakUserRegions(Guid userId, IEnumerable UpdateKeycloakUser(Guid userId, UserRepresentation user) + private async Task UpdateKeycloakUser(string realm, Guid userId, UserRepresentation user) { Serilog.Log.Information($"Keycloak account update for {user.Email}"); - return await this.keycloakClient.UpdateUser(userId, user); + return await this.keycloakClient.UpdateUser(realm, userId, user); } /// @@ -412,7 +412,7 @@ private async Task DetermineUserChanges(ParticipantDetail justi { var justinAgencies = justinUserInfo.assignedAgencies.Select(agency => agency.agencyName).ToList(); - var groups = await this.keycloakClient.GetUserGroups(party.UserId); + var groups = await this.keycloakClient.GetUserGroups(Common.Constants.Auth.RealmConstants.BCPSRealm, party.UserId); // check which groups the user is in now var keycloakGroups = groups.Select(group => group.Name).ToList(); @@ -457,7 +457,7 @@ private async Task DetermineUserChanges(ParticipantDetail justi Serilog.Log.Information($"User {party.Id} has no granted agencies in JUSTIN - disabling account"); userChangeModel.BooleanChangeTypes.Add(ChangeType.ACTIVATION, new BooleanChangeType(true, false)); // see what regions were removed (if any) - var groups = await this.keycloakClient.GetUserGroups(party.UserId); + var groups = await this.keycloakClient.GetUserGroups(Common.Constants.Auth.RealmConstants.BCPSRealm, party.UserId); var keycloakGroups = groups.Select(group => group.Name).ToList(); var keycloakRegions = allRegions.Intersect(keycloakGroups).ToList(); if (keycloakRegions.Count > 0) diff --git a/backend/webapi/Features/AccessRequests/HcimAccountTransfer.cs b/backend/webapi/Features/AccessRequests/HcimAccountTransfer.cs index 47df61bf..901f6d01 100644 --- a/backend/webapi/Features/AccessRequests/HcimAccountTransfer.cs +++ b/backend/webapi/Features/AccessRequests/HcimAccountTransfer.cs @@ -108,7 +108,7 @@ public async Task> HandleAsync(Command command) return DomainResult.Success(new Model(authStatus)); } - if (!await this.UpdateKeycloakUser(dto.UserId, authStatus.OrgDetails, authStatus.HcimUserRole)) + if (!await this.UpdateKeycloakUser(Common.Constants.Auth.RealmConstants.BCPSRealm, dto.UserId, authStatus.OrgDetails, authStatus.HcimUserRole)) { return DomainResult.Failed(); } @@ -128,14 +128,14 @@ public async Task> HandleAsync(Command command) return DomainResult.Success(new Model(authStatus)); } - private async Task UpdateKeycloakUser(Guid userId, LdapLoginResponse.OrgDetails orgDetails, string hcimRole) + private async Task UpdateKeycloakUser(string realm, Guid userId, LdapLoginResponse.OrgDetails orgDetails, string hcimRole) { - if (!await this.keycloakClient.UpdateUser(userId, (user) => user.SetLdapOrgDetails(orgDetails))) + if (!await this.keycloakClient.UpdateUser(realm, userId, (user) => user.SetLdapOrgDetails(orgDetails))) { return false; } - if (!await this.keycloakClient.AssignClientRole(userId, this.hcimClientId, hcimRole)) + if (!await this.keycloakClient.AssignClientRole(realm, userId, this.hcimClientId, hcimRole)) { return false; } diff --git a/backend/webapi/Features/AccessRequests/SAEforms.cs b/backend/webapi/Features/AccessRequests/SAEforms.cs index aa8925f3..39e82149 100644 --- a/backend/webapi/Features/AccessRequests/SAEforms.cs +++ b/backend/webapi/Features/AccessRequests/SAEforms.cs @@ -76,7 +76,7 @@ public async Task HandleAsync(Command command) return DomainResult.Failed(); } - if (!await this.keycloakClient.AssignClientRole(dto.UserId, MohClients.SAEforms.ClientId, MohClients.SAEforms.AccessRole)) + if (!await this.keycloakClient.AssignClientRole(Common.Constants.Auth.RealmConstants.BCPSRealm, dto.UserId, MohClients.SAEforms.ClientId, MohClients.SAEforms.AccessRole)) { return DomainResult.Failed(); } diff --git a/backend/webapi/Features/AccessRequests/Uci.cs b/backend/webapi/Features/AccessRequests/Uci.cs index ca280e83..16929dc8 100644 --- a/backend/webapi/Features/AccessRequests/Uci.cs +++ b/backend/webapi/Features/AccessRequests/Uci.cs @@ -71,7 +71,7 @@ public async Task HandleAsync(Command command) return DomainResult.Failed(); } - if (!await this.keycloakClient.AssignClientRole(dto.UserId, MohClients.Uci.ClientId, MohClients.Uci.AccessRole)) + if (!await this.keycloakClient.AssignClientRole(Common.Constants.Auth.RealmConstants.BCPSRealm, dto.UserId, MohClients.Uci.ClientId, MohClients.Uci.AccessRole)) { return DomainResult.Failed(); } diff --git a/backend/webapi/Features/AccessRequests/ValidateUser.cs b/backend/webapi/Features/AccessRequests/ValidateUser.cs index 4eee18a2..1ac38416 100644 --- a/backend/webapi/Features/AccessRequests/ValidateUser.cs +++ b/backend/webapi/Features/AccessRequests/ValidateUser.cs @@ -94,7 +94,7 @@ public async Task> HandleAsync(Command com throw new DIAMAuthException($"Party {command.PartyId} was not found - request ignored"); } - var keycloakUser = await this.keycloakAdministrationClient.GetUser(party.UserId); + var keycloakUser = await this.keycloakAdministrationClient.GetUser(Common.Constants.Auth.RealmConstants.BCPSRealm, party.UserId); if (keycloakUser == null) { diff --git a/backend/webapi/Features/Admin/IdentityProviders/IdentityProviderQuery.cs b/backend/webapi/Features/Admin/IdentityProviders/IdentityProviderQuery.cs index d1087f14..4c4ed7c5 100644 --- a/backend/webapi/Features/Admin/IdentityProviders/IdentityProviderQuery.cs +++ b/backend/webapi/Features/Admin/IdentityProviders/IdentityProviderQuery.cs @@ -27,7 +27,7 @@ public IdentityProviderQueryHandler(IMapper mapper, PidpDbContext context, IKeyc public async Task> HandleAsync(IdentityProviderQuery query) { - var providers = await this.keycloakAdministrationClient.GetIdentityProviders(); + var providers = await this.keycloakAdministrationClient.GetIdentityProviders(Common.Constants.Auth.RealmConstants.BCPSRealm); return providers.ToList(); } } diff --git a/backend/webapi/Features/Admin/PartyDelete.cs b/backend/webapi/Features/Admin/PartyDelete.cs index 18f48539..6f10d2ce 100644 --- a/backend/webapi/Features/Admin/PartyDelete.cs +++ b/backend/webapi/Features/Admin/PartyDelete.cs @@ -75,7 +75,7 @@ public async Task RemoveClientRoles(Party party) continue; } - if (await this.client.RemoveClientRole(party.UserId, role)) + if (await this.client.RemoveClientRole(Common.Constants.Auth.RealmConstants.BCPSRealm, party.UserId, role)) { this.logger.LogRemoveSuccess(role.Name!, party.UserId); } @@ -97,7 +97,7 @@ public async Task RemoveClientRoles(Party party) var mohClient = MohClients.FromAccessType(accessType); if (mohClient != null) { - role = await this.client.GetClientRole(mohClient.Value.ClientId, mohClient.Value.AccessRole); + role = await this.client.GetClientRole(Common.Constants.Auth.RealmConstants.BCPSRealm, mohClient.Value.ClientId, mohClient.Value.AccessRole); if (role == null) { diff --git a/backend/webapi/Features/Admin/PartyDetailQuery.cs b/backend/webapi/Features/Admin/PartyDetailQuery.cs index bc1dd914..362e0844 100644 --- a/backend/webapi/Features/Admin/PartyDetailQuery.cs +++ b/backend/webapi/Features/Admin/PartyDetailQuery.cs @@ -26,7 +26,7 @@ public PartyDetailQueryHandler(IMapper mapper, PidpDbContext context, IKeycloakA this.context = context; this.keycloakAdministrationClient = keycloakAdministrationClient; this.jumClient = jumClient; - this.httpContextAccessor = httpContextAccessor; + this.httpContextAccessor = httpContextAccessor; } public async Task HandleAsync(PartyDetailQuery query) @@ -49,15 +49,15 @@ public PartyDetailQueryHandler(IMapper mapper, PidpDbContext context, IKeycloakA }; // get the keycloak user - var keycloakUser = await this.keycloakAdministrationClient.GetUser(user.UserId); + var keycloakUser = await this.keycloakAdministrationClient.GetUser(Common.Constants.Auth.RealmConstants.BCPSRealm, user.UserId); - var client = await this.keycloakAdministrationClient.GetClient("PIDP-SERVICE"); + var client = await this.keycloakAdministrationClient.GetClient(Common.Constants.Auth.RealmConstants.BCPSRealm, "PIDP-SERVICE"); if (keycloakUser != null && client != null) { partyModel.Enabled = keycloakUser.Enabled == true; partyModel.IdentityProvider = keycloakUser.Attributes.GetValueOrDefault("identityProvider").FirstOrDefault(); - var roles = await this.keycloakAdministrationClient.GetUserClientRoles(partyModel.KeycloakUserId, Guid.Parse(client.Id)); + var roles = await this.keycloakAdministrationClient.GetUserClientRoles(Common.Constants.Auth.RealmConstants.BCPSRealm, partyModel.KeycloakUserId, Guid.Parse(client.Id)); partyModel.Roles = roles.Select(role => role.Name).ToList(); } diff --git a/backend/webapi/Features/Admin/SubmittingAgencies/SubmittingAgencyQuery.cs b/backend/webapi/Features/Admin/SubmittingAgencies/SubmittingAgencyQuery.cs index c9052463..fdbb8a86 100644 --- a/backend/webapi/Features/Admin/SubmittingAgencies/SubmittingAgencyQuery.cs +++ b/backend/webapi/Features/Admin/SubmittingAgencies/SubmittingAgencyQuery.cs @@ -2,11 +2,11 @@ namespace Pidp.Features.Admin.SubmittingAgencies; using System.Threading.Tasks; using AutoMapper; +using AutoMapper.QueryableExtensions; +using Microsoft.EntityFrameworkCore; using Pidp.Data; using Pidp.Infrastructure.HttpClients.Keycloak; -using Microsoft.EntityFrameworkCore; using Pidp.Models; -using AutoMapper.QueryableExtensions; public record SubmittingAgencyQuery() : IQuery>; @@ -35,7 +35,7 @@ public async Task> HandleAsync(SubmittingAgencyQuery { if (!string.IsNullOrEmpty(response.IdpHint)) { - var provider = await this.keycloakAdministrationClient.GetIdentityProvider(response.IdpHint); + var provider = await this.keycloakAdministrationClient.GetIdentityProvider(Common.Constants.Auth.RealmConstants.BCPSRealm, response.IdpHint); if (provider != null) { response.HasIdentityProvider = true; diff --git a/backend/webapi/Features/Parties/Demographics.cs b/backend/webapi/Features/Parties/Demographics.cs index 9928d062..10382129 100644 --- a/backend/webapi/Features/Parties/Demographics.cs +++ b/backend/webapi/Features/Parties/Demographics.cs @@ -111,7 +111,7 @@ public async Task HandleAsync(Command command) Serilog.Log.Information($"Updating {party.Id} email to {command.Email} from {currentEmail}"); var messageId = Guid.NewGuid().ToString(); - var userInfo = await this.administrationClient.GetUser(party.UserId); + var userInfo = await this.administrationClient.GetUser(Common.Constants.Auth.RealmConstants.BCPSRealm, party.UserId); if (userInfo != null) { var changeModel = new UserChangeModel @@ -143,7 +143,7 @@ public async Task HandleAsync(Command command) await this.context.SaveChangesAsync(); changeModel.ChangeId = changeEntry.Entity.Id; userInfo.Email = command.Email; - await this.administrationClient.UpdateUser(party.UserId, userInfo); + await this.administrationClient.UpdateUser(Common.Constants.Auth.RealmConstants.BCPSRealm, party.UserId, userInfo); if (accessRequests) { diff --git a/backend/webapi/Features/Parties/ProfileUpdateServiceImpl.cs b/backend/webapi/Features/Parties/ProfileUpdateServiceImpl.cs index 8134b884..4329da7d 100644 --- a/backend/webapi/Features/Parties/ProfileUpdateServiceImpl.cs +++ b/backend/webapi/Features/Parties/ProfileUpdateServiceImpl.cs @@ -43,7 +43,7 @@ public async Task UpdateUserProfile(UpdatePersonContactInfoModel updatePer } // get the user from keycloak - var keycloakUserInfo = await this.keycloakAdministrationClient.GetUser(party.UserId); + var keycloakUserInfo = await this.keycloakAdministrationClient.GetUser(Common.Constants.Auth.RealmConstants.BCPSRealm, party.UserId); if (keycloakUserInfo == null) { Serilog.Log.Error($"Keycloak user not found for {updatePerson.PartyId} {party.UserId} - likely deleted from Keycloak"); @@ -83,7 +83,7 @@ public async Task UpdateUserProfile(UpdatePersonContactInfoModel updatePer if (changesDetected) { Serilog.Log.Information($"Updating keycloak info for user {party.UserId}"); - var updated = await this.keycloakAdministrationClient.UpdateUser(party.UserId, keycloakUserInfo); + var updated = await this.keycloakAdministrationClient.UpdateUser(Common.Constants.Auth.RealmConstants.BCPSRealm, party.UserId, keycloakUserInfo); response = updated; if (party.AccessRequests.Any()) diff --git a/backend/webapi/Infrastructure/Auth/AuthenticationSetup.cs b/backend/webapi/Infrastructure/Auth/AuthenticationSetup.cs index df70f6b3..c279145b 100644 --- a/backend/webapi/Infrastructure/Auth/AuthenticationSetup.cs +++ b/backend/webapi/Infrastructure/Auth/AuthenticationSetup.cs @@ -132,13 +132,13 @@ public static IServiceCollection AddKeycloakAuth(this IServiceCollection service private static Task OnForbidden(ForbiddenContext context) { - Serilog.Log.Warning($"Authentication challenge"); + Serilog.Log.Warning($"Authentication failure {context.Result.Failure}"); return Task.CompletedTask; } private static Task OnChallenge(JwtBearerChallengeContext context) { - Serilog.Log.Warning($"Authentication challenge"); + Serilog.Log.Warning($"Authentication challenge {context.Error}"); return Task.CompletedTask; } diff --git a/backend/webapi/Infrastructure/HttpClients/BaseClient.cs b/backend/webapi/Infrastructure/HttpClients/BaseClient.cs index 814271a8..d9ddc39a 100644 --- a/backend/webapi/Infrastructure/HttpClients/BaseClient.cs +++ b/backend/webapi/Infrastructure/HttpClients/BaseClient.cs @@ -154,6 +154,21 @@ private async Task> SendCoreInternalAsync(HttpMethod method, return DomainResult.Failed("Response content was null"); } + var text = await response.Content.ReadAsStringAsync(cancellationToken); + + // resource was created with empty response + if (response.StatusCode == HttpStatusCode.Created && string.IsNullOrEmpty(text)) + { + return DomainResult.Success(default!); + } + + // handle empty response + //if (text.Equals("[]", StringComparison.Ordinal)) + //{ + // this.Logger.LogEmptyResponseContent(); + // return DomainResult.Success(new List()); + //} + var deserializationResult = await response.Content.ReadFromJsonAsync(cancellationToken: cancellationToken); if (deserializationResult == null) { @@ -279,4 +294,6 @@ public static partial class BaseClientLoggingExtensions [LoggerMessage(4, LogLevel.Error, "Received non-success status code {statusCode} with message: {responseMessage}. [{url}]")] public static partial void LogNonSuccessStatusCodeWithURL(this ILogger logger, HttpStatusCode statusCode, string responseMessage, string url); + [LoggerMessage(5, LogLevel.Error, "Response content was empty.")] + public static partial void LogEmptyResponseContent(this ILogger logger); } diff --git a/backend/webapi/Infrastructure/HttpClients/HttpClientSetup.cs b/backend/webapi/Infrastructure/HttpClients/HttpClientSetup.cs index ff50eee8..d94a1549 100644 --- a/backend/webapi/Infrastructure/HttpClients/HttpClientSetup.cs +++ b/backend/webapi/Infrastructure/HttpClients/HttpClientSetup.cs @@ -1,6 +1,7 @@ namespace Pidp.Infrastructure.HttpClients; using System.Net; +using Common.Models.CORNET; using Confluent.Kafka; using IdentityModel.Client; using Pidp.Extensions; @@ -14,6 +15,7 @@ namespace Pidp.Infrastructure.HttpClients; using Pidp.Infrastructure.HttpClients.Plr; using Pidp.Kafka.Consumer; using Pidp.Kafka.Consumer.DomainEventResponses; +using Pidp.Kafka.Consumer.InCustodyProvisioning; using Pidp.Kafka.Consumer.JustinUserChanges; using Pidp.Kafka.Consumer.Notifications; using Pidp.Kafka.Consumer.Responses; @@ -132,10 +134,11 @@ public static IServiceCollection AddHttpClients(this IServiceCollection services services.AddHostedService(); services.AddScoped, JustinUserChangeHandler>(); services.AddScoped, DomainEventResponseHandler>(); + services.AddScoped, InCustodyHandler>(); services.AddHostedService(); services.AddHostedService(); - + services.AddHostedService(); services.AddHostedService(); return services; diff --git a/backend/webapi/Infrastructure/HttpClients/Keycloak/IKeycloakAdministrationClient.cs b/backend/webapi/Infrastructure/HttpClients/Keycloak/IKeycloakAdministrationClient.cs index 37908414..d59daf64 100644 --- a/backend/webapi/Infrastructure/HttpClients/Keycloak/IKeycloakAdministrationClient.cs +++ b/backend/webapi/Infrastructure/HttpClients/Keycloak/IKeycloakAdministrationClient.cs @@ -11,7 +11,7 @@ public interface IKeycloakAdministrationClient /// /// /// - Task AssignClientRole(Guid userId, string clientId, string roleName); + Task AssignClientRole(string realm, Guid userId, string clientId, string roleName); /// /// Assigns a realm-level role to the user, if it exists. @@ -19,14 +19,14 @@ public interface IKeycloakAdministrationClient /// /// /// - Task AssignRealmRole(Guid userId, string roleName); + Task AssignRealmRole(string realm, Guid userId, string roleName); /// /// Gets the Keycloak Client representation by ClientId. /// Returns null if unsuccessful. /// /// - Task GetClient(string clientId); + Task GetClient(string realm, string clientId); /// /// Gets the Keycloak Client Role representation by name. @@ -34,21 +34,39 @@ public interface IKeycloakAdministrationClient /// /// /// - Task GetClientRole(string clientId, string roleName); + Task GetClientRole(string realm, string clientId, string roleName); /// /// Gets the Keycloak Role representation by name. /// Returns null if unsuccessful. /// /// - Task GetRealmRole(string roleName); + Task GetRealmRole(string realm, string roleName); /// /// Gets the Keycloak User Representation for the user. /// Returns null if unsuccessful. /// /// - Task GetUser(Guid userId); + Task GetUser(string realm, Guid userId); + + + /// + /// Gets a keycloak user by username + /// + /// + /// + Task GetUserByUsername(string realm, string username); + + Task GetExtendedUserByUsername(string realm, string username); + + /// + /// Create a new user + /// + /// + /// + /// + Task CreateUser(string realm, ExtendedUserRepresentation user); /// /// Removes the given Client Role from the User. @@ -56,7 +74,7 @@ public interface IKeycloakAdministrationClient /// /// /// - Task RemoveClientRole(Guid userId, Role role); + Task RemoveClientRole(string realm, Guid userId, Role role); /// /// Updates the User with the given Keycloak User Representation. @@ -64,7 +82,7 @@ public interface IKeycloakAdministrationClient /// /// /// - Task UpdateUser(Guid userId, UserRepresentation userRep); + Task UpdateUser(string realm, Guid userId, UserRepresentation userRep); /// /// Fetches the User and updates with the given Action. @@ -72,24 +90,70 @@ public interface IKeycloakAdministrationClient /// /// /// - Task UpdateUser(Guid userId, Action updateAction); - Task AddGrouptoUser(Guid userId, string groupName); + Task UpdateUser(string realm, Guid userId, Action updateAction); + /// + /// + /// + /// + /// + /// + /// + Task AddGrouptoUser(string realm, Guid userId, string groupName); - Task RemoveUserFromGroup(Guid userId, string groupName); + /// + /// + /// + /// + /// + /// + /// + Task RemoveUserFromGroup(string realm, Guid userId, string groupName); - Task> GetUserGroups(Guid userId); + /// + /// + /// + /// + /// + /// + Task> GetUserGroups(string realm, Guid userId); - Task?> GetUserClientRoles(Guid userId, Guid clientId); + /// + /// + /// + /// + /// + /// + /// + Task?> GetUserClientRoles(string realm, Guid userId, Guid clientId); + /// + /// + /// + /// + /// Task GetRealm(string name); - Task GetIdentityProvider(string name); + /// + /// + /// + /// + /// + /// + Task GetIdentityProvider(string realm, string name); /// /// Get Identity providers within realm /// /// - Task> GetIdentityProviders(); + Task> GetIdentityProviders(string realm); + /// + /// + /// + /// + /// + /// + /// + Task LinkUserToIdentityProvider(string realm, ExtendedUserRepresentation user, IdentityProvider idp); } diff --git a/backend/webapi/Infrastructure/HttpClients/Keycloak/KeycloakAdministrationClient.cs b/backend/webapi/Infrastructure/HttpClients/Keycloak/KeycloakAdministrationClient.cs index a036fc61..a847a03a 100644 --- a/backend/webapi/Infrastructure/HttpClients/Keycloak/KeycloakAdministrationClient.cs +++ b/backend/webapi/Infrastructure/HttpClients/Keycloak/KeycloakAdministrationClient.cs @@ -1,25 +1,26 @@ namespace Pidp.Infrastructure.HttpClients.Keycloak; using System.Net; -using DomainResults.Common; +using Common.Exceptions; using global::Keycloak.Net.Models.RealmsAdmin; + // TODO Use DomainResult for success/fail? public class KeycloakAdministrationClient : BaseClient, IKeycloakAdministrationClient { public KeycloakAdministrationClient(HttpClient httpClient, ILogger logger) : base(httpClient, logger) { } - public async Task AssignClientRole(Guid userId, string clientId, string roleName) + public async Task AssignClientRole(string realm, Guid userId, string clientId, string roleName) { // We need both the name and ID of the role to assign it. - var role = await this.GetClientRole(clientId, roleName); + var role = await this.GetClientRole(realm, clientId, roleName); if (role == null) { return false; } // Keycloak expects an array of roles. - var result = await this.PostAsync($"users/{userId}/role-mappings/clients/{role.ContainerId}", new[] { role }); + var result = await this.PostAsync($"{realm}/users/{userId}/role-mappings/clients/{role.ContainerId}", new[] { role }); if (result.IsSuccess) { this.Logger.LogClientRoleAssigned(userId, roleName, clientId); @@ -27,15 +28,15 @@ public async Task AssignClientRole(Guid userId, string clientId, string ro return result.IsSuccess; } - public async Task AddGrouptoUser(Guid userId, string groupName) + public async Task AddGrouptoUser(string realm, Guid userId, string groupName) { - var group = await this.GetRealmGroup(groupName); + var group = await this.GetRealmGroup(realm, groupName); if (group == null) { return false; } //assign user to group - var response = await this.PutAsync($"users/{userId}/groups/{group.Id}"); + var response = await this.PutAsync($"{realm}/users/{userId}/groups/{group.Id}"); if (!response.IsSuccess) { this.Logger.LogRealmGroupAssigned(userId, groupName); @@ -44,15 +45,15 @@ public async Task AddGrouptoUser(Guid userId, string groupName) } - public async Task RemoveUserFromGroup(Guid userId, string groupName) + public async Task RemoveUserFromGroup(string realm, Guid userId, string groupName) { - var group = await this.GetRealmGroup(groupName); + var group = await this.GetRealmGroup(realm, groupName); if (group == null) { return false; } //assign user to group - var response = await this.DeleteAsync($"users/{userId}/groups/{group.Id}"); + var response = await this.DeleteAsync($"{realm}/users/{userId}/groups/{group.Id}"); if (!response.IsSuccess) { this.Logger.LogRealmGroupRemoved(userId, groupName); @@ -61,17 +62,17 @@ public async Task RemoveUserFromGroup(Guid userId, string groupName) } - public async Task AssignRealmRole(Guid userId, string roleName) + public async Task AssignRealmRole(string realm, Guid userId, string roleName) { // We need both the name and ID of the role to assign it. - var role = await this.GetRealmRole(roleName); + var role = await this.GetRealmRole(realm, roleName); if (role == null) { return false; } // Keycloak expects an array of roles. - var response = await this.PostAsync($"users/{userId}/role-mappings/realm", new[] { role }); + var response = await this.PostAsync($"{realm}/users/{userId}/role-mappings/realm", new[] { role }); if (response.IsSuccess) { this.Logger.LogRealmRoleAssigned(userId, roleName); @@ -80,7 +81,7 @@ public async Task AssignRealmRole(Guid userId, string roleName) return response.IsSuccess; } - public async Task GetClient(string clientId) + public async Task GetClient(string realm, string clientId) { var result = await this.GetAsync>("clients"); @@ -99,16 +100,16 @@ public async Task AssignRealmRole(Guid userId, string roleName) return client; } - public async Task GetClientRole(string clientId, string roleName) + public async Task GetClientRole(string realm, string clientId, string roleName) { // Need ID of Client (not the same as ClientId!) to fetch roles. - var client = await this.GetClient(clientId); + var client = await this.GetClient(realm, clientId); if (client == null) { return null; } - var result = await this.GetAsync>($"clients/{client.Id}/roles"); + var result = await this.GetAsync>($"{realm}/clients/{client.Id}/roles"); if (!result.IsSuccess) { @@ -131,17 +132,17 @@ public async Task AssignRealmRole(Guid userId, string roleName) /// /// /// - public async Task?> GetUserClientRoles(Guid userId, Guid clientId) + public async Task?> GetUserClientRoles(string realm, Guid userId, Guid clientId) { - - var response = await this.GetAsync?>($"users/{userId}/role-mappings/clients/{clientId}"); + + var response = await this.GetAsync?>($"{realm}/users/{userId}/role-mappings/clients/{clientId}"); return response.Value; } - public async Task GetRealmRole(string roleName) + public async Task GetRealmRole(string realm, string roleName) { - var result = await this.GetAsync($"roles/{WebUtility.UrlEncode(roleName)}"); + var result = await this.GetAsync($"{realm}/roles/{WebUtility.UrlEncode(roleName)}"); if (!result.IsSuccess) { @@ -151,9 +152,12 @@ public async Task AssignRealmRole(Guid userId, string roleName) return result.Value; } - public async Task GetIdentityProvider(string alias) + public async Task GetIdentityProvider(string realm, string name) { - IDomainResult? result = await this.GetAsync($"identity-provider/instances/{alias}"); + + this.Logger.LogIDPLookup(realm, name); + + var result = await this.GetAsync($"{realm}/identity-provider/instances/{name}"); if (!result.IsSuccess) { return null; @@ -164,7 +168,7 @@ public async Task GetIdentityProvider(string alias) public async Task GetRealm(string realm) { - IDomainResult? result = await this.GetAsync($"realms/{realm}"); + var result = await this.GetAsync($"realms/{realm}"); if (!result.IsSuccess) { @@ -174,9 +178,9 @@ public async Task GetRealm(string realm) return result.Value; } - public async Task GetRealmGroup(string groupName) + public async Task GetRealmGroup(string realm, string groupName) { - IDomainResult>? result = await this.GetAsync>($"groups?search={groupName}"); + var result = await this.GetAsync>($"{realm}/groups?search={groupName}"); if (!result.IsSuccess) { @@ -186,9 +190,9 @@ public async Task GetRealm(string realm) return result.Value.SingleOrDefault(); } - public async Task?> GetUserGroups(Guid userId) + public async Task?> GetUserGroups(string realm, Guid userId) { - var result = await this.GetAsync>($"users/{userId}/groups"); + var result = await this.GetAsync>($"{realm}/users/{userId}/groups"); if (!result.IsSuccess) { @@ -198,13 +202,40 @@ public async Task GetRealm(string realm) return result.Value; } + public async Task CreateUser(string realm, ExtendedUserRepresentation user) + { + var result = await this.PostAsync($"{realm}/users", user); + if (!result.IsSuccess) + { + return false; + } + + return true; + } + + + public async Task GetUserByUsername(string realm, string username) + { + var result = await this.GetAsync($"{realm}/users?username={username}"); + if (!result.IsSuccess) + { + throw new DIAMGeneralException($"Failed to get user by username [{string.Join(",", result.Errors)}]."); + } + else if (result.IsSuccess && result.Value == null) + { + // if the user is missing we should get a null response but a successful one + return null; + } + var userInfo = result.Value; + return userInfo; + } - public async Task GetUser(Guid userId) + public async Task GetUser(string realm, Guid userId) { - var result = await this.GetAsync($"users/{userId}"); + var result = await this.GetAsync($"{realm}/users/{userId}"); if (!result.IsSuccess) { return null; @@ -215,7 +246,7 @@ public async Task GetRealm(string realm) return userInfo; } - public async Task RemoveClientRole(Guid userId, Role role) + public async Task RemoveClientRole(string realm, Guid userId, Role role) { if (role.ClientRole != true) { @@ -223,20 +254,20 @@ public async Task RemoveClientRole(Guid userId, Role role) } // Keycloak expects an array of roles. - var response = await this.DeleteAsync($"users/{userId}/role-mappings/clients/{role.ContainerId}", new[] { role }); + var response = await this.DeleteAsync($"{realm}/users/{userId}/role-mappings/clients/{role.ContainerId}", new[] { role }); return response.IsSuccess; } - public async Task UpdateUser(Guid userId, UserRepresentation userRep) + public async Task UpdateUser(string realm, Guid userId, UserRepresentation userRep) { - var result = await this.PutAsync($"users/{userId}", userRep); + var result = await this.PutAsync($"{realm}/users/{userId}", userRep); return result.IsSuccess; } - public async Task UpdateUser(Guid userId, Action updateAction) + public async Task UpdateUser(string realm, Guid userId, Action updateAction) { - var user = await this.GetUser(userId); + var user = await this.GetUser(realm, userId); if (user == null) { return false; @@ -244,20 +275,66 @@ public async Task UpdateUser(Guid userId, Action updat updateAction(user); - return await this.UpdateUser(userId, user); + return await this.UpdateUser(realm, userId, user); } - public async Task> GetIdentityProviders() + public async Task> GetIdentityProviders(string realm) { - var result = await this.GetAsync>($"identity-provider/instances"); + var result = await this.GetAsync>($"{realm}/identity-provider/instances"); if (!result.IsSuccess) { - Serilog.Log.Error($"Failed to get identity providers [{string.Join(",",result.Errors)}]."); + Serilog.Log.Error($"Failed to get identity providers [{string.Join(",", result.Errors)}]."); return null; } return result.Value; } + + /// + /// Extended info includes ID not typically returned + /// + /// + /// + /// + public async Task GetExtendedUserByUsername(string realm, string username) + { + var result = await this.GetAsync>($"{realm}/users?username={username}"); + if (!result.IsSuccess) + { + return null; + } + + var userInfo = result.Value.FirstOrDefault(); + + return userInfo; + } + + + public async Task LinkUserToIdentityProvider(string realm, ExtendedUserRepresentation user, IdentityProvider idp) + { + var result = await this.PostAsync($"{realm}/users/{user.Id}/federated-identity/{idp.Alias}", new IdpLink() + { + UserId = user.Id.ToString(), + UserName = user.Username + } + ); + if (result.IsSuccess) + { + this.Logger.LogUserLinkedToIdp(user.Id, idp.ProviderId); + return true; + } + else + { + return false; + } + } + + private class IdpLink + { + public string UserId { get; set; } = string.Empty; + public string UserName { get; set; } = string.Empty; + } + } public static partial class KeycloakAdministrationClientLoggingExtensions @@ -277,6 +354,9 @@ public static partial class KeycloakAdministrationClientLoggingExtensions public static partial void LogRealmGroupAssigned(this ILogger logger, Guid userId, string groupName); [LoggerMessage(6, LogLevel.Information, "User {userId} was removed from Realm Group {groupName}.")] public static partial void LogRealmGroupRemoved(this ILogger logger, Guid userId, string groupName); + [LoggerMessage(7, LogLevel.Information, "User {userId} was linked to IDP {idp}.")] + public static partial void LogUserLinkedToIdp(this ILogger logger, Guid userId, string idp); + [LoggerMessage(8, LogLevel.Information, "Getting {realm} IDP {idp}.")] + public static partial void LogIDPLookup(this ILogger logger, string realm, string idp); - } diff --git a/backend/webapi/Infrastructure/HttpClients/Keycloak/KeycloakApiDefinitions.cs b/backend/webapi/Infrastructure/HttpClients/Keycloak/KeycloakApiDefinitions.cs index 8e0a250c..c7835c6b 100644 --- a/backend/webapi/Infrastructure/HttpClients/Keycloak/KeycloakApiDefinitions.cs +++ b/backend/webapi/Infrastructure/HttpClients/Keycloak/KeycloakApiDefinitions.cs @@ -73,13 +73,13 @@ public class UserRepresentation public string? LastName { get; set; } public string? FirstName { get; set; } - public List FederatedIdentities { get; set; } + public List FederatedIdentities { get; set; } = []; public bool Enabled { get; set; } = true; // enabled by default - // public List Groups { get; set; } = new List(); + // public List Groups { get; set; } = new List(); - public Dictionary Attributes { get; set; } = new(); + public Dictionary Attributes { get; set; } = []; internal void SetLdapOrgDetails(LdapLoginResponse.OrgDetails orgDetails) => this.SetAttribute("org_details", JsonSerializer.Serialize(orgDetails, new JsonSerializerOptions() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase })); @@ -92,13 +92,24 @@ public class UserRepresentation public void SetPhoneExtension(string phoneExtension) => this.SetAttribute("phoneExtension", phoneExtension); - private void SetAttribute(string key, string value) => this.Attributes[key] = new string[] { value }; + private void SetAttribute(string key, string value) => this.Attributes[key] = [value]; } - public class FederatedIdentityRepresentation { - public string IdentityProvider { get; set; } - public string UserId { get; set; } - public string UserName { get; set; } + public string? IdentityProvider { get; set; } + public string? UserId { get; set; } + public string? UserName { get; set; } + +} + +public class ExtendedUserRepresentation : UserRepresentation +{ + public Guid Id { get; set; } + public string? Username { get; set; } + public bool EmailVerified { get; set; } } + + + + diff --git a/backend/webapi/Kafka/Consumer/InCustodyProvisioning/IInCustodyService.cs b/backend/webapi/Kafka/Consumer/InCustodyProvisioning/IInCustodyService.cs new file mode 100644 index 00000000..cdcaa382 --- /dev/null +++ b/backend/webapi/Kafka/Consumer/InCustodyProvisioning/IInCustodyService.cs @@ -0,0 +1,10 @@ +namespace Pidp.Kafka.Consumer.InCustodyProvisioning; + +using Common.Models.CORNET; + +public interface IInCustodyService +{ + // process an incoming message + Task ProcessInCustodySubmissionMessage(InCustodyParticipantModel value); + +} diff --git a/backend/webapi/Kafka/Consumer/InCustodyProvisioning/InCustodyHandler.cs b/backend/webapi/Kafka/Consumer/InCustodyProvisioning/InCustodyHandler.cs new file mode 100644 index 00000000..cc32d52e --- /dev/null +++ b/backend/webapi/Kafka/Consumer/InCustodyProvisioning/InCustodyHandler.cs @@ -0,0 +1,49 @@ +namespace Pidp.Kafka.Consumer.InCustodyProvisioning; + +using Common.Models.CORNET; +using Pidp.Data; +using Pidp.Kafka.Interfaces; + +/// +/// Handles incoming in-custody messages from Kafka +/// +/// +/// +/// +public class InCustodyHandler(ILogger logger, PidpDbContext dbContext, IInCustodyService inCustodyService) : IKafkaHandler +{ + + public async Task HandleAsync(string consumerName, string key, InCustodyParticipantModel value) + { + + // check we havent consumed this message before + var processedAlready = await dbContext.HasMessageBeenProcessed(key, consumerName); + if (processedAlready) + { + logger.LogMessageAlreadyProcessed(key); + return Task.CompletedTask; + } + + logger.LogInCustodyMessageReceived(consumerName, key); + + + // service will create the keycloak account (if not present) and then inform Disclosure service to provision account + // and link cases to the user + var result = await inCustodyService.ProcessInCustodySubmissionMessage(value); + + if (result.IsCompleted) + { + + //add to tell message has been processed by consumer + await dbContext.AddIdempotentConsumer(messageId: key, consumer: consumerName); + } + + return result; + } +} +public static partial class InCustodyHandlerLoggingExtensions +{ + [LoggerMessage(1, LogLevel.Information, "InCustody Message received on {consumerName} with key {key}")] + public static partial void LogInCustodyMessageReceived(this ILogger logger, string consumerName, string key); + +} diff --git a/backend/webapi/Kafka/Consumer/InCustodyProvisioning/InCustodyMessageConsumer.cs b/backend/webapi/Kafka/Consumer/InCustodyProvisioning/InCustodyMessageConsumer.cs new file mode 100644 index 00000000..cde55e15 --- /dev/null +++ b/backend/webapi/Kafka/Consumer/InCustodyProvisioning/InCustodyMessageConsumer.cs @@ -0,0 +1,31 @@ +namespace Pidp.Kafka.Consumer.InCustodyProvisioning; + +using System.Net; +using Common.Models.CORNET; +using Pidp.Kafka.Interfaces; + +public class InCustodyMessageConsumer(IKafkaConsumer kafkaConsumer, PidpConfiguration config) : BackgroundService +{ + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + + Serilog.Log.Information("Starting consumer {0}", config.KafkaCluster.ParticipantCSNumberMappingTopic); + try + { + await kafkaConsumer.Consume(config.KafkaCluster.ParticipantCSNumberMappingTopic, stoppingToken); + } + catch (Exception ex) + { + Serilog.Log.Warning($"{(int)HttpStatusCode.InternalServerError} ConsumeFailedOnTopic - {config.KafkaCluster.ParticipantCSNumberMappingTopic}, {ex}"); + } + } + + public override void Dispose() + { + kafkaConsumer.Close(); + kafkaConsumer.Dispose(); + + base.Dispose(); + GC.SuppressFinalize(this); + } +} diff --git a/backend/webapi/Kafka/Consumer/InCustodyProvisioning/InCustodyService.cs b/backend/webapi/Kafka/Consumer/InCustodyProvisioning/InCustodyService.cs new file mode 100644 index 00000000..046e1684 --- /dev/null +++ b/backend/webapi/Kafka/Consumer/InCustodyProvisioning/InCustodyService.cs @@ -0,0 +1,225 @@ +namespace Pidp.Kafka.Consumer.InCustodyProvisioning; + +using Common.Constants.Auth; +using Common.Kafka; +using Common.Logging; +using Common.Models.CORNET; +using NodaTime; +using Pidp.Data; +using Pidp.Exceptions; +using Pidp.Infrastructure.HttpClients.Edt; +using Pidp.Infrastructure.HttpClients.Keycloak; +using Pidp.Models; +using Pidp.Models.Lookups; + +public class InCustodyService(IClock clock, PidpDbContext context, ILogger logger, IKafkaProducer producer, IEdtCoreClient coreClient, IKeycloakAdministrationClient keycloakAdministrationClient, PidpConfiguration pidpConfiguration) : IInCustodyService +{ + + public async Task ProcessInCustodySubmissionMessage(InCustodyParticipantModel value) + { + try + { + // check/add keycloak user + var keycloakUser = await this.AddOrUpdateKeycloakUserAsync(value); + + // create access request + var accessRequest = await this.CreateInCustodyAccessRequest(keycloakUser, value); + + // publish to disclosure portal + var publishResponse = await this.PublishDisclosurePortalMessage(accessRequest, value); + + return Task.CompletedTask; + } + catch (Exception ex) + { + logger.LogInCustodyServiceException(ex.Message, ex); + throw; + } + + } + + private async Task AddOrUpdateKeycloakUserAsync(InCustodyParticipantModel value) + { + try + { + // get the participant from the core service + logger.EdtCoreParticipantLookup(value.ParticipantId, value.CSNumber); + var participant = await coreClient.GetPersonByKey(value.ParticipantId); + + if (participant == null) + { + // log and return + logger.LogParticipantNotFound(value.ParticipantId); + throw new AccessRequestException($"No participant found in core with key {value.ParticipantId}"); + } + + logger.LogParticipantFound(value.ParticipantId, participant.FirstName, participant.LastName); + + // username will always be the cs number @ incustody.bcprosecution.gov.bc.ca + var username = $"{value.CSNumber}@incustody.bcprosecution.gov.bc.ca"; + logger.LogCheckingKeycloakUser(username, value.ParticipantId); + var keycloakUser = await keycloakAdministrationClient.GetExtendedUserByUsername("Corrections", username); + + if (keycloakUser == null) + { + logger.LogKeycloakUserNotPresent(username, value.ParticipantId); + var createdUser = await keycloakAdministrationClient.CreateUser("Corrections", new ExtendedUserRepresentation() + { + FirstName = participant.FirstName, + LastName = participant.LastName, + Username = username, + Email = username, + EmailVerified = true, + Enabled = true + + }); + + if (createdUser) + { + keycloakUser = await keycloakAdministrationClient.GetExtendedUserByUsername("Corrections", username); + + if (keycloakUser == null) + { + logger.LogKeycloakUserCreationFailed(username, value.ParticipantId); + throw new AccessRequestException($"Failed to create keycloak user for {username}"); + } + // get idp + + var correctionsIdp = await keycloakAdministrationClient.GetIdentityProvider(RealmConstants.CorrectionsRealm, pidpConfiguration.CorrectionsIDP); + if (correctionsIdp != null) + { + // link the user to the corrections IDP + await keycloakAdministrationClient.LinkUserToIdentityProvider(RealmConstants.CorrectionsRealm, keycloakUser, correctionsIdp); + logger.LogKeycloakUserCreationSuccess(username, keycloakUser.Id.ToString(), value.ParticipantId); + + } + else + { + logger.LogIDPNotFound(RealmConstants.CorrectionsRealm, pidpConfiguration.CorrectionsIDP); + throw new AccessRequestException($"Error processing keycloak user - PartID: {value.ParticipantId} - IDP not found"); + + + } + } + else + { + logger.LogKeycloakUserCreationFailed(username, value.ParticipantId); + throw new AccessRequestException($"Error processing keycloak user - PartID: {value.ParticipantId} - failed to create user"); + + } + + + } + + + + // return the user + return keycloakUser; + + } + catch (Exception ex) + { + logger.LogKeycloakServiceException(ex.Message, ex); + throw new AccessRequestException($"Error processing keycloak user - PartID: {value.ParticipantId} {ex.Message}"); + } + + + } + + + private async Task CreateInCustodyAccessRequest(ExtendedUserRepresentation keycloakUser, InCustodyParticipantModel value) + { + + + var party = new Party + { + UserId = keycloakUser.Id, + Jpdid = keycloakUser.Username, + FirstName = keycloakUser.FirstName!, + LastName = keycloakUser.LastName!, + Email = keycloakUser.Username + }; + + context.Parties.Add(party); + + + var partyAdded = await context.SaveChangesAsync(); + + if (partyAdded > 0) + { + var accessRequest = new AccessRequest + { + Party = party, + AccessTypeCode = AccessTypeCode.DigitalEvidenceDisclosure, + RequestedOn = clock.GetCurrentInstant() + }; + + context.AccessRequests.Add(accessRequest); + + await context.SaveChangesAsync(); + + return accessRequest; + } + else + { + throw new AccessRequestException($"Failed to create party for {keycloakUser.Username}"); + } + + + } + + + /// + /// Publish a message to tell disclosure service that a new user should be created + /// This will be with the username for corrections users + /// + /// + /// + public async Task PublishDisclosurePortalMessage(AccessRequest accessRequest, InCustodyParticipantModel value) + { + + var msgId = Guid.NewGuid().ToString(); + // publish to the topic for disclosure portal to handle provisioning + var delivered = await producer.ProduceAsync(pidpConfiguration.KafkaCluster.DisclosurePublicUserCreationTopic, msgId, accessRequest); + + if (delivered.Status == Confluent.Kafka.PersistenceStatus.Persisted) + { + logger.LogKafkaMsgSent(msgId, delivered.Partition.Value); + } + else + { + logger.LogKafkaMsgSendFailure(msgId, delivered.Status.ToString()); + } + + return Task.CompletedTask; + } +} + +public static partial class InCustodyServiceLoggingExtensions +{ + [LoggerMessage(1, LogLevel.Information, "Looking up participant by key {key} CS#: [{csNumber}]")] + public static partial void EdtCoreParticipantLookup(this ILogger logger, string key, string csNumber); + [LoggerMessage(2, LogLevel.Warning, "Already processed message with key {key}")] + public static partial void LogMessageAlreadyProcessed(this ILogger logger, string key); + [LoggerMessage(3, LogLevel.Error, "Participant not found in core with key {key}")] + public static partial void LogParticipantNotFound(this ILogger logger, string key); + [LoggerMessage(4, LogLevel.Information, "Participant found {key} [{firstName} {lastName}]")] + public static partial void LogParticipantFound(this ILogger logger, string key, string firstName, string lastName); + [LoggerMessage(5, LogLevel.Information, "Checking for existing Keycloak user [{username}] PartID: {key}")] + public static partial void LogCheckingKeycloakUser(this ILogger logger, string username, string key); + [LoggerMessage(6, LogLevel.Information, "Keycloak user not present for [{username}] PartID: {key} - will be added")] + public static partial void LogKeycloakUserNotPresent(this ILogger logger, string username, string key); + [LoggerMessage(7, LogLevel.Information, "Keycloak user creation success [{username}] ID: {userId} PartID: {key} - will be added")] + public static partial void LogKeycloakUserCreationSuccess(this ILogger logger, string userId, string username, string key); + [LoggerMessage(8, LogLevel.Error, "Keycloak user creation failed [{username}] PartID: {key} - check further logs")] + public static partial void LogKeycloakUserCreationFailed(this ILogger logger, string username, string key); + [LoggerMessage(9, LogLevel.Error, "Keycloak service error {msg}")] + public static partial void LogKeycloakServiceException(this ILogger logger, string msg, Exception ex); + [LoggerMessage(10, LogLevel.Warning, "Keycloak {realm} IDP not found {idp}")] + public static partial void LogIDPNotFound(this ILogger logger, string realm, string idp); + [LoggerMessage(11, LogLevel.Error, "Failed to complete in-custody onboarding {msg}")] + public static partial void LogInCustodyServiceException(this ILogger logger, string msg, Exception ex); + +} + + diff --git a/backend/webapi/Kafka/Consumer/Notifications/NotificationAckHandler.cs b/backend/webapi/Kafka/Consumer/Notifications/NotificationAckHandler.cs index 2675dea1..a40921ba 100644 --- a/backend/webapi/Kafka/Consumer/Notifications/NotificationAckHandler.cs +++ b/backend/webapi/Kafka/Consumer/Notifications/NotificationAckHandler.cs @@ -17,7 +17,7 @@ public async Task HandleAsync(string consumerName, string key, Notificatio using var trx = this.context.Database.BeginTransaction(); - Log.Logger.Information("Message received on {0} with key {1}", consumerName, key); + Log.Logger.Information($"Message received on {consumerName} with key {key}"); //check whether this message has been processed before if (await this.context.HasBeenProcessed(key, consumerName)) { @@ -33,7 +33,7 @@ public async Task HandleAsync(string consumerName, string key, Notificatio .Where(request => request.Id == value.AccessRequestId).SingleOrDefaultAsync(); if (accessRequest != null) { - Log.Information($"Marking access request {value.AccessRequestId} as {value.Status}"); + Log.Information($"Marking access request {value.AccessRequestId} {value.PartId} as {value.Status}"); try { diff --git a/backend/webapi/PidpConfiguration.cs b/backend/webapi/PidpConfiguration.cs index 368deef8..010b9eb5 100644 --- a/backend/webapi/PidpConfiguration.cs +++ b/backend/webapi/PidpConfiguration.cs @@ -10,7 +10,7 @@ public class PidpConfiguration private static readonly string? EnvironmentName = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT"); public string ApplicationUrl { get; set; } = string.Empty; - + public string CorrectionsIDP { get; set; } = "siteminder"; public AddressAutocompleteClientConfiguration AddressAutocompleteClient { get; set; } = new(); public ConnectionStringConfiguration ConnectionStrings { get; set; } = new(); public ChesClientConfiguration ChesClient { get; set; } = new(); @@ -132,6 +132,7 @@ public class KafkaClusterConfiguration public string ApprovalCreationTopic { get; set; } = string.Empty; public string ProcessResponseTopic { get; set; } = string.Empty; public string UserAccountChangeTopicName { get; set; } = string.Empty; + public string ParticipantCSNumberMappingTopic { get; set; } = string.Empty; public string NotificationTopicName { get; set; } = string.Empty; public string UserAccountChangeNotificationTopicName { get; set; } = string.Empty; public string SaslOauthbearerTokenEndpointUrl { get; set; } = string.Empty; diff --git a/backend/webapi/Program.cs b/backend/webapi/Program.cs index 29d8e78a..a4c0bf02 100644 --- a/backend/webapi/Program.cs +++ b/backend/webapi/Program.cs @@ -1,11 +1,10 @@ namespace Pidp; -using Avro.Generic; +using System.Reflection; using Serilog; using Serilog.Events; using Serilog.Formatting.Json; using Serilog.Sinks.SystemConsole.Themes; -using System.Reflection; public class Program @@ -51,23 +50,12 @@ private static void CreateLogger() .AddJsonFile($"appsettings.{environmentName}.json", optional: true) .Build(); - var seqEndpoint = Environment.GetEnvironmentVariable("Seq__Url"); - seqEndpoint ??= config.GetValue("Seq:Url"); var splunkHost = Environment.GetEnvironmentVariable("SplunkConfig__Host"); splunkHost ??= config.GetValue("SplunkConfig:Host"); var splunkToken = Environment.GetEnvironmentVariable("SplunkConfig__CollectorToken"); splunkToken ??= config.GetValue("SplunkConfig:CollectorToken"); - - if (string.IsNullOrEmpty(seqEndpoint)) - { - Console.WriteLine("SEQ Log Host is not configured - check Seq environment"); - Environment.Exit(100); - } - - - try { if (PidpConfiguration.IsDevelopment()) @@ -94,7 +82,6 @@ private static void CreateLogger() .Enrich.WithMachineName() .Enrich.WithProperty("Assembly", $"{name.Name}") .Enrich.WithProperty("Version", $"{name.Version}") - .WriteTo.Seq(seqEndpoint) .WriteTo.Console( outputTemplate: outputTemplate, theme: AnsiConsoleTheme.Code) diff --git a/backend/webapi/Startup.cs b/backend/webapi/Startup.cs index 2f1dfcd1..b4dacfdc 100644 --- a/backend/webapi/Startup.cs +++ b/backend/webapi/Startup.cs @@ -19,7 +19,6 @@ namespace Pidp; using Newtonsoft.Json.Serialization; using NodaTime; using NodaTime.Serialization.SystemTextJson; -using OpenTelemetry; using OpenTelemetry.Exporter; using OpenTelemetry.Logs; using OpenTelemetry.Metrics; @@ -40,6 +39,7 @@ namespace Pidp; using Pidp.Infrastructure.HttpClients; using Pidp.Infrastructure.Services; using Pidp.Infrastructure.Telemetry; +using Pidp.Kafka.Consumer.InCustodyProvisioning; using Prometheus; using Quartz; using Quartz.AspNetCore; @@ -121,7 +121,7 @@ public void ConfigureServices(IServiceCollection services) .AddKeycloakAuth(config) .AddScoped() .AddScoped() - + .AddScoped() .AddSingleton(SystemClock.Instance) .AddScoped(); diff --git a/backend/webapi/appsettings.json b/backend/webapi/appsettings.json index 5bc37968..1cc3abb6 100644 --- a/backend/webapi/appsettings.json +++ b/backend/webapi/appsettings.json @@ -62,6 +62,7 @@ "UserAccountChangeTopicName": "digitalevidence-accountchange-topic", "ProcessResponseTopic": "digitalevidence-processresponse-topic", "CourtLocationAccessRequestTopic": "digitalevidence-courtlocationaccessrequest-topic", + "ParticipantCSNumberMappingTopic": "digitalevidencedisclosure-csnumber-response-topic", "CaseAccessRequestTopicName": "digitalevidence-caseaccessrequest-topic", "ConsumerGroupId": "digitalevidence-webapi-consumer", "IncomingChangeEventTopic": "digitalevidence-justin-user-change-topic", From de2371f1ccd3a6a78b1e846c476597f702cc74a3 Mon Sep 17 00:00:00 2001 From: Lee Wright <258036@NTTDATA.COM> Date: Tue, 16 Jul 2024 09:38:00 -0700 Subject: [PATCH 4/5] Access requests for in-custody --- backend/ApprovalFlow/Program.cs | 2 +- .../common/Logging/CorrelationIdMiddleware.cs | 23 +++++ .../InCustodyProvisioning/InCustodyService.cs | 84 ++++++++++++++----- backend/webapi/Startup.cs | 2 + 4 files changed, 88 insertions(+), 23 deletions(-) create mode 100644 backend/common/Logging/CorrelationIdMiddleware.cs diff --git a/backend/ApprovalFlow/Program.cs b/backend/ApprovalFlow/Program.cs index 299d8371..0e9da493 100644 --- a/backend/ApprovalFlow/Program.cs +++ b/backend/ApprovalFlow/Program.cs @@ -79,13 +79,13 @@ private static void CreateLogger( Console.WriteLine("Creating the logging directory failed: {0}", e.ToString()); } + var name = Assembly.GetExecutingAssembly().GetName(); var outputTemplate = "[{Timestamp:HH:mm:ss} {Level:u3}] {Message:lj}{NewLine}{Exception}"; var loggerConfiguration = new LoggerConfiguration() .MinimumLevel.Information() .Filter.ByExcluding("RequestPath like '/health%'") - .Filter.ByExcluding("RequestPath like '/metrics%'") .MinimumLevel.Override("Microsoft", LogEventLevel.Warning) .MinimumLevel.Override("Microsoft.Hosting.Lifetime", LogEventLevel.Information) .MinimumLevel.Override("System", LogEventLevel.Warning) diff --git a/backend/common/Logging/CorrelationIdMiddleware.cs b/backend/common/Logging/CorrelationIdMiddleware.cs new file mode 100644 index 00000000..168930d4 --- /dev/null +++ b/backend/common/Logging/CorrelationIdMiddleware.cs @@ -0,0 +1,23 @@ +using Microsoft.AspNetCore.Http.Extensions; + +namespace Common.Logging; + +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Serilog.Context; + +public class CorrelationIdMiddleware(RequestDelegate next, ILogger logger) +{ + public const string CorrelationIdHeader = "X-Correlation-ID"; + public async Task InvokeAsync(HttpContext context) + { + var correlationId = context.Request.Headers[CorrelationIdHeader].FirstOrDefault() ?? Guid.NewGuid().ToString(); + context.Items["CorrelationId"] = correlationId; + context.Response.Headers.Append(CorrelationIdHeader, correlationId); + using (LogContext.PushProperty(CorrelationIdHeader, correlationId)) + { + logger.LogInformation($"CorrelationId: {correlationId} {context.Request.GetEncodedUrl}"); + await next(context); + } + } +} diff --git a/backend/webapi/Kafka/Consumer/InCustodyProvisioning/InCustodyService.cs b/backend/webapi/Kafka/Consumer/InCustodyProvisioning/InCustodyService.cs index 046e1684..6451d878 100644 --- a/backend/webapi/Kafka/Consumer/InCustodyProvisioning/InCustodyService.cs +++ b/backend/webapi/Kafka/Consumer/InCustodyProvisioning/InCustodyService.cs @@ -12,7 +12,7 @@ namespace Pidp.Kafka.Consumer.InCustodyProvisioning; using Pidp.Models; using Pidp.Models.Lookups; -public class InCustodyService(IClock clock, PidpDbContext context, ILogger logger, IKafkaProducer producer, IEdtCoreClient coreClient, IKeycloakAdministrationClient keycloakAdministrationClient, PidpConfiguration pidpConfiguration) : IInCustodyService +public class InCustodyService(IClock clock, PidpDbContext context, ILogger logger, IKafkaProducer producer, IEdtCoreClient coreClient, IKeycloakAdministrationClient keycloakAdministrationClient, PidpConfiguration pidpConfiguration) : IInCustodyService { public async Task ProcessInCustodySubmissionMessage(InCustodyParticipantModel value) @@ -129,34 +129,49 @@ private async Task AddOrUpdateKeycloakUserAsync(InCu private async Task CreateInCustodyAccessRequest(ExtendedUserRepresentation keycloakUser, InCustodyParticipantModel value) { + var party = context.Parties.Where(p => p.Jpdid == keycloakUser.Username).FirstOrDefault(); - - var party = new Party + if (party != null) { - UserId = keycloakUser.Id, - Jpdid = keycloakUser.Username, - FirstName = keycloakUser.FirstName!, - LastName = keycloakUser.LastName!, - Email = keycloakUser.Username - }; - - context.Parties.Add(party); - + logger.LogPartyAlreadyPresent(party.Jpdid); + } + else + { + party = new Party + { + UserId = keycloakUser.Id, + Jpdid = keycloakUser.Username, + FirstName = keycloakUser.FirstName!, + LastName = keycloakUser.LastName!, + Email = keycloakUser.Username + }; - var partyAdded = await context.SaveChangesAsync(); + context.Parties.Add(party); + await context.SaveChangesAsync(); + party = context.Parties.Where(p => p.Jpdid == keycloakUser.Username).FirstOrDefault(); + } - if (partyAdded > 0) + if (party != null) { - var accessRequest = new AccessRequest + var accessRequest = context.AccessRequests.Where(req => req.PartyId == party.Id && req.AccessTypeCode == AccessTypeCode.DigitalEvidenceDisclosure).FirstOrDefault(); + + if (accessRequest != null) { - Party = party, - AccessTypeCode = AccessTypeCode.DigitalEvidenceDisclosure, - RequestedOn = clock.GetCurrentInstant() - }; + logger.LogAccessRequestAlreadyPresent(party.Jpdid, accessRequest.Id); + } + else + { + accessRequest = new AccessRequest + { + Party = party, + AccessTypeCode = AccessTypeCode.DigitalEvidenceDisclosure, + RequestedOn = clock.GetCurrentInstant() + }; - context.AccessRequests.Add(accessRequest); + context.AccessRequests.Add(accessRequest); - await context.SaveChangesAsync(); + await context.SaveChangesAsync(); + } return accessRequest; } @@ -169,6 +184,7 @@ private async Task CreateInCustodyAccessRequest(ExtendedUserRepre } + /// /// Publish a message to tell disclosure service that a new user should be created /// This will be with the username for corrections users @@ -180,7 +196,7 @@ public async Task PublishDisclosurePortalMessage(AccessRequest accessReque var msgId = Guid.NewGuid().ToString(); // publish to the topic for disclosure portal to handle provisioning - var delivered = await producer.ProduceAsync(pidpConfiguration.KafkaCluster.DisclosurePublicUserCreationTopic, msgId, accessRequest); + var delivered = await producer.ProduceAsync(pidpConfiguration.KafkaCluster.DisclosurePublicUserCreationTopic, msgId, GetInCustoryDisclosureUserModel(accessRequest, value)); if (delivered.Status == Confluent.Kafka.PersistenceStatus.Persisted) { @@ -193,6 +209,26 @@ public async Task PublishDisclosurePortalMessage(AccessRequest accessReque return Task.CompletedTask; } + + + private static EdtDisclosureUserProvisioning GetInCustoryDisclosureUserModel(AccessRequest accessRequest, InCustodyParticipantModel model) + { + + return new EdtDisclosureUserProvisioning + { + Key = $"{model.ParticipantId}", + UserName = accessRequest.Party.Jpdid, + Email = accessRequest.Party.Email, + FullName = $"{accessRequest.Party.FirstName} {accessRequest.Party.LastName}", + AccountType = "Saml", + Role = "User", + SystemName = AccessTypeCode.DigitalEvidenceDisclosure.ToString(), + AccessRequestId = accessRequest.Id, + OrganizationType = "In-custody", + OrganizationName = "Public", + PersonKey = model.ParticipantId + }; + } } public static partial class InCustodyServiceLoggingExtensions @@ -219,6 +255,10 @@ public static partial class InCustodyServiceLoggingExtensions public static partial void LogIDPNotFound(this ILogger logger, string realm, string idp); [LoggerMessage(11, LogLevel.Error, "Failed to complete in-custody onboarding {msg}")] public static partial void LogInCustodyServiceException(this ILogger logger, string msg, Exception ex); + [LoggerMessage(12, LogLevel.Information, "Party already present {party}")] + public static partial void LogPartyAlreadyPresent(this ILogger logger, string party); + [LoggerMessage(13, LogLevel.Information, "Access request {accessRequestId} already present for party {party}")] + public static partial void LogAccessRequestAlreadyPresent(this ILogger logger, string party, int accessRequestId); } diff --git a/backend/webapi/Startup.cs b/backend/webapi/Startup.cs index b4dacfdc..43ad2689 100644 --- a/backend/webapi/Startup.cs +++ b/backend/webapi/Startup.cs @@ -3,6 +3,7 @@ namespace Pidp; using System.Reflection; using System.Text.Json; using Common.Kafka; +using Common.Logging; using Common.Utils; using FluentValidation.AspNetCore; using MicroElements.Swashbuckle.FluentValidation.AspNetCore; @@ -362,6 +363,7 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) // For example: 200, 201, 203 -> 2xx options.ReduceStatusCodeCardinality(); }); + app.UseMiddleware(); app.UseAuthentication(); app.UseAuthorization(); app.UseEndpoints(endpoints => From 29c6e29816e0c8ef34379880d2c5e9a306d92009 Mon Sep 17 00:00:00 2001 From: Lee Wright <258036@NTTDATA.COM> Date: Tue, 16 Jul 2024 15:13:12 -0700 Subject: [PATCH 5/5] WIP --- .../InCustodyProvisioning/InCustodyService.cs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/backend/webapi/Kafka/Consumer/InCustodyProvisioning/InCustodyService.cs b/backend/webapi/Kafka/Consumer/InCustodyProvisioning/InCustodyService.cs index 6451d878..bb0a47e6 100644 --- a/backend/webapi/Kafka/Consumer/InCustodyProvisioning/InCustodyService.cs +++ b/backend/webapi/Kafka/Consumer/InCustodyProvisioning/InCustodyService.cs @@ -133,10 +133,11 @@ private async Task CreateInCustodyAccessRequest(ExtendedUserRepre if (party != null) { - logger.LogPartyAlreadyPresent(party.Jpdid); + logger.LogPartyAlreadyPresent(party.Jpdid, party.Id); } else { + party = new Party { UserId = keycloakUser.Id, @@ -146,6 +147,13 @@ private async Task CreateInCustodyAccessRequest(ExtendedUserRepre Email = keycloakUser.Username }; + var altId = new PartyAlternateId + { + Name = "CSNumber", + Value = value.CSNumber, + Party = party + }; + context.Parties.Add(party); await context.SaveChangesAsync(); party = context.Parties.Where(p => p.Jpdid == keycloakUser.Username).FirstOrDefault(); @@ -255,8 +263,8 @@ public static partial class InCustodyServiceLoggingExtensions public static partial void LogIDPNotFound(this ILogger logger, string realm, string idp); [LoggerMessage(11, LogLevel.Error, "Failed to complete in-custody onboarding {msg}")] public static partial void LogInCustodyServiceException(this ILogger logger, string msg, Exception ex); - [LoggerMessage(12, LogLevel.Information, "Party already present {party}")] - public static partial void LogPartyAlreadyPresent(this ILogger logger, string party); + [LoggerMessage(12, LogLevel.Information, "Party already present {party} {partyId}")] + public static partial void LogPartyAlreadyPresent(this ILogger logger, string party, int partyId); [LoggerMessage(13, LogLevel.Information, "Access request {accessRequestId} already present for party {party}")] public static partial void LogAccessRequestAlreadyPresent(this ILogger logger, string party, int accessRequestId);