Skip to content

Commit

Permalink
Merge branch 'main' into fix/DTOSS-6549-move-ampls-to-hub
Browse files Browse the repository at this point in the history
  • Loading branch information
patrickmoore-nc authored Jan 6, 2025
2 parents 94554db + 58d9c67 commit dbc4649
Show file tree
Hide file tree
Showing 32 changed files with 1,227 additions and 240 deletions.
16 changes: 8 additions & 8 deletions application/CohortManager/compose.core.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -231,24 +231,24 @@ services:
- ASPNETCORE_URLS=http://*:7995
- DtOsDatabaseConnectionString=Server=localhost,1433;Database=${DB_NAME};User Id=SA;Password=${PASSWORD};TrustServerCertificate=True

participant-demographic-data-service:
container_name: participant-demographic-data-service
participant-management-data-service:
container_name: participant-management-data-service
network_mode: host
build:
context: ./src/Functions/
dockerfile: screeningDataServices/ParticipantDemographicDataService/Dockerfile
dockerfile: screeningDataServices/ParticipantManagementDataService/Dockerfile
environment:
- ASPNETCORE_URLS=http://*:7993
- ASPNETCORE_URLS=http://*:7994
- DtOsDatabaseConnectionString=Server=localhost,1433;Database=${DB_NAME};User Id=SA;Password=${PASSWORD};TrustServerCertificate=True

participant-management-data-service:
container_name: participant-management-data-service
participant-demographic-data-service:
container_name: participant-demographic-data-service
network_mode: host
build:
context: ./src/Functions/
dockerfile: screeningDataServices/ParticipantManagementDataService/Dockerfile
dockerfile: screeningDataServices/ParticipantDemographicDataService/Dockerfile
environment:
- ASPNETCORE_URLS=http://*:7994
- ASPNETCORE_URLS=http://*:7993
- DtOsDatabaseConnectionString=Server=localhost,1433;Database=${DB_NAME};User Id=SA;Password=${PASSWORD};TrustServerCertificate=True

# Screening Validation Service
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,22 +3,22 @@ namespace NHS.Screening.ReceiveCaasFile;
using System.Text;
using System.Text.Json;
using Azure.Storage.Queues;
using Common;
using Microsoft.Extensions.Logging;
using Model;

public class AddBatchToQueue : IAddBatchToQueue
{
private readonly QueueClient _AddQueueClient;

public readonly ILogger<AddBatchToQueue> _logger;

public AddBatchToQueue(ILogger<AddBatchToQueue> logger)
{
var storageConnectionString = Environment.GetEnvironmentVariable("AzureWebJobsStorage") ?? "";
_AddQueueClient = new QueueClient(storageConnectionString, Environment.GetEnvironmentVariable("AddQueueName"));

_AddQueueClient.CreateIfNotExists();
private readonly IAzureQueueStorageHelper _queueHelper;

public AddBatchToQueue(ILogger<AddBatchToQueue> logger, IAzureQueueStorageHelper queueHelper)
{
_logger = logger;
_queueHelper = queueHelper;
}

public async Task ProcessBatch(Batch batch)
Expand Down Expand Up @@ -51,7 +51,7 @@ private async Task AddMessagesAsync(Batch currentBatch)

private async Task AddMessage(BasicParticipantCsvRecord basicParticipantCsvRecord)
{
await _AddQueueClient.SendMessageAsync(ParseMessage(basicParticipantCsvRecord));
await _queueHelper.AddItemToQueueAsync<BasicParticipantCsvRecord>(basicParticipantCsvRecord,Environment.GetEnvironmentVariable("AddQueueName"));
}

private static string ParseMessage(BasicParticipantCsvRecord ParticipantCsvRecord)
Expand All @@ -61,4 +61,4 @@ private static string ParseMessage(BasicParticipantCsvRecord ParticipantCsvRecor

return Convert.ToBase64String(bytes);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@ namespace NHS.Screening.ReceiveCaasFile;

public interface IRecordsProcessedTracker
{
bool RecordNotAlreadyProcessed(string RecordType, string NHSId);
bool RecordAlreadyProcessed(string RecordType, string NHSId);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
namespace NHS.Screening.ReceiveCaasFile;

using Model;

public interface IValidateDates
{
public bool ValidateAllDates(Participant participant);
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
namespace NHS.Screening.ReceiveCaasFile;

using System.Text.Json;
using System.Threading.Tasks.Dataflow;
using Common;
using Common.Interfaces;
using Microsoft.Extensions.Logging;
Expand All @@ -21,11 +22,13 @@ public class ProcessCaasFile : IProcessCaasFile
private readonly IAddBatchToQueue _addBatchToQueue;
private readonly IExceptionHandler _exceptionHandler;

private readonly RecordsProcessedTracker _recordsProcessTracker;
private readonly IRecordsProcessedTracker _recordsProcessTracker;

private readonly IValidateDates _validateDates;

public ProcessCaasFile(ILogger<ProcessCaasFile> logger, ICallFunction callFunction, ICheckDemographic checkDemographic, ICreateBasicParticipantData createBasicParticipantData,
IExceptionHandler handleException, IAddBatchToQueue addBatchToQueue, IReceiveCaasFileHelper receiveCaasFileHelper, IExceptionHandler exceptionHandler
,RecordsProcessedTracker recordsProcessedTracker
, IRecordsProcessedTracker recordsProcessedTracker, IValidateDates validateDates
)
{
_logger = logger;
Expand All @@ -37,6 +40,7 @@ public ProcessCaasFile(ILogger<ProcessCaasFile> logger, ICallFunction callFuncti
_receiveCaasFileHelper = receiveCaasFileHelper;
_exceptionHandler = exceptionHandler;
_recordsProcessTracker = recordsProcessedTracker;
_validateDates = validateDates;
}

/// <summary>
Expand All @@ -61,19 +65,19 @@ await Parallel.ForEachAsync(values, options, async (rec, cancellationToken) =>

if (!ValidationHelper.ValidateNHSNumber(participant.NhsNumber))
{
await _exceptionHandler.CreateSystemExceptionLog(new Exception($"Invalid NHS Number was passed in for participant {participant} and file {name}"),participant,name );
await _exceptionHandler.CreateSystemExceptionLog(new Exception($"Invalid NHS Number was passed in for participant {participant} and file {name}"), participant, name);

return; // skip current participant
}

if (!ValidateDates(participant))
if (!_validateDates.ValidateAllDates(participant))
{

await _exceptionHandler.CreateSystemExceptionLog(new Exception($"Invalid effective date found in participant data {participant} and file name {name}"), participant, name);
return; // Skip current participant
}

if(!_recordsProcessTracker.RecordNotAlreadyProcessed(participant.RecordType,participant.NhsNumber))
if (!_recordsProcessTracker.RecordAlreadyProcessed(participant.RecordType, participant.NhsNumber))
{
await _exceptionHandler.CreateSystemExceptionLog(new Exception($"Duplicate Participant was in the file"), participant, name);
return; // Skip current participant
Expand Down Expand Up @@ -117,7 +121,7 @@ private async Task<Batch> AddRecordToBatch(Participant participant, Batch curren
currentBatch.DeleteRecords.Enqueue(basicParticipantCsvRecord);
break;
default:
await _exceptionHandler.CreateSchemaValidationException(basicParticipantCsvRecord,"RecordType was not set to an expected value");
await _exceptionHandler.CreateSchemaValidationException(basicParticipantCsvRecord, "RecordType was not set to an expected value");
break;
}
return currentBatch;
Expand Down Expand Up @@ -157,7 +161,7 @@ private async Task UpdateParticipant(BasicParticipantCsvRecord basicParticipantC
}
catch (Exception ex)
{
_logger.LogError(ex ,"Update participant function failed.\nMessage: {Message}\nStack Trace: {StackTrace}", ex.Message, ex.StackTrace);
_logger.LogError(ex, "Update participant function failed.\nMessage: {Message}\nStack Trace: {StackTrace}", ex.Message, ex.StackTrace);
await CreateError(basicParticipantCsvRecord.participant, name);
}
}
Expand All @@ -171,7 +175,7 @@ private async Task RemoveParticipant(BasicParticipantCsvRecord basicParticipantC
}
catch (Exception ex)
{
_logger.LogError(ex,"Remove participant function failed.\nMessage: {Message}\nStack Trace: {StackTrace}", ex.Message, ex.StackTrace);
_logger.LogError(ex, "Remove participant function failed.\nMessage: {Message}\nStack Trace: {StackTrace}", ex.Message, ex.StackTrace);
await CreateError(basicParticipantCsvRecord.participant, filename);
}
}
Expand All @@ -186,69 +190,9 @@ private async Task CreateError(Participant participant, string filename)
}
catch (Exception ex)
{
_logger.LogError(ex ,"Handling the exception failed.\nMessage: {Message}\nStack Trace: {StackTrace}", ex.Message, ex.StackTrace);
_handleException.CreateSystemExceptionLog(ex, participant, filename);
}
}
private async Task CreateError(Participant participant, string filename, string errorMessage)
{
try
{
_logger.LogError(errorMessage);
await _handleException.CreateRecordValidationExceptionLog(participant.NhsNumber, filename, errorMessage, "", JsonSerializer.Serialize(participant));
}
catch (Exception ex)
{
_logger.LogError(ex ,"Handling the exception failed.\nMessage: {Message}\nStack Trace: {StackTrace}", ex.Message, ex.StackTrace);
_logger.LogError(ex, "Handling the exception failed.\nMessage: {Message}\nStack Trace: {StackTrace}", ex.Message, ex.StackTrace);
_handleException.CreateSystemExceptionLog(ex, participant, filename);
}
}

private bool ValidateDates(Participant participant)
{
if(!IsValidDate(participant.CurrentPostingEffectiveFromDate)){
_logger.LogWarning("Invalid {datename} found in participant data",nameof(participant.CurrentPostingEffectiveFromDate));
return false;
}
if(!IsValidDate(participant.EmailAddressEffectiveFromDate)){
_logger.LogWarning("Invalid {datename} found in participant data",nameof(participant.EmailAddressEffectiveFromDate));
return false;
}
if(!IsValidDate(participant.MobileNumberEffectiveFromDate)){
_logger.LogWarning("Invalid {datename} found in participant data",nameof(participant.MobileNumberEffectiveFromDate));
return false;
}
if(!IsValidDate(participant.UsualAddressEffectiveFromDate)){
_logger.LogWarning("Invalid {datename} found in participant data",nameof(participant.UsualAddressEffectiveFromDate));
return false;
}
if(!IsValidDate(participant.TelephoneNumberEffectiveFromDate)){
_logger.LogWarning("Invalid {datename} found in participant data",nameof(participant.TelephoneNumberEffectiveFromDate));
return false;
}
if(!IsValidDate(participant.PrimaryCareProviderEffectiveFromDate)){
_logger.LogWarning("Invalid {datename} found in participant data",nameof(participant.PrimaryCareProviderEffectiveFromDate));
return false;
}
if(!IsValidDate(participant.CurrentPostingEffectiveFromDate)){
_logger.LogWarning("Invalid {datename} found in participant data",nameof(participant.CurrentPostingEffectiveFromDate));
return false;
}

return true;
}
private static bool IsValidDate(string? date)
{
if(date== null)
{
return true;
}
if(date.Length > 8)
{
return false;
}
return true;

}

}
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,13 @@ public RecordsProcessedTracker()
_processedRecords = new HashSet<ParticipantRecord>();
}

public bool RecordNotAlreadyProcessed(string RecordType, string NHSId)
public bool RecordAlreadyProcessed(string RecordType, string NHSId)
{
var rec = new ParticipantRecord{ RecordType = RecordType, NHSId = NHSId};
var rec = new ParticipantRecord { RecordType = RecordType, NHSId = NHSId };
//avoiding race conditions on access to the process records hashset
lock(lockObj)
lock (lockObj)
{
if(_processedRecords.Contains(rec))
if (_processedRecords.Contains(rec))
{
return false;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
namespace NHS.Screening.ReceiveCaasFile;

using Microsoft.Extensions.Logging;
using Model;

public class ValidateDates : IValidateDates
{

private readonly ILogger<ValidateDates> _logger;

public ValidateDates(ILogger<ValidateDates> logger)
{
_logger = logger;
}

public bool ValidateAllDates(Participant participant)
{
if (!IsValidDate(participant.CurrentPostingEffectiveFromDate))
{
_logger.LogWarning("Invalid {datename} found in participant data", nameof(participant.CurrentPostingEffectiveFromDate));
return false;
}
if (!IsValidDate(participant.EmailAddressEffectiveFromDate))
{
_logger.LogWarning("Invalid {datename} found in participant data", nameof(participant.EmailAddressEffectiveFromDate));
return false;
}
if (!IsValidDate(participant.MobileNumberEffectiveFromDate))
{
_logger.LogWarning("Invalid {datename} found in participant data", nameof(participant.MobileNumberEffectiveFromDate));
return false;
}
if (!IsValidDate(participant.UsualAddressEffectiveFromDate))
{
_logger.LogWarning("Invalid {datename} found in participant data", nameof(participant.UsualAddressEffectiveFromDate));
return false;
}
if (!IsValidDate(participant.TelephoneNumberEffectiveFromDate))
{
_logger.LogWarning("Invalid {datename} found in participant data", nameof(participant.TelephoneNumberEffectiveFromDate));
return false;
}
if (!IsValidDate(participant.PrimaryCareProviderEffectiveFromDate))
{
_logger.LogWarning("Invalid {datename} found in participant data", nameof(participant.PrimaryCareProviderEffectiveFromDate));
return false;
}
if (!IsValidDate(participant.CurrentPostingEffectiveFromDate))
{
_logger.LogWarning("Invalid {datename} found in participant data", nameof(participant.CurrentPostingEffectiveFromDate));
return false;
}

return true;
}

private bool IsValidDate(string? date)
{
if (date == null)
{
return true;
}
if (date.Length > 8)
{
return false;
}
return true;

}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
using Microsoft.Extensions.Logging;
using NHS.Screening.ReceiveCaasFile;
using receiveCaasFile;
using Microsoft.Extensions.Azure;
using Azure.Identity;

var loggerFactory = LoggerFactory.Create(builder => builder.AddConsole());
var logger = loggerFactory.CreateLogger("program.cs");
Expand All @@ -28,8 +30,10 @@
services.AddScoped<ICheckDemographic, CheckDemographic>();
services.AddScoped<ICreateBasicParticipantData, CreateBasicParticipantData>();
services.AddScoped<IAddBatchToQueue, AddBatchToQueue>();
services.AddScoped<IRecordsProcessedTracker,RecordsProcessedTracker>(); //Do not change the lifetime of this.
services.AddScoped<IRecordsProcessedTracker, RecordsProcessedTracker>(); //Do not change the lifetime of this.
services.AddScoped<IValidateDates, ValidateDates>();
})
.AddAzureQueues()
.AddExceptionHandler()
.AddDatabaseConnection()
.Build();
Expand Down
Loading

0 comments on commit dbc4649

Please sign in to comment.