Skip to content

[PRMP-1227] Bulk upload metadata processor to review SQS #1002

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
NogaNHS merged 4 commits into from
Jan 26, 2026
Merged

[PRMP-1227] Bulk upload metadata processor to review SQS #1002

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9762efc
[PRMP-1227] Bulk upload metadata processor to review SQS
NogaNHS Jan 26, 2026
2fabd44
[PRMP-1227] revert commented out bulk upload script
NogaNHS Jan 26, 2026
26e4ebc
[PRMP-1227] fixing sonar issues
NogaNHS Jan 26, 2026
b9c259e
[PRMP-1227] PR comments
NogaNHS Jan 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions .github/workflows/base-run-bulk-upload.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,21 @@ on:
required: false
type: "string"
default: "3"
lambda_type:
description: "Which lambda to trigger (BulkUploadMetadataLambda or BulkUploadMetadataProcessor)"
required: false
type: "string"
default: "BulkUploadMetadataLambda"
run_bulk_upload_setup:
description: "Run bulk upload setup step"
required: false
type: "boolean"
default: true
run_document_review_setup:
description: "Run document review setup step"
required: false
type: "boolean"
default: true
secrets:
AWS_ASSUME_ROLE:
required: true
Expand Down Expand Up @@ -67,6 +82,7 @@ jobs:
working-directory: ./tests/bulk-upload/scripts

- name: Setup Bulk Upload
if: ${{ inputs.run_bulk_upload_setup }}
run: |
python setup_bulk_upload.py \
--environment "${{ inputs.sandbox }}" \
Expand All @@ -80,6 +96,7 @@ jobs:
working-directory: ./tests/bulk-upload/scripts

- name: Setup Document Review
if: ${{ inputs.run_document_review_setup }}
run: |
python setup_document_review.py
working-directory: ./tests/bulk-upload/scripts
Expand All @@ -90,5 +107,6 @@ jobs:
run: |
python run_bulk_upload.py \
--environment "${{ inputs.sandbox }}" \
--lambda-type "${{ inputs.lambda_type }}" \
--start-bulk-upload
working-directory: ./tests/bulk-upload/scripts
9 changes: 9 additions & 0 deletions .github/workflows/full-deploy-to-sandbox.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,14 @@ on:
required: true
type: boolean
default: false
lambda_type:
description: "Which lambda to trigger for bulk upload (only applies if bulk_upload is true)"
required: false
type: choice
options:
- "Bulk upload metadata"
- "NEW bulk metadata processor"
default: "Bulk upload metadata"
disable_pds:
description: "Do you want to disable the PDS stub?"
required: true
Expand Down Expand Up @@ -169,5 +177,6 @@ jobs:
combi_settings: "combi300"
base_branch: ${{ inputs.build_branch }}
file_count: "3"
lambda_type: ${{ inputs.lambda_type == 'NEW bulk metadata processor' && 'BulkUploadMetadataProcessor' || 'BulkUploadMetadataLambda' }}
secrets:
AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
11 changes: 10 additions & 1 deletion .github/workflows/run-bulk-upload-dev.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
name: Run Bulk Upload - Dev

run-name: "${{ github.event.inputs.sandbox }} | ${{ github.event.inputs.combi_settings }} | ${{ github.event.inputs.file_count }}"
run-name: "${{ github.event.inputs.sandbox }} | ${{ github.event.inputs.combi_settings }} | ${{ github.event.inputs.file_count }} | ${{ github.event.inputs.lambda_type }}"

permissions:
pull-requests: write
Expand All @@ -27,6 +27,14 @@ on:
required: true
type: "string"
default: "3"
lambda_type:
description: "Which lambda to trigger"
required: true
type: choice
options:
- "BulkUploadMetadataLambda"
- "BulkUploadMetadataProcessor"
default: "BulkUploadMetadataLambda"

jobs:
bulk_upload:
Expand All @@ -36,5 +44,6 @@ jobs:
sandbox: "${{ inputs.sandbox }}"
combi_settings: "${{ inputs.combi_settings }}"
file_count: "${{ inputs.file_count }}"
lambda_type: "${{ inputs.lambda_type }}"
secrets:
AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
17 changes: 16 additions & 1 deletion lambdas/handlers/bulk_upload_metadata_processor_handler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
from enums.feature_flags import FeatureFlags
from enums.lloyd_george_pre_process_format import LloydGeorgePreProcessFormat
from services.bulk_upload_metadata_processor_service import (
BulkUploadMetadataProcessorService,
get_formatter_service,
)
from services.feature_flags_service import FeatureFlagService
from services.metadata_mapping_validator_service import MetadataMappingValidatorService
from utils.audit_logging_setup import LoggingService
from utils.decorators.ensure_env_var import ensure_environment_variables
Expand All @@ -20,6 +22,17 @@
)
@handle_lambda_exceptions
def lambda_handler(event, _context):
feature_flag_service = FeatureFlagService()
send_to_review_flag_object = feature_flag_service.get_feature_flags_by_flag(
FeatureFlags.BULK_UPLOAD_SEND_TO_REVIEW_ENABLED.value
)
send_to_review_enabled = send_to_review_flag_object[
FeatureFlags.BULK_UPLOAD_SEND_TO_REVIEW_ENABLED.value
]

if send_to_review_enabled:
logger.info("Bulk upload send to review queue is enabled for metadata processor")

raw_pre_format_type = event.get(
"preFormatType", LloydGeorgePreProcessFormat.GENERAL
)
Expand All @@ -32,6 +45,7 @@ def lambda_handler(event, _context):
metadata_formatter_service=metadata_formatter_service,
metadata_heading_remap=remappings,
input_file_location=input_file_location,
send_to_review_enabled=send_to_review_enabled,
)

if "source" in event and event.get("source") == "aws.s3":
Expand Down Expand Up @@ -60,6 +74,7 @@ def lambda_handler(event, _context):
metadata_formatter_service=metadata_formatter_service,
metadata_heading_remap=remappings,
fixed_values=fixed_values,
input_file_location=input_file_location
input_file_location=input_file_location,
send_to_review_enabled=send_to_review_enabled,
)
metadata_service.process_metadata()
6 changes: 3 additions & 3 deletions lambdas/repositories/bulk_upload/bulk_upload_sqs_repository.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@
class BulkUploadSqsRepository:
def __init__(self):
self.sqs_repository = SQSService()
self.metadata_queue_url = os.environ["METADATA_SQS_QUEUE_URL"]
self.review_queue_url = os.environ["REVIEW_SQS_QUEUE_URL"]
self.metadata_queue_url = os.getenv("METADATA_SQS_QUEUE_URL")
self.review_queue_url = os.getenv("REVIEW_SQS_QUEUE_URL")

def put_staging_metadata_back_to_queue(self, staging_metadata: StagingSqsMetadata):
request_context.patient_nhs_no = staging_metadata.nhs_number
Expand All @@ -40,7 +40,7 @@ def send_message_to_review_queue(
review_files = [
ReviewMessageFile(
file_name=file.stored_file_name.split("/")[-1],
file_path=file.file_path,
file_path=file.file_path.lstrip("/"),
)
for file in staging_metadata.files
]
Expand Down
74 changes: 35 additions & 39 deletions lambdas/services/bulk_upload/metadata_general_preprocessor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,45 +19,41 @@

class MetadataGeneralPreprocessor(MetadataPreprocessorService):
def validate_record_filename(self, file_name: str, *args, **kwargs) -> str:
try:
file_path_prefix, current_file_name = (
extract_document_path_for_lloyd_george_record(file_name)
)
first_document_number, second_document_number, current_file_name = (
extract_document_number_bulk_upload_file_name(current_file_name)
)
current_file_name = extract_lloyd_george_record_from_bulk_upload_file_name(
current_file_name
)
patient_name, current_file_name = (
extract_patient_name_from_bulk_upload_file_name(current_file_name)
)
file_path_prefix, current_file_name = (
extract_document_path_for_lloyd_george_record(file_name)
)
first_document_number, second_document_number, current_file_name = (
extract_document_number_bulk_upload_file_name(current_file_name)
)
current_file_name = extract_lloyd_george_record_from_bulk_upload_file_name(
current_file_name
)
patient_name, current_file_name = (
extract_patient_name_from_bulk_upload_file_name(current_file_name)
)

if sum(c.isdigit() for c in current_file_name) != 18:
logger.info("Failed to find NHS number or date")
raise InvalidFileNameException("Incorrect NHS number or date format")
if sum(c.isdigit() for c in current_file_name) != 18:
logger.info("Failed to find NHS number or date")
raise InvalidFileNameException("Incorrect NHS number or date format")

nhs_number, current_file_name = (
extract_nhs_number_from_bulk_upload_file_name(current_file_name)
)
date, current_file_name = extract_date_from_bulk_upload_file_name(
current_file_name
)
file_extension = extract_file_extension_from_bulk_upload_file_name(
current_file_name
)
file_name = assemble_lg_valid_file_name_full_path(
file_path_prefix,
first_document_number,
second_document_number,
patient_name,
nhs_number,
date,
file_extension,
)
logger.info(f"Finished processing, new file name is: {file_name}")
return file_name
nhs_number, current_file_name = (
extract_nhs_number_from_bulk_upload_file_name(current_file_name)
)
date, current_file_name = extract_date_from_bulk_upload_file_name(
current_file_name
)
file_extension = extract_file_extension_from_bulk_upload_file_name(
current_file_name
)
file_name = assemble_lg_valid_file_name_full_path(
file_path_prefix,
first_document_number,
second_document_number,
patient_name,
nhs_number,
date,
file_extension,
)
logger.info(f"Finished processing, new file name is: {file_name}")
return file_name

except InvalidFileNameException as error:
logger.error(f"Failed to process {file_name} due to error: {error}")
raise error
Loading
Loading