From 1feeabec7b3e65e6678916c21a89f49e04966204 Mon Sep 17 00:00:00 2001
From: Andrew Newton <andy@vitri.ltd>
Date: Thu, 27 Jun 2024 17:07:32 +0100
Subject: [PATCH] docs: add D&DR documentation (#122)

* docs: Initial documentation for DR SPs

* docs: Docusaurus display tweaks.
---
 docs/app/data-retention/_category_.json       |  5 ++
 docs/app/data-retention/cleanup_procedure.md  | 44 +++++++++++++++
 docs/app/data-retention/overview.md           | 53 +++++++++++++++++++
 .../populate-procedures/_category_.json       |  5 ++
 .../sp_populate_admin_cancelled.md            | 13 +++++
 ..._populate_application_not_yet_submitted.md | 12 +++++
 .../sp_populate_application_ntu_withdrawn.md  | 13 +++++
 .../sp_populate_application_refused.md        | 13 +++++
 .../sp_populate_cancelled_reg.md              | 12 +++++
 .../sp_populate_case_impounding.md            | 11 ++++
 .../sp_populate_document_financial_reports.md | 13 +++++
 .../sp_populate_irfo_gv_permit_expired.md     | 11 ++++
 ...pulate_irfo_gv_permit_withdrawn_pending.md | 13 +++++
 .../sp_populate_irfo_operator_expired.md      | 13 +++++
 .../sp_populate_irfo_psv_auth_expired.md      | 11 ++++
 ..._auth_withdrawn_pending_refused_expired.md | 13 +++++
 .../sp_populate_irhp_cancelled_withdrawn.md   | 11 ++++
 .../sp_populate_irhp_cert_road_nys.md         | 12 +++++
 .../sp_populate_irhp_expired.md               | 14 +++++
 .../sp_populate_irhp_replaced_permits.md      | 11 ++++
 .../sp_populate_irhp_terminated_permits.md    | 13 +++++
 .../sp_populate_irhp_unsuccessful.md          | 12 +++++
 .../sp_populate_licence_no_imp_unlicenced.md  | 14 +++++
 ...ate_licence_no_pi_no_imp_not_unlicenced.md | 14 +++++
 .../sp_populate_licence_not_submitted.md      | 13 +++++
 ...opulate_licence_with_imp_not_unlicenced.md | 14 +++++
 ...e_licence_with_pi_no_imp_not_unlicenced.md | 15 ++++++
 .../sp_populate_misc_fees.md                  | 12 +++++
 ...p_populate_new_var_appl_bus_reg_expired.md | 12 +++++
 .../sp_populate_org_no_licence.md             | 14 +++++
 .../sp_populate_orphan_users.md               | 13 +++++
 ...sp_populate_queue_table_items_to_delete.md | 14 +++++
 .../sp_populate_refused_with_var_zero.md      | 13 +++++
 .../populate-procedures/sp_populate_tm_pi.md  | 12 +++++
 .../sp_populate_tm_without_pi.md              | 12 +++++
 .../sp_populate_withdrawn_with_var_zero.md    | 13 +++++
 36 files changed, 513 insertions(+)
 create mode 100644 docs/app/data-retention/_category_.json
 create mode 100644 docs/app/data-retention/cleanup_procedure.md
 create mode 100644 docs/app/data-retention/overview.md
 create mode 100644 docs/app/data-retention/populate-procedures/_category_.json
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_admin_cancelled.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_application_not_yet_submitted.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_application_ntu_withdrawn.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_application_refused.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_cancelled_reg.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_case_impounding.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_document_financial_reports.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_irfo_gv_permit_expired.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_irfo_gv_permit_withdrawn_pending.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_irfo_operator_expired.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_irfo_psv_auth_expired.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_irfo_psv_auth_withdrawn_pending_refused_expired.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_irhp_cancelled_withdrawn.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_irhp_cert_road_nys.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_irhp_expired.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_irhp_replaced_permits.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_irhp_terminated_permits.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_irhp_unsuccessful.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_licence_no_imp_unlicenced.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_licence_no_pi_no_imp_not_unlicenced.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_licence_not_submitted.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_licence_with_imp_not_unlicenced.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_licence_with_pi_no_imp_not_unlicenced.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_misc_fees.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_new_var_appl_bus_reg_expired.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_org_no_licence.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_orphan_users.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_queue_table_items_to_delete.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_refused_with_var_zero.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_tm_pi.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_tm_without_pi.md
 create mode 100644 docs/app/data-retention/populate-procedures/sp_populate_withdrawn_with_var_zero.md

diff --git a/docs/app/data-retention/_category_.json b/docs/app/data-retention/_category_.json
new file mode 100644
index 0000000000..21582d127e
--- /dev/null
+++ b/docs/app/data-retention/_category_.json
@@ -0,0 +1,5 @@
+{
+  "label": "Data Retention",
+  "position": 100,
+  "collapsed": true
+}
diff --git a/docs/app/data-retention/cleanup_procedure.md b/docs/app/data-retention/cleanup_procedure.md
new file mode 100644
index 0000000000..151b9555fc
--- /dev/null
+++ b/docs/app/data-retention/cleanup_procedure.md
@@ -0,0 +1,44 @@
+---
+sidebar_position: 2
+---
+# Delete & Cleanup Procedure
+
+The `sp_dr_cleanup` systematically cleans up data across related tables in the database, based on the data retention rules. This procedure is integral to maintaining the integrity and performance of the system by removing outdated records, and helps remain compliant with data protection and retention legislation.
+
+### Key Steps in the Cleanup Process
+
+1. **Initialization:**
+    - The procedure starts by checking system parameters to ensure that data deletion is not disabled. It sets some system parameters necessary for the run, including disabling foreign key checks and triggers for the remainder of the run.
+
+2. **Transaction Management:**
+    - A transaction is started to ensure that all changes are atomic and can be rolled back in case of any errors during the execution process. This is planned to be removed soon, to allow the DR process to run while the system remains online (albeit at a v.quiet time)
+
+3. **Temporary Tables Creation:**
+    - Temporary tables are created to manage the data that needs to be processed. These tables help isolate and track which records are to be deleted without affecting the live data during the cleanup process.
+
+4. **Deletion Criteria Setup:**
+    - Records eligible for deletion are identified based on the rules specified in the `data_retention` table and related `data_retention_rule`. This includes checking fields like `action_confirmation` and `deleted_date`.
+
+5. **Dependency Management:**
+    - The script handles dependencies carefully by ensuring that related records in dependent tables are also considered for deletion. This includes cascading effects where deleting a record in one table may affect records in multiple related tables.
+
+6. **Record Deletion:**
+    - Actual deletion of records occurs from various tables, based on the dependencies and relationships defined.
+
+7. **Verification and Cleanup:**
+    - After deletion, the script verifies the number of records deleted and ensures that the counts match the expected numbers. Temporary tables are used to compare pre and post-deletion record counts.
+
+8. **Transaction Completion:**
+    - If the `dry_run` parameter is set to `FALSE`, the changes are committed to the database. If `dry_run` is `TRUE`, all changes are rolled back, allowing the operation to be tested without affecting actual data.
+
+9. **Finalization:**
+    - The environment is reset by re-enabling foreign key checks and triggers. Temporary tables are dropped to clean up the session.
+
+### Error Handling
+
+- The procedure includes robust error handling mechanisms that catch SQL exceptions during execution. These exceptions trigger a rollback of the transaction, ensuring that the database state remains consistent and no partial updates are committed.
+- The above mechanism will not be possible when running without transactions, some consideration should be given to the impact of this change.
+
+### Usage
+
+- This procedure can be executed with parameters to specify the user ID performing the cleanup, the limit on the number of records to process, and whether the operation should be a dry run.
diff --git a/docs/app/data-retention/overview.md b/docs/app/data-retention/overview.md
new file mode 100644
index 0000000000..6058076133
--- /dev/null
+++ b/docs/app/data-retention/overview.md
@@ -0,0 +1,53 @@
+---
+sidebar_position: 1
+---
+# Data Retention System Overview
+
+The Data Retention system is designed to manage the lifecycle of records in compliance with regulatory and internal data management policies. This system uses a set of "populate rules" to determine when records are eligible for deletion, ensuring data is handled securely and efficiently.
+
+### Key Components
+
+#### Data Retention Rules
+
+- **Table:** `data_retention_rule`
+- **Description:** Stores the rules that define how data retention is handled for different types of entity. Each rule specifies the retention period, the maximum dataset size for consideration in a single run, and whether the rule is currently enabled.
+- **Key Fields:**
+    - `id`: Unique identifier for the rule.
+    - `description`: A brief description of what the rule does.
+    - `retention_period`: How long (in months) to keep the data after record closure.
+    - `max_data_set`: Limits the number of rows affected in a single operation.
+    - `is_enabled`: Indicates if the rule is active.
+    - `action_type`: Defines the type of action (e.g., automatic deletion, manual review).
+    - `populate_procedure`: The stored procedure associated with populating data for this rule.
+
+#### Data Retention Control Table
+
+- **Table:** `data_retention`
+- **Description:** Records information about data eligible for retention actions. It is populated by the stored procedures indexed in the  `data_retention_rule` table,
+- **Key Fields:**
+    - `entity_name`, `entity_pk`: Identify the database record to delete.
+    - `data_retention_rule_id`: Links to the applicable retention rule that populated the record.
+    - `action_confirmation`: Indicates whether the deletion action is confirmed by an admin user.
+    - `assigned_to`: User ID for accountability.
+    - `next_review_date`: When the record should next be reviewed.
+    - `actioned_date`: When the deletion took place for this record.
+
+### Process Flow
+
+1. **Rule Execution:**
+    - Stored procedures defined per rule in `populate_procedure` are executed regularly. These procedures evaluate records against their respective retention rules and populate the `data_retention` table with records eligible for deletion or review.
+
+2. **Admin UI Interaction:**
+    - An administrative UI in the `olcs-internal` web app allows users to manage retention rules and review records flagged for manual action.
+
+3. **Automated and Manual Deletion:**
+    - Records flagged for automatic deletion are processed in scheduled runs of the `sp_dr_cleanup` stored procedure, which handles the deletion of data and cleanup of any dependent records.
+    - Records requiring manual approval are not deleted in these runs until manually reviewed and confirmed through the admin UI, then deleted in subsequent cleanup runs.
+
+### Security and Compliance
+
+- The system is designed to ensure compliance with data protection regulations by allowing records to be deleted after per-entity retention periods have expired, and allows for review and manual intervention when necessary.
+
+### Conclusion
+
+The Data Retention system provides a framework for managing the lifecycle of data within the organisation. By automating data cleanup the system helps maintain data hygiene and regulatory compliance efficiently.
diff --git a/docs/app/data-retention/populate-procedures/_category_.json b/docs/app/data-retention/populate-procedures/_category_.json
new file mode 100644
index 0000000000..cf23628781
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/_category_.json
@@ -0,0 +1,5 @@
+{
+  "label": "Populate Rules",
+  "position": 100,
+  "collapsed": true
+}
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_admin_cancelled.md b/docs/app/data-retention/populate-procedures/sp_populate_admin_cancelled.md
new file mode 100644
index 0000000000..1bcb6f3567
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_admin_cancelled.md
@@ -0,0 +1,13 @@
+### sp_populate_admin_cancelled
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `bus_reg`
+- **Status Filter:** Selects records where the `bus_reg_status_id` is `'breg_s_admin'` which is "admin cancelled" status 
+- **Date Filter:** Selects records where the `effective_date` is older than the current date minus the retention period.
+
+
+#### Summary:
+- This procedure deletes records from the `bus_reg` table where the `bus_reg_status_id` is `'breg_s_admin'` and the `effective_date` is older than the current date minus the retention period defined in the data retention rules.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_application_not_yet_submitted.md b/docs/app/data-retention/populate-procedures/sp_populate_application_not_yet_submitted.md
new file mode 100644
index 0000000000..cc17fcdf79
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_application_not_yet_submitted.md
@@ -0,0 +1,12 @@
+### sp_populate_application_not_yet_submitted
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `application`
+- **Status Filter:** Selects records where the application status is `'apsts_not_submitted'`.
+- **Date Filter:** Selects records based on the `last_modified_on` or `created_on` date being older than the current date minus the retention period.
+
+#### Summary:
+- This procedure targets records in the `application` table where the status is `'apsts_not_submitted'` and either the last modified date or creation date is older than the current date minus the retention period.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_application_ntu_withdrawn.md b/docs/app/data-retention/populate-procedures/sp_populate_application_ntu_withdrawn.md
new file mode 100644
index 0000000000..15f50be42d
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_application_ntu_withdrawn.md
@@ -0,0 +1,13 @@
+### sp_populate_application_ntu_withdrawn
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `application`
+- **Status Filter:** Selects records where the application status is either `'apsts_ntu'` (not taken up) or `'apsts_withdrawn'` (withdrawn by the applicant).
+- **Event Filter:** Only considers records where the last event related to the application is of type ID `5` or `16`.
+- **Date Filter:** Filters applications based on the date of the last relevant event being older than the current date minus the retention period defined.
+
+#### Summary:
+- This procedure focuses on records from the `application` table with statuses of not taken up or withdrawn, considering only those where the last significant event occurred before the current date minus the retention period.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_application_refused.md b/docs/app/data-retention/populate-procedures/sp_populate_application_refused.md
new file mode 100644
index 0000000000..e482c64b7e
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_application_refused.md
@@ -0,0 +1,13 @@
+### sp_populate_application_refused
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `application`
+- **Status Filter:** Selects records where the application status is `'apsts_refused'`.
+- **Variation Filter:** Considers only applications that are not variations (`is_variation = 0`).
+- **Date Filter:** Filters applications based on the `refused_date` being older than the current date minus the retention period.
+
+#### Summary:
+- This procedure targets records in the `application` table that have been refused and are not variations, considering those whose refusal date is older than the current date minus the retention period.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_cancelled_reg.md b/docs/app/data-retention/populate-procedures/sp_populate_cancelled_reg.md
new file mode 100644
index 0000000000..f738a5be13
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_cancelled_reg.md
@@ -0,0 +1,12 @@
+### sp_populate_cancelled_reg
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `bus_reg`
+- **Status Filter:** Selects records where the bus registration status is `'breg_s_cancelled'`.
+- **Date Filter:** Filters registrations based on the `effective_date` being older than the current date minus the retention period defined for this rule.
+
+#### Summary:
+- This procedure targets records in the `bus_reg` table that have been cancelled, specifically those whose effective date is older than the current date minus the retention period.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_case_impounding.md b/docs/app/data-retention/populate-procedures/sp_populate_case_impounding.md
new file mode 100644
index 0000000000..3c4641efa9
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_case_impounding.md
@@ -0,0 +1,11 @@
+### sp_populate_case_impounding
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `cases`
+- **Closure Filter:** Selects case records where the `closed_date` is older than the current date minus the retention period defined for this rule.
+
+#### Summary:
+- This procedure identifies cases related to impounding that have been closed and whose closed date is older than the current date minus the retention period.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_document_financial_reports.md b/docs/app/data-retention/populate-procedures/sp_populate_document_financial_reports.md
new file mode 100644
index 0000000000..bd908b5563
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_document_financial_reports.md
@@ -0,0 +1,13 @@
+### sp_populate_document_financial_reports
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `document`
+- **Sub-category Filter:** Selects records specifically in the sub-category with ID `180`, which is `Financial reports` .
+- **Association Filters:** Ensures the documents considered for deletion are not associated with any licences, applications, cases, transport managers, operating centres, IRFO organisations, submissions, statements, or continuation details.
+- **Date Filter:** Filters documents based on the last modification or creation date being older than the current date minus the retention period from the data retention rules table.
+
+#### Summary:
+- Targets documents classified under a specific financial report sub-category, which are not linked to any major entity (eg licences or applications), and whose last modification or creation date is older than the retention period.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_irfo_gv_permit_expired.md b/docs/app/data-retention/populate-procedures/sp_populate_irfo_gv_permit_expired.md
new file mode 100644
index 0000000000..362d841958
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_irfo_gv_permit_expired.md
@@ -0,0 +1,11 @@
+### sp_populate_irfo_gv_permit_expired
+
+#### Parameters:
+- `p_user_id INT`: User ID executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `irfo_gv_permit`
+- **Expiry Filter:** Selects records where the `expiry_date` of the permit is older than the current date minus the retention period.
+
+#### Summary:
+- This procedure targets expired IRFO goods vehicle permits, populating rows where the expiry date is older than the current date minus the retention period.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_irfo_gv_permit_withdrawn_pending.md b/docs/app/data-retention/populate-procedures/sp_populate_irfo_gv_permit_withdrawn_pending.md
new file mode 100644
index 0000000000..481aa802b8
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_irfo_gv_permit_withdrawn_pending.md
@@ -0,0 +1,13 @@
+### sp_populate_irfo_gv_permit_withdrawn_pending_refused_expired
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `irfo_gv_permit`
+- **Status Filter:** Selects records where the IRFO goods vehicle permit status is in one of the following states: 'irfo_auth_s_withdrawn' (withdrawn), 'irfo_perm_s_pending' (pending), or 'irfo_perm_s_refused' (refused).
+- **No Expiry Date Filter:** Specifically targets permits that do not have an `expiry_date` set.
+- **Creation Date Filter:** Filters permits based on the `created_on` date being older than the current date minus the retention period
+
+#### Summary:
+- This procedure identifies IRFO goods vehicle permits with specific statuses (withdrawn, pending, or refused) that lack an expiry date and whose creation date is older than the current date minus the retention period.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_irfo_operator_expired.md b/docs/app/data-retention/populate-procedures/sp_populate_irfo_operator_expired.md
new file mode 100644
index 0000000000..7d5f6f7492
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_irfo_operator_expired.md
@@ -0,0 +1,13 @@
+### sp_populate_irfo_operator_expired
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `organisation`
+- **Type Filter:** Focuses on organisations of type 'org_t_ir', IRFO Operators.
+- **Creation Date Filter:** Selects organisations based on the `created_on` date being older than the current date minus the retention period fron the data retention rules.
+- **Association Filters:** Ensures that the organisations are not currently associated with any IRFO PSV authorizations, IRFO goods vehicle permits, or licenses.
+
+#### Summary:
+- This procedure targets IRFO operator organisations that are no longer linked to any active permits, authorizations, or licenses and whose creation date is older than the specified retention period. It stages these organization records for deletion.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_irfo_psv_auth_expired.md b/docs/app/data-retention/populate-procedures/sp_populate_irfo_psv_auth_expired.md
new file mode 100644
index 0000000000..61e577ac70
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_irfo_psv_auth_expired.md
@@ -0,0 +1,11 @@
+### sp_populate_irfo_psv_auth_expired
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `irfo_psv_auth`
+- **Expiry Filter:** Selects records where the `expiry_date` of the PSV authorization is older than the current date minus the retention period parameter.
+
+#### Summary:
+- This procedure identifies expired IRFO PSV authorizations, selecting those whose expiry date is older than the current date minus the retention period.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_irfo_psv_auth_withdrawn_pending_refused_expired.md b/docs/app/data-retention/populate-procedures/sp_populate_irfo_psv_auth_withdrawn_pending_refused_expired.md
new file mode 100644
index 0000000000..caae86f4d4
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_irfo_psv_auth_withdrawn_pending_refused_expired.md
@@ -0,0 +1,13 @@
+### sp_populate_irfo_psv_auth_withdrawn_pending_refused_expired
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `irfo_psv_auth`
+- **Status Filter:** Selects records where the PSV authorization status is in one of the following states: 'irfo_auth_s_withdrawn' (withdrawn), 'irfo_auth_s_pending' (pending), 'irfo_auth_s_refused' (refused), 'irfo_auth_s_granted' (granted), or 'irfo_auth_s_cns' (consented).
+- **No Expiry Date Filter:** Specifically targets authorizations that do not have an `expiry_date` set.
+- **Creation Date Filter:** Filters authorizations based on the `created_on` date being older than the current date minus the retention period.
+
+#### Summary:
+- This procedure identifies IRFO PSV authorizations with specific statuses, particularly those without an expiry date and whose creation date is older than the current date minus the retention period.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_irhp_cancelled_withdrawn.md b/docs/app/data-retention/populate-procedures/sp_populate_irhp_cancelled_withdrawn.md
new file mode 100644
index 0000000000..8a273d18d6
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_irhp_cancelled_withdrawn.md
@@ -0,0 +1,11 @@
+### sp_populate_irhp_cancelled_withdrawn
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `irhp_application`
+- **Status Filter:** Selects records where the IRHP application status is either 'permit_app_cancelled' (cancelled) or 'permit_app_withdrawn' (withdrawn).
+- **Date Filter:** Filters applications based on the `last_modified_on` date being older than the current date minus the retention period.
+#### Summary:
+- This procedure identifies cancelled or withdrawn IRHP applications, selecting those whose last modification date is older than the current date minus the retention period
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_irhp_cert_road_nys.md b/docs/app/data-retention/populate-procedures/sp_populate_irhp_cert_road_nys.md
new file mode 100644
index 0000000000..eb6a61933f
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_irhp_cert_road_nys.md
@@ -0,0 +1,12 @@
+### sp_populate_irhp_cert_road_nys
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `irhp_application`
+- **Status Filter:** Selects records where the IRHP application status is 'permit_app_nys' (not yet started).
+- **Permit Type Filter:** Specifically targets applications for permit types with IDs `6` (Roadworthiness Cert Vehicle) and `7` (Roadworthiness Cert Trailer).
+- **Date Filter:** Filters applications based on either the `last_modified_on` date or the `created_on` date being older than 12 months from the current date. This accounts for cases where the last modified date might be null.
+#### Summary:
+- This procedure identifies IRHP applications that are not yet started (status 'permit_app_nys') for specific permit types, selecting those whose last interaction or creation is older than 12 months.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_irhp_expired.md b/docs/app/data-retention/populate-procedures/sp_populate_irhp_expired.md
new file mode 100644
index 0000000000..ac8cef18ed
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_irhp_expired.md
@@ -0,0 +1,14 @@
+### sp_populate_irhp_expired
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `irhp_permit` and `irhp_application`
+- **Status Filter for IRHP Permit:** Selects records where the IRHP permit status is 'irhp_permit_expired'.
+- **Status Filter for IRHP Application:** Selects records where the IRHP application status is 'permit_app_expired'.
+- **Date Filter for IRHP Permit:** Filters permits based on the `last_modified_on` date being older than the current date minus the retention window.
+- **Date Filter for IRHP Application:** Filters applications based on the `last_modified_on` date being older than 60 months from the current date.
+
+#### Summary:
+- This procedure identifies expired IRHP permits and applications, selecting those permits whose last modification date is older than the defined retention period and applications whose last modification date is older than 60 months.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_irhp_replaced_permits.md b/docs/app/data-retention/populate-procedures/sp_populate_irhp_replaced_permits.md
new file mode 100644
index 0000000000..85c7014a59
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_irhp_replaced_permits.md
@@ -0,0 +1,11 @@
+### sp_populate_irhp_replaced_permits
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `irhp_permit`
+- **Status Filter:** Selects records where the IRHP permit status is 'irhp_permit_ceased'.
+- **Expiry Date Filter:** Filters permits based on the `expiry_date`  being earlier than the current date minus the retention period.
+#### Summary:
+- This procedure targets IRHP permits that have been ceased or replaced, selecting those whose expiry date is older than the defined retention period.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_irhp_terminated_permits.md b/docs/app/data-retention/populate-procedures/sp_populate_irhp_terminated_permits.md
new file mode 100644
index 0000000000..40793a0136
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_irhp_terminated_permits.md
@@ -0,0 +1,13 @@
+### sp_populate_irhp_terminated_permits
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `irhp_permit` and `irhp_application`
+- **Status Filter for IRHP Permit:** Selects records where the IRHP permit status is 'irhp_permit_terminated'.
+- **Date Filter for IRHP Permit:** Filters permits based on the `last_modified_on` date being older than the current date minus the retention period defined in the data retention rules.
+- **Comprehensive Application Cleanup:** Also targets IRHP applications associated solely with terminated permits, ensuring applications that no longer have any active or non-terminated permits and were last modified more than 60 months ago are selected.
+
+#### Summary:
+- This procedure identifies terminated IRHP permits and their associated applications, selecting those permits and applications whose last modification dates are older than the defined retention periods.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_irhp_unsuccessful.md b/docs/app/data-retention/populate-procedures/sp_populate_irhp_unsuccessful.md
new file mode 100644
index 0000000000..278598a6c2
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_irhp_unsuccessful.md
@@ -0,0 +1,12 @@
+### sp_populate_irhp_unsuccessful
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `irhp_application`
+- **Status Filter:** Selects records where the IRHP application status is 'permit_app_unsuccessful'.
+- **Date Filter:** Filters applications based on the `last_modified_on` date being older than the current date minus the retention period defined in config.
+
+#### Summary:
+- This procedure identifies IRHP applications that were unsuccessful, selecting those whose last modification date is older than the defined retention period.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_licence_no_imp_unlicenced.md b/docs/app/data-retention/populate-procedures/sp_populate_licence_no_imp_unlicenced.md
new file mode 100644
index 0000000000..324f854ac8
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_licence_no_imp_unlicenced.md
@@ -0,0 +1,14 @@
+### sp_populate_licence_no_imp_unlicenced
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `licence`
+- **Organisation Filter:** Targets licences linked to organisations flagged as unlicensed (`is_unlicensed = TRUE`).
+- **Licence Status Filter:** Selects licences with the status `'lsts_unlicenced'`.
+- **Case Association Filter:** Excludes licences that are associated with any cases, specifically impounding cases.
+- **Expiry Date Filter:** Filters licences based on their expiry date being earlier than the current date minus the retention period.
+
+#### Summary:
+- This procedure identifies unlicensed licences that are not involved in ongoing cases and whose expiry dates have passed beyond the designated retention period.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_licence_no_pi_no_imp_not_unlicenced.md b/docs/app/data-retention/populate-procedures/sp_populate_licence_no_pi_no_imp_not_unlicenced.md
new file mode 100644
index 0000000000..c7e4011982
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_licence_no_pi_no_imp_not_unlicenced.md
@@ -0,0 +1,14 @@
+### sp_populate_licence_no_pi_no_imp_not_unlicenced
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `licence`
+- **Exclusion of Cases:** Excludes licences that are currently involved in any public inquiry (`pi`) or impounding cases.
+- **Organisation Filter:** Targets licences linked to organisations that are not marked as unlicensed (`is_unlicensed = FALSE`).
+- **Licence Category Filter:** Specifically targets licences categorised under 'lcat_gv' (goods vehicles).
+- **Expiry Date Filter:** Filters licences based on their expiry date being earlier than the current date minus the retention period defined in the data retention rules. 
+
+#### Summary:
+- This procedure identifies licences not involved in public inquiries or impounding cases, which are not marked as unlicensed, specifically within the GV category, and whose expiry dates have passed beyond the retention period.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_licence_not_submitted.md b/docs/app/data-retention/populate-procedures/sp_populate_licence_not_submitted.md
new file mode 100644
index 0000000000..eef563ff8f
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_licence_not_submitted.md
@@ -0,0 +1,13 @@
+### sp_populate_licence_not_submitted
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Summary:
+- This procedure identifies licences that have not been submitted and are not associated with any applications. It targets those whose expiry dates have passed beyond the retention period
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `licence`
+- **Status Filter:** Selects licences with the status `'lsts_not_submitted'`.
+- **Expiry Date Filter:** Filters licences based on their expiry date being earlier than the current date minus the retention period.
+- **Application Association Filter:** Excludes licences that are associated with any applications, ensuring only those without any submitted applications are selected.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_licence_with_imp_not_unlicenced.md b/docs/app/data-retention/populate-procedures/sp_populate_licence_with_imp_not_unlicenced.md
new file mode 100644
index 0000000000..19c68a477c
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_licence_with_imp_not_unlicenced.md
@@ -0,0 +1,14 @@
+### sp_populate_licence_with_imp_not_unlicenced
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Summary:
+- This procedure identifies licences within the goods vehicles category that have been involved in impounding cases and are not associated with unlicensed organisations. It targets those whose expiry dates have passed beyond the designated retention period.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `licence`
+- **Impounding Cases Filter:** Selects licences that are associated with impounding cases.
+- **Organisation Filter:** Targets licences linked to organisations that are not marked as unlicensed (`is_unlicensed = FALSE`).
+- **Licence Category Filter:** Specifically targets licences categorised under 'lcat_gv' (goods vehicles).
+- **Expiry Date Filter:** Filters licences based on their expiry date being earlier than the current date minus the defined retention period.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_licence_with_pi_no_imp_not_unlicenced.md b/docs/app/data-retention/populate-procedures/sp_populate_licence_with_pi_no_imp_not_unlicenced.md
new file mode 100644
index 0000000000..43951140ca
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_licence_with_pi_no_imp_not_unlicenced.md
@@ -0,0 +1,15 @@
+### sp_populate_licence_with_pi_no_imp_not_unlicenced
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Summary:
+- This procedure identifies goods vehicle licences involved in public inquiries but not impounding cases, which are not associated with unlicensed organisations, and whose expiry dates have passed beyond the retention period.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `licence`
+- **Public Inquiry Filter:** Selects licences that are involved in public inquiries.
+- **Exclusion of Impounding Cases:** Excludes licences that are currently involved in any impounding cases, selecting only those with public inquiry cases.
+- **Organisation Filter:** Targets licences linked to organisations that are not marked as unlicensed (`is_unlicensed = FALSE`).
+- **Licence Category Filter:** Specifically targets licences categorised under 'lcat_gv' (goods vehicles).
+- **Expiry Date Filter:** Filters licences based on their expiry date being earlier than the current date minus the retention period defined in the data retention rules.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_misc_fees.md b/docs/app/data-retention/populate-procedures/sp_populate_misc_fees.md
new file mode 100644
index 0000000000..e9f419fd7c
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_misc_fees.md
@@ -0,0 +1,12 @@
+### sp_populate_misc_fees
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `fee`
+- **Association Filters:** Targets fee records that are not associated with any licence, IRFO goods vehicle permit, IRFO PSV authorization, bus registration, application, IRHP application, or task, ensuring only miscellaneous fees with no current relevant entity links are selected.
+- **Date Filter:** Filters fees based on the `last_modified_on` or `created_on` date being older than the current date minus the retention period defined in the retention rules. This identifies fees that have not been updated or relevant for a specified time period.
+
+#### Summary:
+- This procedure identifies miscellaneous fee records that are unlinked to any major operational entity and whose modification or creation date is older than the defined retention period.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_new_var_appl_bus_reg_expired.md b/docs/app/data-retention/populate-procedures/sp_populate_new_var_appl_bus_reg_expired.md
new file mode 100644
index 0000000000..f3dc89cc3b
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_new_var_appl_bus_reg_expired.md
@@ -0,0 +1,12 @@
+### sp_populate_new_var_appl_bus_reg_expired
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `bus_reg` (bus registration)
+- **Status Filter:** Selects records where the bus registration status is 'breg_s_expired'
+- **Expiry Date Filter:** Filters registrations based on the `end_date` being earlier than the current date minus the retention period defined in the data retention rules. 
+
+#### Summary:
+- This procedure identifies bus registrations that have expired and whose end date is older than the defined retention period.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_org_no_licence.md b/docs/app/data-retention/populate-procedures/sp_populate_org_no_licence.md
new file mode 100644
index 0000000000..5c32879ac8
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_org_no_licence.md
@@ -0,0 +1,14 @@
+### sp_populate_org_no_licence
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `organisation`
+- **Licence Association Filter:** Targets organisations that currently do not hold any licences. It excludes organisations that are associated with any record in the `licence` table.
+- **Modification Date Filter:** Filters organisations based on the `last_modified_on` date being earlier than the current date minus the retention period defined.
+- **IRFO Filter:** Excludes IRFO organisations by checking if `is_irfo=0`.
+- **Custom Rule Filter:** Ensures organisations have not been marked for retention under any custom rules.
+
+#### Summary:
+- This procedure identifies organisations without any current licences and which have not been modified within the retention period defined by the rules table.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_orphan_users.md b/docs/app/data-retention/populate-procedures/sp_populate_orphan_users.md
new file mode 100644
index 0000000000..02159582ff
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_orphan_users.md
@@ -0,0 +1,13 @@
+### sp_populate_orphan_users
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `user`
+- **Activity Filter:** Targets users who have not logged in since a specified period defined by `v_retention_period` months ago. This condition also accounts for users who have never logged in since their creation if `last_login_at` is NULL.
+- **Association Filter:** Excludes users who are associated with any `organisation_user` entries, ensuring that only truly orphaned users are targeted. Further, users are excluded if they have associations with local authorities, partner contact details, teams, and certain specified user IDs (1 and 4, reserved for administrative or system accounts).
+- **Expiry Date Filter:** Utilizes the `last_login_at` or `created_on` dates to determine if the user's record has surpassed the retention period without activity or association, qualifying them as stale or orphaned.
+
+#### Summary:
+- This procedure identifies users who are considered orphaned due to lack of login activity, absence of associations with organisations, teams, local authorities, or partners, and are not specially protected user IDs. 
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_queue_table_items_to_delete.md b/docs/app/data-retention/populate-procedures/sp_populate_queue_table_items_to_delete.md
new file mode 100644
index 0000000000..09e05032c5
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_queue_table_items_to_delete.md
@@ -0,0 +1,14 @@
+### sp_populate_queue_table_items_to_delete
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `queue`
+- **Status Filter:** Targets queue items that have a status of 'que_sts_complete'.
+- **Date Filter:** Utilizes the `last_modified_on` or `created_on` date to check if the queue item's record is older than the retention period . The filter checks:
+    - If `last_modified_on` is NULL, then it compares the `created_on` date.
+    - Otherwise, it uses the `last_modified_on` date.
+
+#### Summary:
+- This procedure identifies completed queue items that have not been updated or interacted with past the specified retention period.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_refused_with_var_zero.md b/docs/app/data-retention/populate-procedures/sp_populate_refused_with_var_zero.md
new file mode 100644
index 0000000000..adaa537501
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_refused_with_var_zero.md
@@ -0,0 +1,13 @@
+### sp_populate_refused_with_var_zero
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `bus_reg` (bus registration)
+- **Status Filter:** Targets bus registrations where the status is 'breg_s_refused'.
+- **Variation Number Filter:** Selects registrations that have a variation number of zero (`variation_no = 0`).
+- **Event History Filter:** Includes a subquery that fetches the latest event datetime for events of type `1018` (Registration Refused), and filters out registrations where this occurred before the current date minus the specified retention period.
+
+#### Summary:
+- This procedure identifies bus registrations that have been refused, have no variations, and where the latest relevant event occurred prior to the retention period.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_tm_pi.md b/docs/app/data-retention/populate-procedures/sp_populate_tm_pi.md
new file mode 100644
index 0000000000..dbb26969e0
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_tm_pi.md
@@ -0,0 +1,12 @@
+### sp_populate_tm_pi
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `transport_manager`
+- **Date Filter:** Selects transport managers whose last associated licence date is earlier than the current date minus the specified retention period (`v_retention_period`).
+- **Case Involvement Filter:** Includes transport managers who have been involved in public inquiries (`pi`) specifically related to transport manager cases (`case_t_tm`), or have decisions from transport manager cases marked as 'tm_decision_rl' (revoked licence) or 'tm_decision_rnl' (revoked no licence).
+
+#### Summary:
+- This procedure identifies transport manager records that have regulatory issues, and last licence date is beyond the retention period.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_tm_without_pi.md b/docs/app/data-retention/populate-procedures/sp_populate_tm_without_pi.md
new file mode 100644
index 0000000000..ae490b8151
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_tm_without_pi.md
@@ -0,0 +1,12 @@
+### sp_populate_tm_without_pi
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `transport_manager`
+- **Date Filter:** Targets transport managers whose last associated licence date is earlier than the current date minus the specified retention period (`v_retention_period`).
+- **Case Involvement Filter:** Excludes transport managers who have been involved in public inquiries (`pi`) or have decisions from transport manager cases marked as 'tm_decision_rl' or 'tm_decision_rnl'.
+
+#### Summary:
+- This procedure identifies transport manager records that have not been involved in public inquiries or other decisions about thier their status, whose last licence date has surpassed the retention period.
diff --git a/docs/app/data-retention/populate-procedures/sp_populate_withdrawn_with_var_zero.md b/docs/app/data-retention/populate-procedures/sp_populate_withdrawn_with_var_zero.md
new file mode 100644
index 0000000000..5953a78849
--- /dev/null
+++ b/docs/app/data-retention/populate-procedures/sp_populate_withdrawn_with_var_zero.md
@@ -0,0 +1,13 @@
+### sp_populate_withdrawn_with_var_zero
+
+#### Parameters:
+- `p_user_id INT`: User ID of the person executing the procedure.
+
+#### Record Selection Criteria for Deletion:
+- **Entity:** `bus_reg` (bus registration)
+- **Status Filter:** Selects bus registrations where the status is 'breg_s_withdrawn'.
+- **Variation Number Filter:** Targets registrations that have a variation number of zero (`variation_no = 0`),  selecting initial applications that were withdrawn without any subsequent variations or changes.
+- **Event History Filter:** Includes a join to `event_history` to ensure that only bus registrations with specific events of type '1002' are selected, and only events occurring before the current date minus the retention period.
+
+#### Summary:
+- This procedure identifies bus registrations that have been withdrawn, have no variations, and have specific withdrawal-related events recorded before the retention period expiry.