From 431fa07d7c1097df0b7b6b454950ec8a87e6f682 Mon Sep 17 00:00:00 2001 From: gloskull Date: Sun, 26 Jul 2026 06:25:38 +0100 Subject: [PATCH] Add multi-region disaster recovery primitives --- docs/runbooks/multi-region-dr.md | 43 ++++++ src/lib.rs | 6 + src/replication/mod.rs | 241 +++++++++++++++++++++++++++++++ tests/replication_dr_test.rs | 112 ++++++++++++++ 4 files changed, 402 insertions(+) create mode 100644 docs/runbooks/multi-region-dr.md create mode 100644 src/replication/mod.rs create mode 100644 tests/replication_dr_test.rs diff --git a/docs/runbooks/multi-region-dr.md b/docs/runbooks/multi-region-dr.md new file mode 100644 index 0000000..5e1d8ce --- /dev/null +++ b/docs/runbooks/multi-region-dr.md @@ -0,0 +1,43 @@ +# Multi-Region Replication and Disaster Recovery Runbook + +## Architecture + +VeriNode operators should run at least three regions: one primary writer and +multiple read replicas. Replication health is modeled in +`src/replication/mod.rs` by `ReplicationTopology` and `RegionStatus` so every +service evaluates the same deterministic gates before traffic movement. + +## SLOs and release gates + +- Critical paths must remain at or below 100 ms P99. +- Availability target is 99.99%. +- Replication lag must remain at or below 100 ms for every failover candidate. +- Canary promotion requires a passed security review, at least 99.99% request + success, and P99 latency at or below 100 ms. + +## Blue-green and canary deployment + +1. Deploy the inactive color to all replica regions. +2. Run schema and storage-layout checks before enabling writes. +3. Shift 1%, 10%, 25%, 50%, then 100% of traffic while recording + `CanaryAnalysis` samples. +4. Promote only if `passes_release_gate()` succeeds. +5. Roll back to the previous color immediately on latency, availability, or + security-review failures. + +## Disaster recovery exercise + +1. Capture a `ReplicationTopology` snapshot from monitoring. +2. Require `validate_dr_posture()` to pass before the exercise begins. +3. Generate a `failover_plan()` and freeze writes during DNS cutover. +4. Verify read-after-write behavior in the target region. +5. Record a `DisasterRecoveryTestReport`; the report passes only when canary, + RTO (<= 300 seconds), and RPO (<= 100 ms lag) requirements are satisfied. + +## Monitoring and alerts + +Dashboard panels should display the fields from `ReplicationMetrics`: +configured regions, healthy regions, max replication lag, max critical-path P99, +availability target, P99 target, and DR readiness. Page operators when +`dr_ready` is false, when max replication lag exceeds 100 ms, or when P99 +latency exceeds 100 ms. diff --git a/src/lib.rs b/src/lib.rs index 4ac6c55..6741512 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -53,6 +53,12 @@ pub mod pool_manager; // span semantics for ingestion by observability pipelines. pub mod tracing; +// Multi-region replication and disaster recovery readiness (issue #91). +// Provides deterministic topology validation, failover planning, canary +// release gates, DR test reports, and dashboard snapshots for off-chain +// operators without adding runtime dependencies. +pub mod replication; + // --- ERROR CODES --- #[contracterror] diff --git a/src/replication/mod.rs b/src/replication/mod.rs new file mode 100644 index 0000000..9c2cdce --- /dev/null +++ b/src/replication/mod.rs @@ -0,0 +1,241 @@ +//! Multi-region replication and disaster-recovery primitives (issue #91). +//! +//! The runtime intentionally stays dependency-free and deterministic: node +//! operators can feed health probes, replication-lag samples, and canary +//! results into these plain Rust types from any monitoring stack. The module +//! then produces stable decisions for failover eligibility, blue-green deploy +//! gates, DR-test reports, and dashboard/alert snapshots. + +extern crate alloc; + +use alloc::string::String; +use alloc::vec::Vec; + +/// P99 latency target for critical paths, in milliseconds. +pub const CRITICAL_PATH_P99_TARGET_MS: u64 = 100; +/// Maximum tolerated replication lag before a region is considered stale. +pub const MAX_REPLICATION_LAG_MS: u64 = 100; +/// Availability objective in basis points: 99.99%. +pub const AVAILABILITY_TARGET_BPS: u32 = 9_999; +/// Minimum number of healthy regions needed before failover is safe. +pub const MIN_HEALTHY_REGIONS_FOR_DR: usize = 2; +/// Canary success-rate gate, in basis points. +pub const CANARY_SUCCESS_TARGET_BPS: u32 = 9_999; + +/// Region deployment color used by blue-green rollouts. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DeploymentColor { + Blue, + Green, +} + +/// Coarse region health used by failover and alerting. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RegionHealth { + Healthy, + Degraded, + Unavailable, +} + +/// Configuration and live status for one replication region. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RegionStatus { + pub id: String, + pub priority: u8, + pub health: RegionHealth, + pub replication_lag_ms: u64, + pub critical_path_p99_ms: u64, + pub deployment_color: DeploymentColor, +} + +impl RegionStatus { + pub fn new( + id: String, + priority: u8, + health: RegionHealth, + replication_lag_ms: u64, + critical_path_p99_ms: u64, + deployment_color: DeploymentColor, + ) -> Self { + Self { + id, + priority, + health, + replication_lag_ms, + critical_path_p99_ms, + deployment_color, + } + } + + pub fn is_dr_ready(&self) -> bool { + self.health == RegionHealth::Healthy + && self.replication_lag_ms <= MAX_REPLICATION_LAG_MS + && self.critical_path_p99_ms <= CRITICAL_PATH_P99_TARGET_MS + } +} + +/// Errors returned by replication-planning operations. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ReplicationError { + EmptyTopology, + NoPrimaryRegion, + NoFailoverCandidate, + InsufficientHealthyRegions, + CanaryFailed, + SecurityReviewRequired, +} + +/// Deterministic system-wide topology snapshot. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ReplicationTopology { + primary_region_id: String, + regions: Vec, +} + +impl ReplicationTopology { + pub fn new( + primary_region_id: String, + regions: Vec, + ) -> Result { + if regions.is_empty() { + return Err(ReplicationError::EmptyTopology); + } + if !regions.iter().any(|r| r.id == primary_region_id) { + return Err(ReplicationError::NoPrimaryRegion); + } + Ok(Self { + primary_region_id, + regions, + }) + } + + pub fn primary_region_id(&self) -> &str { + &self.primary_region_id + } + pub fn regions(&self) -> &[RegionStatus] { + &self.regions + } + + pub fn healthy_region_count(&self) -> usize { + self.regions.iter().filter(|r| r.is_dr_ready()).count() + } + + pub fn failover_candidate(&self) -> Option<&RegionStatus> { + self.regions + .iter() + .filter(|r| r.id != self.primary_region_id && r.is_dr_ready()) + .min_by_key(|r| r.priority) + } + + pub fn validate_dr_posture(&self) -> Result<(), ReplicationError> { + if self.healthy_region_count() < MIN_HEALTHY_REGIONS_FOR_DR { + return Err(ReplicationError::InsufficientHealthyRegions); + } + self.failover_candidate() + .map(|_| ()) + .ok_or(ReplicationError::NoFailoverCandidate) + } + + pub fn failover_plan(&self) -> Result { + self.validate_dr_posture()?; + let candidate = self + .failover_candidate() + .ok_or(ReplicationError::NoFailoverCandidate)?; + Ok(FailoverPlan { + from_region_id: self.primary_region_id.clone(), + to_region_id: candidate.id.clone(), + dns_ttl_seconds: 30, + freeze_writes: true, + verify_read_after_write: true, + }) + } + + pub fn dashboard_snapshot(&self) -> ReplicationMetrics { + let max_replication_lag_ms = self + .regions + .iter() + .map(|r| r.replication_lag_ms) + .max() + .unwrap_or(0); + let max_critical_path_p99_ms = self + .regions + .iter() + .map(|r| r.critical_path_p99_ms) + .max() + .unwrap_or(0); + ReplicationMetrics { + configured_regions: self.regions.len(), + healthy_regions: self.healthy_region_count(), + max_replication_lag_ms, + max_critical_path_p99_ms, + availability_target_bps: AVAILABILITY_TARGET_BPS, + p99_target_ms: CRITICAL_PATH_P99_TARGET_MS, + dr_ready: self.validate_dr_posture().is_ok(), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FailoverPlan { + pub from_region_id: String, + pub to_region_id: String, + pub dns_ttl_seconds: u64, + pub freeze_writes: bool, + pub verify_read_after_write: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CanaryAnalysis { + pub requests: u64, + pub successful_requests: u64, + pub p99_latency_ms: u64, + pub security_review_passed: bool, +} + +impl CanaryAnalysis { + pub fn success_rate_bps(&self) -> u32 { + if self.requests == 0 { + return 0; + } + ((self.successful_requests.saturating_mul(10_000)) / self.requests).min(10_000) as u32 + } + + pub fn passes_release_gate(&self) -> Result<(), ReplicationError> { + if !self.security_review_passed { + return Err(ReplicationError::SecurityReviewRequired); + } + if self.success_rate_bps() < CANARY_SUCCESS_TARGET_BPS + || self.p99_latency_ms > CRITICAL_PATH_P99_TARGET_MS + { + return Err(ReplicationError::CanaryFailed); + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DisasterRecoveryTestReport { + pub failover_plan: FailoverPlan, + pub canary: CanaryAnalysis, + pub recovery_time_seconds: u64, + pub recovery_point_lag_ms: u64, +} + +impl DisasterRecoveryTestReport { + pub fn passed(&self) -> bool { + self.canary.passes_release_gate().is_ok() + && self.recovery_time_seconds <= 300 + && self.recovery_point_lag_ms <= MAX_REPLICATION_LAG_MS + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ReplicationMetrics { + pub configured_regions: usize, + pub healthy_regions: usize, + pub max_replication_lag_ms: u64, + pub max_critical_path_p99_ms: u64, + pub availability_target_bps: u32, + pub p99_target_ms: u64, + pub dr_ready: bool, +} diff --git a/tests/replication_dr_test.rs b/tests/replication_dr_test.rs new file mode 100644 index 0000000..cf7c167 --- /dev/null +++ b/tests/replication_dr_test.rs @@ -0,0 +1,112 @@ +use sorosusu_contracts::replication::{ + CanaryAnalysis, DeploymentColor, DisasterRecoveryTestReport, RegionHealth, RegionStatus, + ReplicationError, ReplicationTopology, AVAILABILITY_TARGET_BPS, CRITICAL_PATH_P99_TARGET_MS, +}; + +fn region(id: &str, priority: u8, health: RegionHealth, lag: u64, p99: u64) -> RegionStatus { + RegionStatus::new(id.into(), priority, health, lag, p99, DeploymentColor::Blue) +} + +#[test] +fn failover_plan_selects_lowest_priority_healthy_replica() { + let topology = ReplicationTopology::new( + "us-east-1".into(), + vec![ + region("us-east-1", 0, RegionHealth::Healthy, 10, 80), + region("eu-west-1", 2, RegionHealth::Healthy, 15, 85), + region("us-west-2", 1, RegionHealth::Healthy, 20, 90), + ], + ) + .unwrap(); + + let plan = topology.failover_plan().unwrap(); + + assert_eq!(plan.from_region_id, "us-east-1"); + assert_eq!(plan.to_region_id, "us-west-2"); + assert_eq!(plan.dns_ttl_seconds, 30); + assert!(plan.freeze_writes); + assert!(plan.verify_read_after_write); +} + +#[test] +fn stale_or_slow_regions_block_dr_posture() { + let topology = ReplicationTopology::new( + "us-east-1".into(), + vec![ + region("us-east-1", 0, RegionHealth::Healthy, 10, 80), + region("eu-west-1", 1, RegionHealth::Healthy, 101, 80), + region("us-west-2", 2, RegionHealth::Healthy, 10, 101), + ], + ) + .unwrap(); + + assert_eq!( + topology.failover_plan(), + Err(ReplicationError::InsufficientHealthyRegions) + ); + + let metrics = topology.dashboard_snapshot(); + assert_eq!(metrics.configured_regions, 3); + assert_eq!(metrics.healthy_regions, 1); + assert_eq!(metrics.max_replication_lag_ms, 101); + assert_eq!(metrics.max_critical_path_p99_ms, 101); + assert_eq!(metrics.availability_target_bps, AVAILABILITY_TARGET_BPS); + assert_eq!(metrics.p99_target_ms, CRITICAL_PATH_P99_TARGET_MS); + assert!(!metrics.dr_ready); +} + +#[test] +fn canary_requires_security_review_success_rate_and_latency() { + let secure_fast = CanaryAnalysis { + requests: 10_000, + successful_requests: 9_999, + p99_latency_ms: 100, + security_review_passed: true, + }; + assert_eq!(secure_fast.success_rate_bps(), 9_999); + assert!(secure_fast.passes_release_gate().is_ok()); + + let no_security_review = CanaryAnalysis { + security_review_passed: false, + ..secure_fast.clone() + }; + assert_eq!( + no_security_review.passes_release_gate(), + Err(ReplicationError::SecurityReviewRequired) + ); + + let slow = CanaryAnalysis { + p99_latency_ms: 101, + ..secure_fast + }; + assert_eq!( + slow.passes_release_gate(), + Err(ReplicationError::CanaryFailed) + ); +} + +#[test] +fn disaster_recovery_report_combines_failover_canary_rto_and_rpo() { + let topology = ReplicationTopology::new( + "us-east-1".into(), + vec![ + region("us-east-1", 0, RegionHealth::Healthy, 5, 75), + region("eu-west-1", 1, RegionHealth::Healthy, 7, 85), + ], + ) + .unwrap(); + + let report = DisasterRecoveryTestReport { + failover_plan: topology.failover_plan().unwrap(), + canary: CanaryAnalysis { + requests: 100_000, + successful_requests: 100_000, + p99_latency_ms: 90, + security_review_passed: true, + }, + recovery_time_seconds: 120, + recovery_point_lag_ms: 7, + }; + + assert!(report.passed()); +}