diff --git a/rust/lance-datagen/src/generator.rs b/rust/lance-datagen/src/generator.rs index 39da4734619..387021fdbfe 100644 --- a/rust/lance-datagen/src/generator.rs +++ b/rust/lance-datagen/src/generator.rs @@ -2818,14 +2818,29 @@ pub mod array { Box::new(RandomIntervalGenerator::new(unit)) } + /// The default sampling range for temporal generators: the 365 days ending at + /// 2024-01-01T00:00:00Z (exclusive) + /// + /// The range must be a fixed anchor and not derived from the wall clock + /// (e.g. `Utc::now()`), otherwise the same RNG seed would generate different + /// values depending on when the generator was created, breaking + /// reproducibility (e.g. of saved fuzz inputs). Callers that need a + /// time-relative range can use the `*_in_range` variants. + fn default_temporal_range() -> (chrono::DateTime, chrono::DateTime) { + let end = chrono::DateTime::::from_timestamp(1_704_067_200, 0) + .expect("2024-01-01T00:00:00Z is a valid timestamp"); + let start = end - chrono::TimeDelta::try_days(365).expect("TimeDelta try_days"); + (start, end) + } + /// Create a generator of randomly sampled date32 values /// - /// Instead of sampling the entire range, all values will be drawn from the last year as this - /// is a more common use pattern + /// Instead of sampling the entire range, all values will be drawn from a fixed + /// one-year range (the 365 days ending at 2024-01-01 UTC) as this is a more + /// common use pattern. Use [`rand_date32_in_range`] to control the range. pub fn rand_date32() -> Box { - let now = chrono::Utc::now(); - let one_year_ago = now - chrono::TimeDelta::try_days(365).expect("TimeDelta try days"); - rand_date32_in_range(one_year_ago, now) + let (start, end) = default_temporal_range(); + rand_date32_in_range(start, end) } /// Create a generator of randomly sampled date32 values in the given range @@ -2853,12 +2868,12 @@ pub mod array { /// Create a generator of randomly sampled date64 values /// - /// Instead of sampling the entire range, all values will be drawn from the last year as this - /// is a more common use pattern + /// Instead of sampling the entire range, all values will be drawn from a fixed + /// one-year range (the 365 days ending at 2024-01-01 UTC) as this is a more + /// common use pattern. Use [`rand_date64_in_range`] to control the range. pub fn rand_date64() -> Box { - let now = chrono::Utc::now(); - let one_year_ago = now - chrono::TimeDelta::try_days(365).expect("TimeDelta try_days"); - rand_date64_in_range(one_year_ago, now) + let (start, end) = default_temporal_range(); + rand_date64_in_range(start, end) } /// Create a generator of randomly sampled timestamp values in the given range @@ -2914,10 +2929,14 @@ pub mod array { } } + /// Create a generator of randomly sampled timestamp values + /// + /// Instead of sampling the entire range, all values will be drawn from a fixed + /// one-year range (the 365 days ending at 2024-01-01 UTC) as this is a more + /// common use pattern. Use [`rand_timestamp_in_range`] to control the range. pub fn rand_timestamp(data_type: &DataType) -> Box { - let now = chrono::Utc::now(); - let one_year_ago = now - chrono::Duration::try_days(365).unwrap(); - rand_timestamp_in_range(one_year_ago, now, data_type) + let (start, end) = default_temporal_range(); + rand_timestamp_in_range(start, end, data_type) } /// Create a generator of randomly sampled date64 values @@ -3214,8 +3233,12 @@ pub fn rand(schema: &Schema) -> BatchGeneratorBuilder { #[cfg(test)] mod tests { - use arrow::datatypes::{Float32Type, Int8Type, Int16Type, UInt32Type}; - use arrow_array::{BooleanArray, Float32Array, Int8Array, Int16Array, Int32Array, UInt32Array}; + use arrow::datatypes::{Float32Type, Int8Type, Int16Type, TimeUnit, UInt32Type}; + use arrow_array::{ + BooleanArray, Date32Array, Date64Array, Float32Array, Int8Array, Int16Array, Int32Array, + TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray, + TimestampSecondArray, UInt32Array, + }; use super::*; @@ -3395,6 +3418,72 @@ mod tests { ); } + #[test] + fn test_rng_temporal_deterministic() { + // The default temporal generators must not depend on the wall clock: the + // same seed must produce the same values no matter when the generator is + // created (https://github.com/lance-format/lance/issues/7913). These + // exact values pin both the RNG stream and the fixed default sampling + // range (the 365 days ending at 2024-01-01 UTC). + fn gen_values(mut genn: Box) -> Arc { + let mut rng = rand_xoshiro::Xoshiro256PlusPlus::seed_from_u64(DEFAULT_SEED.0); + genn.generate(RowCount::from(3), &mut rng).unwrap() + } + + assert_eq!( + *gen_values(array::rand_date32()), + Date32Array::from(vec![19655, 19474, 19717]) + ); + assert_eq!( + *gen_values(array::rand_date64()), + Date64Array::from(vec![ + 1_698_192_000_000, + 1_682_553_600_000, + 1_703_548_800_000 + ]) + ); + assert_eq!( + *gen_values(array::rand_timestamp(&DataType::Timestamp( + TimeUnit::Second, + None + ))), + TimestampSecondArray::from(vec![1_698_211_127, 1_682_585_540, 1_703_559_286]) + ); + assert_eq!( + *gen_values(array::rand_timestamp(&DataType::Timestamp( + TimeUnit::Millisecond, + None + ))), + TimestampMillisecondArray::from(vec![ + 1_698_211_127_056, + 1_682_585_540_319, + 1_703_559_286_487 + ]) + ); + assert_eq!( + *gen_values(array::rand_timestamp(&DataType::Timestamp( + TimeUnit::Microsecond, + None + ))), + TimestampMicrosecondArray::from(vec![ + 1_698_211_127_056_596, + 1_682_585_540_319_384, + 1_703_559_286_487_645 + ]) + ); + assert_eq!( + *gen_values(array::rand_timestamp(&DataType::Timestamp( + TimeUnit::Nanosecond, + None + ))), + TimestampNanosecondArray::from(vec![ + 1_698_211_127_056_596_085, + 1_682_585_540_319_384_548, + 1_703_559_286_487_645_287 + ]) + ); + } + #[test] fn test_rng_list() { // Note: these tests are heavily dependent on the default seed. diff --git a/rust/lance/tests/query/primitives.rs b/rust/lance/tests/query/primitives.rs index 65fa6f4e4d3..349c005a23b 100644 --- a/rust/lance/tests/query/primitives.rs +++ b/rust/lance/tests/query/primitives.rs @@ -260,7 +260,9 @@ async fn test_query_date(#[case] data_type: DataType) { test_scan(&original, &ds).await; test_take(&original, &ds).await; test_filter(&original, &ds, "value < current_date()").await; - test_filter(&original, &ds, "value > DATE '2024-01-01'").await; + // Mid-range literal: rand_type samples dates from the fixed range + // [2023-01-01, 2024-01-01), so this splits the generated values + test_filter(&original, &ds, "value > DATE '2023-07-01'").await; test_filter(&original, &ds, "value is null").await; test_filter(&original, &ds, "value is not null").await; }) @@ -295,7 +297,9 @@ async fn test_query_timestamp(#[case] data_type: DataType) { test_scan(&original, &ds).await; test_take(&original, &ds).await; test_filter(&original, &ds, "value < current_timestamp()").await; - test_filter(&original, &ds, "value > TIMESTAMP '2024-01-01 00:00:00'").await; + // Mid-range literal: rand_type samples timestamps from the fixed range + // [2023-01-01, 2024-01-01), so this splits the generated values + test_filter(&original, &ds, "value > TIMESTAMP '2023-07-01 00:00:00'").await; test_filter(&original, &ds, "value is null").await; test_filter(&original, &ds, "value is not null").await; })