Skip to main content

happenstance_sync_testkit/
watermark_suite.rs

1//! The watermark family (#143): where a runner keeps what a peer confirmed
2//! (SY-31).
3//!
4//! Not a peer's bar and not an ingest store's, but a **projection store's**:
5//! the store a runner keeps its per-peer watermark in, beside the read model
6//! whose rows depend on it. The rule takes a
7//! [`ProjectionFixture`](happenstance_testkit::ProjectionFixture), the fixture
8//! the projection suite already mounts, so a projection adapter calls it over
9//! the fixture it has. There is no mounting macro, as in the convergence
10//! family: one rule, called directly, with no capability to decline.
11//!
12//! # What the rule plays
13//!
14//! The runner. It drives a real [`Edge`](happenstance_sync::runner::Edge) from
15//! an in-memory local log to an in-memory far side that names its store, keeps
16//! the edge's [`Watermark`](happenstance_sync::runner::Watermark) through the
17//! fixture's store in a batch that also carries a read-model row, and reads
18//! both back through a fresh handle. So what it holds the store to is what the
19//! shipped runner asks of it: a checkpoint under a `sync/` id the store did
20//! not mint, committed with the batch, kept apart per peer, refused when it
21//! would move backwards, and seen from outside the handle that wrote it.
22
23/// The watermark rule. See the [crate documentation](crate) for what it
24/// covers.
25///
26/// Returns `Result<RuleOutcome, Violation>`, as every rule in this crate does.
27///
28/// # What the rule does not do
29///
30/// **Assert a local position it did not read back.** Every position compared
31/// is one the local log assigned, read through
32/// [`WriteGroupStore::read_groups`](happenstance_sync::WriteGroupStore::read_groups)
33/// or reported by the edge.
34pub mod watermark_rules {
35    use alloc::format;
36    use alloc::vec::Vec;
37    use core::num::NonZeroUsize;
38
39    use happenstance_core::{
40        CommitError, Event, EventStore, ProjectionProbe, ProjectionStore, SequencePosition, StoreId,
41    };
42    use happenstance_sync::runner::{Edge, Watermark};
43    use happenstance_sync::{MemoryIngestStore, StoreResume, StoreSyncPeer, WriteGroupStore};
44    use happenstance_testkit::{ProjectionFixture, RuleOutcome};
45
46    use crate::violation::{Violation, ensure};
47    use crate::{IngestFixture, MemoryIngestFixture, MemoryIngestHandle};
48
49    /// The rule's clause.
50    const CLAUSE: &str = "SY-31";
51
52    /// The local log's incarnation.
53    const LOCAL: StoreId = StoreId::from_bytes([0x31; 16]);
54
55    /// The most steps an edge takes before the rule decides it will never sync
56    /// (H-13): far above what three small groups need.
57    const MAX_STEPS: usize = 16;
58
59    /// A page large enough for the whole local log.
60    const WHOLE_LOG: NonZeroUsize = NonZeroUsize::MIN.saturating_add(63);
61
62    /// The read-model row each commit carries, by key.
63    const ROW_WITH_THE_HUB: &str = "sync-watermark/confirmed-by-hub";
64    const ROW_WITH_THE_SPOKE: &str = "sync-watermark/confirmed-by-spoke";
65    const ROW_WITH_THE_STALE: &str = "sync-watermark/stale";
66
67    /// A far side over the in-memory reference store, which names its
68    /// incarnation on every answer, so the edge learns whose watermark it holds.
69    type Far = StoreSyncPeer<MemoryIngestHandle, MemoryIngestHandle>;
70
71    /// **SY-31.** A runner's per-peer watermark is kept through the projection
72    /// port, under the reserved id, in the same commit as a read model.
73    ///
74    /// The rule appends two groups to a local log and syncs an edge with a
75    /// hub, then commits the edge's watermark through the fixture's store in a
76    /// batch carrying one probe row. Through a fresh handle it requires:
77    ///
78    /// 1. the row and the watermark both, so they landed in one commit;
79    /// 2. an edge restored from that watermark to push nothing more;
80    /// 3. after a third group and a second edge, to a spoke, committed with a
81    ///    row of its own, each peer's watermark its own;
82    /// 4. a hub watermark below the kept one refused as
83    ///    [`CommitError::CheckpointRegression`], with neither its row nor the
84    ///    watermark changed.
85    ///
86    /// # Rejects
87    ///
88    /// `tests/wrong_watermark_stores.rs` carries four projection stores, each
89    /// wrong only for a `sync/` id: one that refuses the reserved prefix as an
90    /// application id it does not accept, one that keeps sync ids in its
91    /// handle rather than its store, one that keys every peer's watermark
92    /// under the bare prefix, and one that upserts a watermark into a side
93    /// table without the port's regression check.
94    ///
95    /// # Errors
96    ///
97    /// A [`Violation`] of SY-31 if the fixture declines a second handle, if the
98    /// store refuses or loses a watermark, keeps it apart from its batch,
99    /// shares it between peers or lets it move backwards, or if the reference
100    /// edge fails to sync.
101    pub async fn watermark_advances_transactionally_with_a_read_model<F: ProjectionFixture>(
102        open: impl AsyncFn() -> F,
103    ) -> Result<RuleOutcome, Violation> {
104        if let Some(reason) = F::SECOND_HANDLE.reason() {
105            return Err(Violation::new(
106                CLAUSE,
107                format!(
108                    "a watermark kept through the port is a claim about what a fresh handle \
109                     reads, and this fixture cannot open one ({reason}); SECOND_HANDLE is a MUST"
110                ),
111            ));
112        }
113        let fixture = open().await;
114        let writer = fixture.connect().await;
115        let local = MemoryIngestStore::with_store_id(LOCAL);
116        let hub_store = MemoryIngestFixture::new();
117        let hub = far_side(&hub_store).await;
118
119        append(&local, 2).await?;
120        append(&local, 1).await?;
121        let first = synced_watermark(&mut Edge::new(), &hub, &local).await?;
122        commit_with_row(&writer, &first, ROW_WITH_THE_HUB).await?;
123        let observer = fixture.connect().await;
124        kept_with_its_row(&observer, &first, ROW_WITH_THE_HUB).await?;
125        restored_pushes_nothing(&first, &hub, &local).await?;
126
127        append(&local, 1).await?;
128        let spoke_store = MemoryIngestFixture::new();
129        let second =
130            synced_watermark(&mut Edge::new(), &far_side(&spoke_store).await, &local).await?;
131        commit_with_row(&writer, &second, ROW_WITH_THE_SPOKE).await?;
132        let observer = fixture.connect().await;
133        kept_with_its_row(&observer, &second, ROW_WITH_THE_SPOKE).await?;
134        kept(&observer, &first, Requirement::EachPeersOwn).await?;
135
136        stale_is_refused(&fixture, &local, &first).await?;
137        Ok(RuleOutcome::Ran)
138    }
139
140    /// Requirement 4: a watermark below the kept one, in a batch with a row,
141    /// is refused as a regression, and leaves the row out and the watermark
142    /// where it was.
143    async fn stale_is_refused<F: ProjectionFixture>(
144        fixture: &F,
145        local: &MemoryIngestStore,
146        kept_one: &Watermark,
147    ) -> Result<(), Violation> {
148        let lowest = first_group(local).await?;
149        ensure(lowest < kept_one.confirmed(), CLAUSE, || {
150            format!(
151                "the rule's own arrangement: the first local group, at {lowest}, is not below \
152                 the watermark {}",
153                kept_one.confirmed()
154            )
155        })?;
156        let stale = Watermark::new(kept_one.peer(), lowest);
157        let writer = fixture.connect().await;
158        let mut batch = begin(&writer).await?;
159        probe_write(&writer, &mut batch, ROW_WITH_THE_STALE).await?;
160        let refused = stale.commit(&writer, batch).await;
161        ensure(
162            matches!(
163                refused,
164                Err(CommitError::CheckpointRegression { current, attempted })
165                    if current == kept_one.confirmed() && attempted == lowest
166            ),
167            CLAUSE,
168            || {
169                format!(
170                    "a watermark of {lowest} for a peer whose kept watermark is {} must be \
171                     refused as a checkpoint regression, the port's answer for any projection; \
172                     the store answered {refused:?}. A watermark that moves backwards re-pushes \
173                     what the peer already holds, and one written around the port's check is \
174                     one written around its transaction",
175                    kept_one.confirmed()
176                )
177            },
178        )?;
179        let observer = fixture.connect().await;
180        kept(&observer, kept_one, Requirement::UnmovedByARegression).await?;
181        let row = probe_read(&observer, ROW_WITH_THE_STALE).await?;
182        ensure(row.is_none(), CLAUSE, || {
183            format!(
184                "the refused commit's row is in the read model ({row:?}): a refused watermark \
185                 must leave the batch it rode in uncommitted"
186            )
187        })
188    }
189
190    /// A far side over a fresh reference store, named on every answer.
191    async fn far_side(store: &MemoryIngestFixture) -> Far {
192        StoreSyncPeer::new(store.connect().await, store.connect().await)
193    }
194
195    /// Appends one group of `events` opaque events to `local`.
196    async fn append(local: &MemoryIngestStore, events: usize) -> Result<(), Violation> {
197        let group = (0..events)
198            .map(|_| Event::new("WatermarkWritten", b"opaque".to_vec()))
199            .collect::<Result<Vec<_>, _>>()
200            .map_err(|error| {
201                Violation::new(CLAUSE, format!("the rule's own event is invalid: {error}"))
202            })?;
203        EventStore::append(local, &group, None)
204            .await
205            .map(|_| ())
206            .map_err(|error| {
207                Violation::new(
208                    CLAUSE,
209                    format!("the reference local log refused: {error:?}"),
210                )
211            })
212    }
213
214    /// Steps `edge` until it is synced, and hands back its watermark.
215    async fn synced_watermark(
216        edge: &mut Edge<StoreResume>,
217        peer: &Far,
218        local: &MemoryIngestStore,
219    ) -> Result<Watermark, Violation> {
220        for _ in 0..MAX_STEPS {
221            edge.step(peer, local).await.map_err(|error| {
222                Violation::new(
223                    CLAUSE,
224                    format!("the reference edge failed to step: {error:?}"),
225                )
226            })?;
227            if edge.status().is_synced() {
228                return edge.watermark().ok_or_else(|| {
229                    Violation::new(
230                        CLAUSE,
231                        format!(
232                            "the reference edge synced with a far side that names itself and \
233                             holds no watermark: {:?}",
234                            edge.status()
235                        ),
236                    )
237                });
238            }
239        }
240        Err(Violation::new(
241            CLAUSE,
242            format!(
243                "the reference edge did not sync in {MAX_STEPS} steps: {:?}",
244                edge.status()
245            ),
246        ))
247    }
248
249    /// Begins a batch, writes one probe row into it, and commits it with
250    /// `watermark`.
251    async fn commit_with_row<S: ProjectionProbe>(
252        store: &S,
253        watermark: &Watermark,
254        key: &str,
255    ) -> Result<(), Violation> {
256        let mut batch = begin(store).await?;
257        probe_write(store, &mut batch, key).await?;
258        watermark.commit(store, batch).await.map_err(|error| {
259            Violation::new(
260                CLAUSE,
261                format!(
262                    "the store refused to commit a watermark under `{}` with its batch: {error}. \
263                     Every id but a sync watermark's refuses the `sync/` prefix (ADR-0082), so a \
264                     store that refuses it too has nowhere to keep a runner's watermark",
265                    watermark.id()
266                ),
267            )
268        })
269    }
270
271    /// Requirement 1's two halves, through `observer`: the row, and the
272    /// watermark.
273    async fn kept_with_its_row<S: ProjectionProbe>(
274        observer: &S,
275        watermark: &Watermark,
276        key: &str,
277    ) -> Result<(), Violation> {
278        let row = probe_read(observer, key).await?;
279        ensure(row == Some(PROBE_VALUE), CLAUSE, || {
280            format!(
281                "the row committed with a watermark reads back as {row:?} through a fresh \
282                 handle; the watermark and the read model must land in one commit"
283            )
284        })?;
285        kept(observer, watermark, Requirement::KeptWithItsRow).await
286    }
287
288    /// Which numbered requirement a [`kept`] check serves, so a violation says
289    /// which one failed.
290    #[derive(Debug, Clone, Copy)]
291    enum Requirement {
292        /// 1: read back with the row it was committed with.
293        KeptWithItsRow,
294        /// 3: one peer's watermark untouched by another's.
295        EachPeersOwn,
296        /// 4: unchanged by a refused regression.
297        UnmovedByARegression,
298    }
299
300    impl core::fmt::Display for Requirement {
301        fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
302            f.write_str(match self {
303                Self::KeptWithItsRow => "requirement 1, a watermark kept with its row",
304                Self::EachPeersOwn => "requirement 3, each peer's watermark its own",
305                Self::UnmovedByARegression => {
306                    "requirement 4, a watermark unmoved by a refused regression"
307                }
308            })
309        }
310    }
311
312    /// `watermark` is what `observer` loads for its peer, as `requirement`
313    /// asks.
314    async fn kept<S: ProjectionStore>(
315        observer: &S,
316        watermark: &Watermark,
317        requirement: Requirement,
318    ) -> Result<(), Violation> {
319        let loaded = Watermark::load(observer, watermark.peer())
320            .await
321            .map_err(|error| {
322                Violation::new(
323                    CLAUSE,
324                    format!(
325                        "{requirement}: a fresh handle could not read a kept watermark: {error:?}"
326                    ),
327                )
328            })?;
329        ensure(loaded == Some(*watermark), CLAUSE, || {
330            format!(
331                "{requirement}: a fresh handle loads {loaded:?} under `{}` where {watermark:?} was \
332                 committed: a \
333                 watermark is a checkpoint of the store, kept per peer, not of the handle that \
334                 wrote it nor one shared by every peer",
335                watermark.id()
336            )
337        })
338    }
339
340    /// Requirement 2: an edge restored from the kept watermark, which
341    /// [`kept`] has just read back, pushes nothing the hub already
342    /// acknowledged.
343    async fn restored_pushes_nothing(
344        watermark: &Watermark,
345        hub: &Far,
346        local: &MemoryIngestStore,
347    ) -> Result<(), Violation> {
348        let mut restored = Edge::restored(*watermark);
349        let stepped = restored.step(hub, local).await.map_err(|error| {
350            Violation::new(
351                CLAUSE,
352                format!("the restored edge failed to step: {error:?}"),
353            )
354        })?;
355        ensure(stepped.pushed == 0, CLAUSE, || {
356            format!(
357                "an edge restored from the kept watermark pushed {} events the hub had \
358                 already acknowledged",
359                stepped.pushed
360            )
361        })
362    }
363
364    /// The first position of the local log's first write group.
365    async fn first_group(local: &MemoryIngestStore) -> Result<SequencePosition, Violation> {
366        let page = WriteGroupStore::read_groups(local, None, WHOLE_LOG)
367            .await
368            .map_err(|error| {
369                Violation::new(CLAUSE, format!("the reference local log failed: {error:?}"))
370            })?;
371        page.groups
372            .first()
373            .map(happenstance_sync::WriteGroup::first)
374            .ok_or_else(|| Violation::new(CLAUSE, "the reference local log holds no group"))
375    }
376
377    /// The value every probe row carries.
378    const PROBE_VALUE: u64 = 31;
379
380    async fn begin<S: ProjectionStore>(store: &S) -> Result<S::Batch, Violation> {
381        store.begin().await.map_err(|error| {
382            Violation::new(
383                CLAUSE,
384                format!("the store could not begin a batch: {error}"),
385            )
386        })
387    }
388
389    async fn probe_write<S: ProjectionProbe>(
390        store: &S,
391        batch: &mut S::Batch,
392        key: &str,
393    ) -> Result<(), Violation> {
394        store
395            .probe_write(batch, key, PROBE_VALUE)
396            .await
397            .map_err(|error| {
398                Violation::new(CLAUSE, format!("the store refused a probe row: {error}"))
399            })
400    }
401
402    async fn probe_read<S: ProjectionProbe>(
403        store: &S,
404        key: &str,
405    ) -> Result<Option<u64>, Violation> {
406        store.probe_read(key).await.map_err(|error| {
407            Violation::new(
408                CLAUSE,
409                format!("the store could not read a probe row: {error}"),
410            )
411        })
412    }
413}
414
415/// Hands the complete watermark rule set to `$callback`.
416///
417/// This family's single enumeration (CF-22), beside the rule it names. No
418/// mounting macro reads it: a projection adapter calls the rule over its own
419/// fixture.
420///
421/// # Examples
422///
423/// ```
424/// macro_rules! rule_names {
425///     ($($name:ident),* $(,)?) => { [ $( stringify!($name) ),* ] };
426/// }
427///
428/// let names = happenstance_sync_testkit::for_each_watermark_rule!(rule_names);
429/// assert_eq!(names, ["watermark_advances_transactionally_with_a_read_model"]);
430/// ```
431#[macro_export]
432macro_rules! for_each_watermark_rule {
433    ($($callback:tt)+) => {
434        $($callback)+! {
435            // --- The watermark is kept through the port (SY-31) ---------------
436            watermark_advances_transactionally_with_a_read_model,
437        }
438    };
439}