1pub mod watermark_rules {
35 use alloc::format;
36 use alloc::vec::Vec;
37 use core::num::NonZeroUsize;
38
39 use happenstance_core::{
40 CommitError, Event, EventStore, ProjectionProbe, ProjectionStore, SequencePosition, StoreId,
41 };
42 use happenstance_sync::runner::{Edge, Watermark};
43 use happenstance_sync::{MemoryIngestStore, StoreResume, StoreSyncPeer, WriteGroupStore};
44 use happenstance_testkit::{ProjectionFixture, RuleOutcome};
45
46 use crate::violation::{Violation, ensure};
47 use crate::{IngestFixture, MemoryIngestFixture, MemoryIngestHandle};
48
49 const CLAUSE: &str = "SY-31";
51
52 const LOCAL: StoreId = StoreId::from_bytes([0x31; 16]);
54
55 const MAX_STEPS: usize = 16;
58
59 const WHOLE_LOG: NonZeroUsize = NonZeroUsize::MIN.saturating_add(63);
61
62 const ROW_WITH_THE_HUB: &str = "sync-watermark/confirmed-by-hub";
64 const ROW_WITH_THE_SPOKE: &str = "sync-watermark/confirmed-by-spoke";
65 const ROW_WITH_THE_STALE: &str = "sync-watermark/stale";
66
67 type Far = StoreSyncPeer<MemoryIngestHandle, MemoryIngestHandle>;
70
71 pub async fn watermark_advances_transactionally_with_a_read_model<F: ProjectionFixture>(
102 open: impl AsyncFn() -> F,
103 ) -> Result<RuleOutcome, Violation> {
104 if let Some(reason) = F::SECOND_HANDLE.reason() {
105 return Err(Violation::new(
106 CLAUSE,
107 format!(
108 "a watermark kept through the port is a claim about what a fresh handle \
109 reads, and this fixture cannot open one ({reason}); SECOND_HANDLE is a MUST"
110 ),
111 ));
112 }
113 let fixture = open().await;
114 let writer = fixture.connect().await;
115 let local = MemoryIngestStore::with_store_id(LOCAL);
116 let hub_store = MemoryIngestFixture::new();
117 let hub = far_side(&hub_store).await;
118
119 append(&local, 2).await?;
120 append(&local, 1).await?;
121 let first = synced_watermark(&mut Edge::new(), &hub, &local).await?;
122 commit_with_row(&writer, &first, ROW_WITH_THE_HUB).await?;
123 let observer = fixture.connect().await;
124 kept_with_its_row(&observer, &first, ROW_WITH_THE_HUB).await?;
125 restored_pushes_nothing(&first, &hub, &local).await?;
126
127 append(&local, 1).await?;
128 let spoke_store = MemoryIngestFixture::new();
129 let second =
130 synced_watermark(&mut Edge::new(), &far_side(&spoke_store).await, &local).await?;
131 commit_with_row(&writer, &second, ROW_WITH_THE_SPOKE).await?;
132 let observer = fixture.connect().await;
133 kept_with_its_row(&observer, &second, ROW_WITH_THE_SPOKE).await?;
134 kept(&observer, &first, Requirement::EachPeersOwn).await?;
135
136 stale_is_refused(&fixture, &local, &first).await?;
137 Ok(RuleOutcome::Ran)
138 }
139
140 async fn stale_is_refused<F: ProjectionFixture>(
144 fixture: &F,
145 local: &MemoryIngestStore,
146 kept_one: &Watermark,
147 ) -> Result<(), Violation> {
148 let lowest = first_group(local).await?;
149 ensure(lowest < kept_one.confirmed(), CLAUSE, || {
150 format!(
151 "the rule's own arrangement: the first local group, at {lowest}, is not below \
152 the watermark {}",
153 kept_one.confirmed()
154 )
155 })?;
156 let stale = Watermark::new(kept_one.peer(), lowest);
157 let writer = fixture.connect().await;
158 let mut batch = begin(&writer).await?;
159 probe_write(&writer, &mut batch, ROW_WITH_THE_STALE).await?;
160 let refused = stale.commit(&writer, batch).await;
161 ensure(
162 matches!(
163 refused,
164 Err(CommitError::CheckpointRegression { current, attempted })
165 if current == kept_one.confirmed() && attempted == lowest
166 ),
167 CLAUSE,
168 || {
169 format!(
170 "a watermark of {lowest} for a peer whose kept watermark is {} must be \
171 refused as a checkpoint regression, the port's answer for any projection; \
172 the store answered {refused:?}. A watermark that moves backwards re-pushes \
173 what the peer already holds, and one written around the port's check is \
174 one written around its transaction",
175 kept_one.confirmed()
176 )
177 },
178 )?;
179 let observer = fixture.connect().await;
180 kept(&observer, kept_one, Requirement::UnmovedByARegression).await?;
181 let row = probe_read(&observer, ROW_WITH_THE_STALE).await?;
182 ensure(row.is_none(), CLAUSE, || {
183 format!(
184 "the refused commit's row is in the read model ({row:?}): a refused watermark \
185 must leave the batch it rode in uncommitted"
186 )
187 })
188 }
189
190 async fn far_side(store: &MemoryIngestFixture) -> Far {
192 StoreSyncPeer::new(store.connect().await, store.connect().await)
193 }
194
195 async fn append(local: &MemoryIngestStore, events: usize) -> Result<(), Violation> {
197 let group = (0..events)
198 .map(|_| Event::new("WatermarkWritten", b"opaque".to_vec()))
199 .collect::<Result<Vec<_>, _>>()
200 .map_err(|error| {
201 Violation::new(CLAUSE, format!("the rule's own event is invalid: {error}"))
202 })?;
203 EventStore::append(local, &group, None)
204 .await
205 .map(|_| ())
206 .map_err(|error| {
207 Violation::new(
208 CLAUSE,
209 format!("the reference local log refused: {error:?}"),
210 )
211 })
212 }
213
214 async fn synced_watermark(
216 edge: &mut Edge<StoreResume>,
217 peer: &Far,
218 local: &MemoryIngestStore,
219 ) -> Result<Watermark, Violation> {
220 for _ in 0..MAX_STEPS {
221 edge.step(peer, local).await.map_err(|error| {
222 Violation::new(
223 CLAUSE,
224 format!("the reference edge failed to step: {error:?}"),
225 )
226 })?;
227 if edge.status().is_synced() {
228 return edge.watermark().ok_or_else(|| {
229 Violation::new(
230 CLAUSE,
231 format!(
232 "the reference edge synced with a far side that names itself and \
233 holds no watermark: {:?}",
234 edge.status()
235 ),
236 )
237 });
238 }
239 }
240 Err(Violation::new(
241 CLAUSE,
242 format!(
243 "the reference edge did not sync in {MAX_STEPS} steps: {:?}",
244 edge.status()
245 ),
246 ))
247 }
248
249 async fn commit_with_row<S: ProjectionProbe>(
252 store: &S,
253 watermark: &Watermark,
254 key: &str,
255 ) -> Result<(), Violation> {
256 let mut batch = begin(store).await?;
257 probe_write(store, &mut batch, key).await?;
258 watermark.commit(store, batch).await.map_err(|error| {
259 Violation::new(
260 CLAUSE,
261 format!(
262 "the store refused to commit a watermark under `{}` with its batch: {error}. \
263 Every id but a sync watermark's refuses the `sync/` prefix (ADR-0082), so a \
264 store that refuses it too has nowhere to keep a runner's watermark",
265 watermark.id()
266 ),
267 )
268 })
269 }
270
271 async fn kept_with_its_row<S: ProjectionProbe>(
274 observer: &S,
275 watermark: &Watermark,
276 key: &str,
277 ) -> Result<(), Violation> {
278 let row = probe_read(observer, key).await?;
279 ensure(row == Some(PROBE_VALUE), CLAUSE, || {
280 format!(
281 "the row committed with a watermark reads back as {row:?} through a fresh \
282 handle; the watermark and the read model must land in one commit"
283 )
284 })?;
285 kept(observer, watermark, Requirement::KeptWithItsRow).await
286 }
287
288 #[derive(Debug, Clone, Copy)]
291 enum Requirement {
292 KeptWithItsRow,
294 EachPeersOwn,
296 UnmovedByARegression,
298 }
299
300 impl core::fmt::Display for Requirement {
301 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
302 f.write_str(match self {
303 Self::KeptWithItsRow => "requirement 1, a watermark kept with its row",
304 Self::EachPeersOwn => "requirement 3, each peer's watermark its own",
305 Self::UnmovedByARegression => {
306 "requirement 4, a watermark unmoved by a refused regression"
307 }
308 })
309 }
310 }
311
312 async fn kept<S: ProjectionStore>(
315 observer: &S,
316 watermark: &Watermark,
317 requirement: Requirement,
318 ) -> Result<(), Violation> {
319 let loaded = Watermark::load(observer, watermark.peer())
320 .await
321 .map_err(|error| {
322 Violation::new(
323 CLAUSE,
324 format!(
325 "{requirement}: a fresh handle could not read a kept watermark: {error:?}"
326 ),
327 )
328 })?;
329 ensure(loaded == Some(*watermark), CLAUSE, || {
330 format!(
331 "{requirement}: a fresh handle loads {loaded:?} under `{}` where {watermark:?} was \
332 committed: a \
333 watermark is a checkpoint of the store, kept per peer, not of the handle that \
334 wrote it nor one shared by every peer",
335 watermark.id()
336 )
337 })
338 }
339
340 async fn restored_pushes_nothing(
344 watermark: &Watermark,
345 hub: &Far,
346 local: &MemoryIngestStore,
347 ) -> Result<(), Violation> {
348 let mut restored = Edge::restored(*watermark);
349 let stepped = restored.step(hub, local).await.map_err(|error| {
350 Violation::new(
351 CLAUSE,
352 format!("the restored edge failed to step: {error:?}"),
353 )
354 })?;
355 ensure(stepped.pushed == 0, CLAUSE, || {
356 format!(
357 "an edge restored from the kept watermark pushed {} events the hub had \
358 already acknowledged",
359 stepped.pushed
360 )
361 })
362 }
363
364 async fn first_group(local: &MemoryIngestStore) -> Result<SequencePosition, Violation> {
366 let page = WriteGroupStore::read_groups(local, None, WHOLE_LOG)
367 .await
368 .map_err(|error| {
369 Violation::new(CLAUSE, format!("the reference local log failed: {error:?}"))
370 })?;
371 page.groups
372 .first()
373 .map(happenstance_sync::WriteGroup::first)
374 .ok_or_else(|| Violation::new(CLAUSE, "the reference local log holds no group"))
375 }
376
377 const PROBE_VALUE: u64 = 31;
379
380 async fn begin<S: ProjectionStore>(store: &S) -> Result<S::Batch, Violation> {
381 store.begin().await.map_err(|error| {
382 Violation::new(
383 CLAUSE,
384 format!("the store could not begin a batch: {error}"),
385 )
386 })
387 }
388
389 async fn probe_write<S: ProjectionProbe>(
390 store: &S,
391 batch: &mut S::Batch,
392 key: &str,
393 ) -> Result<(), Violation> {
394 store
395 .probe_write(batch, key, PROBE_VALUE)
396 .await
397 .map_err(|error| {
398 Violation::new(CLAUSE, format!("the store refused a probe row: {error}"))
399 })
400 }
401
402 async fn probe_read<S: ProjectionProbe>(
403 store: &S,
404 key: &str,
405 ) -> Result<Option<u64>, Violation> {
406 store.probe_read(key).await.map_err(|error| {
407 Violation::new(
408 CLAUSE,
409 format!("the store could not read a probe row: {error}"),
410 )
411 })
412 }
413}
414
415#[macro_export]
432macro_rules! for_each_watermark_rule {
433 ($($callback:tt)+) => {
434 $($callback)+! {
435 watermark_advances_transactionally_with_a_read_model,
437 }
438 };
439}