[{"data":1,"prerenderedAt":2130},["ShallowReactive",2],{"navigation":3,"\u002Fblog\u002Ftesting-and-operating-event-time":334,"\u002Fblog\u002Ftesting-and-operating-event-time-surround":2119},[4,43,91,154,181,224,262,293],{"title":5,"path":6,"stem":7,"children":8,"icon":42},"Getting Started","\u002Fdocs\u002Fgetting-started","1.docs\u002F1.getting-started\u002F1.index",[9,12,17,22,27,32,37],{"title":10,"path":6,"stem":7,"icon":11},"Getting started","i-lucide-flag",{"title":13,"path":14,"stem":15,"icon":16},"Modules overview","\u002Fdocs\u002Fgetting-started\u002Fmodules-overview","1.docs\u002F1.getting-started\u002F2.modules-overview","i-lucide-boxes",{"title":18,"path":19,"stem":20,"icon":21},"Installation","\u002Fdocs\u002Fgetting-started\u002Finstallation","1.docs\u002F1.getting-started\u002F3.installation","i-lucide-download",{"title":23,"path":24,"stem":25,"icon":26},"License configuration","\u002Fdocs\u002Fgetting-started\u002Flicense-configuration","1.docs\u002F1.getting-started\u002F4.license-configuration","i-lucide-key-round",{"title":28,"path":29,"stem":30,"icon":31},"Your first app","\u002Fdocs\u002Fgetting-started\u002Ffirst-app","1.docs\u002F1.getting-started\u002F5.first-app","i-lucide-square-play",{"title":33,"path":34,"stem":35,"icon":36},"Project setup","\u002Fdocs\u002Fgetting-started\u002Fproject-setup","1.docs\u002F1.getting-started\u002F6.project-setup","i-lucide-package",{"title":38,"path":39,"stem":40,"icon":41},"AI assistants","\u002Fdocs\u002Fgetting-started\u002Fai-assistants","1.docs\u002F1.getting-started\u002F7.ai-assistants","i-lucide-bot",false,{"title":44,"path":45,"stem":46,"children":47,"icon":50},"Core Concepts","\u002Fdocs\u002Fconcepts","1.docs\u002F2.concepts\u002F1.index",[48,51,56,61,66,71,76,81,86],{"title":49,"path":45,"stem":46,"icon":50},"Core concepts","i-lucide-book-open",{"title":52,"path":53,"stem":54,"icon":55},"Architecture","\u002Fdocs\u002Fconcepts\u002Farchitecture","1.docs\u002F2.concepts\u002F2.architecture","i-lucide-layers",{"title":57,"path":58,"stem":59,"icon":60},"Exactly-once semantics","\u002Fdocs\u002Fconcepts\u002Fexactly-once","1.docs\u002F2.concepts\u002F3.exactly-once","i-lucide-shield-check",{"title":62,"path":63,"stem":64,"icon":65},"Lanes and parallelism","\u002Fdocs\u002Fconcepts\u002Flanes-and-parallelism","1.docs\u002F2.concepts\u002F4.lanes-and-parallelism","i-lucide-split",{"title":67,"path":68,"stem":69,"icon":70},"State and thread-safety","\u002Fdocs\u002Fconcepts\u002Fstate-and-thread-safety","1.docs\u002F2.concepts\u002F5.state-and-thread-safety","i-lucide-lock",{"title":72,"path":73,"stem":74,"icon":75},"Event time and watermarks","\u002Fdocs\u002Fconcepts\u002Fevent-time-and-watermarks","1.docs\u002F2.concepts\u002F6.event-time-and-watermarks","i-lucide-clock",{"title":77,"path":78,"stem":79,"icon":80},"The configuration model","\u002Fdocs\u002Fconcepts\u002Fconfiguration-model","1.docs\u002F2.concepts\u002F7.configuration-model","i-lucide-sliders-horizontal",{"title":82,"path":83,"stem":84,"icon":85},"The error-handling model","\u002Fdocs\u002Fconcepts\u002Ferror-handling-model","1.docs\u002F2.concepts\u002F8.error-handling-model","i-lucide-triangle-alert",{"title":87,"path":88,"stem":89,"icon":90},"How StoatFlow differs from Kafka Streams","\u002Fdocs\u002Fconcepts\u002Fhow-stoatflow-differs-from-ks","1.docs\u002F2.concepts\u002F9.how-stoatflow-differs-from-ks","i-lucide-arrow-left-right",{"title":92,"path":93,"stem":94,"children":95,"icon":98},"Building Topologies","\u002Fdocs\u002Fbuilding","1.docs\u002F3.building\u002F1.index",[96,99,104,109,114,119,124,129,134,139,144,149],{"title":97,"path":93,"stem":94,"icon":98},"Building topologies","i-lucide-blocks",{"title":100,"path":101,"stem":102,"icon":103},"Error handling and DLQ","\u002Fdocs\u002Fbuilding\u002Ferror-handling-dlq","1.docs\u002F3.building\u002F10.error-handling-dlq","i-lucide-circle-x",{"title":105,"path":106,"stem":107,"icon":108},"State stores","\u002Fdocs\u002Fbuilding\u002Fstate-stores","1.docs\u002F3.building\u002F11.state-stores","i-lucide-database",{"title":110,"path":111,"stem":112,"icon":113},"Testing topologies","\u002Fdocs\u002Fbuilding\u002Ftesting","1.docs\u002F3.building\u002F12.testing","i-lucide-flask-conical",{"title":115,"path":116,"stem":117,"icon":118},"Sources and sinks","\u002Fdocs\u002Fbuilding\u002Fstreams-builder","1.docs\u002F3.building\u002F2.streams-builder","i-lucide-import",{"title":120,"path":121,"stem":122,"icon":123},"KStream and KTable operations","\u002Fdocs\u002Fbuilding\u002Fkstream-ktable","1.docs\u002F3.building\u002F3.kstream-ktable","i-lucide-waypoints",{"title":125,"path":126,"stem":127,"icon":128},"Aggregations","\u002Fdocs\u002Fbuilding\u002Faggregations","1.docs\u002F3.building\u002F4.aggregations","i-lucide-sigma",{"title":130,"path":131,"stem":132,"icon":133},"Windowing","\u002Fdocs\u002Fbuilding\u002Fwindowing","1.docs\u002F3.building\u002F5.windowing","i-lucide-calendar-clock",{"title":135,"path":136,"stem":137,"icon":138},"Joins","\u002Fdocs\u002Fbuilding\u002Fjoins","1.docs\u002F3.building\u002F6.joins","i-lucide-git-merge",{"title":140,"path":141,"stem":142,"icon":143},"The Processor API","\u002Fdocs\u002Fbuilding\u002Fprocessor-api","1.docs\u002F3.building\u002F7.processor-api","i-lucide-cpu",{"title":145,"path":146,"stem":147,"icon":148},"Scheduled sources","\u002Fdocs\u002Fbuilding\u002Fscheduled-sources","1.docs\u002F3.building\u002F8.scheduled-sources","i-lucide-timer",{"title":150,"path":151,"stem":152,"icon":153},"Serdes and Avro","\u002Fdocs\u002Fbuilding\u002Fserdes","1.docs\u002F3.building\u002F9.serdes","i-lucide-binary",{"title":155,"path":156,"stem":157,"children":158,"icon":80},"Configuration","\u002Fdocs\u002Fconfiguration","1.docs\u002F4.configuration\u002F1.index",[159,161,166,171,176],{"title":160,"path":156,"stem":157,"icon":80},"How configuration works",{"title":162,"path":163,"stem":164,"icon":165},"Engine configuration (:core)","\u002Fdocs\u002Fconfiguration\u002Fcore-config","1.docs\u002F4.configuration\u002F2.core-config","i-lucide-settings-2",{"title":167,"path":168,"stem":169,"icon":170},"Runtime configuration (:runtime)","\u002Fdocs\u002Fconfiguration\u002Fruntime-config","1.docs\u002F4.configuration\u002F3.runtime-config","i-lucide-server-cog",{"title":172,"path":173,"stem":174,"icon":175},"Defaults, adaptivity, and presets","\u002Fdocs\u002Fconfiguration\u002Fdefaults-and-presets","1.docs\u002F4.configuration\u002F4.defaults-and-presets","i-lucide-gauge",{"title":177,"path":178,"stem":179,"icon":180},"Kafka client configuration","\u002Fdocs\u002Fconfiguration\u002Fkafka-client-config","1.docs\u002F4.configuration\u002F5.kafka-client-config","i-lucide-plug",{"title":182,"path":183,"stem":184,"children":185,"icon":188},"Running in Production","\u002Fdocs\u002Fruntime","1.docs\u002F5.runtime\u002F1.index",[186,189,194,199,204,209,214,219],{"title":187,"path":183,"stem":184,"icon":188},"The runtime","i-lucide-server",{"title":190,"path":191,"stem":192,"icon":193},"The REST API","\u002Fdocs\u002Fruntime\u002Frest-api","1.docs\u002F5.runtime\u002F2.rest-api","i-lucide-globe",{"title":195,"path":196,"stem":197,"icon":198},"Health checks","\u002Fdocs\u002Fruntime\u002Fhealth-checks","1.docs\u002F5.runtime\u002F3.health-checks","i-lucide-heart-pulse",{"title":200,"path":201,"stem":202,"icon":203},"Metrics","\u002Fdocs\u002Fruntime\u002Fmetrics","1.docs\u002F5.runtime\u002F4.metrics","i-lucide-activity",{"title":205,"path":206,"stem":207,"icon":208},"Pause and resume","\u002Fdocs\u002Fruntime\u002Fpause-unpause","1.docs\u002F5.runtime\u002F5.pause-unpause","i-lucide-pause",{"title":210,"path":211,"stem":212,"icon":213},"Plugins and lifecycle hooks","\u002Fdocs\u002Fruntime\u002Fplugins","1.docs\u002F5.runtime\u002F6.plugins","i-lucide-puzzle",{"title":215,"path":216,"stem":217,"icon":218},"Docker images","\u002Fdocs\u002Fruntime\u002Fdocker","1.docs\u002F5.runtime\u002F7.docker","i-lucide-container",{"title":220,"path":221,"stem":222,"icon":223},"GraalVM native image","\u002Fdocs\u002Fruntime\u002Fnative-image","1.docs\u002F5.runtime\u002F8.native-image","i-lucide-zap",{"title":225,"path":226,"stem":227,"children":228,"icon":231},"Deploying & Operating","\u002Fdocs\u002Foperating","1.docs\u002F6.operating\u002F1.index",[229,232,237,242,247,252,257],{"title":230,"path":226,"stem":227,"icon":231},"Deploying and operating","i-lucide-life-buoy",{"title":233,"path":234,"stem":235,"icon":236},"Running on Kubernetes","\u002Fdocs\u002Foperating\u002Fkubernetes","1.docs\u002F6.operating\u002F2.kubernetes","i-lucide-ship",{"title":238,"path":239,"stem":240,"icon":241},"High availability","\u002Fdocs\u002Foperating\u002Fhigh-availability","1.docs\u002F6.operating\u002F3.high-availability","i-lucide-copy",{"title":243,"path":244,"stem":245,"icon":246},"Liveness and readiness probes","\u002Fdocs\u002Foperating\u002Fprobes","1.docs\u002F6.operating\u002F4.probes","i-lucide-stethoscope",{"title":248,"path":249,"stem":250,"icon":251},"Observability","\u002Fdocs\u002Foperating\u002Fobservability","1.docs\u002F6.operating\u002F5.observability","i-lucide-telescope",{"title":253,"path":254,"stem":255,"icon":256},"Tuning under load","\u002Fdocs\u002Foperating\u002Ftuning","1.docs\u002F6.operating\u002F6.tuning","i-lucide-sliders",{"title":258,"path":259,"stem":260,"icon":261},"Production checklist","\u002Fdocs\u002Foperating\u002Fproduction-checklist","1.docs\u002F6.operating\u002F7.production-checklist","i-lucide-clipboard-check",{"title":263,"path":264,"stem":265,"children":266,"icon":90},"Migrating from Kafka Streams","\u002Fdocs\u002Fmigration","1.docs\u002F7.migration\u002F1.index",[267,268,273,278,283,288],{"title":263,"path":264,"stem":265,"icon":90},{"title":269,"path":270,"stem":271,"icon":272},"Automated port","\u002Fdocs\u002Fmigration\u002Fautomated-port","1.docs\u002F7.migration\u002F2.automated-port","i-lucide-wand-sparkles",{"title":274,"path":275,"stem":276,"icon":277},"Migration without carrying state","\u002Fdocs\u002Fmigration\u002Fwithout-data-migration","1.docs\u002F7.migration\u002F3.without-data-migration","i-lucide-sparkles",{"title":279,"path":280,"stem":281,"icon":282},"Migration carrying state","\u002Fdocs\u002Fmigration\u002Fwith-data-migration","1.docs\u002F7.migration\u002F4.with-data-migration","i-lucide-database-backup",{"title":284,"path":285,"stem":286,"icon":287},"The migration tool","\u002Fdocs\u002Fmigration\u002Fmigration-tool","1.docs\u002F7.migration\u002F5.migration-tool","i-lucide-truck",{"title":289,"path":290,"stem":291,"icon":292},"Reusing your Kafka Streams dashboards","\u002Fdocs\u002Fmigration\u002Freusing-kafka-streams-dashboards","1.docs\u002F7.migration\u002F6.reusing-kafka-streams-dashboards","i-lucide-line-chart",{"title":294,"path":295,"stem":296,"children":297,"icon":299},"Reference","\u002Fdocs\u002Freference","1.docs\u002F8.reference\u002F1.index",[298,300,305,310,315,320,325,329],{"title":294,"path":295,"stem":296,"icon":299},"i-lucide-list",{"title":301,"path":302,"stem":303,"icon":304},"Configuration reference","\u002Fdocs\u002Freference\u002Fconfiguration-reference","1.docs\u002F8.reference\u002F2.configuration-reference","i-lucide-table",{"title":306,"path":307,"stem":308,"icon":309},"REST API reference","\u002Fdocs\u002Freference\u002Frest-api-reference","1.docs\u002F8.reference\u002F3.rest-api-reference","i-lucide-network",{"title":311,"path":312,"stem":313,"icon":314},"Gradle plugin reference","\u002Fdocs\u002Freference\u002Fgradle-plugin-reference","1.docs\u002F8.reference\u002F4.gradle-plugin-reference","i-lucide-box",{"title":316,"path":317,"stem":318,"icon":319},"Maven reference","\u002Fdocs\u002Freference\u002Fmaven-reference","1.docs\u002F8.reference\u002F5.maven-reference","i-simple-icons-apachemaven",{"title":321,"path":322,"stem":323,"icon":324},"Kafka Streams compatibility matrix","\u002Fdocs\u002Freference\u002Fks-compatibility-matrix","1.docs\u002F8.reference\u002F6.ks-compatibility-matrix","i-lucide-table-2",{"title":326,"path":327,"stem":328,"icon":175},"Metrics reference","\u002Fdocs\u002Freference\u002Fmetrics-reference","1.docs\u002F8.reference\u002F7.metrics-reference",{"title":330,"path":331,"stem":332,"icon":333},"Glossary","\u002Fdocs\u002Freference\u002Fglossary","1.docs\u002F8.reference\u002F8.glossary","i-lucide-book-a",{"id":335,"title":336,"authors":337,"badge":343,"body":345,"date":2107,"description":2108,"draft":42,"extension":2109,"image":2110,"meta":2112,"navigation":736,"ogImage":2113,"path":2114,"seo":2115,"sitemap":2116,"stem":2117,"updated":2113,"__hash__":2118},"posts\u002F3.blog\u002F18.testing-and-operating-event-time.md","Testing and operating event time",[338],{"name":339,"to":340,"avatar":341},"Hartmut Armbruster","https:\u002F\u002Fwww.linkedin.com\u002Fin\u002Fhartmut-co-uk\u002F",{"src":342},"\u002Fassets\u002Fhartmut_armbruster_monochromatic.jpg",{"label":344},"Deep Dive",{"type":346,"value":347,"toc":2096},"minimark",[348,380,404,503,508,558,561,580,590,597,600,643,659,797,806,809,848,1273,1322,1325,1331,1531,1554,1578,1581,1584,1615,1629,1635,1652,1727,1730,1744,1825,1867,1883,1886,1899,1906,1917,1920,1926,1942,1966,1972,1983,1993,1996,2012,2028,2033,2092],[349,350,353],"callout",{"color":351,"icon":352},"neutral","i-lucide-list-ordered",[354,355,356,360,361,366,367,366,371,375,376],"p",{},[357,358,359],"strong",{},"Event time in StoatFlow — part 4 of 4."," ",[362,363,365],"a",{"href":364},"\u002Fblog\u002Fthree-clocks-stream-time-vs-watermarks","Three clocks"," · ",[362,368,370],{"href":369},"\u002Fblog\u002Fmaking-the-watermark-move","Making the watermark move",[362,372,374],{"href":373},"\u002Fblog\u002Fclosing-a-window-under-exactly-once","Closing a window under exactly-once"," · Testing and operating event time ",[377,378,379],"em",{},"(this post)",[354,381,382,383,387,388,391,392,395,396,403],{},"The ",[384,385,386],"code",{},"TopologyTestDriver"," keeps one clock: stream time is the maximum event time it has seen, advanced by every record you pipe in, with no out-of-orderness lag and no minimum: one watermark for every topic you pipe, the largest timestamp seen on any of them. Production keeps the clock ",[362,389,390],{"href":364},"part 1"," described: a 10-second lag by default, and a minimum across every partition of every source. So a window-close or join-timing test that passes in the driver emits ",[377,393,394],{},"later"," on a real broker, and can count a record the driver dropped. It is the loudest rule in the ",[362,397,399,402],{"href":398},"\u002Fblog\u002Fai-assistant-skills",[384,400,401],{},"stoatflow-test"," skill"," and a callout on the testing page; this post is the operator's half of the series.",[405,406,407,412],"blockquote",{},[354,408,409],{},[357,410,411],{},"TL;DR",[413,414,415,437,443,468,486],"ul",{},[416,417,418,421,422,424,425,428,429,432,433,436],"li",{},[357,419,420],{},"What:"," the test-time and run-time surface of event time — ",[384,423,386],{}," divergences and mitigations, ",[384,426,427],{},"GET \u002Fwatermarks",", the ",[384,430,431],{},"stoatflow.watermark.*"," and ",[384,434,435],{},"stoatflow.dropped.records.total"," meters, the two alerts, the three config keys, and restart \u002F failover behaviour.",[416,438,439,442],{},[357,440,441],{},"Why:"," the driver's clock is Kafka Streams-shaped and production's is not; a green unit test says nothing about emission timing under a real out-of-orderness bound and a real partition minimum.",[416,444,445,448,449,452,453,456,457,432,460,463,464,467],{},[357,446,447],{},"How:"," drive the watermark explicitly in tests; if production must behave like the driver, give it zero lag (",[384,450,451],{},"forMonotonousTimestamps()"," or ",[384,454,455],{},"max-out-of-orderness-ms: 0"," — in the driver itself the generator never runs); put timing assertions in a broker test; in production read ",[384,458,459],{},"stoatflow.watermark.current",[384,461,462],{},"lag.ms"," at info level and alert on a stalled watermark while the app is ",[384,465,466],{},"RUNNING",".",[416,469,470,473,474,477,478,481,482,485],{},[357,471,472],{},"The catch:"," the driver's clock can drop an out-of-order record the engine still counts; ",[384,475,476],{},"stoatflow.watermark.late.records.total"," is debug-level, off in a default deployment; ",[384,479,480],{},"Joined.withGracePeriod"," is advisory and silently ignored (grace comes from the window definition); ",[384,483,484],{},"stoatflow.topology.event.time.tracking = 0"," means the engine decided your topology has no event time, and you should check that it is right.",[416,487,488,360,491,366,495,366,497,366,500,467],{},[357,489,490],{},"Docs:",[362,492,494],{"href":493},"\u002Fdocs\u002Fbuilding\u002Ftesting#advancing-time","Testing",[362,496,200],{"href":201},[362,498,326],{"href":499},"\u002Fdocs\u002Freference\u002Fmetrics-reference#watermarks-and-event-time",[362,501,306],{"href":502},"\u002Fdocs\u002Freference\u002Frest-api-reference#get-watermarks",[504,505,507],"h2",{"id":506},"in-this-post","In this post",[413,509,510,516,522,528,534,540,546,552],{},[416,511,512],{},[362,513,515],{"href":514},"#the-driver-is-not-the-engine","The driver is not the engine",[416,517,518],{},[362,519,521],{"href":520},"#reading-watermarks","Reading \u002Fwatermarks",[416,523,524],{},[362,525,527],{"href":526},"#the-meters-that-matter","The meters that matter",[416,529,530],{},[362,531,533],{"href":532},"#one-alert-on-day-one","One alert on day one",[416,535,536],{},[362,537,539],{"href":538},"#the-three-knobs","The three knobs",[416,541,542],{},[362,543,545],{"href":544},"#restart-failover-migration","Restart, failover, migration",[416,547,548],{},[362,549,551],{"href":550},"#tradeoffs-and-limits","Tradeoffs and limits",[416,553,554],{},[362,555,557],{"href":556},"#whats-coming-next","What's coming next",[504,559,515],{"id":560},"the-driver-is-not-the-engine",[354,562,563,564,567,568,571,572,575,576,579],{},"The divergence is exact, and it is recorded as such in the compatibility register. The driver holds one number for event time and raises it to the maximum of itself and each piped record's timestamp. The source's ",[384,565,566],{},"WatermarkStrategy"," is consulted for its timestamp assigner and nothing else — its watermark generator is never created — so a ",[384,569,570],{},"forBoundedOutOfOrderness(30 s)"," source contributes zero lag in a test. It is one number for every topic you pipe, so there is no minimum across partitions. The final emission runs synchronously, on the very pipe that carries the watermark past ",[384,573,574],{},"windowEnd + grace",": no punctuation lane and no commit-interval lag, because with one thread and no lanes the race ",[362,577,578],{"href":373},"part 3"," closes cannot occur. What the driver does keep is the engine's late-drop floors: a piped record is judged against the current watermark, punctuators and an advance's closing flush against the watermark the previous advance flushed, and timers and the barrier-gated suppress flush against the last barrier's watermark. The rules that decide whether a record counts are the engine's; only the clock under them differs.",[354,581,582,583,586,587,467],{},"The engine has none of those simplifications. Its watermark trails the newest data by the bound, follows the slowest partition, and a final result appears up to two commit intervals after it crosses the close. So a test that pipes a record past the close and asserts the final result on the next line is correct in the driver and early on a broker — by the bound, then by up to two commit intervals — and nothing is wrong with either side. The rule for a late record is the same in both: it is late when its window's close time is at or before the floor of the thread processing it. But the floors stand on different clocks — the driver's is the largest timestamp it has seen, the engine's is the watermark stamped on the last barrier, which trails the newest data by the bound and follows the slowest partition. So the driver can drop an out-of-order record that the engine still counts, and a test of ",[377,584,585],{},"what"," is counted needs the same care as one of ",[377,588,589],{},"when",[354,591,592],{},[593,594],"img",{"alt":595,"src":596},"Six records arrive in order across two partitions. The driver's stream time, the maximum seen, passes the window's 10:06:00 close at arrival four, and the driver emits the final count there. The engine's watermark, the minimum over partitions less the ten-second bound, passes the close only at arrival six, and the engine emits up to two commit intervals later. A fifth in-window record arriving between the two crossings is dropped by the driver, whose clock has passed the close, and counted by the engine, whose watermark has not: the driver counts three, the engine four.","\u002Fassets\u002Fblog\u002Ftesting-and-operating-event-time_driver-vs-engine.svg",[354,598,599],{},"Read the two clock rows against the same six arrivals. The driver's stream time crosses 10:06:00 at ④, a record from the next window, and emits the final count of three on that pipe. The engine's watermark is the minimum over the two partitions, less ten seconds, and P1 is behind: it crosses only at ⑥, when P1's own record from the next window arrives, and the emission itself follows at a barrier the axis cannot show. Record ⑤ — inside the window, arriving between the two crossings — is where they part: the driver's clock has already passed the close, so the driver drops it, while the engine's watermark still sits below 10:06:00, so the engine counts it. The driver emits three, the engine four.",[354,601,602,603,606,607,610,611,614,615,618,619,622,623,626,627,630,631,634,635,638,639,642],{},"Three smaller facts matter for test design. ",[384,604,605],{},"advanceWatermark"," is lenient by default — a backward request is a silent no-op, since every pipe already drives event time forward — and throws only under ",[384,608,609],{},"test.driver.commit-per-pipe=false",", the legacy manual mode. ",[384,612,613],{},"advanceWallClockTime"," fires processing-time timers, ",[384,616,617],{},"WALL_CLOCK_TIME"," punctuators and wall-clock and cron scheduled sources, then commits a barrier; it never moves event time. And because the default commits after every pipe, ",[384,620,621],{},"Suppressed.untilBarrier()"," looks eager in the driver — one emission per record — so a test of epoch consolidation has to turn the flag off and call ",[384,624,625],{},"commitBarrier()"," itself. The driver also refuses, when it is built, every topology the engine refuses to start because a watermark can never drive it — a window, ",[384,628,629],{},"suppress",", stream–stream join, event-time timer or ",[384,632,633],{},"STREAM_TIME"," scheduled source beside a ",[384,636,637],{},"noWatermarks()"," source, under ",[384,640,641],{},"DISABLED"," coordination, or with no Kafka topic to read — so that failure shows in the unit test, not on deploy.",[354,644,645,646,648,649,651,652,655,656,658],{},"The mitigations, in order. First, if a broker must behave like the driver, run production with zero lag — ",[384,647,451],{}," on the source, or ",[384,650,455],{}," — so \"the watermark passed ",[384,653,654],{},"end + grace","\" means the same thing in both. In the driver itself it changes nothing: the generator never runs. Second, drive event time explicitly with ",[384,657,605],{}," rather than through the next record's timestamp — it states the intent, and survives a reordered fixture.",[660,661,666],"pre",{"className":662,"code":663,"language":664,"meta":665,"style":665},"language-kotlin shiki shiki-themes vitesse-light","\u002F\u002F No lag in production either, so \"the watermark passed end + grace\" means the same thing on a broker.\nval consumed = Consumed.with(Serdes.String(), Serdes.Long())\n    .withWatermarkStrategy(WatermarkStrategy.forMonotonousTimestamps())\n\n\u002F\u002F ...pipe records, then advance the watermark explicitly instead of relying on the next record's timestamp:\ndriver.advanceWatermark(windowEnd + graceMs + 1)\nassertThat(output.readKeyValuesToList()).containsExactly(\u002F* the final result *\u002F)\n","kotlin","",[384,667,668,677,714,731,738,744,771],{"__ignoreMap":665},[669,670,673],"span",{"class":671,"line":672},"line",1,[669,674,676],{"class":675},"s8zF2","\u002F\u002F No lag in production either, so \"the watermark passed end + grace\" means the same thing on a broker.\n",[669,678,680,684,688,692,695,699,702,705,708,711],{"class":671,"line":679},2,[669,681,683],{"class":682},"sbBg2","val",[669,685,687],{"class":686},"suHK_"," consumed ",[669,689,691],{"class":690},"sYZai","=",[669,693,694],{"class":686}," Consumed.",[669,696,698],{"class":697},"sySUi","with",[669,700,701],{"class":686},"(Serdes.",[669,703,704],{"class":697},"String",[669,706,707],{"class":686},"(), Serdes.",[669,709,710],{"class":697},"Long",[669,712,713],{"class":686},"())\n",[669,715,717,720,723,726,729],{"class":671,"line":716},3,[669,718,719],{"class":686},"    .",[669,721,722],{"class":697},"withWatermarkStrategy",[669,724,725],{"class":686},"(WatermarkStrategy.",[669,727,728],{"class":697},"forMonotonousTimestamps",[669,730,713],{"class":686},[669,732,734],{"class":671,"line":733},4,[669,735,737],{"emptyLinePlaceholder":736},true,"\n",[669,739,741],{"class":671,"line":740},5,[669,742,743],{"class":675},"\u002F\u002F ...pipe records, then advance the watermark explicitly instead of relying on the next record's timestamp:\n",[669,745,747,750,752,755,759,762,764,768],{"class":671,"line":746},6,[669,748,749],{"class":686},"driver.",[669,751,605],{"class":697},[669,753,754],{"class":686},"(windowEnd ",[669,756,758],{"class":757},"si04Y","+",[669,760,761],{"class":686}," graceMs ",[669,763,758],{"class":757},[669,765,767],{"class":766},"s-TwI"," 1",[669,769,770],{"class":686},")\n",[669,772,774,777,780,783,786,789,792,795],{"class":671,"line":773},7,[669,775,776],{"class":697},"assertThat",[669,778,779],{"class":686},"(output.",[669,781,782],{"class":697},"readKeyValuesToList",[669,784,785],{"class":686},"()).",[669,787,788],{"class":697},"containsExactly",[669,790,791],{"class":686},"(",[669,793,794],{"class":675},"\u002F* the final result *\u002F",[669,796,770],{"class":686},[354,798,799,800,802,803,805],{},"Third, put timing assertions in a broker test — Testcontainers, the real bound, the real minimum, the real barrier cadence — and let the driver test assert ",[377,801,585],{}," is emitted rather than ",[377,804,589],{},". Fourth, never assert on wall clock in either. The testing page now carries this as a callout; it should have from the start.",[504,807,521],{"id":808},"reading-watermarks",[354,810,811,813,814,817,818,821,822,432,825,828,829,832,833,835,836,839,840,843,844,847],{},[384,812,427],{}," is the endpoint that makes a stall attributable. It returns 503 only while the engine is not initialised, or when no partition has registered ",[377,815,816],{},"and"," no watermark has been emitted; a topology that has registered partitions but not yet emitted a watermark answers 200 with ",[384,819,820],{},"globalMs"," at ",[384,823,824],{},"Long.MIN_VALUE",[384,826,827],{},"globalIso: \"N\u002FA\"",". ",[384,830,831],{},"\u002Finfo"," carries the global watermark alone (",[384,834,820],{},", ",[384,837,838],{},"globalIso",") unless ",[384,841,842],{},"runtime.endpoints.info.show-watermarks"," is off; the per-partition view is only on ",[384,845,846],{},"\u002Fwatermarks",". Trimmed to two partitions, one of them idle:",[660,849,853],{"className":850,"code":851,"language":852,"meta":665,"style":665},"language-json shiki shiki-themes vitesse-light","{\n  \"globalMs\": 1790071560000,\n  \"globalIso\": \"2026-09-22T10:06:00Z\",\n  \"alignmentGroupMins\": {},\n  \"alignmentBinding\": null,\n  \"partitions\": {\n    \"orders:0\": {\n      \"watermarkMs\": 1790071560000, \"watermarkIso\": \"2026-09-22T10:06:00Z\",\n      \"idle\": false, \"lastEventTimeMs\": 1790071592000, \"lastEventTimeIso\": \"2026-09-22T10:06:32Z\",\n      \"alignmentGroup\": null, \"alignmentHeld\": false, \"driftFromGroupMinMs\": 0,\n      \"idlenessTimeoutMs\": 300000\n    },\n    \"orders:1\": {\n      \"watermarkMs\": 1790071058000, \"watermarkIso\": \"2026-09-22T09:57:38Z\",\n      \"idle\": true, \"lastEventTimeMs\": 1790071080000, \"lastEventTimeIso\": \"2026-09-22T09:58:00Z\",\n      \"alignmentGroup\": null, \"alignmentHeld\": false, \"driftFromGroupMinMs\": 0,\n      \"idlenessTimeoutMs\": 120000\n    }\n  }\n}\n","json",[384,854,855,860,881,903,917,933,947,961,996,1045,1088,1103,1109,1123,1156,1202,1241,1255,1261,1267],{"__ignoreMap":665},[669,856,857],{"class":671,"line":672},[669,858,859],{"class":690},"{\n",[669,861,862,866,869,872,875,878],{"class":671,"line":679},[669,863,865],{"class":864},"s61at","  \"",[669,867,820],{"class":868},"su6XF",[669,870,871],{"class":864},"\"",[669,873,874],{"class":690},":",[669,876,877],{"class":766}," 1790071560000",[669,879,880],{"class":690},",\n",[669,882,883,885,887,889,891,895,899,901],{"class":671,"line":716},[669,884,865],{"class":864},[669,886,838],{"class":868},[669,888,871],{"class":864},[669,890,874],{"class":690},[669,892,894],{"class":893},"sSP4y"," \"",[669,896,898],{"class":897},"spphp","2026-09-22T10:06:00Z",[669,900,871],{"class":893},[669,902,880],{"class":690},[669,904,905,907,910,912,914],{"class":671,"line":733},[669,906,865],{"class":864},[669,908,909],{"class":868},"alignmentGroupMins",[669,911,871],{"class":864},[669,913,874],{"class":690},[669,915,916],{"class":690}," {},\n",[669,918,919,921,924,926,928,931],{"class":671,"line":740},[669,920,865],{"class":864},[669,922,923],{"class":868},"alignmentBinding",[669,925,871],{"class":864},[669,927,874],{"class":690},[669,929,930],{"class":682}," null",[669,932,880],{"class":690},[669,934,935,937,940,942,944],{"class":671,"line":746},[669,936,865],{"class":864},[669,938,939],{"class":868},"partitions",[669,941,871],{"class":864},[669,943,874],{"class":690},[669,945,946],{"class":690}," {\n",[669,948,949,952,955,957,959],{"class":671,"line":773},[669,950,951],{"class":864},"    \"",[669,953,954],{"class":868},"orders:0",[669,956,871],{"class":864},[669,958,874],{"class":690},[669,960,946],{"class":690},[669,962,964,967,970,972,974,976,979,981,984,986,988,990,992,994],{"class":671,"line":963},8,[669,965,966],{"class":864},"      \"",[669,968,969],{"class":868},"watermarkMs",[669,971,871],{"class":864},[669,973,874],{"class":690},[669,975,877],{"class":766},[669,977,978],{"class":690},",",[669,980,894],{"class":864},[669,982,983],{"class":868},"watermarkIso",[669,985,871],{"class":864},[669,987,874],{"class":690},[669,989,894],{"class":893},[669,991,898],{"class":897},[669,993,871],{"class":893},[669,995,880],{"class":690},[669,997,999,1001,1004,1006,1008,1011,1013,1015,1018,1020,1022,1025,1027,1029,1032,1034,1036,1038,1041,1043],{"class":671,"line":998},9,[669,1000,966],{"class":864},[669,1002,1003],{"class":868},"idle",[669,1005,871],{"class":864},[669,1007,874],{"class":690},[669,1009,1010],{"class":682}," false",[669,1012,978],{"class":690},[669,1014,894],{"class":864},[669,1016,1017],{"class":868},"lastEventTimeMs",[669,1019,871],{"class":864},[669,1021,874],{"class":690},[669,1023,1024],{"class":766}," 1790071592000",[669,1026,978],{"class":690},[669,1028,894],{"class":864},[669,1030,1031],{"class":868},"lastEventTimeIso",[669,1033,871],{"class":864},[669,1035,874],{"class":690},[669,1037,894],{"class":893},[669,1039,1040],{"class":897},"2026-09-22T10:06:32Z",[669,1042,871],{"class":893},[669,1044,880],{"class":690},[669,1046,1048,1050,1053,1055,1057,1059,1061,1063,1066,1068,1070,1072,1074,1076,1079,1081,1083,1086],{"class":671,"line":1047},10,[669,1049,966],{"class":864},[669,1051,1052],{"class":868},"alignmentGroup",[669,1054,871],{"class":864},[669,1056,874],{"class":690},[669,1058,930],{"class":682},[669,1060,978],{"class":690},[669,1062,894],{"class":864},[669,1064,1065],{"class":868},"alignmentHeld",[669,1067,871],{"class":864},[669,1069,874],{"class":690},[669,1071,1010],{"class":682},[669,1073,978],{"class":690},[669,1075,894],{"class":864},[669,1077,1078],{"class":868},"driftFromGroupMinMs",[669,1080,871],{"class":864},[669,1082,874],{"class":690},[669,1084,1085],{"class":766}," 0",[669,1087,880],{"class":690},[669,1089,1091,1093,1096,1098,1100],{"class":671,"line":1090},11,[669,1092,966],{"class":864},[669,1094,1095],{"class":868},"idlenessTimeoutMs",[669,1097,871],{"class":864},[669,1099,874],{"class":690},[669,1101,1102],{"class":766}," 300000\n",[669,1104,1106],{"class":671,"line":1105},12,[669,1107,1108],{"class":690},"    },\n",[669,1110,1112,1114,1117,1119,1121],{"class":671,"line":1111},13,[669,1113,951],{"class":864},[669,1115,1116],{"class":868},"orders:1",[669,1118,871],{"class":864},[669,1120,874],{"class":690},[669,1122,946],{"class":690},[669,1124,1126,1128,1130,1132,1134,1137,1139,1141,1143,1145,1147,1149,1152,1154],{"class":671,"line":1125},14,[669,1127,966],{"class":864},[669,1129,969],{"class":868},[669,1131,871],{"class":864},[669,1133,874],{"class":690},[669,1135,1136],{"class":766}," 1790071058000",[669,1138,978],{"class":690},[669,1140,894],{"class":864},[669,1142,983],{"class":868},[669,1144,871],{"class":864},[669,1146,874],{"class":690},[669,1148,894],{"class":893},[669,1150,1151],{"class":897},"2026-09-22T09:57:38Z",[669,1153,871],{"class":893},[669,1155,880],{"class":690},[669,1157,1159,1161,1163,1165,1167,1170,1172,1174,1176,1178,1180,1183,1185,1187,1189,1191,1193,1195,1198,1200],{"class":671,"line":1158},15,[669,1160,966],{"class":864},[669,1162,1003],{"class":868},[669,1164,871],{"class":864},[669,1166,874],{"class":690},[669,1168,1169],{"class":682}," true",[669,1171,978],{"class":690},[669,1173,894],{"class":864},[669,1175,1017],{"class":868},[669,1177,871],{"class":864},[669,1179,874],{"class":690},[669,1181,1182],{"class":766}," 1790071080000",[669,1184,978],{"class":690},[669,1186,894],{"class":864},[669,1188,1031],{"class":868},[669,1190,871],{"class":864},[669,1192,874],{"class":690},[669,1194,894],{"class":893},[669,1196,1197],{"class":897},"2026-09-22T09:58:00Z",[669,1199,871],{"class":893},[669,1201,880],{"class":690},[669,1203,1205,1207,1209,1211,1213,1215,1217,1219,1221,1223,1225,1227,1229,1231,1233,1235,1237,1239],{"class":671,"line":1204},16,[669,1206,966],{"class":864},[669,1208,1052],{"class":868},[669,1210,871],{"class":864},[669,1212,874],{"class":690},[669,1214,930],{"class":682},[669,1216,978],{"class":690},[669,1218,894],{"class":864},[669,1220,1065],{"class":868},[669,1222,871],{"class":864},[669,1224,874],{"class":690},[669,1226,1010],{"class":682},[669,1228,978],{"class":690},[669,1230,894],{"class":864},[669,1232,1078],{"class":868},[669,1234,871],{"class":864},[669,1236,874],{"class":690},[669,1238,1085],{"class":766},[669,1240,880],{"class":690},[669,1242,1244,1246,1248,1250,1252],{"class":671,"line":1243},17,[669,1245,966],{"class":864},[669,1247,1095],{"class":868},[669,1249,871],{"class":864},[669,1251,874],{"class":690},[669,1253,1254],{"class":766}," 120000\n",[669,1256,1258],{"class":671,"line":1257},18,[669,1259,1260],{"class":690},"    }\n",[669,1262,1264],{"class":671,"line":1263},19,[669,1265,1266],{"class":690},"  }\n",[669,1268,1270],{"class":671,"line":1269},20,[669,1271,1272],{"class":690},"}\n",[354,1274,1275,1276,1278,1279,1282,1283,1285,1286,1288,1289,1291,1292,835,1294,432,1296,1298,1299,1302,1303,1306,1307,1309,1310,1313,1314,1316,1317,1321],{},"Three steps. Find the partition whose ",[384,1277,969],{}," — or ",[384,1280,1281],{},"previewWatermarkMs",", where a partition shows one — equals ",[384,1284,820],{},": that is the minimum, the one every window is waiting for — here ",[384,1287,954],{},", because ",[384,1290,1116],{}," is idle and excluded; its watermark is lower but no longer counts. Then read ",[384,1293,1003],{},[384,1295,1017],{},[384,1297,1095],{}," together: the timeout is the one in force for ",[377,1300,1301],{},"that"," partition — its source's ",[384,1304,1305],{},"withIdleness"," if set, otherwise the global key — and without it the other two are uninterpretable once timeouts differ per source. ",[384,1308,1116],{}," went quiet at 09:58, its source asked for two minutes, so it idled out at 10:00 and the application moved on. Finally, a minimum partition that is ",[377,1311,1312],{},"not"," idle with a fresh ",[384,1315,1017],{}," is delivering old timestamps — the lagging-but-live case ",[362,1318,1320],{"href":1319},"\u002Fblog\u002Fmaking-the-watermark-move#one-minimum-n-sources","part 2"," introduced, which idleness deliberately does not touch.",[504,1323,527],{"id":1324},"the-meters-that-matter",[354,1326,1327,1328,1330],{},"The watermark family sits under ",[384,1329,431],{},", and the split between info and debug level is the first thing to know about it.",[1332,1333,1334,1350],"table",{},[1335,1336,1337],"thead",{},[1338,1339,1340,1344,1347],"tr",{},[1341,1342,1343],"th",{},"Meter",[1341,1345,1346],{},"Level",[1341,1348,1349],{},"What it tells you",[1351,1352,1353,1366,1382,1394,1410,1422,1448,1467,1479,1492,1504,1516],"tbody",{},[1338,1354,1355,1360,1363],{},[1356,1357,1358],"td",{},[384,1359,459],{},[1356,1361,1362],{},"info",[1356,1364,1365],{},"the global watermark, epoch ms — the number every window is judged against",[1338,1367,1368,1373,1375],{},[1356,1369,1370],{},[384,1371,1372],{},"stoatflow.watermark.lag.ms",[1356,1374,1362],{},[1356,1376,1377,1378,1381],{},"wall clock minus the watermark; sampled ~1 Hz whether or not the watermark moved, ",[384,1379,1380],{},"-1"," before the first one",[1338,1383,1384,1389,1391],{},[1356,1385,1386],{},[384,1387,1388],{},"stoatflow.watermark.advance.total",[1356,1390,1362],{},[1356,1392,1393],{},"how often the watermark moved",[1338,1395,1396,1401,1403],{},[1356,1397,1398],{},[384,1399,1400],{},"stoatflow.dropped.records.total{reason=\"late_window\"|\"late_session\"}",[1356,1402,1362],{},[1356,1404,1405,1406,1409],{},"records dropped because their window had already closed — the real drop signal; ",[384,1407,1408],{},"reason=\"no_event_time\""," counts records skipped at the source because they carry no event time",[1338,1411,1412,1417,1419],{},[1356,1413,1414],{},[384,1415,1416],{},"stoatflow.watermark.preview.partitions",[1356,1418,1362],{},[1356,1420,1421],{},"partitions whose next record waits behind older records elsewhere and counts toward the watermark as if processed — a record stamped in the future, or a skewed catch-up",[1338,1423,1424,1439,1441],{},[1356,1425,1426,1429,1430,1429,1433,360,1436],{},[384,1427,1428],{},"stoatflow.watermark.alignment.partitions.held"," \u002F ",[384,1431,1432],{},".held.time.total",[384,1434,1435],{},".yields.total",[384,1437,1438],{},"{group}",[1356,1440,1362],{},[1356,1442,1443,1444,1447],{},"partitions held for alignment, the time a hold stopped all dispatch (",[384,1445,1446],{},"rate()"," is the fraction of time spent waiting), and in-order yields",[1338,1449,1450,1455,1457],{},[1356,1451,1452],{},[384,1453,1454],{},"stoatflow.topology.event.time.tracking",[1356,1456,1362],{},[1356,1458,1459,1462,1463,1466],{},[384,1460,1461],{},"1"," when the engine tracks watermarks, ",[384,1464,1465],{},"0"," when it decided this topology has no event time",[1338,1468,1469,1474,1476],{},[1356,1470,1471],{},[384,1472,1473],{},"stoatflow.store.segments{store_name,store_type}",[1356,1475,1362],{},[1356,1477,1478],{},"live RocksDB segments per window\u002Fsession store — tracks watermark progress, not write volume",[1338,1480,1481,1486,1489],{},[1356,1482,1483],{},[384,1484,1485],{},"stoatflow.watermark.partition.current{topic,partition}",[1356,1487,1488],{},"debug",[1356,1490,1491],{},"each partition's own watermark",[1338,1493,1494,1499,1501],{},[1356,1495,1496],{},[384,1497,1498],{},"stoatflow.watermark.late.records.total{topic,partition}",[1356,1500,1488],{},[1356,1502,1503],{},"arrivals behind the watermark, most of which are still folded in",[1338,1505,1506,1511,1513],{},[1356,1507,1508],{},[384,1509,1510],{},"stoatflow.watermark.out.of.orderness.max",[1356,1512,1488],{},[1356,1514,1515],{},"the spread between the most and least advanced partition watermarks — partition skew",[1338,1517,1518,1526,1528],{},[1356,1519,1520,1429,1523],{},[384,1521,1522],{},"stoatflow.watermark.alignment.drift.ms",[384,1524,1525],{},".group.min",[1356,1527,1488],{},[1356,1529,1530],{},"alignment drift and group minimum, only when alignment is configured",[354,1532,1533,1534,1537,1538,1541,1542,1545,1546,1549,1550,1553],{},"Two traps. The first is the pair of \"late\" counters. ",[384,1535,1536],{},"dropped.records.total"," with a ",[384,1539,1540],{},"late_*"," reason is Kafka Streams' ",[384,1543,1544],{},"dropped-records"," parity and it is at info level; ",[384,1547,1548],{},"late.records.total"," counts ",[377,1551,1552],{},"arrivals"," behind the watermark, most of which land inside grace and are folded in, and it is at debug level — a default deployment does not have it. Alert on the first; turn the debug level on when you are tuning out-of-orderness, and let the second tell you how disordered the input really is.",[354,1555,1556,1557,1560,1561,1564,1565,1567,1568,1570,1571,1573,1574,1577],{},"The second trap is ",[384,1558,1559],{},"event.time.tracking",". With ",[384,1562,1563],{},"timestamp-coordination-mode: AUTO",", the default, the engine turns watermark tracking on only when the compiled topology needs it — a watermark-aware processor, an explicit ",[384,1566,566],{}," on a source, a ",[384,1569,633],{}," scheduled source, or a processor declaring event-time timers. If you expected windows to close and this gauge reads ",[384,1572,1465],{},", the topology is not what you think it is; force ",[384,1575,1576],{},"ENABLED"," and compare before looking anywhere else.",[504,1579,533],{"id":1580},"one-alert-on-day-one",[354,1582,1583],{},"The metrics page ships two rules; set at least the first before the application takes traffic.",[660,1585,1589],{"className":1586,"code":1587,"language":1588,"meta":665,"style":665},"language-promql shiki shiki-themes vitesse-light","# The watermark has not moved in five minutes while the app reports RUNNING (== 4)\ndelta(stoatflow_watermark_current[5m]) == 0 and stoatflow_client_state == 4\n\n# Event time has fallen more than five minutes behind wall clock while RUNNING\nstoatflow_watermark_lag_ms > 300000 and stoatflow_client_state == 4\n","promql",[384,1590,1591,1596,1601,1605,1610],{"__ignoreMap":665},[669,1592,1593],{"class":671,"line":672},[669,1594,1595],{},"# The watermark has not moved in five minutes while the app reports RUNNING (== 4)\n",[669,1597,1598],{"class":671,"line":679},[669,1599,1600],{},"delta(stoatflow_watermark_current[5m]) == 0 and stoatflow_client_state == 4\n",[669,1602,1603],{"class":671,"line":716},[669,1604,737],{"emptyLinePlaceholder":736},[669,1606,1607],{"class":671,"line":733},[669,1608,1609],{},"# Event time has fallen more than five minutes behind wall clock while RUNNING\n",[669,1611,1612],{"class":671,"line":740},[669,1613,1614],{},"stoatflow_watermark_lag_ms > 300000 and stoatflow_client_state == 4\n",[354,1616,1617,1618,1621,1622,1624,1625,1628],{},"Both are joined on ",[384,1619,1620],{},"stoatflow_client_state == 4",", so a restoring or paused application does not page you. What they catch is the failure mode this series keeps returning to: a stalled watermark closes nothing. Windows stay open, ",[384,1623,629],{}," buffers grow, timers do not fire, stores stop expiring, and segmented stores add one RocksDB instance per segment interval on the partitions that ",[377,1626,1627],{},"are"," advancing. We measured it: a partition that keeps delivering records with old timestamps is never idle-excluded — deliberately, since excluding it would over-advance the watermark and drop its records as late — and in the measurement it pinned the watermark for thirty minutes of lag and grew a three-minute-retention store from its plateau of four segments to thirty. The count was back at four within one barrier of the partition catching up.",[354,1630,1631],{},[593,1632],{"alt":1633,"src":1634},"A modelled stall on a wall-clock axis: the global watermark goes flat from 10:03 when one partition keeps delivering old timestamps; the event-time lag climbs a minute per minute and crosses the five-minute alert line at 10:08, where both documented alerts fire while the application reports RUNNING; the segment count climbs past its plateau of four, one per minute, to twelve. At catch-up, 10:11:30, the watermark jumps, the lag collapses and the segments return to the plateau within one barrier.","\u002Fassets\u002Fblog\u002Ftesting-and-operating-event-time_stall-as-the-meters-see-it.svg",[354,1636,1637,1638,1641,1642,1644,1645,1648,1649,1651],{},"The diagram is a model of that measurement — minutes rather than the test's compressed 400 ms batches, an eight-minute stall rather than thirty — but its shape is the one you will see on a dashboard: ",[384,1639,1640],{},"watermark.current"," flat, ",[384,1643,462],{}," climbing a minute per minute until it crosses the 300 000 ms line, both alerts firing at the same minute because the stall alert needs five flat ones, and ",[384,1646,1647],{},"store.segments"," climbing past the dashed plateau with nothing in the count to say why — pair it with ",[384,1650,462],{},", which does.",[354,1653,1654,1655,1657,1658,1661,1662,1664,1665,1668,1669,1672,1673,1675,1676,1678,1679,1684,1685,1688,1689,1692,1693,1695,1696,1699,1700,1702,1703,432,1705,1708,1709,1712,1713,1717,1718,1720,1721,1723,1724,1726],{},"The runbook when either fires is ",[384,1656,846],{},", then one of five branches. The minimum partition is ",[357,1659,1660],{},"idle-eligible but not yet idle"," — it went quiet and its timeout has not elapsed — so either wait, or lower that source's ",[384,1663,1305],{},". It is ",[357,1666,1667],{},"live but delivering old timestamps"," — ",[384,1670,1671],{},"idle: false",", a fresh ",[384,1674,1017],{},", an old ",[384,1677,969],{}," — which is the producer's clock or a replay, and no watermark setting fixes it: find the producer, or accept the stall for the duration of the catch-up. ",[357,1680,1681,1683],{},[384,1682,923],{}," is set",": a watermark-alignment hold is stopping every source while a lagging member of its group has nothing buffered — ",[384,1686,1687],{},"stoatflow.watermark.alignment.held.time.total"," climbs, and a hold past a minute is logged with the sizing rule; size ",[384,1690,1691],{},"maxAllowedWatermarkDrift"," to at least that source's normal gap plus the difference in bounds, or give it a shorter ",[384,1694,1305],{},". A partition's next record is ",[357,1697,1698],{},"stamped in the future"," and waits behind older records: with a built-in strategy that is not a stall — within about a second event time is set by the other partitions, and ",[384,1701,846],{}," shows ",[384,1704,1281],{},[384,1706,1707],{},"headHeldSinceMs"," — but with a custom ",[384,1710,1711],{},"WatermarkGenerator"," there is no preview, so the watermark stands still until the record is dispatched or the partition idles out; either way a warning names the partition once the record is more than a minute ahead of the wall clock, and the fix is the producer's clock or the timestamp unit (",[362,1714,1716],{"href":1715},"\u002Fblog\u002Fnever-reorder-a-partition#a-record-stamped-in-the-future","the full story","). Or ",[384,1719,1559],{}," reads ",[384,1722,1465],{}," and there was never a watermark to stall. Part 2's ",[384,1725,637],{}," trap no longer reaches production for anything the watermark drives: the engine, and the driver, refuse that topology when it is built.",[504,1728,539],{"id":1729},"the-three-knobs",[354,1731,1732,1733,1736,1737,1740,1741,467],{},"The engine-wide event-time configuration is three keys under ",[384,1734,1735],{},"stoatflow.watermark",", plus one under ",[384,1738,1739],{},"processing","; everything per source lives in code, on ",[384,1742,1743],{},"Consumed",[660,1745,1749],{"className":1746,"code":1747,"language":1748,"meta":665,"style":665},"language-yaml shiki shiki-themes vitesse-light","stoatflow:\n  watermark:\n    max-out-of-orderness-ms: 10000   # default strategy lag — size to measured skew\n    idleness-timeout-ms: 300000      # 5 min; 0 disables idleness for sources without withIdleness\n    auto-interval-ms: 200            # periodic emit cadence\n  processing:\n    timestamp-coordination-mode: AUTO  # AUTO | ENABLED | DISABLED\n","yaml",[384,1750,1751,1759,1766,1779,1792,1805,1812],{"__ignoreMap":665},[669,1752,1753,1756],{"class":671,"line":672},[669,1754,1755],{"class":868},"stoatflow",[669,1757,1758],{"class":690},":\n",[669,1760,1761,1764],{"class":671,"line":679},[669,1762,1763],{"class":868},"  watermark",[669,1765,1758],{"class":690},[669,1767,1768,1771,1773,1776],{"class":671,"line":716},[669,1769,1770],{"class":868},"    max-out-of-orderness-ms",[669,1772,874],{"class":690},[669,1774,1775],{"class":766}," 10000",[669,1777,1778],{"class":675},"   # default strategy lag — size to measured skew\n",[669,1780,1781,1784,1786,1789],{"class":671,"line":733},[669,1782,1783],{"class":868},"    idleness-timeout-ms",[669,1785,874],{"class":690},[669,1787,1788],{"class":766}," 300000",[669,1790,1791],{"class":675},"      # 5 min; 0 disables idleness for sources without withIdleness\n",[669,1793,1794,1797,1799,1802],{"class":671,"line":740},[669,1795,1796],{"class":868},"    auto-interval-ms",[669,1798,874],{"class":690},[669,1800,1801],{"class":766}," 200",[669,1803,1804],{"class":675},"            # periodic emit cadence\n",[669,1806,1807,1810],{"class":671,"line":746},[669,1808,1809],{"class":868},"  processing",[669,1811,1758],{"class":690},[669,1813,1814,1817,1819,1822],{"class":671,"line":773},[669,1815,1816],{"class":868},"    timestamp-coordination-mode",[669,1818,874],{"class":690},[669,1820,1821],{"class":897}," AUTO",[669,1823,1824],{"class":675},"  # AUTO | ENABLED | DISABLED\n",[354,1826,1827,1830,1831,1833,1834,1837,1838,1841,1842,1844,1845,1848,1849,1852,1853,1856,1857,1860,1861,1863,1864,1866],{},[384,1828,1829],{},"max-out-of-orderness-ms"," is the lag of the default strategy, and the number to size to the disorder the data actually has — ",[384,1832,1548],{}," at debug level is the closer measure; ",[384,1835,1836],{},"out.of.orderness.max"," shows skew between partitions. ",[384,1839,1840],{},"idleness-timeout-ms"," is the fallback for sources that set no ",[384,1843,1305],{}," of their own; zero switches idleness off for them, which pins a never-written partition forever, and the engine logs one warning at start-up naming the sources left uncovered. ",[384,1846,1847],{},"auto-interval-ms"," is the periodic recompute cadence, Flink's ",[384,1850,1851],{},"pipeline.auto-watermark-interval"," under another name, and rarely worth touching. ",[384,1854,1855],{},"timestamp-coordination-mode"," is a switch over the detection described above: ",[384,1858,1859],{},"AUTO"," derives it from the topology, ",[384,1862,1576],{}," forces tracking on as a safety valve, ",[384,1865,641],{}," forces it off for benchmarking — and is refused, at start-up and by the driver, for a topology with anything the watermark drives.",[354,1868,1869,1870,1872,1873,1875,1876,1878,1879,1882],{},"Two things are deliberately ",[377,1871,1312],{}," here. Alignment has no YAML keys — it is a property of a source, configured on ",[384,1874,1743],{},". And a per-source ",[384,1877,566],{}," overrides both the assigner and the generator, whereas the Kafka Streams-shaped ",[384,1880,1881],{},"default.timestamp.extractor"," overrides only the event-time basis.",[504,1884,545],{"id":1885},"restart-failover-migration",[354,1887,1888,1889,1892,1893,1895,1896,1898],{},"The watermark survives a restart in two parts. The floor every lane judges late records against — the watermark stamped on the last committed barriers — lives in a changelogged store, ",[384,1890,1891],{},"\u003Capplication.id>-stoatflow-watermark-changelog",", committed in the barrier's own transaction, and seeds every lane at start and at an HA promotion. Each source partition's partition time rides its committed offset, in Kafka Streams' own metadata format, and seeds the watermark tracker as the partition registers. So after a crash restart, an in-place restart or a promotion, ",[384,1894,846],{}," answers 503 or ",[384,1897,824],{}," for a moment and then resumes from the committed cut — it does not wait for new records. A crash restart resumes the floor one barrier back, a graceful one exactly where it stopped; either way a window closed before the restart stays closed, and a late record for it is dropped as it would have been. Offset tools cannot erase the floor; wiping the state takes it with it, and the engine then resumes from the partition times.",[354,1900,1901,1902,1905],{},"The rest survives with it. Event-time ",[377,1903,1904],{},"timers"," are persisted with the state: a timer armed before a shutdown — even one followed by a wiped state directory — is rebuilt from its changelog on restart and fires once the resumed watermark passes it, which is exactly what the timer-recovery test drives. On an HA promotion the standby reloads its timer heap from the replicated store, so a timer the old active already fired and committed does not fire again on the new one — the failover test pins that to exactly one firing. And the emit frontier is restored, so a window whose final result was already emitted is not emitted again; a window that was pending is emitted once, later.",[354,1907,1908,1909,1912,1913,1916],{},"The same applies at a Kafka Streams cutover. The ",[362,1910,1911],{"href":280},"migration tool"," translates a windowed changelog into StoatFlow's layout; with ",[384,1914,1915],{},"emitFrontier: true"," it also seeds the frontier from the store's own maximum timestamp, so the first watermark tick after cutover does not re-emit every restored closed window in one burst.",[504,1918,551],{"id":1919},"tradeoffs-and-limits",[354,1921,1922,1925],{},[357,1923,1924],{},"Late arrivals are invisible at info level."," A default deployment sees drops, not the disorder that causes them. Enable the debug level for the watermark family while tuning the bound, then turn it off.",[354,1927,1928,1933,1934,1937,1938,1941],{},[357,1929,1930,1932],{},[384,1931,480],{}," does nothing."," It is stored and round-tripped so Kafka Streams code compiles and reads back what it wrote, but the effective grace of a stream-stream join comes from ",[384,1935,1936],{},"JoinWindows.ofTimeDifferenceAndGrace",", and a stream-table join has no grace at all. Setting it on ",[384,1939,1940],{},"Joined"," produces no error and no effect.",[354,1943,1944,1947,1948,1950,1951,1954,1955,1958,1959,1962,1963,1965],{},[357,1945,1946],{},"DIRECT dispatch has no watermarks, and refuses event time."," The processing-time fast path never advances the watermark. ",[384,1949,1859],{}," resolves an event-time topology to ",[384,1952,1953],{},"LANE","; an explicit ",[384,1956,1957],{},"DIRECT"," with event time fails at start-up with a message that says so. A legitimately processing-time topology therefore shows ",[384,1960,1961],{},"event.time.tracking = 0"," and a 503 from ",[384,1964,846],{}," — by design, not by fault.",[354,1967,1968,1971],{},[357,1969,1970],{},"Idleness cannot be tested in the driver."," It is wall-clock based and the driver has no partitions to idle; test it on a broker, or accept a slow test.",[354,1973,1974,1977,1978,1980,1981,467],{},[357,1975,1976],{},"The driver cannot show the frontier lag at all."," Emission-timing assertions belong in an integration test; the driver is for ",[377,1979,585],{},", not ",[377,1982,589],{},[354,1984,1985,1988,1989,1992],{},[357,1986,1987],{},"Ingestion time is not supported."," Event time comes from the record timestamp or an assigner — on a ",[384,1990,1991],{},"LogAppendTime"," topic that is the broker's append time — or you work in processing time; there is no Flink-style ingestion time stamped as the source reads.",[504,1994,557],{"id":1995},"whats-coming-next",[354,1997,1998,1999,2002,2003,2006,2007,2011],{},"Part 2 named a problem the engine does not solve today: a bulk reload stamped in the past starves the live side, because ordered dispatch — correctly — serves the oldest partition head first, each partition in offset order, and the watermark stands still until the reload has drained and its partition idled out. Alignment holds back the source that is ",[377,2000,2001],{},"ahead","; this needs the inverse, a way to let the source that is ",[377,2004,2005],{},"behind"," defer. Dispatch tiers and deferred watermarks are designed and on the ",[362,2008,2010],{"href":2009},"\u002Fproduct\u002Froadmap","roadmap",". When they ship, the post that follows will cover them alongside alignment in full — including why we built alignment for Flink parity and reframed its motivation after our own design review.",[354,2013,2014,2015,2017,2018,2020,2021,2023,2024,2027],{},"That closes the series. ",[362,2016,365],{"href":364}," set StoatFlow's watermark beside Kafka Streams' and Flink's clocks; ",[362,2019,370],{"href":369}," was the configuration discipline a minimum demands; ",[362,2022,374],{"href":373}," followed one advance to a committed result; this part is what you test and what you watch. The ",[362,2025,2026],{"href":73},"event-time concept page"," is the durable summary of all four.",[354,2029,2030],{},[357,2031,2032],{},"Read on:",[413,2034,2035,2044,2053,2061,2068,2075,2082],{},[416,2036,2037,1668,2039,835,2041,2043],{},[362,2038,494],{"href":493},[384,2040,605],{},[384,2042,613],{}," and the driver's time surface.",[416,2045,2046,2048,2049,2052],{},[362,2047,200],{"href":201}," and the ",[362,2050,2051],{"href":499},"metrics reference"," — the curated set and the full catalogue.",[416,2054,2055,2057,2058,2060],{},[362,2056,306],{"href":502}," — every ",[384,2059,846],{}," field.",[416,2062,2063,2067],{},[362,2064,2066],{"href":2065},"\u002Fdocs\u002Fconfiguration\u002Fcore-config#event-time-and-watermarks","Core configuration"," — the watermark keys.",[416,2069,2070,2074],{},[362,2071,2073],{"href":2072},"\u002Fdocs\u002Fconfiguration\u002Fcore-config#which-changelog-topics-an-application-creates","Changelog topics"," — the watermark changelog that carries the late-drop floor across a restart or a promotion.",[416,2076,2077,2081],{},[362,2078,2080],{"href":2079},"\u002Fblog\u002Fnever-reorder-a-partition","Why a stream processor must never reorder a partition"," — offset order, and the record stamped in the future.",[416,2083,2084,2087,2088,2091],{},[362,2085,2086],{"href":280},"Migrating with data"," — the ",[384,2089,2090],{},"emitFrontier"," cutover flag.",[2093,2094,2095],"style",{},"html pre.shiki code .s8zF2, html code.shiki .s8zF2{--shiki-default:#A0ADA0}html pre.shiki code .sbBg2, html code.shiki .sbBg2{--shiki-default:#1E754F}html pre.shiki code .suHK_, html code.shiki .suHK_{--shiki-default:#393A34}html pre.shiki code .sYZai, html code.shiki .sYZai{--shiki-default:#999999}html pre.shiki code .sySUi, html code.shiki .sySUi{--shiki-default:#59873A}html pre.shiki code .si04Y, html code.shiki .si04Y{--shiki-default:#AB5959}html pre.shiki code .s-TwI, html code.shiki .s-TwI{--shiki-default:#2F798A}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html pre.shiki code .s61at, html code.shiki .s61at{--shiki-default:#99841877}html pre.shiki code .su6XF, html code.shiki .su6XF{--shiki-default:#998418}html pre.shiki code .sSP4y, html code.shiki .sSP4y{--shiki-default:#B5695977}html pre.shiki code .spphp, html code.shiki .spphp{--shiki-default:#B56959}",{"title":665,"searchDepth":679,"depth":679,"links":2097},[2098,2099,2100,2101,2102,2103,2104,2105,2106],{"id":506,"depth":679,"text":507},{"id":560,"depth":679,"text":515},{"id":808,"depth":679,"text":521},{"id":1324,"depth":679,"text":527},{"id":1580,"depth":679,"text":533},{"id":1729,"depth":679,"text":539},{"id":1885,"depth":679,"text":545},{"id":1919,"depth":679,"text":551},{"id":1995,"depth":679,"text":557},"2026-10-06","The TopologyTestDriver's stream time is the maximum it has seen, with no out-of-orderness lag and no cross-partition minimum; production keeps a 10-second lag and the minimum. A window-close test that passes in the driver emits later on a real broker, and can count a record the driver dropped. Part 4 of the event-time series: which tests transfer, how to read \u002Fwatermarks, the meters that matter, the one alert to set on day one, and what a restart does to the watermark.","md",{"src":2111},"\u002Fassets\u002Fblog\u002Fog\u002Ftesting-and-operating-event-time.png",{},null,"\u002Fblog\u002Ftesting-and-operating-event-time",{"title":336,"description":2108},{"loc":2114},"3.blog\u002F18.testing-and-operating-event-time","Jvgx-gA_9ie0WmhqSF0lY3sEMX_7EfH79bjODZ8PvtM",[2120,2125],{"title":2121,"path":2122,"stem":2123,"description":2124,"children":-1},"The self-join that emitted nothing","\u002Fblog\u002Fthe-self-join-that-emitted-nothing","3.blog\u002F26.the-self-join-that-emitted-nothing","On Kafka Streams, s.join(s) over two records of one key emits four results. On StoatFlow it emitted none — no error, no log line, no metric. A stream–stream join chose which side a record belonged to by the name of the source it came from, and a self-join's records all come from one source. The same rule dropped records in chained joins, merged legs and cogroups. How we found every affected shape by running Kafka Streams' own test driver as the oracle, and how the fix makes the edge carry the input.",{"title":2126,"path":2127,"stem":2128,"description":2129,"children":-1},"A use-after-free in a RocksDB iterator, and the test that passed on Linux","\u002Fblog\u002Frocksdb-iterator-use-after-free","3.blog\u002F25.rocksdb-iterator-use-after-free","A StoatFlow state store layers each epoch's uncommitted writes over RocksDB in native write batches, and an iterator reads through all of them. Four defects let a batch be freed, or rewritten, under an iterator still reading it: segmentation faults, keys read twice, a value read back as garbage. None was reachable through a store StoatFlow builds. The interesting part is how we proved them — one JVM per scenario, the allocator scribbling freed memory — and why the first version of that test passed on Linux.",1792981295894]