[{"data":1,"prerenderedAt":1179},["ShallowReactive",2],{"navigation":3,"\u002Fblog\u002Fnever-reorder-a-partition":334,"\u002Fblog\u002Fnever-reorder-a-partition-surround":1168},[4,43,91,154,181,224,262,293],{"title":5,"path":6,"stem":7,"children":8,"icon":42},"Getting Started","\u002Fdocs\u002Fgetting-started","1.docs\u002F1.getting-started\u002F1.index",[9,12,17,22,27,32,37],{"title":10,"path":6,"stem":7,"icon":11},"Getting started","i-lucide-flag",{"title":13,"path":14,"stem":15,"icon":16},"Modules overview","\u002Fdocs\u002Fgetting-started\u002Fmodules-overview","1.docs\u002F1.getting-started\u002F2.modules-overview","i-lucide-boxes",{"title":18,"path":19,"stem":20,"icon":21},"Installation","\u002Fdocs\u002Fgetting-started\u002Finstallation","1.docs\u002F1.getting-started\u002F3.installation","i-lucide-download",{"title":23,"path":24,"stem":25,"icon":26},"License configuration","\u002Fdocs\u002Fgetting-started\u002Flicense-configuration","1.docs\u002F1.getting-started\u002F4.license-configuration","i-lucide-key-round",{"title":28,"path":29,"stem":30,"icon":31},"Your first app","\u002Fdocs\u002Fgetting-started\u002Ffirst-app","1.docs\u002F1.getting-started\u002F5.first-app","i-lucide-square-play",{"title":33,"path":34,"stem":35,"icon":36},"Project setup","\u002Fdocs\u002Fgetting-started\u002Fproject-setup","1.docs\u002F1.getting-started\u002F6.project-setup","i-lucide-package",{"title":38,"path":39,"stem":40,"icon":41},"AI assistants","\u002Fdocs\u002Fgetting-started\u002Fai-assistants","1.docs\u002F1.getting-started\u002F7.ai-assistants","i-lucide-bot",false,{"title":44,"path":45,"stem":46,"children":47,"icon":50},"Core Concepts","\u002Fdocs\u002Fconcepts","1.docs\u002F2.concepts\u002F1.index",[48,51,56,61,66,71,76,81,86],{"title":49,"path":45,"stem":46,"icon":50},"Core concepts","i-lucide-book-open",{"title":52,"path":53,"stem":54,"icon":55},"Architecture","\u002Fdocs\u002Fconcepts\u002Farchitecture","1.docs\u002F2.concepts\u002F2.architecture","i-lucide-layers",{"title":57,"path":58,"stem":59,"icon":60},"Exactly-once semantics","\u002Fdocs\u002Fconcepts\u002Fexactly-once","1.docs\u002F2.concepts\u002F3.exactly-once","i-lucide-shield-check",{"title":62,"path":63,"stem":64,"icon":65},"Lanes and parallelism","\u002Fdocs\u002Fconcepts\u002Flanes-and-parallelism","1.docs\u002F2.concepts\u002F4.lanes-and-parallelism","i-lucide-split",{"title":67,"path":68,"stem":69,"icon":70},"State and thread-safety","\u002Fdocs\u002Fconcepts\u002Fstate-and-thread-safety","1.docs\u002F2.concepts\u002F5.state-and-thread-safety","i-lucide-lock",{"title":72,"path":73,"stem":74,"icon":75},"Event time and watermarks","\u002Fdocs\u002Fconcepts\u002Fevent-time-and-watermarks","1.docs\u002F2.concepts\u002F6.event-time-and-watermarks","i-lucide-clock",{"title":77,"path":78,"stem":79,"icon":80},"The configuration model","\u002Fdocs\u002Fconcepts\u002Fconfiguration-model","1.docs\u002F2.concepts\u002F7.configuration-model","i-lucide-sliders-horizontal",{"title":82,"path":83,"stem":84,"icon":85},"The error-handling model","\u002Fdocs\u002Fconcepts\u002Ferror-handling-model","1.docs\u002F2.concepts\u002F8.error-handling-model","i-lucide-triangle-alert",{"title":87,"path":88,"stem":89,"icon":90},"How StoatFlow differs from Kafka Streams","\u002Fdocs\u002Fconcepts\u002Fhow-stoatflow-differs-from-ks","1.docs\u002F2.concepts\u002F9.how-stoatflow-differs-from-ks","i-lucide-arrow-left-right",{"title":92,"path":93,"stem":94,"children":95,"icon":98},"Building Topologies","\u002Fdocs\u002Fbuilding","1.docs\u002F3.building\u002F1.index",[96,99,104,109,114,119,124,129,134,139,144,149],{"title":97,"path":93,"stem":94,"icon":98},"Building topologies","i-lucide-blocks",{"title":100,"path":101,"stem":102,"icon":103},"Error handling and DLQ","\u002Fdocs\u002Fbuilding\u002Ferror-handling-dlq","1.docs\u002F3.building\u002F10.error-handling-dlq","i-lucide-circle-x",{"title":105,"path":106,"stem":107,"icon":108},"State stores","\u002Fdocs\u002Fbuilding\u002Fstate-stores","1.docs\u002F3.building\u002F11.state-stores","i-lucide-database",{"title":110,"path":111,"stem":112,"icon":113},"Testing topologies","\u002Fdocs\u002Fbuilding\u002Ftesting","1.docs\u002F3.building\u002F12.testing","i-lucide-flask-conical",{"title":115,"path":116,"stem":117,"icon":118},"Sources and sinks","\u002Fdocs\u002Fbuilding\u002Fstreams-builder","1.docs\u002F3.building\u002F2.streams-builder","i-lucide-import",{"title":120,"path":121,"stem":122,"icon":123},"KStream and KTable operations","\u002Fdocs\u002Fbuilding\u002Fkstream-ktable","1.docs\u002F3.building\u002F3.kstream-ktable","i-lucide-waypoints",{"title":125,"path":126,"stem":127,"icon":128},"Aggregations","\u002Fdocs\u002Fbuilding\u002Faggregations","1.docs\u002F3.building\u002F4.aggregations","i-lucide-sigma",{"title":130,"path":131,"stem":132,"icon":133},"Windowing","\u002Fdocs\u002Fbuilding\u002Fwindowing","1.docs\u002F3.building\u002F5.windowing","i-lucide-calendar-clock",{"title":135,"path":136,"stem":137,"icon":138},"Joins","\u002Fdocs\u002Fbuilding\u002Fjoins","1.docs\u002F3.building\u002F6.joins","i-lucide-git-merge",{"title":140,"path":141,"stem":142,"icon":143},"The Processor API","\u002Fdocs\u002Fbuilding\u002Fprocessor-api","1.docs\u002F3.building\u002F7.processor-api","i-lucide-cpu",{"title":145,"path":146,"stem":147,"icon":148},"Scheduled sources","\u002Fdocs\u002Fbuilding\u002Fscheduled-sources","1.docs\u002F3.building\u002F8.scheduled-sources","i-lucide-timer",{"title":150,"path":151,"stem":152,"icon":153},"Serdes and Avro","\u002Fdocs\u002Fbuilding\u002Fserdes","1.docs\u002F3.building\u002F9.serdes","i-lucide-binary",{"title":155,"path":156,"stem":157,"children":158,"icon":80},"Configuration","\u002Fdocs\u002Fconfiguration","1.docs\u002F4.configuration\u002F1.index",[159,161,166,171,176],{"title":160,"path":156,"stem":157,"icon":80},"How configuration works",{"title":162,"path":163,"stem":164,"icon":165},"Engine configuration (:core)","\u002Fdocs\u002Fconfiguration\u002Fcore-config","1.docs\u002F4.configuration\u002F2.core-config","i-lucide-settings-2",{"title":167,"path":168,"stem":169,"icon":170},"Runtime configuration (:runtime)","\u002Fdocs\u002Fconfiguration\u002Fruntime-config","1.docs\u002F4.configuration\u002F3.runtime-config","i-lucide-server-cog",{"title":172,"path":173,"stem":174,"icon":175},"Defaults, adaptivity, and presets","\u002Fdocs\u002Fconfiguration\u002Fdefaults-and-presets","1.docs\u002F4.configuration\u002F4.defaults-and-presets","i-lucide-gauge",{"title":177,"path":178,"stem":179,"icon":180},"Kafka client configuration","\u002Fdocs\u002Fconfiguration\u002Fkafka-client-config","1.docs\u002F4.configuration\u002F5.kafka-client-config","i-lucide-plug",{"title":182,"path":183,"stem":184,"children":185,"icon":188},"Running in Production","\u002Fdocs\u002Fruntime","1.docs\u002F5.runtime\u002F1.index",[186,189,194,199,204,209,214,219],{"title":187,"path":183,"stem":184,"icon":188},"The runtime","i-lucide-server",{"title":190,"path":191,"stem":192,"icon":193},"The REST API","\u002Fdocs\u002Fruntime\u002Frest-api","1.docs\u002F5.runtime\u002F2.rest-api","i-lucide-globe",{"title":195,"path":196,"stem":197,"icon":198},"Health checks","\u002Fdocs\u002Fruntime\u002Fhealth-checks","1.docs\u002F5.runtime\u002F3.health-checks","i-lucide-heart-pulse",{"title":200,"path":201,"stem":202,"icon":203},"Metrics","\u002Fdocs\u002Fruntime\u002Fmetrics","1.docs\u002F5.runtime\u002F4.metrics","i-lucide-activity",{"title":205,"path":206,"stem":207,"icon":208},"Pause and resume","\u002Fdocs\u002Fruntime\u002Fpause-unpause","1.docs\u002F5.runtime\u002F5.pause-unpause","i-lucide-pause",{"title":210,"path":211,"stem":212,"icon":213},"Plugins and lifecycle hooks","\u002Fdocs\u002Fruntime\u002Fplugins","1.docs\u002F5.runtime\u002F6.plugins","i-lucide-puzzle",{"title":215,"path":216,"stem":217,"icon":218},"Docker images","\u002Fdocs\u002Fruntime\u002Fdocker","1.docs\u002F5.runtime\u002F7.docker","i-lucide-container",{"title":220,"path":221,"stem":222,"icon":223},"GraalVM native image","\u002Fdocs\u002Fruntime\u002Fnative-image","1.docs\u002F5.runtime\u002F8.native-image","i-lucide-zap",{"title":225,"path":226,"stem":227,"children":228,"icon":231},"Deploying & Operating","\u002Fdocs\u002Foperating","1.docs\u002F6.operating\u002F1.index",[229,232,237,242,247,252,257],{"title":230,"path":226,"stem":227,"icon":231},"Deploying and operating","i-lucide-life-buoy",{"title":233,"path":234,"stem":235,"icon":236},"Running on Kubernetes","\u002Fdocs\u002Foperating\u002Fkubernetes","1.docs\u002F6.operating\u002F2.kubernetes","i-lucide-ship",{"title":238,"path":239,"stem":240,"icon":241},"High availability","\u002Fdocs\u002Foperating\u002Fhigh-availability","1.docs\u002F6.operating\u002F3.high-availability","i-lucide-copy",{"title":243,"path":244,"stem":245,"icon":246},"Liveness and readiness probes","\u002Fdocs\u002Foperating\u002Fprobes","1.docs\u002F6.operating\u002F4.probes","i-lucide-stethoscope",{"title":248,"path":249,"stem":250,"icon":251},"Observability","\u002Fdocs\u002Foperating\u002Fobservability","1.docs\u002F6.operating\u002F5.observability","i-lucide-telescope",{"title":253,"path":254,"stem":255,"icon":256},"Tuning under load","\u002Fdocs\u002Foperating\u002Ftuning","1.docs\u002F6.operating\u002F6.tuning","i-lucide-sliders",{"title":258,"path":259,"stem":260,"icon":261},"Production checklist","\u002Fdocs\u002Foperating\u002Fproduction-checklist","1.docs\u002F6.operating\u002F7.production-checklist","i-lucide-clipboard-check",{"title":263,"path":264,"stem":265,"children":266,"icon":90},"Migrating from Kafka Streams","\u002Fdocs\u002Fmigration","1.docs\u002F7.migration\u002F1.index",[267,268,273,278,283,288],{"title":263,"path":264,"stem":265,"icon":90},{"title":269,"path":270,"stem":271,"icon":272},"Automated port","\u002Fdocs\u002Fmigration\u002Fautomated-port","1.docs\u002F7.migration\u002F2.automated-port","i-lucide-wand-sparkles",{"title":274,"path":275,"stem":276,"icon":277},"Migration without carrying state","\u002Fdocs\u002Fmigration\u002Fwithout-data-migration","1.docs\u002F7.migration\u002F3.without-data-migration","i-lucide-sparkles",{"title":279,"path":280,"stem":281,"icon":282},"Migration carrying state","\u002Fdocs\u002Fmigration\u002Fwith-data-migration","1.docs\u002F7.migration\u002F4.with-data-migration","i-lucide-database-backup",{"title":284,"path":285,"stem":286,"icon":287},"The migration tool","\u002Fdocs\u002Fmigration\u002Fmigration-tool","1.docs\u002F7.migration\u002F5.migration-tool","i-lucide-truck",{"title":289,"path":290,"stem":291,"icon":292},"Reusing your Kafka Streams dashboards","\u002Fdocs\u002Fmigration\u002Freusing-kafka-streams-dashboards","1.docs\u002F7.migration\u002F6.reusing-kafka-streams-dashboards","i-lucide-line-chart",{"title":294,"path":295,"stem":296,"children":297,"icon":299},"Reference","\u002Fdocs\u002Freference","1.docs\u002F8.reference\u002F1.index",[298,300,305,310,315,320,325,329],{"title":294,"path":295,"stem":296,"icon":299},"i-lucide-list",{"title":301,"path":302,"stem":303,"icon":304},"Configuration reference","\u002Fdocs\u002Freference\u002Fconfiguration-reference","1.docs\u002F8.reference\u002F2.configuration-reference","i-lucide-table",{"title":306,"path":307,"stem":308,"icon":309},"REST API reference","\u002Fdocs\u002Freference\u002Frest-api-reference","1.docs\u002F8.reference\u002F3.rest-api-reference","i-lucide-network",{"title":311,"path":312,"stem":313,"icon":314},"Gradle plugin reference","\u002Fdocs\u002Freference\u002Fgradle-plugin-reference","1.docs\u002F8.reference\u002F4.gradle-plugin-reference","i-lucide-box",{"title":316,"path":317,"stem":318,"icon":319},"Maven reference","\u002Fdocs\u002Freference\u002Fmaven-reference","1.docs\u002F8.reference\u002F5.maven-reference","i-simple-icons-apachemaven",{"title":321,"path":322,"stem":323,"icon":324},"Kafka Streams compatibility matrix","\u002Fdocs\u002Freference\u002Fks-compatibility-matrix","1.docs\u002F8.reference\u002F6.ks-compatibility-matrix","i-lucide-table-2",{"title":326,"path":327,"stem":328,"icon":175},"Metrics reference","\u002Fdocs\u002Freference\u002Fmetrics-reference","1.docs\u002F8.reference\u002F7.metrics-reference",{"title":330,"path":331,"stem":332,"icon":333},"Glossary","\u002Fdocs\u002Freference\u002Fglossary","1.docs\u002F8.reference\u002F8.glossary","i-lucide-book-a",{"id":335,"title":336,"authors":337,"badge":343,"body":345,"date":1156,"description":1157,"draft":42,"extension":1158,"image":1159,"meta":1161,"navigation":937,"ogImage":1162,"path":1163,"seo":1164,"sitemap":1165,"stem":1166,"updated":1162,"__hash__":1167},"posts\u002F3.blog\u002F22.never-reorder-a-partition.md","Why a stream processor must never reorder a partition",[338],{"name":339,"to":340,"avatar":341},"Hartmut Armbruster","https:\u002F\u002Fwww.linkedin.com\u002Fin\u002Fhartmut-co-uk\u002F",{"src":342},"\u002Fassets\u002Fhartmut_armbruster_monochromatic.jpg",{"label":344},"Deep Dive",{"type":346,"value":347,"toc":1144},"minimark",[348,352,355,411,416,472,475,483,495,498,501,504,510,520,534,537,540,547,550,652,659,672,683,686,693,700,703,706,721,724,727,734,778,781,784,787,799,810,813,832,835,841,991,994,1007,1018,1040,1043,1049,1063,1069,1075,1081,1087,1093,1098,1140],[349,350,351],"p",{},"A Kafka consumer commits an offset to say \"everything before this has been processed\". That sentence is only true if the records before the offset were processed before it. From February until this autumn, StoatFlow broke that assumption: it sorted each partition's buffered records by timestamp, so the oldest event went first, while the offset it committed was the highest one it had dispatched. When a newer-stamped record sat in the buffer behind older-stamped ones, a commit could move past it. A crash at the wrong moment lost it for good — under exactly-once and at-least-once alike, with no error, no log line and no metric.",[349,353,354],{},"Every partition is now dispatched in offset order, the way Kafka Streams and Flink do it. This post follows the defect from the buffer to the committed offset, explains why the obvious repair would have broken exactly-once, and shows how the one new cost of offset order — a record stamped far in the future holding the whole application's event time still — was removed as well.",[356,357,358,364],"blockquote",{},[349,359,360],{},[361,362,363],"strong",{},"TL;DR",[365,366,367,374,380,386,392,398],"ul",{},[368,369,370,373],"li",{},[361,371,372],{},"What:"," each partition's buffer was a priority queue sorted by record timestamp. Equal timestamps, timestamps that go backwards, and an overflow path let a later offset be dispatched before an earlier one.",[368,375,376,379],{},[361,377,378],{},"Why it lost data:"," the committed offset was the highest offset dispatched, plus one. A commit between the two dispatches covered the record still waiting; a crash before it was processed meant the restart never read it again.",[368,381,382,385],{},[361,383,384],{},"The fix:"," Kafka Streams' model — each partition is a FIFO queue in offset order, and timestamp order is chosen only between partition heads. A tripwire now counts any offset that goes backwards.",[368,387,388,391],{},[361,389,390],{},"The rejected fix:"," committing only up to the first gap keeps the reordering and duplicates output under exactly-once.",[368,393,394,397],{},[361,395,396],{},"The catch:"," a record out of order within its partition is now late data, judged by the watermark and grace. A record stamped far in the future still waits at its partition's head — but that partition now contributes the watermark it will have once the head goes out, so event time keeps moving everywhere else.",[368,399,400,403,404,407,408,410],{},[361,401,402],{},"Docs:"," ",[405,406,72],"a",{"href":73}," · ",[405,409,326],{"href":327},".",[412,413,415],"h2",{"id":414},"in-this-post","In this post",[365,417,418,424,430,436,442,448,454,460,466],{},[368,419,420],{},[405,421,423],{"href":422},"#the-optimisation","The optimisation",[368,425,426],{},[405,427,429],{"href":428},"#three-ways-a-partition-got-reordered","Three ways a partition got reordered",[368,431,432],{},[405,433,435],{"href":434},"#from-reordering-to-a-lost-record","From reordering to a lost record",[368,437,438],{},[405,439,441],{"href":440},"#why-nothing-noticed","Why nothing noticed",[368,443,444],{},[405,445,447],{"href":446},"#the-fix-that-would-have-broken-exactly-once","The fix that would have broken exactly-once",[368,449,450],{},[405,451,453],{"href":452},"#offset-order-end-to-end","Offset order, end to end",[368,455,456],{},[405,457,459],{"href":458},"#a-record-stamped-in-the-future","A record stamped in the future",[368,461,462],{},[405,463,465],{"href":464},"#what-changes-for-an-application","What changes for an application",[368,467,468],{},[405,469,471],{"href":470},"#tradeoffs-and-limits","Tradeoffs and limits",[412,473,423],{"id":474},"the-optimisation",[349,476,477,478,482],{},"StoatFlow is one process. A consumer thread polls every assigned partition into a bounded buffer per partition — 2,000 records by default — and a dispatcher hands records from those buffers to the processing lanes. When several partitions have records waiting, the dispatcher takes the partition whose next record is oldest. That is the same rule Kafka Streams applies in its ",[479,480,481],"code",{},"PartitionGroup",", there among the partitions of one task, and it is what keeps a fast partition from racing hours ahead of a slow one in event time.",[349,484,485,486,489,490,494],{},"The defect was one step further. The per-partition buffer was not a queue but a ",[479,487,488],{},"PriorityBlockingQueue"," ordered by record timestamp, so the oldest record ",[491,492,493],"em",{},"within"," a partition also went first. The reasoning was attractive: if a producer wrote slightly out of order, the engine would put it right before any window saw it. A unit test pinned the behaviour as intended — \"records returned by timestamp, not insertion order\".",[349,496,497],{},"The design that introduced the cross-partition rule had only ever assumed each partition was already in timestamp order. Nothing reconciled the in-partition sort with how offsets were committed.",[412,499,429],{"id":500},"three-ways-a-partition-got-reordered",[349,502,503],{},"Three triggers, all ordinary:",[349,505,506,509],{},[361,507,508],{},"Equal timestamps."," A binary heap is not a stable sort: records that compare equal come out in heap order, not insertion order. A burst produced within one millisecond — routine at a few thousand records a second — is enough. In our unit reproduction, six equal-timestamp records dispatched as offset 0, then 5, with offset 1 left behind.",[349,511,512,515,516,519],{},[361,513,514],{},"Timestamps that go backwards."," With producer-assigned ",[479,517,518],{},"CreateTime",", a record can carry an older timestamp than the records written after it: several producers on one partition, a retried send, a producer with a skewed clock, an upstream application that stamps the event time from its payload. The heap sorted the newer-stamped record behind the older-stamped ones that followed it.",[349,521,522,525,526,529,530,533],{},[361,523,524],{},"The overflow path, with ascending timestamps too."," When a polled record did not fit its buffer, it was set aside and retried on the next poll — but the loop carried on offering the rest of that poll. If the dispatcher freed a slot in between, record ",[491,527,528],{},"k+1"," got into the buffer while record ",[491,531,532],{},"k"," was still set aside. This one did not need the heap at all; it hit the plain FIFO mode as well. That mode had a second problem: its queue held 1,024 slots while its \"full\" check compared against 2,000, so the partition was never paused and the set-aside list could grow without bound under backpressure.",[349,535,536],{},"For the first two triggers, the reordering was bounded by what was buffered at once — up to 2,000 records per partition. All three depended on arrival timing, so the same input could be processed in different orders on different runs.",[412,538,435],{"id":539},"from-reordering-to-a-lost-record",[349,541,542,543,546],{},"Reordering alone is a correctness problem; the commit turned it into loss. For every dispatched record, the engine updated the partition's tracked offset, and the update kept the ",[361,544,545],{},"highest"," offset seen — a lower one was silently ignored. When a commit barrier fixed the offsets to commit, it took that highest offset plus one.",[349,548,549],{},"That rule is correct exactly when a partition is read in order: the highest offset dispatched implies everything below it was dispatched. With a reordered partition, it implies nothing. Take three records on one partition, where offset 0 carries the newest timestamp:",[551,552,557],"pre",{"className":553,"code":554,"language":555,"meta":556,"style":556},"language-mermaid shiki shiki-themes vitesse-light","sequenceDiagram\n    participant K as Partition 0\n    participant B as Partition buffer\n    participant D as Dispatcher\n    participant C as Commit\n    K->>B: offset 0 at 10:00:05\u003Cbr\u002F>offset 1 at 10:00:01\u003Cbr\u002F>offset 2 at 10:00:02\n    Note over B: sorted by timestamp — 1, 2, 0\n    B->>D: offset 1\n    B->>D: offset 2\n    Note over D: highest dispatched offset is 2\n    D->>C: barrier fixes offset 3\n    C->>K: transaction commits offset 3\n    Note over B: offset 0 still buffered\n    Note over K,C: crash before the next commit\n    K-->>D: restart resumes at offset 3 — offset 0 is never read again\n","mermaid","",[479,558,559,568,574,580,586,592,598,604,610,616,622,628,634,640,646],{"__ignoreMap":556},[560,561,564],"span",{"class":562,"line":563},"line",1,[560,565,567],{"class":566},"suHK_","sequenceDiagram\n",[560,569,571],{"class":562,"line":570},2,[560,572,573],{"class":566},"    participant K as Partition 0\n",[560,575,577],{"class":562,"line":576},3,[560,578,579],{"class":566},"    participant B as Partition buffer\n",[560,581,583],{"class":562,"line":582},4,[560,584,585],{"class":566},"    participant D as Dispatcher\n",[560,587,589],{"class":562,"line":588},5,[560,590,591],{"class":566},"    participant C as Commit\n",[560,593,595],{"class":562,"line":594},6,[560,596,597],{"class":566},"    K->>B: offset 0 at 10:00:05\u003Cbr\u002F>offset 1 at 10:00:01\u003Cbr\u002F>offset 2 at 10:00:02\n",[560,599,601],{"class":562,"line":600},7,[560,602,603],{"class":566},"    Note over B: sorted by timestamp — 1, 2, 0\n",[560,605,607],{"class":562,"line":606},8,[560,608,609],{"class":566},"    B->>D: offset 1\n",[560,611,613],{"class":562,"line":612},9,[560,614,615],{"class":566},"    B->>D: offset 2\n",[560,617,619],{"class":562,"line":618},10,[560,620,621],{"class":566},"    Note over D: highest dispatched offset is 2\n",[560,623,625],{"class":562,"line":624},11,[560,626,627],{"class":566},"    D->>C: barrier fixes offset 3\n",[560,629,631],{"class":562,"line":630},12,[560,632,633],{"class":566},"    C->>K: transaction commits offset 3\n",[560,635,637],{"class":562,"line":636},13,[560,638,639],{"class":566},"    Note over B: offset 0 still buffered\n",[560,641,643],{"class":562,"line":642},14,[560,644,645],{"class":566},"    Note over K,C: crash before the next commit\n",[560,647,649],{"class":562,"line":648},15,[560,650,651],{"class":566},"    K-->>D: restart resumes at offset 3 — offset 0 is never read again\n",[349,653,654,655,658],{},"Offsets 1 and 2 are processed and committed with offset 3. Offset 0 is still in the buffer behind them. If the process dies before the next commit, the consumer group's position is already past offset 0, and the restart starts reading at 3. Under exactly-once the transaction that committed offset 3 is valid; under at-least-once the same commit goes out through ",[479,656,657],{},"commitSync",". Neither guarantee can help, because both are defined in terms of the committed offset, and the committed offset was wrong.",[349,660,661,662,665,666,668,669,671],{},"We reproduced the loss on a real broker, red first: one lane deliberately parked, a record named ",[479,663,664],{},"late"," at offset 1 carrying the newest timestamp, followed in the log by 60 filler records stamped up to an hour earlier — so the buffer sorted ",[479,667,664],{}," behind all of them. A pause commits the processed prefix, the application crashes, the application restarts, and the output topic is counted per key to its end offsets. Result: ",[479,670,664],{}," absent from the output, on every run. The control — the same flow with timestamps in offset order — produced every key exactly once.",[349,673,674,675,678,679,682],{},"Without any crash, the reordering still changed results. Two updates of one key in one partition could be applied in reverse offset order — a table fed by an update and a later correction could end on the older value, which ",[479,676,677],{},"reduce",", ",[479,680,681],{},"aggregate"," and a stream–table join would then carry downstream. Kafka Streams never does this; it processes a partition in offset order.",[412,684,441],{"id":685},"why-nothing-noticed",[349,687,688,689,692],{},"Three things kept it hidden. Every test seeded timestamps that ascended with offsets, or produced one record per millisecond, and none overflowed a buffer while the dispatcher drained it. A reorder only loses data when a barrier falls between the two dispatches ",[361,690,691],{},"and"," the process crashes before the lower offset's next commit; in steady state the left-behind record goes out a batch later. And the tracked-offset update dropped the lower offset without a signal: nothing was logged, no metric moved, and a lost record leaves no trace in the output: it is never there.",[349,694,695,696,699],{},"It surfaced through a test for an unrelated dead-letter bug, which sent a burst of poison records sharing one millisecond. The dead-letter handler logged them in ",[491,697,698],{},"descending"," offset order: 100 down to 38. The harness check that failed was not looking for reordering at all.",[412,701,447],{"id":702},"the-fix-that-would-have-broken-exactly-once",[349,704,705],{},"The tempting repair keeps the sort and fixes the arithmetic: track which offsets were dispatched, and commit only up to the first gap. Offset 0 has not been dispatched, so commit offset 0, not 3.",[349,707,708,709,712,713,716,717,720],{},"It ",[361,710,711],{},"breaks exactly-once",". The transaction that commits \"up to the gap\" also commits the output records and state-store writes of everything already processed ",[491,714,715],{},"above"," the gap — offsets 1 and 2 in the example. After a crash, the restart resumes at the gap and processes offsets 1 and 2 again. Their output is emitted a second time, visible to every ",[479,718,719],{},"read_committed"," consumer, and an aggregate applies them twice. The commit position and the transaction's contents have to describe the same set of records, and with a gap they cannot. It would also have left the no-crash reordering untouched.",[349,722,723],{},"An offset tie-break in the heap was the other candidate. It fixes equal timestamps and nothing else.",[412,725,453],{"id":726},"offset-order-end-to-end",[349,728,729,730,733],{},"The fix adopts Kafka Streams' ",[479,731,732],{},"RecordQueue"," model and drops the in-partition sort:",[365,735,736,746,763,772],{},[368,737,738,741,742,745],{},[361,739,740],{},"Each partition is a FIFO queue in offset order",", in every mode. Timestamp order is chosen only ",[491,743,744],{},"between"," partitions — the partition with the oldest head goes first, as before.",[368,747,748,751,752,755,756,759,760,410],{},[361,749,750],{},"The buffer always appends."," Nothing is set aside, so nothing can overtake an earlier offset on the way in. The buffer's capacity is now a pause threshold: once a partition holds that many records, the consumer pauses it before the next poll, so a partition holds at most ",[479,753,754],{},"capacity − 1 + max.poll.records",". The old start-up rule that capacity be at least ",[479,757,758],{},"max.poll.records"," is gone, as in Kafka Streams. Every poll goes through one path that applies the pause first; a poll that still reaches a full buffer — a pause that did not take effect — is not appended but fetched again, and counted as ",[479,761,762],{},"partition_buffer_over_bound",[368,764,765,403,768,771],{},[361,766,767],{},"A tripwire counts any offset that goes backwards.",[479,769,770],{},"stoatflow.epoch.invariant.violations.total{kind=\"partition_offset_regression\"}"," increments, and a rate-limited WARN names the partition and both offsets. It counts instead of failing, so a silent loss becomes a visible signal without stopping the application.",[368,773,774,777],{},[361,775,776],{},"Tracking follows the consumer."," A broker-side seek-back — log truncation after an unclean leader election, or an offset reset — legitimately re-fetches lower offsets. The tracked offset now moves down with it once the first re-fetched record is dispatched, so later commits no longer skip the re-fetched records. A barrier fixed between the seek and that first dispatch still commits the old position — a narrow window Kafka Streams shares. Each seek-back counts once on the same tripwire, and a reordering bug would now cost duplicates rather than a loss.",[349,779,780],{},"When every partition's timestamps ascend, dispatch is still in exact global timestamp order. The FIFO queue is also the cheaper structure: constant-time instead of logarithmic.",[349,782,783],{},"The same work closed two neighbouring holes. With a custom timestamp assigner or extractor, the dispatcher compared partition heads by extracted event time while the buffer had sorted them by Kafka timestamp; a cached head could then be dispatched while a different record was removed — one record processed twice, another never. A FIFO buffer only appends at the tail, so the head cannot change under it. And records polled while the application was pausing could be lost, because the consumer's position had already moved past them; that is fixed too.",[412,785,459],{"id":786},"a-record-stamped-in-the-future",[349,788,789,790,793,794,798],{},"Offset order has one cost the old buffer did not have. A partition's head is now a commitment: nothing behind it goes first. A record stamped hours ahead — a producer clock that runs ahead, a timestamp field read in the wrong unit, a replayed topic that jumps to a later period — waits at the head of its partition until every older head elsewhere has been served. While it waits, its partition dispatches nothing, so that partition's watermark does not move, and the application's watermark is the minimum across partitions. Left alone, windows would stay open, ",[479,791,792],{},"suppress"," would hold and event-time timers would not fire, across every partition, until the record was dispatched or the partition was ",[405,795,797],{"href":796},"\u002Fblog\u002Fmaking-the-watermark-move#idleness-the-escape-hatch","declared idle",": up to five minutes with the default idleness timeout, and indefinitely with idleness switched off. The old buffer pushed such a record to the back of its window, so this stall did not exist before.",[349,800,801,802,805,806,809],{},"The commitment that causes the stall also removes it. Because nothing behind the head can go first, the watermark that partition will have the moment its head is dispatched is already known — for a bounded-out-of-orderness source, the head's timestamp minus the bound. So once a head is ",[361,803,804],{},"held"," — unchanged for a full refresh interval, while other partitions are being served and an older head waits elsewhere — the partition contributes that ",[361,807,808],{},"preview"," to the global minimum instead of its current watermark. The global watermark becomes the minimum over the other partitions, which is where idleness would have put it after five minutes, within about a second and the next commit barrier. Dispatch order does not change; only the watermark moves.",[349,811,812],{},"The gate keeps the preview out of steady state. A caught-up application never holds a head for a full refresh interval. A pause, a slow commit, full lane backpressure, a binding watermark-alignment hold or a partition waiting for data stops every partition from being served, so no head starts to count as held behind the others. A preview already published stays until its record is dispatched, through a pause too: the value remains the partition's commitment. And a partition with records in a batch that is waiting for lane capacity gets no new preview, because its head would then not be the next record to reach a lane.",[349,814,815,816,819,820,823,824,827,828,831],{},"You can see it happen. ",[479,817,818],{},"stoatflow.watermark.preview.partitions"," counts the partitions currently held behind older heads, and the ",[479,821,822],{},"\u002Fwatermarks"," endpoint shows each one's preview (",[479,825,826],{},"previewWatermarkMs",") and since when its head has been held (",[479,829,830],{},"headHeldSinceMs","). StoatFlow still logs a warning naming the partition when its next record is stamped more than a minute ahead of the wall clock. While the partition waits, the warning says what is true of event time: with a built-in strategy it is set by the other partitions; with a custom watermark generator, which gets no preview, it stands still.",[412,833,465],{"id":834},"what-changes-for-an-application",[349,836,837,840],{},[361,838,839],{},"Out-of-order data is late data, judged by the watermark."," A record whose timestamp is older than records before it in the same partition is no longer moved forward by up to 2,000 positions. It is processed in offset order, and the source's watermark strategy and the window's grace decide whether it still counts. If your producers write with real disorder, say so in the strategy:",[551,842,846],{"className":843,"code":844,"language":845,"meta":556,"style":556},"language-kotlin shiki shiki-themes vitesse-light","val strategy = WatermarkStrategy\n    .forBoundedOutOfOrderness\u003CString, Order>(Duration.ofSeconds(30))\n    .withTimestampAssigner { _, value, _ -> value?.eventTime ?: 0L }\n\nbuilder.stream(\n    \"orders\",\n    Consumed.with(Serdes.String(), orderSerde)\n        .withWatermarkStrategy(strategy),\n)\n","kotlin",[479,847,848,864,901,933,939,950,959,975,986],{"__ignoreMap":556},[560,849,850,854,857,861],{"class":562,"line":563},[560,851,853],{"class":852},"sbBg2","val",[560,855,856],{"class":566}," strategy ",[560,858,860],{"class":859},"sYZai","=",[560,862,863],{"class":566}," WatermarkStrategy\n",[560,865,866,869,873,876,880,882,885,888,891,894,898],{"class":562,"line":570},[560,867,868],{"class":566},"    .",[560,870,872],{"class":871},"sySUi","forBoundedOutOfOrderness",[560,874,875],{"class":566},"\u003C",[560,877,879],{"class":878},"sUxyF","String",[560,881,678],{"class":566},[560,883,884],{"class":878},"Order",[560,886,887],{"class":566},">(Duration.",[560,889,890],{"class":871},"ofSeconds",[560,892,893],{"class":566},"(",[560,895,897],{"class":896},"s-TwI","30",[560,899,900],{"class":566},"))\n",[560,902,903,905,908,911,915,918,921,924,927,930],{"class":562,"line":576},[560,904,868],{"class":566},[560,906,907],{"class":871},"withTimestampAssigner",[560,909,910],{"class":566}," { _, ",[560,912,914],{"class":913},"si04Y","value",[560,916,917],{"class":566},", _ ",[560,919,920],{"class":859},"->",[560,922,923],{"class":913}," value",[560,925,926],{"class":566},"?.eventTime ?: ",[560,928,929],{"class":896},"0L",[560,931,932],{"class":566}," }\n",[560,934,935],{"class":562,"line":582},[560,936,938],{"emptyLinePlaceholder":937},true,"\n",[560,940,941,944,947],{"class":562,"line":588},[560,942,943],{"class":566},"builder.",[560,945,946],{"class":871},"stream",[560,948,949],{"class":566},"(\n",[560,951,952,956],{"class":562,"line":594},[560,953,955],{"class":954},"spphp","    \"orders\"",[560,957,958],{"class":566},",\n",[560,960,961,964,967,970,972],{"class":562,"line":600},[560,962,963],{"class":566},"    Consumed.",[560,965,966],{"class":871},"with",[560,968,969],{"class":566},"(Serdes.",[560,971,879],{"class":871},[560,973,974],{"class":566},"(), orderSerde)\n",[560,976,977,980,983],{"class":562,"line":606},[560,978,979],{"class":566},"        .",[560,981,982],{"class":871},"withWatermarkStrategy",[560,984,985],{"class":566},"(strategy),\n",[560,987,988],{"class":562,"line":612},[560,989,990],{"class":566},")\n",[349,992,993],{},"A topology that tolerates no out-of-orderness may count more late drops than before. That is the honest number: the old buffer hid a bounded amount of disorder and paid for it with the commit.",[349,995,996,403,999,1002,1003,1006],{},[361,997,998],{},"One setting is gone.",[479,1000,1001],{},"stoatflow.processing.timestamp-ordered-dispatch"," and ",[479,1004,1005],{},"StreamsConfig.Builder.timestampOrderedDispatch(...)"," are removed — with every buffer FIFO, both values behaved identically. A YAML file that still sets the key starts and ignores it; code that calls the builder method no longer compiles, so delete the call.",[349,1008,1009,1015,1016,410],{},[361,1010,1011,1014],{},[479,1012,1013],{},"stoatflow.processing.buffered-records-per-partition"," accepts any positive value."," It no longer has to be at least ",[479,1017,758],{},[349,1019,1020,1023,1024,1028,1029,1032,1033,1036,1037,1039],{},[361,1021,1022],{},"Watch the tripwire."," The ",[405,1025,1027],{"href":1026},"\u002Fdocs\u002Freference\u002Fmetrics-reference#commit-barriers-and-transactions","metrics reference"," lists every kind on ",[479,1030,1031],{},"stoatflow.epoch.invariant.violations.total",". ",[479,1034,1035],{},"partition_offset_regression"," also fires once per broker seek-back, so read it next to your cluster's leader-election events. ",[479,1038,762],{}," means the pause that bounds a partition's buffer did not take effect, usually right after a WARN about a failed consumer pause; those records are fetched again, so memory stays bounded. Every other kind is unreachable by design, so any non-zero value there is a bug to report.",[412,1041,471],{"id":1042},"tradeoffs-and-limits",[349,1044,1045,1048],{},[361,1046,1047],{},"Records behind a future-stamped head can become late."," While a partition's preview stands, a record behind the future-stamped head that is older than it by more than the source's out-of-orderness bound, plus its window's close margin, is late; records in order behind it never are. It no longer depends on whether a commit barrier happened to fall between the two records.",[349,1050,1051,1054,1055,1058,1059,1062],{},[361,1052,1053],{},"A custom watermark generator gets no preview."," The preview is built into the built-in strategies — bounded out-of-orderness, and monotonous timestamps, which a source with a Kafka Streams ",[479,1056,1057],{},"TimestampExtractor"," also uses. A source with its own ",[479,1060,1061],{},"WatermarkGenerator"," keeps the old behaviour: a held partition holds the watermark until its head is dispatched or idleness excludes it. Keep an idleness timeout on such sources.",[349,1064,1065,1068],{},[361,1066,1067],{},"It takes about a second, not zero."," The gate needs the head unchanged across one full refresh interval of 500 ms; then the next watermark emission and a commit barrier carry the new value to the lanes. A window downstream of another sub-topology's emissions adds its usual lag on top.",[349,1070,1071,1074],{},[361,1072,1073],{},"An application that is always behind can see previews in steady state."," With skewed partitions and a permanent backlog, a partition whose next record is far enough ahead of the backlog qualifies as held. Its preview is a value it is committed to reach, so the watermark stays sound, but that partition's later records are judged against it sooner — the first point above. The gauge shows how often it happens.",[349,1076,1077,1080],{},[361,1078,1079],{},"Kafka Streams and Flink make different trades here."," Kafka Streams' clock is stream time, a maximum, so a held head never stalls anything — but once the future-stamped record is processed, stream time jumps forward and every normal record after it in that task is late for its windowed operators. Flink emits a split's records in arrival order with no timestamp merge, so the future-stamped record raises only its own split's watermark; the source's watermark is the minimum over its splits, so the other splits carry event time on, and the records after it on that split become late only once those splits have moved past them. With the preview, StoatFlow reaches Flink's outcome while keeping its single global watermark: a bad record can make far older records on its own partition late, and nothing else.",[349,1082,1083,1086],{},[361,1084,1085],{},"Per-partition ordering is the contract — cross-partition timing is not."," Two partitions' records still interleave by their heads' timestamps, so a key that appears in two partitions gets no ordering between them. That was true before and is true in Kafka Streams.",[349,1088,1089,1092],{},[361,1090,1091],{},"The tripwire detects; it does not prevent."," A future reordering bug would surface as a counted regression and a WARN, and since tracking follows the lower offset it would cost duplicates rather than a loss — but it would still be a bug. The test that reproduced the loss stays in the build as a regression guard, alongside a unit test that fails if any dispatch before a barrier is not the contiguous range below the committed offset.",[349,1094,1095],{},[361,1096,1097],{},"Read on:",[365,1099,1100,1105,1110,1117,1124,1131],{},[368,1101,1102,1104],{},[405,1103,72],{"href":73}," — offset order within a partition, out-of-order records, and records stamped in the future.",[368,1106,1107,1109],{},[405,1108,326],{"href":1026}," — the epoch-invariant tripwire and its kinds.",[368,1111,1112,1116],{},[405,1113,1115],{"href":1114},"\u002Fblog\u002Fthree-clocks-stream-time-vs-watermarks","Three clocks: stream time, Flink watermarks, and the one watermark StoatFlow computes"," — why the global watermark is a minimum, and what that costs.",[368,1118,1119,1123],{},[405,1120,1122],{"href":1121},"\u002Fblog\u002Fmaking-the-watermark-move","Making the watermark move: sources, idleness, and the partition that stopped the world"," — idleness, and what one source does to the minimum.",[368,1125,1126,1130],{},[405,1127,1129],{"href":1128},"\u002Fblog\u002Finternal-consistency","Internal consistency on Kafka"," — what one commit barrier across the whole application makes possible.",[368,1132,1133,1139],{},[405,1134,1138],{"href":1135,"rel":1136},"https:\u002F\u002Fcwiki.apache.org\u002Fconfluence\u002Fdisplay\u002FKAFKA\u002FKIP-695%3A+Further+improve+Kafka+Streams+timestamp+synchronization",[1137],"nofollow","KIP-695: Further improve Kafka Streams timestamp synchronization"," — how Kafka Streams chooses between partition heads.",[1141,1142,1143],"style",{},"html pre.shiki code .suHK_, html code.shiki .suHK_{--shiki-default:#393A34}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html pre.shiki code .sbBg2, html code.shiki .sbBg2{--shiki-default:#1E754F}html pre.shiki code .sYZai, html code.shiki .sYZai{--shiki-default:#999999}html pre.shiki code .sySUi, html code.shiki .sySUi{--shiki-default:#59873A}html pre.shiki code .sUxyF, html code.shiki .sUxyF{--shiki-default:#2E8F82}html pre.shiki code .s-TwI, html code.shiki .s-TwI{--shiki-default:#2F798A}html pre.shiki code .si04Y, html code.shiki .si04Y{--shiki-default:#AB5959}html pre.shiki code .spphp, html code.shiki .spphp{--shiki-default:#B56959}",{"title":556,"searchDepth":570,"depth":570,"links":1145},[1146,1147,1148,1149,1150,1151,1152,1153,1154,1155],{"id":414,"depth":570,"text":415},{"id":474,"depth":570,"text":423},{"id":500,"depth":570,"text":429},{"id":539,"depth":570,"text":435},{"id":685,"depth":570,"text":441},{"id":702,"depth":570,"text":447},{"id":726,"depth":570,"text":453},{"id":786,"depth":570,"text":459},{"id":834,"depth":570,"text":465},{"id":1042,"depth":570,"text":471},"2026-10-11","StoatFlow used to sort each partition's buffered records by timestamp, so the oldest event went first. Under the right timing that lost a record for good after a crash, under exactly-once and at-least-once alike, without a log line or a metric. This post walks the sequence from a buffered record to a committed offset that runs past it, why the obvious patch would have broken exactly-once, and how the one stall that offset order introduced was removed as well.","md",{"src":1160},"\u002Fassets\u002Fblog\u002Fog\u002Fnever-reorder-a-partition.png",{},null,"\u002Fblog\u002Fnever-reorder-a-partition",{"title":336,"description":1157},{"loc":1163},"3.blog\u002F22.never-reorder-a-partition","8CBeMS2G0ph-NxioUIEnngyTVHKXx_rZfpItDdEj3tg",[1169,1174],{"title":1170,"path":1171,"stem":1172,"description":1173,"children":-1},"Kafka Streams 4.4.0: transactional state stores, headers-aware partitioning and queries","\u002Fblog\u002Fkafka-streams-4-4","3.blog\u002F20.kafka-streams-4-4","What Apache Kafka 4.4.0 brings to Kafka Streams: opt-in transactional state stores (KIP-892), headers in the partitioner and IQv2, the GlobalKTable joiner fix, and static membership and plugins for the Streams Rebalance Protocol — plus where StoatFlow stands.",{"title":1175,"path":1176,"stem":1177,"description":1178,"children":-1},"Your timer fired on another thread","\u002Fblog\u002Ftimer-on-the-wrong-lane","3.blog\u002F23.timer-on-the-wrong-lane","StoatFlow runs a key's timers on the same lane as its records, so onTimer never races process() for that key. The lane is chosen by hashing key bytes — and a timer key is serialised again, by its own serde. Behind a Processor API source declared with only a deserializer, that serde was the default, and when the default wrote different bytes than the topic, a key's timer fired on another lane, concurrently with its records, without a word. How it happens, why Kafka Streams never meets it, and how StoatFlow now refuses to run a proven split.",1792981295564]