[{"data":1,"prerenderedAt":1440},["ShallowReactive",2],{"navigation":3,"\u002Fblog\u002Ftimer-on-the-wrong-lane":334,"\u002Fblog\u002Ftimer-on-the-wrong-lane-surround":1429},[4,43,91,154,181,224,262,293],{"title":5,"path":6,"stem":7,"children":8,"icon":42},"Getting Started","\u002Fdocs\u002Fgetting-started","1.docs\u002F1.getting-started\u002F1.index",[9,12,17,22,27,32,37],{"title":10,"path":6,"stem":7,"icon":11},"Getting started","i-lucide-flag",{"title":13,"path":14,"stem":15,"icon":16},"Modules overview","\u002Fdocs\u002Fgetting-started\u002Fmodules-overview","1.docs\u002F1.getting-started\u002F2.modules-overview","i-lucide-boxes",{"title":18,"path":19,"stem":20,"icon":21},"Installation","\u002Fdocs\u002Fgetting-started\u002Finstallation","1.docs\u002F1.getting-started\u002F3.installation","i-lucide-download",{"title":23,"path":24,"stem":25,"icon":26},"License configuration","\u002Fdocs\u002Fgetting-started\u002Flicense-configuration","1.docs\u002F1.getting-started\u002F4.license-configuration","i-lucide-key-round",{"title":28,"path":29,"stem":30,"icon":31},"Your first app","\u002Fdocs\u002Fgetting-started\u002Ffirst-app","1.docs\u002F1.getting-started\u002F5.first-app","i-lucide-square-play",{"title":33,"path":34,"stem":35,"icon":36},"Project setup","\u002Fdocs\u002Fgetting-started\u002Fproject-setup","1.docs\u002F1.getting-started\u002F6.project-setup","i-lucide-package",{"title":38,"path":39,"stem":40,"icon":41},"AI assistants","\u002Fdocs\u002Fgetting-started\u002Fai-assistants","1.docs\u002F1.getting-started\u002F7.ai-assistants","i-lucide-bot",false,{"title":44,"path":45,"stem":46,"children":47,"icon":50},"Core Concepts","\u002Fdocs\u002Fconcepts","1.docs\u002F2.concepts\u002F1.index",[48,51,56,61,66,71,76,81,86],{"title":49,"path":45,"stem":46,"icon":50},"Core concepts","i-lucide-book-open",{"title":52,"path":53,"stem":54,"icon":55},"Architecture","\u002Fdocs\u002Fconcepts\u002Farchitecture","1.docs\u002F2.concepts\u002F2.architecture","i-lucide-layers",{"title":57,"path":58,"stem":59,"icon":60},"Exactly-once semantics","\u002Fdocs\u002Fconcepts\u002Fexactly-once","1.docs\u002F2.concepts\u002F3.exactly-once","i-lucide-shield-check",{"title":62,"path":63,"stem":64,"icon":65},"Lanes and parallelism","\u002Fdocs\u002Fconcepts\u002Flanes-and-parallelism","1.docs\u002F2.concepts\u002F4.lanes-and-parallelism","i-lucide-split",{"title":67,"path":68,"stem":69,"icon":70},"State and thread-safety","\u002Fdocs\u002Fconcepts\u002Fstate-and-thread-safety","1.docs\u002F2.concepts\u002F5.state-and-thread-safety","i-lucide-lock",{"title":72,"path":73,"stem":74,"icon":75},"Event time and watermarks","\u002Fdocs\u002Fconcepts\u002Fevent-time-and-watermarks","1.docs\u002F2.concepts\u002F6.event-time-and-watermarks","i-lucide-clock",{"title":77,"path":78,"stem":79,"icon":80},"The configuration model","\u002Fdocs\u002Fconcepts\u002Fconfiguration-model","1.docs\u002F2.concepts\u002F7.configuration-model","i-lucide-sliders-horizontal",{"title":82,"path":83,"stem":84,"icon":85},"The error-handling model","\u002Fdocs\u002Fconcepts\u002Ferror-handling-model","1.docs\u002F2.concepts\u002F8.error-handling-model","i-lucide-triangle-alert",{"title":87,"path":88,"stem":89,"icon":90},"How StoatFlow differs from Kafka Streams","\u002Fdocs\u002Fconcepts\u002Fhow-stoatflow-differs-from-ks","1.docs\u002F2.concepts\u002F9.how-stoatflow-differs-from-ks","i-lucide-arrow-left-right",{"title":92,"path":93,"stem":94,"children":95,"icon":98},"Building Topologies","\u002Fdocs\u002Fbuilding","1.docs\u002F3.building\u002F1.index",[96,99,104,109,114,119,124,129,134,139,144,149],{"title":97,"path":93,"stem":94,"icon":98},"Building topologies","i-lucide-blocks",{"title":100,"path":101,"stem":102,"icon":103},"Error handling and DLQ","\u002Fdocs\u002Fbuilding\u002Ferror-handling-dlq","1.docs\u002F3.building\u002F10.error-handling-dlq","i-lucide-circle-x",{"title":105,"path":106,"stem":107,"icon":108},"State stores","\u002Fdocs\u002Fbuilding\u002Fstate-stores","1.docs\u002F3.building\u002F11.state-stores","i-lucide-database",{"title":110,"path":111,"stem":112,"icon":113},"Testing topologies","\u002Fdocs\u002Fbuilding\u002Ftesting","1.docs\u002F3.building\u002F12.testing","i-lucide-flask-conical",{"title":115,"path":116,"stem":117,"icon":118},"Sources and sinks","\u002Fdocs\u002Fbuilding\u002Fstreams-builder","1.docs\u002F3.building\u002F2.streams-builder","i-lucide-import",{"title":120,"path":121,"stem":122,"icon":123},"KStream and KTable operations","\u002Fdocs\u002Fbuilding\u002Fkstream-ktable","1.docs\u002F3.building\u002F3.kstream-ktable","i-lucide-waypoints",{"title":125,"path":126,"stem":127,"icon":128},"Aggregations","\u002Fdocs\u002Fbuilding\u002Faggregations","1.docs\u002F3.building\u002F4.aggregations","i-lucide-sigma",{"title":130,"path":131,"stem":132,"icon":133},"Windowing","\u002Fdocs\u002Fbuilding\u002Fwindowing","1.docs\u002F3.building\u002F5.windowing","i-lucide-calendar-clock",{"title":135,"path":136,"stem":137,"icon":138},"Joins","\u002Fdocs\u002Fbuilding\u002Fjoins","1.docs\u002F3.building\u002F6.joins","i-lucide-git-merge",{"title":140,"path":141,"stem":142,"icon":143},"The Processor API","\u002Fdocs\u002Fbuilding\u002Fprocessor-api","1.docs\u002F3.building\u002F7.processor-api","i-lucide-cpu",{"title":145,"path":146,"stem":147,"icon":148},"Scheduled sources","\u002Fdocs\u002Fbuilding\u002Fscheduled-sources","1.docs\u002F3.building\u002F8.scheduled-sources","i-lucide-timer",{"title":150,"path":151,"stem":152,"icon":153},"Serdes and Avro","\u002Fdocs\u002Fbuilding\u002Fserdes","1.docs\u002F3.building\u002F9.serdes","i-lucide-binary",{"title":155,"path":156,"stem":157,"children":158,"icon":80},"Configuration","\u002Fdocs\u002Fconfiguration","1.docs\u002F4.configuration\u002F1.index",[159,161,166,171,176],{"title":160,"path":156,"stem":157,"icon":80},"How configuration works",{"title":162,"path":163,"stem":164,"icon":165},"Engine configuration (:core)","\u002Fdocs\u002Fconfiguration\u002Fcore-config","1.docs\u002F4.configuration\u002F2.core-config","i-lucide-settings-2",{"title":167,"path":168,"stem":169,"icon":170},"Runtime configuration (:runtime)","\u002Fdocs\u002Fconfiguration\u002Fruntime-config","1.docs\u002F4.configuration\u002F3.runtime-config","i-lucide-server-cog",{"title":172,"path":173,"stem":174,"icon":175},"Defaults, adaptivity, and presets","\u002Fdocs\u002Fconfiguration\u002Fdefaults-and-presets","1.docs\u002F4.configuration\u002F4.defaults-and-presets","i-lucide-gauge",{"title":177,"path":178,"stem":179,"icon":180},"Kafka client configuration","\u002Fdocs\u002Fconfiguration\u002Fkafka-client-config","1.docs\u002F4.configuration\u002F5.kafka-client-config","i-lucide-plug",{"title":182,"path":183,"stem":184,"children":185,"icon":188},"Running in Production","\u002Fdocs\u002Fruntime","1.docs\u002F5.runtime\u002F1.index",[186,189,194,199,204,209,214,219],{"title":187,"path":183,"stem":184,"icon":188},"The runtime","i-lucide-server",{"title":190,"path":191,"stem":192,"icon":193},"The REST API","\u002Fdocs\u002Fruntime\u002Frest-api","1.docs\u002F5.runtime\u002F2.rest-api","i-lucide-globe",{"title":195,"path":196,"stem":197,"icon":198},"Health checks","\u002Fdocs\u002Fruntime\u002Fhealth-checks","1.docs\u002F5.runtime\u002F3.health-checks","i-lucide-heart-pulse",{"title":200,"path":201,"stem":202,"icon":203},"Metrics","\u002Fdocs\u002Fruntime\u002Fmetrics","1.docs\u002F5.runtime\u002F4.metrics","i-lucide-activity",{"title":205,"path":206,"stem":207,"icon":208},"Pause and resume","\u002Fdocs\u002Fruntime\u002Fpause-unpause","1.docs\u002F5.runtime\u002F5.pause-unpause","i-lucide-pause",{"title":210,"path":211,"stem":212,"icon":213},"Plugins and lifecycle hooks","\u002Fdocs\u002Fruntime\u002Fplugins","1.docs\u002F5.runtime\u002F6.plugins","i-lucide-puzzle",{"title":215,"path":216,"stem":217,"icon":218},"Docker images","\u002Fdocs\u002Fruntime\u002Fdocker","1.docs\u002F5.runtime\u002F7.docker","i-lucide-container",{"title":220,"path":221,"stem":222,"icon":223},"GraalVM native image","\u002Fdocs\u002Fruntime\u002Fnative-image","1.docs\u002F5.runtime\u002F8.native-image","i-lucide-zap",{"title":225,"path":226,"stem":227,"children":228,"icon":231},"Deploying & Operating","\u002Fdocs\u002Foperating","1.docs\u002F6.operating\u002F1.index",[229,232,237,242,247,252,257],{"title":230,"path":226,"stem":227,"icon":231},"Deploying and operating","i-lucide-life-buoy",{"title":233,"path":234,"stem":235,"icon":236},"Running on Kubernetes","\u002Fdocs\u002Foperating\u002Fkubernetes","1.docs\u002F6.operating\u002F2.kubernetes","i-lucide-ship",{"title":238,"path":239,"stem":240,"icon":241},"High availability","\u002Fdocs\u002Foperating\u002Fhigh-availability","1.docs\u002F6.operating\u002F3.high-availability","i-lucide-copy",{"title":243,"path":244,"stem":245,"icon":246},"Liveness and readiness probes","\u002Fdocs\u002Foperating\u002Fprobes","1.docs\u002F6.operating\u002F4.probes","i-lucide-stethoscope",{"title":248,"path":249,"stem":250,"icon":251},"Observability","\u002Fdocs\u002Foperating\u002Fobservability","1.docs\u002F6.operating\u002F5.observability","i-lucide-telescope",{"title":253,"path":254,"stem":255,"icon":256},"Tuning under load","\u002Fdocs\u002Foperating\u002Ftuning","1.docs\u002F6.operating\u002F6.tuning","i-lucide-sliders",{"title":258,"path":259,"stem":260,"icon":261},"Production checklist","\u002Fdocs\u002Foperating\u002Fproduction-checklist","1.docs\u002F6.operating\u002F7.production-checklist","i-lucide-clipboard-check",{"title":263,"path":264,"stem":265,"children":266,"icon":90},"Migrating from Kafka Streams","\u002Fdocs\u002Fmigration","1.docs\u002F7.migration\u002F1.index",[267,268,273,278,283,288],{"title":263,"path":264,"stem":265,"icon":90},{"title":269,"path":270,"stem":271,"icon":272},"Automated port","\u002Fdocs\u002Fmigration\u002Fautomated-port","1.docs\u002F7.migration\u002F2.automated-port","i-lucide-wand-sparkles",{"title":274,"path":275,"stem":276,"icon":277},"Migration without carrying state","\u002Fdocs\u002Fmigration\u002Fwithout-data-migration","1.docs\u002F7.migration\u002F3.without-data-migration","i-lucide-sparkles",{"title":279,"path":280,"stem":281,"icon":282},"Migration carrying state","\u002Fdocs\u002Fmigration\u002Fwith-data-migration","1.docs\u002F7.migration\u002F4.with-data-migration","i-lucide-database-backup",{"title":284,"path":285,"stem":286,"icon":287},"The migration tool","\u002Fdocs\u002Fmigration\u002Fmigration-tool","1.docs\u002F7.migration\u002F5.migration-tool","i-lucide-truck",{"title":289,"path":290,"stem":291,"icon":292},"Reusing your Kafka Streams dashboards","\u002Fdocs\u002Fmigration\u002Freusing-kafka-streams-dashboards","1.docs\u002F7.migration\u002F6.reusing-kafka-streams-dashboards","i-lucide-line-chart",{"title":294,"path":295,"stem":296,"children":297,"icon":299},"Reference","\u002Fdocs\u002Freference","1.docs\u002F8.reference\u002F1.index",[298,300,305,310,315,320,325,329],{"title":294,"path":295,"stem":296,"icon":299},"i-lucide-list",{"title":301,"path":302,"stem":303,"icon":304},"Configuration reference","\u002Fdocs\u002Freference\u002Fconfiguration-reference","1.docs\u002F8.reference\u002F2.configuration-reference","i-lucide-table",{"title":306,"path":307,"stem":308,"icon":309},"REST API reference","\u002Fdocs\u002Freference\u002Frest-api-reference","1.docs\u002F8.reference\u002F3.rest-api-reference","i-lucide-network",{"title":311,"path":312,"stem":313,"icon":314},"Gradle plugin reference","\u002Fdocs\u002Freference\u002Fgradle-plugin-reference","1.docs\u002F8.reference\u002F4.gradle-plugin-reference","i-lucide-box",{"title":316,"path":317,"stem":318,"icon":319},"Maven reference","\u002Fdocs\u002Freference\u002Fmaven-reference","1.docs\u002F8.reference\u002F5.maven-reference","i-simple-icons-apachemaven",{"title":321,"path":322,"stem":323,"icon":324},"Kafka Streams compatibility matrix","\u002Fdocs\u002Freference\u002Fks-compatibility-matrix","1.docs\u002F8.reference\u002F6.ks-compatibility-matrix","i-lucide-table-2",{"title":326,"path":327,"stem":328,"icon":175},"Metrics reference","\u002Fdocs\u002Freference\u002Fmetrics-reference","1.docs\u002F8.reference\u002F7.metrics-reference",{"title":330,"path":331,"stem":332,"icon":333},"Glossary","\u002Fdocs\u002Freference\u002Fglossary","1.docs\u002F8.reference\u002F8.glossary","i-lucide-book-a",{"id":335,"title":336,"authors":337,"badge":343,"body":345,"date":1417,"description":1418,"draft":42,"extension":1419,"image":1420,"meta":1422,"navigation":757,"ogImage":1423,"path":1424,"seo":1425,"sitemap":1426,"stem":1427,"updated":1423,"__hash__":1428},"posts\u002F3.blog\u002F23.timer-on-the-wrong-lane.md","Your timer fired on another thread",[338],{"name":339,"to":340,"avatar":341},"Hartmut Armbruster","https:\u002F\u002Fwww.linkedin.com\u002Fin\u002Fhartmut-co-uk\u002F",{"src":342},"\u002Fassets\u002Fhartmut_armbruster_monochromatic.jpg",{"label":344},"Deep Dive",{"type":346,"value":347,"toc":1406},"minimark",[348,361,372,463,468,518,521,527,530,540,543,546,629,643,657,669,687,690,705,712,715,725,822,858,894,910,924,927,934,959,977,1054,1064,1067,1081,1256,1259,1262,1284,1307,1310,1316,1322,1342,1351,1360,1365,1402],[349,350,351,352,356,357,360],"p",{},"StoatFlow promises that a key's timer callbacks run with the same serialised execution as ",[353,354,355],"code",{},"process(...)"," for that key: one lane, one key's events at a time, so ",[353,358,359],{},"onTimer"," can read and write the key's state without a lock. The lane is chosen by hashing the key's bytes. A timer's lane is chosen the same way — by hashing the timer key's bytes, which the engine serialises again, with the processor's timer key serde. When those two byte strings differ, the timer lands on another lane and runs concurrently with the key's records.",[349,362,363,364,367,368,371],{},"Behind a Processor API source declared with only a key deserializer, that is what happened. The timer key serde fell back to ",[353,365,366],{},"default.key.serde","; if the default wrote different bytes than the topic's producer, the key's timer fired on another lane, and nothing said so. Our reproduction put key ",[353,369,370],{},"k0","'s records on lane 0 and its timer on lane 6. This post explains where the two byte strings come from, why Kafka Streams never has to care, and what StoatFlow does now: declare the serde, check the lanes at build time, and stop the application on a proven split.",[373,374,375,381],"blockquote",{},[349,376,377],{},[378,379,380],"strong",{},"TL;DR",[382,383,384,398,411,425,444,450],"ul",{},[385,386,387,390,391,394,395,397],"li",{},[378,388,389],{},"What:"," a record's lane is the hash of its key bytes; a timer's lane is the hash of its key re-serialised by the timer key serde. Behind ",[353,392,393],{},"addSource(name, keyDeserializer, …)",", that serde was ",[353,396,366],{}," — different bytes, different lane, same key.",[385,399,400,403,404,406,407,410],{},[378,401,402],{},"Why it matters:"," ",[353,405,359],{}," and ",[353,408,409],{},"process"," for the same key ran at the same time on two virtual threads, racing on the key's state. No error, no metric.",[385,412,413,416,417,420,421,424],{},[378,414,415],{},"The fix:"," declare the key serde — ",[353,418,419],{},"Topology.addSource(name, Consumed.with(keySerde, valueSerde), topics)",". A build-time rule, ",[353,422,423],{},"timer-key-lane-affinity",", warns where you have not.",[385,426,427,430,431,435,436,439,440,443],{},[378,428,429],{},"The guarantee:"," at runtime StoatFlow compares ",[432,433,434],"em",{},"lanes",", not bytes. A timer registered for the key being processed that picks another lane throws a fatal ",[353,437,438],{},"TimerLaneAffinityException"," by default; a sampled check covers the rest and counts ",[353,441,442],{},"stoatflow.timer.lane.mismatch.total",".",[385,445,446,449],{},[378,447,448],{},"The catch:"," the sample can miss a mismatch that only some keys show, and the timer key serde is part of the stored timer format — changing it strands timers already persisted.",[385,451,452,403,455,460,461,443],{},[378,453,454],{},"Docs:",[456,457,459],"a",{"href":458},"\u002Fdocs\u002Fbuilding\u002Fprocessor-api#timer-key-serde","Timer key serde"," · ",[456,462,62],{"href":63},[464,465,467],"h2",{"id":466},"in-this-post","In this post",[382,469,470,476,482,488,494,500,506,512],{},[385,471,472],{},[456,473,475],{"href":474},"#a-lane-is-a-keys-serialisation-point","A lane is a key's serialisation point",[385,477,478],{},[456,479,481],{"href":480},"#where-the-two-byte-strings-diverge","Where the two byte strings diverge",[385,483,484],{},[456,485,487],{"href":486},"#why-kafka-streams-never-meets-this","Why Kafka Streams never meets this",[385,489,490],{},[456,491,493],{"href":492},"#declare-the-key-serde","Declare the key serde",[385,495,496],{},[456,497,499],{"href":498},"#prove-the-split-then-stop","Prove the split, then stop",[385,501,502],{},[456,503,505],{"href":504},"#catch-it-in-a-unit-test","Catch it in a unit test",[385,507,508],{},[456,509,511],{"href":510},"#timers-for-another-key","Timers for another key",[385,513,514],{},[456,515,517],{"href":516},"#tradeoffs-and-limits","Tradeoffs and limits",[464,519,475],{"id":520},"a-lane-is-a-keys-serialisation-point",[349,522,523,524,526],{},"A StoatFlow application is one process. Inside it, records are routed to ",[378,525,434],{}," — StoatFlow's unit of parallel processing — by key: the engine hashes the key's bytes and takes the result modulo the lane count, so the same key always lands on the same lane. Each lane runs on its own virtual thread and processes its keys' records one at a time, in the order Kafka delivered them. That is what makes per-key state safe without locks: two records of one key are never processed at the same moment.",[349,528,529],{},"The bytes that are hashed depend on where the record is. In the first stage of a topology they are the topic's raw key bytes, exactly as the producer wrote them — no serde involved. Behind a re-key, where a record crosses to another lane in memory, they are the bytes the boundary's key serde writes.",[349,531,532,533,536,537,539],{},"A timer has its own bytes. When a processor registers or deletes a timer, the engine serialises the timer key with the processor's ",[378,534,535],{},"timer key serde",": those bytes are what the timer store persists, what ",[353,538,359],{}," gets decoded from, and what picks the lane the timer fires on (behind a windowed aggregation, the inner key's bytes). Nothing on the timer path reads the bytes the record was laned by. So \"a key's timers run on its records' lane\" holds only if the timer key serde writes exactly the bytes the record was laned by.",[464,541,481],{"id":542},"where-the-two-byte-strings-diverge",[349,544,545],{},"The Kafka Streams-shaped Processor API declares a source with deserializers:",[547,548,553],"pre",{"className":549,"code":550,"language":551,"meta":552,"style":552},"language-kotlin shiki shiki-themes vitesse-light","val topology = Topology()\n    .addSource(\"events\", keyDeserializer, eventDeserializer, \"events\")\n    .addProcessor(\"sessionize\", { SessionProcessor() }, \"events\")\n","kotlin","",[353,554,555,579,603],{"__ignoreMap":552},[556,557,560,564,568,572,576],"span",{"class":558,"line":559},"line",1,[556,561,563],{"class":562},"sbBg2","val",[556,565,567],{"class":566},"suHK_"," topology ",[556,569,571],{"class":570},"sYZai","=",[556,573,575],{"class":574},"sySUi"," Topology",[556,577,578],{"class":566},"()\n",[556,580,582,585,588,591,595,598,600],{"class":558,"line":581},2,[556,583,584],{"class":566},"    .",[556,586,587],{"class":574},"addSource",[556,589,590],{"class":566},"(",[556,592,594],{"class":593},"spphp","\"events\"",[556,596,597],{"class":566},", keyDeserializer, eventDeserializer, ",[556,599,594],{"class":593},[556,601,602],{"class":566},")\n",[556,604,606,608,611,613,616,619,622,625,627],{"class":558,"line":605},3,[556,607,584],{"class":566},[556,609,610],{"class":574},"addProcessor",[556,612,590],{"class":566},[556,614,615],{"class":593},"\"sessionize\"",[556,617,618],{"class":566},", { ",[556,620,621],{"class":574},"SessionProcessor",[556,623,624],{"class":566},"() }, ",[556,626,594],{"class":593},[556,628,602],{"class":566},[349,630,631,632,634,635,638,639,642],{},"A deserializer can read a key but not write one. With nothing else declared on the records' path, the timer key serde fell back to ",[353,633,366],{},". Two outcomes followed. If the default could not serialise the key type at all, the first timer registration threw a ",[353,636,637],{},"ClassCastException"," — loud, and easy to fix. If it serialised the key ",[432,640,641],{},"successfully but differently"," from the producer, the timer was registered, persisted and fired, on whatever lane its bytes hashed to.",[349,644,645,646,649,650,653,654,656],{},"Different bytes are common. A producer writes the key as JSON — ",[353,647,648],{},"\"k0\"",", quotes included — and the source's deserializer strips the quotes; the default ",[353,651,652],{},"String"," serde then writes ",[353,655,370],{}," without them. Other causes: a Schema Registry serde whose id has evolved, header-mode input, or a boundary laned by a different key serde than the one the timer uses.",[349,658,659,660,662,663,665,666,668],{},"We built the case on a real broker, red first: eight lanes, JSON-quoted keys on the wire, a deserializer that unquotes them, ",[353,661,366],{}," set to ",[353,664,652],{},". The test derives keys whose two encodings hash to different lanes rather than hard-coding them. Result: key ",[353,667,370],{},"'s records ran on lane 0 while its timer fired on lane 6, with no log line and no metric. The control — keys whose encodings happen to share a lane — behaved.",[349,670,671,672,674,675,677,678,680,681,683,684,686],{},"The application kept running and broke its promise: ",[353,673,359],{}," for ",[353,676,370],{}," ran on lane 6 while lane 0 processed ",[353,679,370],{},"'s records. Any read-modify-write on the key's state could interleave: a session processor closing a session from ",[353,682,359],{}," while ",[353,685,409],{}," extends it, a timeout firing while the record that should have cancelled it is half-processed. Global stores do not make that safe — the lane is the key's serialisation point, and the timer had left it.",[464,688,487],{"id":689},"why-kafka-streams-never-meets-this",[349,691,692,693,696,697,700,701,704],{},"Kafka Streams has no keyed timers — only punctuators, which belong to a task, not a key — and no in-memory boundaries: when a downstream operation needs a changed key's partitioning — a ",[353,694,695],{},"groupBy",", a join, an explicit ",[353,698,699],{},"repartition()"," — the records go through a repartition topic. A source declares its key with a ",[353,702,703],{},"Deserializer",", and a key is serialised only where it is written to a topic or a store. So Kafka Streams never has to re-serialise a key it only read, and never has to agree with itself about which bytes a key has.",[349,706,707,708,711],{},"StoatFlow added both features on purpose. Keyed timers are what make per-key timeouts and debounce windows natural, as in Flink. In-memory re-keying is what removes the repartition topics. Together they make the key's ",[432,709,710],{},"bytes"," decide which thread runs the code, as well as how the key is stored, and that needs a declared serde instead of a default.",[464,713,493],{"id":714},"declare-the-key-serde",[349,716,717,718,721,722,724],{},"The fix is to say what the key is. StoatFlow adds a ",[353,719,720],{},"Consumed","-taking ",[353,723,587],{},":",[547,726,728],{"className":549,"code":727,"language":551,"meta":552,"style":552},"import io.stoatflow.core.topology.Consumed\nimport io.stoatflow.core.topology.Topology\nimport org.apache.kafka.common.serialization.Serdes\n\nval topology = Topology()\n    .addSource(\"events\", Consumed.with(Serdes.String(), eventSerde), \"events\")\n    .addProcessor(\"sessionize\", { SessionProcessor() }, \"events\")\n",[353,729,730,738,745,752,759,772,801],{"__ignoreMap":552},[556,731,732,735],{"class":558,"line":559},[556,733,734],{"class":562},"import",[556,736,737],{"class":574}," io.stoatflow.core.topology.Consumed\n",[556,739,740,742],{"class":558,"line":581},[556,741,734],{"class":562},[556,743,744],{"class":574}," io.stoatflow.core.topology.Topology\n",[556,746,747,749],{"class":558,"line":605},[556,748,734],{"class":562},[556,750,751],{"class":574}," org.apache.kafka.common.serialization.Serdes\n",[556,753,755],{"class":558,"line":754},4,[556,756,758],{"emptyLinePlaceholder":757},true,"\n",[556,760,762,764,766,768,770],{"class":558,"line":761},5,[556,763,563],{"class":562},[556,765,567],{"class":566},[556,767,571],{"class":570},[556,769,575],{"class":574},[556,771,578],{"class":566},[556,773,775,777,779,781,783,786,789,792,794,797,799],{"class":558,"line":774},6,[556,776,584],{"class":566},[556,778,587],{"class":574},[556,780,590],{"class":566},[556,782,594],{"class":593},[556,784,785],{"class":566},", Consumed.",[556,787,788],{"class":574},"with",[556,790,791],{"class":566},"(Serdes.",[556,793,652],{"class":574},[556,795,796],{"class":566},"(), eventSerde), ",[556,798,594],{"class":593},[556,800,602],{"class":566},[556,802,804,806,808,810,812,814,816,818,820],{"class":558,"line":803},7,[556,805,584],{"class":566},[556,807,610],{"class":574},[556,809,590],{"class":566},[556,811,615],{"class":593},[556,813,618],{"class":566},[556,815,621],{"class":574},[556,817,624],{"class":566},[556,819,594],{"class":593},[556,821,602],{"class":566},[349,823,824,825,827,828,831,832,835,836,406,839,842,843,845,846,849,850,853,854,857],{},"The ",[353,826,720],{}," applies exactly as it does for ",[353,829,830],{},"StreamsBuilder.stream(...)"," — serdes, timestamp extractor or watermark strategy, offset reset, lane overrides — and a ",[353,833,834],{},"Pattern"," overload takes a topic pattern. ",[353,837,838],{},"addGlobalStore(...)",[353,840,841],{},"addReadOnlyStateStore(...)"," get the same ",[353,844,720],{}," twin, so a state-update processor's keys, timers included, use the source's own codec; their sources always read from the beginning, so an offset reset other than ",[353,847,848],{},"earliest"," is refused there. A source with a ",[353,851,852],{},"ByteArrayDeserializer"," key counts as declaring ",[353,855,856],{},"Serdes.ByteArray()",", since the bytes it reads are the bytes it would write. The deserializer overloads are unchanged.",[349,859,860,861,863,864,867,868,871,872,867,874,877,878,880,881,884,885,888,889,891,892,443],{},"Behind the source, the timer key serde is the key serde declared for the key a processor receives: upstream on a ",[353,862,720],{},", ",[353,865,866],{},"Grouped"," or ",[353,869,870],{},"Repartitioned",", or downstream before any key change on a ",[353,873,870],{},[353,875,876],{},"Produced",", otherwise ",[353,879,366],{},". When nothing on the records' path declares the key, an upstream DSL store's key serde counts too — a ",[353,882,883],{},"Materialized.withKeySerde(…)"," on the aggregation before ",[353,886,887],{},"toStream()",". A Processor API node in between counts as a key change, so the source's ",[353,890,720],{}," reaches a timer processor fed by that source directly; behind another Processor API node, declare the serde downstream or set ",[353,893,366],{},[349,895,896,897,899,900,903,904,906,907,443],{},"At build time, the ",[353,898,423],{}," validation rule reports a timer-aware processor whose timer bytes may differ from its records' lane bytes — behind a deserializer-only source, or across a boundary laned by a different key serde. It defaults to ",[353,901,902],{},"warn",", because a configured default may be exactly right; if ",[353,905,366],{}," already writes the topic's key bytes, set the rule to ",[353,908,909],{},"off",[349,911,912,913,916,917,406,920,923],{},"This is one case of a wider change in the same release: the DSL now carries only the serdes you declared. Defaults are applied once, from the application's ",[353,914,915],{},"StreamsConfig",", when the topology is compiled. ",[353,918,919],{},"StreamsBuilder",[353,921,922],{},"TopologyConfig"," no longer carry default serdes of their own — Kafka Streams parity — and a windowed key keeps its serde through every operator downstream of the aggregation.",[464,925,499],{"id":926},"prove-the-split-then-stop",[349,928,929,930,933],{},"A build-time rule can only say \"may differ\". At runtime StoatFlow checks what happens, and it compares ",[378,931,932],{},"lanes, not bytes",": two encodings that differ but hash to the same lane are no split, and are never reported. That matters because a byte comparison would flag every harmless difference — and a check that cries wolf gets switched off.",[349,935,936,939,940,943,944,946,947,950,951,954,955,958],{},[378,937,938],{},"The guard"," runs on every timer registered or deleted for the key being processed, on that record's lane. If the timer key's bytes pick another lane than the record's, the split is proven: this key's timers would run beside its records. The default verdict is ",[353,941,942],{},"error",". The registration does not happen; StoatFlow logs the split at ERROR and throws ",[353,945,438],{},", a fatal ",[353,948,949],{},"StreamsException"," that bypasses ",[353,952,953],{},"processing.exception.handler",", so the epoch aborts and its offsets are not committed. Catching it does not hide it — a processor that catches it around the timer call, or an upstream processor that catches it around ",[353,956,957],{},"context.forward",", sees it rethrown once that processor returns.",[349,960,961,964,965,967,968,970,971,973,974,976],{},[378,962,963],{},"The sampled check"," covers what the guard cannot see: processors that only register timers from punctuators or ",[353,966,359],{},", timers for other keys, and processors that never time the record's own key. Before ",[353,969,409],{},", it re-serialises the record key with the timer key serde and compares lanes — for the first 64 records per processor and lane, then one in 4,096. The default verdict is ",[353,972,902],{},": it logs once per processor and counts every split it samples on ",[353,975,442],{},", pre-registered at zero so an alert on its increase sees the first hit. It is not fatal by default because it judges more than proven splits, and because some wire encodings legitimately differ from a re-serialisation: an evolved Schema Registry id, JSON written by another producer.",[978,979,980,996],"table",{},[981,982,983],"thead",{},[984,985,986,990,993],"tr",{},[987,988,989],"th",{},"Setting (YAML)",[987,991,992],{},"Default",[987,994,995],{},"What it governs",[997,998,999,1020,1040],"tbody",{},[984,1000,1001,1007,1011],{},[1002,1003,1004],"td",{},[353,1005,1006],{},"stoatflow.processing.timer-lane-affinity-guard",[1002,1008,1009],{},[353,1010,942],{},[1002,1012,1013,1014,863,1016,867,1018],{},"A timer registered or deleted for the key being processed that picks another lane — ",[353,1015,942],{},[353,1017,902],{},[353,1019,909],{},[984,1021,1022,1027,1031],{},[1002,1023,1024],{},[353,1025,1026],{},"stoatflow.processing.timer-lane-affinity-sampled-check",[1002,1028,1029],{},[353,1030,902],{},[1002,1032,1033,1034,863,1036,867,1038],{},"The sampled record-key check — ",[353,1035,942],{},[353,1037,902],{},[353,1039,909],{},[984,1041,1042,1047,1051],{},[1002,1043,1044],{},[353,1045,1046],{},"stoatflow.topology.validation.timer-key-lane-affinity",[1002,1048,1049],{},[353,1050,902],{},[1002,1052,1053],{},"The build-time rule",[349,1055,1056,1057,1060,1061,1063],{},"Neither check runs on a sub-topology with a single lane, where no split is possible, or on records whose lane a custom ",[353,1058,1059],{},"StreamPartitioner"," on ",[353,1062,699],{}," chose without hashing the key.",[464,1065,505],{"id":1066},"catch-it-in-a-unit-test",[349,1068,824,1069,1072,1073,1076,1077,1080],{},[353,1070,1071],{},"TopologyTestDriver"," runs every piped record through the sampled check and judges a split over the processor's sub-topology lane count: its ",[353,1074,1075],{},"numberOfLanes"," override, else ",[353,1078,1079],{},"num.stream.threads",", else a fixed reference count of 16 — never the build machine's core count, so the same test gives the same answer on a laptop and in CI. To judge over your production lane count and fail the test on a systematic mismatch:",[547,1082,1084],{"className":549,"code":1083,"language":551,"meta":552,"style":552},"val props = Properties().apply {\n    put(\"application.id\", \"sessionize-test\")\n    put(\"bootstrap.servers\", \"unused:9092\")\n    put(\"num.stream.threads\", \"8\")                          \u002F\u002F your production lane count\n    put(\"timer.lane-affinity.sampled-check\", \"error\")\n}\n\nTopologyTestDriver(topology, props).use { driver ->\n    val input = driver.createInputTopic(\"events\", Serdes.String(), eventSerde)\n    input.pipeInput(\"k0\", event)   \u002F\u002F a lane split throws here, before deployment\n}\n",[353,1085,1086,1107,1124,1140,1161,1177,1182,1186,1203,1232,1251],{"__ignoreMap":552},[556,1087,1088,1090,1093,1095,1098,1101,1104],{"class":558,"line":559},[556,1089,563],{"class":562},[556,1091,1092],{"class":566}," props ",[556,1094,571],{"class":570},[556,1096,1097],{"class":574}," Properties",[556,1099,1100],{"class":566},"().",[556,1102,1103],{"class":574},"apply",[556,1105,1106],{"class":566}," {\n",[556,1108,1109,1112,1114,1117,1119,1122],{"class":558,"line":581},[556,1110,1111],{"class":574},"    put",[556,1113,590],{"class":566},[556,1115,1116],{"class":593},"\"application.id\"",[556,1118,863],{"class":566},[556,1120,1121],{"class":593},"\"sessionize-test\"",[556,1123,602],{"class":566},[556,1125,1126,1128,1130,1133,1135,1138],{"class":558,"line":605},[556,1127,1111],{"class":574},[556,1129,590],{"class":566},[556,1131,1132],{"class":593},"\"bootstrap.servers\"",[556,1134,863],{"class":566},[556,1136,1137],{"class":593},"\"unused:9092\"",[556,1139,602],{"class":566},[556,1141,1142,1144,1146,1149,1151,1154,1157],{"class":558,"line":754},[556,1143,1111],{"class":574},[556,1145,590],{"class":566},[556,1147,1148],{"class":593},"\"num.stream.threads\"",[556,1150,863],{"class":566},[556,1152,1153],{"class":593},"\"8\"",[556,1155,1156],{"class":566},")                          ",[556,1158,1160],{"class":1159},"s8zF2","\u002F\u002F your production lane count\n",[556,1162,1163,1165,1167,1170,1172,1175],{"class":558,"line":761},[556,1164,1111],{"class":574},[556,1166,590],{"class":566},[556,1168,1169],{"class":593},"\"timer.lane-affinity.sampled-check\"",[556,1171,863],{"class":566},[556,1173,1174],{"class":593},"\"error\"",[556,1176,602],{"class":566},[556,1178,1179],{"class":558,"line":774},[556,1180,1181],{"class":566},"}\n",[556,1183,1184],{"class":558,"line":803},[556,1185,758],{"emptyLinePlaceholder":757},[556,1187,1189,1191,1194,1197,1200],{"class":558,"line":1188},8,[556,1190,1071],{"class":574},[556,1192,1193],{"class":566},"(topology, props).",[556,1195,1196],{"class":574},"use",[556,1198,1199],{"class":566}," { driver ",[556,1201,1202],{"class":570},"->\n",[556,1204,1206,1209,1212,1214,1217,1220,1222,1224,1227,1229],{"class":558,"line":1205},9,[556,1207,1208],{"class":562},"    val",[556,1210,1211],{"class":566}," input ",[556,1213,571],{"class":570},[556,1215,1216],{"class":566}," driver.",[556,1218,1219],{"class":574},"createInputTopic",[556,1221,590],{"class":566},[556,1223,594],{"class":593},[556,1225,1226],{"class":566},", Serdes.",[556,1228,652],{"class":574},[556,1230,1231],{"class":566},"(), eventSerde)\n",[556,1233,1235,1238,1241,1243,1245,1248],{"class":558,"line":1234},10,[556,1236,1237],{"class":566},"    input.",[556,1239,1240],{"class":574},"pipeInput",[556,1242,590],{"class":566},[556,1244,648],{"class":593},[556,1246,1247],{"class":566},", event)   ",[556,1249,1250],{"class":1159},"\u002F\u002F a lane split throws here, before deployment\n",[556,1252,1254],{"class":558,"line":1253},11,[556,1255,1181],{"class":566},[349,1257,1258],{},"The driver is single-threaded, so it cannot show the race itself. It does not need to: it computes the two lanes and fails on the mismatch that would cause it.",[464,1260,511],{"id":1261},"timers-for-another-key",[349,1263,1264,1265,863,1267,1269,1270,1273,1274,1277,1278,1280,1281,1283],{},"A StoatFlow timer may be for any key the timer key serde can serialise, not only the record being processed — from ",[353,1266,409],{},[353,1268,359],{}," or a punctuator. A timer for key ",[353,1271,1272],{},"y",", armed while processing key ",[353,1275,1276],{},"x",", fires on ",[353,1279,1272],{},"'s lane, serially with ",[353,1282,1272],{},"'s records. That goes beyond Flink, which scopes a timer to the current key.",[349,1285,1286,1287,1289,1290,1292,1293,1295,1296,1298,1299,1303,1304,1306],{},"The cost is ordering across lanes. Registering or deleting ",[353,1288,1272],{},"'s timer from ",[353,1291,1276],{},"'s lane is not ordered against what ",[353,1294,1272],{},"'s own lane does to ",[353,1297,1272],{},"'s timers at the same moment. The timer set stays consistent: a write from a later commit wins over one from an earlier commit, even when it was made earlier, the same way while the application runs and after a restart. But which commit each write lands in depends on timing, and a replay after a failure may place them differently. The same holds for moving a ",[456,1300,1302],{"href":1301},"\u002Fdocs\u002Fbuilding\u002Fprocessor-api#named-timers","named timer"," from another lane. When the order matters, use the Flink-style pattern: forward a record re-keyed to ",[353,1305,1272],{},", and arm the timer while processing that record, so every key's timers have one writer.",[464,1308,517],{"id":1309},"tradeoffs-and-limits",[349,1311,1312,1315],{},[378,1313,1314],{},"The sample can miss."," After the first 64 records per processor and lane it looks at one record in 4,096. A mismatch that every key shows is caught in the first few records; one that only a few keys show — a rare schema version, one producer among many — can go unseen for a long time. The guard has no such gap for timers registered on the key being processed, but it judges only those. Declaring the key serde removes the question instead of sampling for it.",[349,1317,1318,1321],{},[378,1319,1320],{},"Lanes depend on the lane count."," A key whose two encodings share a lane at eight lanes may not at sixteen. A check that passed before a lane-count change can fail after it. That is another reason to declare the serde rather than rely on a lucky hash.",[349,1323,1324,1327,1328,1331,1332,1334,1335,1338,1339,1341],{},[378,1325,1326],{},"The timer key serde is part of the timer store's format."," Changing it between deployments — for example, adding ",[353,1329,1330],{},"addSource(..., Consumed...)"," to an application that ran on ",[353,1333,366],{}," — changes the bytes of every future registration, while timers already stored keep the old bytes. Those fail to decode when they fire, are never deleted, or, worst, decode as a ",[432,1336,1337],{},"different"," key, so ",[353,1340,359],{}," acts on the wrong key's state. Named timers are keyed by those bytes too: after a change, a registration by name no longer replaces a timer stored under the old bytes, so both fire. Settle the key serde before you rely on persisted timers, and treat a change as a state migration.",[349,1343,1344,1350],{},[378,1345,1346,1347,1349],{},"A custom ",[353,1348,1059],{}," is outside the contract."," It picks a record's lane without hashing the key bytes, so neither check can judge it, and timers behind it are not guaranteed to share their records' lane.",[349,1352,1353,1356,1357,1359],{},[378,1354,1355],{},"A proven split now stops the application."," An application that ran with split timers on an earlier release now stops at its first proven split, until the serde is declared or the guard is set to ",[353,1358,902],{},". We chose a loud failure over a silent race; the build-time rule is there to make that failure happen in a code review rather than in production.",[349,1361,1362],{},[378,1363,1364],{},"Read on:",[382,1366,1367,1372,1377,1388,1395],{},[385,1368,1369,1371],{},[456,1370,459],{"href":458}," — the full contract, the two checks and their settings.",[385,1373,1374,1376],{},[456,1375,62],{"href":63}," — key affinity, virtual threads and in-memory re-keying.",[385,1378,1379,1381,1382,406,1385,443],{},[456,1380,301],{"href":302}," — ",[353,1383,1384],{},"timer-lane-affinity-*",[353,1386,1387],{},"topology.validation.*",[385,1389,1390,1394],{},[456,1391,1393],{"href":1392},"\u002Fblog\u002Fscheduled-sources","Scheduled sources: a topology source that isn't a topic"," — the other kind of callback-driven input.",[385,1396,1397,1401],{},[456,1398,1400],{"href":1399},"\u002Fblog\u002Ferror-handling-three-gates-one-transaction","Three gates, one transaction: error handling in StoatFlow"," — why a fatal exception aborts the epoch instead of reaching the handler.",[1403,1404,1405],"style",{},"html pre.shiki code .sbBg2, html code.shiki .sbBg2{--shiki-default:#1E754F}html pre.shiki code .suHK_, html code.shiki .suHK_{--shiki-default:#393A34}html pre.shiki code .sYZai, html code.shiki .sYZai{--shiki-default:#999999}html pre.shiki code .sySUi, html code.shiki .sySUi{--shiki-default:#59873A}html pre.shiki code .spphp, html code.shiki .spphp{--shiki-default:#B56959}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html pre.shiki code .s8zF2, html code.shiki .s8zF2{--shiki-default:#A0ADA0}",{"title":552,"searchDepth":581,"depth":581,"links":1407},[1408,1409,1410,1411,1412,1413,1414,1415,1416],{"id":466,"depth":581,"text":467},{"id":520,"depth":581,"text":475},{"id":542,"depth":581,"text":481},{"id":689,"depth":581,"text":487},{"id":714,"depth":581,"text":493},{"id":926,"depth":581,"text":499},{"id":1066,"depth":581,"text":505},{"id":1261,"depth":581,"text":511},{"id":1309,"depth":581,"text":517},"2026-10-10","StoatFlow runs a key's timers on the same lane as its records, so onTimer never races process() for that key. The lane is chosen by hashing key bytes — and a timer key is serialised again, by its own serde. Behind a Processor API source declared with only a deserializer, that serde was the default, and when the default wrote different bytes than the topic, a key's timer fired on another lane, concurrently with its records, without a word. How it happens, why Kafka Streams never meets it, and how StoatFlow now refuses to run a proven split.","md",{"src":1421},"\u002Fassets\u002Fblog\u002Fog\u002Ftimer-on-the-wrong-lane.png",{},null,"\u002Fblog\u002Ftimer-on-the-wrong-lane",{"title":336,"description":1418},{"loc":1424},"3.blog\u002F23.timer-on-the-wrong-lane","VcFIC9UtO_YEmYo6_ny4S0gLadV7Ictif-uSSzAzUlU",[1430,1435],{"title":1431,"path":1432,"stem":1433,"description":1434,"children":-1},"Why a stream processor must never reorder a partition","\u002Fblog\u002Fnever-reorder-a-partition","3.blog\u002F22.never-reorder-a-partition","StoatFlow used to sort each partition's buffered records by timestamp, so the oldest event went first. Under the right timing that lost a record for good after a crash, under exactly-once and at-least-once alike, without a log line or a metric. This post walks the sequence from a buffered record to a committed offset that runs past it, why the obvious patch would have broken exactly-once, and how the one stall that offset order introduced was removed as well.",{"title":1436,"path":1437,"stem":1438,"description":1439,"children":-1},"The self-join that emitted nothing","\u002Fblog\u002Fthe-self-join-that-emitted-nothing","3.blog\u002F26.the-self-join-that-emitted-nothing","On Kafka Streams, s.join(s) over two records of one key emits four results. On StoatFlow it emitted none — no error, no log line, no metric. A stream–stream join chose which side a record belonged to by the name of the source it came from, and a self-join's records all come from one source. The same rule dropped records in chained joins, merged legs and cogroups. How we found every affected shape by running Kafka Streams' own test driver as the oracle, and how the fix makes the edge carry the input.",1792981295677]