[{"data":1,"prerenderedAt":935},["ShallowReactive",2],{"navigation":3,"\u002Fblog\u002Fkip-1035-state-store-managed-offsets":314,"\u002Fblog\u002Fkip-1035-state-store-managed-offsets-surround":924},[4,38,86,149,176,219,257,273],{"title":5,"path":6,"stem":7,"children":8,"icon":37},"Getting Started","\u002Fdocs\u002Fgetting-started","1.docs\u002F1.getting-started\u002F1.index",[9,12,17,22,27,32],{"title":10,"path":6,"stem":7,"icon":11},"Getting started","i-lucide-flag",{"title":13,"path":14,"stem":15,"icon":16},"Modules overview","\u002Fdocs\u002Fgetting-started\u002Fmodules-overview","1.docs\u002F1.getting-started\u002F2.modules-overview","i-lucide-boxes",{"title":18,"path":19,"stem":20,"icon":21},"Installation","\u002Fdocs\u002Fgetting-started\u002Finstallation","1.docs\u002F1.getting-started\u002F3.installation","i-lucide-download",{"title":23,"path":24,"stem":25,"icon":26},"License configuration","\u002Fdocs\u002Fgetting-started\u002Flicense-configuration","1.docs\u002F1.getting-started\u002F4.license-configuration","i-lucide-key-round",{"title":28,"path":29,"stem":30,"icon":31},"Your first app","\u002Fdocs\u002Fgetting-started\u002Ffirst-app","1.docs\u002F1.getting-started\u002F5.first-app","i-lucide-square-play",{"title":33,"path":34,"stem":35,"icon":36},"Project setup","\u002Fdocs\u002Fgetting-started\u002Fproject-setup","1.docs\u002F1.getting-started\u002F6.project-setup","i-lucide-package",false,{"title":39,"path":40,"stem":41,"children":42,"icon":45},"Core Concepts","\u002Fdocs\u002Fconcepts","1.docs\u002F2.concepts\u002F1.index",[43,46,51,56,61,66,71,76,81],{"title":44,"path":40,"stem":41,"icon":45},"Core concepts","i-lucide-book-open",{"title":47,"path":48,"stem":49,"icon":50},"Architecture","\u002Fdocs\u002Fconcepts\u002Farchitecture","1.docs\u002F2.concepts\u002F2.architecture","i-lucide-layers",{"title":52,"path":53,"stem":54,"icon":55},"Exactly-once semantics","\u002Fdocs\u002Fconcepts\u002Fexactly-once","1.docs\u002F2.concepts\u002F3.exactly-once","i-lucide-shield-check",{"title":57,"path":58,"stem":59,"icon":60},"Lanes and parallelism","\u002Fdocs\u002Fconcepts\u002Flanes-and-parallelism","1.docs\u002F2.concepts\u002F4.lanes-and-parallelism","i-lucide-split",{"title":62,"path":63,"stem":64,"icon":65},"State and thread-safety","\u002Fdocs\u002Fconcepts\u002Fstate-and-thread-safety","1.docs\u002F2.concepts\u002F5.state-and-thread-safety","i-lucide-lock",{"title":67,"path":68,"stem":69,"icon":70},"Event time and watermarks","\u002Fdocs\u002Fconcepts\u002Fevent-time-and-watermarks","1.docs\u002F2.concepts\u002F6.event-time-and-watermarks","i-lucide-clock",{"title":72,"path":73,"stem":74,"icon":75},"The configuration model","\u002Fdocs\u002Fconcepts\u002Fconfiguration-model","1.docs\u002F2.concepts\u002F7.configuration-model","i-lucide-sliders-horizontal",{"title":77,"path":78,"stem":79,"icon":80},"The error-handling model","\u002Fdocs\u002Fconcepts\u002Ferror-handling-model","1.docs\u002F2.concepts\u002F8.error-handling-model","i-lucide-triangle-alert",{"title":82,"path":83,"stem":84,"icon":85},"How StoatFlow differs from Kafka Streams","\u002Fdocs\u002Fconcepts\u002Fhow-stoatflow-differs-from-ks","1.docs\u002F2.concepts\u002F9.how-stoatflow-differs-from-ks","i-lucide-arrow-left-right",{"title":87,"path":88,"stem":89,"children":90,"icon":93},"Building Topologies","\u002Fdocs\u002Fbuilding","1.docs\u002F3.building\u002F1.index",[91,94,99,104,109,114,119,124,129,134,139,144],{"title":92,"path":88,"stem":89,"icon":93},"Building topologies","i-lucide-blocks",{"title":95,"path":96,"stem":97,"icon":98},"Error handling and DLQ","\u002Fdocs\u002Fbuilding\u002Ferror-handling-dlq","1.docs\u002F3.building\u002F10.error-handling-dlq","i-lucide-circle-x",{"title":100,"path":101,"stem":102,"icon":103},"State stores","\u002Fdocs\u002Fbuilding\u002Fstate-stores","1.docs\u002F3.building\u002F11.state-stores","i-lucide-database",{"title":105,"path":106,"stem":107,"icon":108},"Testing topologies","\u002Fdocs\u002Fbuilding\u002Ftesting","1.docs\u002F3.building\u002F12.testing","i-lucide-flask-conical",{"title":110,"path":111,"stem":112,"icon":113},"Sources and sinks","\u002Fdocs\u002Fbuilding\u002Fstreams-builder","1.docs\u002F3.building\u002F2.streams-builder","i-lucide-import",{"title":115,"path":116,"stem":117,"icon":118},"KStream and KTable operations","\u002Fdocs\u002Fbuilding\u002Fkstream-ktable","1.docs\u002F3.building\u002F3.kstream-ktable","i-lucide-waypoints",{"title":120,"path":121,"stem":122,"icon":123},"Aggregations","\u002Fdocs\u002Fbuilding\u002Faggregations","1.docs\u002F3.building\u002F4.aggregations","i-lucide-sigma",{"title":125,"path":126,"stem":127,"icon":128},"Windowing","\u002Fdocs\u002Fbuilding\u002Fwindowing","1.docs\u002F3.building\u002F5.windowing","i-lucide-calendar-clock",{"title":130,"path":131,"stem":132,"icon":133},"Joins","\u002Fdocs\u002Fbuilding\u002Fjoins","1.docs\u002F3.building\u002F6.joins","i-lucide-git-merge",{"title":135,"path":136,"stem":137,"icon":138},"The Processor API","\u002Fdocs\u002Fbuilding\u002Fprocessor-api","1.docs\u002F3.building\u002F7.processor-api","i-lucide-cpu",{"title":140,"path":141,"stem":142,"icon":143},"Scheduled sources","\u002Fdocs\u002Fbuilding\u002Fscheduled-sources","1.docs\u002F3.building\u002F8.scheduled-sources","i-lucide-timer",{"title":145,"path":146,"stem":147,"icon":148},"Serdes and Avro","\u002Fdocs\u002Fbuilding\u002Fserdes","1.docs\u002F3.building\u002F9.serdes","i-lucide-binary",{"title":150,"path":151,"stem":152,"children":153,"icon":75},"Configuration","\u002Fdocs\u002Fconfiguration","1.docs\u002F4.configuration\u002F1.index",[154,156,161,166,171],{"title":155,"path":151,"stem":152,"icon":75},"How configuration works",{"title":157,"path":158,"stem":159,"icon":160},"Engine configuration (:core)","\u002Fdocs\u002Fconfiguration\u002Fcore-config","1.docs\u002F4.configuration\u002F2.core-config","i-lucide-settings-2",{"title":162,"path":163,"stem":164,"icon":165},"Runtime configuration (:runtime)","\u002Fdocs\u002Fconfiguration\u002Fruntime-config","1.docs\u002F4.configuration\u002F3.runtime-config","i-lucide-server-cog",{"title":167,"path":168,"stem":169,"icon":170},"Defaults, adaptivity, and presets","\u002Fdocs\u002Fconfiguration\u002Fdefaults-and-presets","1.docs\u002F4.configuration\u002F4.defaults-and-presets","i-lucide-gauge",{"title":172,"path":173,"stem":174,"icon":175},"Kafka client configuration","\u002Fdocs\u002Fconfiguration\u002Fkafka-client-config","1.docs\u002F4.configuration\u002F5.kafka-client-config","i-lucide-plug",{"title":177,"path":178,"stem":179,"children":180,"icon":183},"Running in Production","\u002Fdocs\u002Fruntime","1.docs\u002F5.runtime\u002F1.index",[181,184,189,194,199,204,209,214],{"title":182,"path":178,"stem":179,"icon":183},"The runtime","i-lucide-server",{"title":185,"path":186,"stem":187,"icon":188},"The REST API","\u002Fdocs\u002Fruntime\u002Frest-api","1.docs\u002F5.runtime\u002F2.rest-api","i-lucide-globe",{"title":190,"path":191,"stem":192,"icon":193},"Health checks","\u002Fdocs\u002Fruntime\u002Fhealth-checks","1.docs\u002F5.runtime\u002F3.health-checks","i-lucide-heart-pulse",{"title":195,"path":196,"stem":197,"icon":198},"Metrics","\u002Fdocs\u002Fruntime\u002Fmetrics","1.docs\u002F5.runtime\u002F4.metrics","i-lucide-activity",{"title":200,"path":201,"stem":202,"icon":203},"Pause and resume","\u002Fdocs\u002Fruntime\u002Fpause-unpause","1.docs\u002F5.runtime\u002F5.pause-unpause","i-lucide-pause",{"title":205,"path":206,"stem":207,"icon":208},"Plugins and lifecycle hooks","\u002Fdocs\u002Fruntime\u002Fplugins","1.docs\u002F5.runtime\u002F6.plugins","i-lucide-puzzle",{"title":210,"path":211,"stem":212,"icon":213},"Docker images","\u002Fdocs\u002Fruntime\u002Fdocker","1.docs\u002F5.runtime\u002F7.docker","i-lucide-container",{"title":215,"path":216,"stem":217,"icon":218},"GraalVM native image","\u002Fdocs\u002Fruntime\u002Fnative-image","1.docs\u002F5.runtime\u002F8.native-image","i-lucide-zap",{"title":220,"path":221,"stem":222,"children":223,"icon":226},"Deploying & Operating","\u002Fdocs\u002Foperating","1.docs\u002F6.operating\u002F1.index",[224,227,232,237,242,247,252],{"title":225,"path":221,"stem":222,"icon":226},"Deploying and operating","i-lucide-life-buoy",{"title":228,"path":229,"stem":230,"icon":231},"Running on Kubernetes","\u002Fdocs\u002Foperating\u002Fkubernetes","1.docs\u002F6.operating\u002F2.kubernetes","i-lucide-ship",{"title":233,"path":234,"stem":235,"icon":236},"High availability","\u002Fdocs\u002Foperating\u002Fhigh-availability","1.docs\u002F6.operating\u002F3.high-availability","i-lucide-copy",{"title":238,"path":239,"stem":240,"icon":241},"Liveness and readiness probes","\u002Fdocs\u002Foperating\u002Fprobes","1.docs\u002F6.operating\u002F4.probes","i-lucide-stethoscope",{"title":243,"path":244,"stem":245,"icon":246},"Observability","\u002Fdocs\u002Foperating\u002Fobservability","1.docs\u002F6.operating\u002F5.observability","i-lucide-telescope",{"title":248,"path":249,"stem":250,"icon":251},"Tuning under load","\u002Fdocs\u002Foperating\u002Ftuning","1.docs\u002F6.operating\u002F6.tuning","i-lucide-sliders",{"title":253,"path":254,"stem":255,"icon":256},"Production checklist","\u002Fdocs\u002Foperating\u002Fproduction-checklist","1.docs\u002F6.operating\u002F7.production-checklist","i-lucide-clipboard-check",{"title":258,"path":259,"stem":260,"children":261,"icon":85},"Migrating from Kafka Streams","\u002Fdocs\u002Fmigration","1.docs\u002F7.migration\u002F1.index",[262,263,268],{"title":258,"path":259,"stem":260,"icon":85},{"title":264,"path":265,"stem":266,"icon":267},"Migration without carrying state","\u002Fdocs\u002Fmigration\u002Fwithout-data-migration","1.docs\u002F7.migration\u002F2.without-data-migration","i-lucide-sparkles",{"title":269,"path":270,"stem":271,"icon":272},"Migration carrying state","\u002Fdocs\u002Fmigration\u002Fwith-data-migration","1.docs\u002F7.migration\u002F3.with-data-migration","i-lucide-database-backup",{"title":274,"path":275,"stem":276,"children":277,"icon":279},"Reference","\u002Fdocs\u002Freference","1.docs\u002F8.reference\u002F1.index",[278,280,285,290,295,300,305,309],{"title":274,"path":275,"stem":276,"icon":279},"i-lucide-list",{"title":281,"path":282,"stem":283,"icon":284},"Configuration reference","\u002Fdocs\u002Freference\u002Fconfiguration-reference","1.docs\u002F8.reference\u002F2.configuration-reference","i-lucide-table",{"title":286,"path":287,"stem":288,"icon":289},"REST API reference","\u002Fdocs\u002Freference\u002Frest-api-reference","1.docs\u002F8.reference\u002F3.rest-api-reference","i-lucide-network",{"title":291,"path":292,"stem":293,"icon":294},"Gradle plugin reference","\u002Fdocs\u002Freference\u002Fgradle-plugin-reference","1.docs\u002F8.reference\u002F4.gradle-plugin-reference","i-lucide-box",{"title":296,"path":297,"stem":298,"icon":299},"Maven reference","\u002Fdocs\u002Freference\u002Fmaven-reference","1.docs\u002F8.reference\u002F5.maven-reference","i-simple-icons-apachemaven",{"title":301,"path":302,"stem":303,"icon":304},"Kafka Streams compatibility matrix","\u002Fdocs\u002Freference\u002Fks-compatibility-matrix","1.docs\u002F8.reference\u002F6.ks-compatibility-matrix","i-lucide-table-2",{"title":306,"path":307,"stem":308,"icon":170},"Metrics reference","\u002Fdocs\u002Freference\u002Fmetrics-reference","1.docs\u002F8.reference\u002F7.metrics-reference",{"title":310,"path":311,"stem":312,"icon":313},"Glossary","\u002Fdocs\u002Freference\u002Fglossary","1.docs\u002F8.reference\u002F8.glossary","i-lucide-book-a",{"id":315,"title":316,"authors":317,"badge":323,"body":325,"date":914,"description":915,"draft":37,"extension":916,"image":917,"meta":919,"navigation":574,"path":920,"seo":921,"stem":922,"__hash__":923},"posts\u002F3.blog\u002F2.kip-1035-state-store-managed-offsets.md","KIP-1035: Why Kafka Streams 4.3 lets state stores own their offsets",[318],{"name":319,"to":320,"avatar":321},"Hartmut Armbruster","https:\u002F\u002Fwww.linkedin.com\u002Fin\u002Fhartmutarmbruster",{"src":322},"\u002Fassets\u002Fhartmut_armbruster_monochromatic.jpg",{"label":324},"Deep Dive",{"type":326,"value":327,"toc":905},"minimark",[328,371,374,379,385,388,402,408,418,422,425,428,431,434,441,444,451,454,459,465,469,484,487,490,494,497,504,700,706,713,718,724,727,767,777,781,784,804,807,811,818,834,837,840,844,855,867,872,901],[329,330,331,338],"blockquote",{},[332,333,334],"p",{},[335,336,337],"strong",{},"TL;DR",[339,340,341,353,359,365],"ul",{},[342,343,344,347,348,352],"li",{},[335,345,346],{},"What changed:"," Kafka Streams 4.3 (22 May 2026) ships KIP-1035 — state stores now manage their own changelog offsets, persisted atomically with the data, instead of in a separate ",[349,350,351],"code",{},".checkpoint"," file.",[342,354,355,358],{},[335,356,357],{},"Why it mattered:"," offsets and state lived in different places with no atomicity. On an unclean shutdown they could disagree — and under exactly-once that meant discarding local state and restoring the whole changelog.",[342,360,361,364],{},[335,362,363],{},"What it unlocks:"," consistent offset-and-data recovery, no full re-restore after a crash, and — the real prize — the hard prerequisite for transactional state stores (KIP-892).",[342,366,367,370],{},[335,368,369],{},"Who:"," designed and contributed by Nick Telford (Meltwater), split out of his KIP-892 work. His Kafka Summit London 2024 talk is the best primer.",[332,372,373],{},"The Kafka 4.3 release notes describe KIP-1035 as \"an internal runtime change, and only relevant for custom StateStore implementations.\" That's accurate, and it undersells it. The change is small in surface area and large in consequence: it fixes a structural weakness that has shaped how Kafka Streams recovers state for years, and it removes the last roadblock in front of transactional state stores.",[375,376,378],"h2",{"id":377},"the-checkpoint-file-and-what-it-was-for","The checkpoint file, and what it was for",[332,380,381,382,384],{},"Before 4.3, a Kafka Streams instance tracked changelog offsets in a small file on disk — one ",[349,383,351],{}," file per task directory, maintained by the Streams engine itself, independently of whatever StateStore you were running.",[332,386,387],{},"The file's job is simple. When a task commits, Streams records the changelog offset that corresponds to the local state it has written: \"the state for this store is caught up to changelog offset N.\" That record is read in two situations:",[339,389,390,396],{},[342,391,392,395],{},[335,393,394],{},"At restore."," On startup, Streams compares the checkpointed offset to the end of the changelog. Match, and the local state is current — no restore needed. Behind, and it replays the changelog from the checkpoint forward.",[342,397,398,401],{},[335,399,400],{},"At rebalance."," The offsets are read to work out how far behind each task's local state is, so the assignor can hand a task to whichever instance holds the warmest copy.",[332,403,404,407],{},[335,405,406],{},"Diagram (placeholder — replace with a KSTD\u002FExcalidraw visual):"," before — state and offsets are written separately.",[409,410,416],"pre",{"className":411,"code":413,"language":414,"meta":415},[412],"language-text","  Task directory on disk\n  ┌─────────────────────────────────────────────┐\n  │   ┌─────────────────┐      ┌──────────────┐  │\n  │   │  RocksDB        │      │ .checkpoint  │  │\n  │   │  (state data)   │      │  offset = N  │  │\n  │   └─────────────────┘      └──────────────┘  │\n  │     written by RocksDB     written by engine │\n  └─────────────────────────────────────────────┘\n   Two stores. Two write paths. No shared commit.\n","text","",[349,417,413],{"__ignoreMap":415},[375,419,421],{"id":420},"where-it-broke","Where it broke",[332,423,424],{},"The weakness is in that last line: two write paths, no shared commit. The state data and the offset that describes it are persisted separately, and nothing makes those two writes atomic.",[332,426,427],{},"On a clean shutdown that's fine — everything flushes in order. On an unclean shutdown it isn't. RocksDB buffers writes in memtables and may not have flushed them to disk when the process dies; the checkpoint file, written on its own path, can end up ahead of the data it points at. After the crash, the offset and the state disagree.",[332,429,430],{},"Kafka Streams handles that disagreement conservatively. Under exactly-once, the on-disk state can be inconsistent with the checkpoint, so on restart it discards the local state and restores the entire changelog from scratch. For a small store that's seconds. For a large one it is not.",[332,432,433],{},"How not-small? Nick Telford put real numbers on it at Kafka Summit London 2024, from Meltwater's production Streams deployment: around 45 state stores, changelogs totalling roughly 8 TiB, restorations spiking to about 10 million records in flight, and restore time scaling roughly linearly with total changelog size. When recovery time is proportional to how much state you hold rather than how much you were mid-processing, growing your state grows your worst-case downtime with it.",[332,435,436,437,440],{},"There's a second, quieter cost. To keep the on-disk state in step with the checkpoint, Streams force-flushes RocksDB memtables on commit — under at-least-once, once at least 10,000 records have been processed. That flush schedule is opaque and not configurable; it overrides whatever you set through ",[349,438,439],{},"RocksDBConfigSetter",", so the engine and your tuning pull in different directions.",[332,442,443],{},"And then the one that blocks everything else: you cannot build transactional state stores on top of this. The KIP says it plainly:",[329,445,446],{},[332,447,448,449,352],{},"this KIP is a hard-dependency of KIP-892, as transactional behaviour requires that we can atomically sync the changelog offsets with their corresponding state, which is not possible while those offsets are tracked in the separate ",[349,450,351],{},[332,452,453],{},"If offsets live in a file the store doesn't control, the store can't commit its data and its offset as one unit — and atomicity is the whole point of a transaction.",[332,455,456,458],{},[335,457,406],{}," the gap on an unclean shutdown.",[409,460,463],{"className":461,"code":462,"language":414,"meta":415},[412],"  Unclean shutdown, mid-commit\n\n  ┌──────────────────────┐     ┌──────────────┐\n  │ RocksDB              │     │ .checkpoint  │\n  │ memtable not flushed │     │ offset = N   │\n  │ → data at offset M   │     │              │\n  └──────────────────────┘     └──────────────┘\n              M    ≠    N   →   they disagree\n                      │\n                      ▼\n       discard local state, restore the\n       entire changelog from scratch\n",[349,464,462],{"__ignoreMap":415},[375,466,468],{"id":467},"kip-892-and-the-missing-piece","KIP-892 and the missing piece",[332,470,471,472,475,476,479,480,483],{},"The transactional-store work that needs this is also Nick's: KIP-892, Transactional StateStores. Its idea is to stop writing straight into RocksDB as records arrive, and instead buffer a task's writes — in a RocksDB ",[349,473,474],{},"WriteBatchWithIndex"," — applying them only when the Kafka transaction commits. Reads during the transaction are served from the buffer, so you still see your own writes, and interactive queries gain an isolation level: ",[349,477,478],{},"READ_UNCOMMITTED"," to see in-flight data, ",[349,481,482],{},"READ_COMMITTED"," to see only what's durable.",[332,485,486],{},"Done right, this means local state never gets ahead of the changelog. Uncommitted writes sit in memory and are simply dropped if the process dies, because they were never committed. Recovery stops depending on total state size.",[332,488,489],{},"But all of it rests on one requirement: committing the buffered data and its changelog offset together, atomically. With offsets stranded in a separate checkpoint file, you can't. So that requirement was lifted out of KIP-892 into its own proposal — \"broken out to provide more focus and make it easier to contribute and review.\" That proposal is KIP-1035, and it had to land first.",[375,491,493],{"id":492},"kip-1035-the-state-store-owns-its-offsets","KIP-1035: the state store owns its offsets",[332,495,496],{},"KIP-1035's move is to make the StateStore responsible for its own changelog offsets, and to persist them atomically with the data. The checkpoint file stops being the source of truth.",[332,498,499,500,503],{},"It adds three methods to the ",[349,501,502],{},"StateStore"," interface, each with a default so existing stores keep compiling:",[409,505,509],{"className":506,"code":507,"language":508,"meta":415,"style":415},"language-java shiki shiki-themes vitesse-light","\u002F\u002F Does this store manage its own offsets? New stores return true.\n@Deprecated\ndefault boolean managesOffsets() {\n    return false;\n}\n\n\u002F\u002F Commit written records. If managesOffsets() is true, the given\n\u002F\u002F offsets are persisted to disk atomically with those records.\ndefault void commit(final Map\u003CTopicPartition, Long> changelogOffsets) {\n    flush();\n}\n\n\u002F\u002F The offset of the most recently committed changelog record for a\n\u002F\u002F partition — read back from the store itself.\ndefault Long committedOffset(final TopicPartition partition) {\n    return null;\n}\n","java",[349,510,511,520,531,551,563,569,576,582,588,632,641,646,651,657,663,685,695],{"__ignoreMap":415},[512,513,516],"span",{"class":514,"line":515},"line",1,[512,517,519],{"class":518},"s8zF2","\u002F\u002F Does this store manage its own offsets? New stores return true.\n",[512,521,523,527],{"class":514,"line":522},2,[512,524,526],{"class":525},"sYZai","@",[512,528,530],{"class":529},"si04Y","Deprecated\n",[512,532,534,538,541,545,548],{"class":514,"line":533},3,[512,535,537],{"class":536},"sbBg2","default",[512,539,540],{"class":529}," boolean",[512,542,544],{"class":543},"sySUi"," managesOffsets",[512,546,547],{"class":525},"()",[512,549,550],{"class":525}," {\n",[512,552,554,557,560],{"class":514,"line":553},4,[512,555,556],{"class":536},"    return",[512,558,559],{"class":536}," false",[512,561,562],{"class":525},";\n",[512,564,566],{"class":514,"line":565},5,[512,567,568],{"class":525},"}\n",[512,570,572],{"class":514,"line":571},6,[512,573,575],{"emptyLinePlaceholder":574},true,"\n",[512,577,579],{"class":514,"line":578},7,[512,580,581],{"class":518},"\u002F\u002F Commit written records. If managesOffsets() is true, the given\n",[512,583,585],{"class":514,"line":584},8,[512,586,587],{"class":518},"\u002F\u002F offsets are persisted to disk atomically with those records.\n",[512,589,591,593,596,599,602,605,609,612,615,618,621,624,627,630],{"class":514,"line":590},9,[512,592,537],{"class":536},[512,594,595],{"class":529}," void",[512,597,598],{"class":543}," commit",[512,600,601],{"class":525},"(",[512,603,604],{"class":529},"final",[512,606,608],{"class":607},"suHK_"," Map",[512,610,611],{"class":529},"\u003C",[512,613,614],{"class":607},"TopicPartition",[512,616,617],{"class":525},",",[512,619,620],{"class":607}," Long",[512,622,623],{"class":529},">",[512,625,626],{"class":607}," changelogOffsets",[512,628,629],{"class":525},")",[512,631,550],{"class":525},[512,633,635,638],{"class":514,"line":634},10,[512,636,637],{"class":543},"    flush",[512,639,640],{"class":525},"();\n",[512,642,644],{"class":514,"line":643},11,[512,645,568],{"class":525},[512,647,649],{"class":514,"line":648},12,[512,650,575],{"emptyLinePlaceholder":574},[512,652,654],{"class":514,"line":653},13,[512,655,656],{"class":518},"\u002F\u002F The offset of the most recently committed changelog record for a\n",[512,658,660],{"class":514,"line":659},14,[512,661,662],{"class":518},"\u002F\u002F partition — read back from the store itself.\n",[512,664,666,668,671,674,676,678,681,683],{"class":514,"line":665},15,[512,667,537],{"class":536},[512,669,670],{"class":607}," Long ",[512,672,673],{"class":543},"committedOffset",[512,675,601],{"class":525},[512,677,604],{"class":529},[512,679,680],{"class":607}," TopicPartition partition",[512,682,629],{"class":525},[512,684,550],{"class":525},[512,686,688,690,693],{"class":514,"line":687},16,[512,689,556],{"class":536},[512,691,692],{"class":536}," null",[512,694,562],{"class":525},[512,696,698],{"class":514,"line":697},17,[512,699,568],{"class":525},[332,701,702,705],{},[349,703,704],{},"managesOffsets()"," is deprecated the moment it ships. It exists only so the ecosystem can migrate; eventually every store is expected to manage its own offsets and the method goes away.",[332,707,708,709,712],{},"The built-in ",[349,710,711],{},"RocksDBStore"," opts in. It keeps its offsets in a dedicated RocksDB column family next to the data, and opens the database with atomic flush enabled — so when RocksDB flushes memtables to disk, the data and the offsets flush together, or not at all. The offset can no longer get ahead of the data it describes, because they're written in the same batch and flushed in the same operation.",[332,714,715,717],{},[335,716,406],{}," after — offsets live in a column family, flushed atomically with the data.",[409,719,722],{"className":720,"code":721,"language":414,"meta":415},[412],"  RocksDB (one database)\n  ┌────────────────────────────────────────────────┐\n  │  ┌──── default CF ────┐  ┌──── _offsets CF ───┐ │\n  │  │   key → value      │  │  changelog:tp → N  │ │\n  │  └────────────────────┘  └────────────────────┘ │\n  │            └──── one WriteBatch ────┘            │\n  │                       │                          │\n  │                       ▼                          │\n  │           atomic flush (both CFs together)       │\n  └────────────────────────────────────────────────┘\n   Data and offset commit as one unit, or not at all.\n",[349,723,721],{"__ignoreMap":415},[332,725,726],{},"Two consequences fall out of that:",[339,728,729,738],{},[342,730,731,734,735,737],{},[335,732,733],{},"Flushing goes back to RocksDB."," Because offsets ride along with the data automatically, Streams no longer forces a flush on commit to keep a separate file in sync. Flush timing is dictated by your RocksDB configuration again — the way ",[349,736,439],{}," always implied it should be.",[342,739,740,743,744,746,747,750,751,754,755,758,759,762,763,766],{},[335,741,742],{},"The side files retire."," The ",[349,745,351],{}," file is replaced; the ",[349,748,749],{},".position"," file used by interactive queries (KIP-796) moves into the same column family. Both migrate automatically on first start under 4.3. ",[349,752,753],{},"flush()"," is deprecated in favour of ",[349,756,757],{},"commit()",", and the ",[349,760,761],{},"flush-*"," metrics give way to ",[349,764,765],{},"commit-*"," ones.",[332,768,769,770,773,774,776],{},"For custom StateStores, nothing breaks. The defaults keep the legacy checkpoint-file behaviour until you opt in by implementing the three methods and returning ",[349,771,772],{},"true"," from ",[349,775,704],{},".",[375,778,780],{"id":779},"what-it-unlocks","What it unlocks",[332,782,783],{},"The headline is consistency: after KIP-1035 the recorded offset can never be ahead of the data it names, on any kind of shutdown. That single guarantee changes recovery.",[339,785,786,792,798],{},[342,787,788,791],{},[335,789,790],{},"No full re-restore after a crash."," With offsets and data consistent, a restart only replays the gap between the store's committed offset and the end of the changelog — the uncommitted tail — not the whole topic. Recovery time tracks how much you were mid-processing, not how much state you hold.",[342,793,794,797],{},[335,795,796],{},"RocksDB tuning means what it says."," Flush scheduling is yours again; the engine isn't forcing flushes behind your back to keep a file in step.",[342,799,800,803],{},[335,801,802],{},"Transactional state stores become buildable."," This is the real prize. With atomic offset-and-data commits in place, KIP-892 can keep local state across restarts instead of rebuilding it, and the isolation levels for interactive queries have a consistent store to read from.",[332,805,806],{},"The release notes were right that it's an internal change. They were modest about what it makes possible.",[375,808,810],{"id":809},"stoatflow-was-built-on-this-from-the-ground-up","StoatFlow was built on this from the ground up",[332,812,813,814,817],{},"StoatFlow took this as a founding decision rather than a migration. Its state engine never had a checkpoint file to retire — changelog offsets have always lived inside RocksDB, in a dedicated ",[349,815,816],{},"_offsets"," column family, flushed atomically with the data through the same atomic-flush mechanism KIP-1035 standardises for Kafka Streams.",[332,819,820,821,823,824,827,828,830,831,833],{},"Building greenfield, and building single-instance, meant we carried none of the legacy. There's no ",[349,822,704],{}," defaulting to ",[349,825,826],{},"false",", no opt-in for custom stores, no automatic migration from ",[349,829,351],{}," and ",[349,832,749],{}," files — store-managed offsets are the only mode. And without a consumer group or rebalancing, the offset story is simpler still: one instance, one set of offsets, fixated at each commit barrier.",[332,835,836],{},"We push the same idea a step further in two places. Commits flow through Flink-style commit barriers rather than per-task Kafka transactions, and we enforce a strict data-before-offsets ordering — an epoch's offset is written only once that epoch's data has landed, so a timeout can leave the offset behind the data but never ahead of it. The payoff is the one KIP-892 is chasing: delta restore in under a second regardless of total state size — reached from a different starting point.",[332,838,839],{},"None of this is our idea. It's Nick Telford's design, adopted early because it was the right one.",[375,841,843],{"id":842},"credit-where-its-due","Credit where it's due",[332,845,846,847,854],{},"KIP-1035 — and the transactional state stores it makes possible — are the work of ",[848,849,853],"a",{"href":850,"rel":851},"https:\u002F\u002Fwww.linkedin.com\u002Fin\u002Fnicholastelford\u002F",[852],"nofollow","Nick Telford",", who proposed, designed, and contributed both KIPs while running Kafka Streams at scale at Meltwater. It is a substantial piece of engineering: a change to a foundational interface, carried through with the backward-compatibility and migration care that a system as widely deployed as Kafka Streams demands.",[332,856,857,858,866],{},"If you want the full story from the source, his Kafka Summit London 2024 talk, ",[848,859,862],{"href":860,"rel":861},"https:\u002F\u002Fwww.confluent.io\u002Fevents\u002Fkafka-summit-london-2024\u002Fimproving-streams-scalability-with-transactional-statestores-kip-892\u002F",[852],[863,864,865],"em",{},"Improving Streams Scalability with Transactional StateStores (KIP-892)",", is the best primer — it's where the Meltwater numbers above come from, and it walks the problem and the design far better than a release note can.",[332,868,869],{},[335,870,871],{},"Read on:",[339,873,874,881,888,894],{},[342,875,876],{},[848,877,880],{"href":878,"rel":879},"https:\u002F\u002Fcwiki.apache.org\u002Fconfluence\u002Fdisplay\u002FKAFKA\u002FKIP-1035%3A+StateStore+managed+changelog+offsets",[852],"KIP-1035: StateStore managed changelog offsets",[342,882,883],{},[848,884,887],{"href":885,"rel":886},"https:\u002F\u002Fcwiki.apache.org\u002Fconfluence\u002Fdisplay\u002FKAFKA\u002FKIP-892%3A+Transactional+Semantics+for+StateStores",[852],"KIP-892: Transactional StateStores",[342,889,890],{},[848,891,893],{"href":860,"rel":892},[852],"Nick Telford — Improving Streams Scalability with Transactional StateStores (Kafka Summit London 2024)",[342,895,896],{},[848,897,900],{"href":898,"rel":899},"https:\u002F\u002Fkafka.apache.org\u002Fblog\u002F2026\u002F05\u002F22\u002Fapache-kafka-4.3.0-release-announcement\u002F",[852],"Apache Kafka 4.3.0 release announcement",[902,903,904],"style",{},"html pre.shiki code .s8zF2, html code.shiki .s8zF2{--shiki-default:#A0ADA0}html pre.shiki code .sYZai, html code.shiki .sYZai{--shiki-default:#999999}html pre.shiki code .si04Y, html code.shiki .si04Y{--shiki-default:#AB5959}html pre.shiki code .sbBg2, html code.shiki .sbBg2{--shiki-default:#1E754F}html pre.shiki code .sySUi, html code.shiki .sySUi{--shiki-default:#59873A}html pre.shiki code .suHK_, html code.shiki .suHK_{--shiki-default:#393A34}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}",{"title":415,"searchDepth":522,"depth":522,"links":906},[907,908,909,910,911,912,913],{"id":377,"depth":522,"text":378},{"id":420,"depth":522,"text":421},{"id":467,"depth":522,"text":468},{"id":492,"depth":522,"text":493},{"id":779,"depth":522,"text":780},{"id":809,"depth":522,"text":810},{"id":842,"depth":522,"text":843},"2026-06-06","Kafka Streams kept each task's changelog offsets in a separate checkpoint file that could drift from the state it described. KIP-1035, in Kafka 4.3, moves them inside the state store — atomic with the data, and the keystone for transactional state stores.","md",{"src":918},"\u002Fassets\u002Fblog\u002Fog\u002FStoatFlow_og_20260517.jpg",{},"\u002Fblog\u002Fkip-1035-state-store-managed-offsets",{"title":316,"description":915},"3.blog\u002F2.kip-1035-state-store-managed-offsets","cgwJ5oR_FH2Sf5A8rfDBQaaYS3EoR_zl565oDf8nIXw",[925,930],{"title":926,"path":927,"stem":928,"description":929,"children":-1},"StoatFlow: Kafka Streams compatible engine built to scale up — not out","\u002Fblog\u002Fintroducing-stoatflow","3.blog\u002F1.introducing-stoatflow","The first alpha is here. Kafka Streams DSL on a single-replica runtime with virtual-thread parallelism — measurably less CPU, memory, and latency on the same hardware.",{"title":931,"path":932,"stem":933,"description":934,"children":-1},"Compiling StoatFlow to a GraalVM native image: G1, PGO, and why JNI beats FFM under AOT","\u002Fblog\u002Fnative-image-g1-pgo-jni-vs-ffm","3.blog\u002F2.native-image-g1-pgo-jni-vs-ffm","Native image is viable for a throughput-first stream processor — but only with Oracle G1 + PGO. The wrong turn (we blamed FFM; it was the GC) and the reversal (FFM is faster on the JVM, slower under AOT) are the most interesting parts.",1783713348615]