[{"data":1,"prerenderedAt":1036},["ShallowReactive",2],{"navigation":3,"\u002Fblog\u002Fkafka-streams-4-4":334,"\u002Fblog\u002Fkafka-streams-4-4-surround":1030},[4,43,91,154,181,224,262,293],{"title":5,"path":6,"stem":7,"children":8,"icon":42},"Getting Started","\u002Fdocs\u002Fgetting-started","1.docs\u002F1.getting-started\u002F1.index",[9,12,17,22,27,32,37],{"title":10,"path":6,"stem":7,"icon":11},"Getting started","i-lucide-flag",{"title":13,"path":14,"stem":15,"icon":16},"Modules overview","\u002Fdocs\u002Fgetting-started\u002Fmodules-overview","1.docs\u002F1.getting-started\u002F2.modules-overview","i-lucide-boxes",{"title":18,"path":19,"stem":20,"icon":21},"Installation","\u002Fdocs\u002Fgetting-started\u002Finstallation","1.docs\u002F1.getting-started\u002F3.installation","i-lucide-download",{"title":23,"path":24,"stem":25,"icon":26},"License configuration","\u002Fdocs\u002Fgetting-started\u002Flicense-configuration","1.docs\u002F1.getting-started\u002F4.license-configuration","i-lucide-key-round",{"title":28,"path":29,"stem":30,"icon":31},"Your first app","\u002Fdocs\u002Fgetting-started\u002Ffirst-app","1.docs\u002F1.getting-started\u002F5.first-app","i-lucide-square-play",{"title":33,"path":34,"stem":35,"icon":36},"Project setup","\u002Fdocs\u002Fgetting-started\u002Fproject-setup","1.docs\u002F1.getting-started\u002F6.project-setup","i-lucide-package",{"title":38,"path":39,"stem":40,"icon":41},"AI assistants","\u002Fdocs\u002Fgetting-started\u002Fai-assistants","1.docs\u002F1.getting-started\u002F7.ai-assistants","i-lucide-bot",false,{"title":44,"path":45,"stem":46,"children":47,"icon":50},"Core Concepts","\u002Fdocs\u002Fconcepts","1.docs\u002F2.concepts\u002F1.index",[48,51,56,61,66,71,76,81,86],{"title":49,"path":45,"stem":46,"icon":50},"Core concepts","i-lucide-book-open",{"title":52,"path":53,"stem":54,"icon":55},"Architecture","\u002Fdocs\u002Fconcepts\u002Farchitecture","1.docs\u002F2.concepts\u002F2.architecture","i-lucide-layers",{"title":57,"path":58,"stem":59,"icon":60},"Exactly-once semantics","\u002Fdocs\u002Fconcepts\u002Fexactly-once","1.docs\u002F2.concepts\u002F3.exactly-once","i-lucide-shield-check",{"title":62,"path":63,"stem":64,"icon":65},"Lanes and parallelism","\u002Fdocs\u002Fconcepts\u002Flanes-and-parallelism","1.docs\u002F2.concepts\u002F4.lanes-and-parallelism","i-lucide-split",{"title":67,"path":68,"stem":69,"icon":70},"State and thread-safety","\u002Fdocs\u002Fconcepts\u002Fstate-and-thread-safety","1.docs\u002F2.concepts\u002F5.state-and-thread-safety","i-lucide-lock",{"title":72,"path":73,"stem":74,"icon":75},"Event time and watermarks","\u002Fdocs\u002Fconcepts\u002Fevent-time-and-watermarks","1.docs\u002F2.concepts\u002F6.event-time-and-watermarks","i-lucide-clock",{"title":77,"path":78,"stem":79,"icon":80},"The configuration model","\u002Fdocs\u002Fconcepts\u002Fconfiguration-model","1.docs\u002F2.concepts\u002F7.configuration-model","i-lucide-sliders-horizontal",{"title":82,"path":83,"stem":84,"icon":85},"The error-handling model","\u002Fdocs\u002Fconcepts\u002Ferror-handling-model","1.docs\u002F2.concepts\u002F8.error-handling-model","i-lucide-triangle-alert",{"title":87,"path":88,"stem":89,"icon":90},"How StoatFlow differs from Kafka Streams","\u002Fdocs\u002Fconcepts\u002Fhow-stoatflow-differs-from-ks","1.docs\u002F2.concepts\u002F9.how-stoatflow-differs-from-ks","i-lucide-arrow-left-right",{"title":92,"path":93,"stem":94,"children":95,"icon":98},"Building Topologies","\u002Fdocs\u002Fbuilding","1.docs\u002F3.building\u002F1.index",[96,99,104,109,114,119,124,129,134,139,144,149],{"title":97,"path":93,"stem":94,"icon":98},"Building topologies","i-lucide-blocks",{"title":100,"path":101,"stem":102,"icon":103},"Error handling and DLQ","\u002Fdocs\u002Fbuilding\u002Ferror-handling-dlq","1.docs\u002F3.building\u002F10.error-handling-dlq","i-lucide-circle-x",{"title":105,"path":106,"stem":107,"icon":108},"State stores","\u002Fdocs\u002Fbuilding\u002Fstate-stores","1.docs\u002F3.building\u002F11.state-stores","i-lucide-database",{"title":110,"path":111,"stem":112,"icon":113},"Testing topologies","\u002Fdocs\u002Fbuilding\u002Ftesting","1.docs\u002F3.building\u002F12.testing","i-lucide-flask-conical",{"title":115,"path":116,"stem":117,"icon":118},"Sources and sinks","\u002Fdocs\u002Fbuilding\u002Fstreams-builder","1.docs\u002F3.building\u002F2.streams-builder","i-lucide-import",{"title":120,"path":121,"stem":122,"icon":123},"KStream and KTable operations","\u002Fdocs\u002Fbuilding\u002Fkstream-ktable","1.docs\u002F3.building\u002F3.kstream-ktable","i-lucide-waypoints",{"title":125,"path":126,"stem":127,"icon":128},"Aggregations","\u002Fdocs\u002Fbuilding\u002Faggregations","1.docs\u002F3.building\u002F4.aggregations","i-lucide-sigma",{"title":130,"path":131,"stem":132,"icon":133},"Windowing","\u002Fdocs\u002Fbuilding\u002Fwindowing","1.docs\u002F3.building\u002F5.windowing","i-lucide-calendar-clock",{"title":135,"path":136,"stem":137,"icon":138},"Joins","\u002Fdocs\u002Fbuilding\u002Fjoins","1.docs\u002F3.building\u002F6.joins","i-lucide-git-merge",{"title":140,"path":141,"stem":142,"icon":143},"The Processor API","\u002Fdocs\u002Fbuilding\u002Fprocessor-api","1.docs\u002F3.building\u002F7.processor-api","i-lucide-cpu",{"title":145,"path":146,"stem":147,"icon":148},"Scheduled sources","\u002Fdocs\u002Fbuilding\u002Fscheduled-sources","1.docs\u002F3.building\u002F8.scheduled-sources","i-lucide-timer",{"title":150,"path":151,"stem":152,"icon":153},"Serdes and Avro","\u002Fdocs\u002Fbuilding\u002Fserdes","1.docs\u002F3.building\u002F9.serdes","i-lucide-binary",{"title":155,"path":156,"stem":157,"children":158,"icon":80},"Configuration","\u002Fdocs\u002Fconfiguration","1.docs\u002F4.configuration\u002F1.index",[159,161,166,171,176],{"title":160,"path":156,"stem":157,"icon":80},"How configuration works",{"title":162,"path":163,"stem":164,"icon":165},"Engine configuration (:core)","\u002Fdocs\u002Fconfiguration\u002Fcore-config","1.docs\u002F4.configuration\u002F2.core-config","i-lucide-settings-2",{"title":167,"path":168,"stem":169,"icon":170},"Runtime configuration (:runtime)","\u002Fdocs\u002Fconfiguration\u002Fruntime-config","1.docs\u002F4.configuration\u002F3.runtime-config","i-lucide-server-cog",{"title":172,"path":173,"stem":174,"icon":175},"Defaults, adaptivity, and presets","\u002Fdocs\u002Fconfiguration\u002Fdefaults-and-presets","1.docs\u002F4.configuration\u002F4.defaults-and-presets","i-lucide-gauge",{"title":177,"path":178,"stem":179,"icon":180},"Kafka client configuration","\u002Fdocs\u002Fconfiguration\u002Fkafka-client-config","1.docs\u002F4.configuration\u002F5.kafka-client-config","i-lucide-plug",{"title":182,"path":183,"stem":184,"children":185,"icon":188},"Running in Production","\u002Fdocs\u002Fruntime","1.docs\u002F5.runtime\u002F1.index",[186,189,194,199,204,209,214,219],{"title":187,"path":183,"stem":184,"icon":188},"The runtime","i-lucide-server",{"title":190,"path":191,"stem":192,"icon":193},"The REST API","\u002Fdocs\u002Fruntime\u002Frest-api","1.docs\u002F5.runtime\u002F2.rest-api","i-lucide-globe",{"title":195,"path":196,"stem":197,"icon":198},"Health checks","\u002Fdocs\u002Fruntime\u002Fhealth-checks","1.docs\u002F5.runtime\u002F3.health-checks","i-lucide-heart-pulse",{"title":200,"path":201,"stem":202,"icon":203},"Metrics","\u002Fdocs\u002Fruntime\u002Fmetrics","1.docs\u002F5.runtime\u002F4.metrics","i-lucide-activity",{"title":205,"path":206,"stem":207,"icon":208},"Pause and resume","\u002Fdocs\u002Fruntime\u002Fpause-unpause","1.docs\u002F5.runtime\u002F5.pause-unpause","i-lucide-pause",{"title":210,"path":211,"stem":212,"icon":213},"Plugins and lifecycle hooks","\u002Fdocs\u002Fruntime\u002Fplugins","1.docs\u002F5.runtime\u002F6.plugins","i-lucide-puzzle",{"title":215,"path":216,"stem":217,"icon":218},"Docker images","\u002Fdocs\u002Fruntime\u002Fdocker","1.docs\u002F5.runtime\u002F7.docker","i-lucide-container",{"title":220,"path":221,"stem":222,"icon":223},"GraalVM native image","\u002Fdocs\u002Fruntime\u002Fnative-image","1.docs\u002F5.runtime\u002F8.native-image","i-lucide-zap",{"title":225,"path":226,"stem":227,"children":228,"icon":231},"Deploying & Operating","\u002Fdocs\u002Foperating","1.docs\u002F6.operating\u002F1.index",[229,232,237,242,247,252,257],{"title":230,"path":226,"stem":227,"icon":231},"Deploying and operating","i-lucide-life-buoy",{"title":233,"path":234,"stem":235,"icon":236},"Running on Kubernetes","\u002Fdocs\u002Foperating\u002Fkubernetes","1.docs\u002F6.operating\u002F2.kubernetes","i-lucide-ship",{"title":238,"path":239,"stem":240,"icon":241},"High availability","\u002Fdocs\u002Foperating\u002Fhigh-availability","1.docs\u002F6.operating\u002F3.high-availability","i-lucide-copy",{"title":243,"path":244,"stem":245,"icon":246},"Liveness and readiness probes","\u002Fdocs\u002Foperating\u002Fprobes","1.docs\u002F6.operating\u002F4.probes","i-lucide-stethoscope",{"title":248,"path":249,"stem":250,"icon":251},"Observability","\u002Fdocs\u002Foperating\u002Fobservability","1.docs\u002F6.operating\u002F5.observability","i-lucide-telescope",{"title":253,"path":254,"stem":255,"icon":256},"Tuning under load","\u002Fdocs\u002Foperating\u002Ftuning","1.docs\u002F6.operating\u002F6.tuning","i-lucide-sliders",{"title":258,"path":259,"stem":260,"icon":261},"Production checklist","\u002Fdocs\u002Foperating\u002Fproduction-checklist","1.docs\u002F6.operating\u002F7.production-checklist","i-lucide-clipboard-check",{"title":263,"path":264,"stem":265,"children":266,"icon":90},"Migrating from Kafka Streams","\u002Fdocs\u002Fmigration","1.docs\u002F7.migration\u002F1.index",[267,268,273,278,283,288],{"title":263,"path":264,"stem":265,"icon":90},{"title":269,"path":270,"stem":271,"icon":272},"Automated port","\u002Fdocs\u002Fmigration\u002Fautomated-port","1.docs\u002F7.migration\u002F2.automated-port","i-lucide-wand-sparkles",{"title":274,"path":275,"stem":276,"icon":277},"Migration without carrying state","\u002Fdocs\u002Fmigration\u002Fwithout-data-migration","1.docs\u002F7.migration\u002F3.without-data-migration","i-lucide-sparkles",{"title":279,"path":280,"stem":281,"icon":282},"Migration carrying state","\u002Fdocs\u002Fmigration\u002Fwith-data-migration","1.docs\u002F7.migration\u002F4.with-data-migration","i-lucide-database-backup",{"title":284,"path":285,"stem":286,"icon":287},"The migration tool","\u002Fdocs\u002Fmigration\u002Fmigration-tool","1.docs\u002F7.migration\u002F5.migration-tool","i-lucide-truck",{"title":289,"path":290,"stem":291,"icon":292},"Reusing your Kafka Streams dashboards","\u002Fdocs\u002Fmigration\u002Freusing-kafka-streams-dashboards","1.docs\u002F7.migration\u002F6.reusing-kafka-streams-dashboards","i-lucide-line-chart",{"title":294,"path":295,"stem":296,"children":297,"icon":299},"Reference","\u002Fdocs\u002Freference","1.docs\u002F8.reference\u002F1.index",[298,300,305,310,315,320,325,329],{"title":294,"path":295,"stem":296,"icon":299},"i-lucide-list",{"title":301,"path":302,"stem":303,"icon":304},"Configuration reference","\u002Fdocs\u002Freference\u002Fconfiguration-reference","1.docs\u002F8.reference\u002F2.configuration-reference","i-lucide-table",{"title":306,"path":307,"stem":308,"icon":309},"REST API reference","\u002Fdocs\u002Freference\u002Frest-api-reference","1.docs\u002F8.reference\u002F3.rest-api-reference","i-lucide-network",{"title":311,"path":312,"stem":313,"icon":314},"Gradle plugin reference","\u002Fdocs\u002Freference\u002Fgradle-plugin-reference","1.docs\u002F8.reference\u002F4.gradle-plugin-reference","i-lucide-box",{"title":316,"path":317,"stem":318,"icon":319},"Maven reference","\u002Fdocs\u002Freference\u002Fmaven-reference","1.docs\u002F8.reference\u002F5.maven-reference","i-simple-icons-apachemaven",{"title":321,"path":322,"stem":323,"icon":324},"Kafka Streams compatibility matrix","\u002Fdocs\u002Freference\u002Fks-compatibility-matrix","1.docs\u002F8.reference\u002F6.ks-compatibility-matrix","i-lucide-table-2",{"title":326,"path":327,"stem":328,"icon":175},"Metrics reference","\u002Fdocs\u002Freference\u002Fmetrics-reference","1.docs\u002F8.reference\u002F7.metrics-reference",{"title":330,"path":331,"stem":332,"icon":333},"Glossary","\u002Fdocs\u002Freference\u002Fglossary","1.docs\u002F8.reference\u002F8.glossary","i-lucide-book-a",{"id":335,"title":336,"authors":337,"badge":343,"body":345,"date":1017,"description":1018,"draft":42,"extension":1019,"image":1020,"meta":1022,"navigation":1023,"ogImage":1024,"path":1025,"seo":1026,"sitemap":1027,"stem":1028,"updated":1024,"__hash__":1029},"posts\u002F3.blog\u002F20.kafka-streams-4-4.md","Kafka Streams 4.4.0: transactional state stores, headers-aware partitioning and queries",[338],{"name":339,"to":340,"avatar":341},"Hartmut Armbruster","https:\u002F\u002Fwww.linkedin.com\u002Fin\u002Fhartmut-co-uk\u002F",{"src":342},"\u002Fassets\u002Fhartmut_armbruster_monochromatic.jpg",{"label":344},"Announcement",{"type":346,"value":347,"toc":1006},"minimark",[348,365,368,433,438,451,474,497,500,513,517,529,555,592,610,614,668,675,679,691,703,717,766,779,783,879,901,905,981,985,995],[349,350,351,352,359,360,364],"p",{},"Kafka Streams 4.4.0 delivers a change that exactly-once users have been waiting years for: opt-in transactional state stores (KIP-892). Turn them on, and an error under exactly-once no longer means wiping local state and rebuilding it from the changelog. The work traces back to 2021, and 4.3 laid the groundwork by moving changelog offsets into the stores themselves (",[353,354,358],"a",{"href":355,"rel":356},"https:\u002F\u002Fcwiki.apache.org\u002Fconfluence\u002Fdisplay\u002FKAFKA\u002FKIP-1035%3A+StateStore+managed+changelog+offsets",[357],"nofollow","KIP-1035","; see our ",[353,361,363],{"href":362},"\u002Fblog\u002Fkip-1035-state-store-managed-offsets","deep dive","). Around it, record headers reach the partitioner and IQv2, a long-standing GlobalKTable joiner bug gets a proper fix, and the Streams Rebalance Protocol gains static membership and broker-side plugins.",[349,366,367],{},"In StoatFlow we already ship the partitioner, joiner, close and config changes; transactional-store parity, IQv2 and the headers-aware queries are planned for 1.0.0. Where StoatFlow behaves differently, or a change does not apply to a single-instance engine, a short note says so.",[369,370,371,377],"blockquote",{},[349,372,373],{},[374,375,376],"strong",{},"TL;DR",[378,379,380,387,398,404,410,427],"ul",{},[381,382,383,386],"li",{},[374,384,385],{},"KIP-892:"," opt-in transactional state stores — no more state wipe after an error under exactly-once — plus an uncommitted-bytes budget and isolation levels for interactive queries.",[381,388,389,392,393,397],{},[374,390,391],{},"KIP-1321 + KIP-1356:"," ",[394,395,396],"code",{},"StreamPartitioner"," and IQv2 become headers-aware. Windowed sinks with a header-sensitive key serialiser can move partitions on upgrade.",[381,399,400,403],{},[374,401,402],{},"KIP-1340:"," stream–GlobalKTable joiners can see the join key, not just the stream key.",[381,405,406,409],{},[374,407,408],{},"Streams Rebalance Protocol (KIP-1071):"," static membership, topology descriptions (KIP-1331) and broker-side custom assignors (KIP-1357).",[381,411,412,392,415,418,419,422,423,426],{},[374,413,414],{},"Smaller changes:",[394,416,417],{},"GroupMembershipOperation.DEFAULT"," (KIP-1284), ",[394,420,421],{},"application.server"," validated at startup (KIP-1245), clearer ",[394,424,425],{},"cleanUp()"," (KIP-1283).",[381,428,429,432],{},[374,430,431],{},"StoatFlow:"," the partitioner, joiner, close and config changes ship today; the KIP-892 APIs, IQv2 and KIP-1356 are planned for 1.0.0.",[434,435,437],"h2",{"id":436},"transactional-state-stores-kip-892","Transactional state stores (KIP-892)",[349,439,440,441,444,445,450],{},"Set ",[394,442,443],{},"enable.transactional.statestores=true"," and state stores buffer their writes until the corresponding changelog transaction commits (",[353,446,449],{"href":447,"rel":448},"https:\u002F\u002Fcwiki.apache.org\u002Fconfluence\u002Fdisplay\u002FKAFKA\u002FKIP-892%3A+Transactional+Semantics+for+StateStores",[357],"KIP-892","). The consequence matters most under exactly-once. Without them, an error there means the store may hold uncommitted writes, so Kafka Streams wipes it and restores it from the changelog before processing can resume — on a large store, the most expensive recovery path there is. With transactional stores, local state stays consistent with the changelog, and under either guarantee only the writes since the last successful commit are replayed.",[349,452,453,454,457,458,461,462,465,466,469,470,473],{},"The buffer is bounded by ",[394,455,456],{},"statestore.uncommitted.max.bytes",": 64 MiB per instance by default, split across the stream threads (and the global thread, when the topology has global stores); ",[394,459,460],{},"-1"," disables the bound. Crossing it triggers an early commit, whatever ",[394,463,464],{},"commit.interval.ms"," says. Each store reports its share through the new ",[394,467,468],{},"uncommitted-bytes"," metric and ",[394,471,472],{},"StateStore#approximateNumUncommittedBytes()",".",[349,475,476,477,480,481,484,485,488,489,492,493,496],{},"Interactive queries gain an isolation level. ",[394,478,479],{},"READ_UNCOMMITTED"," (the default) sees buffered writes; ",[394,482,483],{},"READ_COMMITTED"," sees only what the changelog has committed. ",[394,486,487],{},"default.interactive.query.isolation.level"," sets it for IQv1, and IQv2 can override it per query with ",[394,490,491],{},"StateQueryRequest#withIsolationLevel(...)",". Store views come from the new ",[394,494,495],{},"readOnly(IsolationLevel)"," on the key-value, window, session and versioned store interfaces.",[349,498,499],{},"The flag is off by default, so nothing changes until you opt in. The 4.4 upgrade guide does not cover it yet; the KIP and the Streams config reference are the documentation.",[369,501,502],{},[349,503,504,506,507,509,510,512],{},[374,505,431],{}," every store the DSL selects already buffers each epoch's writes until the commit barrier's transaction commits. The 4.4 API surface — the two config keys, isolation levels, ",[394,508,495],{}," and the ",[394,511,468],{}," gauge — is planned for 1.0.0.",[434,514,516],{"id":515},"headers-end-to-end-kip-1321-kip-1356","Headers, end to end (KIP-1321, KIP-1356)",[349,518,519,520,359,525,528],{},"Headers carry the metadata a payload shouldn't — a schema id, a trace context, a tenant tag — so every place a stream processor drops them is a place lineage breaks. Kafka Streams 4.3 let state stores persist them (",[353,521,524],{"href":522,"rel":523},"https:\u002F\u002Fcwiki.apache.org\u002Fconfluence\u002Fspaces\u002FKAFKA\u002Fpages\u002F406618944\u002FKIP-1271+Allow+to+Store+Record+Headers+in+State+Stores",[357],"KIP-1271",[353,526,363],{"href":527},"\u002Fblog\u002Fkip-1271-record-headers-in-state-stores","). 4.4 closes two more of the gaps.",[349,530,531,392,540,542,543,546,547,550,551,554],{},[374,532,533,534,539],{},"Partitioning (",[353,535,538],{"href":536,"rel":537},"https:\u002F\u002Fcwiki.apache.org\u002Fconfluence\u002Fx\u002FpYwmGQ",[357],"KIP-1321",").",[394,541,396],{}," gains ",[394,544,545],{},"partitions(topic, key, value, headers, numPartitions)",". The old four-argument method is deprecated for removal in 5.0, but it stays the interface's single abstract method, so existing lambdas still compile. The built-in partitioners now pass the record's headers to the key serialiser, and ",[394,548,549],{},"KafkaStreams#queryMetadataForKey"," has matching ",[394,552,553],{},"Headers"," overloads so interactive-query routing agrees with where the key was written.",[349,556,557,565,566,569,570,569,573,576,577,580,581,584,585,588,589,591],{},[374,558,559,560,539],{},"Querying (",[353,561,564],{"href":562,"rel":563},"https:\u002F\u002Fcwiki.apache.org\u002Fconfluence\u002Fspaces\u002FKAFKA\u002Fpages\u002F430408653\u002FKIP-1356+Introduce+IQv2+for+headers-aware+state+stores",[357],"KIP-1356"," Four new IQv2 query types — ",[394,567,568],{},"TimestampedKeyWithHeadersQuery",", ",[394,571,572],{},"TimestampedRangeWithHeadersQuery",[394,574,575],{},"TimestampedWindowKeyWithHeadersQuery"," and ",[394,578,579],{},"TimestampedWindowRangeWithHeadersQuery"," — return headers-aware stores' contents as ",[394,582,583],{},"ReadOnlyRecord","s: key, value, timestamp and read-only headers. ",[394,586,587],{},"Record"," now implements ",[394,590,583],{},". Existing query types against headers-aware stores keep working and simply strip the headers.",[369,593,594],{},[349,595,596,598,599,602,603,605,606,609],{},[374,597,431],{}," the partitioning half ships today — the five-argument ",[394,600,601],{},"partitions",", header-aware serialisation at sinks and in in-memory repartitioning, and the ",[394,604,553],{}," overloads of ",[394,607,608],{},"queryMetadataForKey",". IQv2, and with it the four headers-aware query types, is planned for 1.0.0.",[434,611,613],{"id":612},"the-globalktable-joiner-fix-kip-1340","The GlobalKTable joiner fix (KIP-1340)",[349,615,616,617,620,621,624,625,629,630,633,634,639,640,643,644,647,648,651,652,654,655,658,659,661,662,664,665,667],{},"In a stream–GlobalKTable join, the join key is computed by your ",[394,618,619],{},"KeyValueMapper",", yet ",[394,622,623],{},"ValueJoinerWithKey"," always received the ",[626,627,628],"em",{},"stream"," record's key. The types lined up, so nothing flagged it — the kind of bug that survives code review. 4.4 fixes it with a small API addition: ",[394,631,632],{},"ValueJoinerWithStreamAndMappedKey"," (",[353,635,638],{"href":636,"rel":637},"https:\u002F\u002Fcwiki.apache.org\u002Fconfluence\u002Fdisplay\u002FKAFKA\u002FKIP-1340%3A+Expose+Both+Mapped+Key+and+Stream+Key+in+Streams-GlobalKTable+Joins",[357],"KIP-1340","), whose ",[394,641,642],{},"apply(streamKey, mappedKey, streamValue, tableValue)"," gets both keys, through four new ",[394,645,646],{},"join","\u002F",[394,649,650],{},"leftJoin"," overloads. The ",[394,653,623],{}," global-table overloads are deprecated; their behaviour is unchanged. In a left join, a mapper that returns ",[394,656,657],{},"null"," still emits: the joiner sees a ",[394,660,657],{}," mapped key and a ",[394,663,657],{}," table value. Stream records with a ",[394,666,657],{}," value are still dropped.",[369,669,670],{},[349,671,672,674],{},[374,673,431],{}," the same fix and the same deprecations, plus Kotlin-lambda overloads.",[434,676,678],{"id":677},"the-streams-rebalance-protocol-kip-1071-kip-1331-kip-1357","The Streams Rebalance Protocol (KIP-1071, KIP-1331, KIP-1357)",[349,680,681,682,569,687,690],{},"Some context first. Kafka 4.0 made ",[353,683,686],{"href":684,"rel":685},"https:\u002F\u002Fcwiki.apache.org\u002Fconfluence\u002Fdisplay\u002FKAFKA\u002FKIP-848%3A+The+Next+Generation+of+the+Consumer+Rebalance+Protocol",[357],"KIP-848",[626,688,689],{},"the Next Generation of the Consumer Rebalance Protocol",", generally available. It moves group coordination to the broker and says goodbye to \"stop-the-world\" rebalances: instead of every member pausing while the group renegotiates, the coordinator reconciles assignments incrementally and members keep working on the partitions they retain.",[349,692,693,694,699,700,473],{},"The Streams Rebalance Protocol (",[353,695,698],{"href":696,"rel":697},"https:\u002F\u002Fcwiki.apache.org\u002Fconfluence\u002Fdisplay\u002FKAFKA\u002FKIP-1071%3A+Streams+Rebalance+Protocol",[357],"KIP-1071",") builds on it and brings the same model to Kafka Streams. Task assignment, internal-topic creation and the topology metadata the group needs all move from the client into the group coordinator, which makes Streams tasks a first-class citizen of the Kafka protocol. It arrived as early access in 4.1, became generally available for a core feature set in 4.2, and is enabled with ",[394,701,702],{},"group.protocol=streams",[349,704,705,706,709,710,713,714,716],{},"4.4 fills in more of the picture. Static membership arrives: ",[394,707,708],{},"group.instance.id"," now works with the Streams Rebalance Protocol, so a restarting instance can keep its tasks instead of triggering a reassignment. It needs 4.4 on both clients and brokers — an older broker rejects it with ",[394,711,712],{},"INVALID_REQUEST",", which shuts the client down, so remove ",[394,715,708],{}," before downgrading a broker. For topologies without persistent state stores, Kafka Streams generates a new process ID on every restart, which triggers a reassignment and largely cancels the benefit.",[349,718,719,720,725,726,729,730,733,734,737,738,741,742,745,746,749,750,755,756,745,759,762,763,473],{},"Two new KIPs extend the protocol on the operations side. ",[353,721,724],{"href":722,"rel":723},"https:\u002F\u002Fcwiki.apache.org\u002Fconfluence\u002Fdisplay\u002FKAFKA\u002FKIP-1331%3A+Streams+Group+Topology+Description+Plugin",[357],"KIP-1331"," lets clients push their topology description to the group coordinator, where a pluggable backend stores it — so an operator can see what a group runs without access to its source code. ",[394,727,728],{},"Admin#describeStreamsGroups"," returns it with ",[394,731,732],{},"DescribeStreamsGroupsOptions#includeTopologyDescription(true)",", as does ",[394,735,736],{},"kafka-streams-groups.sh --describe --topology",". Nothing is pushed until the broker sets ",[394,739,740],{},"group.streams.topology.description.plugin.class","; clients can opt out with ",[394,743,744],{},"topology.description.push.enabled"," (default ",[394,747,748],{},"true","). ",[353,751,754],{"href":752,"rel":753},"https:\u002F\u002Fcwiki.apache.org\u002Fconfluence\u002Fspaces\u002FKAFKA\u002Fpages\u002F430408758\u002FKIP-1357+Add+broker+side+custom+assignors+for+streams+groups",[357],"KIP-1357"," makes the broker-side task assignor pluggable: the broker registers assignors in ",[394,757,758],{},"group.streams.assignors",[394,760,761],{},"sticky","), and a group picks one with ",[394,764,765],{},"streams.assignor.name",[369,767,768],{},[349,769,770,772,773,775,776,473],{},[374,771,431],{}," not applicable, by design. A StoatFlow application is a single instance in a classic consumer group, so there are no tasks to assign and no streams group to push a topology to. ",[394,774,744],{}," is accepted and ignored; the same description is always available from ",[394,777,778],{},"GET \u002Ftopology\u002Fks",[434,780,782],{"id":781},"smaller-changes","Smaller changes",[378,784,785,808,829,844,863],{},[381,786,787,796,797,800,801,576,804,807],{},[374,788,789,633,791,539],{},[394,790,417],{},[353,792,795],{"href":793,"rel":794},"https:\u002F\u002Fcwiki.apache.org\u002Fconfluence\u002Fspaces\u002FKAFKA\u002Fpages\u002F406621398\u002FKIP-1284+Introduce+CloseOptions.DEFAULT+for+Kafka+Streams",[357],"KIP-1284"," A new default for ",[394,798,799],{},"CloseOptions"," adapts to the protocol: under the classic protocol the member stays in the group; under the Streams Rebalance Protocol dynamic members leave and static members stay until the session timeout. ",[394,802,803],{},"close()",[394,805,806],{},"close(Duration)"," now use it.",[381,809,810,820,821,824,825,828],{},[374,811,812,814,815,539],{},[394,813,421],{}," validated at startup (",[353,816,819],{"href":817,"rel":818},"https:\u002F\u002Fcwiki.apache.org\u002Fconfluence\u002Fdisplay\u002FKAFKA\u002FKIP-1245%3A+Enforce+%27application.server%27+%3Cserver%3E%3A%3Cport%3E+format+at+config+level",[357],"KIP-1245"," A malformed value now fails with a ",[394,822,823],{},"ConfigException"," when ",[394,826,827],{},"StreamsConfig"," is created, not later during assignment.",[381,830,831,392,841,843],{},[374,832,833,835,836,539],{},[394,834,425],{}," semantics (",[353,837,840],{"href":838,"rel":839},"https:\u002F\u002Fcwiki.apache.org\u002Fconfluence\u002Fdisplay\u002FKAFKA\u002FKIP-1283%3A+Clarify+KafkaStreams+cleanUp+semantics+to+preserve+process+metadata+and+state+directory+lock+file",[357],"KIP-1283",[394,842,425],{}," may leave the state directory behind when it holds only process metadata and the lock file. That is by design, and no longer logs a warning.",[381,845,846,854,855,858,859,862],{},[374,847,848,849,539],{},"Async bootstrap DNS forced off (",[353,850,853],{"href":851,"rel":852},"https:\u002F\u002Fcwiki.apache.org\u002Fconfluence\u002Fdisplay\u002FKAFKA\u002FKIP-909%3A+DNS+Resolution+Failure+Should+Not+Fail+the+Clients",[357],"KIP-909"," Kafka Streams does not support the asynchronous bootstrap resolution mode yet: it forces ",[394,856,857],{},"bootstrap.resolve.timeout.ms"," to ",[394,860,861],{},"0"," for every client it creates and ignores a positive value with a warning.",[381,864,865,392,868,871,872,875,876,473],{},[374,866,867],{},"Metrics and upgrade.",[394,869,870],{},"record-e2e-latency-*"," at terminal nodes now measures to the end of processing; every record in a batch gets the batch's completion time, so read the values as upper bounds. ",[394,873,874],{},"upgrade.from"," accepts ",[394,877,878],{},"4.3",[369,880,881],{},[349,882,883,392,885,887,888,891,892,894,895,897,898,900],{},[374,884,431],{},[394,886,417],{}," resolves to the ",[394,889,890],{},"leaveGroupOnClose"," setting, which defaults to staying in the group, matching the classic protocol. ",[394,893,421],{}," is validated when the config is built, and ",[394,896,425],{}," always removes the whole state directory — StoatFlow keeps no process-metadata file there. Its end-to-end latency already included processing time, and ",[394,899,874],{}," is accepted and ignored.",[434,902,904],{"id":903},"what-to-watch-when-upgrading","What to watch when upgrading",[378,906,907,920,942,960,972],{},[381,908,909,392,912,915,916,919],{},[374,910,911],{},"Windowed sinks with header-sensitive key serialisers can move partitions.",[394,913,914],{},"WindowedStreamPartitioner"," now passes the record's headers to the key serialiser, where 4.3 passed empty headers. If your serialiser overrides ",[394,917,918],{},"serialize(topic, headers, data)"," and its output depends on the headers, windowed records may land on different partitions than before the upgrade, which breaks co-partitioning with existing data. Wrap the serialiser, or supply a partitioner that ignores headers. Other sinks already serialised the key with headers in 4.3.",[381,921,922,934,935,937,938,941],{},[374,923,924,925,928,929,539],{},"Processors sharing a ",[394,926,927],{},"StoreBuilder"," instance are regrouped (",[353,930,933],{"href":931,"rel":932},"https:\u002F\u002Fissues.apache.org\u002Fjira\u002Fbrowse\u002FKAFKA-20464",[357],"KAFKA-20464"," Processors that return the same ",[394,936,927],{}," instance from ",[394,939,940],{},"ConnectedStoreProvider#stores()"," now land in one subtopology. Task IDs change, so state restores from the changelog on the first start after the upgrade — plan for the restore time. Linking processors to a store by name is unaffected.",[381,943,944,950,951,953,954,956,957,959],{},[374,945,946,949],{},[394,947,948],{},"REMAIN_IN_GROUP"," now means it."," Under the Streams Rebalance Protocol, 4.3 members left the group on every close, whatever ",[394,952,799],{}," said. In 4.4 ",[394,955,948],{}," is honoured, so a member closed with it keeps its tasks until the session timeout. Dynamic members still leave on a plain ",[394,958,803],{},"; static members stay.",[381,961,962,965,966,968,969,971],{},[374,963,964],{},"Deprecation warnings on GlobalKTable joins."," Move ",[394,967,623],{}," joiners to ",[394,970,632],{},"; ignoring the mapped key keeps today's behaviour.",[381,973,974,977,978,980],{},[374,975,976],{},"Transactional state stores are opt-in."," Enable them deliberately and size ",[394,979,456],{}," for your write volume per commit interval.",[434,982,984],{"id":983},"worth-the-upgrade","Worth the upgrade",[349,986,987,988,991,992,994],{},"4.4.0 is a substantial Kafka Streams release, and a good one. If you run exactly-once with large stores, transactional state stores alone change what a failure costs you: try ",[394,989,990],{},"enable.transactional.statestores"," on a staging copy first and watch the ",[394,993,468],{}," gauge against your commit interval. Everyone else gets a partitioner and IQv2 that see record headers, a joiner that sees the key it joined on, and a rebalance protocol that keeps getting better. Congratulations to the Apache Kafka community on the release.",[349,996,997,998,1003,1004,473],{},"The ",[353,999,1002],{"href":1000,"rel":1001},"https:\u002F\u002Fkafka.apache.org\u002Fdocumentation\u002Fstreams\u002Fupgrade-guide\u002F",[357],"Kafka Streams upgrade guide"," and the KIPs linked above carry the detail. For the method-by-method status in StoatFlow, see the ",[353,1005,321],{"href":322},{"title":1007,"searchDepth":1008,"depth":1008,"links":1009},"",2,[1010,1011,1012,1013,1014,1015,1016],{"id":436,"depth":1008,"text":437},{"id":515,"depth":1008,"text":516},{"id":612,"depth":1008,"text":613},{"id":677,"depth":1008,"text":678},{"id":781,"depth":1008,"text":782},{"id":903,"depth":1008,"text":904},{"id":983,"depth":1008,"text":984},"2026-10-13","What Apache Kafka 4.4.0 brings to Kafka Streams: opt-in transactional state stores (KIP-892), headers in the partitioner and IQv2, the GlobalKTable joiner fix, and static membership and plugins for the Streams Rebalance Protocol — plus where StoatFlow stands.","md",{"src":1021},"\u002Fassets\u002Fblog\u002Fog\u002Fkafka-streams-4-4.png",{},true,null,"\u002Fblog\u002Fkafka-streams-4-4",{"title":336,"description":1018},{"loc":1025},"3.blog\u002F20.kafka-streams-4-4","vFh8QlY0UEpW9Ajme0cwkvGhavI0QyXIQp-6VVRotYU",[1024,1031],{"title":1032,"path":1033,"stem":1034,"description":1035,"children":-1},"Three clocks: stream time, Flink watermarks, and the one watermark StoatFlow computes","\u002Fblog\u002Fthree-clocks-stream-time-vs-watermarks","3.blog\u002F15.three-clocks-stream-time-vs-watermarks","Kafka Streams, Flink and StoatFlow answer \"is this window done?\" with three different clocks. StoatFlow runs two of them: it accepts a late record the way Kafka Streams does, on the operator's stream time, and closes the window the way Flink does, on a single global watermark. Part 1 of a four-part series on event time — including two corrections to a claim our own compatibility matrix made about grace.",1791929251800]