[{"data":1,"prerenderedAt":2184},["ShallowReactive",2],{"navigation":3,"\u002Fblog\u002Ftags\u002Fengineering":314},[4,38,86,149,176,219,257,273],{"title":5,"path":6,"stem":7,"children":8,"icon":37},"Getting Started","\u002Fdocs\u002Fgetting-started","1.docs\u002F1.getting-started\u002F1.index",[9,12,17,22,27,32],{"title":10,"path":6,"stem":7,"icon":11},"Getting started","i-lucide-flag",{"title":13,"path":14,"stem":15,"icon":16},"Modules overview","\u002Fdocs\u002Fgetting-started\u002Fmodules-overview","1.docs\u002F1.getting-started\u002F2.modules-overview","i-lucide-boxes",{"title":18,"path":19,"stem":20,"icon":21},"Installation","\u002Fdocs\u002Fgetting-started\u002Finstallation","1.docs\u002F1.getting-started\u002F3.installation","i-lucide-download",{"title":23,"path":24,"stem":25,"icon":26},"License configuration","\u002Fdocs\u002Fgetting-started\u002Flicense-configuration","1.docs\u002F1.getting-started\u002F4.license-configuration","i-lucide-key-round",{"title":28,"path":29,"stem":30,"icon":31},"Your first app","\u002Fdocs\u002Fgetting-started\u002Ffirst-app","1.docs\u002F1.getting-started\u002F5.first-app","i-lucide-square-play",{"title":33,"path":34,"stem":35,"icon":36},"Project setup","\u002Fdocs\u002Fgetting-started\u002Fproject-setup","1.docs\u002F1.getting-started\u002F6.project-setup","i-lucide-package",false,{"title":39,"path":40,"stem":41,"children":42,"icon":45},"Core Concepts","\u002Fdocs\u002Fconcepts","1.docs\u002F2.concepts\u002F1.index",[43,46,51,56,61,66,71,76,81],{"title":44,"path":40,"stem":41,"icon":45},"Core concepts","i-lucide-book-open",{"title":47,"path":48,"stem":49,"icon":50},"Architecture","\u002Fdocs\u002Fconcepts\u002Farchitecture","1.docs\u002F2.concepts\u002F2.architecture","i-lucide-layers",{"title":52,"path":53,"stem":54,"icon":55},"Exactly-once semantics","\u002Fdocs\u002Fconcepts\u002Fexactly-once","1.docs\u002F2.concepts\u002F3.exactly-once","i-lucide-shield-check",{"title":57,"path":58,"stem":59,"icon":60},"Lanes and parallelism","\u002Fdocs\u002Fconcepts\u002Flanes-and-parallelism","1.docs\u002F2.concepts\u002F4.lanes-and-parallelism","i-lucide-split",{"title":62,"path":63,"stem":64,"icon":65},"State and thread-safety","\u002Fdocs\u002Fconcepts\u002Fstate-and-thread-safety","1.docs\u002F2.concepts\u002F5.state-and-thread-safety","i-lucide-lock",{"title":67,"path":68,"stem":69,"icon":70},"Event time and watermarks","\u002Fdocs\u002Fconcepts\u002Fevent-time-and-watermarks","1.docs\u002F2.concepts\u002F6.event-time-and-watermarks","i-lucide-clock",{"title":72,"path":73,"stem":74,"icon":75},"The configuration model","\u002Fdocs\u002Fconcepts\u002Fconfiguration-model","1.docs\u002F2.concepts\u002F7.configuration-model","i-lucide-sliders-horizontal",{"title":77,"path":78,"stem":79,"icon":80},"The error-handling model","\u002Fdocs\u002Fconcepts\u002Ferror-handling-model","1.docs\u002F2.concepts\u002F8.error-handling-model","i-lucide-triangle-alert",{"title":82,"path":83,"stem":84,"icon":85},"How StoatFlow differs from Kafka Streams","\u002Fdocs\u002Fconcepts\u002Fhow-stoatflow-differs-from-ks","1.docs\u002F2.concepts\u002F9.how-stoatflow-differs-from-ks","i-lucide-arrow-left-right",{"title":87,"path":88,"stem":89,"children":90,"icon":93},"Building Topologies","\u002Fdocs\u002Fbuilding","1.docs\u002F3.building\u002F1.index",[91,94,99,104,109,114,119,124,129,134,139,144],{"title":92,"path":88,"stem":89,"icon":93},"Building topologies","i-lucide-blocks",{"title":95,"path":96,"stem":97,"icon":98},"Error handling and DLQ","\u002Fdocs\u002Fbuilding\u002Ferror-handling-dlq","1.docs\u002F3.building\u002F10.error-handling-dlq","i-lucide-circle-x",{"title":100,"path":101,"stem":102,"icon":103},"State stores","\u002Fdocs\u002Fbuilding\u002Fstate-stores","1.docs\u002F3.building\u002F11.state-stores","i-lucide-database",{"title":105,"path":106,"stem":107,"icon":108},"Testing topologies","\u002Fdocs\u002Fbuilding\u002Ftesting","1.docs\u002F3.building\u002F12.testing","i-lucide-flask-conical",{"title":110,"path":111,"stem":112,"icon":113},"Sources and sinks","\u002Fdocs\u002Fbuilding\u002Fstreams-builder","1.docs\u002F3.building\u002F2.streams-builder","i-lucide-import",{"title":115,"path":116,"stem":117,"icon":118},"KStream and KTable operations","\u002Fdocs\u002Fbuilding\u002Fkstream-ktable","1.docs\u002F3.building\u002F3.kstream-ktable","i-lucide-waypoints",{"title":120,"path":121,"stem":122,"icon":123},"Aggregations","\u002Fdocs\u002Fbuilding\u002Faggregations","1.docs\u002F3.building\u002F4.aggregations","i-lucide-sigma",{"title":125,"path":126,"stem":127,"icon":128},"Windowing","\u002Fdocs\u002Fbuilding\u002Fwindowing","1.docs\u002F3.building\u002F5.windowing","i-lucide-calendar-clock",{"title":130,"path":131,"stem":132,"icon":133},"Joins","\u002Fdocs\u002Fbuilding\u002Fjoins","1.docs\u002F3.building\u002F6.joins","i-lucide-git-merge",{"title":135,"path":136,"stem":137,"icon":138},"The Processor API","\u002Fdocs\u002Fbuilding\u002Fprocessor-api","1.docs\u002F3.building\u002F7.processor-api","i-lucide-cpu",{"title":140,"path":141,"stem":142,"icon":143},"Scheduled sources","\u002Fdocs\u002Fbuilding\u002Fscheduled-sources","1.docs\u002F3.building\u002F8.scheduled-sources","i-lucide-timer",{"title":145,"path":146,"stem":147,"icon":148},"Serdes and Avro","\u002Fdocs\u002Fbuilding\u002Fserdes","1.docs\u002F3.building\u002F9.serdes","i-lucide-binary",{"title":150,"path":151,"stem":152,"children":153,"icon":75},"Configuration","\u002Fdocs\u002Fconfiguration","1.docs\u002F4.configuration\u002F1.index",[154,156,161,166,171],{"title":155,"path":151,"stem":152,"icon":75},"How configuration works",{"title":157,"path":158,"stem":159,"icon":160},"Engine configuration (:core)","\u002Fdocs\u002Fconfiguration\u002Fcore-config","1.docs\u002F4.configuration\u002F2.core-config","i-lucide-settings-2",{"title":162,"path":163,"stem":164,"icon":165},"Runtime configuration (:runtime)","\u002Fdocs\u002Fconfiguration\u002Fruntime-config","1.docs\u002F4.configuration\u002F3.runtime-config","i-lucide-server-cog",{"title":167,"path":168,"stem":169,"icon":170},"Defaults, adaptivity, and presets","\u002Fdocs\u002Fconfiguration\u002Fdefaults-and-presets","1.docs\u002F4.configuration\u002F4.defaults-and-presets","i-lucide-gauge",{"title":172,"path":173,"stem":174,"icon":175},"Kafka client configuration","\u002Fdocs\u002Fconfiguration\u002Fkafka-client-config","1.docs\u002F4.configuration\u002F5.kafka-client-config","i-lucide-plug",{"title":177,"path":178,"stem":179,"children":180,"icon":183},"Running in Production","\u002Fdocs\u002Fruntime","1.docs\u002F5.runtime\u002F1.index",[181,184,189,194,199,204,209,214],{"title":182,"path":178,"stem":179,"icon":183},"The runtime","i-lucide-server",{"title":185,"path":186,"stem":187,"icon":188},"The REST API","\u002Fdocs\u002Fruntime\u002Frest-api","1.docs\u002F5.runtime\u002F2.rest-api","i-lucide-globe",{"title":190,"path":191,"stem":192,"icon":193},"Health checks","\u002Fdocs\u002Fruntime\u002Fhealth-checks","1.docs\u002F5.runtime\u002F3.health-checks","i-lucide-heart-pulse",{"title":195,"path":196,"stem":197,"icon":198},"Metrics","\u002Fdocs\u002Fruntime\u002Fmetrics","1.docs\u002F5.runtime\u002F4.metrics","i-lucide-activity",{"title":200,"path":201,"stem":202,"icon":203},"Pause and resume","\u002Fdocs\u002Fruntime\u002Fpause-unpause","1.docs\u002F5.runtime\u002F5.pause-unpause","i-lucide-pause",{"title":205,"path":206,"stem":207,"icon":208},"Plugins and lifecycle hooks","\u002Fdocs\u002Fruntime\u002Fplugins","1.docs\u002F5.runtime\u002F6.plugins","i-lucide-puzzle",{"title":210,"path":211,"stem":212,"icon":213},"Docker images","\u002Fdocs\u002Fruntime\u002Fdocker","1.docs\u002F5.runtime\u002F7.docker","i-lucide-container",{"title":215,"path":216,"stem":217,"icon":218},"GraalVM native image","\u002Fdocs\u002Fruntime\u002Fnative-image","1.docs\u002F5.runtime\u002F8.native-image","i-lucide-zap",{"title":220,"path":221,"stem":222,"children":223,"icon":226},"Deploying & Operating","\u002Fdocs\u002Foperating","1.docs\u002F6.operating\u002F1.index",[224,227,232,237,242,247,252],{"title":225,"path":221,"stem":222,"icon":226},"Deploying and operating","i-lucide-life-buoy",{"title":228,"path":229,"stem":230,"icon":231},"Running on Kubernetes","\u002Fdocs\u002Foperating\u002Fkubernetes","1.docs\u002F6.operating\u002F2.kubernetes","i-lucide-ship",{"title":233,"path":234,"stem":235,"icon":236},"High availability","\u002Fdocs\u002Foperating\u002Fhigh-availability","1.docs\u002F6.operating\u002F3.high-availability","i-lucide-copy",{"title":238,"path":239,"stem":240,"icon":241},"Liveness and readiness probes","\u002Fdocs\u002Foperating\u002Fprobes","1.docs\u002F6.operating\u002F4.probes","i-lucide-stethoscope",{"title":243,"path":244,"stem":245,"icon":246},"Observability","\u002Fdocs\u002Foperating\u002Fobservability","1.docs\u002F6.operating\u002F5.observability","i-lucide-telescope",{"title":248,"path":249,"stem":250,"icon":251},"Tuning under load","\u002Fdocs\u002Foperating\u002Ftuning","1.docs\u002F6.operating\u002F6.tuning","i-lucide-sliders",{"title":253,"path":254,"stem":255,"icon":256},"Production checklist","\u002Fdocs\u002Foperating\u002Fproduction-checklist","1.docs\u002F6.operating\u002F7.production-checklist","i-lucide-clipboard-check",{"title":258,"path":259,"stem":260,"children":261,"icon":85},"Migrating from Kafka Streams","\u002Fdocs\u002Fmigration","1.docs\u002F7.migration\u002F1.index",[262,263,268],{"title":258,"path":259,"stem":260,"icon":85},{"title":264,"path":265,"stem":266,"icon":267},"Migration without carrying state","\u002Fdocs\u002Fmigration\u002Fwithout-data-migration","1.docs\u002F7.migration\u002F2.without-data-migration","i-lucide-sparkles",{"title":269,"path":270,"stem":271,"icon":272},"Migration carrying state","\u002Fdocs\u002Fmigration\u002Fwith-data-migration","1.docs\u002F7.migration\u002F3.with-data-migration","i-lucide-database-backup",{"title":274,"path":275,"stem":276,"children":277,"icon":279},"Reference","\u002Fdocs\u002Freference","1.docs\u002F8.reference\u002F1.index",[278,280,285,290,295,300,305,309],{"title":274,"path":275,"stem":276,"icon":279},"i-lucide-list",{"title":281,"path":282,"stem":283,"icon":284},"Configuration reference","\u002Fdocs\u002Freference\u002Fconfiguration-reference","1.docs\u002F8.reference\u002F2.configuration-reference","i-lucide-table",{"title":286,"path":287,"stem":288,"icon":289},"REST API reference","\u002Fdocs\u002Freference\u002Frest-api-reference","1.docs\u002F8.reference\u002F3.rest-api-reference","i-lucide-network",{"title":291,"path":292,"stem":293,"icon":294},"Gradle plugin reference","\u002Fdocs\u002Freference\u002Fgradle-plugin-reference","1.docs\u002F8.reference\u002F4.gradle-plugin-reference","i-lucide-box",{"title":296,"path":297,"stem":298,"icon":299},"Maven reference","\u002Fdocs\u002Freference\u002Fmaven-reference","1.docs\u002F8.reference\u002F5.maven-reference","i-simple-icons-apachemaven",{"title":301,"path":302,"stem":303,"icon":304},"Kafka Streams compatibility matrix","\u002Fdocs\u002Freference\u002Fks-compatibility-matrix","1.docs\u002F8.reference\u002F6.ks-compatibility-matrix","i-lucide-table-2",{"title":306,"path":307,"stem":308,"icon":170},"Metrics reference","\u002Fdocs\u002Freference\u002Fmetrics-reference","1.docs\u002F8.reference\u002F7.metrics-reference",{"title":310,"path":311,"stem":312,"icon":313},"Glossary","\u002Fdocs\u002Freference\u002Fglossary","1.docs\u002F8.reference\u002F8.glossary","i-lucide-book-a",[315],{"id":316,"title":317,"authors":318,"badge":324,"body":326,"date":2173,"description":2174,"draft":37,"extension":2175,"image":2176,"meta":2178,"navigation":2179,"path":2180,"seo":2181,"stem":2182,"__hash__":2183},"posts\u002F3.blog\u002F2.native-image-g1-pgo-jni-vs-ffm.md","Compiling StoatFlow to a GraalVM native image: G1, PGO, and why JNI beats FFM under AOT",[319],{"name":320,"to":321,"avatar":322},"Hartmut Armbruster","https:\u002F\u002Fwww.linkedin.com\u002Fin\u002Fhartmutarmbruster",{"src":323},"\u002Fassets\u002Fhartmut_armbruster_monochromatic.jpg",{"label":325},"Engineering",{"type":327,"value":328,"toc":2145},"minimark",[329,412,419,424,427,479,486,489,507,514,518,525,532,535,596,607,614,619,623,634,641,700,707,714,717,723,727,737,743,771,774,819,829,833,836,850,863,866,940,954,961,974,980,984,987,997,1012,1033,1043,1047,1054,1059,1085,1099,1103,1134,1139,1143,1146,1190,1193,1197,1215,1218,1250,1267,1271,1274,1305,1322,1326,1336,1339,1380,1390,1394,1404,1408,1559,1566,1570,1709,1718,1722,1804,1810,1814,1820,1942,1964,1971,1975,1990,1993,1997,2022,2039,2049,2086,2090,2100,2139],[330,331,332,339],"blockquote",{},[333,334,335],"p",{},[336,337,338],"strong",{},"TL;DR",[340,341,342,353,359,377,392,406],"ul",{},[343,344,345,348,349,352],"li",{},[336,346,347],{},"Native image is viable today"," — but only with ",[336,350,351],{},"Oracle GraalVM + G1 + PGO",". GraalVM Community Edition ships Serial GC only, and Serial GC can't carry an allocation-heavy stateful workload.",[343,354,355,358],{},[336,356,357],{},"We blamed FFM; it was the GC."," A stateful app that \"couldn't keep up\" under native looked like a RocksDB FFM problem. Switching Serial → G1 fixed it outright, to JVM parity. The bottleneck was the garbage collector, not the foreign-function path.",[343,360,361,364,365,369,370,373,374],{},[336,362,363],{},"JNI beats FFM under AOT"," — the reverse of the JVM. On HotSpot the JIT intrinsifies FFM's ",[366,367,368],"code",{},"critical(false)"," downcalls; SubstrateVM has no JIT, so a plain JNI call ends up cheaper. This drove a new ",[366,371,372],{},"AUTO"," backend default: ",[336,375,376],{},"FFM on the JVM, JNI under native image.",[343,378,379,382,383,386,387,391],{},[336,380,381],{},"The cost, across all four apps:"," CPU within ",[336,384,385],{},"−4% to +10%"," of the JVM (the Avro stateful app is ",[388,389,390],"em",{},"cheaper","), latency equal-or-better everywhere, after PGO. Down from +42% \u002F +75% on the Serial floor.",[343,393,394,397,398,401,402,405],{},[336,395,396],{},"Memory is tunable, not free."," Serial native is tiny (78 MB RSS for a stateless app vs 496 MB on the JVM). G1 sizes its heap from ",[366,399,400],{},"-Xmx"," and sprawls unless you pin it; the stateful app still lands 23% ",[388,403,404],{},"below"," the JVM.",[343,407,408,411],{},[336,409,410],{},"The serde \u002F security surface is still being widened"," — Avro + Schema Registry, SASL, OAuth, mTLS, and non-zstd codecs all need native metadata we haven't fully mapped yet.",[333,413,414,415,418],{},"We've been building and running StoatFlow applications as GraalVM native images. For a throughput-first stream processor the interesting question isn't the one native image is usually sold on — cold-start and footprint — it's whether steady-state CPU and tail latency hold up under sustained load. They do: with Oracle GraalVM, G1, and Profile-Guided Optimisation, the four apps we measured run within ",[336,416,417],{},"−4% to +10% CPU"," of the JVM, latency equal-or-better everywhere — after one wrong turn worth the telling.",[420,421,423],"h2",{"id":422},"where-we-started","Where we started",[333,425,426],{},"A StoatFlow app doesn't begin from a blank GraalVM configuration. The framework already bundles native-image support, discovered automatically the moment an app depends on it:",[340,428,429,445,453],{},[343,430,431,436,437,440,441,444],{},[336,432,433],{},[366,434,435],{},":core"," ships the metadata for everything the engine touches — the Kafka serdes, assignors and partitioners, and the RocksDB API behind both backends: the JNI bindings ",[388,438,439],{},"and"," the FFM foreign downcalls to RocksDB's C API, plus JCTools and zstd. GraalVM finds it by classpath (",[366,442,443],{},"META-INF\u002Fnative-image\u002Fio.stoatflow\u002Fcore\u002F","), so any dependent app inherits it.",[343,446,447,452],{},[336,448,449],{},[366,450,451],{},":runtime"," adds the configuration layer — the Hoplite YAML binding for the runtime config classes, Logback, Micrometer\u002FHdrHistogram, and the rest of the runtime stack.",[343,454,455,462,463,466,467,470,471,474,475,478],{},[336,456,457,458,461],{},"The ",[366,459,460],{},"io.stoatflow"," Gradle plugin"," turns it on in one line: ",[366,464,465],{},"nativeImage { enabled = true }"," applies GraalVM's build plugin, the ",[366,468,469],{},"--initialize-at-run-time"," flags that RocksDB and the Kafka SASL client need, and a ",[366,472,473],{},"nativeDockerBuild"," task that drives a multi-stage ",[366,476,477],{},"Dockerfile.native",".",[333,480,481,482,485],{},"So compiling a StoatFlow app to a native image is a one-line opt-in: the framework covers its own reflective, JNI and FFM surface, and the only per-app work is registering the metadata for your own serde value classes. That gets you a binary that runs. This post is about the next question — does it run ",[388,483,484],{},"well","? The honest answer runs through a wrong turn, a reversal, and a memory result you have to tune for.",[333,487,488],{},"We benchmarked four apps; the deep dive that follows uses two representatives:",[340,490,491,499],{},[343,492,493,498],{},[336,494,495],{},[366,496,497],{},"stateless-simple"," — a stateless 1 KB transform at 20K msg\u002Fs. No state stores.",[343,500,501,506],{},[336,502,503],{},[366,504,505],{},"word-count"," — stateful, RocksDB-backed, ~300K records\u002Fs processed (7.5K msg\u002Fs in, fanned out by aggregation).",[333,508,509,510,513],{},"Both ran under exactly-once semantics, four configurations each, 10-minute warmup + 10-minute measurement on a Hetzner ",[366,511,512],{},"ccx33"," worker (8 dedicated vCPU); the full four-app matrix closes the post. What follows is the honest version, wrong turns included — because the wrong turns are where the interesting engineering lives.",[420,515,517],{"id":516},"act-1-the-floor-graalvm-ce-and-serial-gc","Act 1 — The floor: GraalVM CE and Serial GC",[333,519,520,521,524],{},"GraalVM Community Edition only ships ",[336,522,523],{},"Serial GC"," for native image. G1 and the parallel collector are Oracle-GraalVM-only. So the first build we measured had no choice in collector — Serial it was.",[333,526,527,528,531],{},"The stateless app worked, but at a price: ",[336,529,530],{},"+42% CPU"," vs the JVM (1.7 cores vs 1.2). Tolerable, if not free.",[333,533,534],{},"The stateful app didn't work at all. It couldn't keep up:",[536,537,538,555],"table",{},[539,540,541],"thead",{},[542,543,544,548,552],"tr",{},[545,546,547],"th",{},"Metric",[545,549,551],{"align":550},"right","JVM",[545,553,554],{"align":550},"Serial native",[556,557,558,572,585],"tbody",{},[542,559,560,564,567],{},[561,562,563],"td",{},"True E2E P50",[561,565,566],{"align":550},"120.6 ms",[561,568,569],{"align":550},[336,570,571],{},"13,504.7 ms",[542,573,574,577,580],{},[561,575,576],{},"Max consumer lag",[561,578,579],{"align":550},"100",[561,581,582],{"align":550},[336,583,584],{},"17,498",[542,586,587,590,593],{},[561,588,589],{},"Avg CPU",[561,591,592],{"align":550},"2.0 cores",[561,594,595],{"align":550},"3.5 cores",[333,597,598,599,602,603,606],{},"That's a ",[336,600,601],{},"112× regression"," in end-to-end latency and a ",[336,604,605],{},"175× regression"," in lag. The app was falling behind the input and never catching up.",[333,608,609,610,613],{},"The obvious suspect was the RocksDB ",[336,611,612],{},"FFM"," (Foreign Function & Memory API) backend. StoatFlow talks to RocksDB's C API through FFM downcalls, and native image has to do special gymnastics to make those work (more on that below). It's the most exotic part of the native build, and when the workload that touches state explodes, you blame the exotic thing.",[333,615,616],{},[336,617,618],{},"We were wrong.",[420,620,622],{"id":621},"act-2-the-reversal-oracle-graalvm-and-g1","Act 2 — The reversal: Oracle GraalVM and G1",[333,624,625,626,629,630,633],{},"Oracle GraalVM is now free for production use under the GFTC licence, and it unlocks two things CE doesn't: the ",[336,627,628],{},"G1 garbage collector"," and ",[336,631,632],{},"PGO"," (Profile-Guided Optimisation).",[333,635,636,637,640],{},"We rebuilt the stateful app with nothing changed but ",[366,638,639],{},"--gc=G1",":",[536,642,643,659],{},[539,644,645],{},[542,646,647,649,651,654],{},[545,648,547],{},[545,650,551],{"align":550},[545,652,653],{"align":550},"Serial",[545,655,656],{"align":550},[336,657,658],{},"G1",[556,660,661,674,687],{},[542,662,663,665,667,669],{},[561,664,563],{},[561,666,566],{"align":550},[561,668,571],{"align":550},[561,670,671],{"align":550},[336,672,673],{},"128.1 ms",[542,675,676,678,680,682],{},[561,677,576],{},[561,679,579],{"align":550},[561,681,584],{"align":550},[561,683,684],{"align":550},[336,685,686],{},"91",[542,688,689,691,694,697],{},[561,690,589],{},[561,692,693],{"align":550},"2.0",[561,695,696],{"align":550},"3.5",[561,698,699],{"align":550},"2.9",[333,701,702,703,706],{},"G1 fixed it ",[336,704,705],{},"outright",". 128 ms P50, 91 lag — that's JVM parity. The 112× latency blowup was gone with a one-flag change to the garbage collector.",[333,708,709,710,713],{},"So the \"FFM is the bottleneck\" hypothesis was simply false. ",[336,711,712],{},"The stateful failure was GC-bound, not FFM-bound."," Serial GC, single-threaded and stop-the-world, was choking on the allocation churn of the stateful path (RocksDB read\u002Fwrite buffers, aggregation state, serialisation). G1's concurrent and parallel collection kept up where Serial stalled. The FFM path had been carrying the heat for the GC the whole time.",[333,715,716],{},"This is the beat worth slowing down on: the exotic component is rarely the bottleneck. The boring component — the collector that runs on every allocation — was. We reached for the interesting explanation and missed the mundane one. The benchmark matrix is what caught it, not intuition.",[333,718,719,720,722],{},"One caveat we name in the report: this comparison mixes two variables. Serial was GraalVM CE; G1 was Oracle GraalVM, which also applies optimisations CE lacks. So \"CE-Serial → Oracle-G1\" is edition ",[388,721,439],{}," collector. The actionable comparison is JVM vs the best shippable native build — and that's what every headline number here uses.",[420,724,726],{"id":725},"act-3-the-cpu-lever-pgo","Act 3 — The CPU lever: PGO",[333,728,729,730,629,733,736],{},"G1 brought latency and throughput to parity, but CPU was still high: ",[336,731,732],{},"+33% (stateless)",[336,734,735],{},"+45% (stateful)"," vs the JVM. The JVM's advantage here is the JIT — it profiles the running code and recompiles hot paths with real branch and type information. A native image is compiled ahead of time with static heuristics; it never sees the workload.",[333,738,739,742],{},[336,740,741],{},"PGO closes that gap."," It's a two-pass build:",[744,745,746,757,764],"ol",{},[343,747,748,749,752,753,756],{},"Compile an ",[336,750,751],{},"instrumented"," binary (",[366,754,755],{},"--pgo-instrument",").",[343,758,759,760,763],{},"Run it under representative load to collect a ",[366,761,762],{},".iprof"," profile.",[343,765,766,767,770],{},"Rebuild with the profile (",[366,768,769],{},"--pgo=\u003Capp>.iprof",") so the AOT compiler gets the hot-path inlining and code layout the static heuristics miss.",[333,772,773],{},"PGO is purely a CPU lever — it doesn't touch latency or throughput, which were already at parity. It cut CPU:",[536,775,776,791],{},[539,777,778],{},[542,779,780,783,786],{},[545,781,782],{},"App",[545,784,785],{"align":550},"G1 (no PGO)",[545,787,788],{"align":550},[336,789,790],{},"G1 + PGO",[556,792,793,806],{},[542,794,795,798,801],{},[561,796,797],{},"stateless",[561,799,800],{"align":550},"1.6 cores",[561,802,803],{"align":550},[336,804,805],{},"1.4 cores",[542,807,808,811,814],{},[561,809,810],{},"stateful",[561,812,813],{"align":550},"2.9 cores",[561,815,816],{"align":550},[336,817,818],{},"2.4 cores",[333,820,821,822,629,825,828],{},"That's −13% and −17% on top of G1. The residual gap to the JVM lands at ",[336,823,824],{},"+17% (stateless)",[336,826,827],{},"+20% (stateful)"," — down from the Serial floor of +42% \u002F +75%. That residual is the irreducible AOT-vs-JIT peak-code difference, plus the FFM downcall path under SubstrateVM, which is the next act.",[420,830,832],{"id":831},"act-4-the-counterintuitive-finding-jni-beats-ffm-under-aot","Act 4 — The counterintuitive finding: JNI beats FFM under AOT",[333,834,835],{},"Here's the part that inverts conventional wisdom.",[333,837,838,841,842,845,846,849],{},[336,839,840],{},"On the HotSpot JVM, StoatFlow's FFM RocksDB backend is faster than the JNI one."," The reason is ",[366,843,844],{},"Linker.Option.critical(false)"," — it marks a downcall as not needing a full safepoint transition, and the JIT ",[388,847,848],{},"intrinsifies"," it, compiling the foreign call down to nearly a direct C call. FFM on a warm JVM is genuinely excellent.",[333,851,852,855,856,858,859,862],{},[336,853,854],{},"Under SubstrateVM (native image), there is no JIT."," That intrinsification never happens — there's no profiling compiler to perform it. So the ",[366,857,368],{}," optimisation that makes FFM win on the JVM simply doesn't fire. And once you strip it away, an FFM downcall ends up ",[388,860,861],{},"slower"," than a plain JNI call. (Both pin the virtual thread for the duration of the native call anyway, so there's no Loom advantage either way.)",[333,864,865],{},"We measured it on word-count, G1+PGO, swapping only the RocksDB backend:",[536,867,868,882],{},[539,869,870],{},[542,871,872,874,877],{},[545,873,547],{},[545,875,876],{"align":550},"FFM (native)",[545,878,879],{"align":550},[336,880,881],{},"JNI (native)",[556,883,884,895,906,919,930],{},[542,885,886,888,890],{},[561,887,589],{},[561,889,818],{"align":550},[561,891,892],{"align":550},[336,893,894],{},"2.2 cores",[542,896,897,900,903],{},[561,898,899],{},"Avg RSS",[561,901,902],{"align":550},"606 MB",[561,904,905],{"align":550},"601 MB",[542,907,908,911,914],{},[561,909,910],{},"True E2E P99",[561,912,913],{"align":550},"226.9 ms",[561,915,916],{"align":550},[336,917,918],{},"211.8 ms",[542,920,921,924,927],{},[561,922,923],{},"Produced rec\u002Fs",[561,925,926],{"align":550},"304,725",[561,928,929],{"align":550},"307,418",[542,931,932,934,937],{},[561,933,576],{},[561,935,936],{"align":550},"113",[561,938,939],{"align":550},"102",[333,941,942,943,946,947,950,951],{},"JNI uses ",[336,944,945],{},"~8% less CPU"," (2.2 vs 2.4 cores) with a ",[336,948,949],{},"better P99"," (211.8 vs 226.9 ms, −6.7%) and marginally higher throughput. ",[336,952,953],{},"The optimisation that's correct on one runtime is wrong on the other.",[333,955,956,957,960],{},"So the fix is to pick per-runtime. We added a new ",[366,958,959],{},"RocksDbBackendType.AUTO"," default that resolves to:",[340,962,963,968],{},[343,964,965,967],{},[336,966,612],{}," on the JVM — where the JIT intrinsifies it and it wins.",[343,969,970,973],{},[336,971,972],{},"JNI"," under a native image — where there's no JIT and JNI wins.",[333,975,976,977,979],{},"Each runtime gets its faster path automatically. This also has a pleasant side effect on the native build's fragility, which the snares section gets into: the FFM-under-native machinery is genuinely brittle, and ",[366,978,372],{}," takes it off the default path for native images entirely.",[420,981,983],{"id":982},"memory-real-but-gc-dependent","Memory: real, but GC-dependent",[333,985,986],{},"Native image's headline memory win is real — and more nuanced than the marketing suggests.",[333,988,989,992,993,996],{},[336,990,991],{},"Under Serial GC, native is tiny."," The stateless app's RSS was ",[336,994,995],{},"78 MB"," vs the JVM's 496 MB — an 84% reduction. Serial keeps a compact heap and reclaims aggressively. That is a genuine, large win.",[333,998,999,1002,1003,1008,1009,1011],{},[336,1000,1001],{},"Under G1, it depends on the workload's live footprint",", because G1 ",[388,1004,1005,1006],{},"sizes its heap reservation from ",[366,1007,400],{}," — and the container entrypoint hands ",[366,1010,400],{}," almost the whole container (≈15.6 GB). So G1 reserves generously:",[340,1013,1014,1024],{},[343,1015,1016,1019,1020,1023],{},[336,1017,1018],{},"Stateless app"," (live heap ~40 MB): G1's reservation dominates RSS, which jumps to ",[336,1021,1022],{},"587 MB — above the JVM's 496 MB",", and far above Serial's 78 MB. The Serial memory win is gone.",[343,1025,1026,1029,1030,478],{},[336,1027,1028],{},"Stateful app"," (larger live footprint): G1's reservation is comparable-to-lower than the JVM's heap + metaspace + JIT code cache, so RSS lands at ",[336,1031,1032],{},"606 MB — 23% below the JVM's 791 MB",[333,1034,1035,1036,1042],{},"The takeaway: ",[336,1037,1038,1039,1041],{},"G1 native wants a tight ",[366,1040,400],{}," to keep the memory win."," The stateless app needs ~40 MB of heap; giving G1 15 GB to reserve from is the whole problem. That's a tuning follow-up — an env-var override, no rebuild — and we haven't done it yet. So the honest statement is: native memory is a win you tune for under G1, not one you get for free.",[420,1044,1046],{"id":1045},"the-snares","The snares",[333,1048,1049,1050,1053],{},"These are the discoveries — the things that cost an afternoon each and that you only hit because native image moves failures from compile time to first-record time. Closed-world compilation means ",[388,1051,1052],{},"anything"," reflective, JNI, or FFM must be declared up front; miss one and it surfaces as a runtime error the first time that code path executes, never at build.",[1055,1056,1058],"h3",{"id":1057},"glibc-version-pinning","glibc version pinning",[333,1060,1061,1062,1065,1066,1069,1070,1073,1074,1077,1078,1081,1082,478],{},"Oracle's default ",[366,1063,1064],{},"native-image:25"," builder image is built on Oracle Linux 10 (glibc 2.39). The binary it produces then ",[388,1067,1068],{},"requires"," ",[366,1071,1072],{},"GLIBC_2.38"," at runtime. But the distroless ",[366,1075,1076],{},"cc-debian12"," runtime image ships glibc ",[336,1079,1080],{},"2.36"," — so the binary crashloops on startup with ",[366,1083,1084],{},"version GLIBC_2.38 not found",[333,1086,1087,1090,1091,1094,1095,1098],{},[336,1088,1089],{},"Fix:"," pin the builder to the ",[366,1092,1093],{},":25-ol9"," tag (Oracle Linux 9, glibc 2.34). 2.34 ≤ 2.36, so the binary runs. The rule: ",[336,1096,1097],{},"a native binary requires the build host's glibc version or newer at runtime"," — build on the oldest glibc you can, run anywhere newer.",[1055,1100,1102],{"id":1101},"reachability-metadatajson-silently-shadows-the-legacy-reflection-configjson","reachability-metadata.json silently shadows the legacy reflection-config.json",[333,1104,1105,1106,1109,1110,1113,1114,1117,1118,1121,1122,1125,1126,1129,1130,1133],{},"GraalVM has two metadata formats: the older ",[366,1107,1108],{},"reflection-config.json"," and the newer ",[366,1111,1112],{},"reachability-metadata.json",". When ",[336,1115,1116],{},"both"," are present, the newer file wins — and silently drops the old file's registrations, including the ",[366,1119,1120],{},"allDeclaredConstructors: true"," on our runtime config data classes. The symptom: a runtime ",[366,1123,1124],{},"MissingReflectionRegistrationError"," on a config class that ",[388,1127,1128],{},"was"," registered, in the file that ",[388,1131,1132],{},"used"," to win.",[333,1135,1136,1138],{},[336,1137,1089],{}," consolidate the registrations into the reachability-metadata file. Don't run the two formats side by side and expect them to merge — they don't.",[1055,1140,1142],{"id":1141},"everything-reflective-must-be-declared-and-you-find-out-at-first-contact","Everything reflective must be declared, and you find out at first contact",[333,1144,1145],{},"Each of these surfaced only when its code path first executed, never at build time:",[340,1147,1148,1161,1172,1181],{},[343,1149,1150,1153,1154,1157,1158],{},[336,1151,1152],{},"JCTools"," lane-queue field-holder classes (accessed via ",[366,1155,1156],{},"Unsafe",") → ",[366,1159,1160],{},"NoSuchFieldException",[343,1162,1163,1069,1166,1169,1170],{},[336,1164,1165],{},"HdrHistogram",[366,1167,1168],{},"ConcurrentHistogram"," → ",[366,1171,1160],{},[343,1173,1174,1177,1178],{},[336,1175,1176],{},"zstd-jni"," native field handles → ",[366,1179,1180],{},"NoSuchFieldError",[343,1182,1183,1186,1187],{},[336,1184,1185],{},"Kafka serialisers"," instantiated by class name → ",[366,1188,1189],{},"NoSuchMethodException",[333,1191,1192],{},"There's no shortcut here other than discipline: a closed world means the failure budget moves entirely from compile time to first-record time. Drive every code path before you trust the binary.",[1055,1194,1196],{"id":1195},"ffm-under-native-image-needs-hand-rolled-symbol-resolution","FFM under native image needs hand-rolled symbol resolution",[333,1198,1199,1200,1203,1204,1069,1207,1210,1211,1214],{},"This is the brittle one. ",[366,1201,1202],{},"SymbolLookup.loaderLookup()"," — the normal way to find a native function — ",[336,1205,1206],{},"can't see the symbols of a JNI library loaded at runtime under SubstrateVM.",[366,1208,1209],{},"librocksdbjni"," is loaded by RocksDB's JNI bootstrap at runtime; the FFM backend then can't resolve ",[366,1212,1213],{},"rocksdb_*"," symbols against it.",[333,1216,1217],{},"The workaround:",[744,1219,1220,1227,1238,1243],{},[343,1221,1222,1223,1226],{},"Detect native image via the ",[366,1224,1225],{},"org.graalvm.nativeimage.imagecode"," system property.",[343,1228,1229,1230,1233,1234,1237],{},"Read the library's mapped path out of ",[366,1231,1232],{},"\u002Fproc\u002Fself\u002Fmaps",", then ",[366,1235,1236],{},"libraryLookup"," it explicitly.",[343,1239,1240,1242],{},[366,1241,469],{}," for the FFM holder class.",[343,1244,1245,1246,1249],{},"Register ",[336,1247,1248],{},"every"," foreign downcall descriptor — and the leaf type includes the implicit function-pointer first argument, which is trivially easy to miss on one of forty descriptors.",[333,1251,1252,1253,1256,1257,1259,1260,1262,1263,1266],{},"It works, but it's fragile and ",[336,1254,1255],{},"Linux-only"," (the ",[366,1258,1232],{}," trick has no portable equivalent). The good news: the ",[366,1261,372],{}," backend change (JNI under native) means this whole apparatus is ",[336,1264,1265],{},"off the default path now",". It's there if you force FFM on native, but you have to ask for it.",[1055,1268,1270],{"id":1269},"avro-schema-registry-is-the-worst-offender","Avro + Schema Registry is the worst offender",[333,1272,1273],{},"Confluent's Avro stack is reflection-heavy in three compounding ways, and native image trips on all of them:",[340,1275,1276,1290,1296],{},[343,1277,1278,1281,1282,1285,1286,1289],{},[336,1279,1280],{},"The serde config instantiates classes by name from its own defaults",", even properties you never set — e.g. ",[366,1283,1284],{},"NullContextNameStrategy",". Symptom: ",[366,1287,1288],{},"ConfigException: … could not be found"," at startup, for a class you've never heard of.",[343,1291,1292,1295],{},[336,1293,1294],{},"The schema-id serialisers and the registry REST entities"," (Jackson-deserialised from the registry's HTTP responses) all need registering.",[343,1297,1298,1301,1302,478],{},[336,1299,1300],{},"Native image disables URL protocols by default",", so the HTTP schema fetch fails until you add ",[366,1303,1304],{},"--enable-url-protocols=http,https",[333,1306,1307,1308,1311,1312,1315,1316,1321],{},"Out of this fell a clean ownership pattern: ",[336,1309,1310],{},"keep serde reachability metadata in the app module, not in the engine."," The app is the closed-world boundary. StoatFlow ships metadata for everything ",[388,1313,1314],{},"it"," touches — the engine, Kafka client, RocksDB (both backends), JCTools, Micrometer\u002FHdrHistogram, the config layer. Customers add only their own value classes. The tracing agent catches the config-default classes in one pass and is the surer route for anything Avro. (Full recipe in the ",[1317,1318,1320],"a",{"href":1319},"\u002Fdocs\u002Fruntime\u002Fnative-image#custom-serde-metadata","custom-serde native-image guide",".)",[1055,1323,1325],{"id":1324},"extracting-a-pgo-profile-from-a-server-that-never-exits","Extracting a PGO profile from a server that never exits",[333,1327,1328,1329,1331,1332,1335],{},"PGO's instrumented binary only writes its ",[366,1330,762],{}," profile ",[336,1333,1334],{},"on clean shutdown",". There's no on-demand dump API. A streaming app runs forever and never exits on its own, so getting the profile out takes some plumbing.",[333,1337,1338],{},"The trick that's race-free:",[340,1340,1341,1360,1367],{},[343,1342,1343,1344,1347,1348,1351,1352,1355,1356,1359],{},"An entrypoint that runs the binary as a ",[336,1345,1346],{},"child process"," (not ",[366,1349,1350],{},"exec","), and ",[336,1353,1354],{},"forwards SIGTERM"," to it — so ",[366,1357,1358],{},"kubectl exec -- kill -TERM 1"," drains and dumps the profile.",[343,1361,1362,1363,1366],{},"After the child exits, the entrypoint runs ",[366,1364,1365],{},"sleep infinity"," so the pod stays alive (and the liveness probe's eventual restart doesn't race the extraction).",[343,1368,1369,1370,1375,1376,1379],{},"Dump to an ",[336,1371,1372],{},[366,1373,1374],{},"emptyDir"," so the file survives that restart, then ",[366,1377,1378],{},"kubectl cp"," it out.",[333,1381,1382,1383,1386,1387,1389],{},"Drive ~4 minutes of representative load, ",[366,1384,1385],{},"kill -TERM 1",", copy the ",[366,1388,762],{},", rebuild with it. Done.",[420,1391,1393],{"id":1392},"results","Results",[333,1395,1396,1397,1403],{},"The full four-configuration matrix, both apps, JVM as the baseline. Every number traces to the ",[1317,1398,1402],{"href":1399,"rel":1400},"https:\u002F\u002Fstoatflow.io\u002Fproduct\u002Fbenchmarks",[1401],"nofollow","benchmark report","; deltas are vs the JVM.",[1055,1405,1407],{"id":1406},"stateless-simple-20k-msgs-stateless","stateless-simple (20K msg\u002Fs, stateless)",[536,1409,1410,1430],{},[539,1411,1412],{},[542,1413,1414,1416,1418,1420,1422,1427],{},[545,1415,547],{},[545,1417,551],{"align":550},[545,1419,653],{"align":550},[545,1421,658],{"align":550},[545,1423,1424],{"align":550},[336,1425,1426],{},"G1+PGO",[545,1428,1429],{},"Δ vs JVM",[556,1431,1432,1451,1470,1489,1511,1535],{},[542,1433,1434,1436,1439,1442,1445,1448],{},[561,1435,923],{},[561,1437,1438],{"align":550},"19,962",[561,1440,1441],{"align":550},"19,942",[561,1443,1444],{"align":550},"19,954",[561,1446,1447],{"align":550},"19,947",[561,1449,1450],{},"≈ 0%",[542,1452,1453,1455,1458,1461,1464,1467],{},[561,1454,576],{},[561,1456,1457],{"align":550},"326",[561,1459,1460],{"align":550},"353",[561,1462,1463],{"align":550},"323",[561,1465,1466],{"align":550},"339",[561,1468,1469],{},"≈",[542,1471,1472,1475,1478,1481,1484,1487],{},[561,1473,1474],{},"True E2E P50 (ms)",[561,1476,1477],{"align":550},"128.9",[561,1479,1480],{"align":550},"131.7",[561,1482,1483],{"align":550},"129.1",[561,1485,1486],{"align":550},"128.8",[561,1488,1450],{},[542,1490,1491,1494,1497,1500,1503,1506],{},[561,1492,1493],{},"True E2E P99 (ms)",[561,1495,1496],{"align":550},"252",[561,1498,1499],{"align":550},"233.7",[561,1501,1502],{"align":550},"250.2",[561,1504,1505],{"align":550},"229.6",[561,1507,1508],{},[336,1509,1510],{},"−9%",[542,1512,1513,1516,1519,1522,1525,1530],{},[561,1514,1515],{},"Avg CPU (cores)",[561,1517,1518],{"align":550},"1.2",[561,1520,1521],{"align":550},"1.7",[561,1523,1524],{"align":550},"1.6",[561,1526,1527],{"align":550},[336,1528,1529],{},"1.4",[561,1531,1532],{},[336,1533,1534],{},"+17%",[542,1536,1537,1540,1543,1548,1551,1554],{},[561,1538,1539],{},"Avg container RSS (MB)",[561,1541,1542],{"align":550},"496",[561,1544,1545],{"align":550},[336,1546,1547],{},"78",[561,1549,1550],{"align":550},"596",[561,1552,1553],{"align":550},"587",[561,1555,1556],{},[336,1557,1558],{},"+18%",[333,1560,1561,1562,1565],{},"Throughput and latency parity (P99 actually ",[388,1563,1564],{},"beats"," the JVM by 9%) at +17% CPU. The memory catch is real: G1's heap reservation pushes RSS 18% above the JVM. Serial native's 78 MB is the memory-optimal build here; G1+PGO is the CPU\u002Flatency-optimal one.",[1055,1567,1569],{"id":1568},"word-count-75k-msgs-in-300ks-processed-stateful-rocksdb","word-count (7.5K msg\u002Fs in, ~300K\u002Fs processed, stateful — RocksDB)",[536,1571,1572,1591],{},[539,1573,1574],{},[542,1575,1576,1578,1580,1582,1584,1589],{},[545,1577,547],{},[545,1579,551],{"align":550},[545,1581,653],{"align":550},[545,1583,658],{"align":550},[545,1585,1586],{"align":550},[336,1587,1588],{},"G1+PGO (FFM)",[545,1590,1429],{},[556,1592,1593,1610,1626,1647,1666,1686],{},[542,1594,1595,1597,1600,1603,1606,1608],{},[561,1596,923],{},[561,1598,1599],{"align":550},"303,770",[561,1601,1602],{"align":550},"308,180",[561,1604,1605],{"align":550},"296,387",[561,1607,926],{"align":550},[561,1609,1450],{},[542,1611,1612,1614,1616,1620,1622,1624],{},[561,1613,576],{},[561,1615,579],{"align":550},[561,1617,1618],{"align":550},[336,1619,584],{},[561,1621,686],{"align":550},[561,1623,936],{"align":550},[561,1625,1469],{},[542,1627,1628,1630,1633,1638,1641,1644],{},[561,1629,1474],{},[561,1631,1632],{"align":550},"120.6",[561,1634,1635],{"align":550},[336,1636,1637],{},"13,504.7",[561,1639,1640],{"align":550},"128.1",[561,1642,1643],{"align":550},"124.8",[561,1645,1646],{},"+3%",[542,1648,1649,1651,1654,1657,1660,1663],{},[561,1650,1493],{},[561,1652,1653],{"align":550},"211.3",[561,1655,1656],{"align":550},"17,233.9",[561,1658,1659],{"align":550},"248.8",[561,1661,1662],{"align":550},"226.9",[561,1664,1665],{},"+7%",[542,1667,1668,1670,1672,1674,1676,1681],{},[561,1669,1515],{},[561,1671,693],{"align":550},[561,1673,696],{"align":550},[561,1675,699],{"align":550},[561,1677,1678],{"align":550},[336,1679,1680],{},"2.4",[561,1682,1683],{},[336,1684,1685],{},"+20%",[542,1687,1688,1690,1693,1696,1699,1704],{},[561,1689,1539],{},[561,1691,1692],{"align":550},"791",[561,1694,1695],{"align":550},"142",[561,1697,1698],{"align":550},"779",[561,1700,1701],{"align":550},[336,1702,1703],{},"606",[561,1705,1706],{},[336,1707,1708],{},"−23%",[333,1710,1711,1712,1714,1715,1717],{},"Serial couldn't keep up (112× E2E latency). G1 fixed it to parity; PGO trimmed CPU from +75% to +20%. And because this app's live footprint is larger, G1's reservation lands ",[388,1713,404],{}," the JVM's total — so the stateful native image keeps a 23% memory advantage ",[336,1716,439],{}," keeps up.",[1055,1719,1721],{"id":1720},"ffm-vs-jni-backend-under-native-image-word-count-g1pgo","FFM vs JNI backend, under native image (word-count, G1+PGO)",[536,1723,1724,1739],{},[539,1725,1726],{},[542,1727,1728,1730,1732,1736],{},[545,1729,547],{},[545,1731,876],{"align":550},[545,1733,1734],{"align":550},[336,1735,881],{},[545,1737,1738],{},"Δ (JNI vs FFM)",[556,1740,1741,1757,1768,1784,1794],{},[542,1742,1743,1745,1747,1752],{},[561,1744,1515],{},[561,1746,1680],{"align":550},[561,1748,1749],{"align":550},[336,1750,1751],{},"2.2",[561,1753,1754],{},[336,1755,1756],{},"−8%",[542,1758,1759,1761,1763,1766],{},[561,1760,1539],{},[561,1762,1703],{"align":550},[561,1764,1765],{"align":550},"601",[561,1767,1469],{},[542,1769,1770,1772,1774,1779],{},[561,1771,1493],{},[561,1773,1662],{"align":550},[561,1775,1776],{"align":550},[336,1777,1778],{},"211.8",[561,1780,1781],{},[336,1782,1783],{},"−7%",[542,1785,1786,1788,1790,1792],{},[561,1787,923],{},[561,1789,926],{"align":550},[561,1791,929],{"align":550},[561,1793,1469],{},[542,1795,1796,1798,1800,1802],{},[561,1797,576],{},[561,1799,936],{"align":550},[561,1801,939],{"align":550},[561,1803,1469],{},[333,1805,1806,1807,1809],{},"JNI is the better native backend on every axis that moves — which is exactly the inversion of the JVM, and exactly why ",[366,1808,372],{}," resolves to JNI under native image.",[1055,1811,1813],{"id":1812},"all-four-apps-under-g1pgo-with-the-auto-backend","All four apps under G1+PGO (with the AUTO backend)",[333,1815,1816,1817,1819],{},"We then re-benchmarked all four apps on the best config — G1 + PGO + ",[366,1818,372],{}," (so the two stateful apps run on JNI). The two serde-heavy apps ran as native images for the first time; Protobuf and Avro + Schema Registry both processed cleanly under load. Numbers are native vs the JVM baseline:",[536,1821,1822,1842],{},[539,1823,1824],{},[542,1825,1826,1828,1831,1834,1837,1840],{},[545,1827,782],{},[545,1829,1830],{},"Serde",[545,1832,1833],{"align":550},"Avg CPU (nat \u002F jvm)",[545,1835,1836],{"align":550},"RSS MB (nat \u002F jvm)",[545,1838,1839],{"align":550},"E2E P99 ms (nat \u002F jvm)",[545,1841,923],{"align":550},[556,1843,1844,1867,1888,1914],{},[542,1845,1846,1848,1851,1854,1857,1864],{},[561,1847,497],{},[561,1849,1850],{},"String",[561,1852,1853],{"align":550},"1.3 \u002F 1.2 (+8%)",[561,1855,1856],{"align":550},"577 \u002F 496 (+16%)",[561,1858,1859,1860,1863],{"align":550},"216 \u002F 252 (",[336,1861,1862],{},"−14%",")",[561,1865,1866],{"align":550},"19,966",[542,1868,1869,1871,1873,1876,1882,1885],{},[561,1870,505],{},[561,1872,1850],{},[561,1874,1875],{"align":550},"2.2 \u002F 2.0 (+10%)",[561,1877,1878,1879,1863],{"align":550},"600 \u002F 791 (",[336,1880,1881],{},"−24%",[561,1883,1884],{"align":550},"210 \u002F 211 (≈0%)",[561,1886,1887],{"align":550},"303,424",[542,1889,1890,1893,1896,1902,1905,1911],{},[561,1891,1892],{},"stateless-advanced",[561,1894,1895],{},"Protobuf",[561,1897,1898,1899,1863],{"align":550},"1.2 \u002F 1.2 (",[336,1900,1901],{},"0%",[561,1903,1904],{"align":550},"537 \u002F 522 (+3%)",[561,1906,1907,1908,1863],{"align":550},"206 \u002F 244 (",[336,1909,1910],{},"−16%",[561,1912,1913],{"align":550},"13,532",[542,1915,1916,1919,1922,1928,1934,1939],{},[561,1917,1918],{},"stateful-joins",[561,1920,1921],{},"Avro",[561,1923,1924,1925,1863],{"align":550},"2.7 \u002F 2.8 (",[336,1926,1927],{},"−4%",[561,1929,1930,1931,1863],{"align":550},"920 \u002F 1151 (",[336,1932,1933],{},"−20%",[561,1935,1936,1937,1863],{"align":550},"251 \u002F 273 (",[336,1938,1756],{},[561,1940,1941],{"align":550},"20,213",[333,1943,1944,1945,1948,1949,1952,1953,1956,1957,1960,1961,1963],{},"That is the bottom line of this whole exercise. CPU lands between ",[336,1946,1947],{},"−4% and +10%"," of the JVM; ",[336,1950,1951],{},"latency is equal-or-better on every app"," (P99 −8% to −16% on three, parity on the fourth); and ",[336,1954,1955],{},"memory beats the JVM on both stateful apps"," (−20%, −24%), where the RocksDB working set is large enough that G1's reservation comes in under the JVM's heap + metaspace + code cache. The Protobuf app is at outright ",[336,1958,1959],{},"parity"," on CPU. Only the two tiny-heap stateless apps give back a little memory — the same G1 ",[366,1962,400],{}," sprawl, tunable. Throughput and lag are at parity throughout.",[333,1965,1966,1967,1970],{},"The standout: the Avro stateful-joins app — the most complex of the four, with windowed joins over five Avro streams — runs ",[336,1968,1969],{},"leaner than the JVM on CPU, memory, and tail latency at once."," A long way from \"native can't do stateful.\"",[1055,1972,1974],{"id":1973},"smaller-on-disk-and-what-the-build-costs","Smaller on disk — and what the build costs",[333,1976,1977,1978,1981,1982,1985,1986,1989],{},"There's a footprint win too, the one native image is actually known for. word-count's native image is ",[336,1979,1980],{},"147 MB"," on disk against ",[336,1983,1984],{},"222 MB"," for the JVM build — a ",[336,1987,1988],{},"34% reduction",". The other three apps land within a few MB of that same split (the native binaries are near-identical in size, ~147–155 MB; the JVM images ~222–233 MB), so one number stands for all four. Most of the native image is the GraalVM binary plus the distroless base; the JVM image carries a JRE and the shadow JAR.",[333,1991,1992],{},"What you pay for it is build time. A native PGO build is two passes — an instrumented compile, a few minutes of profiling load, then the optimised compile — each running minutes on a dedicated amd64 worker, where the JVM image is a shadow JAR plus a Jib layer in seconds. Call it a 10–50× build-time multiplier, on Oracle GraalVM with a buildx-on-Kubernetes builder to keep alive. It's a release-time cost rather than a per-change one — but it's real, and it's the honest other half of the footprint win.",[1055,1994,1996],{"id":1995},"can-ffm-be-fixed-on-native","Can FFM be fixed on native?",[333,1998,1999,2000,2005,2006,2009,2010,2013,2014,2017,2018,2021],{},"The honest answer, after digging into it: not from our code, not today. The slowness is GraalVM's, and Oracle says so — their FFM tracking issue (",[1317,2001,2004],{"href":2002,"rel":2003},"https:\u002F\u002Fgithub.com\u002Foracle\u002Fgraal\u002Fissues\u002F8113",[1401],"oracle\u002Fgraal #8113",") lists ",[388,2007,2008],{},"\"improve downcall performance (currently always unoptimized)\""," as an ",[336,2011,2012],{},"open goal",", undelivered in GraalVM 25, with no committed date, and they explicitly intend to keep JNI and the ",[366,2015,2016],{},"@CFunction"," C-interface as the performance-grade native call paths. SubstrateVM emits a generic downcall stub and has no JIT to intrinsify the ",[366,2019,2020],{},"critical()"," hint the way HotSpot's C2 does.",[333,2023,2024,2025,2028,2029,2031,2032,2034,2035,2038],{},"We tried the one lever that lives in application code: ",[366,2026,2027],{},"critical(true)"," (allow-heap-access), which lets the hot path pass the on-heap key directly instead of copying it off-heap first. It doesn't recover the intrinsification — it only saves the copy — and a RocksDB key is a handful of bytes, so we didn't expect much. It turned out to be worse than \"not much\": under native image ",[366,2030,2027],{}," changes the downcall's leaf types, so it needs a whole new set of foreign-metadata registrations and crashes at startup without them. It's not a flag, it's a rebuild-per-missing-entry slog — and it leaves the actual cost, the unoptimised stub, untouched. The other option, a ",[366,2033,2016],{}," no-transition binding, ",[388,2036,2037],{},"would"," match HotSpot's fast path, but it's native-image-only: a third call path, not \"FFM everywhere.\"",[333,2040,2041,2042,2045,2046,2048],{},"So the foreign-function path doesn't have a cheap fix on native, and — this is the part that matters — it doesn't need one ",[388,2043,2044],{},"to ship today",". JNI already gets the stateful apps to JVM-parity-or-better. ",[366,2047,372],{}," (JNI on native, FFM on the JVM) is the pragmatic default that routes each runtime to its faster path without anyone having to know any of this.",[333,2050,2051,2052,2054,2055,2058,2059,2062,2063,2066,2067,2069,2070,2073,2074,2079,2080,2082,2083,2085],{},"But ",[366,2053,372],{}," is a stopgap, not the destination. The goal is the single most efficient, most resource-frugal path on each runtime — and on the JVM that's FFM, so what we actually want is FFM ",[388,2056,2057],{},"everywhere",", not a runtime-dependent split. The GraalVM team has named the route: the slow stub is what ",[336,2060,2061],{},"bound"," downcall handles (the jextract pattern we use today) compile to; an ",[336,2064,2065],{},"unbound"," handle — the function pointer passed as a call argument, the holder class initialised at build time — is reported to reach near-",[366,2068,2016],{}," speed under native image. There's no flag for it yet, and the ",[366,2071,2072],{},"registerForDirectDowncall"," API that would formalise it is proposed but uncommitted (",[1317,2075,2078],{"href":2076,"rel":2077},"https:\u002F\u002Fgithub.com\u002Foracle\u002Fgraal\u002Fissues\u002F12219",[1401],"oracle\u002Fgraal #12219 \u002F GR-75754","). So what's next is concrete and not ours to ship: rewrite the RocksDB FFM backend to unbound handles, validate it on GraalVM 25.1+, and the day native FFM matches JNI, ",[366,2081,372],{}," collapses to FFM everywhere — one path, the most efficient one. Until that API lands, ",[366,2084,372],{}," routes each runtime to its current best — eyes open that \"current best\" isn't yet \"best possible.\"",[420,2087,2089],{"id":2088},"whats-next","What's next",[333,2091,2092,2093,2096,2097,2099],{},"Native image is viable ",[336,2094,2095],{},"today"," for the surface we've covered: G1 + PGO + the ",[366,2098,372],{}," backend get you JVM-parity-or-better latency and throughput, CPU within −4% to +10%, and memory that beats the JVM on the stateful apps. The work from here is widening that covered surface, and being honest that it isn't complete:",[340,2101,2102,2112,2118,2127],{},[343,2103,2104,2107,2108,2111],{},[336,2105,2106],{},"Kafka client security isn't covered yet."," SASL mechanisms (SCRAM, GSSAPI\u002FKerberos), OAuth bearer (",[366,2109,2110],{},"OAUTHBEARER","), and mTLS each pull in JAAS, login-module, and JCA reflection that will need native metadata. None of it is mapped yet.",[343,2113,2114,2117],{},[336,2115,2116],{},"Compression codecs beyond zstd."," snappy, lz4, and gzip each have their own JNI\u002Fnative or reflective surface to register. Only zstd is covered today.",[343,2119,2120,2123,2124,2126],{},[336,2121,2122],{},"More as real workloads exercise more of the Kafka client and serde surface."," Closed-world compilation guarantees that every code path a customer hits for the first time is a potential ",[366,2125,1124],{}," until we (or they) declare it.",[343,2128,2129,2135,2136,2138],{},[336,2130,2131,2132,2134],{},"The G1 ",[366,2133,400],{}," tuning follow-up."," A tight ",[366,2137,400],{}," should recover the stateless memory win that G1's default container-fill reservation currently gives away — cheap to do, not yet done.",[333,2140,2141,2142,2144],{},"The shape of the thing is clear: native image earns its keep for the covered workloads, the wrong turns taught us where the real costs live (the collector, not the foreign-function path), and the ",[366,2143,372],{}," backend means you don't have to know any of this to get the faster path on each runtime. The surface we widen next is set by the workloads that hit it first — which serde and security paths a real deployment exercises is exactly what tells us what to map.",{"title":2146,"searchDepth":2147,"depth":2147,"links":2148},"",2,[2149,2150,2151,2152,2153,2154,2155,2164,2172],{"id":422,"depth":2147,"text":423},{"id":516,"depth":2147,"text":517},{"id":621,"depth":2147,"text":622},{"id":725,"depth":2147,"text":726},{"id":831,"depth":2147,"text":832},{"id":982,"depth":2147,"text":983},{"id":1045,"depth":2147,"text":1046,"children":2156},[2157,2159,2160,2161,2162,2163],{"id":1057,"depth":2158,"text":1058},3,{"id":1101,"depth":2158,"text":1102},{"id":1141,"depth":2158,"text":1142},{"id":1195,"depth":2158,"text":1196},{"id":1269,"depth":2158,"text":1270},{"id":1324,"depth":2158,"text":1325},{"id":1392,"depth":2147,"text":1393,"children":2165},[2166,2167,2168,2169,2170,2171],{"id":1406,"depth":2158,"text":1407},{"id":1568,"depth":2158,"text":1569},{"id":1720,"depth":2158,"text":1721},{"id":1812,"depth":2158,"text":1813},{"id":1973,"depth":2158,"text":1974},{"id":1995,"depth":2158,"text":1996},{"id":2088,"depth":2147,"text":2089},"2026-06-06","Native image is viable for a throughput-first stream processor — but only with Oracle G1 + PGO. The wrong turn (we blamed FFM; it was the GC) and the reversal (FFM is faster on the JVM, slower under AOT) are the most interesting parts.","md",{"src":2177},"\u002Fassets\u002Fblog\u002Fog\u002FStoatFlow_og_20260517.jpg",{},true,"\u002Fblog\u002Fnative-image-g1-pgo-jni-vs-ffm",{"title":317,"description":2174},"3.blog\u002F2.native-image-g1-pgo-jni-vs-ffm","lsLRQhFBJrz097XqIrDIaFfznFgvJdYhnDAcJLKtRbI",1783713349153]