[{"data":1,"prerenderedAt":2307},["ShallowReactive",2],{"navigation":3,"\u002Fdocs\u002Foperating\u002Fobservability":314,"\u002Fdocs\u002Foperating\u002Fobservability-surround":2302},[4,38,86,149,176,219,257,273],{"title":5,"path":6,"stem":7,"children":8,"icon":37},"Getting Started","\u002Fdocs\u002Fgetting-started","1.docs\u002F1.getting-started\u002F1.index",[9,12,17,22,27,32],{"title":10,"path":6,"stem":7,"icon":11},"Getting started","i-lucide-flag",{"title":13,"path":14,"stem":15,"icon":16},"Modules overview","\u002Fdocs\u002Fgetting-started\u002Fmodules-overview","1.docs\u002F1.getting-started\u002F2.modules-overview","i-lucide-boxes",{"title":18,"path":19,"stem":20,"icon":21},"Installation","\u002Fdocs\u002Fgetting-started\u002Finstallation","1.docs\u002F1.getting-started\u002F3.installation","i-lucide-download",{"title":23,"path":24,"stem":25,"icon":26},"License configuration","\u002Fdocs\u002Fgetting-started\u002Flicense-configuration","1.docs\u002F1.getting-started\u002F4.license-configuration","i-lucide-key-round",{"title":28,"path":29,"stem":30,"icon":31},"Your first app","\u002Fdocs\u002Fgetting-started\u002Ffirst-app","1.docs\u002F1.getting-started\u002F5.first-app","i-lucide-square-play",{"title":33,"path":34,"stem":35,"icon":36},"Project setup","\u002Fdocs\u002Fgetting-started\u002Fproject-setup","1.docs\u002F1.getting-started\u002F6.project-setup","i-lucide-package",false,{"title":39,"path":40,"stem":41,"children":42,"icon":45},"Core Concepts","\u002Fdocs\u002Fconcepts","1.docs\u002F2.concepts\u002F1.index",[43,46,51,56,61,66,71,76,81],{"title":44,"path":40,"stem":41,"icon":45},"Core concepts","i-lucide-book-open",{"title":47,"path":48,"stem":49,"icon":50},"Architecture","\u002Fdocs\u002Fconcepts\u002Farchitecture","1.docs\u002F2.concepts\u002F2.architecture","i-lucide-layers",{"title":52,"path":53,"stem":54,"icon":55},"Exactly-once semantics","\u002Fdocs\u002Fconcepts\u002Fexactly-once","1.docs\u002F2.concepts\u002F3.exactly-once","i-lucide-shield-check",{"title":57,"path":58,"stem":59,"icon":60},"Lanes and parallelism","\u002Fdocs\u002Fconcepts\u002Flanes-and-parallelism","1.docs\u002F2.concepts\u002F4.lanes-and-parallelism","i-lucide-split",{"title":62,"path":63,"stem":64,"icon":65},"State and thread-safety","\u002Fdocs\u002Fconcepts\u002Fstate-and-thread-safety","1.docs\u002F2.concepts\u002F5.state-and-thread-safety","i-lucide-lock",{"title":67,"path":68,"stem":69,"icon":70},"Event time and watermarks","\u002Fdocs\u002Fconcepts\u002Fevent-time-and-watermarks","1.docs\u002F2.concepts\u002F6.event-time-and-watermarks","i-lucide-clock",{"title":72,"path":73,"stem":74,"icon":75},"The configuration model","\u002Fdocs\u002Fconcepts\u002Fconfiguration-model","1.docs\u002F2.concepts\u002F7.configuration-model","i-lucide-sliders-horizontal",{"title":77,"path":78,"stem":79,"icon":80},"The error-handling model","\u002Fdocs\u002Fconcepts\u002Ferror-handling-model","1.docs\u002F2.concepts\u002F8.error-handling-model","i-lucide-triangle-alert",{"title":82,"path":83,"stem":84,"icon":85},"How StoatFlow differs from Kafka Streams","\u002Fdocs\u002Fconcepts\u002Fhow-stoatflow-differs-from-ks","1.docs\u002F2.concepts\u002F9.how-stoatflow-differs-from-ks","i-lucide-arrow-left-right",{"title":87,"path":88,"stem":89,"children":90,"icon":93},"Building Topologies","\u002Fdocs\u002Fbuilding","1.docs\u002F3.building\u002F1.index",[91,94,99,104,109,114,119,124,129,134,139,144],{"title":92,"path":88,"stem":89,"icon":93},"Building topologies","i-lucide-blocks",{"title":95,"path":96,"stem":97,"icon":98},"Error handling and DLQ","\u002Fdocs\u002Fbuilding\u002Ferror-handling-dlq","1.docs\u002F3.building\u002F10.error-handling-dlq","i-lucide-circle-x",{"title":100,"path":101,"stem":102,"icon":103},"State stores","\u002Fdocs\u002Fbuilding\u002Fstate-stores","1.docs\u002F3.building\u002F11.state-stores","i-lucide-database",{"title":105,"path":106,"stem":107,"icon":108},"Testing topologies","\u002Fdocs\u002Fbuilding\u002Ftesting","1.docs\u002F3.building\u002F12.testing","i-lucide-flask-conical",{"title":110,"path":111,"stem":112,"icon":113},"Sources and sinks","\u002Fdocs\u002Fbuilding\u002Fstreams-builder","1.docs\u002F3.building\u002F2.streams-builder","i-lucide-import",{"title":115,"path":116,"stem":117,"icon":118},"KStream and KTable operations","\u002Fdocs\u002Fbuilding\u002Fkstream-ktable","1.docs\u002F3.building\u002F3.kstream-ktable","i-lucide-waypoints",{"title":120,"path":121,"stem":122,"icon":123},"Aggregations","\u002Fdocs\u002Fbuilding\u002Faggregations","1.docs\u002F3.building\u002F4.aggregations","i-lucide-sigma",{"title":125,"path":126,"stem":127,"icon":128},"Windowing","\u002Fdocs\u002Fbuilding\u002Fwindowing","1.docs\u002F3.building\u002F5.windowing","i-lucide-calendar-clock",{"title":130,"path":131,"stem":132,"icon":133},"Joins","\u002Fdocs\u002Fbuilding\u002Fjoins","1.docs\u002F3.building\u002F6.joins","i-lucide-git-merge",{"title":135,"path":136,"stem":137,"icon":138},"The Processor API","\u002Fdocs\u002Fbuilding\u002Fprocessor-api","1.docs\u002F3.building\u002F7.processor-api","i-lucide-cpu",{"title":140,"path":141,"stem":142,"icon":143},"Scheduled sources","\u002Fdocs\u002Fbuilding\u002Fscheduled-sources","1.docs\u002F3.building\u002F8.scheduled-sources","i-lucide-timer",{"title":145,"path":146,"stem":147,"icon":148},"Serdes and Avro","\u002Fdocs\u002Fbuilding\u002Fserdes","1.docs\u002F3.building\u002F9.serdes","i-lucide-binary",{"title":150,"path":151,"stem":152,"children":153,"icon":75},"Configuration","\u002Fdocs\u002Fconfiguration","1.docs\u002F4.configuration\u002F1.index",[154,156,161,166,171],{"title":155,"path":151,"stem":152,"icon":75},"How configuration works",{"title":157,"path":158,"stem":159,"icon":160},"Engine configuration (:core)","\u002Fdocs\u002Fconfiguration\u002Fcore-config","1.docs\u002F4.configuration\u002F2.core-config","i-lucide-settings-2",{"title":162,"path":163,"stem":164,"icon":165},"Runtime configuration (:runtime)","\u002Fdocs\u002Fconfiguration\u002Fruntime-config","1.docs\u002F4.configuration\u002F3.runtime-config","i-lucide-server-cog",{"title":167,"path":168,"stem":169,"icon":170},"Defaults, adaptivity, and presets","\u002Fdocs\u002Fconfiguration\u002Fdefaults-and-presets","1.docs\u002F4.configuration\u002F4.defaults-and-presets","i-lucide-gauge",{"title":172,"path":173,"stem":174,"icon":175},"Kafka client configuration","\u002Fdocs\u002Fconfiguration\u002Fkafka-client-config","1.docs\u002F4.configuration\u002F5.kafka-client-config","i-lucide-plug",{"title":177,"path":178,"stem":179,"children":180,"icon":183},"Running in Production","\u002Fdocs\u002Fruntime","1.docs\u002F5.runtime\u002F1.index",[181,184,189,194,199,204,209,214],{"title":182,"path":178,"stem":179,"icon":183},"The runtime","i-lucide-server",{"title":185,"path":186,"stem":187,"icon":188},"The REST API","\u002Fdocs\u002Fruntime\u002Frest-api","1.docs\u002F5.runtime\u002F2.rest-api","i-lucide-globe",{"title":190,"path":191,"stem":192,"icon":193},"Health checks","\u002Fdocs\u002Fruntime\u002Fhealth-checks","1.docs\u002F5.runtime\u002F3.health-checks","i-lucide-heart-pulse",{"title":195,"path":196,"stem":197,"icon":198},"Metrics","\u002Fdocs\u002Fruntime\u002Fmetrics","1.docs\u002F5.runtime\u002F4.metrics","i-lucide-activity",{"title":200,"path":201,"stem":202,"icon":203},"Pause and resume","\u002Fdocs\u002Fruntime\u002Fpause-unpause","1.docs\u002F5.runtime\u002F5.pause-unpause","i-lucide-pause",{"title":205,"path":206,"stem":207,"icon":208},"Plugins and lifecycle hooks","\u002Fdocs\u002Fruntime\u002Fplugins","1.docs\u002F5.runtime\u002F6.plugins","i-lucide-puzzle",{"title":210,"path":211,"stem":212,"icon":213},"Docker images","\u002Fdocs\u002Fruntime\u002Fdocker","1.docs\u002F5.runtime\u002F7.docker","i-lucide-container",{"title":215,"path":216,"stem":217,"icon":218},"GraalVM native image","\u002Fdocs\u002Fruntime\u002Fnative-image","1.docs\u002F5.runtime\u002F8.native-image","i-lucide-zap",{"title":220,"path":221,"stem":222,"children":223,"icon":226},"Deploying & Operating","\u002Fdocs\u002Foperating","1.docs\u002F6.operating\u002F1.index",[224,227,232,237,242,247,252],{"title":225,"path":221,"stem":222,"icon":226},"Deploying and operating","i-lucide-life-buoy",{"title":228,"path":229,"stem":230,"icon":231},"Running on Kubernetes","\u002Fdocs\u002Foperating\u002Fkubernetes","1.docs\u002F6.operating\u002F2.kubernetes","i-lucide-ship",{"title":233,"path":234,"stem":235,"icon":236},"High availability","\u002Fdocs\u002Foperating\u002Fhigh-availability","1.docs\u002F6.operating\u002F3.high-availability","i-lucide-copy",{"title":238,"path":239,"stem":240,"icon":241},"Liveness and readiness probes","\u002Fdocs\u002Foperating\u002Fprobes","1.docs\u002F6.operating\u002F4.probes","i-lucide-stethoscope",{"title":243,"path":244,"stem":245,"icon":246},"Observability","\u002Fdocs\u002Foperating\u002Fobservability","1.docs\u002F6.operating\u002F5.observability","i-lucide-telescope",{"title":248,"path":249,"stem":250,"icon":251},"Tuning under load","\u002Fdocs\u002Foperating\u002Ftuning","1.docs\u002F6.operating\u002F6.tuning","i-lucide-sliders",{"title":253,"path":254,"stem":255,"icon":256},"Production checklist","\u002Fdocs\u002Foperating\u002Fproduction-checklist","1.docs\u002F6.operating\u002F7.production-checklist","i-lucide-clipboard-check",{"title":258,"path":259,"stem":260,"children":261,"icon":85},"Migrating from Kafka Streams","\u002Fdocs\u002Fmigration","1.docs\u002F7.migration\u002F1.index",[262,263,268],{"title":258,"path":259,"stem":260,"icon":85},{"title":264,"path":265,"stem":266,"icon":267},"Migration without carrying state","\u002Fdocs\u002Fmigration\u002Fwithout-data-migration","1.docs\u002F7.migration\u002F2.without-data-migration","i-lucide-sparkles",{"title":269,"path":270,"stem":271,"icon":272},"Migration carrying state","\u002Fdocs\u002Fmigration\u002Fwith-data-migration","1.docs\u002F7.migration\u002F3.with-data-migration","i-lucide-database-backup",{"title":274,"path":275,"stem":276,"children":277,"icon":279},"Reference","\u002Fdocs\u002Freference","1.docs\u002F8.reference\u002F1.index",[278,280,285,290,295,300,305,309],{"title":274,"path":275,"stem":276,"icon":279},"i-lucide-list",{"title":281,"path":282,"stem":283,"icon":284},"Configuration reference","\u002Fdocs\u002Freference\u002Fconfiguration-reference","1.docs\u002F8.reference\u002F2.configuration-reference","i-lucide-table",{"title":286,"path":287,"stem":288,"icon":289},"REST API reference","\u002Fdocs\u002Freference\u002Frest-api-reference","1.docs\u002F8.reference\u002F3.rest-api-reference","i-lucide-network",{"title":291,"path":292,"stem":293,"icon":294},"Gradle plugin reference","\u002Fdocs\u002Freference\u002Fgradle-plugin-reference","1.docs\u002F8.reference\u002F4.gradle-plugin-reference","i-lucide-box",{"title":296,"path":297,"stem":298,"icon":299},"Maven reference","\u002Fdocs\u002Freference\u002Fmaven-reference","1.docs\u002F8.reference\u002F5.maven-reference","i-simple-icons-apachemaven",{"title":301,"path":302,"stem":303,"icon":304},"Kafka Streams compatibility matrix","\u002Fdocs\u002Freference\u002Fks-compatibility-matrix","1.docs\u002F8.reference\u002F6.ks-compatibility-matrix","i-lucide-table-2",{"title":306,"path":307,"stem":308,"icon":170},"Metrics reference","\u002Fdocs\u002Freference\u002Fmetrics-reference","1.docs\u002F8.reference\u002F7.metrics-reference",{"title":310,"path":311,"stem":312,"icon":313},"Glossary","\u002Fdocs\u002Freference\u002Fglossary","1.docs\u002F8.reference\u002F8.glossary","i-lucide-book-a",{"id":315,"title":243,"body":316,"description":2295,"extension":2296,"meta":2297,"navigation":2298,"pageType":2299,"path":244,"seo":2300,"stem":245,"__hash__":2301},"docs\u002F1.docs\u002F6.operating\u002F5.observability.md",{"type":317,"value":318,"toc":2277},"minimark",[319,332,389,394,401,469,475,479,486,614,617,684,691,757,877,889,893,904,935,938,1055,1084,1098,1141,1163,1167,1172,1363,1404,1408,1419,1425,1478,1483,1494,1525,1528,1584,1601,1615,1619,1642,1656,1662,1669,1740,1743,1803,1812,1818,1824,1838,1847,1922,1939,1953,1957,1960,2034,2038,2053,2107,2133,2174,2222,2226,2273],[320,321,322,323,327,328,331],"p",{},"A StoatFlow application is one process, so observability is simpler than a clustered stream processor — there is one set of metrics, one log stream, and one HTTP surface to point your monitoring stack at. This page wires those three together: scraping ",[324,325,326],"code",{},"\u002Fmetrics"," into Prometheus and Grafana, the meters worth alerting on, reading and correlating the logs, and using the ",[324,329,330],{},"\u002Fdebug\u002F*"," endpoints when a pipeline goes quiet or slow.",[333,334,335],"tldr-panel",{},[336,337,338,355,361,374],"ul",{},[339,340,341,345,346,349,350,354],"li",{},[342,343,344],"strong",{},"Metrics:"," ",[324,347,348],{},"GET \u002Fmetrics"," in Prometheus text format — scrape it from your existing stack. The full meter catalogue is on the ",[351,352,353],"a",{"href":307},"metrics reference",".",[339,356,357,360],{},[342,358,359],{},"Logs:"," plain text to stdout out of the box; per-package levels are configurable. Enable MDC to stamp each line with the lane and barrier it belongs to, and switch the encoder to JSON if you want structured ingestion.",[339,362,363,345,366,369,370,373],{},[342,364,365],{},"Diagnosis:",[324,367,368],{},"\u002Fdebug\u002Fthreads"," and ",[324,371,372],{},"\u002Fdebug\u002Fbarriers"," give a live snapshot of what every lane is doing and where a commit is stuck — for when the dashboards say \"stopped\" but not \"why\".",[339,375,376,379,380,382,383,385,386,354],{},[342,377,378],{},"Exposure:"," keep ",[324,381,326],{},", ",[324,384,330],{},", and the introspection endpoints on the in-cluster network; only the health probes belong on public ingress. See ",[351,387,388],{"href":229},"Kubernetes",[390,391,393],"h2",{"id":392},"the-three-signals","The three signals",[320,395,396,397,400],{},"The runtime exposes everything an operator needs over the same HTTP server (default port ",[324,398,399],{},"8080","):",[402,403,404,420],"table",{},[405,406,407],"thead",{},[408,409,410,414,417],"tr",{},[411,412,413],"th",{},"Signal",[411,415,416],{},"Surface",[411,418,419],{},"Use it for",[421,422,423,438,451],"tbody",{},[408,424,425,430,435],{},[426,427,428],"td",{},[342,429,195],{},[426,431,432,434],{},[324,433,348],{}," (Prometheus)",[426,436,437],{},"Dashboards, alerting, trend lines, capacity planning.",[408,439,440,445,448],{},[426,441,442],{},[342,443,444],{},"Logs",[426,446,447],{},"stdout (plain text by default)",[426,449,450],{},"Correlating a metric spike with a specific record, lane, or barrier; post-mortems.",[408,452,453,458,466],{},[426,454,455],{},[342,456,457],{},"Diagnostic snapshots",[426,459,460,382,463],{},[324,461,462],{},"GET \u002Fdebug\u002Fthreads",[324,464,465],{},"GET \u002Fdebug\u002Fbarriers",[426,467,468],{},"On-demand \"what is it doing right now\" when a pipeline is frozen or slow.",[320,470,471,472,474],{},"Metrics and logs are continuous and cheap; the ",[324,473,330],{}," snapshots are pull-on-demand and read state the engine already keeps, so there is no steady-state cost to having them enabled.",[390,476,478],{"id":477},"scrape-metrics-into-prometheus","Scrape metrics into Prometheus",[320,480,481,482,485],{},"Metrics are enabled by default under ",[324,483,484],{},"runtime.metrics",". The endpoint serves the standard Prometheus exposition format, so any Prometheus-compatible collector scrapes it without translation.",[487,488,493],"pre",{"className":489,"code":490,"language":491,"meta":492,"style":492},"language-yaml shiki shiki-themes vitesse-light","runtime:\n  http:\n    enabled: true\n    port: 8080            # default\n  metrics:\n    enabled: true         # default\n    recording-level: info # info | debug | trace (default: info)\n    common-tags:          # applied to every meter — handy in a shared Prometheus\n      environment: production\n      service: word-count\n","yaml","",[324,494,495,508,516,529,545,553,566,581,592,603],{"__ignoreMap":492},[496,497,500,504],"span",{"class":498,"line":499},"line",1,[496,501,503],{"class":502},"su6XF","runtime",[496,505,507],{"class":506},"sYZai",":\n",[496,509,511,514],{"class":498,"line":510},2,[496,512,513],{"class":502},"  http",[496,515,507],{"class":506},[496,517,519,522,525],{"class":498,"line":518},3,[496,520,521],{"class":502},"    enabled",[496,523,524],{"class":506},":",[496,526,528],{"class":527},"sbBg2"," true\n",[496,530,532,535,537,541],{"class":498,"line":531},4,[496,533,534],{"class":502},"    port",[496,536,524],{"class":506},[496,538,540],{"class":539},"s-TwI"," 8080",[496,542,544],{"class":543},"s8zF2","            # default\n",[496,546,548,551],{"class":498,"line":547},5,[496,549,550],{"class":502},"  metrics",[496,552,507],{"class":506},[496,554,556,558,560,563],{"class":498,"line":555},6,[496,557,521],{"class":502},[496,559,524],{"class":506},[496,561,562],{"class":527}," true",[496,564,565],{"class":543},"         # default\n",[496,567,569,572,574,578],{"class":498,"line":568},7,[496,570,571],{"class":502},"    recording-level",[496,573,524],{"class":506},[496,575,577],{"class":576},"spphp"," info",[496,579,580],{"class":543}," # info | debug | trace (default: info)\n",[496,582,584,587,589],{"class":498,"line":583},8,[496,585,586],{"class":502},"    common-tags",[496,588,524],{"class":506},[496,590,591],{"class":543},"          # applied to every meter — handy in a shared Prometheus\n",[496,593,595,598,600],{"class":498,"line":594},9,[496,596,597],{"class":502},"      environment",[496,599,524],{"class":506},[496,601,602],{"class":576}," production\n",[496,604,606,609,611],{"class":498,"line":605},10,[496,607,608],{"class":502},"      service",[496,610,524],{"class":506},[496,612,613],{"class":576}," word-count\n",[320,615,616],{},"A minimal static scrape config:",[487,618,620],{"className":489,"code":619,"language":491,"meta":492,"style":492},"scrape_configs:\n  - job_name: stoatflow\n    metrics_path: \u002Fmetrics\n    static_configs:\n      - targets: [\"my-app:8080\"]\n",[324,621,622,629,642,652,659],{"__ignoreMap":492},[496,623,624,627],{"class":498,"line":499},[496,625,626],{"class":502},"scrape_configs",[496,628,507],{"class":506},[496,630,631,634,637,639],{"class":498,"line":510},[496,632,633],{"class":506},"  -",[496,635,636],{"class":502}," job_name",[496,638,524],{"class":506},[496,640,641],{"class":576}," stoatflow\n",[496,643,644,647,649],{"class":498,"line":518},[496,645,646],{"class":502},"    metrics_path",[496,648,524],{"class":506},[496,650,651],{"class":576}," \u002Fmetrics\n",[496,653,654,657],{"class":498,"line":531},[496,655,656],{"class":502},"    static_configs",[496,658,507],{"class":506},[496,660,661,664,667,669,672,676,679,681],{"class":498,"line":547},[496,662,663],{"class":506},"      -",[496,665,666],{"class":502}," targets",[496,668,524],{"class":506},[496,670,671],{"class":506}," [",[496,673,675],{"class":674},"sSP4y","\"",[496,677,678],{"class":576},"my-app:8080",[496,680,675],{"class":674},[496,682,683],{"class":506},"]\n",[320,685,686,687,690],{},"On Kubernetes, prefer service discovery over static targets. With a Prometheus Operator, point a ",[324,688,689],{},"ServiceMonitor"," at the HTTP port; without it, the pod annotations are enough for the default Kubernetes SD relabeling:",[487,692,694],{"className":489,"code":693,"language":491,"meta":492,"style":492},"# Pod template metadata — picked up by Prometheus pod service-discovery\nmetadata:\n  annotations:\n    prometheus.io\u002Fscrape: \"true\"\n    prometheus.io\u002Fpath: \"\u002Fmetrics\"\n    prometheus.io\u002Fport: \"8080\"\n",[324,695,696,701,708,715,731,744],{"__ignoreMap":492},[496,697,698],{"class":498,"line":499},[496,699,700],{"class":543},"# Pod template metadata — picked up by Prometheus pod service-discovery\n",[496,702,703,706],{"class":498,"line":510},[496,704,705],{"class":502},"metadata",[496,707,507],{"class":506},[496,709,710,713],{"class":498,"line":518},[496,711,712],{"class":502},"  annotations",[496,714,507],{"class":506},[496,716,717,720,722,725,728],{"class":498,"line":531},[496,718,719],{"class":502},"    prometheus.io\u002Fscrape",[496,721,524],{"class":506},[496,723,724],{"class":674}," \"",[496,726,727],{"class":576},"true",[496,729,730],{"class":674},"\"\n",[496,732,733,736,738,740,742],{"class":498,"line":547},[496,734,735],{"class":502},"    prometheus.io\u002Fpath",[496,737,524],{"class":506},[496,739,724],{"class":674},[496,741,326],{"class":576},[496,743,730],{"class":674},[496,745,746,749,751,753,755],{"class":498,"line":555},[496,747,748],{"class":502},"    prometheus.io\u002Fport",[496,750,524],{"class":506},[496,752,724],{"class":674},[496,754,399],{"class":576},[496,756,730],{"class":674},[487,758,760],{"className":489,"code":759,"language":491,"meta":492,"style":492},"# Or, with the Prometheus Operator: a ServiceMonitor selecting the app Service\napiVersion: monitoring.coreos.com\u002Fv1\nkind: ServiceMonitor\nmetadata:\n  name: word-count\nspec:\n  selector:\n    matchLabels:\n      app: word-count\n  endpoints:\n    - port: http        # the named container\u002FService port that maps to 8080\n      path: \u002Fmetrics\n      interval: 15s\n",[324,761,762,767,777,787,793,802,809,816,823,832,839,856,866],{"__ignoreMap":492},[496,763,764],{"class":498,"line":499},[496,765,766],{"class":543},"# Or, with the Prometheus Operator: a ServiceMonitor selecting the app Service\n",[496,768,769,772,774],{"class":498,"line":510},[496,770,771],{"class":502},"apiVersion",[496,773,524],{"class":506},[496,775,776],{"class":576}," monitoring.coreos.com\u002Fv1\n",[496,778,779,782,784],{"class":498,"line":518},[496,780,781],{"class":502},"kind",[496,783,524],{"class":506},[496,785,786],{"class":576}," ServiceMonitor\n",[496,788,789,791],{"class":498,"line":531},[496,790,705],{"class":502},[496,792,507],{"class":506},[496,794,795,798,800],{"class":498,"line":547},[496,796,797],{"class":502},"  name",[496,799,524],{"class":506},[496,801,613],{"class":576},[496,803,804,807],{"class":498,"line":555},[496,805,806],{"class":502},"spec",[496,808,507],{"class":506},[496,810,811,814],{"class":498,"line":568},[496,812,813],{"class":502},"  selector",[496,815,507],{"class":506},[496,817,818,821],{"class":498,"line":583},[496,819,820],{"class":502},"    matchLabels",[496,822,507],{"class":506},[496,824,825,828,830],{"class":498,"line":594},[496,826,827],{"class":502},"      app",[496,829,524],{"class":506},[496,831,613],{"class":576},[496,833,834,837],{"class":498,"line":605},[496,835,836],{"class":502},"  endpoints",[496,838,507],{"class":506},[496,840,842,845,848,850,853],{"class":498,"line":841},11,[496,843,844],{"class":506},"    -",[496,846,847],{"class":502}," port",[496,849,524],{"class":506},[496,851,852],{"class":576}," http",[496,854,855],{"class":543},"        # the named container\u002FService port that maps to 8080\n",[496,857,859,862,864],{"class":498,"line":858},12,[496,860,861],{"class":502},"      path",[496,863,524],{"class":506},[496,865,651],{"class":576},[496,867,869,872,874],{"class":498,"line":868},13,[496,870,871],{"class":502},"      interval",[496,873,524],{"class":506},[496,875,876],{"class":576}," 15s\n",[878,879,882],"callout",{"color":880,"icon":881},"warning","i-lucide-shield",[320,883,884,886,887,354],{},[324,885,326],{}," carries license tier and timing detail when a license is configured. Scrape it from your in-cluster monitoring stack only — never expose the metrics port through internet-facing ingress. The deployment-level exposure rules are on ",[351,888,388],{"href":229},[390,890,892],{"id":891},"build-the-grafana-view","Build the Grafana view",[320,894,895,896,899,900,903],{},"Grafana reads the same Prometheus series. A single-instance app means no per-replica aggregation to reason about — one target, one set of series, filtered by ",[324,897,898],{},"application_id"," (and your ",[324,901,902],{},"common-tags",") when several apps share a Prometheus.",[320,905,906,907,909,910,913,914,917,918,921,922,921,925,928,929,931,932,934],{},"The names below are the canonical dotted metric IDs; Prometheus rewrites ",[324,908,354],{}," to ",[324,911,912],{},"_"," and appends the type suffix (",[324,915,916],{},"_total"," for counters, ",[324,919,920],{},"_seconds_count"," \u002F ",[324,923,924],{},"_seconds_sum",[324,926,927],{},"_seconds_max"," for timers). The complete catalogue and every tag are on the ",[351,930,353],{"href":307},"; recording levels and scrape setup are on ",[351,933,195],{"href":196}," — this is the operator's working subset.",[320,936,937],{},"A starter overview row, top to bottom:",[487,939,943],{"className":940,"code":941,"language":942,"meta":492,"style":492},"language-promql shiki shiki-themes vitesse-light","# Throughput — records processed per second across all lanes\nsum(rate(stoatflow_lane_records_processed_total[1m]))\n\n# End-to-end latency — mean over the scrape window, source timestamp to processing completion.\n# e2e.latency is a plain timer: it exports _seconds_count \u002F _seconds_sum \u002F _seconds_max, no\n# _bucket series — so use the sum\u002Fcount ratio for the average and the _max gauge for the peak.\nrate(stoatflow_e2e_latency_seconds_sum[5m]) \u002F rate(stoatflow_e2e_latency_seconds_count[5m])\nstoatflow_e2e_latency_seconds_max\n\n# Commit health — barriers completing vs. failing\nrate(stoatflow_barrier_completed_total[1m])\nrate(stoatflow_barrier_failed_total[5m])\n\n# Backpressure — total queued depth as a fraction of capacity\nsum(stoatflow_lane_queue_size) \u002F sum(stoatflow_lane_queue_capacity)\n\n# Event-time progress — how far behind wall-clock the watermark is\nstoatflow_watermark_lag_ms\n\n# Consumer lag — worst partition\nmax(stoatflow_consumer_lag_records)\n","promql",[324,944,945,950,955,961,966,971,976,981,986,990,995,1000,1005,1009,1015,1021,1026,1032,1038,1043,1049],{"__ignoreMap":492},[496,946,947],{"class":498,"line":499},[496,948,949],{},"# Throughput — records processed per second across all lanes\n",[496,951,952],{"class":498,"line":510},[496,953,954],{},"sum(rate(stoatflow_lane_records_processed_total[1m]))\n",[496,956,957],{"class":498,"line":518},[496,958,960],{"emptyLinePlaceholder":959},true,"\n",[496,962,963],{"class":498,"line":531},[496,964,965],{},"# End-to-end latency — mean over the scrape window, source timestamp to processing completion.\n",[496,967,968],{"class":498,"line":547},[496,969,970],{},"# e2e.latency is a plain timer: it exports _seconds_count \u002F _seconds_sum \u002F _seconds_max, no\n",[496,972,973],{"class":498,"line":555},[496,974,975],{},"# _bucket series — so use the sum\u002Fcount ratio for the average and the _max gauge for the peak.\n",[496,977,978],{"class":498,"line":568},[496,979,980],{},"rate(stoatflow_e2e_latency_seconds_sum[5m]) \u002F rate(stoatflow_e2e_latency_seconds_count[5m])\n",[496,982,983],{"class":498,"line":583},[496,984,985],{},"stoatflow_e2e_latency_seconds_max\n",[496,987,988],{"class":498,"line":594},[496,989,960],{"emptyLinePlaceholder":959},[496,991,992],{"class":498,"line":605},[496,993,994],{},"# Commit health — barriers completing vs. failing\n",[496,996,997],{"class":498,"line":841},[496,998,999],{},"rate(stoatflow_barrier_completed_total[1m])\n",[496,1001,1002],{"class":498,"line":858},[496,1003,1004],{},"rate(stoatflow_barrier_failed_total[5m])\n",[496,1006,1007],{"class":498,"line":868},[496,1008,960],{"emptyLinePlaceholder":959},[496,1010,1012],{"class":498,"line":1011},14,[496,1013,1014],{},"# Backpressure — total queued depth as a fraction of capacity\n",[496,1016,1018],{"class":498,"line":1017},15,[496,1019,1020],{},"sum(stoatflow_lane_queue_size) \u002F sum(stoatflow_lane_queue_capacity)\n",[496,1022,1024],{"class":498,"line":1023},16,[496,1025,960],{"emptyLinePlaceholder":959},[496,1027,1029],{"class":498,"line":1028},17,[496,1030,1031],{},"# Event-time progress — how far behind wall-clock the watermark is\n",[496,1033,1035],{"class":498,"line":1034},18,[496,1036,1037],{},"stoatflow_watermark_lag_ms\n",[496,1039,1041],{"class":498,"line":1040},19,[496,1042,960],{"emptyLinePlaceholder":959},[496,1044,1046],{"class":498,"line":1045},20,[496,1047,1048],{},"# Consumer lag — worst partition\n",[496,1050,1052],{"class":498,"line":1051},21,[496,1053,1054],{},"max(stoatflow_consumer_lag_records)\n",[878,1056,1059],{"color":1057,"icon":1058},"info","i-lucide-info",[320,1060,1061,1062,1065,1066,1069,1070,1073,1074,1077,1078,1081,1082,354],{},"True percentiles (P95\u002FP99) are available only for the timers registered with percentile publishing — for example the barrier commit and alignment timers, which export a ",[324,1063,1064],{},"{quantile=\"0.99\"}"," label series you read directly, e.g. ",[324,1067,1068],{},"stoatflow_barrier_latency_seconds{quantile=\"0.99\"}",". The end-to-end and per-record processing-latency timers are plain timers: they publish count, sum, and max only, so a ",[324,1071,1072],{},"histogram_quantile(...)"," over a ",[324,1075,1076],{},"_bucket"," series returns no data for them. Use the sum\u002Fcount average and the ",[324,1079,1080],{},"_max"," gauge for those, as above. Which timers carry percentiles is listed on ",[351,1083,195],{"href":196},[320,1085,1086,1087,1090,1091,369,1094,1097],{},"With ",[324,1088,1089],{},"bind-jvm-metrics: true"," (default) the standard ",[324,1092,1093],{},"jvm_*",[324,1095,1096],{},"system_*"," series land on the same endpoint, so a second row covers the runtime itself:",[487,1099,1101],{"className":940,"code":1100,"language":942,"meta":492,"style":492},"# Heap used vs. max\njvm_memory_used_bytes{area=\"heap\"} \u002F jvm_memory_max_bytes{area=\"heap\"}\n\n# GC pause rate\nrate(jvm_gc_pause_seconds_sum[5m])\n\n# CPU\nprocess_cpu_usage\n",[324,1102,1103,1108,1113,1117,1122,1127,1131,1136],{"__ignoreMap":492},[496,1104,1105],{"class":498,"line":499},[496,1106,1107],{},"# Heap used vs. max\n",[496,1109,1110],{"class":498,"line":510},[496,1111,1112],{},"jvm_memory_used_bytes{area=\"heap\"} \u002F jvm_memory_max_bytes{area=\"heap\"}\n",[496,1114,1115],{"class":498,"line":518},[496,1116,960],{"emptyLinePlaceholder":959},[496,1118,1119],{"class":498,"line":531},[496,1120,1121],{},"# GC pause rate\n",[496,1123,1124],{"class":498,"line":547},[496,1125,1126],{},"rate(jvm_gc_pause_seconds_sum[5m])\n",[496,1128,1129],{"class":498,"line":555},[496,1130,960],{"emptyLinePlaceholder":959},[496,1132,1133],{"class":498,"line":568},[496,1134,1135],{},"# CPU\n",[496,1137,1138],{"class":498,"line":583},[496,1139,1140],{},"process_cpu_usage\n",[878,1142,1143],{"color":1057,"icon":1058},[320,1144,1145,1146,1149,1150,382,1153,382,1156,1159,1160,354],{},"Lane ids follow the Kafka Streams task convention ",[324,1147,1148],{},"{subtopology}_{lane}"," — ",[324,1151,1152],{},"0_0",[324,1154,1155],{},"0_15",[324,1157,1158],{},"1_7",". To break a panel down per sub-topology, group by a regex matcher: ",[324,1161,1162],{},"sum by (lane_id) (rate(stoatflow_lane_records_processed_total{lane_id=~\"1_.*\"}[1m]))",[390,1164,1166],{"id":1165},"alert-on-the-right-meters","Alert on the right meters",[320,1168,1169,1170,354],{},"For a single-instance app, the alerts that matter are the ones that tell you the one process is stuck, falling behind, or about to be evicted — and the license, because an expired license takes the readiness probe down. Wire these into Alertmanager (or your equivalent); the full rationale for each meter is on ",[351,1171,195],{"href":196},[402,1173,1174,1190],{},[405,1175,1176],{},[408,1177,1178,1181,1184,1187],{},[411,1179,1180],{},"Alert",[411,1182,1183],{},"Condition",[411,1185,1186],{},"Severity",[411,1188,1189],{},"What it means",[421,1191,1192,1210,1230,1245,1261,1276,1291,1306,1325,1344],{},[408,1193,1194,1197,1202,1205],{},[426,1195,1196],{},"Barrier failures",[426,1198,1199],{},[324,1200,1201],{},"rate(stoatflow_barrier_failed_total[5m]) > 0",[426,1203,1204],{},"Critical",[426,1206,1207,1208,354],{},"A commit transaction is failing — the process aborts and restarts on a fatal commit error. Pair with ",[324,1209,372],{},[408,1211,1212,1215,1221,1223],{},[426,1213,1214],{},"Not ready",[426,1216,1217,1220],{},[324,1218,1219],{},"stoatflow_client_state != 4"," for 5m (while expected up)",[426,1222,1204],{},[426,1224,1225,1226,1229],{},"The app left ",[324,1227,1228],{},"RUNNING"," — validating, restoring, paused, or stopping.",[408,1231,1232,1235,1240,1242],{},[426,1233,1234],{},"License invalid",[426,1236,1237],{},[324,1238,1239],{},"stoatflow_license_valid == 0",[426,1241,1204],{},[426,1243,1244],{},"Readiness goes 503; renew before the grace budget runs out.",[408,1246,1247,1250,1255,1258],{},[426,1248,1249],{},"License expiring",[426,1251,1252],{},[324,1253,1254],{},"stoatflow_license_days_remaining \u003C 14",[426,1256,1257],{},"Warning",[426,1259,1260],{},"Lead time to renew.",[408,1262,1263,1266,1271,1273],{},[426,1264,1265],{},"Watermark stall",[426,1267,1268],{},[324,1269,1270],{},"delta(stoatflow_watermark_current[5m]) == 0 and stoatflow_client_state == 4",[426,1272,1257],{},[426,1274,1275],{},"Running but event time isn't advancing — windows won't close.",[408,1277,1278,1281,1286,1288],{},[426,1279,1280],{},"High consumer lag",[426,1282,1283],{},[324,1284,1285],{},"max(stoatflow_consumer_lag_records) > 100000",[426,1287,1257],{},[426,1289,1290],{},"Falling behind the input; check throughput and backpressure.",[408,1292,1293,1296,1301,1303],{},[426,1294,1295],{},"Backpressure",[426,1297,1298],{},[324,1299,1300],{},"sum(stoatflow_lane_queue_size) > 0.8 * sum(stoatflow_lane_queue_capacity)",[426,1302,1257],{},[426,1304,1305],{},"Lanes can't keep up; a slow operator or downstream call is filling the queues.",[408,1307,1308,1311,1316,1318],{},[426,1309,1310],{},"Engine restarts on faults",[426,1312,1313],{},[324,1314,1315],{},"rate(stoatflow_engine_restart_total{trigger=\"replace_thread\"}[10m]) > 0",[426,1317,1257],{},[426,1319,1320,1321,1324],{},"Processing faults are recovering via in-place engine restart (",[324,1322,1323],{},"REPLACE_THREAD","). Recoveries are clean, but a sustained rate approaches the restart budget and ends in a terminal shutdown — find the root-cause exception in the logs.",[408,1326,1327,1330,1335,1337],{},[426,1328,1329],{},"Standby lag (HA)",[426,1331,1332],{},[324,1333,1334],{},"stoatflow_ha_standby_replication_lag_ms > 5000",[426,1336,1257],{},[426,1338,1339,1340,1343],{},"Under ",[351,1341,1342],{"href":234},"hot standby",", the standby is falling behind — promotion would have catch-up to do.",[408,1345,1346,1349,1354,1356],{},[426,1347,1348],{},"Redundancy below desired (HA)",[426,1350,1351],{},[324,1352,1353],{},"stoatflow_ha_redundancy_below_desired == 1",[426,1355,1204],{},[426,1357,1358,1359,1362],{},"Under hot standby, the active has fewer caught-up standby spares than ",[324,1360,1361],{},"desired-standbys"," — reduced (or no) warm failover redundancy.",[320,1364,1365,1366,1368,1369,1372,1373,382,1376,382,1379,382,1382,382,1385,1388,1389,1395,1396,1398,1399,369,1401,354],{},"The standby alerts apply only when ",[351,1367,1342],{"href":234}," is enabled. The ",[324,1370,1371],{},"stoatflow.client.state"," gauge reports the ordinal of the application lifecycle state. The states are numbered in the order the app moves through them on startup — ",[324,1374,1375],{},"CREATED=0",[324,1377,1378],{},"STARTING=1",[324,1380,1381],{},"VALIDATING_STATE=2",[324,1383,1384],{},"RESTORING=3",[324,1386,1387],{},"RUNNING=4"," — so ",[342,1390,1391,1394],{},[324,1392,1393],{},"== 4"," is the \"healthy and processing\" predicate"," the alerts above gate on, and any value below 4 means the app has not yet reached (or has left) ",[324,1397,1228],{},". The complete state-to-ordinal mapping and the readiness contract are on ",[351,1400,190],{"href":191},[351,1402,1403],{"href":239},"Probes",[390,1405,1407],{"id":1406},"logs-and-correlation","Logs and correlation",[320,1409,1410,1411,1414,1415,1418],{},"The runtime logs to stdout. Out of the box that is ",[342,1412,1413],{},"plain text"," — Logback's pattern encoder, one line per event — which is fine for ",[324,1416,1417],{},"kubectl logs"," and local development. If you want structured ingestion into Loki, Elasticsearch, CloudWatch, or any pipeline that parses JSON, add a JSON encoder to your application's Logback configuration; StoatFlow does not impose one.",[320,1420,1421,1422,524],{},"Set per-package log levels through config (applied to Logback at startup) without rebuilding or editing ",[324,1423,1424],{},"logback.xml",[487,1426,1428],{"className":489,"code":1427,"language":491,"meta":492,"style":492},"logging:\n  level:\n    root: INFO\n    io.stoatflow: INFO\n    # Quieten the Kafka clients in steady state:\n    org.apache.kafka: WARN\n",[324,1429,1430,1437,1444,1454,1463,1468],{"__ignoreMap":492},[496,1431,1432,1435],{"class":498,"line":499},[496,1433,1434],{"class":502},"logging",[496,1436,507],{"class":506},[496,1438,1439,1442],{"class":498,"line":510},[496,1440,1441],{"class":502},"  level",[496,1443,507],{"class":506},[496,1445,1446,1449,1451],{"class":498,"line":518},[496,1447,1448],{"class":502},"    root",[496,1450,524],{"class":506},[496,1452,1453],{"class":576}," INFO\n",[496,1455,1456,1459,1461],{"class":498,"line":531},[496,1457,1458],{"class":502},"    io.stoatflow",[496,1460,524],{"class":506},[496,1462,1453],{"class":576},[496,1464,1465],{"class":498,"line":547},[496,1466,1467],{"class":543},"    # Quieten the Kafka clients in steady state:\n",[496,1469,1470,1473,1475],{"class":498,"line":555},[496,1471,1472],{"class":502},"    org.apache.kafka",[496,1474,524],{"class":506},[496,1476,1477],{"class":576}," WARN\n",[1479,1480,1482],"h3",{"id":1481},"correlating-a-log-line-to-a-lane-and-barrier","Correlating a log line to a lane and barrier",[320,1484,1485,1486,1489,1490,1493],{},"Lane and barrier identifiers are not in the log output by default. The engine can publish them via SLF4J's MDC (Mapped Diagnostic Context), but ",[342,1487,1488],{},"MDC is off unless you opt in"," — set ",[324,1491,1492],{},"stoatflow.processing.mdc-enabled: true"," in your config:",[487,1495,1497],{"className":489,"code":1496,"language":491,"meta":492,"style":492},"stoatflow:\n  processing:\n    mdc-enabled: true   # default: false\n",[324,1498,1499,1506,1513],{"__ignoreMap":492},[496,1500,1501,1504],{"class":498,"line":499},[496,1502,1503],{"class":502},"stoatflow",[496,1505,507],{"class":506},[496,1507,1508,1511],{"class":498,"line":510},[496,1509,1510],{"class":502},"  processing",[496,1512,507],{"class":506},[496,1514,1515,1518,1520,1522],{"class":498,"line":518},[496,1516,1517],{"class":502},"    mdc-enabled",[496,1519,524],{"class":506},[496,1521,562],{"class":527},[496,1523,1524],{"class":543},"   # default: false\n",[320,1526,1527],{},"With it enabled, the engine populates these MDC keys while processing:",[336,1529,1530,1555,1566],{},[339,1531,1532,1537,1538,1540,1541,382,1543,1545,1546,1549,1550,1552,1553,354],{},[342,1533,1534],{},[324,1535,1536],{},"laneId"," — the lane handling the record or barrier, in the ",[324,1539,1148],{}," form (",[324,1542,1152],{},[324,1544,1158],{},"). This is the same value the metrics carry as the ",[324,1547,1548],{},"lane_id"," tag, so you can pivot from a per-lane Grafana panel to that lane's log lines — note the key name differs by case: MDC uses ",[324,1551,1536],{},", the metric tag is ",[324,1554,1548],{},[339,1556,1557,1562,1563,1565],{},[342,1558,1559],{},[324,1560,1561],{},"barrierId"," — the monotonically increasing commit-barrier identifier. The same id appears in the ",[324,1564,372],{}," snapshot, so a slow commit can be followed across logs and the debug endpoint.",[339,1567,1568,382,1573,382,1578,1583],{},[342,1569,1570],{},[324,1571,1572],{},"sourceTopic",[342,1574,1575],{},[324,1576,1577],{},"sourcePartition",[342,1579,1580],{},[324,1581,1582],{},"sourceOffset"," — the input record's coordinates, for tracing a specific record through the topology.",[320,1585,1586,1587,369,1590,1593,1594,382,1597,1600],{},"These keys only reach your log lines if your Logback pattern references them — for example ",[324,1588,1589],{},"%X{laneId}",[324,1591,1592],{},"%X{barrierId}"," in a pattern encoder, or the equivalent fields in a JSON encoder. Operator names you set explicitly (",[324,1595,1596],{},"Named.as(...)",[324,1598,1599],{},"Consumed.as(...)",", …) appear in the topology-related log messages themselves, so a log entry can still be tied back to a specific node in the DAG without MDC.",[878,1602,1603],{"color":1057,"icon":1058},[320,1604,1605,1606,1608,1609,1611,1612,1614],{},"Correlation works because the identifiers are shared, not because of distributed tracing. Once MDC is enabled and your encoder emits the keys, filter logs to one ",[324,1607,1536],{}," or one ",[324,1610,1561],{}," and you're looking at exactly the records and commit the corresponding metric series and ",[324,1613,372],{}," snapshot describe — no trace context to propagate.",[390,1616,1618],{"id":1617},"diagnose-a-frozen-or-slow-pipeline","Diagnose a frozen or slow pipeline",[320,1620,1621,1622,1626,1627,1629,1630,1633,1634,1637,1638,1641],{},"Metrics tell you ",[1623,1624,1625],"em",{},"that"," throughput dropped to zero or a commit stopped advancing. The two ",[324,1628,330],{}," endpoints tell you ",[1623,1631,1632],{},"why",", by snapshotting state the engine already holds. Reach for them when a dashboard shows a stall — ",[324,1635,1636],{},"stoatflow_barrier_completed_total"," flat, ",[324,1639,1640],{},"stoatflow_consumer_lag_records"," climbing, throughput at zero — and you need to see what the process is actually doing right now.",[320,1643,1644,1645,1648,1649,1651,1652,1655],{},"Both are gated by ",[324,1646,1647],{},"runtime.http.debug.enabled"," (default ",[324,1650,727],{},"); when disabled they return ",[324,1653,1654],{},"404",". They expose internal engine state, so keep them on the in-cluster network and off public ingress.",[1479,1657,1659,1661],{"id":1658},"debugthreads-what-every-lane-is-doing",[324,1660,368],{}," — what every lane is doing",[320,1663,1664,1665,1668],{},"A JSON snapshot of engine-owned and platform threads with their current state and stack traces. Use it to answer \"is a lane blocked, and on what?\" — a lane parked in a user ",[324,1666,1667],{},"Processor"," waiting on a slow REST call looks different from one waiting on a commit, and the stack tells you which.",[487,1670,1674],{"className":1671,"code":1672,"language":1673,"meta":492,"style":492},"language-bash shiki shiki-themes vitesse-light","# Full snapshot\ncurl -s localhost:8080\u002Fdebug\u002Fthreads\n\n# Only non-runnable threads (the usual starting point for a freeze)\ncurl -s 'localhost:8080\u002Fdebug\u002Fthreads?filter=stuck'\n\n# Engine-named threads and lanes only, deeper stacks\ncurl -s 'localhost:8080\u002Fdebug\u002Fthreads?filter=known&depth=80'\n","bash",[324,1675,1676,1681,1694,1698,1703,1718,1722,1727],{"__ignoreMap":492},[496,1677,1678],{"class":498,"line":499},[496,1679,1680],{"class":543},"# Full snapshot\n",[496,1682,1683,1687,1691],{"class":498,"line":510},[496,1684,1686],{"class":1685},"sySUi","curl",[496,1688,1690],{"class":1689},"sEi1f"," -s",[496,1692,1693],{"class":576}," localhost:8080\u002Fdebug\u002Fthreads\n",[496,1695,1696],{"class":498,"line":518},[496,1697,960],{"emptyLinePlaceholder":959},[496,1699,1700],{"class":498,"line":531},[496,1701,1702],{"class":543},"# Only non-runnable threads (the usual starting point for a freeze)\n",[496,1704,1705,1707,1709,1712,1715],{"class":498,"line":547},[496,1706,1686],{"class":1685},[496,1708,1690],{"class":1689},[496,1710,1711],{"class":674}," '",[496,1713,1714],{"class":576},"localhost:8080\u002Fdebug\u002Fthreads?filter=stuck",[496,1716,1717],{"class":674},"'\n",[496,1719,1720],{"class":498,"line":555},[496,1721,960],{"emptyLinePlaceholder":959},[496,1723,1724],{"class":498,"line":568},[496,1725,1726],{"class":543},"# Engine-named threads and lanes only, deeper stacks\n",[496,1728,1729,1731,1733,1735,1738],{"class":498,"line":583},[496,1730,1686],{"class":1685},[496,1732,1690],{"class":1689},[496,1734,1711],{"class":674},[496,1736,1737],{"class":576},"localhost:8080\u002Fdebug\u002Fthreads?filter=known&depth=80",[496,1739,1717],{"class":674},[320,1741,1742],{},"Query params (both optional):",[402,1744,1745,1758],{},[405,1746,1747],{},[408,1748,1749,1752,1755],{},[411,1750,1751],{},"Param",[411,1753,1754],{},"Default",[411,1756,1757],{},"Effect",[421,1759,1760,1775],{},[408,1761,1762,1767,1772],{},[426,1763,1764],{},[324,1765,1766],{},"depth",[426,1768,1769],{},[324,1770,1771],{},"40",[426,1773,1774],{},"Max stack frames per thread.",[408,1776,1777,1782,1787],{},[426,1778,1779],{},[324,1780,1781],{},"filter",[426,1783,1784],{},[324,1785,1786],{},"all",[426,1788,1789,382,1791,1794,1795,1798,1799,1802],{},[324,1790,1786],{},[324,1792,1793],{},"stuck"," (only non-",[324,1796,1797],{},"RUNNABLE"," threads), or ",[324,1800,1801],{},"known"," (engine threads and lanes only).",[320,1804,1805,1808,1809,1811],{},[324,1806,1807],{},"filter=stuck"," is the fast triage view: if throughput is zero, the threads that aren't ",[324,1810,1797],{}," are where the work has piled up. The response also surfaces any threads the JVM has detected as deadlocked.",[1479,1813,1815,1817],{"id":1814},"debugbarriers-where-a-commit-is-stuck",[324,1816,372],{}," — where a commit is stuck",[320,1819,1820,1821,1823],{},"A JSON snapshot of the commit pipeline, built to answer \"which barrier is stuck, at what phase, and what's holding it up?\" from a single call. When ",[324,1822,1636],{}," has gone flat, this is the endpoint that tells you whether the commit is waiting on lanes, queued, mid-transaction, or simply idle.",[487,1825,1827],{"className":1671,"code":1826,"language":1673,"meta":492,"style":492},"curl -s localhost:8080\u002Fdebug\u002Fbarriers\n",[324,1828,1829],{"__ignoreMap":492},[496,1830,1831,1833,1835],{"class":498,"line":499},[496,1832,1686],{"class":1685},[496,1834,1690],{"class":1689},[496,1836,1837],{"class":576}," localhost:8080\u002Fdebug\u002Fbarriers\n",[320,1839,1840,1841,1846],{},"The snapshot includes the last committed barrier and its age, any in-flight commit progress, and a derived ",[342,1842,1843],{},[324,1844,1845],{},"phase"," field — a single-string summary of where the pipeline is:",[402,1848,1849,1860],{},[405,1850,1851],{},[408,1852,1853,1857],{},[411,1854,1855],{},[324,1856,1845],{},[411,1858,1859],{},"Meaning",[421,1861,1862,1872,1882,1892,1902,1912],{},[408,1863,1864,1869],{},[426,1865,1866],{},[324,1867,1868],{},"IDLE",[426,1870,1871],{},"No commit in progress; nothing pending.",[408,1873,1874,1879],{},[426,1875,1876],{},[324,1877,1878],{},"AWAITING_LANE_ACKS",[426,1880,1881],{},"A barrier is in flight, waiting for lanes to reach it.",[408,1883,1884,1889],{},[426,1885,1886],{},[324,1887,1888],{},"WAITING_FOR_COMMIT_GATE",[426,1890,1891],{},"A barrier is pending, waiting to enter the commit.",[408,1893,1894,1899],{},[426,1895,1896],{},[324,1897,1898],{},"QUEUED_FOR_COMMIT",[426,1900,1901],{},"Barrier queued for the commit step.",[408,1903,1904,1909],{},[426,1905,1906],{},[324,1907,1908],{},"IN_TX_COMMIT",[426,1910,1911],{},"The Kafka transaction commit is executing.",[408,1913,1914,1919],{},[426,1915,1916],{},[324,1917,1918],{},"AWAITING_ASYNC_FLUSH",[426,1920,1921],{},"The commit is waiting on the asynchronous state flush.",[320,1923,1924,1925,1927,1928,1930,1931,1934,1935,1938],{},"A healthy app cycles through these quickly; a stuck app sits in one. If ",[324,1926,1845],{}," is ",[324,1929,1878],{}," and not moving, cross-reference ",[324,1932,1933],{},"\u002Fdebug\u002Fthreads?filter=stuck"," to find which lane is blocked and on what. If a commit is genuinely frozen, the runtime detects the stall, aborts the in-flight transaction, and exits so the orchestrator restarts the process — you'll see this as a spike in ",[324,1936,1937],{},"stoatflow.barrier.failed.total"," and the restarting instance entering its restoration phase.",[878,1940,1941],{"color":1057,"icon":1058},[320,1942,1943,1944,1946,1947,1950,1951,354],{},"The ",[324,1945,330],{}," endpoints are for diagnosis, not routine monitoring — they dump internal engine state on demand. Leave them enabled (free at rest), keep them off public ingress, and for hardened deployments that minimise attack surface set ",[324,1948,1949],{},"runtime.http.debug.enabled: false",". The exposure split for every endpoint is on ",[351,1952,185],{"href":186},[390,1954,1956],{"id":1955},"a-diagnosis-workflow","A diagnosis workflow",[320,1958,1959],{},"When the dashboards say something is wrong, the order that gets to a cause fastest:",[1961,1962,1963,1984,1996,2010,2023],"ol",{},[339,1964,1965,1970,1971,1973,1974,1977,1978,1980,1981,354],{},[342,1966,1967,1968],{},"Check ",[324,1969,1371],{}," — is the app ",[324,1972,1228],{}," (",[324,1975,1976],{},"4",")? If it's validating, restoring, or stopping (any value below ",[324,1979,1976],{},"), the answer is lifecycle, not a freeze. Confirm against ",[324,1982,1983],{},"\u002Fhealth\u002Fready",[339,1985,1986,1149,1989,1992,1993,1995],{},[342,1987,1988],{},"Check throughput and lag",[324,1990,1991],{},"rate(stoatflow_lane_records_processed_total[1m])"," at zero with rising ",[324,1994,1640],{}," means the process stopped making progress.",[339,1997,1998,2001,2002,2004,2005,2007,2008,354],{},[342,1999,2000],{},"Check commit progress"," — flat ",[324,2003,1636],{}," points at the commit pipeline. Pull ",[324,2006,372],{}," and read the ",[324,2009,1845],{},[339,2011,2012,2015,2016,2018,2019,2022],{},[342,2013,2014],{},"Find the blocked lane"," — if ",[324,2017,1845],{}," is waiting on lanes, ",[324,2020,2021],{},"curl '…\u002Fdebug\u002Fthreads?filter=stuck'"," and read the stacks for the lane that isn't making progress.",[339,2024,2025,2028,2029,369,2031,2033],{},[342,2026,2027],{},"Correlate with logs"," — with MDC enabled, filter the logs to the offending ",[324,2030,1536],{},[324,2032,1561],{}," to see the records and errors around the stall.",[390,2035,2037],{"id":2036},"rocksdb-metrics","RocksDB metrics",[320,2039,2040,2041,2044,2045,2048,2049,2052],{},"For persistent (RocksDB-backed) stores, StoatFlow can expose the same signals Kafka Streams does — cache hit ratios, compaction and flush activity, write stalls, memtable and SST sizes — under ",[324,2042,2043],{},"stoatflow.rocksdb.*",", tagged ",[324,2046,2047],{},"store_name",". They are ",[342,2050,2051],{},"off by default","; two flags gate them:",[487,2054,2056],{"className":489,"code":2055,"language":491,"meta":492,"style":492},"stoatflow:\n  rocks-db:\n    metrics:\n      enabled: true             # property + block-cache gauges (recording-level info) — negligible cost\n      statistics-enabled: true  # + ticker\u002Fhistogram\u002Fratio family (recording-level debug) — RocksDB write-path\n                                # cost, commonly cited ~5–10% on write-heavy loads; wired at store open (restart to toggle)\n",[324,2057,2058,2064,2071,2078,2090,2102],{"__ignoreMap":492},[496,2059,2060,2062],{"class":498,"line":499},[496,2061,1503],{"class":502},[496,2063,507],{"class":506},[496,2065,2066,2069],{"class":498,"line":510},[496,2067,2068],{"class":502},"  rocks-db",[496,2070,507],{"class":506},[496,2072,2073,2076],{"class":498,"line":518},[496,2074,2075],{"class":502},"    metrics",[496,2077,507],{"class":506},[496,2079,2080,2083,2085,2087],{"class":498,"line":531},[496,2081,2082],{"class":502},"      enabled",[496,2084,524],{"class":506},[496,2086,562],{"class":527},[496,2088,2089],{"class":543},"             # property + block-cache gauges (recording-level info) — negligible cost\n",[496,2091,2092,2095,2097,2099],{"class":498,"line":547},[496,2093,2094],{"class":502},"      statistics-enabled",[496,2096,524],{"class":506},[496,2098,562],{"class":527},[496,2100,2101],{"class":543},"  # + ticker\u002Fhistogram\u002Fratio family (recording-level debug) — RocksDB write-path\n",[496,2103,2104],{"class":498,"line":555},[496,2105,2106],{"class":543},"                                # cost, commonly cited ~5–10% on write-heavy loads; wired at store open (restart to toggle)\n",[320,2108,2109,2110,2113,2114,2117,2118,2121,2122,921,2125,2128,2129,2132],{},"The property\u002Fcache half (",[324,2111,2112],{},"enabled",") is cheap — a handful of native reads per store per minute — and is what confirms a sizing diagnosis: ",[324,2115,2116],{},"stoatflow_rocksdb_shared_block_cache_usage_bytes"," vs ",[324,2119,2120],{},"..._capacity_bytes"," tells you whether the block cache is full, and ",[324,2123,2124],{},"stoatflow_rocksdb_estimate_num_keys",[324,2126,2127],{},"..._total_sst_files_size_bytes"," track state growth. The statistics half (",[324,2130,2131],{},"statistics-enabled",") adds the health signals that need RocksDB's internal counters:",[336,2134,2135,2159,2168],{},[339,2136,2137,2140,2141,2144,2145,2144,2148,2150,2151,2154,2155,2158],{},[324,2138,2139],{},"stoatflow_rocksdb_block_cache_hit_ratio"," (+ ",[324,2142,2143],{},"data","\u002F",[324,2146,2147],{},"index",[324,2149,1781],{}," variants) — ",[342,2152,2153],{},"per-interval","; a falling ratio under load is the classic \"cache too small\" symptom (see ",[351,2156,2157],{"href":249},"Tuning → RocksDB",").",[339,2160,2161,369,2164,2167],{},[324,2162,2163],{},"stoatflow_rocksdb_write_stall_duration_avg_ms",[324,2165,2166],{},"..._total_ms"," — writes stalling on compaction back-pressure.",[339,2169,2170,2173],{},[324,2171,2172],{},"rate(stoatflow_rocksdb_compaction_bytes_written_total[5m])"," and the compaction\u002Fflush time histograms — compaction load, useful when segment-checkpoint churn is suspected.",[320,2175,2176,2177,2144,2180,2144,2183,2186,2187,2190,2191,2194,2195,2198,2199,2201,2202,2205,2206,2144,2208,2210,2211,2214,2215,2217,2218,2221],{},"Ratios and histogram ",[324,2178,2179],{},"avg",[324,2181,2182],{},"min",[324,2184,2185],{},"max"," are per-interval gauges; tickers are ",[324,2188,2189],{},".total"," counters — use PromQL ",[324,2192,2193],{},"rate()",". Two caveats: the counters ",[342,2196,2197],{},"reset on an in-place engine restart"," (a normal Prometheus counter reset, absorbed by ",[324,2200,2193],{},"), and on the default (FFM) backend the histogram ",[324,2203,2204],{},"*.min.ms"," series is unavailable — use ",[324,2207,2179],{},[324,2209,2185],{},". The block cache is shared across all stores, so ",[324,2212,2213],{},"stoatflow_rocksdb_shared_block_cache_*"," (no ",[324,2216,2047],{},") is the global truth; the per-store ",[324,2219,2220],{},"stoatflow_rocksdb_block_cache_*"," gauges repeat it for Kafka Streams dashboard portability.",[390,2223,2225],{"id":2224},"next-steps","Next steps",[336,2227,2228,2237,2244,2251,2259,2266],{},[339,2229,2230,2234,2235,354],{},[342,2231,2232],{},[351,2233,195],{"href":196}," — scraping, recording levels, tags, and license metrics; the complete meter catalogue is on the ",[351,2236,353],{"href":307},[339,2238,2239,2243],{},[342,2240,2241],{},[351,2242,1403],{"href":239}," — liveness and readiness, and how readiness holds traffic off during restoration.",[339,2245,2246,2250],{},[342,2247,2248],{},[351,2249,185],{"href":186}," — every operational endpoint, with the in-cluster vs. public exposure split.",[339,2252,2253,2258],{},[342,2254,2255],{},[351,2256,2257],{"href":249},"Tuning"," — the bound knobs behind the self-tuning commit cadence, and the latency\u002Fthroughput\u002Frestart trade-offs.",[339,2260,2261,2265],{},[342,2262,2263],{},[351,2264,233],{"href":234}," — the hot-standby meters and what to alert on for the cluster.",[339,2267,2268,2272],{},[342,2269,2270],{},[351,2271,253],{"href":254}," — the pre-flight list before going live.",[2274,2275,2276],"style",{},"html pre.shiki code .su6XF, html code.shiki .su6XF{--shiki-default:#998418}html pre.shiki code .sYZai, html code.shiki .sYZai{--shiki-default:#999999}html pre.shiki code .sbBg2, html code.shiki .sbBg2{--shiki-default:#1E754F}html pre.shiki code .s-TwI, html code.shiki .s-TwI{--shiki-default:#2F798A}html pre.shiki code .s8zF2, html code.shiki .s8zF2{--shiki-default:#A0ADA0}html pre.shiki code .spphp, html code.shiki .spphp{--shiki-default:#B56959}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html pre.shiki code .sSP4y, html code.shiki .sSP4y{--shiki-default:#B5695977}html pre.shiki code .sySUi, html code.shiki .sySUi{--shiki-default:#59873A}html pre.shiki code .sEi1f, html code.shiki .sEi1f{--shiki-default:#A65E2B}",{"title":492,"searchDepth":510,"depth":510,"links":2278},[2279,2280,2281,2282,2283,2286,2292,2293,2294],{"id":392,"depth":510,"text":393},{"id":477,"depth":510,"text":478},{"id":891,"depth":510,"text":892},{"id":1165,"depth":510,"text":1166},{"id":1406,"depth":510,"text":1407,"children":2284},[2285],{"id":1481,"depth":518,"text":1482},{"id":1617,"depth":510,"text":1618,"children":2287},[2288,2290],{"id":1658,"depth":518,"text":2289},"\u002Fdebug\u002Fthreads — what every lane is doing",{"id":1814,"depth":518,"text":2291},"\u002Fdebug\u002Fbarriers — where a commit is stuck",{"id":1955,"depth":510,"text":1956},{"id":2036,"depth":510,"text":2037},{"id":2224,"depth":510,"text":2225},"Operating-time visibility for a single-instance app — scrape \u002Fmetrics into Prometheus and Grafana, the meters worth alerting on, correlating logs to lanes and barriers via opt-in MDC, and the \u002Fdebug\u002Fthreads and \u002Fdebug\u002Fbarriers endpoints for diagnosing a frozen or slow pipeline.","md",{},{"icon":246},"how-to",{"title":243,"description":2295},"nEOdljlF5cMyDv_8kf_N2rcm4CR1ymXhALsDj_cr65c",[2303,2305],{"title":238,"path":239,"stem":240,"description":2304,"icon":241,"children":-1},"Wire StoatFlow's \u002Fhealth\u002Flive and \u002Fhealth\u002Fready endpoints to Kubernetes probes — what readiness gates, why a frozen commit pipeline fails liveness, and how to tune probe periods.",{"title":248,"path":249,"stem":250,"description":2306,"icon":251,"children":-1},"Practical tuning for StoatFlow under load — lane count, commit-cadence bounds, the memory and uncommitted-state knobs, and RocksDB sizing for large state, each tied to the symptom that tells you to reach for it.",1783713362991]