[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":-5,"_8":9,"_10":11},"actionData","errors","root",{"_221":-5,"_335":-5,"_336":337,"_338":339,"_340":341,"_342":343,"_344":345,"_346":347,"_348":349,"_350":351,"_352":353},"routes/_store+/_readers+/read_/route","routes/_store+/_readers+/read_.$slug/route",{"_16":227,"_228":229,"_230":231,"_232":51,"_233":-7},"routes/_store+/_readers+/read_.$slug.$chapterId/route",{"_12":13,"_14":15,"_16":17,"_18":19,"_20":21,"_22":23,"_24":25},"title","Beyond Uptime: Reliability Engineering for AI Infrastructure","content","

Operation Slowburn

https://leanpub.com/forensic-ai

How can a system be healthy and still be failing?

https://leanpub.com/forensic-ai","book",{"_43":15,"_12":13,"_44":45,"_46":47,"_48":49,"_50":51,"_52":53,"_54":55,"_56":57},"prevChapter",{"_27":38,"_12":41,"_30":-5,"_31":42,"_33":34,"_35":-5,"_36":-5,"_37":-5},"nextChapter",{"_27":38,"_12":39,"_30":-5,"_31":40,"_33":34,"_35":-5,"_36":-5,"_37":-5},"sections",[26],"chapterTitle","Operation Slowburn",{"_27":28,"_12":29,"_30":-5,"_31":32,"_33":34,"_35":-5,"_36":-5,"_37":-5},"level",3,"How can a system be healthy and still be failing?","description","path","#how-can-a-system-be-healthy-and-still-be-failing","fileCount",1,"quizType","sectionNum","youtubeId",2,"The Synchronization Problem","#chapter3-sync-prob","How AI Infrastructure Actually Works","#chapter1-main-block","publicChapterContent","metaText","Master AI infrastructure reliability! Debug silent GPU cluster stalls, PFC pause cascades, PCIe downtraining, and MFU drops across distributed compute.","slug","forensic-ai","showReadLinkString","sample","embargo",false,"coverFeatured",{"_225":226},"authors",[216],"tocStructure",[58,59,19,60,61,62,63,64,65,66,26,21,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110],{"_27":34,"_12":214,"_30":-5,"_31":215,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":34,"_12":212,"_30":-5,"_31":213,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":210,"_30":-5,"_31":211,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":208,"_30":-5,"_31":209,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":206,"_30":-5,"_31":207,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":204,"_30":-5,"_31":205,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":202,"_30":-5,"_31":203,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":200,"_30":-5,"_31":201,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":38,"_12":25,"_30":-5,"_31":199,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":197,"_30":-5,"_31":198,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":195,"_30":-5,"_31":196,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":193,"_30":-5,"_31":194,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":191,"_30":-5,"_31":192,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":38,"_12":189,"_30":-5,"_31":190,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":187,"_30":-5,"_31":188,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":185,"_30":-5,"_31":186,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":38,"_12":183,"_30":-5,"_31":184,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":181,"_30":-5,"_31":182,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":179,"_30":-5,"_31":180,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":177,"_30":-5,"_31":178,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":175,"_30":-5,"_31":176,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":173,"_30":-5,"_31":174,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":38,"_12":171,"_30":-5,"_31":172,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":169,"_30":-5,"_31":170,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":167,"_30":-5,"_31":168,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":165,"_30":-5,"_31":166,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":163,"_30":-5,"_31":164,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":38,"_12":161,"_30":-5,"_31":162,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":159,"_30":-5,"_31":160,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":157,"_30":-5,"_31":158,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":155,"_30":-5,"_31":156,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":153,"_30":-5,"_31":154,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":151,"_30":-5,"_31":152,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":149,"_30":-5,"_31":150,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":38,"_12":147,"_30":-5,"_31":148,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":145,"_30":-5,"_31":146,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":143,"_30":-5,"_31":144,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":141,"_30":-5,"_31":142,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":139,"_30":-5,"_31":140,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":137,"_30":-5,"_31":138,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":38,"_12":135,"_30":-5,"_31":136,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":133,"_30":-5,"_31":134,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":131,"_30":-5,"_31":132,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":129,"_30":-5,"_31":130,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":127,"_30":-5,"_31":128,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":125,"_30":-5,"_31":126,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":123,"_30":-5,"_31":124,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":38,"_12":121,"_30":-5,"_31":122,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":119,"_30":-5,"_31":120,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":117,"_30":-5,"_31":118,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":115,"_30":-5,"_31":116,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":113,"_30":-5,"_31":114,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":34,"_12":111,"_30":-5,"_31":112,"_33":34,"_35":-5,"_36":-5,"_37":-5},"References","#references","An Agenda for Further Research","#an-agenda-for-further-research","An example of a Harnessed Agent Cluster running an AI model","#an-example-of-a-harnessed-agent-cluster-running-an-ai-model","Toward Autonomous AI Infrastructure","#toward-autonomous-ai-infrastructure","Machine Learning as the Investigator, Not Only the Instrument","#machine-learning-as-the-investigator-not-only-the-instrument","AI Investigating AI : Towards Autonomous Infrastructure","#chapter10-auto-ai","Organizational Practices: Building the Team Around the Method","#organizational-practices-building-the-team-around-the-method","Preventive Engineering Practices","#preventive-engineering-practices","A Concrete Detection Formalism: From Fleet Averages to Per-Rank Tail Statistics","#chapter9-sec4-per-rank","Toward Automated Detection","#chap9-sec3-auto-detect","From Forensics to Foresight","#chapter9-sec1-forensic","Why Generalization Is Possible at All","#why-generalization-is-possible-at-all","Generalizing the Incident :A Practitioner Framework","#chapter9-prac-framework","The Failure Graph as a Formal Structure","#the-failure-graph-as-a-formal-structure","A Divergent Presentation of the Same Failure Class","#a-divergent-presentation-of-the-same-failure-class","Systematic , Layer-by-Layer Method","#systematic--layer-by-layer-method","Observability as Evidence Collection","#observability-as-evidence-collection","The Fleet Aggregation Paradox","#the-fleet-aggregation-paradox","Systematic RCA - The Failure Graph","#chapter8-fail-graph","The Structural Argument","#the-structural-argument","Memory Subsystem Degradation","#memory-subsystem-degradation","Passive Hardware Telemetry Checklist","#passive-hardware-telemetry-checklist","Manufacturing Variance & “Mercurial Cores”","#manufacturing-variance--mercurial-cores","Cosmic Radiation & Single-Event Upsets (SEUs)","#cosmic-radiation--single-event-upsets-seus","Transient Voltage Fluctuations & Thermal Noise","#transient-voltage-fluctuations--thermal-noise","The Anatomy of Silent Data Corruption","#chapter7-silent-dc","The Diagnostic Implication","#the-diagnostic-implication","Hash Polarization: A Second Congestion Mechanism","#chapter6-sec2-hash-pol","Priority Flow Control and Cascading Pause Frames","#priority-flow-control-and-cascading-pause-frames","The Congested Highway","#the-congested-highway","Lossless Fabric Bottlenecks","#chapter6-loss-bottles","Transport Reliability Semantics: What RDMA Guarantees and What It Does Not","#transport-reliability-semantics-what-rdma-guarantees-and-what-it-does-not","Comparing the Three Fabrics Along the Dimensions That Matter","#comparing-the-three-fabrics-along-the-dimensions-that-matter","A Cautionary Case: Fast Transport Is Not Sufficient Alone","#a-cautionary-case-fast-transport-is-not-sufficient-alone","RDMA, RoCE, and InfiniBand","#rdma-roce-and-infiniband","Limits of Ordinary Networking","#limits-of-ordinary-networking","Specialized Networking Technologies","#chapter5-special-net","The Forensic Investigation","#chapter4-sec2-forensic","Incident Premise and Cluster Configuration","#incident-premise-and-cluster-configuration","Operation Slowburn : A Forensic Reconstruction","#chapter4-op-slowburn","A Worked Capacity-Planning Illustration","#chapter3-sec4-cap-plan","Why this is unavoidable","#why-this-is-unavoidable","From the Table to the Training Run","#chapter3-sec2-table-run","An Intuitive Model: Barrier Synchronization","#an-intuitive-model-barrier-synchronization","#chapter2-op-slowburn","Beyond Network Rules : A Systems-Thinking Argument","#beyond-network-rules--a-systems-thinking-argument","The Modern AI Stack : A Glossary Through the Same Factory Lens","#the-modern-ai-stack--a-glossary-through-the-same-factory-lens","How Tiny Delays Compound in AI","#how-tiny-delays-compound-in-ai","Visualizing the AI Factory Floor","#factory-floor-visual","Why this requires GPU’s rather than CPU’s","#why-this-requires-gpus-rather-than-cpus","What is an AI Model","#what-is-ai-model","Disclaimer & Limitation of Liability","#disclaimer--limitation-of-liability","Beyond Uptime : Reliability Engineering for AI Infrastructure","#beyond-uptime--reliability-engineering-for-ai-infrastructure",{"_217":218,"_219":220,"_221":222},"id","Z2lkOi8vbGVhbnB1Yi9TdG9yZTo6UHJvZHVjdEF1dGhvci84OTU4NQ","name","Sreejith Kaimal","user",{"_223":224},"username","sree-kaimal","url","https://d2sofvawe08yqg.cloudfront.net/forensic-ai/s_featured?1788822530&1788822530",{"_12":13,"_46":47,"_44":45,"_50":51,"_52":319,"_320":321,"_54":322,"_48":49,"_323":-5,"_324":-5,"_325":326,"_327":326,"_328":329,"_330":331},"parts",[290,291,292],"toc",[234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289],"isUnavailableOnline","unavailableMessage",{"_27":34,"_12":214,"_30":-5,"_31":215,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":34,"_12":212,"_30":-5,"_31":213,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":38,"_12":41,"_30":-5,"_31":42,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":210,"_30":-5,"_31":211,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":208,"_30":-5,"_31":209,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":206,"_30":-5,"_31":207,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":204,"_30":-5,"_31":205,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":202,"_30":-5,"_31":203,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":200,"_30":-5,"_31":201,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":38,"_12":25,"_30":-5,"_31":199,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":29,"_30":-5,"_31":32,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":38,"_12":39,"_30":-5,"_31":40,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":197,"_30":-5,"_31":198,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":195,"_30":-5,"_31":196,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":193,"_30":-5,"_31":194,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":191,"_30":-5,"_31":192,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":38,"_12":189,"_30":-5,"_31":190,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":187,"_30":-5,"_31":188,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":185,"_30":-5,"_31":186,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":38,"_12":183,"_30":-5,"_31":184,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":181,"_30":-5,"_31":182,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":179,"_30":-5,"_31":180,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":177,"_30":-5,"_31":178,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":175,"_30":-5,"_31":176,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":173,"_30":-5,"_31":174,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":38,"_12":171,"_30":-5,"_31":172,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":169,"_30":-5,"_31":170,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":167,"_30":-5,"_31":168,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":165,"_30":-5,"_31":166,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":163,"_30":-5,"_31":164,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":38,"_12":161,"_30":-5,"_31":162,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":159,"_30":-5,"_31":160,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":157,"_30":-5,"_31":158,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":155,"_30":-5,"_31":156,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":153,"_30":-5,"_31":154,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":151,"_30":-5,"_31":152,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":149,"_30":-5,"_31":150,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":38,"_12":147,"_30":-5,"_31":148,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":145,"_30":-5,"_31":146,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":143,"_30":-5,"_31":144,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":141,"_30":-5,"_31":142,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":139,"_30":-5,"_31":140,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":137,"_30":-5,"_31":138,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":38,"_12":135,"_30":-5,"_31":136,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":133,"_30":-5,"_31":134,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":131,"_30":-5,"_31":132,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":129,"_30":-5,"_31":130,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":127,"_30":-5,"_31":128,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":125,"_30":-5,"_31":126,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":123,"_30":-5,"_31":124,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":38,"_12":121,"_30":-5,"_31":122,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":119,"_30":-5,"_31":120,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":117,"_30":-5,"_31":118,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":115,"_30":-5,"_31":116,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":28,"_12":113,"_30":-5,"_31":114,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_27":34,"_12":111,"_30":-5,"_31":112,"_33":34,"_35":-5,"_36":-5,"_37":-5},{"_12":214,"_30":293,"_33":34,"_31":215,"_294":318},{"_12":212,"_30":293,"_33":34,"_31":213,"_294":296},{"_12":111,"_30":293,"_33":34,"_31":112,"_294":295},"","chapters",[],[297,298,299,300,301,302,303,304,305,306],{"_27":38,"_12":41,"_31":42,"_33":34,"_307":317,"_37":-5},{"_27":38,"_12":25,"_31":199,"_33":34,"_307":316,"_37":-5},{"_27":38,"_12":39,"_31":40,"_33":34,"_307":315,"_37":-5},{"_27":38,"_12":189,"_31":190,"_33":34,"_307":314,"_37":-5},{"_27":38,"_12":183,"_31":184,"_33":34,"_307":313,"_37":-5},{"_27":38,"_12":171,"_31":172,"_33":34,"_307":312,"_37":-5},{"_27":38,"_12":161,"_31":162,"_33":34,"_307":311,"_37":-5},{"_27":38,"_12":147,"_31":148,"_33":34,"_307":310,"_37":-5},{"_27":38,"_12":135,"_31":136,"_33":34,"_307":309,"_37":-5},{"_27":38,"_12":121,"_31":122,"_33":34,"_307":308,"_37":-5},"subsections",[285,286,287,288],[278,279,280,281,282,283],[272,273,274,275,276],[265,266,267,268,269,270],[260,261,262,263],[254,255,256,257,258],[251,252],[246,247,248,249],[244],[237,238,239,240,241,242],[],{"_225":226},"coverFeatured2x",{"_225":334},[332],"freePdfUrl","freeEpubUrl","hasPdfSample",true,"hasEpubSample","samplePdfUrl","https://leanpub.com/forensic-ai/sample.pdf","sampleEpubUrl","https://leanpub.com/forensic-ai/sample.epub",{"_217":218,"_219":220,"_221":333},{"_223":224},"https://d2sofvawe08yqg.cloudfront.net/forensic-ai/s_featured2x?1788822530&1788822530","shoppingCartId","shoppingCart",{"_217":366,"_367":368,"_369":326,"_370":293,"_371":372,"_373":374,"_375":326},"locale","en","readerTheme","light","constants",{"_364":365},"publishers",{"_360":361,"_362":363},"learnerOrganizations",[],"announcements",{"_354":355,"_356":-5},"sentryEnvironment","production","sentryRelease","34b39a99cf18488c56fb0175867f9e503091b00e","announcement_store",{"_357":51,"_358":359},"announcement_authors","flag","text","

Kick off your book project, get started with GhostAI, get better at marketing, or spend the day doing all three! Free live workshops on Zoom. Saturday, June 27, 2026.

\n","companies",[],"universities",[],"paypalClientId","AX53Cwz8jR6rVVzzZiZNWWXl0dGagi-GZrjG-S3Ua14E5nrKMn7DxDQ730O4R8LusLPp49ALn6aRVaEy","no-cart","count",0,"free","updatedAt","paymentGateway","STRIPE","cartItems",[],"loading"]