[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":-5,"_8":9,"_10":11},"actionData","errors","root",{"_221":-5,"_335":-5,"_336":337,"_338":339,"_340":341,"_342":343,"_344":345,"_346":347,"_348":349,"_350":351,"_352":353},"routes/_store+/_readers+/read_/route","routes/_store+/_readers+/read_.$slug/route",{"_16":227,"_228":229,"_230":231,"_232":32,"_233":-7},"routes/_store+/_readers+/read_.$slug.$chapterId/route",{"_12":13,"_14":15,"_16":17,"_18":-5,"_19":-5,"_20":21,"_22":23},"title","Beyond Uptime: Reliability Engineering for AI Infrastructure","content","

Beyond Uptime : Reliability Engineering for AI Infrastructure

https://leanpub.com/forensic-ai

","book",{"_24":15,"_12":13,"_25":26,"_27":28,"_29":30,"_31":32,"_33":34,"_35":36,"_37":38},"prevChapter","nextChapter","sections",[],"chapterTitle","Beyond Uptime : Reliability Engineering for AI Infrastructure","publicChapterContent","metaText","Master AI infrastructure reliability! Debug silent GPU cluster stalls, PFC pause cascades, PCIe downtraining, and MFU drops across distributed compute.","slug","forensic-ai","showReadLinkString","sample","embargo",false,"coverFeatured",{"_225":226},"authors",[216],"tocStructure",[39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94],{"_95":96,"_12":23,"_98":-5,"_99":215,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":96,"_12":213,"_98":-5,"_99":214,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":114,"_12":211,"_98":-5,"_99":212,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":209,"_98":-5,"_99":210,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":207,"_98":-5,"_99":208,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":205,"_98":-5,"_99":206,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":203,"_98":-5,"_99":204,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":201,"_98":-5,"_99":202,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":199,"_98":-5,"_99":200,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":114,"_12":197,"_98":-5,"_99":198,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":195,"_98":-5,"_99":196,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":114,"_12":193,"_98":-5,"_99":194,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":191,"_98":-5,"_99":192,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":189,"_98":-5,"_99":190,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":187,"_98":-5,"_99":188,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":185,"_98":-5,"_99":186,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":114,"_12":183,"_98":-5,"_99":184,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":181,"_98":-5,"_99":182,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":179,"_98":-5,"_99":180,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":114,"_12":177,"_98":-5,"_99":178,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":175,"_98":-5,"_99":176,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":173,"_98":-5,"_99":174,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":171,"_98":-5,"_99":172,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":169,"_98":-5,"_99":170,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":167,"_98":-5,"_99":168,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":114,"_12":165,"_98":-5,"_99":166,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":163,"_98":-5,"_99":164,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":161,"_98":-5,"_99":162,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":159,"_98":-5,"_99":160,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":157,"_98":-5,"_99":158,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":114,"_12":155,"_98":-5,"_99":156,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":153,"_98":-5,"_99":154,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":151,"_98":-5,"_99":152,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":149,"_98":-5,"_99":150,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":147,"_98":-5,"_99":148,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":145,"_98":-5,"_99":146,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":143,"_98":-5,"_99":144,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":114,"_12":141,"_98":-5,"_99":142,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":139,"_98":-5,"_99":140,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":137,"_98":-5,"_99":138,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":135,"_98":-5,"_99":136,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":133,"_98":-5,"_99":134,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":131,"_98":-5,"_99":132,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":114,"_12":129,"_98":-5,"_99":130,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":127,"_98":-5,"_99":128,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":125,"_98":-5,"_99":126,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":123,"_98":-5,"_99":124,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":121,"_98":-5,"_99":122,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":119,"_98":-5,"_99":120,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":117,"_98":-5,"_99":118,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":114,"_12":115,"_98":-5,"_99":116,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":112,"_98":-5,"_99":113,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":110,"_98":-5,"_99":111,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":108,"_98":-5,"_99":109,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":106,"_98":-5,"_99":107,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":96,"_12":97,"_98":-5,"_99":100,"_101":96,"_102":-5,"_103":-5,"_104":-5},"level",1,"References","description","path","#references","fileCount","quizType","sectionNum","youtubeId",3,"An Agenda for Further Research","#an-agenda-for-further-research","An example of a Harnessed Agent Cluster running an AI model","#an-example-of-a-harnessed-agent-cluster-running-an-ai-model","Toward Autonomous AI Infrastructure","#toward-autonomous-ai-infrastructure","Machine Learning as the Investigator, Not Only the Instrument","#machine-learning-as-the-investigator-not-only-the-instrument",2,"AI Investigating AI : Towards Autonomous Infrastructure","#chapter10-auto-ai","Organizational Practices: Building the Team Around the Method","#organizational-practices-building-the-team-around-the-method","Preventive Engineering Practices","#preventive-engineering-practices","A Concrete Detection Formalism: From Fleet Averages to Per-Rank Tail Statistics","#chapter9-sec4-per-rank","Toward Automated Detection","#chap9-sec3-auto-detect","From Forensics to Foresight","#chapter9-sec1-forensic","Why Generalization Is Possible at All","#why-generalization-is-possible-at-all","Generalizing the Incident :A Practitioner Framework","#chapter9-prac-framework","The Failure Graph as a Formal Structure","#the-failure-graph-as-a-formal-structure","A Divergent Presentation of the Same Failure Class","#a-divergent-presentation-of-the-same-failure-class","Systematic , Layer-by-Layer Method","#systematic--layer-by-layer-method","Observability as Evidence Collection","#observability-as-evidence-collection","The Fleet Aggregation Paradox","#the-fleet-aggregation-paradox","Systematic RCA - The Failure Graph","#chapter8-fail-graph","The Structural Argument","#the-structural-argument","Memory Subsystem Degradation","#memory-subsystem-degradation","Passive Hardware Telemetry Checklist","#passive-hardware-telemetry-checklist","Manufacturing Variance & “Mercurial Cores”","#manufacturing-variance--mercurial-cores","Cosmic Radiation & Single-Event Upsets (SEUs)","#cosmic-radiation--single-event-upsets-seus","Transient Voltage Fluctuations & Thermal Noise","#transient-voltage-fluctuations--thermal-noise","The Anatomy of Silent Data Corruption","#chapter7-silent-dc","The Diagnostic Implication","#the-diagnostic-implication","Hash Polarization: A Second Congestion Mechanism","#chapter6-sec2-hash-pol","Priority Flow Control and Cascading Pause Frames","#priority-flow-control-and-cascading-pause-frames","The Congested Highway","#the-congested-highway","Lossless Fabric Bottlenecks","#chapter6-loss-bottles","Transport Reliability Semantics: What RDMA Guarantees and What It Does Not","#transport-reliability-semantics-what-rdma-guarantees-and-what-it-does-not","Comparing the Three Fabrics Along the Dimensions That Matter","#comparing-the-three-fabrics-along-the-dimensions-that-matter","A Cautionary Case: Fast Transport Is Not Sufficient Alone","#a-cautionary-case-fast-transport-is-not-sufficient-alone","RDMA, RoCE, and InfiniBand","#rdma-roce-and-infiniband","Limits of Ordinary Networking","#limits-of-ordinary-networking","Specialized Networking Technologies","#chapter5-special-net","The Forensic Investigation","#chapter4-sec2-forensic","Incident Premise and Cluster Configuration","#incident-premise-and-cluster-configuration","Operation Slowburn : A Forensic Reconstruction","#chapter4-op-slowburn","A Worked Capacity-Planning Illustration","#chapter3-sec4-cap-plan","Why this is unavoidable","#why-this-is-unavoidable","From the Table to the Training Run","#chapter3-sec2-table-run","An Intuitive Model: Barrier Synchronization","#an-intuitive-model-barrier-synchronization","The Synchronization Problem","#chapter3-sync-prob","How can a system be healthy and still be failing?","#how-can-a-system-be-healthy-and-still-be-failing","Operation Slowburn","#chapter2-op-slowburn","Beyond Network Rules : A Systems-Thinking Argument","#beyond-network-rules--a-systems-thinking-argument","The Modern AI Stack : A Glossary Through the Same Factory Lens","#the-modern-ai-stack--a-glossary-through-the-same-factory-lens","How Tiny Delays Compound in AI","#how-tiny-delays-compound-in-ai","Visualizing the AI Factory Floor","#factory-floor-visual","Why this requires GPU’s rather than CPU’s","#why-this-requires-gpus-rather-than-cpus","What is an AI Model","#what-is-ai-model","How AI Infrastructure Actually Works","#chapter1-main-block","Disclaimer & Limitation of Liability","#disclaimer--limitation-of-liability","#beyond-uptime--reliability-engineering-for-ai-infrastructure",{"_217":218,"_219":220,"_221":222},"id","Z2lkOi8vbGVhbnB1Yi9TdG9yZTo6UHJvZHVjdEF1dGhvci84OTU4NQ","name","Sreejith Kaimal","user",{"_223":224},"username","sree-kaimal","url","https://d2sofvawe08yqg.cloudfront.net/forensic-ai/s_featured?1788822530&1788822530",{"_12":13,"_27":28,"_25":26,"_31":32,"_33":319,"_320":321,"_35":322,"_29":30,"_323":-5,"_324":-5,"_325":326,"_327":326,"_328":329,"_330":331},"parts",[290,291,292],"toc",[234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289],"isUnavailableOnline","unavailableMessage",{"_95":96,"_12":23,"_98":-5,"_99":215,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":96,"_12":213,"_98":-5,"_99":214,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":114,"_12":211,"_98":-5,"_99":212,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":209,"_98":-5,"_99":210,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":207,"_98":-5,"_99":208,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":205,"_98":-5,"_99":206,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":203,"_98":-5,"_99":204,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":201,"_98":-5,"_99":202,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":199,"_98":-5,"_99":200,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":114,"_12":197,"_98":-5,"_99":198,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":195,"_98":-5,"_99":196,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":114,"_12":193,"_98":-5,"_99":194,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":191,"_98":-5,"_99":192,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":189,"_98":-5,"_99":190,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":187,"_98":-5,"_99":188,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":185,"_98":-5,"_99":186,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":114,"_12":183,"_98":-5,"_99":184,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":181,"_98":-5,"_99":182,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":179,"_98":-5,"_99":180,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":114,"_12":177,"_98":-5,"_99":178,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":175,"_98":-5,"_99":176,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":173,"_98":-5,"_99":174,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":171,"_98":-5,"_99":172,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":169,"_98":-5,"_99":170,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":167,"_98":-5,"_99":168,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":114,"_12":165,"_98":-5,"_99":166,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":163,"_98":-5,"_99":164,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":161,"_98":-5,"_99":162,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":159,"_98":-5,"_99":160,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":157,"_98":-5,"_99":158,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":114,"_12":155,"_98":-5,"_99":156,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":153,"_98":-5,"_99":154,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":151,"_98":-5,"_99":152,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":149,"_98":-5,"_99":150,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":147,"_98":-5,"_99":148,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":145,"_98":-5,"_99":146,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":143,"_98":-5,"_99":144,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":114,"_12":141,"_98":-5,"_99":142,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":139,"_98":-5,"_99":140,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":137,"_98":-5,"_99":138,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":135,"_98":-5,"_99":136,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":133,"_98":-5,"_99":134,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":131,"_98":-5,"_99":132,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":114,"_12":129,"_98":-5,"_99":130,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":127,"_98":-5,"_99":128,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":125,"_98":-5,"_99":126,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":123,"_98":-5,"_99":124,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":121,"_98":-5,"_99":122,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":119,"_98":-5,"_99":120,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":117,"_98":-5,"_99":118,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":114,"_12":115,"_98":-5,"_99":116,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":112,"_98":-5,"_99":113,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":110,"_98":-5,"_99":111,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":108,"_98":-5,"_99":109,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":105,"_12":106,"_98":-5,"_99":107,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_95":96,"_12":97,"_98":-5,"_99":100,"_101":96,"_102":-5,"_103":-5,"_104":-5},{"_12":23,"_98":293,"_101":96,"_99":215,"_294":318},{"_12":213,"_98":293,"_101":96,"_99":214,"_294":296},{"_12":97,"_98":293,"_101":96,"_99":100,"_294":295},"","chapters",[],[297,298,299,300,301,302,303,304,305,306],{"_95":114,"_12":211,"_99":212,"_101":96,"_307":317,"_104":-5},{"_95":114,"_12":197,"_99":198,"_101":96,"_307":316,"_104":-5},{"_95":114,"_12":193,"_99":194,"_101":96,"_307":315,"_104":-5},{"_95":114,"_12":183,"_99":184,"_101":96,"_307":314,"_104":-5},{"_95":114,"_12":177,"_99":178,"_101":96,"_307":313,"_104":-5},{"_95":114,"_12":165,"_99":166,"_101":96,"_307":312,"_104":-5},{"_95":114,"_12":155,"_99":156,"_101":96,"_307":311,"_104":-5},{"_95":114,"_12":141,"_99":142,"_101":96,"_307":310,"_104":-5},{"_95":114,"_12":129,"_99":130,"_101":96,"_307":309,"_104":-5},{"_95":114,"_12":115,"_99":116,"_101":96,"_307":308,"_104":-5},"subsections",[285,286,287,288],[278,279,280,281,282,283],[272,273,274,275,276],[265,266,267,268,269,270],[260,261,262,263],[254,255,256,257,258],[251,252],[246,247,248,249],[244],[237,238,239,240,241,242],[],{"_225":226},"coverFeatured2x",{"_225":334},[332],"freePdfUrl","freeEpubUrl","hasPdfSample",true,"hasEpubSample","samplePdfUrl","https://leanpub.com/forensic-ai/sample.pdf","sampleEpubUrl","https://leanpub.com/forensic-ai/sample.epub",{"_217":218,"_219":220,"_221":333},{"_223":224},"https://d2sofvawe08yqg.cloudfront.net/forensic-ai/s_featured2x?1788822530&1788822530","shoppingCartId","shoppingCart",{"_217":366,"_367":368,"_369":326,"_370":293,"_371":372,"_373":374,"_375":326},"locale","en","readerTheme","light","constants",{"_364":365},"publishers",{"_360":361,"_362":363},"learnerOrganizations",[],"announcements",{"_354":355,"_356":-5},"sentryEnvironment","production","sentryRelease","34b39a99cf18488c56fb0175867f9e503091b00e","announcement_store",{"_357":32,"_358":359},"announcement_authors","flag","text","

Kick off your book project, get started with GhostAI, get better at marketing, or spend the day doing all three! Free live workshops on Zoom. Saturday, June 27, 2026.

\n","companies",[],"universities",[],"paypalClientId","AX53Cwz8jR6rVVzzZiZNWWXl0dGagi-GZrjG-S3Ua14E5nrKMn7DxDQ730O4R8LusLPp49ALn6aRVaEy","no-cart","count",0,"free","updatedAt","paymentGateway","STRIPE","cartItems",[],"loading"]