[{"data":1,"prerenderedAt":417},["ShallowReactive",2],{"article:column:/column/cloudnative/debugging-capstone":3},{"id":4,"title":5,"body":6,"description":406,"extension":407,"meta":408,"navigation":412,"path":413,"seo":414,"stem":415,"__hash__":416},"column/column/cloudnative/debugging-capstone.md","把一条故障链查到底",{"type":7,"value":8,"toc":373},"minimark",[9,13,16,20,23,26,29,32,36,39,78,83,86,97,100,103,107,110,113,116,119,122,126,128,145,148,151,154,157,160,164,167,170,173,176,179,183,185,202,205,208,211,214,217,221,224,227,230,233,236,240,242,264,267,270,273,276,280,283,286,289,292,295,298,301,304,307,310,320,323,326,329,332,335],[10,11,12],"p",{},"学完前面的章节，已经认识了很多对象：镜像、Pod、Service、配置、卷、探针和控制器。真正遇到故障时，难点往往不在于忘了一条命令，而在于不知道下一条证据该从哪里找。",[10,14,15],{},"这一章把读书笔记服务放进几个教学现场。下面的数据和状态是为推理练习设计的案例，不是某次真实生产事故的日志。你可以先独立作答，再阅读分析；如果在实验集群复现，只使用可丢弃的测试资源与测试数据。",[17,18,19],"h2",{"id":19},"开始之前先保留现场",[10,21,22],{},"用户说“网站打不开”，这句话还不够精确。先问清发生时间、具体路径、影响范围和最近变化。是所有用户都失败，还是只有图片上传？是连接失败，还是HTTP返回500？是持续失败，还是偶发？",[10,24,25],{},"然后保留能关联起来的信息：一次请求的时间与请求ID、实际访问的域名和路径、发布版本、Pod身份以及相关事件。先记录再修改，可以避免修复动作把原来的线索覆盖掉。",[10,27,28],{},"故障现场最危险的习惯，是看到一个熟悉的词就直接操作。遇到CrashLoopBackOff就删除Pod，看到超时就增加副本，看到权限错误就授予管理员权限，都可能暂时改变现象，却没有解释原因。",[10,30,31],{},"把每次排查写成三个短句更有效：现在知道什么；怀疑什么；下一项观察怎样区分这些假设。",[17,33,35],{"id":34},"现场一-pod运行着-请求却到不了应用","现场一 Pod运行着 请求却到不了应用",[10,37,38],{},"已知：",[40,41,42,46,54,57,68],"ul",{},[43,44,45],"li",{},"两个notes-api Pod都显示Running，Ready为True",[43,47,48,49,53],{},"在Pod内部访问 ",[50,51,52],"code",{},"127.0.0.1:8080/readyz"," 成功",[43,55,56],{},"Service名可以解析到ClusterIP",[43,58,59,60,63,64,67],{},"Service配置为 ",[50,61,62],{},"port: 80","、",[50,65,66],{},"targetPort: 8080","，通过Service的80端口访问失败",[43,69,70,71,74,75],{},"Service的selector为 ",[50,72,73],{},"app: note-api","，Pod标签为 ",[50,76,77],{},"app: notes-api",[79,80,82],"h3",{"id":81},"先别归因于dns","先别归因于DNS",[10,84,85],{},"DNS已经返回地址，说明名称解析这一步至少在当前观察中成功。它没有证明Service有可用后端。",[10,87,88,89,92,93,96],{},"这里最有区分力的证据是后端列表。对由selector管理的普通Service，先检查EndpointSlice是否包含预期Pod及端口，然后回看selector是否选中了它们。案例中的 ",[50,90,91],{},"note-api"," 与 ",[50,94,95],{},"notes-api"," 相差一个字母，控制器不会猜测它们本来想表达同一件事。",[10,98,99],{},"合理的修复是纠正声明中的标签契约，再验证EndpointSlice出现预期后端，并重新发起真实业务请求。不要只看到配置提交成功就结束。",[10,101,102],{},"如果selector正确但仍失败，下一层再看targetPort、应用监听地址、网络策略和数据面。应用只监听容器内的127.0.0.1时，来自其他Pod的连接也可能进不来。排查顺序是为了缩小范围，不是规定所有故障都必须执行相同的二十条命令。",[79,104,106],{"id":105},"题目-1","题目 1",[10,108,109],{},"为什么Ready为True与DNS解析成功，仍然不能证明Service可用？",[79,111,112],{"id":112},"提示",[10,114,115],{},"探针、名称解析与后端选择分别验证了什么？",[79,117,118],{"id":118},"答案",[10,120,121],{},"探针验证配置的健康条件；DNS验证名称到地址的解析；Service还需要选中正确后端并使用正确端口，实际流量也要经过可用网络路径。它们是不同层次的证据，不能相互替代。",[17,123,125],{"id":124},"现场二-新密码已经更新-应用仍然连不上数据库","现场二 新密码已经更新 应用仍然连不上数据库",[10,127,38],{},[40,129,130,133,136,139,142],{},[43,131,132],{},"数据库密码完成轮换",[43,134,135],{},"对应Secret已更新",[43,137,138],{},"API从环境变量读取密码",[43,140,141],{},"Deployment的Pod模板没有变化",[43,143,144],{},"运行中的Pod仍是轮换前创建的那一批",[79,146,147],{"id":147},"配置的来源与读取时刻都重要",[10,149,150],{},"Secret对象已经更新，只说明存储的配置变了。容器创建时注入的环境变量，不会因为API对象后来改变就自动改写到运行进程中。",[10,152,153],{},"这里应检查配置通过什么方式传入，以及程序什么时候读取。环境变量通常需要通过受控重建让新容器获得新值。文件投射则有传播延迟，应用也可能只在启动时读取；使用subPath挂载还有不同的更新限制。不能把“配置文件最终会更新”推成“应用立即切换”。",[10,155,156],{},"更稳妥的密码轮换应提前设计新旧凭据的过渡窗口：先让系统能够接受或使用新凭据，确认消费者完成切换，再按授权流程撤销旧凭据。具体顺序取决于数据库和凭据管理方式。直接使旧密码失效，再等待所有实例慢慢更新，容易造成不必要的中断。",[10,158,159],{},"修复后还需确认所有副本都已使用新配置，并观察旧连接退出和新连接建立情况。凭据值不应出现在日志、截图或排障记录里，只记录身份、版本或轮换状态。",[79,161,163],{"id":162},"题目-2","题目 2",[10,165,166],{},"把Secret改成卷挂载，就一定能实现无感热更新吗？",[79,168,112],{"id":169},"提示-1",[10,171,172],{},"考虑API对象、节点文件、应用内存和已有连接四个位置。",[79,174,118],{"id":175},"答案-1",[10,177,178],{},"不能。普通投射卷更新有传播过程，subPath挂载不自动更新；程序还需要主动重读或重载配置，已有数据库连接也可能保留原来的认证状态。应明确每一步的更新机制和失败处理，而不是只改变挂载方式。",[17,180,182],{"id":181},"现场三-扩容发生了-延迟仍在上升","现场三 扩容发生了 延迟仍在上升",[10,184,38],{},[40,186,187,190,193,196,199],{},[43,188,189],{},"API从两个副本扩到六个，全部Ready",[43,191,192],{},"各副本CPU消耗较低",[43,194,195],{},"每个实例的数据库连接池上限为二十",[43,197,198],{},"数据库可供这项服务使用的连接预算为八十",[43,200,201],{},"请求主要等待获取数据库连接，数据库查询也变慢",[79,203,204],{"id":204},"先算清总量",[10,206,207],{},"两个副本最多申请四十个连接，六个副本则可能申请一百二十个，超过给这项服务的八十个预算。还要考虑发布期间额外副本、后台任务和运维连接。",[10,209,210],{},"这并不证明连接数就是唯一根因，但它给出了一个可以验证的假设：扩容增加了数据库侧竞争。下一步应同时检查连接等待时间、活跃查询、慢查询、锁与数据库资源，判断是连接池配置失衡，还是查询本身让连接长期不能归还。",[10,212,213],{},"如果只是把连接池上限一律调大，可能让数据库处理更多并发、排队更长。可以先限制并发、改善慢查询或缩小连接池预算，再评估数据库容量。需要保护用户体验时，也可对昂贵操作限流或排队，但应明确反馈状态，避免无限等待。",[10,215,216],{},"HPA的动作没有错到“没有执行”；它确实增加了实例。问题在于新增实例不能解除当前的瓶颈。排障要区分控制动作是否落地，与这个动作是否能改善业务结果。",[79,218,220],{"id":219},"题目-3","题目 3",[10,222,223],{},"如果需要为一次滚动发布预留一个额外API副本，六个常驻副本每个连接池上限设为十，是否就一定安全？",[79,225,112],{"id":226},"提示-2",[10,228,229],{},"七个实例乘以十只是预算的一部分。",[79,231,118],{"id":232},"答案-2",[10,234,235],{},"七个实例最多七十个连接，小于八十，但仍需考虑终止中的实例、worker、其他客户端、连接泄漏以及数据库能够高效处理的真实并发。连接预算满足是必要检查之一，并不等于查询负载一定安全。",[17,237,239],{"id":238},"现场四-回滚之后旧版本也出错了","现场四 回滚之后旧版本也出错了",[10,241,38],{},[40,243,244,255,258,261],{},[43,245,246,247,250,251,254],{},"新版API不再接受旧的 ",[50,248,249],{},"favorite"," 请求字段，只接受 ",[50,252,253],{},"shelf","；数据库旧列也被移除",[43,256,257],{},"新镜像启动成功，健康接口返回200",[43,259,260],{},"旧前端请求开始失败",[43,262,263],{},"后端回滚到旧镜像，错误仍然持续",[79,265,266],{"id":266},"一次版本号变化包含多个状态",[10,268,269],{},"镜像回滚恢复的是代码与运行模板。数据库已经发生的变化仍然存在；浏览器中的旧前端也不会因为后端回滚就立即更新。",[10,271,272],{},"现在应列出受影响的兼容关系：旧前端调用什么接口，旧后端依赖什么字段，新后端接受哪些数据。基于这些证据选择向前兼容修复或经过评估的数据恢复。不要在没有备份与影响评估时继续反复改表，扩大恢复难度。",[10,274,275],{},"从预防角度看，可以先增加新结构，发布兼容两种形式的代码，完成回填并验证消费者迁移，最后清理旧结构。回退窗口结束之前保留必要兼容能力，也要在测试中覆盖新旧版本并存。",[79,277,279],{"id":278},"题目-4","题目 4",[10,281,282],{},"为什么健康接口通过，没有发现这次发布问题？",[79,284,112],{"id":285},"提示-3",[10,287,288],{},"进程存活与“收藏”这条业务路径是否兼容，是两个判断。",[79,290,118],{"id":291},"答案-3",[10,293,294],{},"健康接口只验证了它被设计验证的条件，没有覆盖旧客户端与新数据库的组合。发布验收应加入代表性的业务请求及新旧兼容测试，并观察分版本错误率，而不能只看Pod数量与健康接口。",[17,296,297],{"id":297},"把排查过程变成团队可以复用的知识",[10,299,300],{},"四个现场看起来不同，但共同点是跨越了责任边界：网络对象到应用监听，配置对象到进程内存，副本数到数据库容量，代码版本到持久数据。",[10,302,303],{},"故障复盘也应该保留这些边界。记录现象、影响、最早证据、关键假设、验证动作和恢复结果。把“重启之后好了”改写为更完整的问题：重启改变了什么状态，它为什么能暂时改善，根因是否仍然存在。",[10,305,306],{},"改进项尽量对应具体机制。selector拼写错误可以增加配置验证和服务连通性测试；密码轮换中断可以补充消费端切换与回退流程；扩容压垮数据库可以统一连接预算并限制并发；回滚失效可以增加兼容迁移与恢复演练。",[10,308,309],{},"不必为了每次事故都新增一个平台。一个可靠的测试、一条清楚的告警、一个避免泄密的排障脚本，有时比再安装一套工具更有价值。",[10,311,312],{},[313,314,316],"a",{"href":315},"/collections-assets/cloudnative/cn13-debugging-evidence-chain.png",[317,318],"img",{"alt":319,"src":315},"四个教学故障分别沿症状、假设、区分证据与修复后验证展开：Service 请求失败核对 selector 和 EndpointSlice，密码问题核对配置消费，扩容变慢核对连接竞争，回滚失败核对代码与数据版本兼容。",[10,321,322],{},"每个动作对应一个可验证假设，修复之后回到用户请求确认结果。",[17,324,325],{"id":325},"你应该能带走的判断能力",[10,327,328],{},"再遇到一个陌生的云原生组件，先看它管理哪种状态、观察什么信号、采取什么动作、失败后怎样恢复。沿着这几个问题，往往能把工具重新放回整个系统中。",[10,330,331],{},"本合集从一次部署走到这里，容器解决了交付与运行边界的一部分，Kubernetes协调工作负载，网络与存储连接请求和数据，交付与观测让变化可验证。最终负责业务正确性的，仍然是把这些机制组合起来的工程设计。",[17,333,334],{"id":334},"参考资料",[40,336,337,345,352,359,366],{},[43,338,339],{},[313,340,344],{"href":341,"rel":342},"https://kubernetes.io/docs/tasks/debug/debug-application/debug-service/",[343],"nofollow","Debug Services",[43,346,347],{},[313,348,351],{"href":349,"rel":350},"https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/",[343],"Debug Pods",[43,353,354],{},[313,355,358],{"href":356,"rel":357},"https://kubernetes.io/docs/concepts/configuration/configmap/",[343],"ConfigMaps",[43,360,361],{},[313,362,365],{"href":363,"rel":364},"https://kubernetes.io/docs/concepts/configuration/secret/",[343],"Secrets",[43,367,368],{},[313,369,372],{"href":370,"rel":371},"https://kubernetes.io/docs/concepts/workloads/controllers/deployment/",[343],"Deployments",{"title":374,"searchDepth":375,"depth":375,"links":376},"",3,[377,379,385,391,397,403,404,405],{"id":19,"depth":378,"text":19},2,{"id":34,"depth":378,"text":35,"children":380},[381,382,383,384],{"id":81,"depth":375,"text":82},{"id":105,"depth":375,"text":106},{"id":112,"depth":375,"text":112},{"id":118,"depth":375,"text":118},{"id":124,"depth":378,"text":125,"children":386},[387,388,389,390],{"id":147,"depth":375,"text":147},{"id":162,"depth":375,"text":163},{"id":169,"depth":375,"text":112},{"id":175,"depth":375,"text":118},{"id":181,"depth":378,"text":182,"children":392},[393,394,395,396],{"id":204,"depth":375,"text":204},{"id":219,"depth":375,"text":220},{"id":226,"depth":375,"text":112},{"id":232,"depth":375,"text":118},{"id":238,"depth":378,"text":239,"children":398},[399,400,401,402],{"id":266,"depth":375,"text":266},{"id":278,"depth":375,"text":279},{"id":285,"depth":375,"text":112},{"id":291,"depth":375,"text":118},{"id":297,"depth":378,"text":297},{"id":325,"depth":378,"text":325},{"id":334,"depth":378,"text":334},"用读书笔记服务的四个教学现场，串联镜像、Pod、网络、配置、容量与发布，练习提出假设和寻找证据。","md",{"collection":409,"order":410,"slug":411},"云原生",13,"debugging-capstone",true,"/column/cloudnative/debugging-capstone",{"title":5,"description":406},"column/cloudnative/debugging-capstone","jCTU2l-XJt-X_iP68Ga5RNgmDogtzrRTh_g76zxIOCg",1791935829025]