[{"data":1,"prerenderedAt":32},["ShallowReactive",2],{"incidents":3,"rca-outcome-01KYEQDGGP2FCFXM4HPS921B76":23},[4],{"id":5,"title":6,"status":7,"severity":8,"componentIds":9,"openedAt":11,"resolvedAt":12,"updates":13},"01KYEQDGGP2FCFXM4HPS921B76","Realtime\u002FMQTT is experiencing issues","resolved","major",[10],"01KYE4M8BJE7E4C661505EW641","2026-07-26T08:07:13.687134Z","2026-07-27T10:37:00.078233Z",[14,17,21],{"status":15,"message":16,"occurredAt":11},"investigating","Automated monitoring detected 5 consecutive failed checks.",{"status":18,"message":19,"occurredAt":20},"monitoring","Automated monitoring detected recovery; watching for stability.","2026-07-27T00:10:03.621354Z",{"status":7,"message":22,"occurredAt":12},"Resolved. The realtime\u002FMQTT broker connection has been restored and has remained healthy across all monitoring checks for over 10 hours. Realtime sync updates are functioning normally.\n",{"rca":24,"notFound":31},{"incidentId":5,"summary":25,"timeline":26,"impact":27,"rootCause":28,"correctiveActions":29,"publishedAt":30},"The realtime\u002FMQTT component was unreachable in production from 2026-07-26 08:07 UTC\nto 2026-07-27 00:24 UTC (~16h17m). A production configuration gap left the\nreal-time messaging service without a working connection. Fixed by correcting the\nconfiguration and verifying sustained recovery. No data loss occurred; the outage\nwas limited to real-time push notifications.","2026-07-26 08:02 UTC — realtime_mqtt component registered (monitoring enabled)\n2026-07-26 08:07 UTC — first health probe fails; component flips to major_outage;\n                        incident auto-opened as draft\n2026-07-27 00:13 UTC — connectivity restored, configuration corrected\n2026-07-27 00:24 UTC — first healthy probe after the fix; recovery confirmed\n2026-07-27 10:35 UTC — component still healthy, zero consecutive probe failures\n                        across 10+ hours of sustained monitoring\n","No impact to data integrity, transaction processing, or core sync functionality —\nthe standard sync path was unaffected throughout. Impact was limited to real-time\npush signaling: clients fell back to normal periodic sync polling instead of\nreceiving an instant push trigger during the affected window. No customer data\nwas lost or at risk.\n","The real-time messaging service was not properly configured in production, so it\nhad no working connection available. This was a configuration\u002Fdeployment gap, not\nan application bug — the underlying code for real-time updates had already been\nbuilt and tested; production simply lacked a working connection to it.\n","- Corrected the production configuration to establish a proper, credentialed\n  connection to the real-time messaging service.\n- Added an internal tool so staff can review, publish, and resolve\n  monitoring-detected incidents directly, reducing time-to-detect for gaps\n  like this one going forward.\n- Scheduled a broader review of production configuration to catch similar\n  gaps proactively.\n","2026-07-27T10:42:02.542942Z",false,1785150443141]