@@ -1090,3 +1090,249 @@ describe.skipIf(!redisAvailable)('DistributedStateManager - cleanup error handli
10901090 expect ( Object . keys ( conn ) . length ) . toBe ( 0 ) ;
10911091 } ) ;
10921092} ) ;
1093+
1094+ describe . skipIf ( ! redisAvailable ) ( 'DistributedStateManager - Dead Instance Cleanup' , ( ) => {
1095+ let redis : Redis ;
1096+ let liveManager : DistributedStateManager ;
1097+
1098+ beforeAll ( async ( ) => {
1099+ redis = new Redis ( REDIS_URL ) ;
1100+ await new Promise < void > ( ( resolve ) => redis . once ( 'ready' , resolve ) ) ;
1101+ } ) ;
1102+
1103+ afterAll ( async ( ) => {
1104+ forceResetDistributedState ( ) ;
1105+ await redis . quit ( ) ;
1106+ } ) ;
1107+
1108+ beforeEach ( async ( ) => {
1109+ forceResetDistributedState ( ) ;
1110+ try {
1111+ const keys = await redis . keys ( 'boardsesh:*' ) ;
1112+ if ( keys . length > 0 ) {
1113+ await redis . del ( ...keys ) ;
1114+ }
1115+ } catch ( err ) {
1116+ console . warn ( 'Failed to clean up test keys:' , err ) ;
1117+ }
1118+ liveManager = new DistributedStateManager ( redis , 'live-instance' ) ;
1119+ } ) ;
1120+
1121+ afterEach ( async ( ) => {
1122+ await liveManager . stop ( ) ;
1123+ forceResetDistributedState ( ) ;
1124+ } ) ;
1125+
1126+ describe ( 'discoverDeadInstances' , ( ) => {
1127+ it ( 'should return empty when no other instances exist' , async ( ) => {
1128+ liveManager . start ( ) ;
1129+ // Give heartbeat time to run
1130+ await new Promise ( ( r ) => setTimeout ( r , 100 ) ) ;
1131+
1132+ const dead = await liveManager . discoverDeadInstances ( ) ;
1133+ expect ( dead ) . toEqual ( [ ] ) ;
1134+ } ) ;
1135+
1136+ it ( 'should skip the current instance' , async ( ) => {
1137+ liveManager . start ( ) ;
1138+ await new Promise ( ( r ) => setTimeout ( r , 100 ) ) ;
1139+
1140+ // Even if we manually clear our own heartbeat, discoverDeadInstances
1141+ // skips the current instance ID.
1142+ await redis . del ( 'boardsesh:instance:live-instance:heartbeat' ) ;
1143+
1144+ const dead = await liveManager . discoverDeadInstances ( ) ;
1145+ expect ( dead ) . toEqual ( [ ] ) ;
1146+ } ) ;
1147+
1148+ it ( 'should find instances whose heartbeat expired' , async ( ) => {
1149+ // Simulate a dead instance: create its conns key but no heartbeat
1150+ await redis . sadd ( 'boardsesh:instance:dead-inst:conns' , 'orphan-conn-1' ) ;
1151+
1152+ const dead = await liveManager . discoverDeadInstances ( ) ;
1153+ expect ( dead ) . toEqual ( [ 'dead-inst' ] ) ;
1154+ } ) ;
1155+
1156+ it ( 'should not report instances with active heartbeats' , async ( ) => {
1157+ // Create another instance with a valid heartbeat
1158+ await redis . sadd ( 'boardsesh:instance:alive-inst:conns' , 'conn-x' ) ;
1159+ await redis . setex ( 'boardsesh:instance:alive-inst:heartbeat' , 60 , Date . now ( ) . toString ( ) ) ;
1160+
1161+ const dead = await liveManager . discoverDeadInstances ( ) ;
1162+ expect ( dead ) . toEqual ( [ ] ) ;
1163+ } ) ;
1164+ } ) ;
1165+
1166+ describe ( 'cleanupDeadInstanceConnections' , ( ) => {
1167+ it ( 'should remove orphaned connections from dead instances' , async ( ) => {
1168+ // Register a live connection
1169+ await liveManager . registerConnection ( 'live-conn' , 'LiveUser' ) ;
1170+ await liveManager . joinSession ( 'live-conn' , 'shared-session' ) ;
1171+
1172+ // Simulate a dead instance with orphaned connections in the same session
1173+ const deadInstanceId = 'dead-instance-abc' ;
1174+ // Create connection hashes for the dead connections
1175+ await redis . hmset ( 'boardsesh:conn:orphan-1' , {
1176+ connectionId : 'orphan-1' ,
1177+ instanceId : deadInstanceId ,
1178+ sessionId : 'shared-session' ,
1179+ userId : '' ,
1180+ username : 'Ghost1' ,
1181+ avatarUrl : '' ,
1182+ isLeader : 'false' ,
1183+ connectedAt : ( Date . now ( ) - 60000 ) . toString ( ) ,
1184+ } ) ;
1185+ await redis . hmset ( 'boardsesh:conn:orphan-2' , {
1186+ connectionId : 'orphan-2' ,
1187+ instanceId : deadInstanceId ,
1188+ sessionId : 'shared-session' ,
1189+ userId : '' ,
1190+ username : 'Ghost2' ,
1191+ avatarUrl : '' ,
1192+ isLeader : 'false' ,
1193+ connectedAt : ( Date . now ( ) - 60000 ) . toString ( ) ,
1194+ } ) ;
1195+ // Add to instance tracking (no heartbeat = dead)
1196+ await redis . sadd ( `boardsesh:instance:${ deadInstanceId } , 'orphan-1' , 'orphan-2' ) ;
1197+ // Add to session members
1198+ await redis . sadd ( 'boardsesh:session:shared-session:members' , 'orphan-1' , 'orphan-2' ) ;
1199+
1200+ // Verify inflated count before cleanup
1201+ const rawCount = await redis . scard ( 'boardsesh:session:shared-session:members' ) ;
1202+ expect ( rawCount ) . toBe ( 3 ) ; // 1 live + 2 ghosts
1203+
1204+ // Run cleanup
1205+ const result = await liveManager . cleanupDeadInstanceConnections ( ) ;
1206+
1207+ expect ( result . deadInstances ) . toEqual ( [ deadInstanceId ] ) ;
1208+ expect ( result . staleConnections . sort ( ) ) . toEqual ( [ 'orphan-1' , 'orphan-2' ] ) ;
1209+ expect ( result . sessionsAffected ) . toEqual ( [ 'shared-session' ] ) ;
1210+
1211+ // Verify connection hashes are deleted
1212+ const conn1 = await redis . hgetall ( 'boardsesh:conn:orphan-1' ) ;
1213+ const conn2 = await redis . hgetall ( 'boardsesh:conn:orphan-2' ) ;
1214+ expect ( Object . keys ( conn1 ) . length ) . toBe ( 0 ) ;
1215+ expect ( Object . keys ( conn2 ) . length ) . toBe ( 0 ) ;
1216+
1217+ // Verify instance tracking key is deleted
1218+ const instanceConns = await redis . smembers ( `boardsesh:instance:${ deadInstanceId } ) ;
1219+ expect ( instanceConns . length ) . toBe ( 0 ) ;
1220+
1221+ // Verify session member count is correct (only live connection)
1222+ const members = await liveManager . getSessionMembers ( 'shared-session' ) ;
1223+ expect ( members . length ) . toBe ( 1 ) ;
1224+ expect ( members [ 0 ] . username ) . toBe ( 'LiveUser' ) ;
1225+ } ) ;
1226+
1227+ it ( 'should return empty summary when no dead instances exist' , async ( ) => {
1228+ const result = await liveManager . cleanupDeadInstanceConnections ( ) ;
1229+ expect ( result . deadInstances ) . toEqual ( [ ] ) ;
1230+ expect ( result . staleConnections ) . toEqual ( [ ] ) ;
1231+ expect ( result . sessionsAffected ) . toEqual ( [ ] ) ;
1232+ } ) ;
1233+
1234+ it ( 'should handle dead instance with no connections' , async ( ) => {
1235+ // Dead instance with empty conns set
1236+ await redis . sadd ( 'boardsesh:instance:empty-dead:conns' , '__placeholder__' ) ;
1237+ await redis . srem ( 'boardsesh:instance:empty-dead:conns' , '__placeholder__' ) ;
1238+ // Actually just create the key with an entry then remove it—
1239+ // simpler: just create it directly
1240+ await redis . del ( 'boardsesh:instance:empty-dead:conns' ) ;
1241+ // Need the key to exist for SCAN to find it
1242+ await redis . sadd ( 'boardsesh:instance:empty-dead:conns' , 'temp' ) ;
1243+ await redis . srem ( 'boardsesh:instance:empty-dead:conns' , 'temp' ) ;
1244+
1245+ // The key may or may not exist (Redis deletes empty sets).
1246+ // Either way, cleanup should not error.
1247+ const result = await liveManager . cleanupDeadInstanceConnections ( ) ;
1248+ // May or may not find the instance depending on whether Redis kept the empty set
1249+ expect ( result . staleConnections ) . toEqual ( [ ] ) ;
1250+ } ) ;
1251+ } ) ;
1252+
1253+ describe ( 'cleanupStaleSessionMembers' , ( ) => {
1254+ it ( 'should remove members whose connection hashes expired' , async ( ) => {
1255+ await liveManager . registerConnection ( 'real-conn' , 'RealUser' ) ;
1256+ await liveManager . joinSession ( 'real-conn' , 'prune-session' ) ;
1257+
1258+ // Add a stale member directly (no connection hash)
1259+ await redis . sadd ( 'boardsesh:session:prune-session:members' , 'stale-conn' ) ;
1260+
1261+ // Verify stale member is in the set
1262+ const rawCount = await redis . scard ( 'boardsesh:session:prune-session:members' ) ;
1263+ expect ( rawCount ) . toBe ( 2 ) ;
1264+
1265+ const removed = await liveManager . cleanupStaleSessionMembers ( 'prune-session' ) ;
1266+ expect ( removed ) . toBe ( 1 ) ;
1267+
1268+ // Only real connection remains
1269+ const count = await liveManager . getSessionMemberCount ( 'prune-session' ) ;
1270+ expect ( count ) . toBe ( 1 ) ;
1271+ } ) ;
1272+
1273+ it ( 'should re-elect leader when stale leader is pruned' , async ( ) => {
1274+ await liveManager . registerConnection ( 'member-conn' , 'Member' ) ;
1275+ await liveManager . joinSession ( 'member-conn' , 'leader-prune-session' ) ;
1276+
1277+ // Add a stale leader directly
1278+ await redis . sadd ( 'boardsesh:session:leader-prune-session:members' , 'stale-leader' ) ;
1279+ await redis . set ( 'boardsesh:session:leader-prune-session:leader' , 'stale-leader' , 'EX' , 14400 ) ;
1280+
1281+ const removed = await liveManager . cleanupStaleSessionMembers ( 'leader-prune-session' ) ;
1282+ expect ( removed ) . toBe ( 1 ) ;
1283+
1284+ // member-conn should now be leader
1285+ const leader = await liveManager . getSessionLeader ( 'leader-prune-session' ) ;
1286+ expect ( leader ) . toBe ( 'member-conn' ) ;
1287+
1288+ const conn = await liveManager . getConnection ( 'member-conn' ) ;
1289+ expect ( conn ! . isLeader ) . toBe ( true ) ;
1290+ } ) ;
1291+
1292+ it ( 'should clean up empty session when all members are stale' , async ( ) => {
1293+ // Add only stale members
1294+ await redis . sadd ( 'boardsesh:session:all-stale:members' , 'gone-1' , 'gone-2' ) ;
1295+ await redis . set ( 'boardsesh:session:all-stale:leader' , 'gone-1' , 'EX' , 14400 ) ;
1296+
1297+ const removed = await liveManager . cleanupStaleSessionMembers ( 'all-stale' ) ;
1298+ expect ( removed ) . toBe ( 2 ) ;
1299+
1300+ // Session should be cleaned up
1301+ const exists = await redis . exists ( 'boardsesh:session:all-stale:members' ) ;
1302+ expect ( exists ) . toBe ( 0 ) ;
1303+
1304+ const leader = await liveManager . getSessionLeader ( 'all-stale' ) ;
1305+ expect ( leader ) . toBeNull ( ) ;
1306+ } ) ;
1307+
1308+ it ( 'should return 0 when no stale members exist' , async ( ) => {
1309+ await liveManager . registerConnection ( 'healthy-conn' , 'Healthy' ) ;
1310+ await liveManager . joinSession ( 'healthy-conn' , 'healthy-session' ) ;
1311+
1312+ const removed = await liveManager . cleanupStaleSessionMembers ( 'healthy-session' ) ;
1313+ expect ( removed ) . toBe ( 0 ) ;
1314+ } ) ;
1315+ } ) ;
1316+
1317+ describe ( 'getSessionMemberCount - stale filtering' , ( ) => {
1318+ it ( 'should not count members whose connection hashes are missing' , async ( ) => {
1319+ await liveManager . registerConnection ( 'counted-conn' , 'Counted' ) ;
1320+ await liveManager . joinSession ( 'counted-conn' , 'count-session' ) ;
1321+
1322+ // Add stale members directly to the session set
1323+ await redis . sadd ( 'boardsesh:session:count-session:members' , 'ghost-1' , 'ghost-2' ) ;
1324+
1325+ // Raw SCARD would return 3, but getSessionMemberCount should return 1
1326+ const rawCount = await redis . scard ( 'boardsesh:session:count-session:members' ) ;
1327+ expect ( rawCount ) . toBe ( 3 ) ;
1328+
1329+ const filteredCount = await liveManager . getSessionMemberCount ( 'count-session' ) ;
1330+ expect ( filteredCount ) . toBe ( 1 ) ;
1331+ } ) ;
1332+
1333+ it ( 'should return 0 for empty session' , async ( ) => {
1334+ const count = await liveManager . getSessionMemberCount ( 'nonexistent-session' ) ;
1335+ expect ( count ) . toBe ( 0 ) ;
1336+ } ) ;
1337+ } ) ;
1338+ } ) ;
0 commit comments