Prevent circular resolve

Fixes: #3246
Co-authored-by: Tom Lau <tomandfatboy@gmail.com>

Author: Cooldude2606

script/vm/sign.lua

@@ -30,10 +30,17 @@ function mt:resolve(uri, args)
 
     ---@type table<string, vm.node>
     local resolved = {}
+    ---@type table<string, boolean>
+    local resolving = {}
 
     ---@param object vm.node|vm.node.object
     ---@param node   vm.node
     local function resolve(object, node)
+        local resolveHash = ("%s|%s"):format(object, node)
+        if resolving[resolveHash] then
+            return -- prevent circular resolve calls
+        end
+        resolving[resolveHash] = true
         if object.type == 'vm.node' then
             for o in object:eachObject() do
                 resolve(o, node)

test/type_inference/common.lua

@@ -4881,3 +4881,25 @@ end
 
 local a, b, <?c?>, d = unpack(t)
 ]]
+
+-- Test for overflow in circular resolve, only pass requirement is no overflow
+TEST 'Callback<<T>>|fun():fun():fun():Success, string' [[
+--- @alias Success fun(): Success
+--- @alias Callback<T> fun(): Success, T
+
+--- @return Success
+local function success()
+    return success
+end
+
+--- @generic T
+--- @param callback Callback<T>
+--- @return Callback<T>
+local function make_callback(callback)
+    return callback
+end
+
+local <?callback?> = make_callback(function()
+    return success, ""
+end)
+]]