'lua_checkstack' doesn't need to check stack overflow

'luaD_growstack' already checks that. This commit also fixes an
internal bug in 'luaD_growstack': a large 'n' could cause an arithmetic
overflow when computing 'needed'.

all

@@ -1,7 +1,7 @@
 make -s -j
 cd testes/libs; make -s
 cd ..     # back to directory 'testes'
-ulimit -S -s 1000
+ulimit -S -s 1100
 if { ../lua -W all.lua; } then
   echo -e "\n\n    final OK!!!!\n\n"
 else

lapi.c

@@ -114,13 +114,8 @@ LUA_API int lua_checkstack (lua_State *L, int n) {
   api_check(L, n >= 0, "negative 'n'");
   if (L->stack_last - L->top > n)  /* stack large enough? */
     res = 1;  /* yes; check is OK */
-  else {  /* no; need to grow stack */
+  else  /* need to grow stack */
-    int inuse = cast_int(L->top - L->stack) + EXTRA_STACK;
+    res = luaD_growstack(L, n, 0);
-    if (inuse > LUAI_MAXSTACK - n)  /* can grow without overflow? */
-      res = 0;  /* no */
-    else  /* try to grow stack */
-      res = luaD_growstack(L, n, 0);
-  }
   if (res && ci->top < L->top + n)
     ci->top = L->top + n;  /* adjust frame top */
   lua_unlock(L);

ldo.c

@@ -227,7 +227,7 @@ int luaD_growstack (lua_State *L, int n, int raiseerror) {
       luaD_throw(L, LUA_ERRERR);  /* error inside message handler */
     return 0;  /* if not 'raiseerror', just signal it */
   }
-  else {
+  else if (n < LUAI_MAXSTACK) {  /* avoids arithmetic overflows */
     int newsize = 2 * size;  /* tentative new size */
     int needed = cast_int(L->top - L->stack) + n;
     if (newsize > LUAI_MAXSTACK)  /* cannot cross the limit */
@@ -236,14 +236,13 @@ int luaD_growstack (lua_State *L, int n, int raiseerror) {
       newsize = needed;
     if (l_likely(newsize <= LUAI_MAXSTACK))
       return luaD_reallocstack(L, newsize, raiseerror);
-    else {  /* stack overflow */
-      /* add extra size to be able to handle the error message */
-      luaD_reallocstack(L, ERRORSTACKSIZE, raiseerror);
-      if (raiseerror)
-        luaG_runerror(L, "stack overflow");
-      return 0;
-    }
   }
+  /* else stack overflow */
+  /* add extra size to be able to handle the error message */
+  luaD_reallocstack(L, ERRORSTACKSIZE, raiseerror);
+  if (raiseerror)
+    luaG_runerror(L, "stack overflow");
+  return 0;
 }
 
 

luaconf.h

@@ -728,7 +728,7 @@
 ** CHANGE it if you need a different limit. This limit is arbitrary;
 ** its only purpose is to stop Lua from consuming unlimited stack
 ** space (and to reserve some numbers for pseudo-indices).
-** (It must fit into max(size_t)/32.)
+** (It must fit into max(size_t)/32 and max(int)/2.)
 */
 #if LUAI_IS32INT
 #define LUAI_MAXSTACK		1000000

testes/coroutine.lua

@@ -741,20 +741,17 @@ _X()
 
 if not _soft then
   -- bug (stack overflow)
-  local j = 2^9
+  local lim = 1000000    -- stack limit; assume 32-bit machine
-  local lim = 1000000    -- (C stack limit; assume 32-bit machine)
+  local t = {lim - 10, lim - 5, lim - 1, lim, lim + 1, lim + 5}
-  local t = {lim - 10, lim - 5, lim - 1, lim, lim + 1}
   for i = 1, #t do
     local j = t[i]
-    co = coroutine.create(function()
+    local co = coroutine.create(function()
-           local t = {}
+           return table.unpack({}, 1, j)
-           for i = 1, j do t[i] = i end
-           return table.unpack(t)
          end)
     local r, msg = coroutine.resume(co)
-    assert(not r)
+    -- must fail for unpacking larger than stack limit
+    assert(j < lim or not r)
   end
-  co = nil
 end