Correctly close VM state after early OOM during open.

Reported by Assumeru.

(cherry picked from commit 5ca25ee)

`lua_newstate()` sets `g->str.mask` to `~(MSize)0` before calling
`cpluaopen(). If OOM happens before the `lj_str_init()` call,
`lj_gc_freeall()` calls `gc_sweepstr()` in a loop with incorrect top
limit `g->str.mask`, which leads to the crash.

This patch changes the order of the loop iteration with the correct
bottom limit.

Sergey Kaplun:
* added the description and the test for the problem

Part of tarantool/tarantool#11691

Author: Mike Pall

src/lj_gc.c

@@ -598,12 +598,11 @@ void lj_gc_finalize_cdata(lua_State *L)
 /* Free all remaining GC objects. */
 void lj_gc_freeall(global_State *g)
 {
-  MSize i, strmask;
+  MSize i;
   /* Free everything, except super-fixed objects (the main thread). */
   g->gc.currentwhite = LJ_GC_WHITES | LJ_GC_SFIXED;
   gc_fullsweep(g, &g->gc.root);
-  strmask = g->strmask;
-  for (i = 0; i <= strmask; i++)  /* Free all string hash chains. */
+  for (i = g->strmask; i != ~(MSize)0; i--)  /* Free all string hash chains. */
     gc_fullsweep(g, &g->strhash[i]);
 }
 

test/tarantool-c-tests/lj-1248-close-state-early-OOM.test.c

@@ -0,0 +1,71 @@
+#include "lua.h"
+/* XXX: The "lj_arch.h" header is included for the skipcond. */
+#include "lj_arch.h"
+
+#include "test.h"
+
+#include <stdlib.h>
+
+/*
+ * LuaJIT requires at least 12000 something bytes for initial
+ * allocations. The `GG_State` requires a little bit more than
+ * 6000 bytes (around 3000 bytes is the `jit_State`).
+ */
+
+/* Currently allocated Lua memory and its limit. */
+static size_t current_memory = 0;
+const size_t memory_limit = 7000;
+
+void *limited_alloc_f(void *msp, void *ptr, size_t osize, size_t nsize)
+{
+	void *ret_ptr = NULL;
+	/* Overflow is OK here. */
+	const size_t requested_diff = nsize - osize;
+	(void)msp;
+
+	if (current_memory + requested_diff > memory_limit)
+		return NULL;
+
+	if (nsize == 0) {
+		free(ptr);
+		current_memory -= osize;
+	} else if (ptr == NULL) {
+		ret_ptr = malloc(nsize);
+		current_memory += ret_ptr ? nsize : 0;
+	} else {
+		ret_ptr = realloc(ptr, nsize);
+		current_memory += ret_ptr ? requested_diff : 0;
+	}
+	return ret_ptr;
+}
+
+static int limited_memory_on_lua_newstate(void *test_state)
+{
+	(void)test_state;
+#if LJ_64 && !LJ_GC64
+	(void)limited_alloc_f;
+	return skip("Can't use custom allocator for 64-bit host without GC64");
+#else
+	/*
+	 * Check that there is no crash and the limit is small enough.
+	 */
+	const lua_State *L = lua_newstate(limited_alloc_f, NULL);
+	assert_true(L == NULL);
+	return TEST_EXIT_SUCCESS;
+#endif
+}
+
+#ifndef LJ_NO_UNWIND
+#  define LJ_NO_UNWIND 0
+#endif
+
+int main(void)
+{
+	/* See https://github.com/LuaJIT/LuaJIT/issues/1311. */
+	if (!LJ_NO_UNWIND)
+		return skip_all("Disabled for external unwinding build due to #1311");
+	const struct test_unit tgroup[] = {
+		test_unit_def(limited_memory_on_lua_newstate),
+	};
+	return test_run_group(tgroup, NULL);
+}