lua: add reference counting for strings - this will need A LOT of testing, but it should finally fix the excessive memory usage problems triggered by luci

git-svn-id: svn://svn.openwrt.org/openwrt/trunk@18136 3c298f89-4303-0410-b956-a3cf2f4a3e73

package/lua/patches/040-memory-limits.patch

@@ -1,289 +0,0 @@
---- a/src/lapi.c
-+++ b/src/lapi.c
-@@ -716,14 +716,14 @@
- 
- LUA_API void lua_setfield (lua_State *L, int idx, const char *k) {
-   StkId t;
--  TValue key;
-   lua_lock(L);
-   api_checknelems(L, 1);
-   t = index2adr(L, idx);
-   api_checkvalidindex(L, t);
--  setsvalue(L, &key, luaS_new(L, k));
--  luaV_settable(L, t, &key, L->top - 1);
--  L->top--;  /* pop value */
-+  setsvalue2s(L, L->top, luaS_new(L, k));
-+  api_incr_top(L);
-+  luaV_settable(L, t, L->top - 1, L->top - 2);
-+  L->top -= 2;  /* pop key and value */
-   lua_unlock(L);
- }
- 
-@@ -971,7 +971,12 @@
-       break;
-     }
-     case LUA_GCCOLLECT: {
--      luaC_fullgc(L);
-+      lu_mem old_thres = g->GCthreshold;
-+      if(g->GCthreshold != MAX_LUMEM) {
-+        g->GCthreshold = MAX_LUMEM;
-+        luaC_fullgc(L);
-+        g->GCthreshold = old_thres;
-+      }
-       break;
-     }
-     case LUA_GCCOUNT: {
---- a/src/ldo.c
-+++ b/src/ldo.c
-@@ -494,6 +494,7 @@
-   struct SParser *p = cast(struct SParser *, ud);
-   int c = luaZ_lookahead(p->z);
-   luaC_checkGC(L);
-+	lua_gc(L, LUA_GCSTOP, 0);  /* stop collector during parsing */
-   tf = ((c == LUA_SIGNATURE[0]) ? luaU_undump : luaY_parser)(L, p->z,
-                                                              &p->buff, p->name);
-   cl = luaF_newLclosure(L, tf->nups, hvalue(gt(L)));
-@@ -502,6 +503,7 @@
-     cl->l.upvals[i] = luaF_newupval(L);
-   setclvalue(L, L->top, cl);
-   incr_top(L);
-+	lua_gc(L, LUA_GCRESTART, 0);
- }
- 
- 
---- a/src/lgc.c
-+++ b/src/lgc.c
-@@ -437,7 +437,10 @@
-   /* check size of buffer */
-   if (luaZ_sizebuffer(&g->buff) > LUA_MINBUFFER*2) {  /* buffer too big? */
-     size_t newsize = luaZ_sizebuffer(&g->buff) / 2;
--    luaZ_resizebuffer(L, &g->buff, newsize);
-+    /* make sure newsize is larger then the buffer's in use size. */
-+    newsize = (luaZ_bufflen(&g->buff) > newsize) ? luaZ_bufflen(&g->buff) : newsize;
-+    if(newsize < luaZ_sizebuffer(&g->buff))
-+      luaZ_resizebuffer(L, &g->buff, newsize);
-   }
- }
- 
---- a/src/lstate.c
-+++ b/src/lstate.c
-@@ -118,7 +118,6 @@
- 
- lua_State *luaE_newthread (lua_State *L) {
-   lua_State *L1 = tostate(luaM_malloc(L, state_size(lua_State)));
--  luaC_link(L, obj2gco(L1), LUA_TTHREAD);
-   preinit_state(L1, G(L));
-   stack_init(L1, L);  /* init stack */
-   setobj2n(L, gt(L1), gt(L));  /* share table of globals */
-@@ -126,6 +125,7 @@
-   L1->basehookcount = L->basehookcount;
-   L1->hook = L->hook;
-   resethookcount(L1);
-+  luaC_link(L, obj2gco(L1), LUA_TTHREAD);
-   lua_assert(iswhite(obj2gco(L1)));
-   return L1;
- }
---- a/src/lstring.c
-+++ b/src/lstring.c
-@@ -53,6 +53,9 @@
-   stringtable *tb;
-   if (l+1 > (MAX_SIZET - sizeof(TString))/sizeof(char))
-     luaM_toobig(L);
-+  tb = &G(L)->strt;
-+  if ((tb->nuse + 1) > cast(lu_int32, tb->size) && tb->size <= MAX_INT/2)
-+    luaS_resize(L, tb->size*2);  /* too crowded */
-   ts = cast(TString *, luaM_malloc(L, (l+1)*sizeof(char)+sizeof(TString)));
-   ts->tsv.len = l;
-   ts->tsv.hash = h;
-@@ -61,13 +64,10 @@
-   ts->tsv.reserved = 0;
-   memcpy(ts+1, str, l*sizeof(char));
-   ((char *)(ts+1))[l] = '\0';  /* ending 0 */
--  tb = &G(L)->strt;
-   h = lmod(h, tb->size);
-   ts->tsv.next = tb->hash[h];  /* chain new entry */
-   tb->hash[h] = obj2gco(ts);
-   tb->nuse++;
--  if (tb->nuse > cast(lu_int32, tb->size) && tb->size <= MAX_INT/2)
--    luaS_resize(L, tb->size*2);  /* too crowded */
-   return ts;
- }
- 
---- a/src/ltable.c
-+++ b/src/ltable.c
-@@ -371,7 +371,6 @@
- 
- Table *luaH_new (lua_State *L, int narray, int nhash) {
-   Table *t = luaM_new(L, Table);
--  luaC_link(L, obj2gco(t), LUA_TTABLE);
-   t->metatable = NULL;
-   t->flags = cast_byte(~0);
-   /* temporary values (kept only if some malloc fails) */
-@@ -381,6 +380,7 @@
-   t->node = cast(Node *, dummynode);
-   setarrayvector(L, t, narray);
-   setnodevector(L, t, nhash);
-+  luaC_link(L, obj2gco(t), LUA_TTABLE);
-   return t;
- }
- 
---- a/src/lvm.c
-+++ b/src/lvm.c
-@@ -375,6 +375,7 @@
-         if (l >= MAX_SIZET - tl) luaG_runerror(L, "string length overflow");
-         tl += l;
-       }
-+      G(L)->buff.n = tl;
-       buffer = luaZ_openspace(L, &G(L)->buff, tl);
-       tl = 0;
-       for (i=n; i>0; i--) {  /* concat all strings */
-@@ -383,6 +384,7 @@
-         tl += l;
-       }
-       setsvalue2s(L, top-n, luaS_newlstr(L, buffer, tl));
-+      luaZ_resetbuffer(&G(L)->buff);
-     }
-     total -= n-1;  /* got `n' strings to create 1 new */
-     last -= n-1;
---- a/src/lua.c
-+++ b/src/lua.c
-@@ -19,6 +19,94 @@
- #include "llimits.h"
- 
- 
-+typedef struct {
-+	char		*name;
-+	lua_State	*L;
-+	size_t		memused;
-+	size_t		peak_memused;
-+	size_t		gc_memused;
-+	size_t		max_memused;
-+	int		collecting;
-+} script_info_t;
-+
-+
-+static void *script_alloc(void *ud, void *ptr, size_t osize, size_t nsize)
-+{
-+	script_info_t *info=(script_info_t *)ud;
-+	size_t old_size = info->memused;
-+
-+	info->memused -= osize;
-+	if (nsize == 0) {
-+		free(ptr);
-+		return NULL;
-+	}
-+	info->memused += nsize;
-+	if(info->max_memused > 0 && nsize > osize &&
-+	  (info->memused >= info->max_memused || info->memused >= info->gc_memused)) {
-+#ifdef LOW_MEM_DEBUG
-+		printf("LOW MEM: 1 osize=%zd, nsize=%zd, used=%zu, peak=%zu, need=%zd\n", osize, nsize,
-+			info->memused, info->peak_memused, (info->memused - info->max_memused));
-+#endif
-+		info->memused = old_size;
-+		/* don't allow a recursive garbage collection call. */
-+		if(info->collecting != 0) {
-+			return NULL;
-+		}
-+		info->collecting = 1;
-+		/* try to free memory by collecting garbage. */
-+		lua_gc(info->L, LUA_GCCOLLECT, 0);
-+		info->collecting = 0;
-+#ifdef LOW_MEM_DEBUG
-+		printf("LOW MEM: 2 used=%zu, peak=%zu\n", info->memused, info->peak_memused);
-+#endif
-+		/* check memory usage again. */
-+		old_size = info->memused;
-+		info->memused -= osize;
-+		info->memused += nsize;
-+		if(info->memused >= info->max_memused) {
-+			info->memused = old_size;
-+#ifdef LOW_MEM_DEBUG
-+			printf("OUT OF MEMORY: memused=%zd, osize=%zd, nsize=%zd\n", info->memused, osize, nsize);
-+#endif
-+			return NULL;
-+		}
-+	}
-+	if(info->memused > info->peak_memused) info->peak_memused = info->memused;
-+	return realloc(ptr, nsize);
-+}
-+
-+static int set_memory_limit(lua_State *L)
-+{
-+	int hardlimit = luaL_checknumber(L, 1);
-+	int softlimit = luaL_optnumber(L, 2, 0);
-+
-+	script_info_t *info;
-+	lua_getallocf(L, (void *)(&info));
-+
-+	if( hardlimit >= 0 )
-+	{
-+		if( softlimit <= 0 )
-+			softlimit = (int)((float)hardlimit * 0.75);
-+
-+		info->max_memused = hardlimit;
-+		info->gc_memused  = softlimit;
-+	}
-+
-+	lua_pushnumber(L, hardlimit);
-+	lua_pushnumber(L, softlimit);
-+	return 2;
-+}
-+
-+static int get_memory_limit(lua_State *L)
-+{
-+	script_info_t *info;
-+	lua_getallocf(L, (void *)(&info));
-+	lua_pushnumber(L, info->max_memused);
-+	lua_pushnumber(L, info->gc_memused);
-+	return 2;
-+}
-+
-+
- static lua_State *globalL = NULL;
- 
- static const char *progname = LUA_PROGNAME;
-@@ -377,11 +465,28 @@
- int main (int argc, char **argv) {
-   int status;
-   struct Smain s;
--  lua_State *L = lua_open();  /* create state */
-+  script_info_t *info;
-+
-+  info = (script_info_t *)calloc(1, sizeof(script_info_t));
-+  info->max_memused = 0;
-+  info->collecting = 0;
-+  info->name = argv[0];
-+  info->memused = 0;
-+  info->peak_memused = 0;
-+
-+  lua_State *L = lua_newstate(script_alloc, info);
-+
-   if (L == NULL) {
-     l_message(argv[0], "cannot create state: not enough memory");
-     return EXIT_FAILURE;
-   }
-+
-+  info->L = L;
-+
-+  luaL_openlibs(L);
-+  lua_register(L, "set_memory_limit", set_memory_limit);
-+  lua_register(L, "get_memory_limit", get_memory_limit);
-+
-   /* Checking 'sizeof(lua_Integer)' cannot be made in preprocessor on all compilers.
-   */
- #ifdef LNUM_INT16
-@@ -396,6 +501,14 @@
-   status = lua_cpcall(L, &pmain, &s);
-   report(L, status);
-   lua_close(L);
-+
-+#ifdef LOW_MEM_DEBUG
-+  printf("%s: memused=%zd, peak_memused=%zd\n", info->name,
-+	info->memused, info->peak_memused);
-+#endif
-+
-+  free(info);
-+
-   return (status || s.status) ? EXIT_FAILURE : EXIT_SUCCESS;
- }
- 

package/lua/patches/300-opcode_performance.patch

@@ -10,7 +10,7 @@
  
  /*
   * If 'obj' is a string, it is tried to be interpreted as a number.
-@@ -564,12 +567,63 @@
+@@ -562,12 +565,63 @@ static inline int arith_mode( const TVal
      ARITH_OP1_END
  #endif
  
@@ -74,7 +74,7 @@
   reentry:  /* entry point */
    lua_assert(isLua(L->ci));
    pc = L->savedpc;
-@@ -594,33 +648,33 @@
+@@ -592,33 +646,33 @@ void luaV_execute (lua_State *L, int nex
      lua_assert(base == L->base && L->base == L->ci->base);
      lua_assert(base <= L->top && L->top <= L->stack + L->stacksize);
      lua_assert(L->top == L->ci->top || luaG_checkopenop(i));
@@ -115,7 +115,7 @@
          TValue g;
          TValue *rb = KBx(i);
          sethvalue(L, &g, cl->env);
-@@ -628,88 +682,88 @@
+@@ -626,88 +680,88 @@ void luaV_execute (lua_State *L, int nex
          Protect(luaV_gettable(L, &g, rb, ra));
          continue;
        }
@@ -219,7 +219,7 @@
          const TValue *rb = RB(i);
          switch (ttype(rb)) {
            case LUA_TTABLE: {
-@@ -729,18 +783,18 @@
+@@ -727,18 +781,18 @@ void luaV_execute (lua_State *L, int nex
          }
          continue;
        }
@@ -241,7 +241,7 @@
          TValue *rb = RKB(i);
          TValue *rc = RKC(i);
          Protect(
-@@ -750,7 +804,7 @@
+@@ -748,7 +802,7 @@ void luaV_execute (lua_State *L, int nex
          pc++;
          continue;
        }
@@ -250,7 +250,7 @@
          Protect(
            if (luaV_lessthan(L, RKB(i), RKC(i)) == GETARG_A(i))
              dojump(L, pc, GETARG_sBx(*pc));
-@@ -758,7 +812,7 @@
+@@ -756,7 +810,7 @@ void luaV_execute (lua_State *L, int nex
          pc++;
          continue;
        }
@@ -259,7 +259,7 @@
          Protect(
            if (lessequal(L, RKB(i), RKC(i)) == GETARG_A(i))
              dojump(L, pc, GETARG_sBx(*pc));
-@@ -766,13 +820,13 @@
+@@ -764,13 +818,13 @@ void luaV_execute (lua_State *L, int nex
          pc++;
          continue;
        }
@@ -275,7 +275,7 @@
          TValue *rb = RB(i);
          if (l_isfalse(rb) != GETARG_C(i)) {
            setobjs2s(L, ra, rb);
-@@ -781,7 +835,7 @@
+@@ -779,7 +833,7 @@ void luaV_execute (lua_State *L, int nex
          pc++;
          continue;
        }
@@ -284,7 +284,7 @@
          int b = GETARG_B(i);
          int nresults = GETARG_C(i) - 1;
          if (b != 0) L->top = ra+b;  /* else previous instruction set top */
-@@ -802,7 +856,7 @@
+@@ -800,7 +854,7 @@ void luaV_execute (lua_State *L, int nex
            }
          }
        }
@@ -293,7 +293,7 @@
          int b = GETARG_B(i);
          if (b != 0) L->top = ra+b;  /* else previous instruction set top */
          L->savedpc = pc;
-@@ -834,7 +888,7 @@
+@@ -832,7 +886,7 @@ void luaV_execute (lua_State *L, int nex
            }
          }
        }
@@ -302,7 +302,7 @@
          int b = GETARG_B(i);
          if (b != 0) L->top = ra+b-1;
          if (L->openupval) luaF_close(L, base);
-@@ -849,7 +903,7 @@
+@@ -847,7 +901,7 @@ void luaV_execute (lua_State *L, int nex
            goto reentry;
          }
        }
@@ -311,7 +311,7 @@
          /* If start,step and limit are all integers, we don't need to check
           * against overflow in the looping.
           */
-@@ -877,7 +931,7 @@
+@@ -875,7 +929,7 @@ void luaV_execute (lua_State *L, int nex
          }
          continue;
        }
@@ -320,7 +320,7 @@
          const TValue *init = ra;
          const TValue *plimit = ra+1;
          const TValue *pstep = ra+2;
-@@ -900,7 +954,7 @@
+@@ -898,7 +952,7 @@ void luaV_execute (lua_State *L, int nex
          dojump(L, pc, GETARG_sBx(i));
          continue;
        }
@@ -329,7 +329,7 @@
          StkId cb = ra + 3;  /* call base */
          setobjs2s(L, cb+2, ra+2);
          setobjs2s(L, cb+1, ra+1);
-@@ -916,7 +970,7 @@
+@@ -914,7 +968,7 @@ void luaV_execute (lua_State *L, int nex
          pc++;
          continue;
        }
@@ -338,7 +338,7 @@
          int n = GETARG_B(i);
          int c = GETARG_C(i);
          int last;
-@@ -938,11 +992,11 @@
+@@ -936,11 +990,11 @@ void luaV_execute (lua_State *L, int nex
          }
          continue;
        }
@@ -352,7 +352,7 @@
          Proto *p;
          Closure *ncl;
          int nup, j;
-@@ -962,7 +1016,7 @@
+@@ -960,7 +1014,7 @@ void luaV_execute (lua_State *L, int nex
          Protect(luaC_checkGC(L));
          continue;
        }