DrSocalkwe3n/luajit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Redesign and harden string interning.

Browse Source 
Up to 40% faster on hash-intensive benchmarks.
With some ideas from Sokolov Yura.
This commit is contained in:
Mike Pall
2020-06-23 03:06:45 +02:00
parent a44f53acf5
commit ff34b48ddd
22 changed files with 394 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files

358
src/lj_str.c
View File

@@ -11,6 +11,7 @@
#include "lj_err.h"
#include "lj_str.h"
#include "lj_char.h"
#include "lj_prng.h"
/* -- String helpers ------------------------------------------------------ */
@@ -37,28 +38,6 @@ int32_t LJ_FASTCALL lj_str_cmp(GCstr *a, GCstr *b)
return (int32_t)(a->len - b->len);
}
/* Fast string data comparison. Caveat: unaligned access to 1st string! */
static LJ_AINLINE int str_fastcmp(const char *a, const char *b, MSize len)
{
MSize i = 0;
lj_assertX(len > 0, "fast string compare with zero length");
lj_assertX((((uintptr_t)a+len-1) & (LJ_PAGESIZE-1)) <= LJ_PAGESIZE-4,
"fast string compare crossing page boundary");
do { /* Note: innocuous access up to end of string + 3. */
uint32_t v = lj_getu32(a+i) ^ *(const uint32_t *)(b+i);
if (v) {
i -= len;
#if LJ_LE
return (int32_t)i >= -3 ? (v << (32+(i<<3))) : 1;
#else
return (int32_t)i >= -3 ? (v >> (32+(i<<3))) : 1;
#endif
}
i += 4;
} while (i < len);
return 0;
}
/* Find fixed string p inside string s. */
const char *lj_str_find(const char *s, const char *p, MSize slen, MSize plen)
{
@@ -91,108 +70,301 @@ int lj_str_haspattern(GCstr *s)
return 0; /* No pattern matching chars found. */
}
/* -- String interning ---------------------------------------------------- */
/* -- String hashing ------------------------------------------------------ */
/* Resize the string hash table (grow and shrink). */
void lj_str_resize(lua_State *L, MSize newmask)
/* Keyed sparse ARX string hash. Constant time. */
static StrHash hash_sparse(uint64_t seed, const char *str, MSize len)
{
global_State *g = G(L);
GCRef *newhash;
MSize i;
if (g->gc.state == GCSsweepstring || newmask >= LJ_MAX_STRTAB-1)
return; /* No resizing during GC traversal or if already too big. */
newhash = lj_mem_newvec(L, newmask+1, GCRef);
memset(newhash, 0, (newmask+1)*sizeof(GCRef));
for (i = g->strmask; i != ~(MSize)0; i--) { /* Rehash old table. */
GCobj *p = gcref(g->strhash[i]);
while (p) { /* Follow each hash chain and reinsert all strings. */
MSize h = gco2str(p)->hash & newmask;
GCobj *next = gcnext(p);
/* NOBARRIER: The string table is a GC root. */
setgcrefr(p->gch.nextgc, newhash[h]);
setgcref(newhash[h], p);
p = next;
}
}
lj_mem_freevec(g, g->strhash, g->strmask+1, GCRef);
g->strmask = newmask;
g->strhash = newhash;
}
/* Intern a string and return string object. */
GCstr *lj_str_new(lua_State *L, const char *str, size_t lenx)
{
global_State *g;
GCstr *s;
GCobj *o;
MSize len = (MSize)lenx;
MSize a, b, h = len;
if (lenx >= LJ_MAX_STR)
lj_err_msg(L, LJ_ERR_STROV);
g = G(L);
/* Compute string hash. Constants taken from lookup3 hash by Bob Jenkins. */
/* Constants taken from lookup3 hash by Bob Jenkins. */
StrHash a, b, h = len ^ (StrHash)seed;
if (len >= 4) { /* Caveat: unaligned access! */
a = lj_getu32(str);
h ^= lj_getu32(str+len-4);
b = lj_getu32(str+(len>>1)-2);
h ^= b; h -= lj_rol(b, 14);
b += lj_getu32(str+(len>>2)-1);
} else if (len > 0) {
} else {
a = *(const uint8_t *)str;
h ^= *(const uint8_t *)(str+len-1);
b = *(const uint8_t *)(str+(len>>1));
h ^= b; h -= lj_rol(b, 14);
} else {
return &g->strempty;
}
a ^= h; a -= lj_rol(h, 11);
b ^= a; b -= lj_rol(a, 25);
h ^= b; h -= lj_rol(b, 16);
/* Check if the string has already been interned. */
o = gcref(g->strhash[h & g->strmask]);
if (LJ_LIKELY((((uintptr_t)str+len-1) & (LJ_PAGESIZE-1)) <= LJ_PAGESIZE-4)) {
while (o != NULL) {
GCstr *sx = gco2str(o);
if (sx->len == len && str_fastcmp(str, strdata(sx), len) == 0) {
/* Resurrect if dead. Can only happen with fixstring() (keywords). */
if (isdead(g, o)) flipwhite(o);
return sx; /* Return existing string. */
return h;
}
#if LUAJIT_SECURITY_STRHASH
/* Keyed dense ARX string hash. Linear time. */
static LJ_NOINLINE StrHash hash_dense(uint64_t seed, StrHash h,
const char *str, MSize len)
{
StrHash b = lj_bswap(lj_rol(h ^ (StrHash)(seed >> 32), 4));
if (len > 12) {
StrHash a = (StrHash)seed;
const char *pe = str+len-12, *p = pe, *q = str;
do {
a += lj_getu32(p);
b += lj_getu32(p+4);
h += lj_getu32(p+8);
p = q; q += 12;
h ^= b; h -= lj_rol(b, 14);
a ^= h; a -= lj_rol(h, 11);
b ^= a; b -= lj_rol(a, 25);
} while (p < pe);
h ^= b; h -= lj_rol(b, 16);
a ^= h; a -= lj_rol(h, 4);
b ^= a; b -= lj_rol(a, 14);
}
return b;
}
#endif
/* -- String interning ---------------------------------------------------- */
#define LJ_STR_MAXCOLL 32
/* Resize the string interning hash table (grow and shrink). */
void lj_str_resize(lua_State *L, MSize newmask)
{
global_State *g = G(L);
GCRef *newtab, *oldtab = g->str.tab;
MSize i;
/* No resizing during GC traversal or if already too big. */
if (g->gc.state == GCSsweepstring || newmask >= LJ_MAX_STRTAB-1)
return;
newtab = lj_mem_newvec(L, newmask+1, GCRef);
memset(newtab, 0, (newmask+1)*sizeof(GCRef));
#if LUAJIT_SECURITY_STRHASH
/* Check which chains need secondary hashes. */
if (g->str.second) {
int newsecond = 0;
/* Compute primary chain lengths. */
for (i = g->str.mask; i != ~(MSize)0; i--) {
GCobj *o = (GCobj *)(gcrefu(oldtab[i]) & ~(uintptr_t)1);
while (o) {
GCstr *s = gco2str(o);
MSize hash = s->hashalg ? hash_sparse(g->str.seed, strdata(s), s->len) :
s->hash;
hash &= newmask;
setgcrefp(newtab[hash], gcrefu(newtab[hash]) + 1);
o = gcnext(o);
}
o = gcnext(o);
}
} else { /* Slow path: end of string is too close to a page boundary. */
while (o != NULL) {
GCstr *sx = gco2str(o);
if (sx->len == len && memcmp(str, strdata(sx), len) == 0) {
/* Resurrect if dead. Can only happen with fixstring() (keywords). */
if (isdead(g, o)) flipwhite(o);
return sx; /* Return existing string. */
/* Mark secondary chains. */
for (i = newmask; i != ~(MSize)0; i--) {
int secondary = gcrefu(newtab[i]) > LJ_STR_MAXCOLL;
newsecond |= secondary;
setgcrefp(newtab[i], secondary);
}
g->str.second = newsecond;
}
#endif
/* Reinsert all strings from the old table into the new table. */
for (i = g->str.mask; i != ~(MSize)0; i--) {
GCobj *o = (GCobj *)(gcrefu(oldtab[i]) & ~(uintptr_t)1);
while (o) {
GCobj *next = gcnext(o);
GCstr *s = gco2str(o);
MSize hash = s->hash;
#if LUAJIT_SECURITY_STRHASH
uintptr_t u;
if (LJ_LIKELY(!s->hashalg)) { /* String hashed with primary hash. */
hash &= newmask;
u = gcrefu(newtab[hash]);
if (LJ_UNLIKELY(u & 1)) { /* Switch string to secondary hash. */
s->hash = hash = hash_dense(g->str.seed, s->hash, strdata(s), s->len);
s->hashalg = 1;
hash &= newmask;
u = gcrefu(newtab[hash]);
}
} else { /* String hashed with secondary hash. */
MSize shash = hash_sparse(g->str.seed, strdata(s), s->len);
u = gcrefu(newtab[shash & newmask]);
if (u & 1) {
hash &= newmask;
u = gcrefu(newtab[hash]);
} else { /* Revert string back to primary hash. */
s->hash = shash;
s->hashalg = 0;
hash = (shash & newmask);
}
}
o = gcnext(o);
/* NOBARRIER: The string table is a GC root. */
setgcrefp(o->gch.nextgc, (u & ~(uintptr_t)1));
setgcrefp(newtab[hash], ((uintptr_t)o | (u & 1)));
#else
hash &= newmask;
/* NOBARRIER: The string table is a GC root. */
setgcrefr(o->gch.nextgc, newtab[hash]);
setgcref(newtab[hash], o);
#endif
o = next;
}
}
/* Nope, create a new string. */
s = lj_mem_newt(L, sizeof(GCstr)+len+1, GCstr);
/* Free old table and replace with new table. */
lj_str_freetab(g);
g->str.tab = newtab;
g->str.mask = newmask;
}
#if LUAJIT_SECURITY_STRHASH
/* Rehash and rechain all strings in a chain. */
static LJ_NOINLINE GCstr *lj_str_rehash_chain(lua_State *L, StrHash hashc,
const char *str, MSize len)
{
global_State *g = G(L);
int ow = g->gc.state == GCSsweepstring ? otherwhite(g) : 0; /* Sweeping? */
GCRef *strtab = g->str.tab;
MSize strmask = g->str.mask;
GCobj *o = gcref(strtab[hashc & strmask]);
setgcrefp(strtab[hashc & strmask], (void *)((uintptr_t)1));
g->str.second = 1;
while (o) {
uintptr_t u;
GCobj *next = gcnext(o);
GCstr *s = gco2str(o);
StrHash hash;
if (ow) { /* Must sweep while rechaining. */
if (((o->gch.marked ^ LJ_GC_WHITES) & ow)) { /* String alive? */
lj_assertG(!isdead(g, o) || (o->gch.marked & LJ_GC_FIXED),
"sweep of undead string");
makewhite(g, o);
} else { /* Free dead string. */
lj_assertG(isdead(g, o) || ow == LJ_GC_SFIXED,
"sweep of unlive string");
lj_str_free(g, s);
o = next;
continue;
}
}
hash = s->hash;
if (!s->hashalg) { /* Rehash with secondary hash. */
hash = hash_dense(g->str.seed, hash, strdata(s), s->len);
s->hash = hash;
s->hashalg = 1;
}
/* Rechain. */
hash &= strmask;
u = gcrefu(strtab[hash]);
setgcrefp(o->gch.nextgc, (u & ~(uintptr_t)1));
setgcrefp(strtab[hash], ((uintptr_t)o | (u & 1)));
o = next;
}
/* Try to insert the pending string again. */
return lj_str_new(L, str, len);
}
#endif
/* Reseed String ID from PRNG after random interval < 2^bits. */
#if LUAJIT_SECURITY_STRID == 1
#define STRID_RESEED_INTERVAL 8
#elif LUAJIT_SECURITY_STRID == 2
#define STRID_RESEED_INTERVAL 4
#elif LUAJIT_SECURITY_STRID >= 3
#define STRID_RESEED_INTERVAL 0
#endif
/* Allocate a new string and add to string interning table. */
static GCstr *lj_str_alloc(lua_State *L, const char *str, MSize len,
StrHash hash, int hashalg)
{
GCstr *s = lj_mem_newt(L, lj_str_size(len), GCstr);
global_State *g = G(L);
uintptr_t u;
newwhite(g, s);
s->gct = ~LJ_TSTR;
s->len = len;
s->hash = h;
s->hash = hash;
#ifndef STRID_RESEED_INTERVAL
s->sid = g->str.id++;
#elif STRID_RESEED_INTERVAL
if (!g->str.idreseed--) {
uint64_t r = lj_prng_u64(&g->prng);
g->str.id = (StrID)r;
g->str.idreseed = (uint8_t)(r >> (64 - STRID_RESEED_INTERVAL));
}
s->sid = g->str.id++;
#else
s->sid = (StrID)lj_prng_u64(&g->prng);
#endif
s->reserved = 0;
s->hashalg = (uint8_t)hashalg;
/* Clear last 4 bytes of allocated memory. Implies zero-termination, too. */
*(uint32_t *)(strdatawr(s)+(len & ~(MSize)3)) = 0;
memcpy(strdatawr(s), str, len);
strdatawr(s)[len] = '\0'; /* Zero-terminate string. */
/* Add it to string hash table. */
h &= g->strmask;
s->nextgc = g->strhash[h];
/* Add to string hash table. */
hash &= g->str.mask;
u = gcrefu(g->str.tab[hash]);
setgcrefp(s->nextgc, (u & ~(uintptr_t)1));
/* NOBARRIER: The string table is a GC root. */
setgcref(g->strhash[h], obj2gco(s));
if (g->strnum++ > g->strmask) /* Allow a 100% load factor. */
lj_str_resize(L, (g->strmask<<1)+1); /* Grow string table. */
setgcrefp(g->str.tab[hash], ((uintptr_t)s | (u & 1)));
if (g->str.num++ > g->str.mask) /* Allow a 100% load factor. */
lj_str_resize(L, (g->str.mask<<1)+1); /* Grow string table. */
return s; /* Return newly interned string. */
}
/* Intern a string and return string object. */
GCstr *lj_str_new(lua_State *L, const char *str, size_t lenx)
{
global_State *g = G(L);
if (lenx-1 < LJ_MAX_STR-1) {
MSize len = (MSize)lenx;
StrHash hash = hash_sparse(g->str.seed, str, len);
MSize coll = 0;
int hashalg = 0;
/* Check if the string has already been interned. */
GCobj *o = gcref(g->str.tab[hash & g->str.mask]);
#if LUAJIT_SECURITY_STRHASH
if (LJ_UNLIKELY((uintptr_t)o & 1)) { /* Secondary hash for this chain? */
hashalg = 1;
hash = hash_dense(g->str.seed, hash, str, len);
o = (GCobj *)(gcrefu(g->str.tab[hash & g->str.mask]) & ~(uintptr_t)1);
}
#endif
while (o != NULL) {
GCstr *sx = gco2str(o);
if (sx->hash == hash && sx->len == len) {
if (memcmp(str, strdata(sx), len) == 0) {
if (isdead(g, o)) flipwhite(o); /* Resurrect if dead. */
return sx; /* Return existing string. */
}
coll++;
}
coll++;
o = gcnext(o);
}
#if LUAJIT_SECURITY_STRHASH
/* Rehash chain if there are too many collisions. */
if (LJ_UNLIKELY(coll > LJ_STR_MAXCOLL) && !hashalg) {
return lj_str_rehash_chain(L, hash, str, len);
}
#endif
/* Otherwise allocate a new string. */
return lj_str_alloc(L, str, len, hash, hashalg);
} else {
if (lenx)
lj_err_msg(L, LJ_ERR_STROV);
return &g->strempty;
}
}
void LJ_FASTCALL lj_str_free(global_State *g, GCstr *s)
{
g->strnum--;
lj_mem_free(g, s, sizestring(s));
g->str.num--;
lj_mem_free(g, s, lj_str_size(s->len));
}
void LJ_FASTCALL lj_str_init(lua_State *L)
{
global_State *g = G(L);
g->str.seed = lj_prng_u64(&g->prng);
lj_str_resize(L, LJ_MIN_STRTAB-1);
}