Implemented Thread Local Storage support.

This is still rough, but the basic idea seems to work!

lx_loader.c

@@ -11,6 +11,8 @@
 #include <dlfcn.h>
 #include <dirent.h>
 #include <pthread.h>
+#include <signal.h>
+#include <ucontext.h>
 
 // 16-bit selector kernel nonsense...
 #include <sys/syscall.h>
@@ -422,7 +424,6 @@ static uint16 initOs2Tib(uint8 *tibspace, void *_topOfStack, const size_t stackl
 
     LxTIB *tib = (LxTIB *) tibspace;
     LxTIB2 *tib2 = (LxTIB2 *) (tib + 1);
-    memset(tib, '\0', LXTIBSIZE);
 
     FIXME("This is probably 50% wrong");
     tib->tib_pexchain = NULL;
@@ -1492,6 +1493,110 @@ static LxModule *loadLxModuleByModuleName(const char *modname)
     return loadLxModuleByModuleNameInternal(modname, 0);
 } // loadLxModuleByModuleName
 
+static __attribute__((noreturn)) void handleThreadLocalStorageAccess(const int slot, ucontext_t *uctx)
+{
+    greg_t *gregs = uctx->uc_mcontext.gregs;
+
+    // use the segfaulting thread's FS register, so we can get its TIB2 pointer.
+    LxTIB2 *ptib2 = NULL;
+    __asm__ __volatile__ (
+        "pushw %%fs            \n\t"
+        "movw %%ax, %%fs       \n\t"
+        "movl %%fs:0xC, %%eax  \n\t"
+        "popw %%fs             \n\t"
+            : "=a" (ptib2)
+            : "eax" (gregs[REG_FS])
+    );
+    uint32 *tls = (uint32 *) (ptib2 + 1);  // The thread's TLS data is stored right after its TIB2 struct.
+    tls += slot;
+
+    printf("We wanted to access thread %p TLS slot %d (currently holds %u)\n", tls - slot, slot, (uint) *tls);
+
+    static const int x86RegisterToUContextEnum[] = {
+        REG_EAX, REG_ECX, REG_EDX, REG_EBX, REG_ESP, REG_EBP, REG_ESI, REG_EDI
+    };
+
+    // this is a hack; we'll want a much more serious x86 instruction decoder before long.
+    int handled = 1;
+    uint8 *eip = (void *) (size_t) gregs[REG_EIP];  // program counter at point of segfault.
+    switch (eip[0]) {
+        case 0xC7:  // mov imm16/32 -> r/m
+            printf("setting TLS slot %d to imm %u.\n", slot, (uint) *((uint32 *) (eip + 2)));
+            *tls = *((uint32 *) (eip + 2));  // !!! FIXME: verify it's a imm32, not 16
+            gregs[REG_EIP] += 6;
+            break;
+
+        case 0x89:  // mov r -> r/m
+            printf("setting TLS slot %d to reg %d (%u).\n", slot, x86RegisterToUContextEnum[eip[1] >> 3], (uint) gregs[x86RegisterToUContextEnum[eip[1] >> 3]]);
+            *tls = (uint32) gregs[x86RegisterToUContextEnum[eip[1] >> 3]];
+            gregs[REG_EIP] += 2;
+            break;
+
+        case 0x8B:  // mov r/m -> r
+            printf("setting to reg %d to TLS slot %d (%u).\n", (uint) x86RegisterToUContextEnum[eip[1] >> 3], slot, *tls);
+            gregs[x86RegisterToUContextEnum[eip[1] >> 3]] = (greg_t) *tls;
+            gregs[REG_EIP] += 2;
+            break;
+
+        default:
+            handled = 0;
+            break;
+    } // switch
+
+    if (!handled) {
+        fprintf(stderr, "Oh no, unhandled opcode 0x%X at %p accessing TLS register! File a bug!\n", (uint) eip[0], eip);
+    } else {
+        // drop out of signal handler to (hopefully) next instruction in the app,
+        //  as if it accessed the TLS slot normally and none of this ever happened.
+        printf("TLS access handler jumping back into app at %p...\n", (void *) gregs[REG_EIP]); fflush(stdout);
+        setcontext(uctx);
+        fprintf(stderr, "panic: setcontext() failed in the TLS access handler! Aborting! (%s)\n", strerror(errno));
+    } // else
+
+    fflush(stderr);
+    abort();
+} // handleThreadLocalStorageAccess
+
+static void segfault_catcher(int sig, siginfo_t *info, void *ctx)
+{
+    const uint32 *addr = (const uint32 *) info->si_addr;
+    const uint32 *tlspage = GLoaderState->tlspage;
+
+    if (tlspage && (addr >= tlspage))
+    {
+        // was the app accessing one of the OS/2 TLS slots?
+        const int slot = (int) (addr - tlspage);
+        if (slot < 32)
+            handleThreadLocalStorageAccess(slot, (ucontext_t *) ctx);
+    } // if
+
+    static int faults = 0;
+    faults++;
+    switch (faults) {
+        // !!! FIXME: case #1 should be a much more detailed crash dump.
+        case 1: fprintf(stderr, "SIGSEGV at addr=%p (eip=%p)\n", addr, (void *) ((ucontext_t *) ctx)->uc_mcontext.gregs[REG_EIP]); break;
+        case 2: write(2, "SIGSEGV, aborting.\n", 19); break;
+    } // switch
+
+    abort();  // cash out.
+} // segfault_catcher
+
+static int installSignalHandlers(void)
+{
+    struct sigaction action;
+
+    memset(&action, '\0', sizeof (action));
+    action.sa_sigaction = segfault_catcher;
+    action.sa_flags = SA_NODEFER | SA_SIGINFO;
+
+    if (sigaction(SIGSEGV, &action, NULL) == -1) {
+        fprintf(stderr, "Couldn't install signal handler! (%s)\n", strerror(errno));
+        return 0;
+    } // if
+
+    return 1;
+} // installSignalHandlers
+
 int main(int argc, char **argv, char **envp)
 {
     if (argc < 2) {
@@ -1499,6 +1604,9 @@ int main(int argc, char **argv, char **envp)
         return 1;
     }
 
+    if (!installSignalHandlers())
+        return 1;
+
     const char *envr = getenv("IS_2INE");
     GLoaderState->subprocess = (envr != NULL);
     GLoaderState->initOs2Tib = initOs2Tib;
@@ -1519,12 +1627,10 @@ int main(int argc, char **argv, char **envp)
     initPib(&GLoaderState->pib, argc, argv, envp);
 
     LxModule *lxmod = loadLxModuleByPath(modulename);
-    if (!lxmod) {
-        return 1;
-    }
-    runLxModule(lxmod, argc, argv, envp);
+    if (lxmod != NULL)
+        runLxModule(lxmod, argc, argv, envp);
 
-    return 1;  // you shouldn't hit this, but if you do, report failure.
+    return 1;
 } // main
 
 // end of lx_loader.c ...

lx_loader.h

@@ -195,7 +195,8 @@ typedef struct LxPIB
 
 #pragma pack(pop)
 
-#define LXTIBSIZE (sizeof (LxTIB) + sizeof (LxTIB2))
+// We put the 128 bytes of TLS slots after the TIB structs.
+#define LXTIBSIZE (sizeof (LxTIB) + sizeof (LxTIB2) + 128)
 
 typedef struct LxLoaderState
 {
@@ -204,6 +205,9 @@ typedef struct LxLoaderState
     LxPIB pib;
     int subprocess;
     int running;
+    uint32 *tlspage;
+    uint32 tlsmask;  // one bit for each TLS slot in use.
+    uint8 tlsallocs[32];  // number of TLS slots allocated in one block, indexed by starting block (zero if not the starting block).
     uint16 (*initOs2Tib)(uint8 *tibspace, void *_topOfStack, const size_t stacklen, const uint32 tid);
     void (*deinitOs2Tib)(const uint16 selector);
     LxModule *(*loadModule)(const char *modname);

native/doscalls.c

@@ -14,6 +14,7 @@
 #include <sys/time.h>
 #include <sys/resource.h>
 #include <sys/stat.h>
+#include <sys/mman.h>
 
 #include "os2native.h"
 #include "doscalls.h"
@@ -218,6 +219,8 @@ LX_NATIVE_MODULE_INIT({ if (!initDoscalls(lx_state)) return NULL; })
     LX_NATIVE_EXPORT(DosExitMustComplete, 381),
     LX_NATIVE_EXPORT(DosSetRelMaxFH, 382),
     LX_NATIVE_EXPORT(DosFlatToSel, 425),
+    LX_NATIVE_EXPORT(DosAllocThreadLocalMemory, 454),
+    LX_NATIVE_EXPORT(DosFreeThreadLocalMemory, 455),
     LX_NATIVE_EXPORT(DosOpenL, 981)
 LX_NATIVE_MODULE_INIT_END()
 
@@ -1309,6 +1312,7 @@ static void *os2ThreadEntry(void *arg)
 {
     // put our thread's TIB structs on the stack and call that the top of the stack.
     uint8 tibspace[LXTIBSIZE];
+    memset(tibspace, '\0', LXTIBSIZE);  // make sure TLS slots are clear, etc.
     os2ThreadEntry2(tibspace, (Thread *) arg);
     return NULL;  // OS/2 threads don't return a value here.
 } // os2ThreadEntry
@@ -2581,6 +2585,106 @@ queryapptype_iofailed:
     return ERROR_INVALID_EXE_SIGNATURE;
 } // DosQueryAppType
 
+static uint32 *initTLSPage(void)
+{
+    const int pagesize = getpagesize();
+
+    void *addr = mmap(NULL, pagesize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    if (addr == ((void *) MAP_FAILED))
+        return NULL;
+
+    // Fill in the page with a debug info string.
+    char *dst = (char *) addr;
+    const char *str = "This is the fake OS/2 TLS page. ";
+    assert(strlen(str) == 32);
+    for (int i = 0; i < 127; i++, dst += 32)
+        strcpy(dst, str);
+    memcpy(dst, str, 31);
+    dst[31] = '\0';
+
+    if (mprotect(addr, pagesize, PROT_NONE) == -1) {
+        munmap(addr, pagesize);
+        return NULL;
+    } // if
+
+    printf("allocated magic OS/2 TLS page at %p\n", addr); fflush(stdout);
+
+    return (uint32 *) addr;
+} // initTLSPage
+
+
+APIRET DosAllocThreadLocalMemory(ULONG cb, PULONG *p)
+{
+    TRACE_NATIVE("DosAllocThreadLocalMemory(%u, %p)", (uint) cb, p);
+
+    if (cb > 8)  // this is a limitation in OS/2. This whole API is weird.
+        return ERROR_INVALID_PARAMETER;
+    else if (!p)
+        return ERROR_INVALID_PARAMETER;
+
+    APIRET retval = NO_ERROR;
+    uint32 mask = 0xFF >> (8 - cb);
+
+    grabLock(&GMutexDosCalls);
+
+    // this is probably expensive to do with the mutex held, but honestly,
+    //  is there a lot of thread contention at the point where your app is
+    //  allocating TLS?
+    if (GLoaderState->tlspage == NULL) {
+        if ((GLoaderState->tlspage = initTLSPage()) == NULL) {
+            ungrabLock(&GMutexDosCalls);
+            return ERROR_NOT_ENOUGH_MEMORY;
+        } // if
+    } // if
+
+    int i;
+    for (i = 0; i < 32; i++, mask <<= 1) {
+        if (((GLoaderState->tlsmask ^ mask) & mask) == mask)
+            break;
+    } // for
+
+    if (i == 32)
+        retval = ERROR_NOT_ENOUGH_MEMORY; // there are only 32 slots, couldn't find anything contiguous.
+    else {
+        assert(GLoaderState->tlsallocs[i] == 0);
+        GLoaderState->tlsallocs[i] = (uint8) cb;
+        GLoaderState->tlsmask |= mask;
+        *p = GLoaderState->tlspage + i;
+        printf("allocated %u OS/2 TLS slot%s at %p\n", (uint) cb, (cb == 1) ? "" : "s", *p); fflush(stdout);
+    } // else
+        
+    ungrabLock(&GMutexDosCalls);
+
+    return retval;
+} // DosAllocThreadLocalMemory
+
+APIRET DosFreeThreadLocalMemory(ULONG *p)
+{
+    TRACE_NATIVE("DosFreeThreadLocalMemory(%p)", p);
+
+    APIRET retval = ERROR_INVALID_PARAMETER;
+
+    grabLock(&GMutexDosCalls);
+
+    if (GLoaderState->tlspage) {
+        const uint32 slot = (uint32) (p - GLoaderState->tlspage);
+        if (slot < 32) {
+            const uint8 slots = GLoaderState->tlsallocs[slot];
+            if (slots > 0) {
+                const uint32 mask = ((uint32) (0xFF >> (8 - slots))) << slot;
+                assert((GLoaderState->tlsmask & mask) == mask);
+                GLoaderState->tlsmask &= ~mask;
+                printf("freed %u OS/2 TLS slot%s at %p\n", (uint) slots, (slots == 1) ? "" : "s", p); fflush(stdout);
+                GLoaderState->tlsallocs[slot] = 0;
+                retval = NO_ERROR;
+            } // if
+        } // if
+    } // if
+
+    ungrabLock(&GMutexDosCalls);
+
+    return retval;
+} // DosFreeThreadLocalMemory
 
 // end of doscalls.c ...
 

native/doscalls.h

@@ -432,6 +432,8 @@ APIRET OS2API DosDevConfig(PVOID pdevinfo, ULONG item);
 APIRET OS2API DosLoadModule(PSZ pszName, ULONG cbName, PSZ pszModname, PHMODULE phmod);
 APIRET OS2API DosResetBuffer(HFILE hFile);
 APIRET OS2API DosQueryAppType(PSZ pszName, PULONG pFlags);
+APIRET OS2API DosAllocThreadLocalMemory(ULONG cb, PULONG *p);
+APIRET OS2API DosFreeThreadLocalMemory(ULONG *p);
 
 #ifdef __cplusplus
 }

tests/testtls.c

@@ -0,0 +1,54 @@
+#define INCL_DOS
+#define INCL_DOSERRORS
+#include <stdio.h>
+#include <stdlib.h>
+#include <os2.h>
+
+static PULONG GAddr = NULL;
+static ULONG x = 0;
+
+static void APIENTRY threadFunc(ULONG arg)
+{
+    const unsigned int t = (unsigned int) arg;
+    DosSleep(t == 1 ? 500 : 1000);
+    *GAddr = t * 100;
+    //printf("thread %u set TLS value to %u\n", t, t * 100);
+    //fflush(stdout);
+    DosSleep(t == 1 ? 1000 : 50);
+    x = *GAddr;
+    //printf("thread %u sees TLS value of %u\n", t, (unsigned int) *GAddr);
+    //fflush(stdout);
+}
+
+int main(void)
+{
+    TID tid = 0;
+
+    if (DosAllocThreadLocalMemory(1, &GAddr) != NO_ERROR) {
+        fprintf(stderr, "DosAllocLocalThreadMemory failed\n");
+        return 1;
+    }
+    printf("the magic address is %p\n", GAddr);
+    *GAddr = 10;
+    printf("Main thread set TLS value to 10\n");
+    fflush(stdout);
+
+    if (DosCreateThread(&tid, threadFunc, 1, 0, 0xFFFF) != NO_ERROR) {
+        fprintf(stderr, "DosCreateThread 1 failed\n");
+        return 1;
+    }
+    if (DosCreateThread(&tid, threadFunc, 2, 0, 0xFFFF) != NO_ERROR) {
+        fprintf(stderr, "DosCreateThread 2 failed\n");
+        return 1;
+    }
+
+    DosSleep(4000);
+
+    printf("Main thread sees TLS value of %u\n", (unsigned int) *GAddr);
+    fflush(stdout);
+
+    DosFreeThreadLocalMemory(GAddr);
+
+    return 0;
+}
+