Improve SQL handling & aggregate memory events (WIP)data_improvements

9 files changed, 406 insertions, 242 deletions

diff --git a/arch/dummy/arch.j2.c b/arch/dummy/arch.j2.c
index c331b34..f51494f 100644
--- a/arch/dummy/arch.j2.c
+++ b/arch/dummy/arch.j2.c
@@ -28,6 +28,12 @@ void arch_core_init(struct Core *core) {
 }
 {% endif %}
 
+void arch_core_free(struct Core *core) {
+    assert(core);
+
+    (void)core;
+}
+
 {% if args.command in ["load", "new"] %}
 void arch_core_save(FILE *f, const struct Core *core) {
     assert(f);
diff --git a/arch/dummy/arch_vars.py b/arch/dummy/arch_vars.py
index e121ca5..bc97ea9 100644
--- a/arch/dummy/arch_vars.py
+++ b/arch/dummy/arch_vars.py
@@ -1,23 +1,26 @@
-core_fields = []
-mvec_loop   = True
+def gen_arch_vars(_):
+    return {
+        "core_fields": [],
+        "mvec_loop": True,
 
-proc_fields = [
-    ("uint64_t", "ip"),
-    ("uint64_t", "sp"),
-    ("uint64_t", "mb0a"),
-    ("uint64_t", "mb0s"),
-    ("uint64_t", "mb1a"),
-    ("uint64_t", "mb1s"),
-]
+        "proc_fields": [
+            ("uint64_t", "ip"),
+            ("uint64_t", "sp"),
+            ("uint64_t", "mb0a"),
+            ("uint64_t", "mb0s"),
+            ("uint64_t", "mb1a"),
+            ("uint64_t", "mb1s"),
+        ],
 
-inst_set = [
-    (["dummy", f"{i:02x}"], symbol)
-    for i, symbol in enumerate(
-        "⠀⠁⠂⠃⠄⠅⠆⠇⡀⡁⡂⡃⡄⡅⡆⡇⠈⠉⠊⠋⠌⠍⠎⠏⡈⡉⡊⡋⡌⡍⡎⡏⠐⠑⠒⠓⠔⠕⠖⠗⡐⡑⡒⡓⡔⡕⡖⡗⠘⠙⠚⠛⠜⠝⠞⠟⡘⡙⡚⡛⡜⡝⡞⡟"
-        "⠠⠡⠢⠣⠤⠥⠦⠧⡠⡡⡢⡣⡤⡥⡦⡧⠨⠩⠪⠫⠬⠭⠮⠯⡨⡩⡪⡫⡬⡭⡮⡯⠰⠱⠲⠳⠴⠵⠶⠷⡰⡱⡲⡳⡴⡵⡶⡷⠸⠹⠺⠻⠼⠽⠾⠿⡸⡹⡺⡻⡼⡽⡾⡿"
-        "⢀⢁⢂⢃⢄⢅⢆⢇⣀⣁⣂⣃⣄⣅⣆⣇⢈⢉⢊⢋⢌⢍⢎⢏⣈⣉⣊⣋⣌⣍⣎⣏⢐⢑⢒⢓⢔⢕⢖⢗⣐⣑⣒⣓⣔⣕⣖⣗⢘⢙⢚⢛⢜⢝⢞⢟⣘⣙⣚⣛⣜⣝⣞⣟"
-        "⢠⢡⢢⢣⢤⢥⢦⢧⣠⣡⣢⣣⣤⣥⣦⣧⢨⢩⢪⢫⢬⢭⢮⢯⣨⣩⣪⣫⣬⣭⣮⣯⢰⢱⢲⢳⢴⢵⢶⢷⣰⣱⣲⣳⣴⣵⣶⣷⢸⢹⢺⢻⢼⢽⢾⢿⣸⣹⣺⣻⣼⣽⣾⣿"
-    )
-]
+        "inst_set": [
+            (["dummy", f"{i:02x}"], symbol)
+            for i, symbol in enumerate(
+                "⠀⠁⠂⠃⠄⠅⠆⠇⡀⡁⡂⡃⡄⡅⡆⡇⠈⠉⠊⠋⠌⠍⠎⠏⡈⡉⡊⡋⡌⡍⡎⡏⠐⠑⠒⠓⠔⠕⠖⠗⡐⡑⡒⡓⡔⡕⡖⡗⠘⠙⠚⠛⠜⠝⠞⠟⡘⡙⡚⡛⡜⡝⡞⡟"
+                "⠠⠡⠢⠣⠤⠥⠦⠧⡠⡡⡢⡣⡤⡥⡦⡧⠨⠩⠪⠫⠬⠭⠮⠯⡨⡩⡪⡫⡬⡭⡮⡯⠰⠱⠲⠳⠴⠵⠶⠷⡰⡱⡲⡳⡴⡵⡶⡷⠸⠹⠺⠻⠼⠽⠾⠿⡸⡹⡺⡻⡼⡽⡾⡿"
+                "⢀⢁⢂⢃⢄⢅⢆⢇⣀⣁⣂⣃⣄⣅⣆⣇⢈⢉⢊⢋⢌⢍⢎⢏⣈⣉⣊⣋⣌⣍⣎⣏⢐⢑⢒⢓⢔⢕⢖⢗⣐⣑⣒⣓⣔⣕⣖⣗⢘⢙⢚⢛⢜⢝⢞⢟⣘⣙⣚⣛⣜⣝⣞⣟"
+                "⢠⢡⢢⢣⢤⢥⢦⢧⣠⣡⣢⣣⣤⣥⣦⣧⢨⢩⢪⢫⢬⢭⢮⢯⣨⣩⣪⣫⣬⣭⣮⣯⢰⢱⢲⢳⢴⢵⢶⢷⣰⣱⣲⣳⣴⣵⣶⣷⢸⢹⢺⢻⢼⢽⢾⢿⣸⣹⣺⣻⣼⣽⣾⣿"
+            )
+        ],
 
-inst_count = len(inst_set)
+        "inst_count": 2 ** 7,
+    }
diff --git a/arch/salis-v1/arch.j2.c b/arch/salis-v1/arch.j2.c
index 9c3ea45..06a701d 100644
--- a/arch/salis-v1/arch.j2.c
+++ b/arch/salis-v1/arch.j2.c
@@ -33,6 +33,11 @@ void arch_core_init(struct Core *core) {
 }
 {% endif %}
 
+void arch_core_free(struct Core *core) {
+    assert(core);
+    (void)core;
+}
+
 {% if args.command in ["load", "new"] %}
 void arch_core_save(FILE *f, const struct Core *core) {
     assert(f);
@@ -47,6 +52,7 @@ void arch_core_save(FILE *f, const struct Core *core) {
     fwrite(&core->wmb0, sizeof(uint64_t), 1, f);
     fwrite(&core->wmb1, sizeof(uint64_t), 1, f);
     fwrite(&core->wdea, sizeof(uint64_t), 1, f);
+    fwrite( core->aeva, sizeof(uint64_t), {{ mvec_size }}, f);
 }
 {% endif %}
 
@@ -64,6 +70,7 @@ void arch_core_load(FILE *f, struct Core *core) {
     fread(&core->wmb0, sizeof(uint64_t), 1, f);
     fread(&core->wmb1, sizeof(uint64_t), 1, f);
     fread(&core->wdea, sizeof(uint64_t), 1, f);
+    fread( core->aeva, sizeof(uint64_t), {{ mvec_size }}, f);
 }
 {% endif %}
 
@@ -119,6 +126,9 @@ void _free_memory_block(struct Core *core, uint64_t addr, uint64_t size) {
 
     for (uint64_t i = 0; i < size; ++i) {
         mvec_free(core, addr + i);
+
+        // Record deallocation event
+        ++core->aeva[addr];
     }
 }
 
@@ -388,6 +398,9 @@ void _alloc(struct Core *core, uint64_t pix, bool fwrd) {
     // Enlarge child block 1 byte
     mvec_alloc(core, proc->sp);
 
+    // Record allocation event
+    ++core->aeva[proc->sp];
+
     if (!proc->mb1s || !fwrd) {
         proc->mb1a = proc->sp;
     }
@@ -828,12 +841,17 @@ const char *arch_mnemonic(uint8_t inst) {
     return NULL;
 }
 
+// ----------------------------------------------------------------------------
+// Data aggregation functions
+// ----------------------------------------------------------------------------
 {% if data_push_path is defined %}
 void arch_push_data_header() {
     assert(g_sim_data);
 
-    // General trends from all cores will be queried together
-    const char *trend_sql = (
+    // Creates general trend table for all cores
+    g_info("Creating 'trend' table in SQLite database");
+    salis_exec_sql(
+        0, NULL, NULL,
         "create table trend ("
         "step int not null, "
         {% for i in range(args.cores) %}
@@ -855,37 +873,54 @@ void arch_push_data_header() {
         ");"
     );
 
-    g_info("Generating 'trend' table in SQLite database");
-    salis_exec_sql(trend_sql);
+    // Creates core-specific instruction data tables
+    char *iprefs[]   = { "pop", "exe", "wrt" };
+    int   iprefs_cnt = sizeof(iprefs) / sizeof(iprefs[0]);
 
-    // Core-specific instruction data will be queried separately
-    // A table is created for each core
-    {% for t in ["pop", "exe", "wrt"] %}
-    {% for i in range(args.cores) %}
-    const char *{{ t }}_sql_{{ i }} = (
-        "create table {{ t }}_{{ i }} ("
-        "step int not null, "
-        // Cycle data allows normalizing against each core's cycle speed
-        {% for j in range(args.cores) %}
-        "cycl_{{ j }} int not null, "
-        {% endfor %}
-        {% for j in arch_vars.inst_set %}
-        "inst_{{ j[0]|join(' ') }} int not null{% if not loop.last %},{% endif %} "
-        {% endfor %}
-        ");"
-    );
+    for (int i = 0; i < {{ args.cores }}; ++i) {
+        for (int j = 0; j < iprefs_cnt; ++j) {
+            g_info("Creating '%s_%d' table in SQLite database", iprefs[j], i);
+            salis_exec_sql(
+                0, NULL, NULL,
+                "create table %s_%d ("
+                "step int not null, "
+                // Cycle data for all cores allows
+                // normalizing against each core's cycle speed
+                {% for i in range(args.cores) %}
+                "cycl_{{ i }} int not null, "
+                {% endfor %}
+                {% for i in arch_vars.inst_set %}
+                "{{ i[0][0] }} int not null{% if not loop.last %},{% endif %} "
+                {% endfor %}
+                ");",
+                iprefs[j], i
+            );
+        }
+    }
 
-    g_info("Generating '{{ t }}_{{ i }}' table in SQLite database");
-    salis_exec_sql({{ t }}_sql_{{ i }});
-    {% endfor %}
-    {% endfor %}
+    // Creates core-specific memory event tables
+    char *eprefs[]   = { "aev" };
+    int   eprefs_cnt = sizeof(eprefs) / sizeof(eprefs[0]);
+
+    for (int i = 0; i < {{ args.cores }}; ++i) {
+        for (int j = 0; j < eprefs_cnt; ++j) {
+            g_info("Creating '%s_%d' table in SQLite database", eprefs[j], i);
+            salis_exec_sql(
+                0, NULL, NULL,
+                "create table %s_%d ("
+                "step int  not null, "
+                "evts blob not null"
+                ");",
+                eprefs[j], i
+            );
+        }
+    }
 }
 
 void arch_push_data_line() {
     assert(g_sim_data);
 
-    // Measure current instruction population and
-    // average memory block sizes
+    // Measure current instruction population and average memory block sizes
     uint64_t ipop[{{ args.cores }}][{{ arch_vars.inst_count }}] = { 0 };
 
     double amb0[{{ args.cores }}] = { 0 };
@@ -920,11 +955,10 @@ void arch_push_data_line() {
         {% endif %}
     }
 
-    // Insert new row
-    char *trend_sql = NULL;
-
-    asprintf(
-        &trend_sql,
+    // Insert row into trend table
+    g_info("Pushing row to 'trend' table in SQLite database");
+    salis_exec_sql(
+        0, NULL, NULL,
         "insert into trend ("
         "step, "
         {% for i in range(args.cores) %}
@@ -950,7 +984,7 @@ void arch_push_data_line() {
         {% endfor %}
         ");",
         g_steps,
-        {% for i in range(args.cores) %}
+        {% for i in range(args.cores) +%}
         g_cores[{{ i }}].cycl,
         g_cores[{{ i }}].mall,
         g_cores[{{ i }}].pnum,
@@ -965,59 +999,118 @@ void arch_push_data_line() {
         g_cores[{{ i }}].wmb0,
         g_cores[{{ i }}].wmb1,
         g_cores[{{ i }}].wdea{% if not loop.last %},{% endif %}
-        {% endfor %}
+        {% endfor +%}
     );
 
-    g_info("Pushing row to 'trend' table in SQLite database");
-    salis_exec_sql(trend_sql);
-    free(trend_sql);
+    // Insert row into instruction data tables
+    char *iprefs[]   = { "pop", "exe", "wrt" };
+    int   iprefs_cnt = sizeof(iprefs) / sizeof(iprefs[0]);
 
-    {% for t in ["pop", "exe", "wrt"] %}
-    {% for i in range(args.cores) %}
+    for (int i = 0; i < {{ args.cores }}; ++i) {
+        for (int j = 0; j < iprefs_cnt; ++j) {
+            uint64_t *ia;
+
+            if (!strcmp("pop", iprefs[j])) {
+                // Population is generated above, prior to push
+                ia = ipop[i];
+            } else if (!strcmp("exe", iprefs[j])) {
+                ia = g_cores[i].iexe;
+            } else if (!strcmp("wrt", iprefs[j])) {
+                ia = g_cores[i].iwrt;
+            }
 
-    {% if t == "pop" %}
-        // Reference instruction population vector that was
-        // manually generated above
-        {% set table = "ipop[%d]"|format(i) %}
-    {% else %}
-        {% set table = "g_cores[%d].i%s"|format(i, t) %}
-    {% endif %}
+            g_info("Pushing row to '%s_%d' table in SQLite database", iprefs[j], i);
+            salis_exec_sql(
+                0, NULL, NULL,
+                "insert into %s_%d ("
+                "step, "
+                {% for i in range(args.cores) %}
+                "cycl_{{ i }}, "
+                {% endfor %}
+                {% for i in arch_vars.inst_set %}
+                "{{ i[0][0] }}{% if not loop.last %},{% endif %} "
+                {% endfor %}
+                ") values ("
+                "%ld, "
+                {% for _ in range(args.cores) %}
+                "%ld, "
+                {% endfor %}
+                {% for _ in range(arch_vars.inst_count) %}
+                "%ld{% if not loop.last %},{% endif %} "
+                {% endfor %}
+                ");",
+                iprefs[j],
+                i,
+                g_steps,
+                {% for j in range(args.cores) %}
+                g_cores[{{ j }}].cycl,
+                {% endfor %}
+                {% for j in range(arch_vars.inst_count) %}
+                ia[{{ j }}]{% if not loop.last %},{% endif +%}
+                {% endfor %}
+            );
+        }
+    }
 
-    char *{{ t }}_sql_{{ i }} = NULL;
+    // Insert row into memory event tables
+    char *eprefs[]   = { "aev" };
+    int   eprefs_cnt = sizeof(eprefs) / sizeof(eprefs[0]);
 
-    asprintf(
-        &{{ t }}_sql_{{ i }},
-        "insert into {{ t }}_{{ i }} ("
-        "step, "
-        {% for j in range(args.cores) %}
-        "cycl_{{ j }}, "
-        {% endfor %}
-        {% for j in arch_vars.inst_set %}
-        "inst_{{ j[0]|join(' ') }}{% if not loop.last %},{% endif %} "
-        {% endfor %}
-        ") values ("
-        "%ld, "
-        {% for _ in range(args.cores) %}
-        "%ld, "
-        {% endfor %}
-        {% for _ in range(arch_vars.inst_count) %}
-        "%ld{% if not loop.last %},{% endif %} "
-        {% endfor %}
-        ");",
-        g_steps,
-        {% for j in range(args.cores) %}
-        g_cores[{{ j }}].cycl,
-        {% endfor %}
-        {% for j in range(arch_vars.inst_count) %}
-        {{ table }}[{{ j }}]{% if not loop.last %},{% endif %}
-        {% endfor %}
-    );
+    // Event run-length data array is defined as static
+    // This prevents heap errors
+    static uint64_t erl_data[{{ mvec_size }} * 2];
 
-    g_info("Pushing row to '{{ t }}_{{ i }}' table in SQLite database");
-    salis_exec_sql({{ t }}_sql_{{ i }});
-    free({{ t }}_sql_{{ i }});
-    {% endfor %}
-    {% endfor %}
+    for (int i = 0; i < {{ args.cores }}; ++i) {
+        for (int j = 0; j < eprefs_cnt; ++j) {
+            uint64_t *eva;
+
+            if (!strcmp("aev", eprefs[j])) {
+                eva = g_cores[i].aeva;
+            }
+
+            // Assume event data to be sparse in most cases
+            // Run-length encoding should help keep database size manageable,
+            // while making it easy to decode events array, wherever the
+            // database is consumed.
+            uint64_t addr = 0;
+            uint64_t eix  = 0;
+
+            while (addr < {{ mvec_size }}) {
+                assert(eix < {{ mvec_size }} * 2);
+
+                erl_data[eix]     = eva[addr];
+                erl_data[eix + 1] = 0;
+
+                while (addr < {{ mvec_size }} && eva[addr] == erl_data[eix]) {
+                    ++erl_data[eix + 1];
+                    ++addr;
+                }
+
+                eix += 2;
+            }
+
+            {% if not args.optimized %}
+            uint64_t el = 0;
+
+            for (uint64_t k = 0; k < eix; k += 2) {
+                el += erl_data[k + 1];
+            }
+
+            assert(el == {{ mvec_size }});
+            {% endif %}
+
+            // Insert blob into database
+            const void *blob_ptr  = erl_data;
+            int         blob_size = eix * sizeof(uint64_t);
+
+            g_info("Pushing row to '%s_%d' table in SQLite database", eprefs[j], i);
+            salis_exec_sql(
+                1, &blob_ptr, &blob_size,
+                "insert into %s_%d (step, evts) values (%ld, ?);",
+                eprefs[j], i, g_steps
+            );
+        }
+    }
 
     // Reset all data aggregation core fields to zero
     for (int i = 0; i < {{ args.cores }}; ++i) {
@@ -1033,6 +1126,9 @@ void arch_push_data_line() {
         core->wmb0 = 0;
         core->wmb1 = 0;
         core->wdea = 0;
+
+        // Event vectors
+        memset(core->aeva, 0, sizeof(uint64_t) * {{ mvec_size }});
     }
 }
 {% endif %}
diff --git a/arch/salis-v1/arch_vars.py b/arch/salis-v1/arch_vars.py
index f469ad6..25687ce 100644
--- a/arch/salis-v1/arch_vars.py
+++ b/arch/salis-v1/arch_vars.py
@@ -1,119 +1,125 @@
-mvec_loop = False
+def gen_arch_vars(args):
+    return {
+        "mvec_loop": False,
 
-# Organisms consist of:
-# - instruction pointer
-# - seeker pointer
-# - main memory block
-# - child memory block
-# - 4 registers
-# - 8 value stack
-proc_fields = [
-    ("uint64_t", "ip"),
-    ("uint64_t", "sp"),
-    ("uint64_t", "mb0a"),
-    ("uint64_t", "mb0s"),
-    ("uint64_t", "mb1a"),
-    ("uint64_t", "mb1s"),
-    ("uint64_t", "r0x"),
-    ("uint64_t", "r1x"),
-    ("uint64_t", "r2x"),
-    ("uint64_t", "r3x"),
-    ("uint64_t", "s0"),
-    ("uint64_t", "s1"),
-    ("uint64_t", "s2"),
-    ("uint64_t", "s3"),
-    ("uint64_t", "s4"),
-    ("uint64_t", "s5"),
-    ("uint64_t", "s6"),
-    ("uint64_t", "s7"),
-]
+        # Organisms consist of:
+        # - instruction pointer
+        # - seeker pointer
+        # - main memory block
+        # - child memory block
+        # - 4 registers
+        # - 8 value stack
+        "proc_fields": [
+            ("uint64_t", "ip"),
+            ("uint64_t", "sp"),
+            ("uint64_t", "mb0a"),
+            ("uint64_t", "mb0s"),
+            ("uint64_t", "mb1a"),
+            ("uint64_t", "mb1s"),
+            ("uint64_t", "r0x"),
+            ("uint64_t", "r1x"),
+            ("uint64_t", "r2x"),
+            ("uint64_t", "r3x"),
+            ("uint64_t", "s0"),
+            ("uint64_t", "s1"),
+            ("uint64_t", "s2"),
+            ("uint64_t", "s3"),
+            ("uint64_t", "s4"),
+            ("uint64_t", "s5"),
+            ("uint64_t", "s6"),
+            ("uint64_t", "s7"),
+        ],
 
-# Salis-v1 instruction set
-inst_set = [
-    (["noop"], " "),
-    (["nop0"], "0"),
-    (["nop1"], "1"),
-    (["nop2"], "2"),
-    (["nop3"], "3"),
-    # -------------
-    (["jmpb"], "("),
-    (["jmpf"], ")"),
-    (["adrb"], "["),
-    (["adrf"], "]"),
-    (["ifnz"], "?"),
-    # -------------
-    (["allb"], "{"),
-    (["allf"], "}"),
-    (["bswp"], "%"),
-    (["bclr"], "|"),
-    (["splt"], "$"),
-    # -------------
-    (["addn"], "+"),
-    (["subn"], "-"),
-    (["muln"], "*"),
-    (["divn"], "/"),
-    (["incn"], "^"),
-    (["decn"], "v"),
-    (["notn"], "!"),
-    (["shfl"], "<"),
-    (["shfr"], ">"),
-    (["zero"], "z"),
-    (["unit"], "u"),
-    # -------------
-    (["pshn"], "#"),
-    (["popn"], "~"),
-    # -------------
-    (["load"], "."),
-    (["wrte"], ":"),
-    (["dupl"], "="),
-    (["swap"], "x"),
-    # -------------
-    (["keya"], "a"),
-    (["keyb"], "b"),
-    (["keyc"], "c"),
-    (["keyd"], "d"),
-    (["keye"], "e"),
-    (["keyf"], "f"),
-    (["keyg"], "g"),
-    (["keyh"], "h"),
-    (["keyi"], "i"),
-    (["keyj"], "j"),
-    (["keyk"], "k"),
-    (["keyl"], "l"),
-    (["keym"], "m"),
-    (["keyn"], "n"),
-    (["keyo"], "o"),
-    (["keyp"], "p"),
-    # -------------
-    (["loka"], "A"),
-    (["lokb"], "B"),
-    (["lokc"], "C"),
-    (["lokd"], "D"),
-    (["loke"], "E"),
-    (["lokf"], "F"),
-    (["lokg"], "G"),
-    (["lokh"], "H"),
-    (["loki"], "I"),
-    (["lokj"], "J"),
-    (["lokk"], "K"),
-    (["lokl"], "L"),
-    (["lokm"], "M"),
-    (["lokn"], "N"),
-    (["loko"], "O"),
-    (["lokp"], "P"),
-]
+        # Salis-v1 instruction set
+        "inst_set": [
+            (["noop"], " "),
+            (["nop0"], "0"),
+            (["nop1"], "1"),
+            (["nop2"], "2"),
+            (["nop3"], "3"),
+            # -------------
+            (["jmpb"], "("),
+            (["jmpf"], ")"),
+            (["adrb"], "["),
+            (["adrf"], "]"),
+            (["ifnz"], "?"),
+            # -------------
+            (["allb"], "{"),
+            (["allf"], "}"),
+            (["bswp"], "%"),
+            (["bclr"], "|"),
+            (["splt"], "$"),
+            # -------------
+            (["addn"], "+"),
+            (["subn"], "-"),
+            (["muln"], "*"),
+            (["divn"], "/"),
+            (["incn"], "^"),
+            (["decn"], "v"),
+            (["notn"], "!"),
+            (["shfl"], "<"),
+            (["shfr"], ">"),
+            (["zero"], "z"),
+            (["unit"], "u"),
+            # -------------
+            (["pshn"], "#"),
+            (["popn"], "~"),
+            # -------------
+            (["load"], "."),
+            (["wrte"], ":"),
+            (["dupl"], "="),
+            (["swap"], "x"),
+            # -------------
+            (["keya"], "a"),
+            (["keyb"], "b"),
+            (["keyc"], "c"),
+            (["keyd"], "d"),
+            (["keye"], "e"),
+            (["keyf"], "f"),
+            (["keyg"], "g"),
+            (["keyh"], "h"),
+            (["keyi"], "i"),
+            (["keyj"], "j"),
+            (["keyk"], "k"),
+            (["keyl"], "l"),
+            (["keym"], "m"),
+            (["keyn"], "n"),
+            (["keyo"], "o"),
+            (["keyp"], "p"),
+            # -------------
+            (["loka"], "A"),
+            (["lokb"], "B"),
+            (["lokc"], "C"),
+            (["lokd"], "D"),
+            (["loke"], "E"),
+            (["lokf"], "F"),
+            (["lokg"], "G"),
+            (["lokh"], "H"),
+            (["loki"], "I"),
+            (["lokj"], "J"),
+            (["lokk"], "K"),
+            (["lokl"], "L"),
+            (["lokm"], "M"),
+            (["lokn"], "N"),
+            (["loko"], "O"),
+            (["lokp"], "P"),
+        ],
 
-inst_count = len(inst_set)
+        "inst_count": 2 ** 6,
 
-# Extra fields used exclusively for data aggregation
-core_fields = [
-    ("uint64_t", f"iexe[{inst_count}]"),
-    ("uint64_t", f"iwrt[{inst_count}]"),
-    ("uint64_t", "emb0"),
-    ("uint64_t", "emb1"),
-    ("uint64_t", "eliv"),
-    ("uint64_t", "edea"),
-    ("uint64_t", "wmb0"),
-    ("uint64_t", "wmb1"),
-    ("uint64_t", "wdea"),
-]
+        # Extra fields used exclusively for data aggregation
+        "core_fields": [
+            ("uint64_t", f"iexe[{2 ** 6}]", False),
+            ("uint64_t", f"iwrt[{2 ** 6}]", False),
+            ("uint64_t", "emb0", True),
+            ("uint64_t", "emb1", True),
+            ("uint64_t", "eliv", True),
+            ("uint64_t", "edea", True),
+            ("uint64_t", "wmb0", True),
+            ("uint64_t", "wmb1", True),
+            ("uint64_t", "wdea", True),
+
+            # Event data aggregators
+            ("uint64_t", f"aeva[{2 ** args.mvec_pow}]", False),
+        ],
+    }
diff --git a/core.j2.c b/core.j2.c
index bd4b266..a7bd9cc 100644
--- a/core.j2.c
+++ b/core.j2.c
@@ -39,7 +39,7 @@ struct Core {
     uint8_t     *iviv;
 
     // Architecture specific custom fields
-    {% for type, val in arch_vars.core_fields %}
+    {% for type, val, _ in arch_vars.core_fields %}
     {{ type }} {{ val }};
     {% endfor %}
 
@@ -73,6 +73,8 @@ void (*g_warn)(const char *fmt, ...);
 void        arch_core_init(struct Core *core);
 {% endif %}
 
+void        arch_core_free(struct Core *core);
+
 {% if args.command in ["load", "new"] %}
 void        arch_core_save(FILE *f, const struct Core *core);
 {% endif %}
@@ -610,27 +612,58 @@ void salis_auto_save() {
 {% endif %}
 
 {% if data_push_path is defined %}
-void salis_exec_sql(const char *sql) {
-    assert(sql);
+void salis_exec_sql(int blob_cnt, const void **blobs, const int *blob_sizes, const char *sql_format, ...) {
+    assert(sql_format);
+
+    va_list args;
+    va_start(args, sql_format);
+    int   sql_len = vsnprintf(NULL, 0, sql_format, args) + 1;
+    char *sql_str = malloc(sql_len);
+    assert(sql_str);
+    va_end(args);
+
+    va_start(args, sql_format);
+    vsprintf(sql_str, sql_format, args);
+    va_end(args);
+
+    // Prepare statement
+    int           sql_res;
+    sqlite3_stmt *sql_stmt;
+
+    sql_res = sqlite3_prepare_v2(g_sim_data, sql_str, -1, &sql_stmt, NULL);
+    assert(sql_res == SQLITE_OK);
+    free(sql_str);
+
+    // Caller may pass multiple binary blobs to the query
+    for (int i = 0; i < blob_cnt; ++i) {
+        assert(blobs[i]);
+        sql_res = sqlite3_bind_blob(sql_stmt, i + 1, blobs[i], blob_sizes[i], SQLITE_STATIC);
+        assert(sql_res == SQLITE_OK);
+    }
 
-    int   sql_res;
-    char *sql_err;
+    // Execute the statement
+    // Only handle SQLITE_BUSY error, in which case we retry the query.
+    // In principle, setting 'journal_mode=wal;' should help prevent busy database errors.
+    while (true) {
+        sql_res = sqlite3_step(sql_stmt);
+
+        if (sql_res == SQLITE_DONE || sql_res == SQLITE_ROW) {
+            break;
+        }
 
-    while ((sql_res = sqlite3_exec(g_sim_data, sql, NULL, NULL, &sql_err)) == SQLITE_BUSY) {
         g_warn("SQLite database returned error '%d' with message:", sql_res);
-        g_warn(sql_err);
-        sqlite3_free(sql_err);
+        g_warn(sqlite3_errmsg(g_sim_data));
 
-        switch (sql_res) {
-        case SQLITE_BUSY:
-            // Only handle busy database errors
+        if (sql_res == SQLITE_BUSY) {
             g_info("Will retry query...");
             continue;
-        default:
-            // Application should fail on all other error conditions
-            assert(false);
         }
+
+        // Fail on unhandled error
+        assert(false);
     }
+
+    sqlite3_finalize(sql_stmt);
 }
 {% endif %}
 
@@ -659,7 +692,7 @@ void salis_init() {
     // Enable Write-Ahead Logging (WAL)
     // This seems to help prevent DB locks when displaying live data
     // See: https://sqlite.org/wal.html
-    salis_exec_sql("pragma journal_mode=wal;");
+    salis_exec_sql(0, NULL, NULL, "pragma journal_mode=wal;");
 
     arch_push_data_header();
     arch_push_data_line();
@@ -871,6 +904,8 @@ void salis_free() {
     {% endif %}
 
     for (int i = 0; i < {{ args.cores }}; ++i) {
+        arch_core_free(&g_cores[i]);
+
         assert(g_cores[i].pvec);
         assert(g_cores[i].iviv);
         assert(g_cores[i].ivav);
diff --git a/salis.py b/salis.py
index 7c07ff0..5f81f2c 100755
--- a/salis.py
+++ b/salis.py
@@ -202,13 +202,15 @@ if args.command in ["new"]:
 arch_path = f"arch/{args.arch}"
 info("Loading architecture specific variables from:", f"{arch_path}/arch_vars.py")
 sys.path.append(arch_path)
-import arch_vars
+from arch_vars import gen_arch_vars
+arch_vars = gen_arch_vars(args)
 
 if args.command in ["load", "new"]:
     ui_path = f"ui/{args.ui}"
     info("Loading UI specific variables from:", f"{ui_path}/ui_vars.py")
     sys.path.append(ui_path)
-    import ui_vars
+    from ui_vars import gen_ui_vars
+    ui_vars = gen_ui_vars(args)
 
 # ------------------------------------------------------------------------------
 # Fill in template variables
@@ -266,7 +268,7 @@ if args.command in ["load", "new"]:
     else:
         warn("Save file compression disabled")
 
-    includes.extend(ui_vars.includes)
+    includes.extend(ui_vars["includes"])
 
 # ------------------------------------------------------------------------------
 # Assemble ancestor organism into byte array
@@ -293,7 +295,7 @@ if args.command in ["bench", "new"] and args.anc is not None:
     for line in lines:
         found = False
 
-        for byte, tup in enumerate(arch_vars.inst_set):
+        for byte, tup in enumerate(arch_vars["inst_set"]):
             if line == tup[0]:
                 anc_bytes.append(byte)
                 found = True
@@ -341,14 +343,13 @@ build_cmd = ["gcc", salis_src, "-o", salis_bin, "-Wall", "-Wextra", "-Werror", "
 build_cmd.extend(["-O3", "-DNDEBUG"] if args.optimized else ["-ggdb"])
 
 if args.command in ["load", "new"]:
-    build_cmd.extend(ui_vars.flags)
+    build_cmd.extend(ui_vars["flags"])
 
-    # Enable POSIX extensions (open_memstream)
+    # Enable POSIX extensions (open_memstream) if compression is enabled
+    # This makes it easy to generate compressed data arrays for lzip using
+    # C's native FILE interface.
     build_cmd.extend(["-lz", "-D_POSIX_C_SOURCE=200809L"] if args.compress else [])
-
-    # Enable GNU extensions (asprintf)
-    # This allows managing large SQL strings more easily
-    build_cmd.extend(["-lsqlite3", "-D_GNU_SOURCE"] if args.data_push_pow != 0 else [])
+    build_cmd.extend(["-lsqlite3"] if args.data_push_pow != 0 else [])
 
 info("Using build command:", " ".join(build_cmd))
 subprocess.run(build_cmd, check=True)
diff --git a/ui/curses/ui.j2.c b/ui/curses/ui.j2.c
index ddeaf3b..2158eeb 100644
--- a/ui/curses/ui.j2.c
+++ b/ui/curses/ui.j2.c
@@ -416,6 +416,17 @@ void ui_print_core(int l) {
     ui_ulx_field(++l, "pcur", g_cores[g_core].pcur);
     ui_ulx_field(++l, "psli", g_cores[g_core].psli);
     ui_ulx_field(++l, "ivpt", g_cores[g_core].ivpt);
+
+    ++l;
+
+    {% if arch_vars.core_fields|length %}
+    ui_line(false, ++l, PAIR_HEADER, A_BOLD, "ARCH SPECIFIC");
+    {% for type, name, print in arch_vars.core_fields if print %}
+    {% if type == "uint64_t" %}
+    ui_ulx_field(++l, "{{ name }}", g_cores[g_core].{{ name }});
+    {% endif %}
+    {% endfor %}
+    {% endif %}
 }
 
 // ----------------------------------------------------------------------------
diff --git a/ui/curses/ui_vars.py b/ui/curses/ui_vars.py
index 4dfdc10..97d2c07 100644
--- a/ui/curses/ui_vars.py
+++ b/ui/curses/ui_vars.py
@@ -1,2 +1,5 @@
-flags    = ["-lncurses", "-DNCURSES_WIDECHAR=1"]
-includes = ["curses.h", "locale.h", "time.h"]
+def gen_ui_vars(_):
+    return {
+        "flags": ["-lncurses", "-DNCURSES_WIDECHAR=1"],
+        "includes": ["curses.h", "locale.h", "time.h"],
+    }
diff --git a/ui/daemon/ui_vars.py b/ui/daemon/ui_vars.py
index 9d0fc33..bb6be7c 100644
--- a/ui/daemon/ui_vars.py
+++ b/ui/daemon/ui_vars.py
@@ -1,2 +1,5 @@
-flags    = []
-includes = ["signal.h", "stdio.h", "unistd.h"]
+def gen_ui_vars(_):
+    return {
+        "flags": [],
+        "includes": ["signal.h", "stdio.h", "unistd.h"],
+    }