Improve SQL handling & aggregate memory events (WIP)data_improvements

1 files changed, 174 insertions, 78 deletions

diff --git a/arch/salis-v1/arch.j2.c b/arch/salis-v1/arch.j2.c
index 9c3ea45..06a701d 100644
--- a/arch/salis-v1/arch.j2.c
+++ b/arch/salis-v1/arch.j2.c
@@ -33,6 +33,11 @@ void arch_core_init(struct Core *core) {
 }
 {% endif %}
 
+void arch_core_free(struct Core *core) {
+    assert(core);
+    (void)core;
+}
+
 {% if args.command in ["load", "new"] %}
 void arch_core_save(FILE *f, const struct Core *core) {
     assert(f);
@@ -47,6 +52,7 @@ void arch_core_save(FILE *f, const struct Core *core) {
     fwrite(&core->wmb0, sizeof(uint64_t), 1, f);
     fwrite(&core->wmb1, sizeof(uint64_t), 1, f);
     fwrite(&core->wdea, sizeof(uint64_t), 1, f);
+    fwrite( core->aeva, sizeof(uint64_t), {{ mvec_size }}, f);
 }
 {% endif %}
 
@@ -64,6 +70,7 @@ void arch_core_load(FILE *f, struct Core *core) {
     fread(&core->wmb0, sizeof(uint64_t), 1, f);
     fread(&core->wmb1, sizeof(uint64_t), 1, f);
     fread(&core->wdea, sizeof(uint64_t), 1, f);
+    fread( core->aeva, sizeof(uint64_t), {{ mvec_size }}, f);
 }
 {% endif %}
 
@@ -119,6 +126,9 @@ void _free_memory_block(struct Core *core, uint64_t addr, uint64_t size) {
 
     for (uint64_t i = 0; i < size; ++i) {
         mvec_free(core, addr + i);
+
+        // Record deallocation event
+        ++core->aeva[addr];
     }
 }
 
@@ -388,6 +398,9 @@ void _alloc(struct Core *core, uint64_t pix, bool fwrd) {
     // Enlarge child block 1 byte
     mvec_alloc(core, proc->sp);
 
+    // Record allocation event
+    ++core->aeva[proc->sp];
+
     if (!proc->mb1s || !fwrd) {
         proc->mb1a = proc->sp;
     }
@@ -828,12 +841,17 @@ const char *arch_mnemonic(uint8_t inst) {
     return NULL;
 }
 
+// ----------------------------------------------------------------------------
+// Data aggregation functions
+// ----------------------------------------------------------------------------
 {% if data_push_path is defined %}
 void arch_push_data_header() {
     assert(g_sim_data);
 
-    // General trends from all cores will be queried together
-    const char *trend_sql = (
+    // Creates general trend table for all cores
+    g_info("Creating 'trend' table in SQLite database");
+    salis_exec_sql(
+        0, NULL, NULL,
         "create table trend ("
         "step int not null, "
         {% for i in range(args.cores) %}
@@ -855,37 +873,54 @@ void arch_push_data_header() {
         ");"
     );
 
-    g_info("Generating 'trend' table in SQLite database");
-    salis_exec_sql(trend_sql);
+    // Creates core-specific instruction data tables
+    char *iprefs[]   = { "pop", "exe", "wrt" };
+    int   iprefs_cnt = sizeof(iprefs) / sizeof(iprefs[0]);
 
-    // Core-specific instruction data will be queried separately
-    // A table is created for each core
-    {% for t in ["pop", "exe", "wrt"] %}
-    {% for i in range(args.cores) %}
-    const char *{{ t }}_sql_{{ i }} = (
-        "create table {{ t }}_{{ i }} ("
-        "step int not null, "
-        // Cycle data allows normalizing against each core's cycle speed
-        {% for j in range(args.cores) %}
-        "cycl_{{ j }} int not null, "
-        {% endfor %}
-        {% for j in arch_vars.inst_set %}
-        "inst_{{ j[0]|join(' ') }} int not null{% if not loop.last %},{% endif %} "
-        {% endfor %}
-        ");"
-    );
+    for (int i = 0; i < {{ args.cores }}; ++i) {
+        for (int j = 0; j < iprefs_cnt; ++j) {
+            g_info("Creating '%s_%d' table in SQLite database", iprefs[j], i);
+            salis_exec_sql(
+                0, NULL, NULL,
+                "create table %s_%d ("
+                "step int not null, "
+                // Cycle data for all cores allows
+                // normalizing against each core's cycle speed
+                {% for i in range(args.cores) %}
+                "cycl_{{ i }} int not null, "
+                {% endfor %}
+                {% for i in arch_vars.inst_set %}
+                "{{ i[0][0] }} int not null{% if not loop.last %},{% endif %} "
+                {% endfor %}
+                ");",
+                iprefs[j], i
+            );
+        }
+    }
 
-    g_info("Generating '{{ t }}_{{ i }}' table in SQLite database");
-    salis_exec_sql({{ t }}_sql_{{ i }});
-    {% endfor %}
-    {% endfor %}
+    // Creates core-specific memory event tables
+    char *eprefs[]   = { "aev" };
+    int   eprefs_cnt = sizeof(eprefs) / sizeof(eprefs[0]);
+
+    for (int i = 0; i < {{ args.cores }}; ++i) {
+        for (int j = 0; j < eprefs_cnt; ++j) {
+            g_info("Creating '%s_%d' table in SQLite database", eprefs[j], i);
+            salis_exec_sql(
+                0, NULL, NULL,
+                "create table %s_%d ("
+                "step int  not null, "
+                "evts blob not null"
+                ");",
+                eprefs[j], i
+            );
+        }
+    }
 }
 
 void arch_push_data_line() {
     assert(g_sim_data);
 
-    // Measure current instruction population and
-    // average memory block sizes
+    // Measure current instruction population and average memory block sizes
     uint64_t ipop[{{ args.cores }}][{{ arch_vars.inst_count }}] = { 0 };
 
     double amb0[{{ args.cores }}] = { 0 };
@@ -920,11 +955,10 @@ void arch_push_data_line() {
         {% endif %}
     }
 
-    // Insert new row
-    char *trend_sql = NULL;
-
-    asprintf(
-        &trend_sql,
+    // Insert row into trend table
+    g_info("Pushing row to 'trend' table in SQLite database");
+    salis_exec_sql(
+        0, NULL, NULL,
         "insert into trend ("
         "step, "
         {% for i in range(args.cores) %}
@@ -950,7 +984,7 @@ void arch_push_data_line() {
         {% endfor %}
         ");",
         g_steps,
-        {% for i in range(args.cores) %}
+        {% for i in range(args.cores) +%}
         g_cores[{{ i }}].cycl,
         g_cores[{{ i }}].mall,
         g_cores[{{ i }}].pnum,
@@ -965,59 +999,118 @@ void arch_push_data_line() {
         g_cores[{{ i }}].wmb0,
         g_cores[{{ i }}].wmb1,
         g_cores[{{ i }}].wdea{% if not loop.last %},{% endif %}
-        {% endfor %}
+        {% endfor +%}
     );
 
-    g_info("Pushing row to 'trend' table in SQLite database");
-    salis_exec_sql(trend_sql);
-    free(trend_sql);
+    // Insert row into instruction data tables
+    char *iprefs[]   = { "pop", "exe", "wrt" };
+    int   iprefs_cnt = sizeof(iprefs) / sizeof(iprefs[0]);
 
-    {% for t in ["pop", "exe", "wrt"] %}
-    {% for i in range(args.cores) %}
+    for (int i = 0; i < {{ args.cores }}; ++i) {
+        for (int j = 0; j < iprefs_cnt; ++j) {
+            uint64_t *ia;
+
+            if (!strcmp("pop", iprefs[j])) {
+                // Population is generated above, prior to push
+                ia = ipop[i];
+            } else if (!strcmp("exe", iprefs[j])) {
+                ia = g_cores[i].iexe;
+            } else if (!strcmp("wrt", iprefs[j])) {
+                ia = g_cores[i].iwrt;
+            }
 
-    {% if t == "pop" %}
-        // Reference instruction population vector that was
-        // manually generated above
-        {% set table = "ipop[%d]"|format(i) %}
-    {% else %}
-        {% set table = "g_cores[%d].i%s"|format(i, t) %}
-    {% endif %}
+            g_info("Pushing row to '%s_%d' table in SQLite database", iprefs[j], i);
+            salis_exec_sql(
+                0, NULL, NULL,
+                "insert into %s_%d ("
+                "step, "
+                {% for i in range(args.cores) %}
+                "cycl_{{ i }}, "
+                {% endfor %}
+                {% for i in arch_vars.inst_set %}
+                "{{ i[0][0] }}{% if not loop.last %},{% endif %} "
+                {% endfor %}
+                ") values ("
+                "%ld, "
+                {% for _ in range(args.cores) %}
+                "%ld, "
+                {% endfor %}
+                {% for _ in range(arch_vars.inst_count) %}
+                "%ld{% if not loop.last %},{% endif %} "
+                {% endfor %}
+                ");",
+                iprefs[j],
+                i,
+                g_steps,
+                {% for j in range(args.cores) %}
+                g_cores[{{ j }}].cycl,
+                {% endfor %}
+                {% for j in range(arch_vars.inst_count) %}
+                ia[{{ j }}]{% if not loop.last %},{% endif +%}
+                {% endfor %}
+            );
+        }
+    }
 
-    char *{{ t }}_sql_{{ i }} = NULL;
+    // Insert row into memory event tables
+    char *eprefs[]   = { "aev" };
+    int   eprefs_cnt = sizeof(eprefs) / sizeof(eprefs[0]);
 
-    asprintf(
-        &{{ t }}_sql_{{ i }},
-        "insert into {{ t }}_{{ i }} ("
-        "step, "
-        {% for j in range(args.cores) %}
-        "cycl_{{ j }}, "
-        {% endfor %}
-        {% for j in arch_vars.inst_set %}
-        "inst_{{ j[0]|join(' ') }}{% if not loop.last %},{% endif %} "
-        {% endfor %}
-        ") values ("
-        "%ld, "
-        {% for _ in range(args.cores) %}
-        "%ld, "
-        {% endfor %}
-        {% for _ in range(arch_vars.inst_count) %}
-        "%ld{% if not loop.last %},{% endif %} "
-        {% endfor %}
-        ");",
-        g_steps,
-        {% for j in range(args.cores) %}
-        g_cores[{{ j }}].cycl,
-        {% endfor %}
-        {% for j in range(arch_vars.inst_count) %}
-        {{ table }}[{{ j }}]{% if not loop.last %},{% endif %}
-        {% endfor %}
-    );
+    // Event run-length data array is defined as static
+    // This prevents heap errors
+    static uint64_t erl_data[{{ mvec_size }} * 2];
 
-    g_info("Pushing row to '{{ t }}_{{ i }}' table in SQLite database");
-    salis_exec_sql({{ t }}_sql_{{ i }});
-    free({{ t }}_sql_{{ i }});
-    {% endfor %}
-    {% endfor %}
+    for (int i = 0; i < {{ args.cores }}; ++i) {
+        for (int j = 0; j < eprefs_cnt; ++j) {
+            uint64_t *eva;
+
+            if (!strcmp("aev", eprefs[j])) {
+                eva = g_cores[i].aeva;
+            }
+
+            // Assume event data to be sparse in most cases
+            // Run-length encoding should help keep database size manageable,
+            // while making it easy to decode events array, wherever the
+            // database is consumed.
+            uint64_t addr = 0;
+            uint64_t eix  = 0;
+
+            while (addr < {{ mvec_size }}) {
+                assert(eix < {{ mvec_size }} * 2);
+
+                erl_data[eix]     = eva[addr];
+                erl_data[eix + 1] = 0;
+
+                while (addr < {{ mvec_size }} && eva[addr] == erl_data[eix]) {
+                    ++erl_data[eix + 1];
+                    ++addr;
+                }
+
+                eix += 2;
+            }
+
+            {% if not args.optimized %}
+            uint64_t el = 0;
+
+            for (uint64_t k = 0; k < eix; k += 2) {
+                el += erl_data[k + 1];
+            }
+
+            assert(el == {{ mvec_size }});
+            {% endif %}
+
+            // Insert blob into database
+            const void *blob_ptr  = erl_data;
+            int         blob_size = eix * sizeof(uint64_t);
+
+            g_info("Pushing row to '%s_%d' table in SQLite database", eprefs[j], i);
+            salis_exec_sql(
+                1, &blob_ptr, &blob_size,
+                "insert into %s_%d (step, evts) values (%ld, ?);",
+                eprefs[j], i, g_steps
+            );
+        }
+    }
 
     // Reset all data aggregation core fields to zero
     for (int i = 0; i < {{ args.cores }}; ++i) {
@@ -1033,6 +1126,9 @@ void arch_push_data_line() {
         core->wmb0 = 0;
         core->wmb1 = 0;
         core->wdea = 0;
+
+        // Event vectors
+        memset(core->aeva, 0, sizeof(uint64_t) * {{ mvec_size }});
     }
 }
 {% endif %}