Evas are now saved to filesHEADmaster

6 files changed, 150 insertions, 76 deletions

diff --git a/core/client.cpp b/core/client.cpp
index 750f2be..771f626 100644
--- a/core/client.cpp
+++ b/core/client.cpp
@@ -36,6 +36,7 @@
 #include <map>
 #include <vector>
 
+#include "common.c"
 #include "logger.c"
 
 // ----------------------------------------------------------------------------
@@ -71,7 +72,6 @@
 #define DEFVAL_X_LOW 0l
 #define DEFVAL_X_HIGH INT64_MAX
 #define DEFVAL_HM_LEFT 0l
-#define DEFVAL_HM_PIXEL_COUNT 0x400l // must equal HM_PIXEL_COUNT in server.c
 
 enum Status {
     STATUS_STOPPED,
diff --git a/core/common.c b/core/common.c
new file mode 100644
index 0000000..786ab38
--- /dev/null
+++ b/core/common.c
@@ -0,0 +1,3 @@
+#define DEFVAL_HM_PIXEL_COUNT 0x400l
+#define EVENT_ARRAYS_SIZE (sizeof(uint64_t) * MVEC_SIZE)
+#define EVENT_ARRAYS_SIZE_COL_MARKER "_evasize_"
diff --git a/core/compress.c b/core/compress.c
index 8790011..e457de5 100644
--- a/core/compress.c
+++ b/core/compress.c
@@ -1,5 +1,3 @@
-#define EVA_SIZE (sizeof(uint64_t) * MVEC_SIZE)
-
 struct DeflateParams {
     z_stream strm;
     size_t size;
diff --git a/core/salis.c b/core/salis.c
index bf7ec44..11c6efb 100644
--- a/core/salis.c
+++ b/core/salis.c
@@ -24,8 +24,9 @@
 #include <threads.h>
 #include <zlib.h>
 
-#include "logger.c"
+#include "common.c"
 #include "compress.c"
+#include "logger.c"
 #include "sql.c"
 
 // ----------------------------------------------------------------------------
@@ -103,6 +104,7 @@ const struct Proc g_dead_proc;
 
 #if defined(COMMAND_NEW) || defined(COMMAND_LOAD)
 char g_asav_pbuf[AUTOSAVE_NAME_LEN];
+char g_eva_pbuf[EVA_SAVE_NAME_LEN];
 #endif
 
 thrd_t g_eva_thrds[CORES][EVENT_ARRAYS_COUNT];
@@ -145,7 +147,7 @@ const char *arch_mnemonic(uint8_t inst);
 #if defined(COMMAND_NEW)
 void arch_push_data_header(void);
 #endif
-void arch_push_data_line(void);
+void arch_push_data_line(FILE *eva_file);
 
 // ----------------------------------------------------------------------------
 // [section] memory vector functions
@@ -725,11 +727,11 @@ void salis_push_data_header(void) {
 
     log_info("Creating core table in SQLite database");
     sql_exec(
-        0, NULL, NULL, NULL, NULL,
+        NULL, NULL,
         "create table core ("
 #define EVENT_ARRAY(core, index, ev) \
-        #ev "_size_" #core " int not null, " \
-        #ev "_" #core " blob not null, "
+        #ev EVENT_ARRAYS_SIZE_COL_MARKER #core " int not null, " \
+        #ev "_" #core " blob, "
 #define FOR_CORE(i) \
         "cycl_" #i " int not null, " \
         "mall_" #i " int not null, " \
@@ -776,8 +778,7 @@ void salis_push_data_line(void) {
     // Compress event arrays
     // Compression (deflation) is CPU intensive so it's done in parallel
     memset(&g_eva_deflate_params, 0, sizeof(struct DeflateParams) * CORES * EVENT_ARRAYS_COUNT);
-    const void *blobs[CORES][EVENT_ARRAYS_COUNT];
-    int blob_sizes[CORES][EVENT_ARRAYS_COUNT];
+    uint64_t blob_sizes[CORES][EVENT_ARRAYS_COUNT];
 
     for (int i = 0; i < CORES; ++i) {
         for (int j = 0; j < EVENT_ARRAYS_COUNT; ++j) {
@@ -793,30 +794,45 @@ void salis_push_data_line(void) {
 
             // Compress event data
             struct DeflateParams *params = &g_eva_deflate_params[i][j];
-            params->size = EVA_SIZE,
+            params->size = EVENT_ARRAYS_SIZE,
             params->in = (Bytef *)in,
-            params->out = (Bytef *)malloc(EVA_SIZE),
+            params->out = (Bytef *)malloc(EVENT_ARRAYS_SIZE),
             thrd_create(&g_eva_thrds[i][j], (thrd_start_t)comp_deflate, params);
         }
     }
 
+    int rem = snprintf(
+        g_eva_pbuf,
+        EVA_SAVE_NAME_LEN,
+        "%s/evas-%#018lx",
+        SIM_EDIR,
+        g_steps
+    );
+
+    assert(rem >= 0);
+    assert(rem < EVA_SAVE_NAME_LEN);
+    (void)rem;
+
+    log_info("Saving event-array data to file: %s", g_eva_pbuf);
+    FILE *eva_file = fopen(g_eva_pbuf, "wb");
+
     for (int i = 0; i < CORES; ++i) {
         for (int j = 0; j < EVENT_ARRAYS_COUNT; ++j) {
             thrd_join(g_eva_thrds[i][j], NULL);
-
             struct DeflateParams *params = &g_eva_deflate_params[i][j];
-            blobs[i][j] = params->out;
             blob_sizes[i][j] = params->strm.total_out;
+            fwrite(params->out, sizeof(char), blob_sizes[i][j], eva_file);
+            comp_deflate_end(params);
+            free(params->out);
         }
     }
 
     log_info("Pushing row to core table in SQLite database");
     sql_exec(
-        CORES * EVENT_ARRAYS_COUNT, (const void **)blobs, (int *)blob_sizes, NULL, NULL,
+        NULL, NULL,
         "insert into core ("
 #define EVENT_ARRAY(core, index, ev) \
-        #ev "_size_" #core ", " \
-        #ev "_" #core ", "
+        #ev EVENT_ARRAYS_SIZE_COL_MARKER #core ", "
 #define FOR_CORE(i) \
         "cycl_" #i ", " \
         "mall_" #i ", " \
@@ -836,9 +852,19 @@ void salis_push_data_line(void) {
         "step"
         ") values ("
 #define EVENT_ARRAY(core, index, ev) \
-        "%ld, ?,"
+        "%ld, "
 #define FOR_CORE(i) \
-        "%ld, %ld, %ld, %ld, %ld, %f, %f, %ld, %ld, %ld, %ld, " \
+        "%ld, " \
+        "%ld, " \
+        "%ld, " \
+        "%ld, " \
+        "%ld, " \
+        "%f, " \
+        "%f, " \
+        "%ld, " \
+        "%ld, " \
+        "%ld, " \
+        "%ld, " \
         EVENT_ARRAYS(i)
         FOR_CORES
 #undef FOR_CORE
@@ -866,14 +892,6 @@ void salis_push_data_line(void) {
         g_steps
     );
 
-    for (int i = 0; i < CORES; ++i) {
-        for (int j = 0; j < EVENT_ARRAYS_COUNT; ++j) {
-            struct DeflateParams *params = &g_eva_deflate_params[i][j];
-            comp_deflate_end(params);
-            free(params->out);
-        }
-    }
-
     // Reset data aggregation fields
     for (int i = 0; i < CORES; ++i) {
         struct Core *core = &g_cores[i];
@@ -884,13 +902,14 @@ void salis_push_data_line(void) {
         core->edea = 0;
 
 #define EVENT_ARRAY(core, index, ev) \
-        memset(core->ev##a, 0, EVA_SIZE);
+        memset(core->ev##a, 0, EVENT_ARRAYS_SIZE);
         EVENT_ARRAYS(core)
 #undef EVENT_ARRAY
     }
 
     // Push arch-specific data
-    arch_push_data_line();
+    arch_push_data_line(eva_file);
+    fclose(eva_file);
 }
 
 #if defined(COMMAND_NEW)
diff --git a/core/server.c b/core/server.c
index 9a06825..6848cf6 100644
--- a/core/server.c
+++ b/core/server.c
@@ -19,6 +19,7 @@
 #include <threads.h>
 #include <zlib.h>
 
+#include "common.c"
 #include "compress.c"
 #include "logger.c"
 #include "sql.c"
@@ -27,8 +28,6 @@
 // [section] macros
 // ----------------------------------------------------------------------------
 #define BACKLOG 10
-#define EVA_SIZE (sizeof(uint64_t) * MVEC_SIZE)
-#define MAX_HM_PIXEL_COUNT 0x400 // must equal DEFVAL_HM_PIXEL_COUNT in client.cpp
 
 // ----------------------------------------------------------------------------
 // [section] structs
@@ -48,17 +47,18 @@ struct CallbackContext {
 
 struct RenderContext {
     const struct CallbackContext *callback_context;
-    const void *blob;
+    void *blob;
     size_t blob_size;
-    uint64_t eva[EVA_SIZE];
-    int64_t out[MAX_HM_PIXEL_COUNT];
+    uint64_t eva[EVENT_ARRAYS_SIZE];
+    int64_t out[DEFVAL_HM_PIXEL_COUNT];
 };
 
 // ----------------------------------------------------------------------------
 // [section] globals
 // ----------------------------------------------------------------------------
 struct json_object *g_response_header;
-size_t g_blob_count;
+size_t g_eva_count;
+char g_eva_pbuf[EVA_SAVE_NAME_LEN];
 
 // ----------------------------------------------------------------------------
 // [section] event array render function
@@ -74,6 +74,8 @@ int eva_render(void *data) {
     const void *blob = render_context->blob;
     size_t blob_size = render_context->blob_size;
 
+    assert(blob);
+
 #if defined(MVEC_LOOP)
     hm_left %= MVEC_SIZE;
 #endif
@@ -89,7 +91,7 @@ int eva_render(void *data) {
     // Inflate blob
     struct InflateParams params = {
         .avail_in = blob_size,
-        .size = EVA_SIZE,
+        .size = EVENT_ARRAYS_SIZE,
         .in = (Bytef *)blob,
         .out = (Bytef *)render_context->eva,
     };
@@ -124,15 +126,14 @@ void sql_callback_add_column_name(sqlite3_stmt *sql_stmt, void *data) {
     assert(!strcmp(sqlite3_column_name(sql_stmt, 1), "type"));
 
     const char *col_name = (const char *)sqlite3_column_text(sql_stmt, 0);
-    const char *col_type = (const char *)sqlite3_column_text(sql_stmt, 1);
     struct json_object *response_header = (struct json_object *)data;
 
     if (!json_object_object_get_ex(response_header, col_name, NULL)) {
         json_object_object_add(response_header, col_name, json_object_new_array());
     }
 
-    if (!strcmp(col_type, "BLOB")) {
-        g_blob_count++;
+    if (strstr(col_name, EVENT_ARRAYS_SIZE_COL_MARKER)) {
+        g_eva_count++;
     }
 }
 
@@ -140,41 +141,103 @@ void sql_callback_add_data(sqlite3_stmt *sql_stmt, void *data) {
     assert(sql_stmt);
     assert(data);
 
+    FILE *eva_file = NULL;
     struct CallbackContext *callback_context = (struct CallbackContext *)data;
-    struct json_object *col_data = NULL;
-    struct RenderContext *render_contexts = calloc(g_blob_count, sizeof(struct RenderContext));
-    thrd_t *threads = calloc(g_blob_count, sizeof(thrd_t));
-
-    for (int i = 0, tid = 0; i < sqlite3_column_count(sql_stmt); i++) {
-        if (json_object_object_get_ex(callback_context->response, sqlite3_column_name(sql_stmt, i), &col_data)) {
-            if (sqlite3_column_type(sql_stmt, i) == SQLITE_BLOB) {
-                render_contexts[tid].callback_context = callback_context;
-                render_contexts[tid].blob = sqlite3_value_blob(sqlite3_column_value(sql_stmt, i));
-                render_contexts[tid].blob_size = sqlite3_value_bytes(sqlite3_column_value(sql_stmt, i));
-                thrd_create(&threads[tid], (thrd_start_t)eva_render, &render_contexts[tid]);
-                tid++;
-            } else {
-                json_object_array_add(col_data, json_object_new_int64(sqlite3_column_int64(sql_stmt, i)));
-            }
+    struct RenderContext *render_contexts = calloc(g_eva_count, sizeof(struct RenderContext));
+    thrd_t *threads = calloc(g_eva_count, sizeof(thrd_t));
+    size_t tid = 0;
+
+    for (int i = 0; i < sqlite3_column_count(sql_stmt); i++) {
+        assert(i < sqlite3_column_count(sql_stmt));
+        assert(sqlite3_column_type(sql_stmt, i) == SQLITE_INTEGER || sqlite3_column_type(sql_stmt, i) == SQLITE_FLOAT);
+
+        const char *col_name = sqlite3_column_name(sql_stmt, i);
+        struct json_object *col_data = json_object_object_get(callback_context->response, col_name);
+        assert(col_name);
+
+        if (!col_data) continue;
+
+        int64_t col_value = sqlite3_column_int64(sql_stmt, i);
+        json_object_array_add(col_data, json_object_new_int64(col_value));
+
+        if (i == 1) {
+            assert(!strcmp(col_name, "step"));
+
+            int rem = snprintf(
+                g_eva_pbuf,
+                EVA_SAVE_NAME_LEN,
+                "%s/evas-%#018lx",
+                SIM_EDIR,
+                col_value
+            );
+
+            assert(rem >= 0);
+            assert(rem < EVA_SAVE_NAME_LEN);
+            (void)rem;
+
+            eva_file = fopen(g_eva_pbuf, "rb");
+            assert(eva_file);
+        }
+
+        if (sqlite3_column_type(sql_stmt, i + 1) == SQLITE_NULL) {
+            assert(eva_file);
+            assert(strstr(col_name, EVENT_ARRAYS_SIZE_COL_MARKER));
+
+            render_contexts[tid].callback_context = callback_context;
+            render_contexts[tid].blob = malloc(col_value);
+            render_contexts[tid].blob_size = col_value;
+            assert(render_contexts[tid].blob);
+
+            size_t red = fread(render_contexts[tid].blob, 1, col_value, eva_file);
+            assert(red == (size_t)col_value);
+            (void)red;
+
+            thrd_create(&threads[tid], (thrd_start_t)eva_render, &render_contexts[tid]);
+
+            tid++;
+            i++;
         }
     }
 
-    for (int i = 0, tid = 0; i < sqlite3_column_count(sql_stmt); i++) {
-        if (json_object_object_get_ex(callback_context->response, sqlite3_column_name(sql_stmt, i), &col_data)) {
-            if (sqlite3_column_type(sql_stmt, i) == SQLITE_BLOB) {
-                thrd_join(threads[tid], NULL);
+    assert(tid == g_eva_count);
+    tid = 0;
+
+    for (int i = 0; i < sqlite3_column_count(sql_stmt); i++) {
+        const char *col_name = sqlite3_column_name(sql_stmt, i);
+        struct json_object *col_data = json_object_object_get(callback_context->response, col_name);
+        assert(col_name);
+
+        if (!col_data) continue;
+
+        if (sqlite3_column_type(sql_stmt, i + 1) == SQLITE_NULL) {
+            assert(strstr(col_name, EVENT_ARRAYS_SIZE_COL_MARKER));
+            assert(render_contexts[tid].blob);
+
+            const char *eva_col_name = sqlite3_column_name(sql_stmt, i + 1);
+            struct json_object *eva_col_data = json_object_object_get(callback_context->response, eva_col_name);
+            assert(eva_col_name);
+            assert(eva_col_data);
 
-                for (int64_t j = 0; j < callback_context->hm_pixel_count; j++) {
-                    json_object_array_add(col_data, json_object_new_int64(render_contexts[tid].out[j]));
-                }
+            thrd_join(threads[tid], NULL);
 
-                tid++;
+            for (int64_t j = 0; j < callback_context->hm_pixel_count; j++) {
+                json_object_array_add(eva_col_data, json_object_new_int64(render_contexts[tid].out[j]));
             }
+
+            free(render_contexts[tid].blob);
+
+            tid++;
+            i++;
         }
     }
 
+    assert(tid == g_eva_count);
+
     callback_context->response_rows++;
     log_info("Processed row #%ld", callback_context->response_rows);
+
+    assert(eva_file);
+    fclose(eva_file);
     free(render_contexts);
     free(threads);
 }
@@ -184,6 +247,7 @@ void sql_callback_add_data(sqlite3_stmt *sql_stmt, void *data) {
 // ----------------------------------------------------------------------------
 void sig_handler(int signo) {
     (void)signo;
+
     log_warn("Signal received, will stop SALIS data server");
     json_object_put(g_response_header);
     sql_close();
@@ -245,7 +309,6 @@ void respond_data(int socket_fd, struct json_object *request) {
     const char *x_axis_pref = (!strcmp(x_axis, "rowid") || !strcmp(x_axis, "step")) ? "core." : "";
 
     sql_exec(
-        0, NULL, NULL,
         sql_callback_add_data,
         &callback_context,
         "select * from ("
@@ -322,14 +385,11 @@ int main(void) {
     g_response_header = json_object_new_object();
     json_object_object_add(g_response_header, "rowid", json_object_new_array());
     sql_exec(
-        0, NULL, NULL,
         sql_callback_add_column_name,
         g_response_header,
-        "select name, type from pragma_table_info('core') union "
-        "select name, type from pragma_table_info('arch');"
+        "select name, type from pragma_table_info('core') union select name, type from pragma_table_info('arch');"
     );
-
-    log_info("Found %lu blob columns in database", g_blob_count);
+    log_info("Found %lu eva-size columns in database", g_eva_count);
 
     log_info("Binding to port: %d", PORT);
     int opt = 1;
diff --git a/core/sql.c b/core/sql.c
index cc59726..760adc4 100644
--- a/core/sql.c
+++ b/core/sql.c
@@ -2,7 +2,7 @@
 
 sqlite3 *g_sim_db;
 
-void sql_exec(int blob_cnt, const void **blobs, const int *blob_sizes, void (*callback)(sqlite3_stmt *sql_stmt, void *data), void *data, const char *sql_format, ...) {
+void sql_exec(void (*callback)(sqlite3_stmt *sql_stmt, void *data), void *data, const char *sql_format, ...) {
     assert(sql_format);
 
     va_list args;
@@ -23,12 +23,6 @@ void sql_exec(int blob_cnt, const void **blobs, const int *blob_sizes, void (*ca
     assert(sql_res == SQLITE_OK);
     free(sql_str);
 
-    for (int i = 0; i < blob_cnt; ++i) {
-        assert(blobs[i]);
-        sql_res = sqlite3_bind_blob(sql_stmt, i + 1, blobs[i], blob_sizes[i], SQLITE_STATIC);
-        assert(sql_res == SQLITE_OK);
-    }
-
     while (true) {
         sql_res = sqlite3_step(sql_stmt);
 
@@ -70,7 +64,7 @@ void sql_open(void) {
     // Enable Write-Ahead Logging (WAL)
     // This seems to help prevent DB locks when displaying live data.
     // See: https://sqlite.org/wal.html
-    sql_exec(0, NULL, NULL, NULL, NULL, "pragma journal_mode=wal;");
+    sql_exec(NULL, NULL, "pragma journal_mode=wal;");
 }
 
 void sql_close(void) {