Evas are now saved to filesHEADmaster

1 files changed, 99 insertions, 39 deletions

diff --git a/core/server.c b/core/server.c
index 9a06825..6848cf6 100644
--- a/core/server.c
+++ b/core/server.c
@@ -19,6 +19,7 @@
 #include <threads.h>
 #include <zlib.h>
 
+#include "common.c"
 #include "compress.c"
 #include "logger.c"
 #include "sql.c"
@@ -27,8 +28,6 @@
 // [section] macros
 // ----------------------------------------------------------------------------
 #define BACKLOG 10
-#define EVA_SIZE (sizeof(uint64_t) * MVEC_SIZE)
-#define MAX_HM_PIXEL_COUNT 0x400 // must equal DEFVAL_HM_PIXEL_COUNT in client.cpp
 
 // ----------------------------------------------------------------------------
 // [section] structs
@@ -48,17 +47,18 @@ struct CallbackContext {
 
 struct RenderContext {
     const struct CallbackContext *callback_context;
-    const void *blob;
+    void *blob;
     size_t blob_size;
-    uint64_t eva[EVA_SIZE];
-    int64_t out[MAX_HM_PIXEL_COUNT];
+    uint64_t eva[EVENT_ARRAYS_SIZE];
+    int64_t out[DEFVAL_HM_PIXEL_COUNT];
 };
 
 // ----------------------------------------------------------------------------
 // [section] globals
 // ----------------------------------------------------------------------------
 struct json_object *g_response_header;
-size_t g_blob_count;
+size_t g_eva_count;
+char g_eva_pbuf[EVA_SAVE_NAME_LEN];
 
 // ----------------------------------------------------------------------------
 // [section] event array render function
@@ -74,6 +74,8 @@ int eva_render(void *data) {
     const void *blob = render_context->blob;
     size_t blob_size = render_context->blob_size;
 
+    assert(blob);
+
 #if defined(MVEC_LOOP)
     hm_left %= MVEC_SIZE;
 #endif
@@ -89,7 +91,7 @@ int eva_render(void *data) {
     // Inflate blob
     struct InflateParams params = {
         .avail_in = blob_size,
-        .size = EVA_SIZE,
+        .size = EVENT_ARRAYS_SIZE,
         .in = (Bytef *)blob,
         .out = (Bytef *)render_context->eva,
     };
@@ -124,15 +126,14 @@ void sql_callback_add_column_name(sqlite3_stmt *sql_stmt, void *data) {
     assert(!strcmp(sqlite3_column_name(sql_stmt, 1), "type"));
 
     const char *col_name = (const char *)sqlite3_column_text(sql_stmt, 0);
-    const char *col_type = (const char *)sqlite3_column_text(sql_stmt, 1);
     struct json_object *response_header = (struct json_object *)data;
 
     if (!json_object_object_get_ex(response_header, col_name, NULL)) {
         json_object_object_add(response_header, col_name, json_object_new_array());
     }
 
-    if (!strcmp(col_type, "BLOB")) {
-        g_blob_count++;
+    if (strstr(col_name, EVENT_ARRAYS_SIZE_COL_MARKER)) {
+        g_eva_count++;
     }
 }
 
@@ -140,41 +141,103 @@ void sql_callback_add_data(sqlite3_stmt *sql_stmt, void *data) {
     assert(sql_stmt);
     assert(data);
 
+    FILE *eva_file = NULL;
     struct CallbackContext *callback_context = (struct CallbackContext *)data;
-    struct json_object *col_data = NULL;
-    struct RenderContext *render_contexts = calloc(g_blob_count, sizeof(struct RenderContext));
-    thrd_t *threads = calloc(g_blob_count, sizeof(thrd_t));
-
-    for (int i = 0, tid = 0; i < sqlite3_column_count(sql_stmt); i++) {
-        if (json_object_object_get_ex(callback_context->response, sqlite3_column_name(sql_stmt, i), &col_data)) {
-            if (sqlite3_column_type(sql_stmt, i) == SQLITE_BLOB) {
-                render_contexts[tid].callback_context = callback_context;
-                render_contexts[tid].blob = sqlite3_value_blob(sqlite3_column_value(sql_stmt, i));
-                render_contexts[tid].blob_size = sqlite3_value_bytes(sqlite3_column_value(sql_stmt, i));
-                thrd_create(&threads[tid], (thrd_start_t)eva_render, &render_contexts[tid]);
-                tid++;
-            } else {
-                json_object_array_add(col_data, json_object_new_int64(sqlite3_column_int64(sql_stmt, i)));
-            }
+    struct RenderContext *render_contexts = calloc(g_eva_count, sizeof(struct RenderContext));
+    thrd_t *threads = calloc(g_eva_count, sizeof(thrd_t));
+    size_t tid = 0;
+
+    for (int i = 0; i < sqlite3_column_count(sql_stmt); i++) {
+        assert(i < sqlite3_column_count(sql_stmt));
+        assert(sqlite3_column_type(sql_stmt, i) == SQLITE_INTEGER || sqlite3_column_type(sql_stmt, i) == SQLITE_FLOAT);
+
+        const char *col_name = sqlite3_column_name(sql_stmt, i);
+        struct json_object *col_data = json_object_object_get(callback_context->response, col_name);
+        assert(col_name);
+
+        if (!col_data) continue;
+
+        int64_t col_value = sqlite3_column_int64(sql_stmt, i);
+        json_object_array_add(col_data, json_object_new_int64(col_value));
+
+        if (i == 1) {
+            assert(!strcmp(col_name, "step"));
+
+            int rem = snprintf(
+                g_eva_pbuf,
+                EVA_SAVE_NAME_LEN,
+                "%s/evas-%#018lx",
+                SIM_EDIR,
+                col_value
+            );
+
+            assert(rem >= 0);
+            assert(rem < EVA_SAVE_NAME_LEN);
+            (void)rem;
+
+            eva_file = fopen(g_eva_pbuf, "rb");
+            assert(eva_file);
+        }
+
+        if (sqlite3_column_type(sql_stmt, i + 1) == SQLITE_NULL) {
+            assert(eva_file);
+            assert(strstr(col_name, EVENT_ARRAYS_SIZE_COL_MARKER));
+
+            render_contexts[tid].callback_context = callback_context;
+            render_contexts[tid].blob = malloc(col_value);
+            render_contexts[tid].blob_size = col_value;
+            assert(render_contexts[tid].blob);
+
+            size_t red = fread(render_contexts[tid].blob, 1, col_value, eva_file);
+            assert(red == (size_t)col_value);
+            (void)red;
+
+            thrd_create(&threads[tid], (thrd_start_t)eva_render, &render_contexts[tid]);
+
+            tid++;
+            i++;
         }
     }
 
-    for (int i = 0, tid = 0; i < sqlite3_column_count(sql_stmt); i++) {
-        if (json_object_object_get_ex(callback_context->response, sqlite3_column_name(sql_stmt, i), &col_data)) {
-            if (sqlite3_column_type(sql_stmt, i) == SQLITE_BLOB) {
-                thrd_join(threads[tid], NULL);
+    assert(tid == g_eva_count);
+    tid = 0;
+
+    for (int i = 0; i < sqlite3_column_count(sql_stmt); i++) {
+        const char *col_name = sqlite3_column_name(sql_stmt, i);
+        struct json_object *col_data = json_object_object_get(callback_context->response, col_name);
+        assert(col_name);
+
+        if (!col_data) continue;
+
+        if (sqlite3_column_type(sql_stmt, i + 1) == SQLITE_NULL) {
+            assert(strstr(col_name, EVENT_ARRAYS_SIZE_COL_MARKER));
+            assert(render_contexts[tid].blob);
+
+            const char *eva_col_name = sqlite3_column_name(sql_stmt, i + 1);
+            struct json_object *eva_col_data = json_object_object_get(callback_context->response, eva_col_name);
+            assert(eva_col_name);
+            assert(eva_col_data);
 
-                for (int64_t j = 0; j < callback_context->hm_pixel_count; j++) {
-                    json_object_array_add(col_data, json_object_new_int64(render_contexts[tid].out[j]));
-                }
+            thrd_join(threads[tid], NULL);
 
-                tid++;
+            for (int64_t j = 0; j < callback_context->hm_pixel_count; j++) {
+                json_object_array_add(eva_col_data, json_object_new_int64(render_contexts[tid].out[j]));
             }
+
+            free(render_contexts[tid].blob);
+
+            tid++;
+            i++;
         }
     }
 
+    assert(tid == g_eva_count);
+
     callback_context->response_rows++;
     log_info("Processed row #%ld", callback_context->response_rows);
+
+    assert(eva_file);
+    fclose(eva_file);
     free(render_contexts);
     free(threads);
 }
@@ -184,6 +247,7 @@ void sql_callback_add_data(sqlite3_stmt *sql_stmt, void *data) {
 // ----------------------------------------------------------------------------
 void sig_handler(int signo) {
     (void)signo;
+
     log_warn("Signal received, will stop SALIS data server");
     json_object_put(g_response_header);
     sql_close();
@@ -245,7 +309,6 @@ void respond_data(int socket_fd, struct json_object *request) {
     const char *x_axis_pref = (!strcmp(x_axis, "rowid") || !strcmp(x_axis, "step")) ? "core." : "";
 
     sql_exec(
-        0, NULL, NULL,
         sql_callback_add_data,
         &callback_context,
         "select * from ("
@@ -322,14 +385,11 @@ int main(void) {
     g_response_header = json_object_new_object();
     json_object_object_add(g_response_header, "rowid", json_object_new_array());
     sql_exec(
-        0, NULL, NULL,
         sql_callback_add_column_name,
         g_response_header,
-        "select name, type from pragma_table_info('core') union "
-        "select name, type from pragma_table_info('arch');"
+        "select name, type from pragma_table_info('core') union select name, type from pragma_table_info('arch');"
     );
-
-    log_info("Found %lu blob columns in database", g_blob_count);
+    log_info("Found %lu eva-size columns in database", g_eva_count);
 
     log_info("Binding to port: %d", PORT);
     int opt = 1;