diff options
Diffstat (limited to 'salis.py')
| -rwxr-xr-x | salis.py | 444 |
1 files changed, 444 insertions, 0 deletions
diff --git a/salis.py b/salis.py new file mode 100755 index 0000000..48436de --- /dev/null +++ b/salis.py @@ -0,0 +1,444 @@ +#!/usr/bin/env -S PYTHONDONTWRITEBYTECODE=1 python + +# file : salis.py +# project : Salis-VM +# author : Paul Oliver <contact@pauloliver.dev> + +# index: +# [section] imports +# [section] argument parser +# [section] unit tests +# [section] build class +# [section] logger +# [section] options dict formatter +# [section] entry-point; source options +# [section] load instruction set +# [section] assemble ancestor +# [section] general defines +# [section] build and launch main executable +# [section] build and launch data server +# [section] build and launch data client + +# ------------------------------------------------------------------------------ +# [section] imports +# ------------------------------------------------------------------------------ +import ctypes +import json +import os +import random +import shutil +import socket +import subprocess +import sys +import unittest + +from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser, ArgumentTypeError, RawTextHelpFormatter +from tempfile import TemporaryDirectory +from types import SimpleNamespace + +# ------------------------------------------------------------------------------ +# [section] argument parser +# ------------------------------------------------------------------------------ +prog = sys.argv[0] +epilog = f"Use '-h' to show command arguments; e.g. '{prog} new -h'" + +parser = ArgumentParser(description="Salis: Simple A-Life Simulator", epilog=epilog, formatter_class=RawTextHelpFormatter, prog=prog) +sub_parsers = parser.add_subparsers(dest="command", required=True) +formatter_class = lambda prog: ArgumentDefaultsHelpFormatter(max_help_position=48, prog=prog) + +new = sub_parsers.add_parser("new", formatter_class=formatter_class, help="create new simulation") +load = sub_parsers.add_parser("load", formatter_class=formatter_class, help="load saved simulation") +server = sub_parsers.add_parser("server", formatter_class=formatter_class, help="run data server") +client = sub_parsers.add_parser("client", formatter_class=formatter_class, help="run data client") +test = sub_parsers.add_parser("test", formatter_class=formatter_class, help="run unit tests") + +def seed(i): + ival = int(i, 0) + if ival < -1: raise ArgumentTypeError("invalid seed value") + return ival + +def pos(i): + ival = int(i, 0) + if ival < 0: raise ArgumentTypeError("value must be positive integer") + return ival + +def nat(i): + ival = int(i, 0) + if ival < 1: raise ArgumentTypeError("value must be greater than zero") + return ival + +def port(i): + ival = int(i, 0) + if not 1024 <= ival <= 49151: raise ArgumentTypeError("value must be valid port number") + return ival + +fmt_id = lambda val: val +fmt_hex = lambda val: hex(int(val, 0)) if type(val) == str else hex(val) + +supported_c_compilers = ("clang", "gcc", "tcc") +supported_cpp_compilers = ("clang++", "g++") + +options = ( + ("a", "anc", (new,), fmt_id, {"metavar": "ANC", "help": "ancestor file name without extension; to be compiled on all cores; ANC points to '{anc-path}/{arch}/{ANC}.asm'", "required": True, "type": str}), + ("A", "anc-path", (new,), fmt_id, {"metavar": "PATH", "help": "path to search for ancestor files", "default": "anc", "required": False, "type": str}), + ("b", "build-only", (new, load, server, client), fmt_id, {"action": "store_true", "help": "only build (do not run) executable", "required": False}), + ("C", "clones", (new,), fmt_id, {"metavar": "N", "help": "number of ancestor clones on each core", "default": 1, "required": False, "type": nat}), + ("c", "cores", (new,), fmt_id, {"metavar": "N", "help": "number of simulator cores", "default": 2, "required": False, "type": nat}), + ("d", "data-push-pow", (new,), fmt_id, {"metavar": "POW", "help": "data aggregation interval exponent; interval = 2^{POW} >= {sync-pow}", "default": 28, "required": False, "type": nat}), + ("f", "font", (client,), fmt_id, {"metavar": "FILE", "help": "font face to use", "default": "/usr/share/fonts/droid/DroidSansMono.ttf", "required": False}), + ("f", "force", (new,), fmt_id, {"action": "store_true", "help": "overwrite existing simulation of given name", "required": False}), + ("g", "c-compiler", (new, load, server, client), fmt_id, {"choices": supported_c_compilers, "help": "C compiler to use", "default": "gcc", "required": False, "type": str}), + ("G", "cpp-compiler", (client,), fmt_id, {"choices": supported_cpp_compilers, "help": "C++ compiler to use", "default": "g++", "required": False, "type": str}), + ("H", "home", (new, load, server), fmt_id, {"metavar": "PATH", "help": "salis home directory", "default": f"{os.environ["HOME"]}/.salis", "required": False, "type": str}), + ("i", "ip", (client,), fmt_id, {"metavar": "IP", "help": "ip address of server", "default": "127.0.0.1", "required": False, "type": str}), + ("l", "log-trace", (new, load, server, client), fmt_id, {"action": "store_true", "help": "log verbose trace messages", "required": False}), + ("M", "muta-pow", (new,), fmt_id, {"metavar": "POW", "help": "mutator range exponent; each step a cosmic ray hits addr = rand_uint64() %% 2^{POW}; lower values of POW mean higher mutation rates", "default": 32, "required": False, "type": pos}), + ("m", "mvec-pow", (new,), fmt_id, {"metavar": "POW", "help": "memory core size exponent; size = 2^{POW}", "default": 20, "required": False, "type": pos}), + ("n", "name", (new, load, server), fmt_id, {"metavar": "NAME", "help": "name of new or loaded simulation", "default": "def.sim", "required": False, "type": str}), + ("o", "optimized", (new, load, server, client), fmt_id, {"action": "store_true", "help": "build with optimizations", "required": False}), + ("p", "pattern", (test,), fmt_id, {"metavar": "PATTERN", "help": "pattern for unit test discovery", "default": "*", "required": False, "type": str}), + ("P", "port", (server, client), fmt_id, {"metavar": "PORT", "help": "port number for data server", "default": 8080, "required": False, "type": port}), + ("p", "pre-cmd", (new, load, server, client), fmt_id, {"metavar": "CMD", "help": "shell command with which to wrap call to executable; e.g. gdb, time, valgrind, etc.", "required": False, "type": str}), + ("s", "seed", (new,), fmt_hex, {"metavar": "SEED", "help": "seed value for new simulation; a value of 0 disables cosmic rays; a value of -1 creates a random seed", "default": 0, "required": False, "type": seed}), + ("T", "keep-temp-dir", (new, load, server, client), fmt_id, {"action": "store_true", "help": "keep temporary directory on exit", "required": False}), + ("u", "ui", (new, load), fmt_id, {"metavar": "UI", "help": "user interface", "default": "curses", "required": False, "type": str}), + ("U", "ui-path", (new, load), fmt_id, {"metavar": "PATH", "help": "path to search for UIs", "default": "ui", "required": False, "type": str}), + ("v", "vm-arch", (new,), fmt_id, {"metavar": "ARCH", "help": "VM architecture", "default": "null", "required": False, "type": str}), + ("y", "sync-pow", (new,), fmt_id, {"metavar": "POW", "help": "core sync interval exponent; sync events occur every N steps, where N = 2^{POW}", "default": 20, "required": False, "type": pos}), + ("z", "auto-save-pow", (new,), fmt_id, {"metavar": "POW", "help": "auto-save interval exponent; interval = 2^{POW} >= {sync_pow}", "default": 36, "required": False, "type": pos}), +) + +[sub_parser.add_argument(f"-{sopt}", f"--{lopt}", **kwargs) for sopt, lopt, sub_parsers, _, kwargs in options for sub_parser in sub_parsers] +args = parser.parse_args() + +# ------------------------------------------------------------------------------ +# [section] unit tests +# ------------------------------------------------------------------------------ +if args.command == "test": + loader = unittest.TestLoader() + loader.testNamePatterns = [args.pattern] + runner = unittest.TextTestRunner(verbosity=2) + result = runner.run(loader.discover("test")) + exit(0 if not result.errors and not result.failures else 1) + +# ------------------------------------------------------------------------------ +# [section] build class +# ------------------------------------------------------------------------------ +class Build: + def __init__(self, path, is_library=False, is_object=False, is_cpp=False, flags=None, defines=None, links=None): + assert [is_library, is_object].count(True) <= 1 + + self.path = path + self.is_library = is_library + self.is_object = is_object + self.is_cpp = is_cpp + + self.flags = flags or [] + self.defines = defines or [] + self.links = links or [] + + self.basename = os.path.splitext(os.path.basename(self.path))[0] + self.extension = ".so" if self.is_library else "" + self.extension = ".o" if self.is_object else "" + self.tempdir = TemporaryDirectory(prefix="salis_", delete=not args.keep_temp_dir) + self.binfile = f"{self.tempdir.name}/{self.basename}{self.extension}" + + self.flags += ["-Wall", "-Wextra", "-Werror", "-Wmissing-declarations", "-Wno-overlength-strings", "-pedantic"] + self.flags += ["-std=c++20"] if self.is_cpp else ["-Wmissing-prototypes"] + self.flags += ["-fPIC", "-shared"] if self.is_library else [] + self.flags += ["-c"] if self.is_object else [] + self.flags += ["-O3", "-flto", "-march=native", "-mtune=native"] if args.optimized else ["-ggdb"] + self.defines += ["-DNDEBUG"] if args.optimized else [] + + self.compiler = args.cpp_compiler if self.is_cpp else args.c_compiler + self.build_cmd = [self.compiler, *self.flags, *self.defines, *self.links, self.path, "-o", self.binfile] + + def build(self): + subprocess.run(self.build_cmd, check=True) + + # Clear exec-stack on dynamic libraries when using TCC. + # Otherwise python will complain when loading the library with CDLL. + # NOTE: Using TCC thus requires patchelf to be accessible. + if self.is_library and args.c_compiler == "tcc": + subprocess.run(["patchelf", "--clear-execstack", self.binfile], check=True) + + def log(self, log): + log.trace(f"Built source '{self.path}' to output: {self.binfile}") + log.trace(f"Using build command: {self.build_cmd}") + + def run(self): + run_cmd = args.pre_cmd.split() if args.pre_cmd else [] + run_cmd.append(self.binfile) + proc = subprocess.Popen(run_cmd, stdout=sys.stdout, stderr=sys.stderr) + + # When a signal is received (e.g. SIGINT), is propagates through the entire process group. + # Handle signal on the interpreter and allow the simulator to shut down gracefully. + try: + proc.wait() + except KeyboardInterrupt: + proc.terminate() + proc.wait() + + code = proc.returncode + + if code != 0: raise RuntimeError(f"Binary returned code: {code}") + +# ------------------------------------------------------------------------------ +# [section] logger +# ------------------------------------------------------------------------------ +logger_defines = ["-DLOG_TRACE"] if args.log_trace else [] +logger_shared_build = Build("core/logger.c", is_library=True, defines=logger_defines) +logger_shared_build.build() +logger_dll = ctypes.CDLL(logger_shared_build.binfile) + +# Pull in logging functions from C +# This way there's just a single logging system to care about +log = SimpleNamespace() +log.trace = lambda msg: logger_dll.log_trace_to_stdout(msg.encode()) +log.info = lambda msg: logger_dll.log_info_to_stdout(msg.encode()) +log.attention = lambda msg: logger_dll.log_attention_to_stdout(msg.encode()) +logger_shared_build.log(log) + +# ------------------------------------------------------------------------------ +# [section] options dict formatter +# ------------------------------------------------------------------------------ +fmt_h2u = lambda field: field.replace("-", "_") +fmt_u2h = lambda field: field.replace("_", "-") + +def fmt_field(field, val): + for _, lopt, _, fmt, _ in options: + if lopt == fmt_u2h(field): + return fmt(val) + + return fmt_id(val) + +def fmt_opts(opts): + opts_out = {field: fmt_field(field, val) for field, val in opts.items()} + return json.dumps(opts_out, indent=2) + +# ------------------------------------------------------------------------------ +# [section] entry-point; source options +# ------------------------------------------------------------------------------ +log.info(f"Called '{prog} {args.command}' with the following options: {fmt_opts(vars(args))}") +paths = SimpleNamespace() + +def gen_paths(): + paths.sim_dir = f"{args.home}/{args.name}" + paths.sim_data = f"{paths.sim_dir}/{args.name}.sqlite3" + paths.sim_evas = f"{paths.sim_dir}/evas" + paths.sim_opts = f"{paths.sim_dir}/opts.json" + paths.sim_path = f"{paths.sim_dir}/{args.name}" + +def source_options(): + if not os.path.isdir(paths.sim_dir): + raise RuntimeError(f"No simulation found named {args.name}") + + with open(paths.sim_opts, "r") as f: + opts = json.loads(f.read()) + + for key, val in opts.items(): + setattr(args, key, val) + + log.info(f"Sourced configuration from '{paths.sim_opts}': {fmt_opts(opts)}") + +def request_from_server(request): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as client: + client.connect((args.ip, args.port)) + client.sendall(json.dumps(request).encode()) + client.shutdown(socket.SHUT_WR) + return json.load(client.makefile(mode="r")) + +# Create new simulation +if args.command == "new": + gen_paths() + + if 0 < args.data_push_pow < args.sync_pow: + raise RuntimeError("Data push power must be equal or greater than core sync power") + + if 0 < args.auto_save_pow < args.sync_pow: + raise RuntimeError("Autosave power must be equal or greater than core sync power") + + if os.path.isdir(paths.sim_dir) and args.force: + log.attention(f"Force flag used! Wiping old simulation at: {paths.sim_dir}") + shutil.rmtree(paths.sim_dir) + + if os.path.isdir(paths.sim_dir): + raise RuntimeError(f"Simulation directory found at: {paths.sim_dir}; use --force flag to remove it") + + if args.seed == -1: + args.seed = random.getrandbits(64) + log.info(f"Using random seed: {hex(args.seed)}") + + log.info(f"Creating new simulation directory at: {paths.sim_dir}") + log.info(f"Creating new event-array storage directory at: {paths.sim_evas}") + log.info(f"Creating configuration file at: {paths.sim_opts}") + os.mkdir(paths.sim_dir) + os.mkdir(paths.sim_evas) + + opts = {fmt_h2u(lopt): getattr(args, fmt_h2u(lopt)) for _, lopt, sub_parsers, _, kwargs in options if new in sub_parsers and load not in sub_parsers} + + with open(paths.sim_opts, "w") as f: + f.write(f"{fmt_opts(opts)}\n") + +# Load saved simulation +if args.command == "load": + gen_paths() + source_options() + +# Data server +if args.command == "server": + gen_paths() + source_options() + +# Data client +if args.command == "client": + # Check provided font file exists + if not os.path.isfile(args.font): + raise RuntimeError(f"Font file {args.font} does not exist; please provide a valid font") + + # Check chosen compilers are compatible + if (args.cpp_compiler == "clang++" and args.c_compiler != "clang") or (args.cpp_compiler == "g++" and args.c_compiler != "gcc"): + raise RuntimeError(f"Compilers {args.cpp_compiler} and {args.c_compiler} might have incompatible ABIs; use either clang++/clang or g++/gcc") + + # Pull basic information from the server + # Required to build the client with correct flags + args.name = request_from_server({"request": "name"})["name"] + log.info(f"Sourced simulation name from '{args.ip}:{args.port}': {args.name}") + + opts = request_from_server({"request": "opts"}) + + for key, val in opts.items(): + setattr(args, key, val) + + log.info(f"Sourced configuration from '{args.ip}:{args.port}': {fmt_opts(opts)}") + + # Server and client should be on the same hash + server_hash = request_from_server({"request": "hash"})["hash"] + client_hash = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode("utf-8").strip() + + if server_hash != client_hash: + raise RuntimeError(f"Server hash '{server_hash}' and client hash '{client_hash}' do not match") + + log.info(f"Confirmed server and client are running against the same git hash: {server_hash}") + +# ------------------------------------------------------------------------------ +# [section] load instruction set +# ------------------------------------------------------------------------------ +arch_inst_path = f"arch/{args.vm_arch}/arch_inst.c" +arch_build = Build(arch_inst_path, is_library=True, flags=["-Icore"]) +arch_build.build() +arch_build.log(log) + +arch_dll = ctypes.CDLL(arch_build.binfile) +arch_dll.arch_inst_mnemonic.argtypes = [ctypes.c_uint8] +arch_dll.arch_inst_mnemonic.restype = ctypes.c_char_p +arch_dll.arch_inst_count.restype = ctypes.c_int + +# ------------------------------------------------------------------------------ +# [section] assemble ancestor +# ------------------------------------------------------------------------------ +anc_path = f"{args.anc_path}/{args.vm_arch}/{args.anc}.asm" + +if not os.path.isfile(anc_path): + raise RuntimeError(f"Could not find ancestor file: {anc_path}") + +with open(anc_path, "r") as f: + lines = f.read().splitlines() + +lines = filter(lambda line: line and not line.startswith(";") and not line.isspace(), lines) +lines = map(lambda line: tuple(line.split()), lines) + +inst_count = arch_dll.arch_inst_count() +mnemo_map = {tuple(arch_dll.arch_inst_mnemonic(byte).decode().split()): byte for byte in range(inst_count)} +anc_bytes = [mnemo_map[line] for line in lines] +anc_repr = f"{{{",".join(map(str, anc_bytes))}}}" + +log.info(f"Compiled ancestor file '{anc_path}' into byte array: {anc_repr}") + +# ------------------------------------------------------------------------------ +# [section] general defines +# ------------------------------------------------------------------------------ +general_defines = [ + f"-DANC=\"{args.anc}\"", + f"-DANC_BYTES={anc_repr}", + f"-DANC_SIZE={len(anc_bytes)}", + f"-DARCH=\"{args.vm_arch}\"", + f"-DAUTOSAVE_INTERVAL={2 ** args.auto_save_pow}ul", + f"-DCLONES={args.clones}", + f"-DCOMMAND_{args.command.upper()}", + f"-DCORES={args.cores}", + f"-DDATA_PUSH_INTERVAL={2 ** args.data_push_pow}ul", + f"-DFOR_CORES={" ".join(f"FOR_CORE({i})" for i in range(args.cores))}", + f"-DMUTA_RANGE={2 ** args.muta_pow}ul", + f"-DMVEC_SIZE={2 ** args.mvec_pow}ul", + f"-DNAME=\"{args.name}\"", + f"-DSEED={args.seed}ul", + f"-DSYNC_INTERVAL={2 ** args.sync_pow}ul", +] + +if args.command == "new" or args.command == "load" or args.command == "server": + local_defines = [ + f"-DAUTOSAVE_NAME_LEN={len(paths.sim_path) + 18}", + f"-DDATA_PUSH_PATH=\"{paths.sim_data}\"", + f"-DEVA_SAVE_NAME_LEN={len(paths.sim_evas) + 23}", + f"-DSIM_DATA=\"{paths.sim_data}\"", + f"-DSIM_DIR=\"{paths.sim_dir}\"", + f"-DSIM_EVAS=\"{paths.sim_evas}\"", + f"-DSIM_OPTS=\"{paths.sim_opts}\"", + f"-DSIM_PATH=\"{paths.sim_path}\"", + ] + +# ------------------------------------------------------------------------------ +# [section] build and launch main executable +# ------------------------------------------------------------------------------ +if args.command == "new" or args.command == "load": + ui_path = f"{args.ui_path}/{args.ui}" + + if not os.path.isdir(ui_path): + raise RuntimeError(f"UI not found at '{ui_path}'") + + with open(f"{ui_path}/links.json", "r") as f: + ui_links = json.load(f) + + ui_flags = [f"-Iarch/{args.vm_arch}", "-Icore", f"arch/{args.vm_arch}/arch.c", arch_inst_path, "core/compress.c", "core/logger.c", "core/salis.c", "core/sql.c", "core/timer.c"] + ui_defines = general_defines + local_defines + logger_defines + ui_links += ["-lsqlite3", "-lz"] + ui_build = Build(f"{ui_path}/ui.c", flags=ui_flags, defines=ui_defines, links=ui_links) + ui_build.build() + ui_build.log(log) + + if not args.build_only: + ui_build.run() + +# ------------------------------------------------------------------------------ +# [section] build and launch data server +# ------------------------------------------------------------------------------ +if args.command == "server": + server_flags = [f"-Iarch/{args.vm_arch}", "-Icore", "core/compress.c", "core/logger.c", "core/sql.c"] + server_defines = general_defines + local_defines + logger_defines + [f"-DPORT={args.port}"] + server_links = ["-ljson-c", "-lsqlite3", "-lz"] + server_build = Build("data/server.c", flags=server_flags, defines=server_defines, links=server_links) + server_build.build() + server_build.log(log) + + if not args.build_only: + server_build.run() + +# ------------------------------------------------------------------------------ +# [section] build and launch data client +# ------------------------------------------------------------------------------ +if args.command == "client": + # Build logger separately as it's written in C + logger_object_build = Build("core/logger.c", is_object=True, defines=logger_defines) + logger_object_build.build() + logger_object_build.log(log) + + client_flags = [f"-Iarch/{args.vm_arch}", "-Icore", logger_object_build.binfile] + client_defines = general_defines + [f"-DFONT_SOURCE=\"{args.font}\"", f"-DIP=\"{args.ip}\"", f"-DPORT={args.port}", f"-DPORT_STR=\"{args.port}\""] + client_links = ["-lGL", "-lglfw", "-limgui", "-limplot", "-ljson-c", "-lm"] + client_build = Build("data/client.cpp", is_cpp=True, flags=client_flags, defines=client_defines, links=client_links) + client_build.build() + client_build.log(log) + + if not args.build_only: + client_build.run() |