chore: upgrade deno_* crates to Mar 2024

The way extensions are injected into the Isolate changed dramatically
between Sept 2021 and Jan 2024. Now ops are available in the virtual
module ext:core/ops.

Cargo.toml

@@ -5,49 +5,46 @@ edition = "2018"
 
 # I decided to use os_pipe becaues I want to pipe stdout of a subprocess into
 # stderr in real time. I want the output of the process to stderr and stdout
-# show up in the original order. I looked an os_pipe uses some unsafe code to
+# show up in the original order. I looked and os_pipe uses some unsafe code to
 # duplicate file descriptors. I am unfamiliar with the correct way of sharing
 # the fds. Therefore I am going to trust that their unsafe code is necessary.
 # Then it makes sense to use a battle tested unsafe code rather than implement
 # it myself.
 [dependencies]
-ahash = "0.7"
+ahash = "0.8"
+clap = { version = "4.5", features = ["derive"] }
 elmi = { path = "../../../infra/rust-elmi", features = [ "genco" ] }
 naive-wadler-prettier= { path = "../../../infra/redwood-lang/compiler/naive-wadler-prettier" }
 os_pipe = "1.0"
 serde = { version = "1.0", features = [ "derive" ] }
 serde_json = { version ="1.0", features = [] }
-structopt = { version = "0.3" }
 elm-project-utils = { path = "../../../infra/rust-elm-project-utils" }
 tracing = { version = "0.1", features = [] }
 rustc-hash = "1.1"
 home = "0.5"
-
 # Required to transpile view functions to Rust
 genco = "0.17"
 # Required to generate fixture Elm files
 genco-extra = { path = "../../../infra/genco-extra" }
-
-
 # All of these are required for deno's javascript runtime. We need to keep the
 # same versions as other projects in our cargo workspace. Multiple different
 # versions of rusty_v8 seem to break its build script.
-deno_runtime = "0.127"
+deno_runtime = "0.154"
 tokio = { version = "1.32", features = ["full"] }
-deno_core = "0.214"
+deno_core = "0.272"
-deno_web = "0.150"
+deno_web = "0.177"
 futures = "0.3"
-
 # Required to add sql query support to interpreter. Because deno expects sync
 # ops to be synchronous, we have to use a second async executor to run the sqlx
 # functions. I read the code for oneshot
 # (https://github.com/faern/oneshot/commit/9aa237f185e1b65d61bf92c20350cf7bee0aa88b)
 # and it looks reasonable.
+oneshot = { version = "0.1.6", features = ["std"]}
 sqlx = { version = "0.7", features = [ "sqlite", "macros", "runtime-tokio", "chrono", "json", "uuid" ] }
-oneshot = "0.1.3"
-
 # required for livetable derive macro
 livetable-core = { path = "../../../infra/livetable/core" }
+# required for clean subcommand
+walkdir = "2.4"
 
 
 [dev-dependencies]

README.md

@@ -0,0 +1,49 @@
+
+When I first created starmelon in September 2021 there was a hacky workaround
+for setTimeout I had to inject into the generated javascript. Now in Nov 2023
+after upgrading deno to 0.127 I find that `globalThis.__boostrap.timers` is no
+longer defined. Now the elm code seems to work without requring this hack.
+However I don't have any correctness tests for starmelon. To reduce the risk of
+forgetting how it was working I have included the hacks from
+src/exec/scripting.rs:168
+
+```rust
+// I think that when I set this script to be the main module, I am skipping the
+// deno/runtime/js/99_main.js script that sets up a bunch of global variables. If I
+// manually add the timer related code below then setTimeout works again.
+// NB. there are 706 lines of setup code that add a bunch of apis to the global window
+// scope. Figure out if I need to include all of them. For example, starmelon does not need
+// to perform http calls right now, but I eventually want to.
+final_script.push_str("const { setTimeout } = globalThis.__bootstrap.timers;\n");
+final_script.push_str(
+    "Deno.core.setMacrotaskCallback(globalThis.__bootstrap.timers.handleTimerMacrotask);\n",
+);
+final_script.push_str("globalThis.setTimeout = setTimeout;\n");
+```
+
+Somewhere between `deno_runtime` version 0.127 and 0.147 they decided to remove
+`deno_core::op` macro and replace it with `deno_core::op2`. As far as I can
+tell, the `op2` macro offers greater control of how values are passed between
+Rust and JavaScript.
+
+In Jan 2024 they moved operations to a virtual module
+`import { op_example } from "ext:core/ops`. You can only access this virtual
+module in the bootstrap ESM scripts of an `deno_core::Extension`. Consequently
+what I have done is write a bootstrap script that imports the ops and reassigns
+them as properties of `globalThis.Extension` object. All of my extensions are
+merged onto the same object. It appears the Deno.core is deleted by the time
+the `deno_runtime::worker::MainWorker` runs the main module.
+`Deno[Deno.internal].core.ops` still exists but does not contain the ops our
+Extensions defined.
+
+An aside is that if you construct a JsRuntime directly and add Extensions then
+those ops will show up on `Deno.core.ops`. But they will not be enumerable
+properties, so you will have to use `Object.getOwnPropertyNames(Deno.core.ops)`
+to visually confirm the ops are there. My current understanding in April 2024
+is that MainWorker includes a JsRuntime, but then also applies all of the
+`deno_runtime` extensions that make the JS enviroment feature comparable with
+Node. For example setTimeout does not exist in a new JsRuntime but does exist
+in a new MainWorker.
+
+You can find some of the extensions that Deno provides in
+[https://github.com/denoland/deno/tree/main/runtime/js]

always-rebuild.do

@@ -1,2 +0,0 @@
-redo-always
-redo-ifchange release

examples/sqlite-integration/README.md

@@ -0,0 +1 @@
+./starmelon exec --sqlite example.sqlite src/ShowDatabaseSchema.elm main --output=schema.html

examples/sqlite-integration/src/ShowDatabaseSchema.elm

@@ -11,7 +11,7 @@ main =
     let
         query =
             fetch
-                "select json_object('type', type, 'name', name, 'sql', sql) from sqlite_master WHERE type = 'table'"
+                "select type, name, sql from sqlite_master WHERE type = 'table'"
                 []
                 (Json.Decode.map3
                     (\kind name sql->

release.do

@@ -1,4 +1,4 @@
 redo-ifchange Cargo.toml
 find src/ -type f | xargs redo-ifchange
 
-cargo build --release
+cargo build --color=always --release

src/debug.rs

@@ -0,0 +1,96 @@
+use deno_core::{Extension, OpState, op2, extension};
+use elm_project_utils::{setup_generator_project, ElmResult};
+use os_pipe::dup_stderr;
+use sqlx::sqlite::SqlitePool;
+use std::{
+    borrow::Cow,
+    cell::RefCell,
+    convert::TryFrom,
+    fs,
+    io::{self, Read, Write},
+    path::PathBuf,
+    process::{Command, Stdio},
+    sync::Arc,
+};
+use tracing::info_span;
+
+use crate::exec::{fixtures, runtime};
+use crate::reporting::{CompilerError, InterpreterError, Problem, TypeError};
+
+#[deno_core::op2(fast)]
+fn op_starmelon_example(
+    state: &mut OpState,
+    #[string] msg: String,
+) -> Result<(), deno_core::error::AnyError> {
+    eprintln!("got a String message from v8 runtime {:?}", &msg);
+    Ok(())
+}
+
+#[op2(fast)]
+fn op_hello(#[string] text: &str) {
+    println!("Hello {} from an op!", text);
+}
+
+extension!(
+  debug_ext,
+  ops = [op_hello],
+  esm_entry_point = "ext:debug_ext/debug_bootstrap.js",
+  esm = [dir "src", "debug_bootstrap.js"],
+  docs = " small example demonstrating op2 usage.", "Contains one op."
+);
+
+pub(crate) fn run(javascript_path: Option<PathBuf>) -> Result<(), Problem> {
+    let final_path: PathBuf = if let Some(ref path) = javascript_path {
+        std::fs::canonicalize(path).unwrap()
+        //std::fs::read_to_string(&path)
+        //    .map_err(|io_err| CompilerError::ReadInputFailed(io_err, "stdout".into()))?
+    } else {
+        return Ok(());
+        //"console.log('hello world')".to_owned()
+    };
+
+    // Create a tokio runtime before registering ops so we can block on futures inside sync ops
+    let span = info_span!("create tokio runtime");
+    let timing_guard = span.enter();
+
+    let sys = tokio::runtime::Builder::new_current_thread()
+        // The default number of additional threads for running blocking FnOnce is 512.
+        .max_blocking_threads(1)
+        .enable_all()
+        .build()
+        .unwrap();
+    drop(timing_guard);
+
+    // step 10 create a v8 isolate. We need to register a different callback depending on
+    // the output type (string, or bytes)
+
+    //let mailbox: Arc<RefCell<Option<Result<Vec<u8>, String>>>> = Arc::new(RefCell::new(None));
+    //let mailbox_clone = Arc::clone(&mailbox);
+
+    let mut extensions = vec![
+        Extension {
+            ops: Cow::Owned(vec![op_starmelon_example::decl()]),
+            op_state_fn: Some(Box::new(move |state| {
+                //state.put(Arc::clone(&mailbox_clone));
+            })),
+            ..Default::default()
+        },
+        debug_ext::init_ops_and_esm(),
+    ];
+
+    let span = info_span!("create v8 isolate");
+    let timing_guard = span.enter();
+    let (worker, main_module) = runtime::setup_worker(extensions, &final_path.to_string_lossy())
+        .map_err(|err| InterpreterError::EventLoop(err))?;
+    drop(timing_guard);
+
+    let entrypoint =
+        move |mut scope: deno_core::v8::HandleScope| -> Result<(), InterpreterError> { Ok(()) };
+
+    let span = info_span!("eval javascript");
+    let timing_guard = span.enter();
+    sys.block_on(async move { runtime::xyz(worker, main_module, entrypoint).await })?;
+    drop(timing_guard);
+
+    Ok(())
+}

src/debug_bootstrap.js

@@ -0,0 +1,7 @@
+// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
+import { op_hello } from "ext:core/ops";
+function hello() {
+  op_hello("world");
+}
+
+globalThis.Extension = { hello };

src/derive/mod.rs

@@ -180,8 +180,8 @@ fn resolve_table_argument(tipe: &elmi::Type) -> Result<i32, TypeError> {
 //ColumnType::MaybeFloat => Maybe Float,
 //ColumnType::Double => Float,
 //ColumnType::MaybeDouble => Maybe Float,
-//ColumnType::Timestamp => #(posix_type.clone()),
+//ColumnType::Timestamp => $(posix_type.clone()),
-//ColumnType::MaybeTimestamp => Maybe #(posix_type.clone()),
+//ColumnType::MaybeTimestamp => Maybe $(posix_type.clone()),
 //ColumnType::ExactlyOneAttachment => LiveTable.Attachment,
 //ColumnType::MaybeAttachment => Maybe LiveTable.Attachment,
 //ColumnType::ListAttachment => List LiveTable.Attachment,

src/elm.rs

@@ -30,6 +30,7 @@ pub fn make(file: &Path, debug: bool, verbosity: u64) -> Result<(), Problem> {
             }
         }
         Err(_io_err) => {
+            eprintln!("{:?}", _io_err);
             // TODO handle this one
             return Err(Problem::Wildcard("elm failed".into()));
         }

src/exec/astrid_pages.rs

@@ -1,22 +1,24 @@
-use crate::exec::fixtures::astrid_pages::ScriptError;
+use deno_core::{v8, Extension, OpState, ExtensionFileSource};
-use crate::exec::{fixtures, runtime};
-use crate::reporting::{CompilerError, InterpreterError, Problem};
-use deno_core::{v8, Extension, OpState};
 use elm_project_utils::{setup_generator_project, ElmPostProcessor, ElmResult};
 use os_pipe::dup_stderr;
 use sqlx::sqlite::SqlitePool;
-use std::borrow::Cow;
+use std::{
-use std::cell::RefCell;
+    borrow::Cow,
-use std::convert::TryFrom;
+    cell::RefCell,
-use std::fs;
+    convert::TryFrom,
-use std::io;
+    fs,
-use std::io::Write;
+    io,
-use std::path::PathBuf;
+    io::Write,
-use std::process::{Command, Stdio};
+    path::PathBuf,
-use std::sync::Arc;
+    process::{Command, Stdio},
-use tokio;
+    sync::Arc,
+};
 use tracing::info_span;
 
+use crate::exec::fixtures::astrid_pages::ScriptError;
+use crate::exec::{fixtures, runtime};
+use crate::reporting::{CompilerError, InterpreterError, Problem};
+
 #[derive(Debug, Copy, Clone)]
 pub enum OutputType {
     Html,
@@ -156,21 +158,6 @@ pub(crate) fn run(
         }
         _ => (),
     }
-    // I think that when I set this script to be the main module, I am skipping the
-    // deno/runtime/js/99_main.js script that sets up a bunch of global variables. If I
-    // manually add the timer related code below then setTimeout works again.
-    // NB. there are 706 lines of setup code that add a bunch of apis to the global window
-    // scope. Figure out if I need to include all of them. For example, starmelon does not need
-    // to perform http calls right now, but I eventually want to.
-    buffer.push_str(
-        r#"
-    
-        const { setTimeout } = globalThis.__bootstrap.timers;
-        Deno.core.setMacrotaskCallback(globalThis.__bootstrap.timers.handleTimerMacrotask);
-        globalThis.setTimeout = setTimeout;
-    
-    "#,
-    );
 
     buffer.push_str(&format!(
         "var worker = Elm.{}.init({{flags: {{ stagename: \"Atago\"}} }});\n",
@@ -183,7 +170,7 @@ pub(crate) fn run(
         globalThis.runOnInput = function(route) { worker.ports.onRequest.send(route) };
 
         worker.ports.onStringOutput.subscribe(function(result) {
-            Deno.core.opSync('op_starmelon_string_output', result);
+            Extension.starmelon_string_output(result);
         });
         // Elm will send a DataView
         if (worker.ports.onBytesOutput) {
@@ -192,7 +179,7 @@ pub(crate) fn run(
                     const ui8 = new Uint8Array(result.a.buffer);
                     output.a = ui8;
                 }
-                Deno.core.opSync('op_starmelon_bytes_output', result)
+                Extension.starmelon_bytes_output(result)
             });
         }"#,
     );
@@ -268,6 +255,36 @@ pub(crate) fn run(
         op_state_fn: Some(Box::new(move |state| {
             state.put(Arc::clone(&mailbox_clone));
         })),
+        esm_files: {
+            const JS: &'static [ExtensionFileSource] = &[ExtensionFileSource::new(
+                "ext:io/bootstrap.js",
+                {
+                    const STR: ::deno_core::v8::OneByteConst = ::deno_core::FastStaticString::create_external_onebyte_const(
+                        r#"
+import { op_starmelon_bytes_output, op_starmelon_string_output, op_starmelon_problem } from "ext:core/ops";
+function starmelon_bytes_output(msg) {
+    return op_starmelon_bytes_output(msg);
+}
+function starmelon_string_output(msg) {
+    return op_starmelon_string_output(msg);
+}
+function starmelon_problem(msg) {
+    return op_starmelon_problem(msg);
+}
+globalThis.Extension = Object.assign(
+    globalThis.Extension || {},
+    { starmelon_bytes_output, starmelon_string_output, starmelon_problem }
+);
+"#
+                                    .as_bytes(),
+                            );
+                    let s: &'static ::deno_core::v8::OneByteConst = &STR;
+                    ::deno_core::FastStaticString::new(s)
+                },
+            )];
+            Cow::Borrowed(JS)
+        },
+        esm_entry_point: Some("ext:io/bootstrap.js"),
         ..Default::default()
     }];
 
@@ -342,10 +359,10 @@ pub(crate) fn run(
 
 type OutputMailbox = Arc<RefCell<Option<Result<Vec<u8>, ScriptError>>>>;
 
-#[deno_core::op]
+#[deno_core::op2]
 fn op_starmelon_problem(
-    state: &mut deno_core::OpState,
+    state: &mut OpState,
-    msg: ScriptError,
+    #[serde] msg: ScriptError,
 ) -> Result<(), deno_core::error::AnyError> {
     eprintln!("got problem from v8 runtime {:?}", &msg);
     let mailbox_clone = state.borrow::<OutputMailbox>();
@@ -355,10 +372,10 @@ fn op_starmelon_problem(
     Ok(())
 }
 
-#[deno_core::op]
+#[deno_core::op2]
 fn op_starmelon_bytes_output(
     state: &mut OpState,
-    msg: ElmResult<deno_core::JsBuffer, ScriptError>,
+    #[serde] msg: ElmResult<deno_core::JsBuffer, ScriptError>,
 ) -> Result<(), deno_core::error::AnyError> {
     let mailbox_clone = state.borrow::<OutputMailbox>();
     if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() {
@@ -375,10 +392,10 @@ fn op_starmelon_bytes_output(
     Ok(())
 }
 
-#[deno_core::op]
+#[deno_core::op2]
 fn op_starmelon_string_output(
     state: &mut OpState,
-    msg: ElmResult<String, ScriptError>,
+    #[serde] msg: ElmResult<String, ScriptError>,
 ) -> Result<(), deno_core::error::AnyError> {
     let mailbox_clone = state.borrow::<OutputMailbox>();
     if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() {

src/exec/css_in_elm.rs

@@ -1,6 +1,6 @@
 use crate::exec::{fixtures, runtime};
 use crate::reporting::{CompilerError, InterpreterError, Problem};
-use deno_core::{Extension, OpState};
+use deno_core::{Extension, ExtensionFileSource, OpState};
 use elm_project_utils::{setup_generator_project, ElmPostProcessor};
 use os_pipe::dup_stderr;
 use serde::Deserialize;
@@ -11,7 +11,6 @@ use std::io::Write;
 use std::path::PathBuf;
 use std::process::{Command, Stdio};
 use std::sync::Arc;
-use tokio;
 use tracing::info_span;
 
 pub(crate) fn run(
@@ -116,21 +115,6 @@ pub(crate) fn run(
         }
         _ => (),
     }
-    // I think that when I set this script to be the main module, I am skipping the
-    // deno/runtime/js/99_main.js script that sets up a bunch of global variables. If I
-    // manually add the timer related code below then setTimeout works again.
-    // NB. there are 706 lines of setup code that add a bunch of apis to the global window
-    // scope. Figure out if I need to include all of them. For example, starmelon does not need
-    // to perform http calls right now, but I eventually want to.
-    buffer.push_str(
-        r#"
-    
-        const { setTimeout } = globalThis.__bootstrap.timers;
-        Deno.core.setMacrotaskCallback(globalThis.__bootstrap.timers.handleTimerMacrotask);
-        globalThis.setTimeout = setTimeout;
-    
-    "#,
-    );
 
     buffer.push_str(&format!("var worker = Elm.{}.init();\n", &gen_module_name));
     // add a shortcut for invoking the function so I don't have to traverse so many object
@@ -139,7 +123,7 @@ pub(crate) fn run(
         r#"
         if (worker.ports.onFilesOutput) {
             worker.ports.onFilesOutput.subscribe(function(result){
-                Deno.core.opSync('op_starmelon_elm_css_files_output', result)
+                Extension.starmelon_elm_css_files_output(result)
             });
         }"#,
     );
@@ -180,6 +164,29 @@ pub(crate) fn run(
         op_state_fn: Some(Box::new(move |state| {
             state.put(Arc::clone(&mailbox_clone));
         })),
+        esm_files: {
+            const JS: &'static [ExtensionFileSource] =
+                &[ExtensionFileSource::new("ext:io/bootstrap.js", {
+                    const STR: ::deno_core::v8::OneByteConst =
+                        ::deno_core::FastStaticString::create_external_onebyte_const(
+                            r#"
+import { op_starmelon_elm_css_files_output } from "ext:core/ops";
+function starmelon_elm_css_files_output(msg) {
+    return op_starmelon_elm_css_files_output(msg);
+}
+globalThis.Extension = Object.assign(
+    globalThis.Extension || {},
+    { starmelon_elm_css_files_output }
+);
+"#
+                            .as_bytes(),
+                        );
+                    let s: &'static ::deno_core::v8::OneByteConst = &STR;
+                    ::deno_core::FastStaticString::new(s)
+                })];
+            Cow::Borrowed(JS)
+        },
+        esm_entry_point: Some("ext:io/bootstrap.js"),
         ..Default::default()
     }];
 
@@ -241,10 +248,10 @@ struct FileDefinition {
     success: bool,
 }
 
-#[deno_core::op]
+#[deno_core::op2]
 fn op_starmelon_elm_css_files_output(
     state: &mut OpState,
-    msg: Vec<FileDefinition>,
+    #[serde] msg: Vec<FileDefinition>,
 ) -> Result<(), deno_core::error::AnyError> {
     let mailbox_clone = state.borrow::<OutputMailbox>();
     if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() {

src/exec/fixtures/astrid_pages.rs

@@ -32,33 +32,33 @@ pub(crate) fn generate(
         import Platform
 
         -- START CUSTOMIZED PART
-        import #(flags_module) exposing (Flags)
+        import $(flags_module) exposing (Flags)
-        #(for (target_module, _) in target_modules.iter() =>
+        $(for (target_module, _) in target_modules.iter() =>
-            import #(target_module)
+            import $(target_module)
-            #<push>
+            $<push>
         )
 
         dispatch route flags =
             case route of
-            #(for (target_module, output_type) in target_modules.iter() =>
+            $(for (target_module, output_type) in target_modules.iter() =>
-                #("    ")#(quoted(target_module)) ->
+                $("    ")$(quoted(target_module)) ->
-                    #( match output_type {
+                    $( match output_type {
                         OutputType::String => {
-                            #("    ")evalRoute (onStringOutput << encodeString) flags #(target_module).route
+                            $("    ")evalRoute (onStringOutput << encodeString) flags $(target_module).route
                         }
                         OutputType::Value => {
-                            #("    ")evalRoute (onStringOutput << encodeJson) flags #(target_module).route
+                            $("    ")evalRoute (onStringOutput << encodeJson) flags $(target_module).route
                         }
                         OutputType::Bytes => {
-                            #("    ")evalRoute (onBytesOutput << encodeBytes) flags #(target_module).route
+                            $("    ")evalRoute (onBytesOutput << encodeBytes) flags $(target_module).route
                         }
                         OutputType::Html =>  {
-                            #("    ")evalRoute (onStringOutput << encodeHtml) flags #(target_module).route
+                            $("    ")evalRoute (onStringOutput << encodeHtml) flags $(target_module).route
                         }
                     })
-                #<push>
+                $<push>
             )
-            #<line>
+            $<line>
 
                 _ ->
                     onStringOutput (encodeFailure NotFound)
@@ -198,7 +198,7 @@ pub(crate) fn generate(
             in
             case
                 D.decodeString
-                    (decodeElmHtml ( #("\\")taggers eventHandler -> D.succeed ()))
+                    (decodeElmHtml ( $("\\")taggers eventHandler -> D.succeed ()))
                     (asJsonString node)
             of
                 Ok elmHtml ->

src/exec/fixtures/css_in_elm.rs

@@ -8,9 +8,9 @@ pub(crate) fn generate(source_checksum: u64, entrypoint: &elmi::Global) -> (Stri
         import Stylesheets
 
         -- START CUSTOMIZED PART
-        import #(&entrypoint.0.module)
+        import $(&entrypoint.0.module)
 
-        entrypoint = #(&entrypoint.0.module).#(&entrypoint.1)
+        entrypoint = $(&entrypoint.0.module).$(&entrypoint.1)
         -- END CUSTOMIZED PART
 
         -- MAIN
@@ -39,7 +39,7 @@ pub(crate) fn generate(source_checksum: u64, entrypoint: &elmi::Global) -> (Stri
         structure =
             Css.File.toFileStructure <|
                 List.map
-                   (#("\\")(fileName, stylesheets) ->
+                   ($("\\")(fileName, stylesheets) ->
                        (fileName, Css.File.compile stylesheets)
                    )
                    entrypoint

src/exec/fixtures/scripting.rs

@@ -180,14 +180,14 @@ decodeString value =
 encodeFailure : String -> E.Value
 encodeFailure msg =
     E.object
-        [ ("ctor", E.string "Err")
+        [ ("$", E.string "Err")
         , ("a", E.string msg)
         ]
 
 encodeSuccess : E.Value -> E.Value
 encodeSuccess value =
     E.object
-        [ ("ctor", E.string "Ok")
+        [ ("$", E.string "Ok")
         , ("a", value)
         ]
 

src/exec/fixtures/sql-client-integration.js

@@ -1,18 +1,18 @@
-function (query) {
+var $author$project$Astrid$Query$execute = function (query) {
     return $author$project$Astrid$Query$dummyExecute;
 };
 
 // CORE QUERIES
 
 function __Debug_print(object) {
-    Deno.core.print(JSON.stringify(object));
+    Deno[Deno.internal].core.print(JSON.stringify(object));
-    Deno.core.print("\n");
+    Deno[Deno.internal].core.print("\n");
 }
 
 function __Debug_print_slots(values) {
     var len = values.length;
     for (var i = 0; i < len; i++) {
-        Deno.core.print(["  ", i, ": ", JSON.stringify(values[i]), "\n"].join(""));
+        Deno[Deno.internal].core.print(["  ", i, ": ", JSON.stringify(values[i]), "\n"].join(""));
     }
 }
 
@@ -129,6 +129,7 @@ function _Query_runDecoder(decoder, sql, xs)
                 return $elm$core$Result$Ok($elm$core$Maybe$Nothing);
             }
             var result = _Json_runOnString.f(decoder.a, xs[0]);
+            console.log("result of decoding string", result)
 
             if (!$elm$core$Result$isOk(result))
             {
@@ -207,6 +208,8 @@ function _Query_runDecoder(decoder, sql, xs)
 
 var _Query_execute = function(query)
 {
+    try {
+        console.log("inside _Query_execute")
     // queries: Array (Int, Query a)
     // values: Array (Maybe a)
     // callbacks: Array (Int, Fn: * -> a)
@@ -262,8 +265,8 @@ var _Query_execute = function(query)
         }
 
         if (statements.length > 0) {
-            var queryResult = Deno.core.opSync(
+            console.log("Extension", Extension)
-                'op_starmelon_batch_queries',
+            var queryResult = Extension.starmelon_batch_queries(
                 statements,
             );
             // I am assuming here that the Rust code is serializing the same
@@ -335,6 +338,9 @@ var _Query_execute = function(query)
             return $elm$core$Result$Ok(values.pop().a)
         }
     }
+    } catch(err) {
+        console.error("Failure:", err)
+    }
 };
 
 var $author$project$Astrid$Query$execute = _Query_execute;

src/exec/fixtures/sqlite.rs

@@ -1,5 +1,5 @@
 use deno_core::futures::StreamExt;
-use deno_core::{Extension, Op, OpState};
+use deno_core::{Extension, ExtensionFileSource, Op, OpState};
 use elm_project_utils::ElmResult;
 use serde::{Deserialize, Serialize};
 use serde_json::{value::Map, value::Number, Value};
@@ -7,7 +7,6 @@ use sqlx::{sqlite::SqlitePool, Column, Row, TypeInfo, ValueRef};
 use std::borrow::Cow;
 use std::thread::JoinHandle;
 use std::time::Instant;
-use tokio;
 use tracing::{info_span, Instrument};
 
 type SQLWorkerMailbox = std::sync::mpsc::Sender<(
@@ -113,20 +112,45 @@ pub(crate) fn init(db_pool: SqlitePool) -> Result<(Extension, JoinHandle<()>), (
     });
     let worker_mailbox_clone = worker_mailbox.clone();
     let extension = Extension {
+        name: "sqlite",
         ops: Cow::Owned(vec![op_starmelon_batch_queries::DECL]),
         op_state_fn: Some(Box::new(move |state| {
             state.put(worker_mailbox_clone.clone());
         })),
+        esm_files: {
+            const JS: &'static [ExtensionFileSource] =
+                &[ExtensionFileSource::new("ext:sqlite/bootstrap.js", {
+                    const STR: ::deno_core::v8::OneByteConst =
+                        ::deno_core::FastStaticString::create_external_onebyte_const(
+                            r#"
+import { op_starmelon_batch_queries } from "ext:core/ops";
+function starmelon_batch_queries(queries) {
+    return op_starmelon_batch_queries(queries);
+}
+globalThis.Extension = Object.assign(globalThis.Extension || {}, { starmelon_batch_queries });
+"#
+                            .as_bytes(),
+                        );
+                    let s: &'static ::deno_core::v8::OneByteConst = &STR;
+                    ::deno_core::FastStaticString::new(s)
+                })];
+            Cow::Borrowed(JS)
+        },
+        esm_entry_point: Some("ext:sqlite/bootstrap.js"),
+        enabled: true,
         ..Default::default()
     };
 
     Ok((extension, sql_worker_thread))
 }
 
-#[deno_core::op]
+deno_core::extension!(hello_runtime, ops = [op_starmelon_batch_queries]);
+
+#[deno_core::op2]
+#[serde]
 fn op_starmelon_batch_queries(
     state: &mut OpState,
-    queries: Vec<(bool, String, Vec<String>)>,
+    #[serde] queries: Vec<(bool, String, Vec<String>)>,
 ) -> Result<ElmResult<Vec<Vec<String>>, AstridQueryError>, deno_core::error::AnyError> {
     let worker_mailbox_clone = state.borrow::<SQLWorkerMailbox>();
 

src/exec/mod.rs

@@ -8,7 +8,7 @@ use tracing::info_span;
 
 mod astrid_pages;
 mod css_in_elm;
-mod fixtures;
+pub(crate) mod fixtures;
 mod scripting;
 
 pub(crate) fn exec(
@@ -169,55 +169,49 @@ fn is_css_in_elm_stylesheet(tipe: &elmi::Type) -> bool {
     false
 }
 
-mod runtime {
+pub(crate) mod runtime {
     use crate::reporting::InterpreterError;
-    use deno_core::error::{type_error, AnyError};
+    use deno_core::error::AnyError;
-    use deno_core::futures::FutureExt;
+    use deno_core::{Extension, FsModuleLoader, ModuleSpecifier};
-    use deno_core::{resolve_url, Extension, FsModuleLoader, ModuleLoader, ModuleSpecifier};
-    use deno_runtime::deno_broadcast_channel::InMemoryBroadcastChannel;
     use deno_runtime::permissions::PermissionsContainer;
     use deno_runtime::worker::MainWorker;
     use deno_runtime::worker::WorkerOptions;
-    use deno_runtime::BootstrapOptions;
+    //use deno_runtime::BootstrapOptions;
-    use deno_web::BlobStore;
-    use std::pin::Pin;
     use std::rc::Rc;
-    use std::sync::Arc;
     use tracing::{info_span, Instrument};
 
-    fn get_error_class_name(e: &AnyError) -> &'static str {
-        deno_runtime::errors::get_error_class_name(e).unwrap_or("Error")
-    }
-
     pub fn setup_worker(
         extensions: Vec<Extension>,
         path_str: &str,
     ) -> Result<(MainWorker, ModuleSpecifier), AnyError> {
-        let module_loader = Rc::new(FsModuleLoader);
+        //let module_loader = Rc::new(FsModuleLoader);
 
         let options = WorkerOptions {
-            bootstrap: BootstrapOptions {
+            //module_loader: Rc::new(FsModuleLoader),
-                args: vec![],
+            //bootstrap: BootstrapOptions {
-                cpu_count: 1,
+            //    args: vec![],
-                enable_testing_features: false,
+            //    cpu_count: 1,
-                location: None,
+            //    enable_testing_features: false,
-                no_color: false,
+            //    location: None,
-                is_tty: false,
+            //    no_color: false,
-                runtime_version: "0.127.0".to_string(),
+            //    is_tty: false,
-                ts_version: "2.0.0".to_string(),
+            //    runtime_version: "0.127.0".to_string(),
-                unstable: false,
+            //    ts_version: "2.0.0".to_string(),
-                user_agent: "starmelon".to_string(),
+            //    unstable: false,
-                ..Default::default()
+            //    user_agent: "starmelon".to_string(),
-            },
+            //    ..Default::default()
+            //},
             extensions,
+            //should_break_on_first_statement: false,
+            //should_wait_for_inspector_session: false,
             ..Default::default()
         };
 
         let main_module = ModuleSpecifier::from_file_path(path_str)
-            .map_err(|_| AnyError::msg("path is not absolute"))?;
+            .map_err(|()| AnyError::msg("path is not absolute"))?;
         let permissions = PermissionsContainer::allow_all();
 
-        let worker = MainWorker::from_options(main_module.clone(), permissions, options);
+        let worker = MainWorker::bootstrap_from_options(main_module.clone(), permissions, options);
         //worker.bootstrap(&options);
 
         Ok((worker, main_module))
@@ -231,12 +225,13 @@ mod runtime {
     where
         F: FnOnce(deno_core::v8::HandleScope) -> Result<(), InterpreterError>,
     {
-        let wait_for_inspector = false;
         // step 10 load the module into our v8 isolate
         worker
             .execute_main_module(&main_module)
             .instrument(info_span!("execute main module"))
             .await?;
+
+        let wait_for_inspector = false;
         worker
             .run_event_loop(wait_for_inspector)
             .instrument(info_span!("run v8 event loop"))

src/exec/scripting.rs

@@ -1,6 +1,6 @@
 use crate::exec::{fixtures, runtime};
 use crate::reporting::{CompilerError, InterpreterError, Problem, TypeError};
-use deno_core::{Extension, OpState};
+use deno_core::{Extension, ExtensionFileSource, OpState};
 use elm_project_utils::{setup_generator_project, ElmResult};
 use os_pipe::dup_stderr;
 use sqlx::sqlite::SqlitePool;
@@ -12,7 +12,6 @@ use std::io::{self, Read, Write};
 use std::path::PathBuf;
 use std::process::{Command, Stdio};
 use std::sync::Arc;
-use tokio;
 use tracing::info_span;
 
 pub(crate) fn run(
@@ -66,6 +65,7 @@ pub(crate) fn run(
 
     command
         .arg("make")
+        .arg("--report=json")
         .arg("--output")
         .arg(&intermediate_file)
         .current_dir(&generator_dir)
@@ -89,10 +89,13 @@ pub(crate) fn run(
     match command.output() {
         Ok(output) => {
             if !output.status.success() {
+                eprintln!("output of elm make: failed building scripting fixture file");
                 return Err(CompilerError::FailedBuildingFixture.into());
             }
+            //command.stdout(Stdio::piped());
         }
-        Err(_) => {
+        Err(err) => {
+            eprintln!("got this kind of failure {:?}", err);
             return Err(CompilerError::FailedBuildingFixture.into());
         }
     }
@@ -158,9 +161,8 @@ pub(crate) fn run(
             // final_script.replace("var $author$project$Astrid$Query$run = ", "JSON.stringify(x)");
             final_script.push_str("\n\n");
             //final_script.push_str(r#"
-            //    Deno.core.print(JSON.stringify(
+            //    Deno[Deno.internal].core.print(JSON.stringify(
-            //        Deno.core.opSync(
+            //        Extension.starmelon_batch_queries(
-            //            'op_starmelon_batch_queries',
             //            [ [true, "select json_object('id', id, 'foo', foo) from foobar", []]
             //            , [false, "select json_object('id', id, 'foo', foo) from foobar", []]
             //            ]
@@ -170,17 +172,6 @@ pub(crate) fn run(
         }
 
         final_script.push_str("\n\n");
-        // I think that when I set this script to be the main module, I am skipping the
-        // deno/runtime/js/99_main.js script that sets up a bunch of global variables. If I
-        // manually add the timer related code below then setTimeout works again.
-        // NB. there are 706 lines of setup code that add a bunch of apis to the global window
-        // scope. Figure out if I need to include all of them. For example, starmelon does not need
-        // to perform http calls right now, but I eventually want to.
-        final_script.push_str("const { setTimeout } = globalThis.__bootstrap.timers;\n");
-        final_script.push_str(
-            "Deno.core.setMacrotaskCallback(globalThis.__bootstrap.timers.handleTimerMacrotask);\n",
-        );
-        final_script.push_str("globalThis.setTimeout = setTimeout;\n");
 
         final_script.push_str(&format!("var worker = Elm.{}.init();\n", &gen_module_name));
         // add a shortcut for invoking the function so I don't have to traverse so many object
@@ -216,11 +207,12 @@ pub(crate) fn run(
                 final_script.push_str(
                     r#"
                     worker.ports.onOutput.subscribe(function(output){
+                        console.log("got output value")
                         if (output.ctor === "Ok") {
                             const json = JSON.stringify(output.a);
-                            Deno.core.opSync('op_starmelon_string_output', output);
+                            Extension.starmelon_string_output(output);
                         } else {
-                            Deno.core.opSync('op_starmelon_problem', output.a);
+                            Extension.starmelon_problem(output.a);
                         }
                     });
                 "#,
@@ -231,11 +223,12 @@ pub(crate) fn run(
                     r#"
                     // Elm will send a DataView
                     worker.ports.onOutput.subscribe(function(output){
+                        console.log("got output bytes")
                         if (output.ctor === "Ok") {
                             const ui8 = new Uint8Array(output.a.buffer);
-                            Deno.core.opSync('op_starmelon_bytes_output', ui8);
+                            Extension.starmelon_bytes_output(ui8);
                         } else {
-                            Deno.core.opSync('op_starmelon_problem', output.a);
+                            Extension.starmelon_problem(output.a);
                         }
                     });
                 "#,
@@ -244,12 +237,10 @@ pub(crate) fn run(
             OutputType::String | OutputType::Html => {
                 final_script.push_str(
                     r#"
-                    worker.ports.onOutput.subscribe(function(output){
+                    worker.ports.onOutput.subscribe(function outputCallback(output){
-                        if (output.ctor === "Ok") {
+                        console.log("calling ports.onOutput callback for HTML", output)
-                            Deno.core.opSync('op_starmelon_string_output', output.a);
+                        // type ElmResult = Ok string | Err string
-                        } else {
+                        Extension.starmelon_string_output(output);
-                            Deno.core.opSync('op_starmelon_problem', output.a);
-                        }
                     });
                 "#,
                 );
@@ -268,6 +259,16 @@ pub(crate) fn run(
         .map_err(|io_err| CompilerError::WriteOutputFailed(io_err, final_file.clone()))?;
     drop(timing_guard);
 
+    // for debugging purposes we have the option to pause here
+    //if true {
+    //    let mut line = String::new();
+    //    eprintln!(
+    //        "Paused after writing `{:?}`. Continue ?",
+    //        final_file.to_string_lossy()
+    //    );
+    //    std::io::stdin().read_line(&mut line).unwrap();
+    //}
+
     // Create a tokio runtime before registering ops so we can block on futures inside sync ops
     let span = info_span!("create tokio runtime");
     let timing_guard = span.enter();
@@ -286,6 +287,7 @@ pub(crate) fn run(
     let mailbox_clone = Arc::clone(&mailbox);
 
     let mut extensions = vec![Extension {
+        name: "io",
         ops: Cow::Owned(vec![
             op_starmelon_bytes_output::decl(),
             op_starmelon_string_output::decl(),
@@ -294,6 +296,36 @@ pub(crate) fn run(
         op_state_fn: Some(Box::new(move |state| {
             state.put(Arc::clone(&mailbox_clone));
         })),
+        esm_files: {
+            const JS: &'static [ExtensionFileSource] = &[ExtensionFileSource::new(
+                "ext:io/bootstrap.js",
+                {
+                    const STR: ::deno_core::v8::OneByteConst = ::deno_core::FastStaticString::create_external_onebyte_const(
+                        r#"
+import { op_starmelon_bytes_output, op_starmelon_string_output, op_starmelon_problem } from "ext:core/ops";
+function starmelon_bytes_output(msg) {
+    return op_starmelon_bytes_output(msg);
+}
+function starmelon_string_output(msg) {
+    return op_starmelon_string_output(msg);
+}
+function starmelon_problem(msg) {
+    return op_starmelon_problem(msg);
+}
+globalThis.Extension = Object.assign(
+    globalThis.Extension || {},
+    { starmelon_bytes_output, starmelon_string_output, starmelon_problem }
+);
+"#
+                                    .as_bytes(),
+                            );
+                    let s: &'static ::deno_core::v8::OneByteConst = &STR;
+                    ::deno_core::FastStaticString::new(s)
+                },
+            )];
+            Cow::Borrowed(JS)
+        },
+        esm_entry_point: Some("ext:io/bootstrap.js"),
         ..Default::default()
     }];
 
@@ -309,6 +341,7 @@ pub(crate) fn run(
             .unwrap();
 
         if let Ok((extension, thread_handle)) = fixtures::sqlite::init(db_pool) {
+            eprintln!("started sqlite extension");
             extensions.push(extension);
             Some(thread_handle)
         } else {
@@ -364,7 +397,7 @@ pub(crate) fn run(
         }
     };
 
-    let foo = move |mut scope: deno_core::v8::HandleScope| -> Result<(), InterpreterError> {
+    let entrypoint = move |mut scope: deno_core::v8::HandleScope| -> Result<(), InterpreterError> {
         let scope = &mut scope;
         let ctx = scope.get_current_context();
         let global = ctx.global(scope);
@@ -429,7 +462,7 @@ pub(crate) fn run(
 
     let span = info_span!("eval javascript");
     let timing_guard = span.enter();
-    sys.block_on(async move { runtime::xyz(worker, main_module, foo).await })?;
+    sys.block_on(async move { runtime::xyz(worker, main_module, entrypoint).await })?;
     drop(timing_guard);
 
     // step 13 receive the callback
@@ -467,10 +500,10 @@ pub(crate) fn run(
 
 type OutputMailbox = Arc<RefCell<Option<Result<Vec<u8>, String>>>>;
 
-#[deno_core::op]
+#[deno_core::op2(fast)]
 fn op_starmelon_problem(
-    state: &mut deno_core::OpState,
+    state: &mut OpState,
-    msg: String,
+    #[string] msg: String,
 ) -> Result<(), deno_core::error::AnyError> {
     eprintln!("got problem from v8 runtime {:?}", &msg);
     let mailbox_clone = state.borrow::<OutputMailbox>();
@@ -480,10 +513,10 @@ fn op_starmelon_problem(
     Ok(())
 }
 
-#[deno_core::op]
+#[deno_core::op2]
 fn op_starmelon_bytes_output(
     state: &mut OpState,
-    msg: ElmResult<deno_core::JsBuffer, String>,
+    #[serde] msg: ElmResult<deno_core::JsBuffer, String>,
 ) -> Result<(), deno_core::error::AnyError> {
     let mailbox_clone = state.borrow::<OutputMailbox>();
     if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() {
@@ -500,10 +533,10 @@ fn op_starmelon_bytes_output(
     Ok(())
 }
 
-#[deno_core::op]
+#[deno_core::op2]
 fn op_starmelon_string_output(
     state: &mut OpState,
-    msg: ElmResult<String, String>,
+    #[serde] msg: ElmResult<String, String>,
 ) -> Result<(), deno_core::error::AnyError> {
     let mailbox_clone = state.borrow::<OutputMailbox>();
     if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() {

src/main.rs

@@ -5,9 +5,10 @@ use std::hash::Hasher;
 use std::io::{self, Write};
 use std::path::PathBuf;
 use std::time::Instant;
-use structopt::StructOpt;
 use tracing::info_span;
+use clap::Parser;
 
+mod debug;
 mod derive;
 mod elm;
 mod exec;
@@ -16,12 +17,46 @@ mod timings;
 mod transpile;
 
 fn main() {
-    let args = Arguments::from_args();
+    let args = Arguments::parse();
 
     let (span_subscriber, timing_report) = Timings::new();
     tracing::subscriber::set_global_default(span_subscriber).unwrap();
 
     match args {
+        Arguments::Clean => {
+            let result = (|| -> Result<(), crate::reporting::Problem> {
+                let elm_project_dir = crate::elm::find_project_root("elm.json", "./")
+                    .map_err(crate::reporting::CompilerError::MissingElmJson)?;
+                for our_temp_dir in [
+                    elm_project_dir.join("elm-stuff").join("starmelon-ce7993"),
+                    elm_project_dir.join("elm-stuff").join("starmelon-5d9ecc")
+                ] {
+                    for entry in walkdir::WalkDir::new(our_temp_dir).contents_first(true) {
+                        if entry.is_err() {
+                            eprintln!("{:?}", entry);
+                            continue;
+                        }
+                        let entry = entry.unwrap();
+                        // TODO implement an error message for failure to delete a file during clean
+                        if entry.file_type().is_file() {
+                            let _ = std::fs::remove_file(entry.path());
+                        }
+                        if entry.file_type().is_dir() {
+                            let _ = std::fs::remove_dir(entry.path());
+                        }
+                        eprintln!("{}", entry.path().display());
+                    }
+                }
+
+                Ok(())
+            })();
+            if let Err(problem) = result {
+                let span = info_span!("pretty print problem");
+                let timing_guard = span.enter();
+                eprintln!("{}", pretty(80, problem.to_doc()));
+                drop(timing_guard);
+            }
+        }
         Arguments::Exec {
             file,
             debug,
@@ -35,7 +70,7 @@ fn main() {
             let start = Instant::now();
             let span = info_span!("exec");
             let timing_guard = span.enter();
-            let result = exec::exec(file, debug, function, input, output, verbosity, sqlite);
+            let result = exec::exec(file, debug, function, input, output, verbosity as u64, sqlite);
             drop(timing_guard);
             if let Err(problem) = result {
                 let span = info_span!("pretty print problem");
@@ -69,8 +104,9 @@ fn main() {
             verbosity,
             debug,
         } => {
-            transpile::transpile(file, debug, function, verbosity).unwrap();
+            transpile::transpile(file, debug, function, verbosity.into()).unwrap();
         }
+        Arguments::Derive(DeriveMacros::Other) => (),
         Arguments::Derive(DeriveMacros::LiveTable {
             file,
             debug,
@@ -82,7 +118,7 @@ fn main() {
             let span = info_span!("derive livetable editor");
             let timing_guard = span.enter();
             let result =
-                derive::derive_livetable(file, debug, output, verbosity /*, sqlite */);
+                derive::derive_livetable(file, debug, output, verbosity.into() /*, sqlite */);
             drop(timing_guard);
             if let Err(problem) = result {
                 let span = info_span!("pretty print problem");
@@ -110,6 +146,40 @@ fn main() {
                 Instant::now() - start
             );
         }
+        Arguments::Debug { javascript, timings } => {
+            let start = Instant::now();
+
+            let span = info_span!("debug javascript");
+            let timing_guard = span.enter();
+            let result = crate::debug::run(javascript);
+
+            if let Err(problem) = result {
+                let span = info_span!("pretty print problem");
+                let timing_guard = span.enter();
+                eprintln!("{}", pretty(80, problem.to_doc()));
+                drop(timing_guard);
+            }
+            {
+                let stdout = io::stdout();
+                let mut handle = stdout.lock();
+                handle.flush().unwrap();
+            }
+
+            if timings {
+                let mut report = timing_report.dump().unwrap();
+                report.sort_by(|(_, a), (_, b)| b.partial_cmp(a).unwrap());
+
+                for (step, duration) in report.iter() {
+                    eprintln!("[{:>10.3?}] {}", duration, step);
+                }
+            }
+            eprintln!(
+                "\t\x1b[1;92mFinished\x1b[0m [{}] in {:?}",
+                "unoptimized",
+                Instant::now() - start
+            );
+
+        }
     }
 }
 
@@ -117,8 +187,9 @@ struct PortableHash(ahash::AHasher);
 
 impl PortableHash {
     fn new() -> Self {
+        use std::hash::BuildHasher;
         // We need constant keys to get the same checksup every time we run the program.
-        Self(ahash::AHasher::new_with_keys(1, 2))
+        Self(ahash::RandomState::with_seeds(1, 2, 3, 4).build_hasher())
     }
 }
 
@@ -143,53 +214,59 @@ impl ::core::hash::Hasher for PortableHash {
     }
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, clap::Parser)]
 enum Arguments {
+    Clean,
     Exec {
-        #[structopt(parse(from_os_str))]
         file: PathBuf,
         function: String,
-        #[structopt(long)]
+        #[arg(long)]
         debug: bool,
-        #[structopt(long)]
+        #[arg(long)]
         input: Option<PathBuf>,
-        #[structopt(long)]
+        #[arg(long)]
         output: Option<PathBuf>,
-        #[structopt(short = "v", parse(from_occurrences))]
+        #[arg(short = 'v', action(clap::ArgAction::Count))]
-        verbosity: u64,
+        verbosity: u8,
-        #[structopt(long)]
+        #[arg(long)]
         sqlite: Option<PathBuf>,
-        #[structopt(long)]
+        #[arg(long)]
         timings: bool,
     },
     Transpile {
-        #[structopt(parse(from_os_str))]
         file: PathBuf,
         function: String,
-        #[structopt(short = "v", parse(from_occurrences))]
+        #[arg(short = 'v', action(clap::ArgAction::Count))]
-        verbosity: u64,
+        verbosity: u8,
-        #[structopt(long)]
+        #[arg(long)]
         debug: bool,
     },
-    #[structopt(name = "derive")]
+    #[command(name = "derive", subcommand)]
     Derive(DeriveMacros),
+    Debug {
+        #[arg(long = "js")]
+        javascript: Option<PathBuf>,
+        #[arg(long)]
+        timings: bool,
+    }
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, clap::Parser)]
 pub enum DeriveMacros {
-    #[structopt(name = "livetable")]
+    #[command(name = "livetable")]
     LiveTable {
-        #[structopt(parse(from_os_str))]
         file: PathBuf,
-        #[structopt(short = "v", parse(from_occurrences))]
+        #[arg(short = 'v', action(clap::ArgAction::Count))]
-        verbosity: u64,
+        verbosity: u8,
-        #[structopt(long)]
+        #[arg(long)]
         debug: bool,
-        #[structopt(long)]
+        #[arg(long)]
         output: Option<PathBuf>,
-        #[structopt(long)]
+        #[arg(long)]
         timings: bool,
     },
+    #[command(subcommand)]
+    Other,
 }
 
 pub mod generated {

src/timings.rs

@@ -1,8 +1,8 @@
 //! This module provides a crude system for timing how long various steps of the program take to
-//! execute using the tracing framework. I seek absolution for suboptimal code. For example I want
+//! execute using the tracing framework. My goal is to give myself permission to write suboptimal
-//! to throw a span around all clones and see exactly how long I spend copying data. If I see that
+//! code. For example I want to throw a span around all clones and see exactly how long I spend
-//! I only spend a few milliseconds overall then that could be a price worth paying for development
+//! copying data. If I see that I only spend a few milliseconds overall then that could be a price
-//! velocity.
+//! worth paying for development velocity.
 //!
 //! This is quite a different use case then flame graphs.
 use rustc_hash::FxHashMap;
@@ -16,7 +16,6 @@ use std::time::{Duration, Instant};
 use tracing::span::{Attributes, Record};
 use tracing::{Event, Id, Metadata};
 
-//
 // Each step will be described with a less than 100 char string. Steps will overlap.
 //
 // I am not tracking the relationship between

src/transpile.rs

@@ -402,45 +402,45 @@ mod rust_codegen {
         body: &elmi::Expr,
     ) {
         quote_in! { *tokens =>
-            fn #name#(if !type_variables.is_empty() =>
+            fn $name$(if !type_variables.is_empty() =>
-                <#(for tvar in type_variables.iter() join (, ) =>
+                <$(for tvar in type_variables.iter() join (, ) =>
-                    #tvar
+                    $tvar
                 )>
-            )(#(for (parameter, tipe) in parameters.iter() join (, ) =>
+            )($(for (parameter, tipe) in parameters.iter() join (, ) =>
-                #parameter: #(ref out { codegen_type(out, tipe) })
+                $parameter: $(ref out { codegen_type(out, tipe) })
-            )) -> #(ref out { codegen_type(out, &return_type) }) {
+            )) -> $(ref out { codegen_type(out, &return_type) }) {
-                #(ref out { codegen_expr(out, symbol_table, body) })
+                $(ref out { codegen_expr(out, symbol_table, body) })
             }
         }
     }
 
     fn codegen_type(tokens: &mut rust::Tokens, tipe: &elmi::Type) {
         quote_in! { *tokens =>
-            #(match tipe {
+            $(match tipe {
                 elmi::Type::TLambda(a, b) => {
-                    ( #(ref out => codegen_type(out, a) ) -> #(ref out => codegen_type(out, b) ) )
+                    ( $(ref out => codegen_type(out, a) ) -> $(ref out => codegen_type(out, b) ) )
                 }
                 elmi::Type::TVar(elmi::Name(variable)) => {
-                    #variable
+                    $variable
                 },
                 elmi::Type::TType(module_name, name, args) if module_name == "elm/core/String" && name == "String" && args.is_empty() => {
                     String
                 }
                 elmi::Type::TType(home, name, args) if args.is_empty() => {
-                    #(ref out => codegen_name_from_global(out, home, name))
+                    $(ref out => codegen_name_from_global(out, home, name))
                 }
                 elmi::Type::TType(home, name, args) => {
-                    #(ref out => codegen_name_from_global(out, home, name))<#(for arg in args join(, ) =>
+                    $(ref out => codegen_name_from_global(out, home, name))<$(for arg in args join(, ) =>
-                        #(ref out => codegen_type(out, arg))
+                        $(ref out => codegen_type(out, arg))
                     )>
                 }
                 //    // Might be a primitive type
-                //    #(if module_name == "elm/core/String" && name == "String" => String)
+                //    $(if module_name == "elm/core/String" && name == "String" => String)
-                //    #(if module_name == "elm/core/Basics" && name == "Int" => i64)
+                //    $(if module_name == "elm/core/Basics" && name == "Int" => i64)
-                //    #(if module_name == "elm/core/Basics" && name == "Float" => f64)
+                //    $(if module_name == "elm/core/Basics" && name == "Float" => f64)
-                //    #(if module_name == "elm/core/Basics" && name == "Bool" => bool)
+                //    $(if module_name == "elm/core/Basics" && name == "Bool" => bool)
-                //    #(if module_name == "elm/core/Maybe" && name == "Maybe" => Option<i32>)
+                //    $(if module_name == "elm/core/Maybe" && name == "Maybe" => Option<i32>)
-                //    #(if module_name == "elm/bytes/Bytes" && name == "Bytes" => Vec<u8>)
+                //    $(if module_name == "elm/bytes/Bytes" && name == "Bytes" => Vec<u8>)
                 //}
                 //elmi::Type::TType(_, _, _) => Err(TypeError::CantEvalCustomType),
                 //elmi::Type::TRecord(_, _) => Err(TypeError::CantEvalRecord),
@@ -479,7 +479,7 @@ mod rust_codegen {
         name: &elmi::Name,
     ) {
         quote_in! { *tokens =>
-            #(ref out => codegen_home_to_builder(out, home) )__#name
+            $(ref out => codegen_home_to_builder(out, home) )__$name
         }
     }
 
@@ -490,7 +490,7 @@ mod rust_codegen {
         } = global;
 
         quote_in! { *tokens =>
-            _#(author.replace("-", "_"))_#(project.replace("-", "_"))__#(home.0.replace(".", "_"))
+            _$(author.replace("-", "_"))_$(project.replace("-", "_"))__$(home.0.replace(".", "_"))
         }
     }
 
@@ -498,18 +498,18 @@ mod rust_codegen {
         match expr {
             elmi::Expr::Bool(true) => quote_in! { *tokens => true },
             elmi::Expr::Bool(false) => quote_in! { *tokens => false },
-            elmi::Expr::Chr(c) => quote_in! { *tokens => #("'")#c#("'") },
+            elmi::Expr::Chr(c) => quote_in! { *tokens => $("'")$c$("'") },
-            elmi::Expr::Str(s) => quote_in! { *tokens => #(quoted(s)) },
+            elmi::Expr::Str(s) => quote_in! { *tokens => $(quoted(s)) },
-            elmi::Expr::Int(x) => quote_in! { *tokens => #(x.to_string()) },
+            elmi::Expr::Int(x) => quote_in! { *tokens => $(x.to_string()) },
-            elmi::Expr::Float(x) => quote_in! { *tokens => #(x.to_string()) },
+            elmi::Expr::Float(x) => quote_in! { *tokens => $(x.to_string()) },
             elmi::Expr::VarLocal(name) => {
                 quote_in! { *tokens =>
-                    #name
+                    $name
                 }
             }
             elmi::Expr::VarGlobal(elmi::Global(home, name)) => {
                 quote_in! { *tokens =>
-                    #(ref out => codegen_name_from_global(out, home, name))
+                    $(ref out => codegen_name_from_global(out, home, name))
                 }
             }
             //elmi::Expr::VarEnum(Global, IndexZeroBased),
@@ -523,7 +523,7 @@ mod rust_codegen {
                 } else {
                     quote_in! { *tokens =>
                         &[
-                            #(for x in xs join (,#<push>) => #(ref out => codegen_expr(out, symbol_table, x) ) )
+                            $(for x in xs join (,$<push>) => $(ref out => codegen_expr(out, symbol_table, x) ) )
                         ]
                     }
                 }
@@ -531,7 +531,7 @@ mod rust_codegen {
             elmi::Expr::Function(_parameters, _body) => {
                 quote_in! { *tokens =>
                     "i don't know how to code gen a function expression"
-                    //#(for elmi::Name(ref parameter) in parameters.iter() join (, ) =>
+                    //$(for elmi::Name(ref parameter) in parameters.iter() join (, ) =>
                     //)
                 }
             }
@@ -548,11 +548,11 @@ mod rust_codegen {
                                         )));
                                     }
                                     quote_in! { *tokens =>
-                                        Box::new(| #(for arg in closure_args.iter() join (, ) => #(ref out => codegen_expr(out, symbol_table, arg))) | {
+                                        Box::new(| $(for arg in closure_args.iter() join (, ) => $(ref out => codegen_expr(out, symbol_table, arg))) | {
-                                            #(ref out => {
+                                            $(ref out => {
                                                 codegen_name_from_global(out, home, name)
                                             })(
-                                                #(for arg in args.iter().chain(closure_args.iter()) join (,#<push>) => #(ref out =>
+                                                $(for arg in args.iter().chain(closure_args.iter()) join (,$<push>) => $(ref out =>
                                                     codegen_expr(out, symbol_table, arg) )
                                                 )
                                             )
@@ -560,10 +560,10 @@ mod rust_codegen {
                                     }
                                 } else {
                                     quote_in! { *tokens =>
-                                        #(ref out => {
+                                        $(ref out => {
                                             codegen_name_from_global(out, home, name)
                                         })(
-                                            #(for arg in args join (,#<push>) => #(ref out =>
+                                            $(for arg in args join (,$<push>) => $(ref out =>
                                                 codegen_expr(out, symbol_table, arg) )
                                             )
                                         )
@@ -581,8 +581,8 @@ mod rust_codegen {
                     }
                     elmi::Expr::VarLocal(name) => {
                         quote_in! { *tokens =>
-                            #name(
+                            $name(
-                                #(for arg in args join (,#<push>) => #(ref out =>
+                                $(for arg in args join (,$<push>) => $(ref out =>
                                     codegen_expr(out, symbol_table, arg) )
                                 )
                             )
@@ -593,7 +593,7 @@ mod rust_codegen {
                         // TODO write a function that can take an expression and return the arity using
                         // the symbol table from the bottom up.
                         quote_in! { *tokens =>
-                            #(format!("{:?}", fexpr))
+                            $(format!("{:?}", fexpr))
                         }
                         //panic!("calling an expression not yet supported");
                     }
@@ -602,19 +602,19 @@ mod rust_codegen {
             //elmi::Expr::TailCall(Name, Vec<(Name, Expr)>),
             elmi::Expr::If(branches, _final_branch) => {
                 quote_in! { *tokens =>
-                    #(for (condition, expr) in branches join (#<push>#("} else")) =>
+                    $(for (condition, expr) in branches join ($<push>$("} else")) =>
-                        if #(ref out => codegen_expr(out, symbol_table, condition)) #("{")
+                        if $(ref out => codegen_expr(out, symbol_table, condition)) $("{")
-                            #(ref out => codegen_expr(out, symbol_table, expr))
+                            $(ref out => codegen_expr(out, symbol_table, expr))
-                    ) #("} else {")
+                    ) $("} else {")
-                        #(ref out => codegen_expr(out, symbol_table, expr))
+                        $(ref out => codegen_expr(out, symbol_table, expr))
-                    #("}")
+                    $("}")
                 }
             }
             elmi::Expr::Let(def, expr) => {
                 quote_in! { *tokens =>
-                    #(ref out => codegen_def(out, symbol_table, def))
+                    $(ref out => codegen_def(out, symbol_table, def))
-                    #<push>
+                    $<push>
-                    #(ref out => codegen_expr(out, symbol_table, expr))
+                    $(ref out => codegen_expr(out, symbol_table, expr))
                 }
             }
             //elmi::Expr::Destruct(Destructor, Box<Expr>),
@@ -624,7 +624,7 @@ mod rust_codegen {
             }
             elmi::Expr::Accessor(name) => {
                 quote_in! { *tokens =>
-                    Box::new(|_v| { _v.#name })
+                    Box::new(|_v| { _v.$name })
                 }
             }
             //elmi::Expr::Access(Box<Expr>, Name),
@@ -633,16 +633,16 @@ mod rust_codegen {
             elmi::Expr::Unit => (),
             elmi::Expr::Tuple(a, b, None) => {
                 quote_in! { *tokens =>
-                    ( #(ref out => codegen_expr(out, symbol_table, a) ), #(ref out => codegen_expr(out, symbol_table, b) ) )
+                    ( $(ref out => codegen_expr(out, symbol_table, a) ), $(ref out => codegen_expr(out, symbol_table, b) ) )
                 }
             }
             elmi::Expr::Tuple(a, b, Some(c)) => {
                 quote_in! { *tokens =>
-                    ( #(ref out => codegen_expr(out, symbol_table, a) ), #(ref out => codegen_expr(out, symbol_table, b) ), #(ref out => codegen_expr(out, symbol_table, c) ) )
+                    ( $(ref out => codegen_expr(out, symbol_table, a) ), $(ref out => codegen_expr(out, symbol_table, b) ), $(ref out => codegen_expr(out, symbol_table, c) ) )
                 }
             }
             //elmi::Expr::Shader(ShaderSource, HashSet<Name>, HashSet<Name>),
-            _ => quote_in! { *tokens => #(format!("{:?}", expr)) },
+            _ => quote_in! { *tokens => $(format!("{:?}", expr)) },
         }
     }
 
@@ -650,14 +650,14 @@ mod rust_codegen {
         match def {
             elmi::Def::Def(name, expr) => {
                 quote_in! { *tokens =>
-                    let #name = #(ref out => codegen_expr(out, symbol_table, expr) );
+                    let $name = $(ref out => codegen_expr(out, symbol_table, expr) );
                 }
             }
             elmi::Def::TailDef(name, arg_names, expr) => {
                 quote_in! { *tokens =>
-                    |#(for arg in arg_names join (, ) => mut #arg) | {
+                    |$(for arg in arg_names join (, ) => mut $arg) | {
-                        #("'")#name : loop {
+                        $("'")$name : loop {
-                            #(ref out => codegen_expr(out, symbol_table, expr))
+                            $(ref out => codegen_expr(out, symbol_table, expr))
                         }
                     }
                 }
@@ -736,10 +736,10 @@ mod lua_codegen {
         body: &elmi::Expr,
     ) {
         quote_in! { *tokens =>
-            function #name(#(for (parameter, _tipe) in parameters.iter() join (, ) =>
+            function $name($(for (parameter, _tipe) in parameters.iter() join (, ) =>
-                #parameter
+                $parameter
             ))
-                #(ref out { codegen_expr(out, symbol_table, body) })
+                $(ref out { codegen_expr(out, symbol_table, body) })
             end
         }
     }
@@ -750,7 +750,7 @@ mod lua_codegen {
         name: &elmi::Name,
     ) {
         quote_in! { *tokens =>
-            #(ref out => codegen_home_to_builder(out, home) )__#name
+            $(ref out => codegen_home_to_builder(out, home) )__$name
         }
     }
 
@@ -761,7 +761,7 @@ mod lua_codegen {
         } = global;
 
         quote_in! { *tokens =>
-            _#(author.replace("-", "_"))_#(project.replace("-", "_"))__#(home.0.replace(".", "_"))
+            _$(author.replace("-", "_"))_$(project.replace("-", "_"))__$(home.0.replace(".", "_"))
         }
     }
 
@@ -769,18 +769,18 @@ mod lua_codegen {
         match expr {
             elmi::Expr::Bool(true) => quote_in! { *tokens => true },
             elmi::Expr::Bool(false) => quote_in! { *tokens => false },
-            elmi::Expr::Chr(c) => quote_in! { *tokens => #(quoted(c)) },
+            elmi::Expr::Chr(c) => quote_in! { *tokens => $(quoted(c)) },
-            elmi::Expr::Str(s) => quote_in! { *tokens => #(quoted(s)) },
+            elmi::Expr::Str(s) => quote_in! { *tokens => $(quoted(s)) },
-            elmi::Expr::Int(x) => quote_in! { *tokens => #(x.to_string()) },
+            elmi::Expr::Int(x) => quote_in! { *tokens => $(x.to_string()) },
-            elmi::Expr::Float(x) => quote_in! { *tokens => #(x.to_string()) },
+            elmi::Expr::Float(x) => quote_in! { *tokens => $(x.to_string()) },
             elmi::Expr::VarLocal(name) => {
                 quote_in! { *tokens =>
-                    #name
+                    $name
                 }
             }
             elmi::Expr::VarGlobal(elmi::Global(home, name)) => {
                 quote_in! { *tokens =>
-                    #(ref out => codegen_name_from_global(out, home, name))
+                    $(ref out => codegen_name_from_global(out, home, name))
                 }
             }
             //elmi::Expr::VarEnum(Global, IndexZeroBased),
@@ -794,7 +794,7 @@ mod lua_codegen {
                 } else {
                     quote_in! { *tokens =>
                         &[
-                            #(for x in xs join (,#<push>) => #(ref out => codegen_expr(out, symbol_table, x) ) )
+                            $(for x in xs join (,$<push>) => $(ref out => codegen_expr(out, symbol_table, x) ) )
                         ]
                     }
                 }
@@ -802,7 +802,7 @@ mod lua_codegen {
             elmi::Expr::Function(_parameters, _body) => {
                 quote_in! { *tokens =>
                     "i don't know how to code gen a function expression"
-                    //#(for elmi::Name(ref parameter) in parameters.iter() join (, ) =>
+                    //$(for elmi::Name(ref parameter) in parameters.iter() join (, ) =>
                     //)
                 }
             }
@@ -819,11 +819,11 @@ mod lua_codegen {
                                         )));
                                     }
                                     quote_in! { *tokens =>
-                                        Box::new(| #(for arg in closure_args.iter() join (, ) => #(ref out => codegen_expr(out, symbol_table, arg))) | {
+                                        Box::new(| $(for arg in closure_args.iter() join (, ) => $(ref out => codegen_expr(out, symbol_table, arg))) | {
-                                            #(ref out => {
+                                            $(ref out => {
                                                 codegen_name_from_global(out, home, name)
                                             })(
-                                                #(for arg in args.iter().chain(closure_args.iter()) join (,#<push>) => #(ref out =>
+                                                $(for arg in args.iter().chain(closure_args.iter()) join (,$<push>) => $(ref out =>
                                                     codegen_expr(out, symbol_table, arg) )
                                                 )
                                             )
@@ -831,10 +831,10 @@ mod lua_codegen {
                                     }
                                 } else {
                                     quote_in! { *tokens =>
-                                        #(ref out => {
+                                        $(ref out => {
                                             codegen_name_from_global(out, home, name)
                                         })(
-                                            #(for arg in args join (,#<push>) => #(ref out =>
+                                            $(for arg in args join (,$<push>) => $(ref out =>
                                                 codegen_expr(out, symbol_table, arg) )
                                             )
                                         )
@@ -852,8 +852,8 @@ mod lua_codegen {
                     }
                     elmi::Expr::VarLocal(name) => {
                         quote_in! { *tokens =>
-                            #name(
+                            $name(
-                                #(for arg in args join (,#<push>) => #(ref out =>
+                                $(for arg in args join (,$<push>) => $(ref out =>
                                     codegen_expr(out, symbol_table, arg) )
                                 )
                             )
@@ -864,7 +864,7 @@ mod lua_codegen {
                         // TODO write a function that can take an expression and return the arity using
                         // the symbol table from the bottom up.
                         quote_in! { *tokens =>
-                            #(format!("{:?}", fexpr))
+                            $(format!("{:?}", fexpr))
                         }
                         //panic!("calling an expression not yet supported");
                     }
@@ -873,19 +873,19 @@ mod lua_codegen {
             //elmi::Expr::TailCall(Name, Vec<(Name, Expr)>),
             elmi::Expr::If(branches, _final_branch) => {
                 quote_in! { *tokens =>
-                    #(for (condition, expr) in branches join (#<push>#("} else")) =>
+                    $(for (condition, expr) in branches join ($<push>$("} else")) =>
-                        if #(ref out => codegen_expr(out, symbol_table, condition)) #("{")
+                        if $(ref out => codegen_expr(out, symbol_table, condition)) $("{")
-                            #(ref out => codegen_expr(out, symbol_table, expr))
+                            $(ref out => codegen_expr(out, symbol_table, expr))
-                    ) #("} else {")
+                    ) $("} else {")
-                        #(ref out => codegen_expr(out, symbol_table, expr))
+                        $(ref out => codegen_expr(out, symbol_table, expr))
-                    #("}")
+                    $("}")
                 }
             }
             elmi::Expr::Let(def, expr) => {
                 quote_in! { *tokens =>
-                    #(ref out => codegen_def(out, symbol_table, def))
+                    $(ref out => codegen_def(out, symbol_table, def))
-                    #<push>
+                    $<push>
-                    #(ref out => codegen_expr(out, symbol_table, expr))
+                    $(ref out => codegen_expr(out, symbol_table, expr))
                 }
             }
             //elmi::Expr::Destruct(Destructor, Box<Expr>),
@@ -896,7 +896,7 @@ mod lua_codegen {
             }
             elmi::Expr::Accessor(name) => {
                 quote_in! { *tokens =>
-                    Box::new(|_v| { _v.#name })
+                    Box::new(|_v| { _v.$name })
                 }
             }
             //elmi::Expr::Access(Box<Expr>, Name),
@@ -905,16 +905,16 @@ mod lua_codegen {
             elmi::Expr::Unit => (),
             elmi::Expr::Tuple(a, b, None) => {
                 quote_in! { *tokens =>
-                    ( #(ref out => codegen_expr(out, symbol_table, a) ), #(ref out => codegen_expr(out, symbol_table, b) ) )
+                    ( $(ref out => codegen_expr(out, symbol_table, a) ), $(ref out => codegen_expr(out, symbol_table, b) ) )
                 }
             }
             elmi::Expr::Tuple(a, b, Some(c)) => {
                 quote_in! { *tokens =>
-                    ( #(ref out => codegen_expr(out, symbol_table, a) ), #(ref out => codegen_expr(out, symbol_table, b) ), #(ref out => codegen_expr(out, symbol_table, c) ) )
+                    ( $(ref out => codegen_expr(out, symbol_table, a) ), $(ref out => codegen_expr(out, symbol_table, b) ), $(ref out => codegen_expr(out, symbol_table, c) ) )
                 }
             }
             //elmi::Expr::Shader(ShaderSource, HashSet<Name>, HashSet<Name>),
-            _ => quote_in! { *tokens => #(format!("{:?}", expr)) },
+            _ => quote_in! { *tokens => $(format!("{:?}", expr)) },
         }
     }
 
@@ -922,14 +922,14 @@ mod lua_codegen {
         match def {
             elmi::Def::Def(name, expr) => {
                 quote_in! { *tokens =>
-                    local #name = #(ref out => codegen_expr(out, symbol_table, expr) )
+                    local $name = $(ref out => codegen_expr(out, symbol_table, expr) )
                 }
             }
             elmi::Def::TailDef(name, arg_names, expr) => {
                 quote_in! { *tokens =>
-                    |#(for arg in arg_names join (, ) => mut #arg) | {
+                    |$(for arg in arg_names join (, ) => mut $arg) | {
-                        #("'")#name : loop {
+                        $("'")$name : loop {
-                            #(ref out => codegen_expr(out, symbol_table, expr))
+                            $(ref out => codegen_expr(out, symbol_table, expr))
                         }
                     }
                 }

starmelon-binary.do

@@ -1,7 +1,15 @@
-if test -f ../../target/release/starmelon; then
+BINARY_PATH="../../target/release/starmelon"
-    redo-ifchange always-rebuild
+
-    ls -al ../../../target/release/starmelon | redo-stamp
+redo-ifchange Cargo.toml
-else;
+find src/ -type f | xargs redo-ifchange
-    cargo build --release
+find ../../../infra/rust-elmi/src -type f | xargs redo-ifchange
-    ls -al ../../../target/release/starmelon | redo-stamp
+find ../../../infra/redwood-lang/compiler/naive-wadler-prettier/src -type f | xargs redo-ifchange
+find ../../../infra/rust-elm-project-utils/src -type f | xargs redo-ifchange
+find ../../../infra/genco-extra/src -type f | xargs redo-ifchange
+find ../../../infra/livetable/core/src -type f | xargs redo-ifchange
+
+if [ ! -f "$BINARY_PATH" ]; then
+    cargo build --release --color=always
+else
+    ls -al "$BINARY_PATH" | redo-stamp
 fi