feat: upgrade v8 runtime crates

Cargo.toml

@@ -32,13 +32,11 @@ elm-quote = { path = "../../../infra/rust-elm-quote" }
 # All of these are required for deno's javascript runtime. We need to keep the
 # same versions as other projects in our cargo workspace. Multiple different
 # versions of rust_v8 seem to break its build script.
-deno_runtime = "0.29.0"
+deno_runtime = "0.50"
-tokio = { version = "1.6", features = ["full"] }
+tokio = { version = "1.17", features = ["full"] }
-deno_core = "0.103.0"
+deno_core = "0.124"
-deno_web = "0.52"
+deno_web = "0.73"
-rusty_v8 = "0.32"
+futures = "0.3"
-futures = "0.3.15"
-serde_v8 = "0.15"
 
 # Required to add sql query support to interpreter. Because deno expects sync
 # ops to be synchronous, we have to use a second async executor to run the sqlx
@@ -48,5 +46,9 @@ serde_v8 = "0.15"
 sqlx = { version = "0.5", features = [ "sqlite", "macros", "runtime-tokio-rustls", "chrono", "json", "uuid" ] }
 oneshot = "0.1.3"
 
+# required for livetable derive macro
+livetable-core = { path = "../../../infra/livetable/core" }
+
+
 [dev-dependencies]
 aho-corasick = "0.7"

benches/string_replacement.rs

@@ -1,5 +1,3 @@
-use aho_corasick::{AhoCorasick, AhoCorasickBuilder};
-
 // TODO figure out why this code takes ~70ms to munge the javascript. By comparison just writing a
 // bunch of chainged string.replace( , ).replace( , ).replace .... takes about (16ms debug) (2.5ms
 // release).

src/exec/astrid_pages.rs

@@ -1,11 +1,9 @@
 use crate::exec::fixtures::astrid_pages::ScriptError;
 use crate::exec::{fixtures, runtime};
-use crate::reporting::{CompilerError, InterpreterError, Problem, TypeError};
+use crate::reporting::{CompilerError, InterpreterError, Problem};
-use deno_core::futures::StreamExt;
+use deno_core::{v8, Extension, OpState};
 use elm_project_utils::{setup_generator_project, ElmPostProcessor, ElmResult};
 use os_pipe::dup_stderr;
-use rusty_v8 as v8;
-use serde::{Deserialize, Serialize};
 use sqlx::sqlite::SqlitePool;
 use sqlx::Row;
 use std::cell::RefCell;
@@ -16,9 +14,8 @@ use std::io::Write;
 use std::path::PathBuf;
 use std::process::{Command, Stdio};
 use std::sync::Arc;
-use std::time::Instant;
 use tokio;
-use tracing::{info_span, Instrument};
+use tracing::info_span;
 
 #[derive(Debug, Copy, Clone)]
 pub enum OutputType {
@@ -105,10 +102,12 @@ pub(crate) fn run(
     match command.output() {
         Ok(output) => {
             if !output.status.success() {
+                eprintln!("{}", String::from_utf8_lossy(&output.stderr));
                 return Err(CompilerError::FailedBuildingFixture.into());
             }
         }
-        Err(_) => {
+        Err(err) => {
+            eprintln!("{:?}", err);
             return Err(CompilerError::FailedBuildingFixture.into());
         }
     }
@@ -209,7 +208,7 @@ pub(crate) fn run(
     drop(timing_guard);
 
     let desired_route = entrypoint.0.module.clone().to_string();
-    let foo = move |mut scope: rusty_v8::HandleScope| -> Result<(), InterpreterError> {
+    let foo = move |mut scope: deno_core::v8::HandleScope| -> Result<(), InterpreterError> {
         let scope = &mut scope;
         let ctx = scope.get_current_context();
         let global = ctx.global(scope);
@@ -253,55 +252,26 @@ pub(crate) fn run(
         .unwrap();
     drop(timing_guard);
 
-    // step 10 create a v8 isolate. We need to register a different callback depending on
-    // the output type (string or bytes)
-
-    let span = info_span!("create v8 isolate");
-    let timing_guard = span.enter();
-    let (mut worker, main_module) = runtime::setup_worker(&buffer_file.to_string_lossy())
-        .map_err(|err| InterpreterError::EventLoop(err))?;
-    drop(timing_guard);
-
     let span = info_span!("register private api");
     let timing_guard = span.enter();
+
+    // Step 10 setup our custom ops for the javascript runtime
     let mailbox: Arc<RefCell<Option<Result<Vec<u8>, ScriptError>>>> = Arc::new(RefCell::new(None));
-
     let mailbox_clone = Arc::clone(&mailbox);
-    worker.js_runtime.register_op(
-        "op_starmelon_bytes_output",
-        deno_core::op_sync(
-            move |_state, msg: ElmResult<deno_core::ZeroCopyBuf, ScriptError>, _: ()| {
-                if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() {
-                    match msg {
-                        ElmResult::Ok { a: buffer } => {
-                            let slice: &[u8] = &buffer;
-                            mailbox.replace(Ok(slice.to_owned()));
-                        }
-                        ElmResult::Err { a: err } => {
-                            mailbox.replace(Err(err));
-                        }
-                    }
-                }
-                Ok(())
-            },
-        ),
-    );
 
-    let mailbox_clone = Arc::clone(&mailbox);
+    let mut extensions = vec![Extension::builder()
-    worker.js_runtime.register_op(
+        .ops(vec![
-        "op_starmelon_string_output",
+            op_starmelon_bytes_output::decl(),
-        deno_core::op_sync(move |_state, msg: ElmResult<String, ScriptError>, _: ()| {
+            op_starmelon_string_output::decl(),
-            if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() {
+            op_starmelon_problem::decl(),
-                match msg {
+        ])
-                    ElmResult::Ok { a: s } => mailbox.replace(Ok(s.into_bytes())),
+        .state(move |state| {
-                    ElmResult::Err { a: err } => mailbox.replace(Err(err)),
+            state.put(Arc::clone(&mailbox_clone));
-                };
+
-            }
+            Ok(())
-            Ok(())
+        })
-        }),
+        .build()];
-    );
 
-    // Step 10.B  setup the sqlite database feature
     let sql_background_thread_handle = if let Some(database_url) = sqlite_path {
         // I want to construct the connection in the initial thread so I can tell if the connection
         // failed
@@ -309,148 +279,23 @@ pub(crate) fn run(
             .block_on(async { SqlitePool::connect(&database_url.to_string_lossy()).await })
             .unwrap();
 
-        let (worker_mailbox, rx) = std::sync::mpsc::channel::<(
+        if let Ok((extension, thread_handle)) = fixtures::sqlite::init(db_pool) {
-            oneshot::Sender<ElmResult<_, _>>,
+            extensions.push(extension);
-            Vec<(bool, String, Vec<String>)>,
+            Some(thread_handle)
-        )>();
-
-        let sql_worker_thread = std::thread::spawn(move || {
-            let worker = tokio::runtime::Builder::new_current_thread()
-                .enable_all()
-                .build()
-                .unwrap();
-            loop {
-                if let Ok((response, queries)) = rx.recv() {
-                    // I am not sure if I should only work on one database task at a time, or
-                    // submit as many takes as possible.  Just spawning the future onto this
-                    // exectutor does not seem to work, even though the docs say the thread pool
-                    // will poll the future until it completes.
-                    let db_pool_clone = db_pool.clone();
-                    let span = info_span!("inside sql queries futures");
-                    let f = async move {
-                        let _start = Instant::now();
-                        let db_pool = db_pool_clone;
-                        let mut result: Vec<Vec<String>> = vec![];
-                        let mut failure: Option<AstridQueryError> = None;
-                        for (fetch_all, sql, _args) in queries {
-                            let mut acc = Vec::new();
-                            if fetch_all {
-                                let mut stream = sqlx::query(&sql).fetch(&db_pool);
-                                loop {
-                                    match stream.next().await {
-                                        None => break,
-                                        Some(Ok(row)) => {
-                                            use sqlx::ValueRef;
-                                            // If we only have one column and it is string type,
-                                            // then we can try to parse it as json
-                                            if row.len() == 1 {
-                                                match row.try_get::<String, _>(0) {
-                                                    Ok(s) => acc.push(s),
-                                                    // TODO set an error flag before returning this one
-                                                    Err(err) => {
-                                                        failure = Some(AstridQueryError::Execute {
-                                                            sql: sql.clone(),
-                                                            message: err.to_string(),
-                                                        });
-                                                        break;
-                                                    }
-                                                };
         } else {
-                                                // Try to auto marshall the row into a javascript
+            None
-                                                // object to make it easier on our no-code users.
-                                                match try_marshal(&row) {
-                                                    Ok(json) => acc.push(json.to_string()),
-                                                    Err(err) => {
-                                                        failure = Some(AstridQueryError::Execute {
-                                                            sql: sql.clone(),
-                                                            message: err.to_string(),
-                                                        });
-                                                        break;
         }
-                                                }
-                                            }
-                                        }
-                                        Some(Err(err)) => {
-                                            eprintln!("got fetch_all sql error {:?}", err);
-                                            failure = Some(AstridQueryError::Execute {
-                                                sql: sql.clone(),
-                                                message: err.to_string(),
-                                            });
-                                            break;
-                                        }
-                                    }
-                                }
-                                result.push(acc);
-                            } else {
-                                match sqlx::query(&sql).fetch_one(&db_pool).await.and_then(|row| {
-                                    if row.len() == 1 {
-                                        row.try_get::<String, _>(0)
-                                    } else {
-                                        try_marshal(&row).map(|json| json.to_string())
-                                    }
-                                }) {
-                                    Ok(s) => result.push(vec![s]),
-                                    Err(sqlx::Error::RowNotFound) => {
-                                        failure = Some(AstridQueryError::NotFound { sql });
-                                    }
-                                    Err(err) => {
-                                        eprintln!("got fetchOne sql error {:?}", err);
-                                        failure = Some(AstridQueryError::Execute {
-                                            sql,
-                                            message: err.to_string(),
-                                        });
-                                    }
-                                }
-                            }
-                            if failure.is_some() {
-                                break;
-                            }
-                        }
-                        if let Some(msg) = failure {
-                            response.send(ElmResult::err(msg))
-                        } else {
-                            response.send(ElmResult::ok(result))
-                        }
-                    };
-
-                    // I found it interesting that the runtime of the future from the viewpoint of
-                    // tracing was around 230us for a trivial select 2 rows query, but walltime I
-                    // measured was around 700us. So polling the future or waiting for file IO is
-                    // more expensive than I thought.
-                    worker.block_on(f.instrument(span)).unwrap();
-                } else {
-                    break;
-                }
-            }
-        });
-        let worker_mailbox_clone = worker_mailbox.clone();
-
-        worker.js_runtime.register_op(
-            "op_starmelon_batch_queries",
-            deno_core::op_sync(
-                move |_state, queries: Vec<(bool, String, Vec<String>)>, _: ()| {
-                    let worker_mailbox = worker_mailbox_clone.clone();
-                    let (sender, receiver) = oneshot::channel::<ElmResult<Vec<Vec<String>>, _>>();
-
-                    let span = info_span!("run sql");
-                    let timing_guard = span.enter();
-
-                    worker_mailbox.send((sender, queries)).unwrap();
-                    let elm_result = receiver.recv().unwrap();
-
-                    drop(timing_guard);
-
-                    Ok(elm_result)
-                },
-            ),
-        );
-
-        Some((worker_mailbox, sql_worker_thread))
     } else {
         None
     };
 
-    worker.js_runtime.sync_ops_cache();
+    drop(timing_guard);
+
+    // step 11 create a v8 isolate.
+    let span = info_span!("create v8 isolate");
+    let timing_guard = span.enter();
+    let (worker, main_module) = runtime::setup_worker(extensions, &buffer_file.to_string_lossy())
+        .map_err(|err| InterpreterError::EventLoop(err))?;
     drop(timing_guard);
 
     let span = info_span!("eval javascript");
@@ -466,16 +311,21 @@ pub(crate) fn run(
     match mailbox.replace(None) {
         Some(Ok(buffer)) => match output {
             None => {
+                io::stdout()
+                    .write_all(b"<!DOCTYPE html>")
+                    .map_err(|io_err| CompilerError::WriteOutputFailed(io_err, "stdout".into()))?;
                 io::stdout()
                     .write_all(&buffer)
                     .map_err(|io_err| CompilerError::WriteOutputFailed(io_err, "stdout".into()))?;
             }
             Some(filename) => {
                 let mut f = fs::File::create(&filename)
-                    .map_err(|io_err| CompilerError::WriteOutputFailed(io_err, filename))?;
+                    .map_err(|io_err| CompilerError::WriteOutputFailed(io_err, filename.clone()))?;
 
+                f.write_all(b"<!DOCTYPE html>")
+                    .map_err(|io_err| CompilerError::WriteOutputFailed(io_err, filename.clone()))?;
                 f.write_all(&buffer)
-                    .map_err(|io_err| CompilerError::WriteOutputFailed(io_err, "stdout".into()))?;
+                    .map_err(|io_err| CompilerError::WriteOutputFailed(io_err, filename))?;
             }
         },
         Some(Err(problem)) => {
@@ -484,84 +334,59 @@ pub(crate) fn run(
         None => println!("nothing in the mailbox"),
     }
 
-    if let Some((tx, thread_handle)) = sql_background_thread_handle {
+    if let Some(thread_handle) = sql_background_thread_handle {
-        drop(tx);
         thread_handle.join().unwrap();
     }
 
     Ok(())
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+type OutputMailbox = Arc<RefCell<Option<Result<Vec<u8>, ScriptError>>>>;
-#[serde(tag = "$")]
+
-enum AstridQueryError {
+#[deno_core::op]
-    Execute {
+fn op_starmelon_problem(
-        #[serde(rename = "a")]
+    state: &mut deno_core::OpState,
-        sql: String,
+    msg: ScriptError,
-        #[serde(rename = "b")]
+) -> Result<(), deno_core::error::AnyError> {
-        message: String,
+    eprintln!("got problem from v8 runtime {:?}", &msg);
-    },
+    let mailbox_clone = state.borrow::<OutputMailbox>();
-    NotFound {
+    if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() {
-        #[serde(rename = "a")]
+        mailbox.replace(Err(msg));
-        sql: String,
+    }
-    },
+    Ok(())
 }
 
-fn try_marshal(row: &sqlx::sqlite::SqliteRow) -> Result<serde_json::Value, sqlx::Error> {
+#[deno_core::op]
-    use serde_json::value::{Map, Number};
+fn op_starmelon_bytes_output(
-    use serde_json::Value;
+    state: &mut OpState,
-    use sqlx::{Column, TypeInfo};
+    msg: ElmResult<deno_core::ZeroCopyBuf, ScriptError>,
+) -> Result<(), deno_core::error::AnyError> {
+    let mailbox_clone = state.borrow::<OutputMailbox>();
+    if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() {
+        match msg {
+            ElmResult::Ok { a: buffer } => {
+                let slice: &[u8] = &buffer;
+                mailbox.replace(Ok(slice.to_owned()));
+            }
+            ElmResult::Err { a: err } => {
+                mailbox.replace(Err(err));
+            }
+        }
+    }
+    Ok(())
+}
 
-    let mut object = Map::new();
+#[deno_core::op]
-    for i in 0..row.len() {
+fn op_starmelon_string_output(
-        let column = row.column(i);
+    state: &mut OpState,
-        let value = match column.type_info().name() {
+    msg: ElmResult<String, ScriptError>,
-            "NULL" => Value::Null,
+) -> Result<(), deno_core::error::AnyError> {
-            "TEXT" => Value::String(row.try_get::<String, _>(i)?),
+    let mailbox_clone = state.borrow::<OutputMailbox>();
-            "REAL" => {
+    if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() {
-                let x = row.try_get::<f64, _>(i)?;
+        match msg {
-                match Number::from_f64(x) {
+            ElmResult::Ok { a: s } => mailbox.replace(Ok(s.into_bytes())),
-                    Some(n) => Value::Number(n),
+            ElmResult::Err { a: err } => mailbox.replace(Err(err)),
-                    None => {
-                        return Err(sqlx::Error::ColumnDecode {
-                            index: column.name().to_owned(),
-                            source: Box::new(StringError(format!("While parsing a SQL type `REAL` I expected a finite number but got {} instead ", x))),
-                        });
-                    }
-                }
-            }
-            //"BLOB" =>
-            "INTEGER" => Value::Number(row.try_get::<i64, _>(i)?.into()),
-            //"NUMERIC" =>
-            "BOOLEAN" => Value::Bool(row.try_get::<bool, _>(i)?),
-            //"DATE" =>
-            //"TIME" =>
-            //"DATETIME" =>
-            unknown => {
-                return Err(sqlx::Error::ColumnDecode {
-                    index: column.name().to_owned(),
-                    source: Box::new(StringError(format!("I don't know how to automatically convert the SQL type `{}`` into a JSON value.", unknown))),
-                });
-            }
         };
-        object.insert(column.name().to_owned(), value);
-    }
-
-    Ok(Value::Object(object))
-}
-
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub struct StringError(String);
-
-impl ::std::fmt::Display for StringError {
-    fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
-        self.0.fmt(f)
-    }
-}
-
-impl std::error::Error for StringError {
-    #[inline]
-    fn description(&self) -> &str {
-        &self.0
     }
+    Ok(())
 }

src/exec/fixtures/astrid_pages.rs

@@ -12,6 +12,7 @@ pub enum ScriptError {
     QueryFailed { a: String },
     HtmlGenerationFailed { a: String },
     NotFound,
+    Other { a: String },
 }
 
 pub(crate) fn generate(
@@ -117,6 +118,7 @@ pub(crate) fn generate(
             | QueryFailed String
             | HtmlGenerationFailed String
             | NotFound
+            | Other String
 
         -- SUBSCRIPTIONS
 
@@ -154,6 +156,12 @@ pub(crate) fn generate(
                             E.object
                                 [ ("$", E.string "NotFound" )
                                 ]
+
+                        Other msg ->
+                            E.object
+                                [ ("$", E.string "Other" )
+                                , ("a", E.string msg)
+                                ]
                   )
                 ]
 

src/exec/fixtures/mod.rs

@@ -1,2 +1,3 @@
 pub mod astrid_pages;
 pub mod scripting;
+pub mod sqlite;

src/exec/fixtures/sqlite.rs

@@ -0,0 +1,241 @@
+use deno_core::futures::StreamExt;
+use deno_core::{Extension, OpState};
+use elm_project_utils::ElmResult;
+use serde::{Deserialize, Serialize};
+use serde_json::value::Number;
+use serde_json::Value;
+use sqlx::sqlite::SqlitePool;
+use sqlx::Row;
+use sqlx::{Column, TypeInfo, ValueRef};
+use std::thread::JoinHandle;
+use std::time::Instant;
+use tokio;
+use tracing::{info_span, Instrument};
+
+type SQLWorkerMailbox = std::sync::mpsc::Sender<(
+    oneshot::Sender<ElmResult<Vec<Vec<String>>, AstridQueryError>>,
+    Vec<(bool, String, Vec<String>)>,
+)>;
+
+pub(crate) fn init(db_pool: SqlitePool) -> Result<(Extension, JoinHandle<()>), ()> {
+    let (worker_mailbox, rx) = std::sync::mpsc::channel::<(
+        oneshot::Sender<ElmResult<_, _>>,
+        Vec<(bool, String, Vec<String>)>,
+    )>();
+
+    let sql_worker_thread = std::thread::spawn(move || {
+        let worker = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .unwrap();
+        loop {
+            if let Ok((response, queries)) = rx.recv() {
+                // I am not sure if I should only work on one database task at a time, or
+                // submit as many takes as possible.  Just spawning the future onto this
+                // exectutor does not seem to work, even though the docs say the thread pool
+                // will poll the future until it completes.
+                let db_pool_clone = db_pool.clone();
+                let span = info_span!("inside sql queries futures");
+                let f = async move {
+                    let _start = Instant::now();
+                    let db_pool = db_pool_clone;
+                    let mut result: Vec<Vec<String>> = vec![];
+                    let mut failure: Option<AstridQueryError> = None;
+                    for (fetch_all, sql, _args) in queries {
+                        let mut acc = Vec::new();
+                        if fetch_all {
+                            let mut stream = sqlx::query(&sql).fetch(&db_pool);
+                            loop {
+                                match stream.next().await {
+                                    None => break,
+                                    Some(Ok(row)) => {
+                                        // Try to auto marshall the row into a javascript
+                                        // object to make it easier on our no-code users.
+                                        match try_marshal(&row) {
+                                            Ok(json) => acc.push(json.to_string()),
+                                            Err(err) => {
+                                                failure = Some(AstridQueryError::Execute {
+                                                    sql: sql.clone(),
+                                                    message: err.to_string(),
+                                                });
+                                                break;
+                                            }
+                                        }
+                                    }
+                                    Some(Err(err)) => {
+                                        eprintln!("got fetch_all sql error {:?}", err);
+                                        failure = Some(AstridQueryError::Execute {
+                                            sql: sql.clone(),
+                                            message: err.to_string(),
+                                        });
+                                        break;
+                                    }
+                                }
+                            }
+                            result.push(acc);
+                        } else {
+                            match sqlx::query(&sql)
+                                .fetch_one(&db_pool)
+                                .await
+                                .and_then(|row| try_marshal(&row).map(|json| json.to_string()))
+                            {
+                                Ok(s) => result.push(vec![s]),
+                                Err(sqlx::Error::RowNotFound) => {
+                                    failure = Some(AstridQueryError::NotFound { sql });
+                                }
+                                Err(err) => {
+                                    eprintln!("got fetchOne sql error {:?}", err);
+                                    failure = Some(AstridQueryError::Execute {
+                                        sql,
+                                        message: err.to_string(),
+                                    });
+                                }
+                            }
+                        }
+                        if failure.is_some() {
+                            break;
+                        }
+                    }
+                    if let Some(msg) = failure {
+                        response.send(ElmResult::err(msg))
+                    } else {
+                        response.send(ElmResult::ok(result))
+                    }
+                };
+
+                // I found it interesting that the runtime of the future from the viewpoint of
+                // tracing was around 230us for a trivial select 2 rows query, but walltime I
+                // measured was around 700us. So polling the future or waiting for file IO is
+                // more expensive than I thought.
+                worker.block_on(f.instrument(span)).unwrap();
+            } else {
+                break;
+            }
+        }
+    });
+    let worker_mailbox_clone = worker_mailbox.clone();
+    let extension = Extension::builder()
+        .ops(vec![op_starmelon_batch_queries::decl()])
+        .state(move |state| {
+            state.put(worker_mailbox_clone.clone());
+
+            Ok(())
+        })
+        .build();
+
+    Ok((extension, sql_worker_thread))
+}
+
+#[deno_core::op]
+fn op_starmelon_batch_queries(
+    state: &mut OpState,
+    queries: Vec<(bool, String, Vec<String>)>,
+) -> Result<ElmResult<Vec<Vec<String>>, AstridQueryError>, deno_core::error::AnyError> {
+    let worker_mailbox_clone = state.borrow::<SQLWorkerMailbox>();
+
+    let worker_mailbox = worker_mailbox_clone.clone();
+    let (sender, receiver) = oneshot::channel::<ElmResult<Vec<Vec<String>>, _>>();
+
+    let span = info_span!("run sql");
+    let timing_guard = span.enter();
+
+    worker_mailbox.send((sender, queries)).unwrap();
+    let elm_result = receiver.recv().unwrap();
+
+    drop(timing_guard);
+
+    Ok(elm_result)
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(tag = "$")]
+enum AstridQueryError {
+    Execute {
+        #[serde(rename = "a")]
+        sql: String,
+        #[serde(rename = "b")]
+        message: String,
+    },
+    NotFound {
+        #[serde(rename = "a")]
+        sql: String,
+    },
+}
+
+fn try_marshal(row: &sqlx::sqlite::SqliteRow) -> Result<serde_json::Value, sqlx::Error> {
+    use serde_json::value::Map;
+    use serde_json::Value;
+    use sqlx::{Column, TypeInfo};
+
+    let mut object = Map::new();
+    for i in 0..row.len() {
+        let value = try_decode_column(row, i)?;
+        let key = row.column(i).name().to_owned();
+        object.insert(key, value);
+    }
+
+    Ok(Value::Object(object))
+}
+
+fn try_decode_column(
+    row: &sqlx::sqlite::SqliteRow,
+    i: usize,
+) -> Result<serde_json::Value, sqlx::Error> {
+    let value = row
+        .try_get_raw(i)
+        .map_err(|err| sqlx::Error::ColumnDecode {
+            index: row.column(i).name().to_owned(),
+            source: Box::new(StringError(format!("{}", err))),
+        })?;
+    match value.type_info().name() {
+        "NULL" => {
+            if let Ok(_x) = row.try_get::<i64, _>(i) {
+                eprintln!("it was actually an int");
+            };
+            Ok(Value::Null)
+        }
+        "TEXT" => Ok(Value::String(row.try_get::<String, _>(i)?)),
+        "REAL" => {
+            let x = row.try_get::<f64, _>(i)?;
+            match Number::from_f64(x) {
+                Some(n) => Ok(Value::Number(n)),
+                None => {
+                    Err(sqlx::Error::ColumnDecode {
+                        index: row.column(i).name().to_owned(),
+                        source: Box::new(StringError(format!("While parsing a SQL type `REAL` I expected a finite number but got {} instead ", x))),
+                    })
+                }
+            }
+        }
+        //"BLOB" =>
+        "INTEGER" => Ok(Value::Number(row.try_get::<i64, _>(i)?.into())),
+        //"NUMERIC" =>
+        "BOOLEAN" => Ok(Value::Bool(row.try_get::<bool, _>(i)?)),
+        //"DATE" =>
+        //"TIME" =>
+        //"DATETIME" =>
+        unknown => Err(sqlx::Error::ColumnDecode {
+            index: row.column(i).name().to_owned(),
+            source: Box::new(StringError(format!(
+                "I don't know how to automatically convert the SQL type `{}`` into a JSON value.",
+                unknown
+            ))),
+        }),
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct StringError(String);
+
+impl ::std::fmt::Display for StringError {
+    fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+impl std::error::Error for StringError {
+    #[inline]
+    fn description(&self) -> &str {
+        &self.0
+    }
+}

src/exec/mod.rs

@@ -33,18 +33,10 @@ pub(crate) fn exec(
         return Err(CompilerError::MissingElmStuff(elm_cache_dir).into());
     }
 
+    // step 2.5 get the module name out of the file.
     let data = std::fs::read(&file)
         .map_err(|io_err| CompilerError::ReadInputFailed(io_err, file.clone()))?;
 
-    let mut hasher = PortableHash::new();
-    hasher.write_all(&data).unwrap();
-    // also include the function name in the checksum so users can run multiple functions
-    // from the same file but still get caching if the file does not change.
-    hasher.write_all(function.as_bytes()).unwrap();
-    let source_checksum = hasher.finish();
-
-    // step 2.5 get the module name out of the file.
-
     let entrypoint = elmi::Global(
         elmi::ModuleNameCanonical {
             package: elmi::PackageName::new("author", "project"),
@@ -77,6 +69,13 @@ pub(crate) fn exec(
     };
     drop(timing_guard);
 
+    let mut hasher = PortableHash::new();
+    hasher.write_all(&data).unwrap();
+    // also include the function name in the checksum so users can run multiple functions
+    // from the same file but still get caching if the file does not change.
+    hasher.write_all(function.as_bytes()).unwrap();
+    let source_checksum = hasher.finish();
+
     match mode {
         ExecutionMode::Scripting(input_type, output_type) => scripting::run(
             debug,
@@ -120,30 +119,34 @@ mod runtime {
     use crate::reporting::InterpreterError;
     use deno_core::error::{type_error, AnyError};
     use deno_core::futures::FutureExt;
-    use deno_core::{resolve_url, FsModuleLoader, ModuleLoader, ModuleSpecifier};
+    use deno_core::{resolve_url, Extension, FsModuleLoader, ModuleLoader, ModuleSpecifier};
     use deno_runtime::deno_broadcast_channel::InMemoryBroadcastChannel;
     use deno_runtime::permissions::Permissions;
     use deno_runtime::worker::MainWorker;
     use deno_runtime::worker::WorkerOptions;
     use deno_runtime::BootstrapOptions;
     use deno_web::BlobStore;
-
     use std::pin::Pin;
     use std::rc::Rc;
     use std::sync::Arc;
-
     use tracing::{info_span, Instrument};
 
     fn get_error_class_name(e: &AnyError) -> &'static str {
         deno_runtime::errors::get_error_class_name(e).unwrap_or("Error")
     }
 
-    pub fn setup_worker(path_str: &str) -> Result<(MainWorker, ModuleSpecifier), AnyError> {
+    pub fn setup_worker(
+        extensions: Vec<Extension>,
+        path_str: &str,
+    ) -> Result<(MainWorker, ModuleSpecifier), AnyError> {
         //let module_loader = Rc::new(EmbeddedModuleLoader("void".to_owned()));
         let module_loader = Rc::new(FsModuleLoader);
         let create_web_worker_cb = Arc::new(|_| {
             todo!("Web workers are not supported in the example");
         });
+        let web_worker_preload_module_cb = Arc::new(|_| {
+            todo!("Web workers are not supported in the example");
+        });
 
         let options = WorkerOptions {
             bootstrap: BootstrapOptions {
@@ -152,19 +155,21 @@ mod runtime {
                 cpu_count: 1,
                 debug_flag: false,
                 enable_testing_features: false,
+                is_tty: false,
                 location: None,
                 no_color: false,
-                runtime_version: "0.29.0".to_string(),
+                runtime_version: "0.50.0".to_string(),
                 ts_version: "2.0.0".to_string(),
                 unstable: false,
             },
-            extensions: vec![],
+            extensions: extensions,
             unsafely_ignore_certificate_errors: None,
             root_cert_store: None,
             user_agent: "hello_runtime".to_string(),
             seed: None,
             module_loader,
             create_web_worker_cb,
+            web_worker_preload_module_cb,
             js_error_create_fn: None,
             maybe_inspector_server: None,
             should_break_on_first_statement: false,
@@ -193,7 +198,7 @@ mod runtime {
         foo: F,
     ) -> Result<(), InterpreterError>
     where
-        F: FnOnce(rusty_v8::HandleScope) -> Result<(), InterpreterError>,
+        F: FnOnce(deno_core::v8::HandleScope) -> Result<(), InterpreterError>,
     {
         let wait_for_inspector = false;
         // step 10 load the module into our v8 isolate
@@ -263,6 +268,7 @@ mod runtime {
 
                 Ok(deno_core::ModuleSource {
                     code,
+                    module_type: deno_core::ModuleType::JavaScript,
                     module_url_specified: module_specifier.to_string(),
                     module_url_found: module_specifier.to_string(),
                 })

src/exec/scripting.rs

@@ -1,9 +1,8 @@
 use crate::exec::{fixtures, runtime};
 use crate::reporting::{CompilerError, InterpreterError, Problem, TypeError};
-use deno_core::futures::StreamExt;
+use deno_core::{Extension, OpState};
 use elm_project_utils::{setup_generator_project, ElmResult};
 use os_pipe::dup_stderr;
-use rusty_v8 as v8;
 use sqlx::sqlite::SqlitePool;
 use sqlx::Row;
 use std::cell::RefCell;
@@ -13,9 +12,8 @@ use std::io::{self, Read, Write};
 use std::path::PathBuf;
 use std::process::{Command, Stdio};
 use std::sync::Arc;
-use std::time::Instant;
 use tokio;
-use tracing::{info_span, Instrument};
+use tracing::info_span;
 
 pub(crate) fn run(
     debug: bool,
@@ -284,51 +282,24 @@ pub(crate) fn run(
     // step 10 create a v8 isolate. We need to register a different callback depending on
     // the output type (string, or bytes)
 
-    let span = info_span!("create v8 isolate");
+    let mailbox: Arc<RefCell<Option<Result<Vec<u8>, String>>>> = Arc::new(RefCell::new(None));
-    let timing_guard = span.enter();
+    let mailbox_clone = Arc::clone(&mailbox);
-    let (mut worker, main_module) = runtime::setup_worker(&final_file.to_string_lossy())
+
-        .map_err(|err| InterpreterError::EventLoop(err))?;
+    let mut extensions = vec![Extension::builder()
-    drop(timing_guard);
+        .ops(vec![
+            op_starmelon_bytes_output::decl(),
+            op_starmelon_string_output::decl(),
+            op_starmelon_problem::decl(),
+        ])
+        .state(move |state| {
+            state.put(Arc::clone(&mailbox_clone));
+
+            Ok(())
+        })
+        .build()];
 
     let span = info_span!("register private api");
     let timing_guard = span.enter();
-    let mailbox: Arc<RefCell<Option<Result<Vec<u8>, String>>>> = Arc::new(RefCell::new(None));
-
-    let mailbox_clone = Arc::clone(&mailbox);
-    worker.js_runtime.register_op(
-        "op_starmelon_bytes_output",
-        deno_core::op_sync(move |_state, msg: deno_core::ZeroCopyBuf, _: ()| {
-            let slice: &[u8] = &msg;
-            eprintln!("got message from v8 runtime {:?}", slice.to_owned());
-            if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() {
-                mailbox.replace(Ok(slice.to_owned()));
-            }
-            Ok(())
-        }),
-    );
-
-    let mailbox_clone = Arc::clone(&mailbox);
-    worker.js_runtime.register_op(
-        "op_starmelon_string_output",
-        deno_core::op_sync(move |_state, msg: String, _: ()| {
-            if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() {
-                mailbox.replace(Ok(msg.into_bytes()));
-            }
-            Ok(())
-        }),
-    );
-
-    let mailbox_clone = Arc::clone(&mailbox);
-    worker.js_runtime.register_op(
-        "op_starmelon_problem",
-        deno_core::op_sync(move |_state, msg: String, _: ()| {
-            eprintln!("got problem from v8 runtime {:?}", &msg);
-            if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() {
-                mailbox.replace(Err(msg));
-            }
-            Ok(())
-        }),
-    );
 
     // Step 10.B  setup the sqlite database feature
     let sql_background_thread_handle = if let Some(database_url) = sqlite_path {
@@ -338,100 +309,21 @@ pub(crate) fn run(
             .block_on(async { SqlitePool::connect(&database_url.to_string_lossy()).await })
             .unwrap();
 
-        let (worker_mailbox, rx) = std::sync::mpsc::channel::<(
+        if let Ok((extension, thread_handle)) = fixtures::sqlite::init(db_pool) {
-            oneshot::Sender<ElmResult<_, _>>,
+            extensions.push(extension);
-            Vec<(bool, String, Vec<String>)>,
+            Some(thread_handle)
-        )>();
-
-        let sql_worker_thread = std::thread::spawn(move || {
-            let worker = tokio::runtime::Builder::new_current_thread()
-                .enable_all()
-                .build()
-                .unwrap();
-            loop {
-                if let Ok((response, queries)) = rx.recv() {
-                    // I am not sure if I should only work on one database task at a time, or
-                    // submit as many takes as possible.  Just spawning the future onto this
-                    // exectutor does not seem to work, even though the docs say the thread pool
-                    // will poll the future until it completes.
-                    let db_pool_clone = db_pool.clone();
-                    let span = info_span!("inside sql queries futures");
-                    let f = async move {
-                        let _start = Instant::now();
-                        let db_pool = db_pool_clone;
-                        let mut result: Vec<Vec<String>> = vec![];
-                        for (fetch_all, sql, _args) in queries {
-                            let mut acc = Vec::new();
-                            if fetch_all {
-                                let mut stream = sqlx::query(&sql).fetch(&db_pool);
-                                loop {
-                                    match stream.next().await {
-                                        None => break,
-                                        Some(Ok(row)) => {
-                                            match row.try_get::<String, _>(0) {
-                                                Ok(s) => acc.push(s),
-                                                // TODO set an error flag before returning this one
-                                                Err(_) => break,
-                                            };
-                                        }
-                                        Some(Err(err)) => {
-                                            eprintln!("got fetch_all sql error {:?}", err);
-                                            break;
-                                        }
-                                    }
-                                }
-                                result.push(acc);
         } else {
-                                let s = sqlx::query(&sql)
+            None
-                                    .fetch_one(&db_pool)
-                                    .await
-                                    .and_then(|row| row.try_get::<String, _>(0))
-                                    .unwrap();
-                                result.push(vec![s]);
         }
-                        }
-                        response.send(ElmResult::ok(result))
-                    };
-
-                    // I found it interesting that the runtime of the future from the viewpoint of
-                    // tracing was around 230us for a trivial select 2 rows query, but walltime I
-                    // measured was around 700us. So polling the future or waiting for file IO is
-                    // more expensive than I thought.
-                    worker.block_on(f.instrument(span)).unwrap();
-                } else {
-                    break;
-                }
-            }
-        });
-        let worker_mailbox_clone = worker_mailbox.clone();
-
-        worker.js_runtime.register_op(
-            "op_starmelon_batch_queries",
-            deno_core::op_sync(
-                move |_state, queries: Vec<(bool, String, Vec<String>)>, _: ()| {
-                    let worker_mailbox = worker_mailbox_clone.clone();
-                    let (sender, receiver) =
-                        oneshot::channel::<ElmResult<Vec<Vec<String>>, String>>();
-
-                    let span = info_span!("run sql");
-                    let timing_guard = span.enter();
-
-                    worker_mailbox.send((sender, queries)).unwrap();
-                    let elm_result = receiver.recv().unwrap();
-
-                    drop(timing_guard);
-
-                    Ok(elm_result)
-                },
-            ),
-        );
-
-        Some((worker_mailbox, sql_worker_thread))
     } else {
         None
     };
+    drop(timing_guard);
 
-    worker.js_runtime.sync_ops_cache();
+    let span = info_span!("create v8 isolate");
+    let timing_guard = span.enter();
+    let (worker, main_module) = runtime::setup_worker(extensions, &final_file.to_string_lossy())
+        .map_err(|err| InterpreterError::EventLoop(err))?;
     drop(timing_guard);
 
     // step 11 marshal the input into the v8 isolate. If we are reading from an
@@ -473,10 +365,11 @@ pub(crate) fn run(
         }
     };
 
-    let foo = move |mut scope: rusty_v8::HandleScope| -> Result<(), InterpreterError> {
+    let foo = move |mut scope: deno_core::v8::HandleScope| -> Result<(), InterpreterError> {
         let scope = &mut scope;
         let ctx = scope.get_current_context();
         let global = ctx.global(scope);
+        use deno_core::v8;
 
         let entrypoint = {
             let x =
@@ -566,14 +459,63 @@ pub(crate) fn run(
         None => println!("nothing in the mailbox"),
     }
 
-    if let Some((tx, thread_handle)) = sql_background_thread_handle {
+    if let Some(thread_handle) = sql_background_thread_handle {
-        drop(tx);
         thread_handle.join().unwrap();
     }
 
     Ok(())
 }
 
+type OutputMailbox = Arc<RefCell<Option<Result<Vec<u8>, String>>>>;
+
+#[deno_core::op]
+fn op_starmelon_problem(
+    state: &mut deno_core::OpState,
+    msg: String,
+) -> Result<(), deno_core::error::AnyError> {
+    eprintln!("got problem from v8 runtime {:?}", &msg);
+    let mailbox_clone = state.borrow::<OutputMailbox>();
+    if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() {
+        mailbox.replace(Err(msg));
+    }
+    Ok(())
+}
+
+#[deno_core::op]
+fn op_starmelon_bytes_output(
+    state: &mut OpState,
+    msg: ElmResult<deno_core::ZeroCopyBuf, String>,
+) -> Result<(), deno_core::error::AnyError> {
+    let mailbox_clone = state.borrow::<OutputMailbox>();
+    if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() {
+        match msg {
+            ElmResult::Ok { a: buffer } => {
+                let slice: &[u8] = &buffer;
+                mailbox.replace(Ok(slice.to_owned()));
+            }
+            ElmResult::Err { a: err } => {
+                mailbox.replace(Err(err));
+            }
+        }
+    }
+    Ok(())
+}
+
+#[deno_core::op]
+fn op_starmelon_string_output(
+    state: &mut OpState,
+    msg: ElmResult<String, String>,
+) -> Result<(), deno_core::error::AnyError> {
+    let mailbox_clone = state.borrow::<OutputMailbox>();
+    if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() {
+        match msg {
+            ElmResult::Ok { a: s } => mailbox.replace(Ok(s.into_bytes())),
+            ElmResult::Err { a: err } => mailbox.replace(Err(err)),
+        };
+    }
+    Ok(())
+}
+
 #[derive(Debug, Copy, Clone)]
 pub enum InputType {
     Value,

src/reporting.rs

@@ -2,8 +2,7 @@ use crate::reporting::doc::reflow;
 use deno_core::error::AnyError;
 use elm_project_utils::{ProjectSetupError, RedoScriptError};
 use elmi;
-use pretty::{self, cyan, vcat, Doc};
+use pretty::{self, cyan, hsep, underline, vcat, vividcyan, vividgreen, Doc};
-use rusty_v8;
 use serde_json;
 use std::cmp::max;
 use std::io;
@@ -114,6 +113,7 @@ pub enum CompilerError {
     CantParseModule(String), // first line
     EmptyModule,
     MissingModuleTypeInformation(elmi::ModuleNameCanonical),
+    MissingLiveTableDeriveMacroTarget(PathBuf, elmi::ModuleNameCanonical),
     BadImport(elmi::Global),
     FailedBuildingFixture,
     ReadInputFailed(io::Error, PathBuf),
@@ -126,7 +126,7 @@ pub enum CompilerError {
 
 #[derive(Debug)]
 pub enum InterpreterError {
-    Setup(rusty_v8::DataError),
+    Setup(deno_core::v8::DataError),
     EventLoop(AnyError),
     AllocationFailed,
     ReferenceError,
@@ -138,8 +138,8 @@ impl From<AnyError> for InterpreterError {
     }
 }
 
-impl From<rusty_v8::DataError> for InterpreterError {
+impl From<deno_core::v8::DataError> for InterpreterError {
-    fn from(error: rusty_v8::DataError) -> Self {
+    fn from(error: deno_core::v8::DataError) -> Self {
         Self::Setup(error)
     }
 }
@@ -186,7 +186,47 @@ impl CompilerError {
             MissingElmStuff(_path) => Doc::text("TODO missing elm-stuff/"),
             CantParseModule(_first_line_prefix) => Doc::text("todo could not parse module"),
             EmptyModule => Doc::text("I did not expect the module file to be empty"),
-            MissingModuleTypeInformation(_) => Doc::text("todo missing module type information"),
+            MissingModuleTypeInformation(module_name) => {
+                // ok ,first check that the file shows up in the source-directories field of
+                // elm.json. Otherwise the elm compiler will not generate artifacts that we can
+                // parse in the derive macro
+                vcat([
+                    Doc::text(format!("I was looking for the `{}`", module_name)),
+                    Doc::text(""),
+                    Doc::text("I checked elm-stuff/0.19.1/ but I cannot find it!"),
+                ])
+            }
+            MissingLiveTableDeriveMacroTarget(filename, module_name) => {
+                return vcat([
+                    to_message_bar("NO DERIVE INPUT", filename.to_str().unwrap_or("")),
+                    Doc::text(""),
+                    reflow(format!("The `{}` module does not expose `LiveTable.Derive.DeriveEditor` value", module_name.module)),
+                    Doc::text(""),
+                    hsep([
+                        Doc::concat(underline("Note".into()), ":".into()),
+                        reflow("When running the livetable derive macro, I require that the given file exposes a value of type `LiveTable.Derive.DeriveEditor`. That way I have something that tells me what code to generate!"),
+                    ]),
+                    Doc::text(""),
+                    vcat([
+                        reflow("Adding a `LiveTable.Derive.DeriveEditor` value can be as brief as adding something like this:"),
+                        Doc::text(""),
+                        hsep([
+                            vividcyan("module"),
+                            module_name.module.to_string().into(),
+                            vividcyan("exposing"),
+                            Doc::text("(deriveEditor)"),
+                        ]),
+                        "".into(),
+                        hsep([vividcyan("import"), "LiveTable.Derive".into()]),
+                        "".into(),
+                        hsep([vividgreen("deriveEditor"), "=".into()]),
+                        Doc::concat(vividcyan("  LiveTable.Derive"), ".deriveEditor".into()),
+                        "    { containerAttributes = [], fieldAttributes = {} }".into(),
+                    ]),
+                    reflow("From there I can generate code for a spreadsheet editor component."),
+                    // I recommend looking through https://guide.example.com for more guidance on how to fill in the `deriveLiveTableEditor` value.
+                ]);
+            }
             BadImport(elmi::Global(module, symbol)) => {
                 // TODO suggest alternatives using edit distance
                 reflow(format!(
@@ -217,6 +257,22 @@ impl CompilerError {
     }
 }
 
+// -- MODULE NAME MISMATCH ------------------------------ src/reactor/ui/Router.elm
+//
+// It looks like this module name is out of sync:
+//
+// 1| module Route exposing (fmain, Table)
+//           ^^^^^
+// I need it to match the file path, so I was expecting to see `Router` here. Make
+// the following change, and you should be all set!
+//
+//     Route -> Router
+//
+// Note: I require that module names correspond to file paths. This makes it much
+// easier to explore unfamiliar codebases! So if you want to keep the current
+// module name, try renaming the file instead.
+//
+
 impl InterpreterError {
     pub fn to_doc(&self) -> Doc {
         let title = "COMPILER ERROR";