diff --git a/Cargo.toml b/Cargo.toml index a85d59d..bb63be9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,13 +32,11 @@ elm-quote = { path = "../../../infra/rust-elm-quote" } # All of these are required for deno's javascript runtime. We need to keep the # same versions as other projects in our cargo workspace. Multiple different # versions of rust_v8 seem to break its build script. -deno_runtime = "0.29.0" -tokio = { version = "1.6", features = ["full"] } -deno_core = "0.103.0" -deno_web = "0.52" -rusty_v8 = "0.32" -futures = "0.3.15" -serde_v8 = "0.15" +deno_runtime = "0.50" +tokio = { version = "1.17", features = ["full"] } +deno_core = "0.124" +deno_web = "0.73" +futures = "0.3" # Required to add sql query support to interpreter. Because deno expects sync # ops to be synchronous, we have to use a second async executor to run the sqlx @@ -48,5 +46,9 @@ serde_v8 = "0.15" sqlx = { version = "0.5", features = [ "sqlite", "macros", "runtime-tokio-rustls", "chrono", "json", "uuid" ] } oneshot = "0.1.3" +# required for livetable derive macro +livetable-core = { path = "../../../infra/livetable/core" } + + [dev-dependencies] aho-corasick = "0.7" diff --git a/benches/string_replacement.rs b/benches/string_replacement.rs index c8c25e0..a0ceb4a 100644 --- a/benches/string_replacement.rs +++ b/benches/string_replacement.rs @@ -1,5 +1,3 @@ -use aho_corasick::{AhoCorasick, AhoCorasickBuilder}; - // TODO figure out why this code takes ~70ms to munge the javascript. By comparison just writing a // bunch of chainged string.replace( , ).replace( , ).replace .... takes about (16ms debug) (2.5ms // release). diff --git a/src/exec/astrid_pages.rs b/src/exec/astrid_pages.rs index cf2cfd1..21b6b1f 100644 --- a/src/exec/astrid_pages.rs +++ b/src/exec/astrid_pages.rs @@ -1,11 +1,9 @@ use crate::exec::fixtures::astrid_pages::ScriptError; use crate::exec::{fixtures, runtime}; -use crate::reporting::{CompilerError, InterpreterError, Problem, TypeError}; -use deno_core::futures::StreamExt; +use crate::reporting::{CompilerError, InterpreterError, Problem}; +use deno_core::{v8, Extension, OpState}; use elm_project_utils::{setup_generator_project, ElmPostProcessor, ElmResult}; use os_pipe::dup_stderr; -use rusty_v8 as v8; -use serde::{Deserialize, Serialize}; use sqlx::sqlite::SqlitePool; use sqlx::Row; use std::cell::RefCell; @@ -16,9 +14,8 @@ use std::io::Write; use std::path::PathBuf; use std::process::{Command, Stdio}; use std::sync::Arc; -use std::time::Instant; use tokio; -use tracing::{info_span, Instrument}; +use tracing::info_span; #[derive(Debug, Copy, Clone)] pub enum OutputType { @@ -105,10 +102,12 @@ pub(crate) fn run( match command.output() { Ok(output) => { if !output.status.success() { + eprintln!("{}", String::from_utf8_lossy(&output.stderr)); return Err(CompilerError::FailedBuildingFixture.into()); } } - Err(_) => { + Err(err) => { + eprintln!("{:?}", err); return Err(CompilerError::FailedBuildingFixture.into()); } } @@ -209,7 +208,7 @@ pub(crate) fn run( drop(timing_guard); let desired_route = entrypoint.0.module.clone().to_string(); - let foo = move |mut scope: rusty_v8::HandleScope| -> Result<(), InterpreterError> { + let foo = move |mut scope: deno_core::v8::HandleScope| -> Result<(), InterpreterError> { let scope = &mut scope; let ctx = scope.get_current_context(); let global = ctx.global(scope); @@ -253,55 +252,26 @@ pub(crate) fn run( .unwrap(); drop(timing_guard); - // step 10 create a v8 isolate. We need to register a different callback depending on - // the output type (string or bytes) - - let span = info_span!("create v8 isolate"); - let timing_guard = span.enter(); - let (mut worker, main_module) = runtime::setup_worker(&buffer_file.to_string_lossy()) - .map_err(|err| InterpreterError::EventLoop(err))?; - drop(timing_guard); - let span = info_span!("register private api"); let timing_guard = span.enter(); + + // Step 10 setup our custom ops for the javascript runtime let mailbox: Arc, ScriptError>>>> = Arc::new(RefCell::new(None)); - let mailbox_clone = Arc::clone(&mailbox); - worker.js_runtime.register_op( - "op_starmelon_bytes_output", - deno_core::op_sync( - move |_state, msg: ElmResult, _: ()| { - if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() { - match msg { - ElmResult::Ok { a: buffer } => { - let slice: &[u8] = &buffer; - mailbox.replace(Ok(slice.to_owned())); - } - ElmResult::Err { a: err } => { - mailbox.replace(Err(err)); - } - } - } - Ok(()) - }, - ), - ); - let mailbox_clone = Arc::clone(&mailbox); - worker.js_runtime.register_op( - "op_starmelon_string_output", - deno_core::op_sync(move |_state, msg: ElmResult, _: ()| { - if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() { - match msg { - ElmResult::Ok { a: s } => mailbox.replace(Ok(s.into_bytes())), - ElmResult::Err { a: err } => mailbox.replace(Err(err)), - }; - } + let mut extensions = vec![Extension::builder() + .ops(vec![ + op_starmelon_bytes_output::decl(), + op_starmelon_string_output::decl(), + op_starmelon_problem::decl(), + ]) + .state(move |state| { + state.put(Arc::clone(&mailbox_clone)); + Ok(()) - }), - ); + }) + .build()]; - // Step 10.B setup the sqlite database feature let sql_background_thread_handle = if let Some(database_url) = sqlite_path { // I want to construct the connection in the initial thread so I can tell if the connection // failed @@ -309,148 +279,23 @@ pub(crate) fn run( .block_on(async { SqlitePool::connect(&database_url.to_string_lossy()).await }) .unwrap(); - let (worker_mailbox, rx) = std::sync::mpsc::channel::<( - oneshot::Sender>, - Vec<(bool, String, Vec)>, - )>(); - - let sql_worker_thread = std::thread::spawn(move || { - let worker = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - loop { - if let Ok((response, queries)) = rx.recv() { - // I am not sure if I should only work on one database task at a time, or - // submit as many takes as possible. Just spawning the future onto this - // exectutor does not seem to work, even though the docs say the thread pool - // will poll the future until it completes. - let db_pool_clone = db_pool.clone(); - let span = info_span!("inside sql queries futures"); - let f = async move { - let _start = Instant::now(); - let db_pool = db_pool_clone; - let mut result: Vec> = vec![]; - let mut failure: Option = None; - for (fetch_all, sql, _args) in queries { - let mut acc = Vec::new(); - if fetch_all { - let mut stream = sqlx::query(&sql).fetch(&db_pool); - loop { - match stream.next().await { - None => break, - Some(Ok(row)) => { - use sqlx::ValueRef; - // If we only have one column and it is string type, - // then we can try to parse it as json - if row.len() == 1 { - match row.try_get::(0) { - Ok(s) => acc.push(s), - // TODO set an error flag before returning this one - Err(err) => { - failure = Some(AstridQueryError::Execute { - sql: sql.clone(), - message: err.to_string(), - }); - break; - } - }; - } else { - // Try to auto marshall the row into a javascript - // object to make it easier on our no-code users. - match try_marshal(&row) { - Ok(json) => acc.push(json.to_string()), - Err(err) => { - failure = Some(AstridQueryError::Execute { - sql: sql.clone(), - message: err.to_string(), - }); - break; - } - } - } - } - Some(Err(err)) => { - eprintln!("got fetch_all sql error {:?}", err); - failure = Some(AstridQueryError::Execute { - sql: sql.clone(), - message: err.to_string(), - }); - break; - } - } - } - result.push(acc); - } else { - match sqlx::query(&sql).fetch_one(&db_pool).await.and_then(|row| { - if row.len() == 1 { - row.try_get::(0) - } else { - try_marshal(&row).map(|json| json.to_string()) - } - }) { - Ok(s) => result.push(vec![s]), - Err(sqlx::Error::RowNotFound) => { - failure = Some(AstridQueryError::NotFound { sql }); - } - Err(err) => { - eprintln!("got fetchOne sql error {:?}", err); - failure = Some(AstridQueryError::Execute { - sql, - message: err.to_string(), - }); - } - } - } - if failure.is_some() { - break; - } - } - if let Some(msg) = failure { - response.send(ElmResult::err(msg)) - } else { - response.send(ElmResult::ok(result)) - } - }; - - // I found it interesting that the runtime of the future from the viewpoint of - // tracing was around 230us for a trivial select 2 rows query, but walltime I - // measured was around 700us. So polling the future or waiting for file IO is - // more expensive than I thought. - worker.block_on(f.instrument(span)).unwrap(); - } else { - break; - } - } - }); - let worker_mailbox_clone = worker_mailbox.clone(); - - worker.js_runtime.register_op( - "op_starmelon_batch_queries", - deno_core::op_sync( - move |_state, queries: Vec<(bool, String, Vec)>, _: ()| { - let worker_mailbox = worker_mailbox_clone.clone(); - let (sender, receiver) = oneshot::channel::>, _>>(); - - let span = info_span!("run sql"); - let timing_guard = span.enter(); - - worker_mailbox.send((sender, queries)).unwrap(); - let elm_result = receiver.recv().unwrap(); - - drop(timing_guard); - - Ok(elm_result) - }, - ), - ); - - Some((worker_mailbox, sql_worker_thread)) + if let Ok((extension, thread_handle)) = fixtures::sqlite::init(db_pool) { + extensions.push(extension); + Some(thread_handle) + } else { + None + } } else { None }; - worker.js_runtime.sync_ops_cache(); + drop(timing_guard); + + // step 11 create a v8 isolate. + let span = info_span!("create v8 isolate"); + let timing_guard = span.enter(); + let (worker, main_module) = runtime::setup_worker(extensions, &buffer_file.to_string_lossy()) + .map_err(|err| InterpreterError::EventLoop(err))?; drop(timing_guard); let span = info_span!("eval javascript"); @@ -466,16 +311,21 @@ pub(crate) fn run( match mailbox.replace(None) { Some(Ok(buffer)) => match output { None => { + io::stdout() + .write_all(b"") + .map_err(|io_err| CompilerError::WriteOutputFailed(io_err, "stdout".into()))?; io::stdout() .write_all(&buffer) .map_err(|io_err| CompilerError::WriteOutputFailed(io_err, "stdout".into()))?; } Some(filename) => { let mut f = fs::File::create(&filename) - .map_err(|io_err| CompilerError::WriteOutputFailed(io_err, filename))?; + .map_err(|io_err| CompilerError::WriteOutputFailed(io_err, filename.clone()))?; + f.write_all(b"") + .map_err(|io_err| CompilerError::WriteOutputFailed(io_err, filename.clone()))?; f.write_all(&buffer) - .map_err(|io_err| CompilerError::WriteOutputFailed(io_err, "stdout".into()))?; + .map_err(|io_err| CompilerError::WriteOutputFailed(io_err, filename))?; } }, Some(Err(problem)) => { @@ -484,84 +334,59 @@ pub(crate) fn run( None => println!("nothing in the mailbox"), } - if let Some((tx, thread_handle)) = sql_background_thread_handle { - drop(tx); + if let Some(thread_handle) = sql_background_thread_handle { thread_handle.join().unwrap(); } Ok(()) } -#[derive(Debug, Serialize, Deserialize)] -#[serde(tag = "$")] -enum AstridQueryError { - Execute { - #[serde(rename = "a")] - sql: String, - #[serde(rename = "b")] - message: String, - }, - NotFound { - #[serde(rename = "a")] - sql: String, - }, +type OutputMailbox = Arc, ScriptError>>>>; + +#[deno_core::op] +fn op_starmelon_problem( + state: &mut deno_core::OpState, + msg: ScriptError, +) -> Result<(), deno_core::error::AnyError> { + eprintln!("got problem from v8 runtime {:?}", &msg); + let mailbox_clone = state.borrow::(); + if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() { + mailbox.replace(Err(msg)); + } + Ok(()) } -fn try_marshal(row: &sqlx::sqlite::SqliteRow) -> Result { - use serde_json::value::{Map, Number}; - use serde_json::Value; - use sqlx::{Column, TypeInfo}; +#[deno_core::op] +fn op_starmelon_bytes_output( + state: &mut OpState, + msg: ElmResult, +) -> Result<(), deno_core::error::AnyError> { + let mailbox_clone = state.borrow::(); + if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() { + match msg { + ElmResult::Ok { a: buffer } => { + let slice: &[u8] = &buffer; + mailbox.replace(Ok(slice.to_owned())); + } + ElmResult::Err { a: err } => { + mailbox.replace(Err(err)); + } + } + } + Ok(()) +} - let mut object = Map::new(); - for i in 0..row.len() { - let column = row.column(i); - let value = match column.type_info().name() { - "NULL" => Value::Null, - "TEXT" => Value::String(row.try_get::(i)?), - "REAL" => { - let x = row.try_get::(i)?; - match Number::from_f64(x) { - Some(n) => Value::Number(n), - None => { - return Err(sqlx::Error::ColumnDecode { - index: column.name().to_owned(), - source: Box::new(StringError(format!("While parsing a SQL type `REAL` I expected a finite number but got {} instead ", x))), - }); - } - } - } - //"BLOB" => - "INTEGER" => Value::Number(row.try_get::(i)?.into()), - //"NUMERIC" => - "BOOLEAN" => Value::Bool(row.try_get::(i)?), - //"DATE" => - //"TIME" => - //"DATETIME" => - unknown => { - return Err(sqlx::Error::ColumnDecode { - index: column.name().to_owned(), - source: Box::new(StringError(format!("I don't know how to automatically convert the SQL type `{}`` into a JSON value.", unknown))), - }); - } +#[deno_core::op] +fn op_starmelon_string_output( + state: &mut OpState, + msg: ElmResult, +) -> Result<(), deno_core::error::AnyError> { + let mailbox_clone = state.borrow::(); + if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() { + match msg { + ElmResult::Ok { a: s } => mailbox.replace(Ok(s.into_bytes())), + ElmResult::Err { a: err } => mailbox.replace(Err(err)), }; - object.insert(column.name().to_owned(), value); - } - - Ok(Value::Object(object)) -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct StringError(String); - -impl ::std::fmt::Display for StringError { - fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { - self.0.fmt(f) - } -} - -impl std::error::Error for StringError { - #[inline] - fn description(&self) -> &str { - &self.0 } + Ok(()) } diff --git a/src/exec/fixtures/astrid_pages.rs b/src/exec/fixtures/astrid_pages.rs index 6409e32..7d428a2 100644 --- a/src/exec/fixtures/astrid_pages.rs +++ b/src/exec/fixtures/astrid_pages.rs @@ -12,6 +12,7 @@ pub enum ScriptError { QueryFailed { a: String }, HtmlGenerationFailed { a: String }, NotFound, + Other { a: String }, } pub(crate) fn generate( @@ -117,6 +118,7 @@ pub(crate) fn generate( | QueryFailed String | HtmlGenerationFailed String | NotFound + | Other String -- SUBSCRIPTIONS @@ -154,6 +156,12 @@ pub(crate) fn generate( E.object [ ("$", E.string "NotFound" ) ] + + Other msg -> + E.object + [ ("$", E.string "Other" ) + , ("a", E.string msg) + ] ) ] diff --git a/src/exec/fixtures/mod.rs b/src/exec/fixtures/mod.rs index 171856a..81c4e15 100644 --- a/src/exec/fixtures/mod.rs +++ b/src/exec/fixtures/mod.rs @@ -1,2 +1,3 @@ pub mod astrid_pages; pub mod scripting; +pub mod sqlite; diff --git a/src/exec/fixtures/sqlite.rs b/src/exec/fixtures/sqlite.rs new file mode 100644 index 0000000..02fe54a --- /dev/null +++ b/src/exec/fixtures/sqlite.rs @@ -0,0 +1,241 @@ +use deno_core::futures::StreamExt; +use deno_core::{Extension, OpState}; +use elm_project_utils::ElmResult; +use serde::{Deserialize, Serialize}; +use serde_json::value::Number; +use serde_json::Value; +use sqlx::sqlite::SqlitePool; +use sqlx::Row; +use sqlx::{Column, TypeInfo, ValueRef}; +use std::thread::JoinHandle; +use std::time::Instant; +use tokio; +use tracing::{info_span, Instrument}; + +type SQLWorkerMailbox = std::sync::mpsc::Sender<( + oneshot::Sender>, AstridQueryError>>, + Vec<(bool, String, Vec)>, +)>; + +pub(crate) fn init(db_pool: SqlitePool) -> Result<(Extension, JoinHandle<()>), ()> { + let (worker_mailbox, rx) = std::sync::mpsc::channel::<( + oneshot::Sender>, + Vec<(bool, String, Vec)>, + )>(); + + let sql_worker_thread = std::thread::spawn(move || { + let worker = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + loop { + if let Ok((response, queries)) = rx.recv() { + // I am not sure if I should only work on one database task at a time, or + // submit as many takes as possible. Just spawning the future onto this + // exectutor does not seem to work, even though the docs say the thread pool + // will poll the future until it completes. + let db_pool_clone = db_pool.clone(); + let span = info_span!("inside sql queries futures"); + let f = async move { + let _start = Instant::now(); + let db_pool = db_pool_clone; + let mut result: Vec> = vec![]; + let mut failure: Option = None; + for (fetch_all, sql, _args) in queries { + let mut acc = Vec::new(); + if fetch_all { + let mut stream = sqlx::query(&sql).fetch(&db_pool); + loop { + match stream.next().await { + None => break, + Some(Ok(row)) => { + // Try to auto marshall the row into a javascript + // object to make it easier on our no-code users. + match try_marshal(&row) { + Ok(json) => acc.push(json.to_string()), + Err(err) => { + failure = Some(AstridQueryError::Execute { + sql: sql.clone(), + message: err.to_string(), + }); + break; + } + } + } + Some(Err(err)) => { + eprintln!("got fetch_all sql error {:?}", err); + failure = Some(AstridQueryError::Execute { + sql: sql.clone(), + message: err.to_string(), + }); + break; + } + } + } + result.push(acc); + } else { + match sqlx::query(&sql) + .fetch_one(&db_pool) + .await + .and_then(|row| try_marshal(&row).map(|json| json.to_string())) + { + Ok(s) => result.push(vec![s]), + Err(sqlx::Error::RowNotFound) => { + failure = Some(AstridQueryError::NotFound { sql }); + } + Err(err) => { + eprintln!("got fetchOne sql error {:?}", err); + failure = Some(AstridQueryError::Execute { + sql, + message: err.to_string(), + }); + } + } + } + if failure.is_some() { + break; + } + } + if let Some(msg) = failure { + response.send(ElmResult::err(msg)) + } else { + response.send(ElmResult::ok(result)) + } + }; + + // I found it interesting that the runtime of the future from the viewpoint of + // tracing was around 230us for a trivial select 2 rows query, but walltime I + // measured was around 700us. So polling the future or waiting for file IO is + // more expensive than I thought. + worker.block_on(f.instrument(span)).unwrap(); + } else { + break; + } + } + }); + let worker_mailbox_clone = worker_mailbox.clone(); + let extension = Extension::builder() + .ops(vec![op_starmelon_batch_queries::decl()]) + .state(move |state| { + state.put(worker_mailbox_clone.clone()); + + Ok(()) + }) + .build(); + + Ok((extension, sql_worker_thread)) +} + +#[deno_core::op] +fn op_starmelon_batch_queries( + state: &mut OpState, + queries: Vec<(bool, String, Vec)>, +) -> Result>, AstridQueryError>, deno_core::error::AnyError> { + let worker_mailbox_clone = state.borrow::(); + + let worker_mailbox = worker_mailbox_clone.clone(); + let (sender, receiver) = oneshot::channel::>, _>>(); + + let span = info_span!("run sql"); + let timing_guard = span.enter(); + + worker_mailbox.send((sender, queries)).unwrap(); + let elm_result = receiver.recv().unwrap(); + + drop(timing_guard); + + Ok(elm_result) +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "$")] +enum AstridQueryError { + Execute { + #[serde(rename = "a")] + sql: String, + #[serde(rename = "b")] + message: String, + }, + NotFound { + #[serde(rename = "a")] + sql: String, + }, +} + +fn try_marshal(row: &sqlx::sqlite::SqliteRow) -> Result { + use serde_json::value::Map; + use serde_json::Value; + use sqlx::{Column, TypeInfo}; + + let mut object = Map::new(); + for i in 0..row.len() { + let value = try_decode_column(row, i)?; + let key = row.column(i).name().to_owned(); + object.insert(key, value); + } + + Ok(Value::Object(object)) +} + +fn try_decode_column( + row: &sqlx::sqlite::SqliteRow, + i: usize, +) -> Result { + let value = row + .try_get_raw(i) + .map_err(|err| sqlx::Error::ColumnDecode { + index: row.column(i).name().to_owned(), + source: Box::new(StringError(format!("{}", err))), + })?; + match value.type_info().name() { + "NULL" => { + if let Ok(_x) = row.try_get::(i) { + eprintln!("it was actually an int"); + }; + Ok(Value::Null) + } + "TEXT" => Ok(Value::String(row.try_get::(i)?)), + "REAL" => { + let x = row.try_get::(i)?; + match Number::from_f64(x) { + Some(n) => Ok(Value::Number(n)), + None => { + Err(sqlx::Error::ColumnDecode { + index: row.column(i).name().to_owned(), + source: Box::new(StringError(format!("While parsing a SQL type `REAL` I expected a finite number but got {} instead ", x))), + }) + } + } + } + //"BLOB" => + "INTEGER" => Ok(Value::Number(row.try_get::(i)?.into())), + //"NUMERIC" => + "BOOLEAN" => Ok(Value::Bool(row.try_get::(i)?)), + //"DATE" => + //"TIME" => + //"DATETIME" => + unknown => Err(sqlx::Error::ColumnDecode { + index: row.column(i).name().to_owned(), + source: Box::new(StringError(format!( + "I don't know how to automatically convert the SQL type `{}`` into a JSON value.", + unknown + ))), + }), + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StringError(String); + +impl ::std::fmt::Display for StringError { + fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { + self.0.fmt(f) + } +} + +impl std::error::Error for StringError { + #[inline] + fn description(&self) -> &str { + &self.0 + } +} diff --git a/src/exec/mod.rs b/src/exec/mod.rs index ce97eb5..7b615d6 100644 --- a/src/exec/mod.rs +++ b/src/exec/mod.rs @@ -33,18 +33,10 @@ pub(crate) fn exec( return Err(CompilerError::MissingElmStuff(elm_cache_dir).into()); } + // step 2.5 get the module name out of the file. let data = std::fs::read(&file) .map_err(|io_err| CompilerError::ReadInputFailed(io_err, file.clone()))?; - let mut hasher = PortableHash::new(); - hasher.write_all(&data).unwrap(); - // also include the function name in the checksum so users can run multiple functions - // from the same file but still get caching if the file does not change. - hasher.write_all(function.as_bytes()).unwrap(); - let source_checksum = hasher.finish(); - - // step 2.5 get the module name out of the file. - let entrypoint = elmi::Global( elmi::ModuleNameCanonical { package: elmi::PackageName::new("author", "project"), @@ -77,6 +69,13 @@ pub(crate) fn exec( }; drop(timing_guard); + let mut hasher = PortableHash::new(); + hasher.write_all(&data).unwrap(); + // also include the function name in the checksum so users can run multiple functions + // from the same file but still get caching if the file does not change. + hasher.write_all(function.as_bytes()).unwrap(); + let source_checksum = hasher.finish(); + match mode { ExecutionMode::Scripting(input_type, output_type) => scripting::run( debug, @@ -120,30 +119,34 @@ mod runtime { use crate::reporting::InterpreterError; use deno_core::error::{type_error, AnyError}; use deno_core::futures::FutureExt; - use deno_core::{resolve_url, FsModuleLoader, ModuleLoader, ModuleSpecifier}; + use deno_core::{resolve_url, Extension, FsModuleLoader, ModuleLoader, ModuleSpecifier}; use deno_runtime::deno_broadcast_channel::InMemoryBroadcastChannel; use deno_runtime::permissions::Permissions; use deno_runtime::worker::MainWorker; use deno_runtime::worker::WorkerOptions; use deno_runtime::BootstrapOptions; use deno_web::BlobStore; - use std::pin::Pin; use std::rc::Rc; use std::sync::Arc; - use tracing::{info_span, Instrument}; fn get_error_class_name(e: &AnyError) -> &'static str { deno_runtime::errors::get_error_class_name(e).unwrap_or("Error") } - pub fn setup_worker(path_str: &str) -> Result<(MainWorker, ModuleSpecifier), AnyError> { + pub fn setup_worker( + extensions: Vec, + path_str: &str, + ) -> Result<(MainWorker, ModuleSpecifier), AnyError> { //let module_loader = Rc::new(EmbeddedModuleLoader("void".to_owned())); let module_loader = Rc::new(FsModuleLoader); let create_web_worker_cb = Arc::new(|_| { todo!("Web workers are not supported in the example"); }); + let web_worker_preload_module_cb = Arc::new(|_| { + todo!("Web workers are not supported in the example"); + }); let options = WorkerOptions { bootstrap: BootstrapOptions { @@ -152,19 +155,21 @@ mod runtime { cpu_count: 1, debug_flag: false, enable_testing_features: false, + is_tty: false, location: None, no_color: false, - runtime_version: "0.29.0".to_string(), + runtime_version: "0.50.0".to_string(), ts_version: "2.0.0".to_string(), unstable: false, }, - extensions: vec![], + extensions: extensions, unsafely_ignore_certificate_errors: None, root_cert_store: None, user_agent: "hello_runtime".to_string(), seed: None, module_loader, create_web_worker_cb, + web_worker_preload_module_cb, js_error_create_fn: None, maybe_inspector_server: None, should_break_on_first_statement: false, @@ -193,7 +198,7 @@ mod runtime { foo: F, ) -> Result<(), InterpreterError> where - F: FnOnce(rusty_v8::HandleScope) -> Result<(), InterpreterError>, + F: FnOnce(deno_core::v8::HandleScope) -> Result<(), InterpreterError>, { let wait_for_inspector = false; // step 10 load the module into our v8 isolate @@ -263,6 +268,7 @@ mod runtime { Ok(deno_core::ModuleSource { code, + module_type: deno_core::ModuleType::JavaScript, module_url_specified: module_specifier.to_string(), module_url_found: module_specifier.to_string(), }) diff --git a/src/exec/scripting.rs b/src/exec/scripting.rs index 36cde16..52488e2 100644 --- a/src/exec/scripting.rs +++ b/src/exec/scripting.rs @@ -1,9 +1,8 @@ use crate::exec::{fixtures, runtime}; use crate::reporting::{CompilerError, InterpreterError, Problem, TypeError}; -use deno_core::futures::StreamExt; +use deno_core::{Extension, OpState}; use elm_project_utils::{setup_generator_project, ElmResult}; use os_pipe::dup_stderr; -use rusty_v8 as v8; use sqlx::sqlite::SqlitePool; use sqlx::Row; use std::cell::RefCell; @@ -13,9 +12,8 @@ use std::io::{self, Read, Write}; use std::path::PathBuf; use std::process::{Command, Stdio}; use std::sync::Arc; -use std::time::Instant; use tokio; -use tracing::{info_span, Instrument}; +use tracing::info_span; pub(crate) fn run( debug: bool, @@ -284,51 +282,24 @@ pub(crate) fn run( // step 10 create a v8 isolate. We need to register a different callback depending on // the output type (string, or bytes) - let span = info_span!("create v8 isolate"); - let timing_guard = span.enter(); - let (mut worker, main_module) = runtime::setup_worker(&final_file.to_string_lossy()) - .map_err(|err| InterpreterError::EventLoop(err))?; - drop(timing_guard); + let mailbox: Arc, String>>>> = Arc::new(RefCell::new(None)); + let mailbox_clone = Arc::clone(&mailbox); + + let mut extensions = vec![Extension::builder() + .ops(vec![ + op_starmelon_bytes_output::decl(), + op_starmelon_string_output::decl(), + op_starmelon_problem::decl(), + ]) + .state(move |state| { + state.put(Arc::clone(&mailbox_clone)); + + Ok(()) + }) + .build()]; let span = info_span!("register private api"); let timing_guard = span.enter(); - let mailbox: Arc, String>>>> = Arc::new(RefCell::new(None)); - - let mailbox_clone = Arc::clone(&mailbox); - worker.js_runtime.register_op( - "op_starmelon_bytes_output", - deno_core::op_sync(move |_state, msg: deno_core::ZeroCopyBuf, _: ()| { - let slice: &[u8] = &msg; - eprintln!("got message from v8 runtime {:?}", slice.to_owned()); - if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() { - mailbox.replace(Ok(slice.to_owned())); - } - Ok(()) - }), - ); - - let mailbox_clone = Arc::clone(&mailbox); - worker.js_runtime.register_op( - "op_starmelon_string_output", - deno_core::op_sync(move |_state, msg: String, _: ()| { - if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() { - mailbox.replace(Ok(msg.into_bytes())); - } - Ok(()) - }), - ); - - let mailbox_clone = Arc::clone(&mailbox); - worker.js_runtime.register_op( - "op_starmelon_problem", - deno_core::op_sync(move |_state, msg: String, _: ()| { - eprintln!("got problem from v8 runtime {:?}", &msg); - if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() { - mailbox.replace(Err(msg)); - } - Ok(()) - }), - ); // Step 10.B setup the sqlite database feature let sql_background_thread_handle = if let Some(database_url) = sqlite_path { @@ -338,100 +309,21 @@ pub(crate) fn run( .block_on(async { SqlitePool::connect(&database_url.to_string_lossy()).await }) .unwrap(); - let (worker_mailbox, rx) = std::sync::mpsc::channel::<( - oneshot::Sender>, - Vec<(bool, String, Vec)>, - )>(); - - let sql_worker_thread = std::thread::spawn(move || { - let worker = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - loop { - if let Ok((response, queries)) = rx.recv() { - // I am not sure if I should only work on one database task at a time, or - // submit as many takes as possible. Just spawning the future onto this - // exectutor does not seem to work, even though the docs say the thread pool - // will poll the future until it completes. - let db_pool_clone = db_pool.clone(); - let span = info_span!("inside sql queries futures"); - let f = async move { - let _start = Instant::now(); - let db_pool = db_pool_clone; - let mut result: Vec> = vec![]; - for (fetch_all, sql, _args) in queries { - let mut acc = Vec::new(); - if fetch_all { - let mut stream = sqlx::query(&sql).fetch(&db_pool); - loop { - match stream.next().await { - None => break, - Some(Ok(row)) => { - match row.try_get::(0) { - Ok(s) => acc.push(s), - // TODO set an error flag before returning this one - Err(_) => break, - }; - } - Some(Err(err)) => { - eprintln!("got fetch_all sql error {:?}", err); - break; - } - } - } - result.push(acc); - } else { - let s = sqlx::query(&sql) - .fetch_one(&db_pool) - .await - .and_then(|row| row.try_get::(0)) - .unwrap(); - result.push(vec![s]); - } - } - response.send(ElmResult::ok(result)) - }; - - // I found it interesting that the runtime of the future from the viewpoint of - // tracing was around 230us for a trivial select 2 rows query, but walltime I - // measured was around 700us. So polling the future or waiting for file IO is - // more expensive than I thought. - worker.block_on(f.instrument(span)).unwrap(); - } else { - break; - } - } - }); - let worker_mailbox_clone = worker_mailbox.clone(); - - worker.js_runtime.register_op( - "op_starmelon_batch_queries", - deno_core::op_sync( - move |_state, queries: Vec<(bool, String, Vec)>, _: ()| { - let worker_mailbox = worker_mailbox_clone.clone(); - let (sender, receiver) = - oneshot::channel::>, String>>(); - - let span = info_span!("run sql"); - let timing_guard = span.enter(); - - worker_mailbox.send((sender, queries)).unwrap(); - let elm_result = receiver.recv().unwrap(); - - drop(timing_guard); - - Ok(elm_result) - }, - ), - ); - - Some((worker_mailbox, sql_worker_thread)) + if let Ok((extension, thread_handle)) = fixtures::sqlite::init(db_pool) { + extensions.push(extension); + Some(thread_handle) + } else { + None + } } else { None }; + drop(timing_guard); - worker.js_runtime.sync_ops_cache(); + let span = info_span!("create v8 isolate"); + let timing_guard = span.enter(); + let (worker, main_module) = runtime::setup_worker(extensions, &final_file.to_string_lossy()) + .map_err(|err| InterpreterError::EventLoop(err))?; drop(timing_guard); // step 11 marshal the input into the v8 isolate. If we are reading from an @@ -473,10 +365,11 @@ pub(crate) fn run( } }; - let foo = move |mut scope: rusty_v8::HandleScope| -> Result<(), InterpreterError> { + let foo = move |mut scope: deno_core::v8::HandleScope| -> Result<(), InterpreterError> { let scope = &mut scope; let ctx = scope.get_current_context(); let global = ctx.global(scope); + use deno_core::v8; let entrypoint = { let x = @@ -566,14 +459,63 @@ pub(crate) fn run( None => println!("nothing in the mailbox"), } - if let Some((tx, thread_handle)) = sql_background_thread_handle { - drop(tx); + if let Some(thread_handle) = sql_background_thread_handle { thread_handle.join().unwrap(); } Ok(()) } +type OutputMailbox = Arc, String>>>>; + +#[deno_core::op] +fn op_starmelon_problem( + state: &mut deno_core::OpState, + msg: String, +) -> Result<(), deno_core::error::AnyError> { + eprintln!("got problem from v8 runtime {:?}", &msg); + let mailbox_clone = state.borrow::(); + if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() { + mailbox.replace(Err(msg)); + } + Ok(()) +} + +#[deno_core::op] +fn op_starmelon_bytes_output( + state: &mut OpState, + msg: ElmResult, +) -> Result<(), deno_core::error::AnyError> { + let mailbox_clone = state.borrow::(); + if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() { + match msg { + ElmResult::Ok { a: buffer } => { + let slice: &[u8] = &buffer; + mailbox.replace(Ok(slice.to_owned())); + } + ElmResult::Err { a: err } => { + mailbox.replace(Err(err)); + } + } + } + Ok(()) +} + +#[deno_core::op] +fn op_starmelon_string_output( + state: &mut OpState, + msg: ElmResult, +) -> Result<(), deno_core::error::AnyError> { + let mailbox_clone = state.borrow::(); + if let Ok(mut mailbox) = mailbox_clone.try_borrow_mut() { + match msg { + ElmResult::Ok { a: s } => mailbox.replace(Ok(s.into_bytes())), + ElmResult::Err { a: err } => mailbox.replace(Err(err)), + }; + } + Ok(()) +} + #[derive(Debug, Copy, Clone)] pub enum InputType { Value, diff --git a/src/reporting.rs b/src/reporting.rs index 151b131..91d51bc 100644 --- a/src/reporting.rs +++ b/src/reporting.rs @@ -2,8 +2,7 @@ use crate::reporting::doc::reflow; use deno_core::error::AnyError; use elm_project_utils::{ProjectSetupError, RedoScriptError}; use elmi; -use pretty::{self, cyan, vcat, Doc}; -use rusty_v8; +use pretty::{self, cyan, hsep, underline, vcat, vividcyan, vividgreen, Doc}; use serde_json; use std::cmp::max; use std::io; @@ -114,6 +113,7 @@ pub enum CompilerError { CantParseModule(String), // first line EmptyModule, MissingModuleTypeInformation(elmi::ModuleNameCanonical), + MissingLiveTableDeriveMacroTarget(PathBuf, elmi::ModuleNameCanonical), BadImport(elmi::Global), FailedBuildingFixture, ReadInputFailed(io::Error, PathBuf), @@ -126,7 +126,7 @@ pub enum CompilerError { #[derive(Debug)] pub enum InterpreterError { - Setup(rusty_v8::DataError), + Setup(deno_core::v8::DataError), EventLoop(AnyError), AllocationFailed, ReferenceError, @@ -138,8 +138,8 @@ impl From for InterpreterError { } } -impl From for InterpreterError { - fn from(error: rusty_v8::DataError) -> Self { +impl From for InterpreterError { + fn from(error: deno_core::v8::DataError) -> Self { Self::Setup(error) } } @@ -186,7 +186,47 @@ impl CompilerError { MissingElmStuff(_path) => Doc::text("TODO missing elm-stuff/"), CantParseModule(_first_line_prefix) => Doc::text("todo could not parse module"), EmptyModule => Doc::text("I did not expect the module file to be empty"), - MissingModuleTypeInformation(_) => Doc::text("todo missing module type information"), + MissingModuleTypeInformation(module_name) => { + // ok ,first check that the file shows up in the source-directories field of + // elm.json. Otherwise the elm compiler will not generate artifacts that we can + // parse in the derive macro + vcat([ + Doc::text(format!("I was looking for the `{}`", module_name)), + Doc::text(""), + Doc::text("I checked elm-stuff/0.19.1/ but I cannot find it!"), + ]) + } + MissingLiveTableDeriveMacroTarget(filename, module_name) => { + return vcat([ + to_message_bar("NO DERIVE INPUT", filename.to_str().unwrap_or("")), + Doc::text(""), + reflow(format!("The `{}` module does not expose `LiveTable.Derive.DeriveEditor` value", module_name.module)), + Doc::text(""), + hsep([ + Doc::concat(underline("Note".into()), ":".into()), + reflow("When running the livetable derive macro, I require that the given file exposes a value of type `LiveTable.Derive.DeriveEditor`. That way I have something that tells me what code to generate!"), + ]), + Doc::text(""), + vcat([ + reflow("Adding a `LiveTable.Derive.DeriveEditor` value can be as brief as adding something like this:"), + Doc::text(""), + hsep([ + vividcyan("module"), + module_name.module.to_string().into(), + vividcyan("exposing"), + Doc::text("(deriveEditor)"), + ]), + "".into(), + hsep([vividcyan("import"), "LiveTable.Derive".into()]), + "".into(), + hsep([vividgreen("deriveEditor"), "=".into()]), + Doc::concat(vividcyan(" LiveTable.Derive"), ".deriveEditor".into()), + " { containerAttributes = [], fieldAttributes = {} }".into(), + ]), + reflow("From there I can generate code for a spreadsheet editor component."), + // I recommend looking through https://guide.example.com for more guidance on how to fill in the `deriveLiveTableEditor` value. + ]); + } BadImport(elmi::Global(module, symbol)) => { // TODO suggest alternatives using edit distance reflow(format!( @@ -217,6 +257,22 @@ impl CompilerError { } } +// -- MODULE NAME MISMATCH ------------------------------ src/reactor/ui/Router.elm +// +// It looks like this module name is out of sync: +// +// 1| module Route exposing (fmain, Table) +// ^^^^^ +// I need it to match the file path, so I was expecting to see `Router` here. Make +// the following change, and you should be all set! +// +// Route -> Router +// +// Note: I require that module names correspond to file paths. This makes it much +// easier to explore unfamiliar codebases! So if you want to keep the current +// module name, try renaming the file instead. +// + impl InterpreterError { pub fn to_doc(&self) -> Doc { let title = "COMPILER ERROR";