From 5d770d0d0ae73dbbd02623496c436ddd801c7c60 Mon Sep 17 00:00:00 2001 From: YetAnotherMinion Date: Tue, 12 Oct 2021 21:00:25 +0100 Subject: [PATCH] refactor: share checksum rebuild logic Move some of the caching logic (source file checksum) out of Starmelon into a shared infra crate. I have 4 similar tooling use cases for working with elm projects in Rust. 1. Starmelon needs to create a modified copy of an existing elm application with extra dependencies and source directories. The extension elm application points to the source directories of the parent 2. Elm reactor needs to compile an elm module into javascript with caching. 3. SQL reactor needs to generate an elm application for each database, generate elm source code and compile that elm code with caching. It uses Starmelon to run derive macros written in Elm. 4. AstridLabs needs to create a heavily modified copy of an existing elm application with tons of generated code. It uses Starmelon to run derive macros written in Elm. For 3 and 4 I could speed up the code generation step by using part of starmelon as library. A proc-macro could include the Elm derive macro javascript at crate build time and reuse the same v8 isolate over and over for every web request. My plan for 1,2,3,4 has a lot of shared functionality. I am thinking that I should consolidate all the components into one library crate. --- Cargo.toml | 1 + examples/single-page/starmelon-release | 1 + src/fixture.rs | 236 ++++++++++++++++ src/main.rs | 364 ++++--------------------- src/reporting.rs | 38 ++- 5 files changed, 310 insertions(+), 330 deletions(-) create mode 120000 examples/single-page/starmelon-release create mode 100644 src/fixture.rs diff --git a/Cargo.toml b/Cargo.toml index 823237f..bd11e48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ os_pipe = "0.9" serde = { version = "1.0", features = [ "derive" ] } serde_json = { version ="1.0", features = [] } structopt = { version = "0.3" } +elm-project-utils = { path = "../../../infra/rust-elm-project-utils" } # All of these are required for deno's javascript runtime. We need to keep the # same versions as other projects in our cargo workspace. Multiple different diff --git a/examples/single-page/starmelon-release b/examples/single-page/starmelon-release new file mode 120000 index 0000000..12e9f9d --- /dev/null +++ b/examples/single-page/starmelon-release @@ -0,0 +1 @@ +../../../../../target/release/starmelon \ No newline at end of file diff --git a/src/fixture.rs b/src/fixture.rs new file mode 100644 index 0000000..c78c19b --- /dev/null +++ b/src/fixture.rs @@ -0,0 +1,236 @@ +use std::fmt::Display; +use crate::{OutputType, InputType}; + +pub(crate) fn generate>( + source_checksum: u64, + target_module: S, + target_function: S, + input: Option, + output: OutputType, +) -> (String, String) { + // The generated fixture Elm files are around 3007 bytes which includes the import of the + // target module and function + let mut buffer = + String::with_capacity(3000 + target_module.as_ref().len() + target_function.as_ref().len()); + + let module_name = format!("Gen{:020}", source_checksum); + + use std::fmt::Write; + write!(buffer, "port module {} exposing (main)\n", &module_name).unwrap(); + buffer.push_str(GENERATOR_IMPORTS); + buffer.push_str("-- START CUSTOMIZED PART\n"); + write!(buffer, "import {}\n\n", target_module.as_ref()).unwrap(); + + // if the input type is none then we don't have to generate an apply, and the port input type + // is () + match output { + OutputType::String => buffer.push_str("encodeOutput = encodeString\n"), + OutputType::Value => buffer.push_str("encodeOutput = encodeJson\n"), + OutputType::Bytes => buffer.push_str("encodeOutput = encodeBytes\n"), + OutputType::Html => buffer.push_str("encodeOutput = encodeHtml\n"), + } + + match input { + None => { + buffer.push_str(&zero_arg_apply( + target_module.as_ref(), + target_function.as_ref(), + )); + } + Some(InputType::Value) => { + buffer.push_str("decodeInput = decodeValue\n"); + buffer.push_str(&one_arg_apply( + target_module.as_ref(), + target_function.as_ref(), + )); + } + Some(InputType::Bytes) => { + buffer.push_str("decodeInput = decodeBytes\n"); + buffer.push_str(&one_arg_apply( + target_module.as_ref(), + target_function.as_ref(), + )); + } + Some(InputType::String) => { + buffer.push_str("decodeInput = decodeString\n"); + buffer.push_str(&one_arg_apply( + target_module.as_ref(), + target_function.as_ref(), + )); + } + } + buffer.push_str("-- END CUSTOMIZED PART\n"); + buffer.push_str(GENERATOR_BODY); + + (module_name, buffer) +} + +// port module Generator exposing (main) +const GENERATOR_IMPORTS: &'static str = r#" +import Bytes exposing (Bytes) +import ElmHtml.InternalTypes exposing (decodeElmHtml) +import ElmHtml.ToString exposing (nodeToStringWithOptions, defaultFormatOptions) +import Html exposing (Html) +import Json.Decode as D +import Json.Encode as E +import Platform +"#; + +fn zero_arg_apply(target_module: S, function: S) -> String { + format!( + r#" + +apply : E.Value -> Cmd Msg +apply _ = + {target_module}.{function} + |> encodeOutput + |> onOutput +"#, + target_module = target_module, + function = function, + ) +} + +// The user just has to alias the decodeInput and encodeOutput +fn one_arg_apply(target_module: S, function: S) -> String { + format!( + r#" + +apply : E.Value -> Cmd Msg +apply value = + case decodeInput value of + Ok input -> + input + |> {target_module}.{function} + |> encodeOutput + |> onOutput + + Err error -> + error + |> D.errorToString + |> encodeFailure + |> onOutput +"#, + target_module = target_module, + function = function, + ) +} + +const GENERATOR_BODY: &'static str = r#" +-- MAIN + +main = + Platform.worker + { init = init + , update = update + , subscriptions = subscriptions + } + +-- PORTS + +port onInput : (E.Value -> msg) -> Sub msg +port onOutput : E.Value -> Cmd msg + +-- MODEL + + +type alias Model = () + +init : () -> (Model, Cmd Msg) +init _ = + ((), Cmd.none) + +-- UPDATE + +type Msg + = Input E.Value + +update : Msg -> Model -> (Model, Cmd Msg) +update msg model = + case msg of + Input value -> + let + cmd = + apply value + in + (model, cmd) + +-- SUBSCRIPTIONS + +subscriptions : Model -> Sub Msg +subscriptions _ = + onInput Input + +-- DECODERS + +decodeBytes : E.Value -> Result D.Error Bytes +decodeBytes value = + D.decodeValue decodeBytesHelp value + +decodeBytesHelp : D.Decoder Bytes +decodeBytesHelp = + D.fail "REPLACE_ME_WITH_BYTES_DECODER" + +decodeString : E.Value -> Result D.Error String +decodeString value = + D.decodeValue D.string value + +-- ENCODERS + +encodeFailure : String -> E.Value +encodeFailure msg = + E.object + [ ("ctor", E.string "Err") + , ("a", E.string msg) + ] + +encodeSuccess : E.Value -> E.Value +encodeSuccess value = + E.object + [ ("ctor", E.string "Ok") + , ("a", value) + ] + +encodeJson : E.Value -> E.Value +encodeJson = + encodeSuccess + +encodeBytes : Bytes -> E.Value +encodeBytes bytes = + bytes + |> encodeBytesHelp + |> encodeSuccess + +encodeBytesHelp : Bytes -> E.Value +encodeBytesHelp bytes = + E.string "REPLACE_ME_WITH_BYTES_ENCODER" + +encodeString : String -> E.Value +encodeString s = + encodeSuccess (E.string s) + +encodeHtml : Html msg -> E.Value +encodeHtml node = + let + options = + { defaultFormatOptions | newLines = True, indent = 2 } + in + case + D.decodeString + (decodeElmHtml (\taggers eventHandler -> D.succeed ())) + (asJsonString node) + of + Ok elmHtml -> + elmHtml + |> nodeToStringWithOptions options + |> encodeString + + Err error -> + error + |> D.errorToString + |> encodeFailure + + +asJsonString : Html msg -> String +asJsonString x = "REPLACE_ME_WITH_JSON_STRINGIFY" +"#; diff --git a/src/main.rs b/src/main.rs index 072b70e..9a8aa5d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,18 +6,19 @@ use pretty::pretty; use serde::{Deserialize, Serialize}; use std::cell::RefCell; use std::collections::HashMap; -use std::fmt::Display; use std::fs::{self, canonicalize, metadata}; use std::hash::Hasher; -use std::io::{self, BufRead, Error, ErrorKind, Read, Seek, Write}; +use std::io::{self, BufRead, Error, ErrorKind, Read, Write}; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::sync::Arc; use std::time::Instant; use structopt::StructOpt; +use elm_project_utils::ChecksumConstraint; use tokio; mod reporting; +mod fixture; fn main() { let args = Arguments::from_args(); @@ -90,7 +91,7 @@ fn exec( io::stderr().write_all(&output.stderr).unwrap(); } } - Err(io_err) => { + Err(_io_err) => { // TODO handle this one return Err(Problem::Wildcard("elm failed".into())); } @@ -164,7 +165,7 @@ fn exec( // step 7 create an Elm fixture file to run our function let start = Instant::now(); - let (gen_module_name, source) = generate_fixture( + let (gen_module_name, source) = fixture::generate( source_checksum, target_module, function, @@ -175,16 +176,15 @@ fn exec( let mut source_filename = generator_dir.join("src").join(&gen_module_name); source_filename.set_extension("elm"); - let mut source_file = fs::File::create(&source_filename) - .map_err(|io_err| CompilerError::WriteOutputFailed(io_err, source_filename.clone()))?; - source_file + fs::File::create(&source_filename) + .map_err(|io_err| CompilerError::WriteOutputFailed(io_err, source_filename.clone()))? .write_all(source.as_bytes()) .map_err(|io_err| CompilerError::WriteOutputFailed(io_err, source_filename.clone()))?; - drop(source_file); eprintln!( "[{:?}] created generator Elm program", Instant::now() - start ); + // step 8 compile the fixture let mut intermediate_file = generator_dir.join("obj").join(&gen_module_name); @@ -383,36 +383,24 @@ globalThis.runOnInput = function(data) { // step 11 marshal the input into the v8 isolate. If we are reading from an // input file load that, if we are reading from stdin read that. - let input = match (input_source, input_type) { - (None, None) => None, - (Some(path), None) => None, - (None, Some(input_type)) => { - // read from stdin when input_source was not present - let mut stdin = io::stdin(); - let mut buffer = Vec::new(); - stdin.read_to_end(&mut buffer).unwrap(); - - match input_type { - InputType::String => { - let s = String::from_utf8(buffer).map_err(|_| CompilerError::BadInput)?; - - Some(ValidatedInput::String(s)) + let input = match input_type { + None => None, + Some(input_type) => { + let buffer = match input_source { + None => { + // read from stdin when input_source was not present + let mut stdin = io::stdin(); + let mut buffer = Vec::new(); + stdin.read_to_end(&mut buffer).unwrap(); + buffer } - InputType::Bytes => Some(ValidatedInput::Bytes(buffer)), - InputType::Value => { - let _: serde_json::Value = - serde_json::from_slice(&buffer).map_err(|_| CompilerError::BadInput)?; - - let s = String::from_utf8(buffer).map_err(|_| CompilerError::BadInput)?; - - Some(ValidatedInput::Value(s)) + Some(path) => { + let buffer = std::fs::read(&path) + .map_err(|io_err| CompilerError::ReadInputFailed(io_err, path.clone()))?; + + buffer } - } - } - (Some(path), Some(input_type)) => { - let buffer = std::fs::read(&path) - .map_err(|io_err| CompilerError::ReadInputFailed(io_err, path.clone()))?; - + }; match input_type { InputType::String => { let s = String::from_utf8(buffer).map_err(|_| CompilerError::BadInput)?; @@ -492,7 +480,7 @@ fn load_interfaces(elm_cache_dir: &Path) -> Result Result

() - .map_err(move |_parse_error| { - raw_checksum.truncate(60); - CompilerError::CorruptedChecksum(String::from_utf8_lossy(&raw_checksum).to_string()) - })?; - - if existing_checksum == checksum { - return Ok(our_temp_dir); - } + + let mut constraint = ChecksumConstraint::new( + &elm_json_path, + our_temp_dir.join("elm-json-checksum"), + )?; + + if !constraint.dirty()? { + return Ok(our_temp_dir) } - let mut f = std::fs::File::create(&checksum_file) - .map_err(|io_err| CompilerError::WriteOutputFailed(io_err, checksum_file.clone()))?; - ::write(&mut f, checksum.to_string().as_bytes()) - .map_err(|io_err| CompilerError::WriteOutputFailed(io_err, checksum_file.clone()))?; - drop(f); - - let generator_elm_json_path = our_temp_dir.join("elm.json"); - let mut generator_elm_json_file = - std::fs::File::create(&generator_elm_json_path).map_err(|io_err| { - CompilerError::WriteOutputFailed(io_err, generator_elm_json_path.clone()) - })?; - - elm_json_file - .rewind() - .map_err(|io_err| CompilerError::ReadInputFailed(io_err, elm_json_path.clone()))?; - - let mut data = std::fs::read(&elm_json_path) + let mut data = fs::read(&elm_json_path) .map_err(|io_err| CompilerError::ReadInputFailed(io_err, elm_json_path.clone()))?; let mut elm_json = serde_json::from_slice::(&mut data) @@ -773,6 +731,12 @@ fn setup_generator_project(verbosity: u64, elm_project_dir: PathBuf) -> Result

Result

, S: AsRef>( Ok(()) } -fn generate_fixture>( - source_checksum: u64, - target_module: S, - target_function: S, - input: Option, - output: OutputType, -) -> (String, String) { - // The generated fixture Elm files are around 3007 bytes which includes the import of the - // target module and function - let mut buffer = - String::with_capacity(3000 + target_module.as_ref().len() + target_function.as_ref().len()); - - let module_name = format!("Gen{:020}", source_checksum); - - use std::fmt::Write; - write!(buffer, "port module {} exposing (main)\n", &module_name).unwrap(); - buffer.push_str(GENERATOR_IMPORTS); - buffer.push_str("-- START CUSTOMIZED PART\n"); - write!(buffer, "import {}\n\n", target_module.as_ref()).unwrap(); - - // if the input type is none then we don't have to generate an apply, and the port input type - // is () - match output { - OutputType::String => buffer.push_str("encodeOutput = encodeString\n"), - OutputType::Value => buffer.push_str("encodeOutput = encodeJson\n"), - OutputType::Bytes => buffer.push_str("encodeOutput = encodeBytes\n"), - OutputType::Html => buffer.push_str("encodeOutput = encodeHtml\n"), - } - - match input { - None => { - buffer.push_str(&zero_arg_apply( - target_module.as_ref(), - target_function.as_ref(), - )); - } - Some(InputType::Value) => { - buffer.push_str("decodeInput = decodeValue\n"); - buffer.push_str(&one_arg_apply( - target_module.as_ref(), - target_function.as_ref(), - )); - } - Some(InputType::Bytes) => { - buffer.push_str("decodeInput = decodeBytes\n"); - buffer.push_str(&one_arg_apply( - target_module.as_ref(), - target_function.as_ref(), - )); - } - Some(InputType::String) => { - buffer.push_str("decodeInput = decodeString\n"); - buffer.push_str(&one_arg_apply( - target_module.as_ref(), - target_function.as_ref(), - )); - } - } - buffer.push_str("-- END CUSTOMIZED PART\n"); - buffer.push_str(GENERATOR_BODY); - - (module_name, buffer) -} mod runtime { use crate::reporting::InterpreterError; @@ -979,13 +882,13 @@ mod runtime { let ctx = scope.get_current_context(); let global = ctx.global(scope); - let runOnInput = { + let entrypoint = { let x = v8::String::new(scope, "runOnInput") .ok_or(InterpreterError::AllocationFailed)?; v8::Local::new(scope, x).into() }; let v8_value = global - .get(scope, runOnInput) + .get(scope, entrypoint) .ok_or(InterpreterError::ReferenceError)?; // step 12 invoke the function @@ -1017,8 +920,6 @@ mod runtime { function.call(scope, this, &[arg1]); } Some(ValidatedInput::Bytes(data)) => { - // Apparently this works with any type that implements serialize - //let arg1 = serde_v8::to_v8(scope, data).unwrap(); let length = data.len(); let y = data.into_boxed_slice(); @@ -1031,11 +932,8 @@ mod runtime { let arg2 = v8::Local::new(scope, c).into(); - //let x = v8::Local::::try_from(data).unwrap(); - function.call(scope, this, &[arg2]); } - Some(_) => (), } eprintln!("\tcall dispatched {:?}", Instant::now() - start); } @@ -1281,173 +1179,3 @@ fn find_project_root>(filename: S, search: S) -> io::Result(target_module: S, function: S) -> String { - format!( - r#" - -apply : E.Value -> Cmd Msg -apply _ = - {target_module}.{function} - |> encodeOutput - |> onOutput -"#, - target_module = target_module, - function = function, - ) -} - -// The user just has to alias the decodeInput and encodeOutput -fn one_arg_apply(target_module: S, function: S) -> String { - format!( - r#" - -apply : E.Value -> Cmd Msg -apply value = - case decodeInput value of - Ok input -> - input - |> {target_module}.{function} - |> encodeOutput - |> onOutput - - Err error -> - error - |> D.errorToString - |> encodeFailure - |> onOutput -"#, - target_module = target_module, - function = function, - ) -} - -const GENERATOR_BODY: &'static str = r#" --- MAIN - -main = - Platform.worker - { init = init - , update = update - , subscriptions = subscriptions - } - --- PORTS - -port onInput : (E.Value -> msg) -> Sub msg -port onOutput : E.Value -> Cmd msg - --- MODEL - - -type alias Model = () - -init : () -> (Model, Cmd Msg) -init _ = - ((), Cmd.none) - --- UPDATE - -type Msg - = Input E.Value - -update : Msg -> Model -> (Model, Cmd Msg) -update msg model = - case msg of - Input value -> - let - cmd = - apply value - in - (model, cmd) - --- SUBSCRIPTIONS - -subscriptions : Model -> Sub Msg -subscriptions _ = - onInput Input - --- DECODERS - -decodeBytes : E.Value -> Result D.Error Bytes -decodeBytes value = - D.decodeValue decodeBytesHelp value - -decodeBytesHelp : D.Decoder Bytes -decodeBytesHelp = - D.fail "REPLACE_ME_WITH_BYTES_DECODER" - -decodeString : E.Value -> Result D.Error String -decodeString value = - D.decodeValue D.string value - --- ENCODERS - -encodeFailure : String -> E.Value -encodeFailure msg = - E.object - [ ("ctor", E.string "Err") - , ("a", E.string msg) - ] - -encodeSuccess : E.Value -> E.Value -encodeSuccess value = - E.object - [ ("ctor", E.string "Ok") - , ("a", value) - ] - -encodeJson : E.Value -> E.Value -encodeJson = - encodeSuccess - -encodeBytes : Bytes -> E.Value -encodeBytes bytes = - bytes - |> encodeBytesHelp - |> encodeSuccess - -encodeBytesHelp : Bytes -> E.Value -encodeBytesHelp bytes = - E.string "REPLACE_ME_WITH_BYTES_ENCODER" - -encodeString : String -> E.Value -encodeString s = - encodeSuccess (E.string s) - -encodeHtml : Html msg -> E.Value -encodeHtml node = - let - options = - { defaultFormatOptions | newLines = True, indent = 2 } - in - case - D.decodeString - (decodeElmHtml (\taggers eventHandler -> D.succeed ())) - (asJsonString node) - of - Ok elmHtml -> - elmHtml - |> nodeToStringWithOptions options - |> encodeString - - Err error -> - error - |> D.errorToString - |> encodeFailure - - -asJsonString : Html msg -> String -asJsonString x = "REPLACE_ME_WITH_JSON_STRINGIFY" -"#; diff --git a/src/reporting.rs b/src/reporting.rs index b9d5a55..d28bb11 100644 --- a/src/reporting.rs +++ b/src/reporting.rs @@ -1,11 +1,12 @@ use deno_core::error::AnyError; use elmi; -use pretty::{self, cyan, hang, hardline, hcat, hsep, line, sep, space_join, vcat, Doc}; +use pretty::{self, cyan, vcat, Doc}; use rusty_v8; use serde_json; use std::cmp::max; use std::io; use std::path::PathBuf; +use elm_project_utils::RedoScriptError; #[derive(Debug)] pub enum Problem { @@ -52,6 +53,23 @@ impl From for Problem { } } +impl From for Problem { + fn from(error: RedoScriptError) -> Self { + match error { + RedoScriptError::WriteFile { filename, source } => { + CompilerError::WriteOutputFailed(source, filename).into() + } + RedoScriptError::ReadFile { filename, source } => { + CompilerError::ReadInputFailed(source, filename).into() + } + RedoScriptError::NeedDistinctChecksumFile { filename: _ } => { + unimplemented!() + } + } + } +} + + #[derive(Debug)] pub enum TypeError { CantEvalRecord, @@ -77,7 +95,6 @@ pub enum CompilerError { FailedBuildingFixture, ReadInputFailed(io::Error, PathBuf), WriteOutputFailed(io::Error, PathBuf), - CorruptedChecksum(String), WriteElmJsonFailed(serde_json::Error, PathBuf), FailedParseElmJson(serde_json::Error), FailedElmiParse(String), @@ -126,9 +143,9 @@ impl CompilerError { let mut title = "COMPILER ERROR"; use CompilerError::*; let message = match self { - MissingElmJson(io_err) => Doc::text("TODO missing elm.json"), - MissingElmStuff(path) => Doc::text("TODO missing elm-stuff/"), - CantParseModule(first_line_prefix) => Doc::text("todo could not parse module"), + MissingElmJson(_io_err) => Doc::text("TODO missing elm.json"), + MissingElmStuff(_path) => Doc::text("TODO missing elm-stuff/"), + CantParseModule(_first_line_prefix) => Doc::text("todo could not parse module"), EmptyModule => Doc::text("I did not expect the module file to be empty"), MissingModuleTypeInformation(_) => Doc::text("todo missing module type information"), BadImport(_) => { @@ -141,23 +158,20 @@ impl CompilerError { //) } FailedBuildingFixture => Doc::text("TODO failed building fixture elm"), - ReadInputFailed(io_err, path) => { + ReadInputFailed(_io_err, _path) => { title = "IO ERROR"; Doc::text("TODO read file failed") }, WriteOutputFailed(io_err, path) => { Doc::text(format!("TODO write file failed {:?} {:?}", io_err, path)) } - CorruptedChecksum(got_checksum) => { - Doc::text("TODO failed to parse checksum, I expected an Integer") - } - WriteElmJsonFailed(serde_json_err, path) => Doc::text("serialize elm.json failed"), - FailedParseElmJson(serde_json_err) => Doc::text("TODO deserialize elm.json failed"), + WriteElmJsonFailed(_serde_json_err, _path) => Doc::text("serialize elm.json failed"), + FailedParseElmJson(_serde_json_err) => Doc::text("TODO deserialize elm.json failed"), FailedElmiParse(_) => Doc::text("failed to parse .elmi"), BadInput => Doc::text("todo bad input"), }; - vcat([to_message_bar("COMPILER ERROR", ""), Doc::text(""), message]) + vcat([to_message_bar(title, ""), Doc::text(""), message]) } }