feat: starmelon reads files or stdin

Cargo.toml

@@ -5,13 +5,21 @@ edition = "2018"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+# I decided to use os_pipe becaues I want to pipe stdout of a subprocess into
+# stderr in real time. I want the outupt of the process to stderr and stdout
+# show up in the original order. I looked an os_pipe uses some unsafe code to
+# duplicate file descriptors. I am unfamiliar with the correct way of sharing
+# the fds. Therefore I am going to trust that their unsafe code is necessary.
+# Then it makes sense to use a battle tested unsafe code rather than implement
+# it myself.
 [dependencies]
-structopt = { version = "0.3" }
-elmi = { path = "../../../infra/rust-elmi" }
 ahash = "0.7"
+elmi = { path = "../../../infra/rust-elmi" }
+naive-wadler-prettier= { path = "../../../infra/redwood-lang/compiler/naive-wadler-prettier" }
+os_pipe = "0.9"
 serde = { version = "1.0", features = [ "derive" ] }
 serde_json = { version ="1.0", features = [] }
-naive-wadler-prettier= { path = "../../../infra/redwood-lang/compiler/naive-wadler-prettier" }
+structopt = { version = "0.3" }
 
 # All of these are required for deno's javascript runtime. We need to keep the
 # same versions as other projects in our cargo workspace. Multiple different
@@ -22,3 +30,4 @@ deno_core = "0.95.0"
 deno_web = "0.44"
 rusty_v8 = "0.25.3"
 futures = "0.3.15"
+serde_v8 = "0.8"

examples/single-page/elm.json

@@ -7,6 +7,7 @@
     "dependencies": {
         "direct": {
             "elm/browser": "1.0.2",
+            "elm/bytes": "1.0.8",
             "elm/core": "1.0.5",
             "elm/html": "1.0.0",
             "elm/svg": "1.0.1"

examples/single-page/src/InvalidModule.elm

@@ -0,0 +1 @@
+not a valid module exposing (..)

examples/single-page/src/Main.elm

@@ -1,8 +1,11 @@
-module Main exposing (view, view2)
+module Main exposing (view, view2, view3)
 
 import Html exposing (Html, div, text)
 import Svg exposing (Svg, svg)
 import Array exposing (Array)
+import Bytes exposing (Bytes)
+import Bytes.Decode
+
 
 type alias Model =
     { a : Int
@@ -17,3 +20,21 @@ view : String -> Html msg
 view model =
     div []
         [ text <| "Hello world" ++ model ]
+
+view3: Bytes -> Html msg
+view3 model =
+    case
+        Bytes.Decode.decode
+            (Bytes.Decode.string (Bytes.width model))
+            model
+    of
+        Just decoded ->
+            div []
+                [ text <| "Hello world" ++ decoded ]
+
+        Nothing ->
+            text "Failed to decode"
+
+badReturnType : String -> Int
+badReturnType _ =
+    42

src/main.rs

@@ -1,6 +1,7 @@
 extern crate naive_wadler_prettier as pretty;
 use crate::reporting::{CompilerError, InterpreterError, Problem, SetupError, TypeError};
 use elmi::DataBinary;
+use os_pipe::dup_stderr;
 use pretty::pretty;
 use serde::{Deserialize, Serialize};
 use std::cell::RefCell;
@@ -8,7 +9,7 @@ use std::collections::HashMap;
 use std::fmt::Display;
 use std::fs::{self, canonicalize, metadata};
 use std::hash::Hasher;
-use std::io::{self, BufRead, Error, ErrorKind, Seek, Write};
+use std::io::{self, BufRead, Error, ErrorKind, Read, Seek, Write};
 use std::path::{Path, PathBuf};
 use std::process::{Command, Stdio};
 use std::sync::Arc;
@@ -27,11 +28,27 @@ fn main() {
             file,
             debug,
             function,
+            input,
             output,
             verbosity,
         } => {
-            if let Err(problem) = exec(file, debug, function, output, verbosity) {
+            let start = Instant::now();
-                println!("{}", pretty(80, problem.to_doc()));
+            if let Err(problem) = exec(file, debug, function, input, output, verbosity) {
+                eprintln!(
+                    "\t\x1b[1;92mFinished\x1b[0m in {:?}",
+                    Instant::now() - start
+                );
+                eprintln!("{}", pretty(80, problem.to_doc()));
+            } else {
+                {
+                    let stdout = io::stdout();
+                    let mut handle = stdout.lock();
+                    handle.flush().unwrap();
+                }
+                eprintln!(
+                    "\n\t\x1b[1;92mFinished\x1b[0m in {:?}",
+                    Instant::now() - start
+                );
             }
         }
     }
@@ -41,6 +58,7 @@ fn exec(
     file: PathBuf,
     debug: bool,
     function: String,
+    input_source: Option<PathBuf>,
     output: Option<PathBuf>,
     verbosity: u64,
 ) -> Result<(), Problem> {
@@ -51,6 +69,16 @@ fn exec(
         .arg("/dev/null")
         .stdin(Stdio::null());
 
+    if verbosity < 1 {
+        command.stderr(Stdio::null());
+        command.stdout(Stdio::null());
+    } else {
+        let pipe = dup_stderr()
+            .map_err(|io_err| CompilerError::ReadInputFailed(io_err, "stdout".into()))?;
+        command.stdout(pipe);
+        command.stderr(Stdio::piped());
+    }
+
     if debug {
         command.arg("--debug");
     }
@@ -126,14 +154,14 @@ fn exec(
 
                 resolve_function_type(tipe)?
             }
-            None => return Err(CompilerError::CantFindFunction(function).into()),
+            None => return Err(CompilerError::BadImport(function).into()),
         },
         None => return Err(CompilerError::MissingModuleTypeInformation(target_module).into()),
     };
     eprintln!("[{:?}] resolved target function", Instant::now() - start);
 
     // step 6 create our private project
-    let generator_dir = setup_generator_project(elm_project_dir.clone())?;
+    let generator_dir = setup_generator_project(verbosity, elm_project_dir.clone())?;
 
     // step 7 create an Elm fixture file to run our function
     let start = Instant::now();
@@ -170,7 +198,15 @@ fn exec(
         .arg("--output")
         .arg(&intermediate_file)
         .current_dir(&generator_dir)
-        .stdin(Stdio::null());
+        .stdin(Stdio::null())
+        .stderr(Stdio::piped());
+    if verbosity < 1 {
+        command.stdout(Stdio::piped());
+    } else {
+        let pipe = dup_stderr()
+            .map_err(|io_err| CompilerError::ReadInputFailed(io_err, "stdout".into()))?;
+        command.stdout(pipe);
+    }
 
     if debug {
         command.arg("--debug");
@@ -178,9 +214,9 @@ fn exec(
     command.arg(&source_filename);
 
     let start = Instant::now();
-    match command.status() {
+    match command.output() {
-        Ok(exit_status) => {
+        Ok(output) => {
-            if !exit_status.success() {
+            if !output.status.success() {
                 return Err(CompilerError::FailedBuildingFixture.into());
             }
         }
@@ -188,11 +224,13 @@ fn exec(
             return Err(CompilerError::FailedBuildingFixture.into());
         }
     }
-    eprintln!(
+    if verbosity > 1 {
-        "[{:?}] compiled {:?}",
+        eprintln!(
-        Instant::now() - start,
+            "compiled {:?} in {:?}",
-        intermediate_file
+            intermediate_file,
-    );
+            Instant::now() - start,
+        );
+    }
 
     // Step 9 fixup the compiled script to run in Deno
 
@@ -202,19 +240,51 @@ fn exec(
     // TODO figure out how to replace multiple substrings in a single pass. One neat trick
     // might be to allocate enough space in our starting buffer to write the new code by
     // having a really large replace block. For example if we are replacing a string
-    // `"REPLACE_ME" + "abc" * 2000` we have over 6k bytes to write out the new code. We
+    // `"REPLACE_ME" + "abc" * 2000` we would have over 6k bytes to write out the new code. We
     // will have to do some extra book keeping to make sure the buffer space is big enough
     // for the replacement code.
     let mut final_script = data
         .replace("'REPLACE_ME_WITH_JSON_STRINGIFY'", "JSON.stringify(x)")
+        .replace(
+            "$elm$json$Json$Decode$fail('REPLACE_ME_WITH_BYTES_DECODER');",
+            r#" _Json_decodePrim(function(value) {
+    return (typeof value === 'object' && value instanceof DataView)
+        ? $elm$core$Result$Ok(value)
+            : _Json_expecting('a DataView', value);
+});"#,
+        )
         .replace(";}(this));", ";}(globalThis));");
 
     final_script.push_str("\n\n");
     final_script.push_str(&format!("var worker = Elm.{}.init();\n", gen_module_name));
     // add a short cut for invoking the function so I don't have to traverse so many object
     // lookups using the rust v8 API.
-    final_script
+    match input_type {
-        .push_str("globalThis.runOnInput = function(data) { worker.ports.onInput.send(data) };\n");
+        None => {
+            final_script.push_str(
+                "globalThis.runOnInput = function() { worker.ports.onInput.send(null)) };\n",
+            );
+        }
+        Some(InputType::Value) => {
+            final_script
+                .push_str("globalThis.runOnInput = function(data) { worker.ports.onInput.send(JSON.parse(data)) };\n");
+        }
+        Some(InputType::String) => {
+            final_script.push_str(
+                "globalThis.runOnInput = function(data) { worker.ports.onInput.send(data) };",
+            );
+        }
+        Some(InputType::Bytes) => {
+            final_script.push_str(
+                r#"
+globalThis.runOnInput = function(data) {
+    const dv = new DataView(data.buffer)
+    worker.ports.onInput.send(dv)
+};
+"#,
+            );
+        }
+    }
 
     match output_type {
         OutputType::Value => {
@@ -312,13 +382,64 @@ fn exec(
 
     worker.js_runtime.sync_ops_cache();
 
+    // step 11 marshal the input into the v8 isolate. If we are reading from an
+    // input file load that, if we are reading from stdin read that.
+    let input = match (input_source, input_type) {
+        (None, None) => None,
+        (Some(path), None) => None,
+        (None, Some(input_type)) => {
+            // read from stdin when input_source was not present
+            let mut stdin = io::stdin();
+            let mut buffer = Vec::new();
+            stdin.read_to_end(&mut buffer).unwrap();
+
+            match input_type {
+                InputType::String => {
+                    let s = String::from_utf8(buffer).map_err(|_| CompilerError::BadInput)?;
+
+                    Some(ValidatedInput::String(s))
+                }
+                InputType::Bytes => Some(ValidatedInput::Bytes(buffer)),
+                InputType::Value => {
+                    let _: serde_json::Value =
+                        serde_json::from_slice(&buffer).map_err(|_| CompilerError::BadInput)?;
+
+                    let s = String::from_utf8(buffer).map_err(|_| CompilerError::BadInput)?;
+
+                    Some(ValidatedInput::Value(s))
+                }
+            }
+        }
+        (Some(path), Some(input_type)) => {
+            let buffer = std::fs::read(&path)
+                .map_err(|io_err| CompilerError::ReadInputFailed(io_err, path.clone()))?;
+
+            match input_type {
+                InputType::String => {
+                    let s = String::from_utf8(buffer).map_err(|_| CompilerError::BadInput)?;
+
+                    Some(ValidatedInput::String(s))
+                }
+                InputType::Bytes => Some(ValidatedInput::Bytes(buffer)),
+                InputType::Value => {
+                    let _: serde_json::Value =
+                        serde_json::from_slice(&buffer).map_err(|_| CompilerError::BadInput)?;
+
+                    let s = String::from_utf8(buffer).map_err(|_| CompilerError::BadInput)?;
+
+                    Some(ValidatedInput::Value(s))
+                }
+            }
+        }
+    };
+
     let start = Instant::now();
     tokio::runtime::Builder::new_current_thread()
         .enable_all()
         .build()
         .unwrap()
-        .block_on(async move { runtime::xyz(worker, main_module).await })?;
+        .block_on(async move { runtime::xyz(worker, main_module, input).await })?;
-    eprintln!("[{:?}] eval javascript", Instant::now() - start);
+    eprintln!("eval javascript in {:?}", Instant::now() - start);
 
     // step 13 receive the callback
     // If I understood which combination of run_event_loop was required to execute
@@ -433,6 +554,8 @@ enum Arguments {
         #[structopt(long)]
         debug: bool,
         #[structopt(long)]
+        input: Option<PathBuf>,
+        #[structopt(long)]
         output: Option<PathBuf>,
         #[structopt(short = "v", parse(from_occurrences))]
         verbosity: u64,
@@ -446,6 +569,12 @@ enum InputType {
     Bytes,
 }
 
+pub enum ValidatedInput {
+    String(String),
+    Value(String),
+    Bytes(Vec<u8>),
+}
+
 #[derive(Debug, Copy, Clone)]
 enum OutputType {
     Html,
@@ -547,7 +676,7 @@ fn resolve_output_type(tipe: &elmi::Type) -> Result<OutputType, TypeError> {
     }
 }
 
-fn setup_generator_project(elm_project_dir: PathBuf) -> Result<PathBuf, Problem> {
+fn setup_generator_project(verbosity: u64, elm_project_dir: PathBuf) -> Result<PathBuf, Problem> {
     // I added a couple of random bytes to the directory name to reduce the risk of
     // collisions with other programs that also use elm-stuff for their scratch space
     let our_temp_dir = elm_project_dir.join("elm-stuff").join("starmelon-5d9ecc");
@@ -566,8 +695,6 @@ fn setup_generator_project(elm_project_dir: PathBuf) -> Result<PathBuf, Problem>
     let elm_json_path = elm_project_dir.join("elm.json");
     let mut elm_json_file = std::fs::File::open(&elm_json_path)
         .map_err(|io_err| CompilerError::ReadInputFailed(io_err, elm_json_path.clone()))?;
-    // TODO CompilerError::ElmJsonChecksumFailed(io::Error, PathBuf),
-    // ReadInputFailed(io::Error, PathBuf),
 
     let mut hasher = PortableHash::new();
     std::io::copy(&mut elm_json_file, &mut hasher).unwrap();
@@ -640,9 +767,9 @@ fn setup_generator_project(elm_project_dir: PathBuf) -> Result<PathBuf, Problem>
         CompilerError::WriteElmJsonFailed(io_err, generator_elm_json_path.clone())
     })?;
 
-    elm_install(&our_temp_dir, "ThinkAlexandria/elm-html-in-elm")?;
+    elm_install(verbosity, &our_temp_dir, "ThinkAlexandria/elm-html-in-elm")?;
-    elm_install(&our_temp_dir, "elm/json")?;
+    elm_install(verbosity, &our_temp_dir, "elm/json")?;
-    elm_install(&our_temp_dir, "elm/bytes")?;
+    elm_install(verbosity, &our_temp_dir, "elm/bytes")?;
 
     Ok(our_temp_dir)
 }
@@ -655,14 +782,30 @@ struct ElmApplication {
     other_fields: serde_json::Value,
 }
 
-fn elm_install<P: AsRef<Path>, S: AsRef<str>>(our_temp_dir: P, package: S) -> Result<(), Problem> {
+fn elm_install<P: AsRef<Path>, S: AsRef<str>>(
-    let mut child = Command::new("elm")
+    verbosity: u64,
+    our_temp_dir: P,
+    package: S,
+) -> Result<(), Problem> {
+    let mut command = Command::new("elm");
+
+    command
         .arg("install")
         .arg(package.as_ref())
         .stdin(Stdio::piped())
-        .current_dir(&our_temp_dir)
+        .current_dir(&our_temp_dir);
-        .spawn()
+
-        .unwrap();
+    if verbosity < 1 {
+        command.stderr(Stdio::null());
+        command.stdout(Stdio::null());
+    } else {
+        let pipe = dup_stderr()
+            .map_err(|io_err| CompilerError::ReadInputFailed(io_err, "stdout".into()))?;
+        command.stdout(pipe);
+        command.stderr(Stdio::piped());
+    }
+
+    let mut child = command.spawn().unwrap();
 
     let child_stdin = child.stdin.as_mut().unwrap();
     child_stdin.write_all(b"y\n").unwrap();
@@ -748,6 +891,7 @@ fn generate_fixture<S: AsRef<str>>(
 
 mod runtime {
     use crate::reporting::InterpreterError;
+    use crate::ValidatedInput;
     use deno_core::error::{type_error, AnyError};
     use deno_core::futures::FutureExt;
     use deno_core::{resolve_url, FsModuleLoader, ModuleLoader, ModuleSpecifier, OpState};
@@ -813,6 +957,7 @@ mod runtime {
     pub async fn xyz(
         mut worker: MainWorker,
         main_module: ModuleSpecifier,
+        input: Option<ValidatedInput>,
     ) -> Result<(), InterpreterError> {
         let wait_for_inspector = false;
         // step 10 load the module into our v8 isolate
@@ -833,23 +978,55 @@ mod runtime {
                 .get(scope, runOnInput)
                 .ok_or(InterpreterError::ReferenceError)?;
 
-            // step 11 marshal the input into the v8 isolate. If we are reading from an
-            // input file load that, if we are reading from stdin read that.
-            let string_input = "Foo bar zap".to_owned();
-
             // step 12 invoke the function
             let function = v8::Local::<v8::Function>::try_from(v8_value)?;
 
             let this = v8::undefined(scope).into();
 
-            let arg1 = {
-                let x = v8::String::new(scope, &string_input)
-                    .ok_or(InterpreterError::AllocationFailed)?;
-                v8::Local::new(scope, x).into()
-            };
-
             let start = Instant::now();
-            function.call(scope, this, &[arg1]);
+            match input {
+                None => {
+                    function.call(scope, this, &[]);
+                }
+                Some(ValidatedInput::String(s)) => {
+                    let arg1 = {
+                        let x =
+                            v8::String::new(scope, &s).ok_or(InterpreterError::AllocationFailed)?;
+                        v8::Local::new(scope, x).into()
+                    };
+
+                    function.call(scope, this, &[arg1]);
+                }
+                Some(ValidatedInput::Value(v)) => {
+                    let arg1 = {
+                        let x =
+                            v8::String::new(scope, &v).ok_or(InterpreterError::AllocationFailed)?;
+                        v8::Local::new(scope, x).into()
+                    };
+
+                    function.call(scope, this, &[arg1]);
+                }
+                Some(ValidatedInput::Bytes(data)) => {
+                    // Apparently this works with any type that implements serialize
+                    //let arg1 = serde_v8::to_v8(scope, data).unwrap();
+                    let length = data.len();
+
+                    let y = data.into_boxed_slice();
+                    let k = v8::ArrayBuffer::new_backing_store_from_boxed_slice(y).make_shared();
+                    let x = v8::ArrayBuffer::with_backing_store(scope, &k);
+
+                    let arg1 = v8::Local::new(scope, x).into();
+
+                    let c = v8::Uint8Array::new(scope, arg1, 0, length).unwrap();
+
+                    let arg2 = v8::Local::new(scope, c).into();
+
+                    //let x = v8::Local::<v8::Uint8Array>::try_from(data).unwrap();
+
+                    function.call(scope, this, &[arg2]);
+                }
+                Some(_) => (),
+            }
             eprintln!("\tcall dispatched {:?}", Instant::now() - start);
         }
         worker.run_event_loop(wait_for_inspector).await?;

src/reporting.rs

@@ -1,11 +1,10 @@
 use deno_core::error::AnyError;
 use elmi;
-use pretty::{self, cyan, hang, hardline, hcat, hsep, sep, space_join, Doc};
+use pretty::{self, cyan, hang, hardline, hcat, hsep, line, sep, space_join, vcat, Doc};
 use rusty_v8;
 use serde_json;
 use std::cmp::max;
 use std::io;
-use std::path::Path;
 use std::path::PathBuf;
 
 #[derive(Debug)]
@@ -74,15 +73,15 @@ pub enum CompilerError {
     CantParseModule(String), // first line
     EmptyModule,
     MissingModuleTypeInformation(String),
-    CantFindFunction(String),
+    BadImport(String),
     FailedBuildingFixture,
-    ElmJsonChecksumFailed(io::Error, PathBuf),
     ReadInputFailed(io::Error, PathBuf),
-    CorruptedChecksum(String),
     WriteOutputFailed(io::Error, PathBuf),
+    CorruptedChecksum(String),
     WriteElmJsonFailed(serde_json::Error, PathBuf),
     FailedParseElmJson(serde_json::Error),
     FailedElmiParse(String),
+    BadInput,
 }
 
 #[derive(Debug)]
@@ -124,7 +123,37 @@ impl SetupError {
 
 impl CompilerError {
     pub fn to_doc(&self) -> Doc {
-        Doc::text("compiler error")
+        use CompilerError::*;
+        let message = match self {
+            MissingElmJson(io_err) => Doc::text("TODO missing elm.json"),
+            MissingElmStuff(path) => Doc::text("TODO missing elm-stuff/"),
+            CantParseModule(first_line_prefix) => Doc::text("todo could not parse module"),
+            EmptyModule => Doc::text("I did not expect the module file to be empty"),
+            MissingModuleTypeInformation(_) => Doc::text("todo missing module type information"),
+            BadImport(_) => {
+                // TODO suggest alternatives using edit distance
+                Doc::text("The `Html` module does not expose `view5`")
+                // TODO
+                // Doc::text("These names seem close though")
+                //Doc::group(
+                //  Doc::nest(4, Doc::concat(Doc::Line, x)),
+                //)
+            }
+            FailedBuildingFixture => Doc::text("TODO failed building fixture elm"),
+            ReadInputFailed(io_err, path) => Doc::text("TODO read file failed"),
+            WriteOutputFailed(io_err, path) => {
+                Doc::text(format!("TODO write file failed {:?} {:?}", io_err, path))
+            }
+            CorruptedChecksum(got_checksum) => {
+                Doc::text("TODO failed to parse checksum, I expected an Integer")
+            }
+            WriteElmJsonFailed(serde_json_err, path) => Doc::text("serialize elm.json failed"),
+            FailedParseElmJson(serde_json_err) => Doc::text("TODO deserialize elm.json failed"),
+            FailedElmiParse(_) => Doc::text("failed to parse .elmi"),
+            BadInput => Doc::text("todo bad input"),
+        };
+
+        vcat([to_message_bar("COMPILER ERROR", ""), Doc::text(""), message])
     }
 }
 
@@ -134,16 +163,16 @@ impl InterpreterError {
     }
 }
 
-fn to_message_bar(title: String, file_path: &Path) -> Doc {
+fn to_message_bar<S1: AsRef<str>, S2: AsRef<str>>(title: S1, file_path: S2) -> Doc {
-    let used_space = 4 + title.len() + 1 + file_path.to_string_lossy().len();
+    let used_space = 4 + title.as_ref().len() + 1 + file_path.as_ref().len();
 
     cyan(Doc::text(format!(
         "-- {} {} {}",
-        title,
+        title.as_ref(),
         std::iter::repeat('-')
             .take(max(1, 80 - used_space))
             .collect::<String>(),
-        file_path.display(),
+        file_path.as_ref(),
     )))
 }