oss/starmelon
oss/starmelon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: starmelon reads files or stdin

Browse source
This commit is contained in:
YetAnotherMinion 2021-09-20 07:34:43 +01:00 • committed by nobody
commit ee7ce47f48
Signed by: GrocerPublishAgent
GPG key ID: D460CD54A9E3AB86
7 changed files with 293 additions and 55 deletions
Download patch file Download diff file Expand all files Collapse all files

15
Cargo.toml
View file

@ -5,13 +5,21 @@ edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
# I decided to use os_pipe becaues I want to pipe stdout of a subprocess into
# stderr in real time. I want the outupt of the process to stderr and stdout
# show up in the original order. I looked an os_pipe uses some unsafe code to
# duplicate file descriptors. I am unfamiliar with the correct way of sharing
# the fds. Therefore I am going to trust that their unsafe code is necessary.
# Then it makes sense to use a battle tested unsafe code rather than implement
# it myself.
[dependencies]
structopt = { version = "0.3" }
elmi = { path = "../../../infra/rust-elmi" }
ahash = "0.7"
elmi = { path = "../../../infra/rust-elmi" }
naive-wadler-prettier= { path = "../../../infra/redwood-lang/compiler/naive-wadler-prettier" }
os_pipe = "0.9"
serde = { version = "1.0", features = [ "derive" ] }
serde_json = { version ="1.0", features = [] }
naive-wadler-prettier= { path = "../../../infra/redwood-lang/compiler/naive-wadler-prettier" }
structopt = { version = "0.3" }
# All of these are required for deno's javascript runtime. We need to keep the
# same versions as other projects in our cargo workspace. Multiple different
@ -22,3 +30,4 @@ deno_core = "0.95.0"
deno_web = "0.44"
rusty_v8 = "0.25.3"
futures = "0.3.15"
serde_v8 = "0.8"

1
examples/single-page/elm.json
View file

@ -7,6 +7,7 @@
"dependencies": {
"direct": {
"elm/browser": "1.0.2",
"elm/bytes": "1.0.8",
"elm/core": "1.0.5",
"elm/html": "1.0.0",
"elm/svg": "1.0.1"

0
examples/single-page/src/Empty.elm Normal file
View file

1
examples/single-page/src/InvalidModule.elm Normal file
View file

@ -0,0 +1 @@
not a valid module exposing (..)

23
examples/single-page/src/Main.elm
View file

@ -1,8 +1,11 @@
module Main exposing (view, view2)
module Main exposing (view, view2, view3)
import Html exposing (Html, div, text)
import Svg exposing (Svg, svg)
import Array exposing (Array)
import Bytes exposing (Bytes)
import Bytes.Decode
type alias Model =
{ a : Int
@ -17,3 +20,21 @@ view : String -> Html msg
view model =
div []
[ text <| "Hello world" ++ model ]
view3: Bytes -> Html msg
view3 model =
case
Bytes.Decode.decode
(Bytes.Decode.string (Bytes.width model))
model
of
Just decoded ->
div []
[ text <| "Hello world" ++ decoded ]
Nothing ->
text "Failed to decode"
badReturnType : String -> Int
badReturnType _ =
42

259
src/main.rs
View file

@ -1,6 +1,7 @@
extern crate naive_wadler_prettier as pretty;
use crate::reporting::{CompilerError, InterpreterError, Problem, SetupError, TypeError};
use elmi::DataBinary;
use os_pipe::dup_stderr;
use pretty::pretty;
use serde::{Deserialize, Serialize};
use std::cell::RefCell;
@ -8,7 +9,7 @@ use std::collections::HashMap;
use std::fmt::Display;
use std::fs::{self, canonicalize, metadata};
use std::hash::Hasher;
use std::io::{self, BufRead, Error, ErrorKind, Seek, Write};
use std::io::{self, BufRead, Error, ErrorKind, Read, Seek, Write};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::sync::Arc;
@ -27,11 +28,27 @@ fn main() {
file,
debug,
function,
input,
output,
verbosity,
} => {
if let Err(problem) = exec(file, debug, function, output, verbosity) {
println!("{}", pretty(80, problem.to_doc()));
let start = Instant::now();
if let Err(problem) = exec(file, debug, function, input, output, verbosity) {
eprintln!(
"\t\x1b[1;92mFinished\x1b[0m in {:?}",
Instant::now() - start
);
eprintln!("{}", pretty(80, problem.to_doc()));
} else {
{
let stdout = io::stdout();
let mut handle = stdout.lock();
handle.flush().unwrap();
}
eprintln!(
"\n\t\x1b[1;92mFinished\x1b[0m in {:?}",
Instant::now() - start
);
}
}
}
@ -41,6 +58,7 @@ fn exec(
file: PathBuf,
debug: bool,
function: String,
input_source: Option<PathBuf>,
output: Option<PathBuf>,
verbosity: u64,
) -> Result<(), Problem> {
@ -51,6 +69,16 @@ fn exec(
.arg("/dev/null")
.stdin(Stdio::null());
if verbosity < 1 {
command.stderr(Stdio::null());
command.stdout(Stdio::null());
} else {
let pipe = dup_stderr()
.map_err(|io_err| CompilerError::ReadInputFailed(io_err, "stdout".into()))?;
command.stdout(pipe);
command.stderr(Stdio::piped());
}
if debug {
command.arg("--debug");
}
@ -126,14 +154,14 @@ fn exec(
resolve_function_type(tipe)?
}
None => return Err(CompilerError::CantFindFunction(function).into()),
None => return Err(CompilerError::BadImport(function).into()),
},
None => return Err(CompilerError::MissingModuleTypeInformation(target_module).into()),
};
eprintln!("[{:?}] resolved target function", Instant::now() - start);
// step 6 create our private project
let generator_dir = setup_generator_project(elm_project_dir.clone())?;
let generator_dir = setup_generator_project(verbosity, elm_project_dir.clone())?;
// step 7 create an Elm fixture file to run our function
let start = Instant::now();
@ -170,7 +198,15 @@ fn exec(
.arg("--output")
.arg(&intermediate_file)
.current_dir(&generator_dir)
.stdin(Stdio::null());
.stdin(Stdio::null())
.stderr(Stdio::piped());
if verbosity < 1 {
command.stdout(Stdio::piped());
} else {
let pipe = dup_stderr()
.map_err(|io_err| CompilerError::ReadInputFailed(io_err, "stdout".into()))?;
command.stdout(pipe);
}
if debug {
command.arg("--debug");
@ -178,9 +214,9 @@ fn exec(
command.arg(&source_filename);
let start = Instant::now();
match command.status() {
Ok(exit_status) => {
if !exit_status.success() {
match command.output() {
Ok(output) => {
if !output.status.success() {
return Err(CompilerError::FailedBuildingFixture.into());
}
}
@ -188,11 +224,13 @@ fn exec(
return Err(CompilerError::FailedBuildingFixture.into());
}
}
eprintln!(
"[{:?}] compiled {:?}",
Instant::now() - start,
intermediate_file
);
if verbosity > 1 {
eprintln!(
"compiled {:?} in {:?}",
intermediate_file,
Instant::now() - start,
);
}
// Step 9 fixup the compiled script to run in Deno
@ -202,19 +240,51 @@ fn exec(
// TODO figure out how to replace multiple substrings in a single pass. One neat trick
// might be to allocate enough space in our starting buffer to write the new code by
// having a really large replace block. For example if we are replacing a string
// `"REPLACE_ME" + "abc" * 2000` we have over 6k bytes to write out the new code. We
// `"REPLACE_ME" + "abc" * 2000` we would have over 6k bytes to write out the new code. We
// will have to do some extra book keeping to make sure the buffer space is big enough
// for the replacement code.
let mut final_script = data
.replace("'REPLACE_ME_WITH_JSON_STRINGIFY'", "JSON.stringify(x)")
.replace(
"$elm$json$Json$Decode$fail('REPLACE_ME_WITH_BYTES_DECODER');",
r#" _Json_decodePrim(function(value) {
return (typeof value === 'object' && value instanceof DataView)
? $elm$core$Result$Ok(value)
: _Json_expecting('a DataView', value);
});"#,
)
.replace(";}(this));", ";}(globalThis));");
final_script.push_str("\n\n");
final_script.push_str(&format!("var worker = Elm.{}.init();\n", gen_module_name));
// add a short cut for invoking the function so I don't have to traverse so many object
// lookups using the rust v8 API.
final_script
.push_str("globalThis.runOnInput = function(data) { worker.ports.onInput.send(data) };\n");
match input_type {
None => {
final_script.push_str(
"globalThis.runOnInput = function() { worker.ports.onInput.send(null)) };\n",
);
}
Some(InputType::Value) => {
final_script
.push_str("globalThis.runOnInput = function(data) { worker.ports.onInput.send(JSON.parse(data)) };\n");
}
Some(InputType::String) => {
final_script.push_str(
"globalThis.runOnInput = function(data) { worker.ports.onInput.send(data) };",
);
}
Some(InputType::Bytes) => {
final_script.push_str(
r#"
globalThis.runOnInput = function(data) {
const dv = new DataView(data.buffer)
worker.ports.onInput.send(dv)
};
"#,
);
}
}
match output_type {
OutputType::Value => {
@ -312,13 +382,64 @@ fn exec(
worker.js_runtime.sync_ops_cache();
// step 11 marshal the input into the v8 isolate. If we are reading from an
// input file load that, if we are reading from stdin read that.
let input = match (input_source, input_type) {
(None, None) => None,
(Some(path), None) => None,
(None, Some(input_type)) => {
// read from stdin when input_source was not present
let mut stdin = io::stdin();
let mut buffer = Vec::new();
stdin.read_to_end(&mut buffer).unwrap();
match input_type {
InputType::String => {
let s = String::from_utf8(buffer).map_err(|_| CompilerError::BadInput)?;
Some(ValidatedInput::String(s))
}
InputType::Bytes => Some(ValidatedInput::Bytes(buffer)),
InputType::Value => {
let _: serde_json::Value =
serde_json::from_slice(&buffer).map_err(|_| CompilerError::BadInput)?;
let s = String::from_utf8(buffer).map_err(|_| CompilerError::BadInput)?;
Some(ValidatedInput::Value(s))
}
}
}
(Some(path), Some(input_type)) => {
let buffer = std::fs::read(&path)
.map_err(|io_err| CompilerError::ReadInputFailed(io_err, path.clone()))?;
match input_type {
InputType::String => {
let s = String::from_utf8(buffer).map_err(|_| CompilerError::BadInput)?;
Some(ValidatedInput::String(s))
}
InputType::Bytes => Some(ValidatedInput::Bytes(buffer)),
InputType::Value => {
let _: serde_json::Value =
serde_json::from_slice(&buffer).map_err(|_| CompilerError::BadInput)?;
let s = String::from_utf8(buffer).map_err(|_| CompilerError::BadInput)?;
Some(ValidatedInput::Value(s))
}
}
}
};
let start = Instant::now();
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap()
.block_on(async move { runtime::xyz(worker, main_module).await })?;
eprintln!("[{:?}] eval javascript", Instant::now() - start);
.block_on(async move { runtime::xyz(worker, main_module, input).await })?;
eprintln!("eval javascript in {:?}", Instant::now() - start);
// step 13 receive the callback
// If I understood which combination of run_event_loop was required to execute
@ -433,6 +554,8 @@ enum Arguments {
#[structopt(long)]
debug: bool,
#[structopt(long)]
input: Option<PathBuf>,
#[structopt(long)]
output: Option<PathBuf>,
#[structopt(short = "v", parse(from_occurrences))]
verbosity: u64,
@ -446,6 +569,12 @@ enum InputType {
Bytes,
}
pub enum ValidatedInput {
String(String),
Value(String),
Bytes(Vec<u8>),
}
#[derive(Debug, Copy, Clone)]
enum OutputType {
Html,
@ -547,7 +676,7 @@ fn resolve_output_type(tipe: &elmi::Type) -> Result<OutputType, TypeError> {
}
}
fn setup_generator_project(elm_project_dir: PathBuf) -> Result<PathBuf, Problem> {
fn setup_generator_project(verbosity: u64, elm_project_dir: PathBuf) -> Result<PathBuf, Problem> {
// I added a couple of random bytes to the directory name to reduce the risk of
// collisions with other programs that also use elm-stuff for their scratch space
let our_temp_dir = elm_project_dir.join("elm-stuff").join("starmelon-5d9ecc");
@ -566,8 +695,6 @@ fn setup_generator_project(elm_project_dir: PathBuf) -> Result<PathBuf, Problem>
let elm_json_path = elm_project_dir.join("elm.json");
let mut elm_json_file = std::fs::File::open(&elm_json_path)
.map_err(|io_err| CompilerError::ReadInputFailed(io_err, elm_json_path.clone()))?;
// TODO CompilerError::ElmJsonChecksumFailed(io::Error, PathBuf),
// ReadInputFailed(io::Error, PathBuf),
let mut hasher = PortableHash::new();
std::io::copy(&mut elm_json_file, &mut hasher).unwrap();
@ -640,9 +767,9 @@ fn setup_generator_project(elm_project_dir: PathBuf) -> Result<PathBuf, Problem>
CompilerError::WriteElmJsonFailed(io_err, generator_elm_json_path.clone())
})?;
elm_install(&our_temp_dir, "ThinkAlexandria/elm-html-in-elm")?;
elm_install(&our_temp_dir, "elm/json")?;
elm_install(&our_temp_dir, "elm/bytes")?;
elm_install(verbosity, &our_temp_dir, "ThinkAlexandria/elm-html-in-elm")?;
elm_install(verbosity, &our_temp_dir, "elm/json")?;
elm_install(verbosity, &our_temp_dir, "elm/bytes")?;
Ok(our_temp_dir)
}
@ -655,14 +782,30 @@ struct ElmApplication {
other_fields: serde_json::Value,
}
fn elm_install<P: AsRef<Path>, S: AsRef<str>>(our_temp_dir: P, package: S) -> Result<(), Problem> {
let mut child = Command::new("elm")
fn elm_install<P: AsRef<Path>, S: AsRef<str>>(
verbosity: u64,
our_temp_dir: P,
package: S,
) -> Result<(), Problem> {
let mut command = Command::new("elm");
command
.arg("install")
.arg(package.as_ref())
.stdin(Stdio::piped())
.current_dir(&our_temp_dir)
.spawn()
.unwrap();
.current_dir(&our_temp_dir);
if verbosity < 1 {
command.stderr(Stdio::null());
command.stdout(Stdio::null());
} else {
let pipe = dup_stderr()
.map_err(|io_err| CompilerError::ReadInputFailed(io_err, "stdout".into()))?;
command.stdout(pipe);
command.stderr(Stdio::piped());
}
let mut child = command.spawn().unwrap();
let child_stdin = child.stdin.as_mut().unwrap();
child_stdin.write_all(b"y\n").unwrap();
@ -748,6 +891,7 @@ fn generate_fixture<S: AsRef<str>>(
mod runtime {
use crate::reporting::InterpreterError;
use crate::ValidatedInput;
use deno_core::error::{type_error, AnyError};
use deno_core::futures::FutureExt;
use deno_core::{resolve_url, FsModuleLoader, ModuleLoader, ModuleSpecifier, OpState};
@ -813,6 +957,7 @@ mod runtime {
pub async fn xyz(
mut worker: MainWorker,
main_module: ModuleSpecifier,
input: Option<ValidatedInput>,
) -> Result<(), InterpreterError> {
let wait_for_inspector = false;
// step 10 load the module into our v8 isolate
@ -833,23 +978,55 @@ mod runtime {
.get(scope, runOnInput)
.ok_or(InterpreterError::ReferenceError)?;
// step 11 marshal the input into the v8 isolate. If we are reading from an
// input file load that, if we are reading from stdin read that.
let string_input = "Foo bar zap".to_owned();
// step 12 invoke the function
let function = v8::Local::<v8::Function>::try_from(v8_value)?;
let this = v8::undefined(scope).into();
let arg1 = {
let x = v8::String::new(scope, &string_input)
.ok_or(InterpreterError::AllocationFailed)?;
v8::Local::new(scope, x).into()
};
let start = Instant::now();
function.call(scope, this, &[arg1]);
match input {
None => {
function.call(scope, this, &[]);
}
Some(ValidatedInput::String(s)) => {
let arg1 = {
let x =
v8::String::new(scope, &s).ok_or(InterpreterError::AllocationFailed)?;
v8::Local::new(scope, x).into()
};
function.call(scope, this, &[arg1]);
}
Some(ValidatedInput::Value(v)) => {
let arg1 = {
let x =
v8::String::new(scope, &v).ok_or(InterpreterError::AllocationFailed)?;
v8::Local::new(scope, x).into()
};
function.call(scope, this, &[arg1]);
}
Some(ValidatedInput::Bytes(data)) => {
// Apparently this works with any type that implements serialize
//let arg1 = serde_v8::to_v8(scope, data).unwrap();
let length = data.len();
let y = data.into_boxed_slice();
let k = v8::ArrayBuffer::new_backing_store_from_boxed_slice(y).make_shared();
let x = v8::ArrayBuffer::with_backing_store(scope, &k);
let arg1 = v8::Local::new(scope, x).into();
let c = v8::Uint8Array::new(scope, arg1, 0, length).unwrap();
let arg2 = v8::Local::new(scope, c).into();
//let x = v8::Local::<v8::Uint8Array>::try_from(data).unwrap();
function.call(scope, this, &[arg2]);
}
Some(_) => (),
}
eprintln!("\tcall dispatched {:?}", Instant::now() - start);
}
worker.run_event_loop(wait_for_inspector).await?;

49
src/reporting.rs
View file

@ -1,11 +1,10 @@
use deno_core::error::AnyError;
use elmi;
use pretty::{self, cyan, hang, hardline, hcat, hsep, sep, space_join, Doc};
use pretty::{self, cyan, hang, hardline, hcat, hsep, line, sep, space_join, vcat, Doc};
use rusty_v8;
use serde_json;
use std::cmp::max;
use std::io;
use std::path::Path;
use std::path::PathBuf;
#[derive(Debug)]
@ -74,15 +73,15 @@ pub enum CompilerError {
CantParseModule(String), // first line
EmptyModule,
MissingModuleTypeInformation(String),
CantFindFunction(String),
BadImport(String),
FailedBuildingFixture,
ElmJsonChecksumFailed(io::Error, PathBuf),
ReadInputFailed(io::Error, PathBuf),
CorruptedChecksum(String),
WriteOutputFailed(io::Error, PathBuf),
CorruptedChecksum(String),
WriteElmJsonFailed(serde_json::Error, PathBuf),
FailedParseElmJson(serde_json::Error),
FailedElmiParse(String),
BadInput,
}
#[derive(Debug)]
@ -124,7 +123,37 @@ impl SetupError {
impl CompilerError {
pub fn to_doc(&self) -> Doc {
Doc::text("compiler error")
use CompilerError::*;
let message = match self {
MissingElmJson(io_err) => Doc::text("TODO missing elm.json"),
MissingElmStuff(path) => Doc::text("TODO missing elm-stuff/"),
CantParseModule(first_line_prefix) => Doc::text("todo could not parse module"),
EmptyModule => Doc::text("I did not expect the module file to be empty"),
MissingModuleTypeInformation(_) => Doc::text("todo missing module type information"),
BadImport(_) => {
// TODO suggest alternatives using edit distance
Doc::text("The `Html` module does not expose `view5`")
// TODO
// Doc::text("These names seem close though")
//Doc::group(
// Doc::nest(4, Doc::concat(Doc::Line, x)),
//)
}
FailedBuildingFixture => Doc::text("TODO failed building fixture elm"),
ReadInputFailed(io_err, path) => Doc::text("TODO read file failed"),
WriteOutputFailed(io_err, path) => {
Doc::text(format!("TODO write file failed {:?} {:?}", io_err, path))
}
CorruptedChecksum(got_checksum) => {
Doc::text("TODO failed to parse checksum, I expected an Integer")
}
WriteElmJsonFailed(serde_json_err, path) => Doc::text("serialize elm.json failed"),
FailedParseElmJson(serde_json_err) => Doc::text("TODO deserialize elm.json failed"),
FailedElmiParse(_) => Doc::text("failed to parse .elmi"),
BadInput => Doc::text("todo bad input"),
};
vcat([to_message_bar("COMPILER ERROR", ""), Doc::text(""), message])
}
}
@ -134,16 +163,16 @@ impl InterpreterError {
}
}
fn to_message_bar(title: String, file_path: &Path) -> Doc {
let used_space = 4 + title.len() + 1 + file_path.to_string_lossy().len();
fn to_message_bar<S1: AsRef<str>, S2: AsRef<str>>(title: S1, file_path: S2) -> Doc {
let used_space = 4 + title.as_ref().len() + 1 + file_path.as_ref().len();
cyan(Doc::text(format!(
"-- {} {} {}",
title,
title.as_ref(),
std::iter::repeat('-')
.take(max(1, 80 - used_space))
.collect::<String>(),
file_path.display(),
file_path.as_ref(),
)))
}