From c7657ce8019be60e7c4f1945784bba1180a0f0c0 Mon Sep 17 00:00:00 2001 From: Riley-Kilgore Date: Tue, 14 Jan 2025 05:16:08 -0800 Subject: [PATCH] Addressed comments on benchmarking PR --- crates/aiken-lang/src/format.rs | 57 +++---- .../src/parser/definition/benchmark.rs | 123 +-------------- .../aiken-lang/src/parser/definition/mod.rs | 1 + .../aiken-lang/src/parser/definition/test.rs | 121 +-------------- .../src/parser/definition/test_like.rs | 143 ++++++++++++++++++ crates/aiken-lang/src/test_framework.rs | 22 ++- crates/aiken-lang/src/tipo/error.rs | 17 +++ crates/aiken-lang/src/tipo/infer.rs | 16 +- crates/aiken-project/src/lib.rs | 47 +----- crates/aiken-project/src/options.rs | 1 - crates/aiken-project/src/telemetry.rs | 1 - crates/aiken-project/src/telemetry/json.rs | 24 +++ .../aiken-project/src/telemetry/terminal.rs | 21 ++- crates/aiken-project/src/test_framework.rs | 1 - crates/aiken/src/cmd/benchmark.rs | 6 - crates/aiken/src/cmd/mod.rs | 2 +- crates/aiken/src/main.rs | 2 +- examples/acceptance_tests/117/lib/tests.ak | 1 + 18 files changed, 254 insertions(+), 352 deletions(-) create mode 100644 crates/aiken-lang/src/parser/definition/test_like.rs diff --git a/crates/aiken-lang/src/format.rs b/crates/aiken-lang/src/format.rs index dea702471..c65ad6027 100644 --- a/crates/aiken-lang/src/format.rs +++ b/crates/aiken-lang/src/format.rs @@ -606,8 +606,9 @@ impl<'comments> Formatter<'comments> { } #[allow(clippy::too_many_arguments)] - fn definition_test<'a>( + fn definition_test_or_bench<'a>( &mut self, + keyword: &'static str, name: &'a str, args: &'a [UntypedArgVia], body: &'a UntypedExpr, @@ -615,14 +616,19 @@ impl<'comments> Formatter<'comments> { on_test_failure: &'a OnTestFailure, ) -> Document<'a> { // Fn name and args - let head = "test " + let head = keyword .to_doc() + .append(" ") .append(name) .append(wrap_args(args.iter().map(|e| (self.fn_arg_via(e), false)))) - .append(match on_test_failure { - OnTestFailure::FailImmediately => "", - OnTestFailure::SucceedEventually => " fail", - OnTestFailure::SucceedImmediately => " fail once", + .append(if keyword == "test" { + match on_test_failure { + OnTestFailure::FailImmediately => "", + OnTestFailure::SucceedEventually => " fail", + OnTestFailure::SucceedImmediately => " fail once", + } + } else { + "" }) .group(); @@ -643,7 +649,7 @@ impl<'comments> Formatter<'comments> { } #[allow(clippy::too_many_arguments)] - fn definition_benchmark<'a>( + fn definition_test<'a>( &mut self, name: &'a str, args: &'a [UntypedArgVia], @@ -651,32 +657,19 @@ impl<'comments> Formatter<'comments> { end_location: usize, on_test_failure: &'a OnTestFailure, ) -> Document<'a> { - // Fn name and args - let head = "bench " - .to_doc() - .append(name) - .append(wrap_args(args.iter().map(|e| (self.fn_arg_via(e), false)))) - .append(match on_test_failure { - OnTestFailure::FailImmediately => "", - OnTestFailure::SucceedEventually => "", - OnTestFailure::SucceedImmediately => "", - }) - .group(); - - // Format body - let body = self.expr(body, true); - - // Add any trailing comments - let body = match printed_comments(self.pop_comments(end_location), false) { - Some(comments) => body.append(line()).append(comments), - None => body, - }; + self.definition_test_or_bench("test", name, args, body, end_location, on_test_failure) + } - // Stick it all together - head.append(" {") - .append(line().append(body).nest(INDENT).group()) - .append(line()) - .append("}") + #[allow(clippy::too_many_arguments)] + fn definition_benchmark<'a>( + &mut self, + name: &'a str, + args: &'a [UntypedArgVia], + body: &'a UntypedExpr, + end_location: usize, + on_test_failure: &'a OnTestFailure, + ) -> Document<'a> { + self.definition_test_or_bench("bench", name, args, body, end_location, on_test_failure) } fn definition_validator<'a>( diff --git a/crates/aiken-lang/src/parser/definition/benchmark.rs b/crates/aiken-lang/src/parser/definition/benchmark.rs index c09609e24..185827e94 100644 --- a/crates/aiken-lang/src/parser/definition/benchmark.rs +++ b/crates/aiken-lang/src/parser/definition/benchmark.rs @@ -1,130 +1,11 @@ use crate::{ ast, - ast::OnTestFailure, - expr::UntypedExpr, - parser::{ - annotation, - chain::{call::parser as call, field_access, tuple_index::parser as tuple_index, Chain}, - error::ParseError, - expr::{self, bytearray, int as uint, list, string, tuple, var}, - pattern, - token::Token, - }, + parser::{error::ParseError, token::Token}, }; use chumsky::prelude::*; pub fn parser() -> impl Parser { - just(Token::Benchmark) - .ignore_then(select! {Token::Name {name} => name}) - .then( - via() - .separated_by(just(Token::Comma)) - .allow_trailing() - .delimited_by(just(Token::LeftParen), just(Token::RightParen)), - ) - .then( - just(Token::Fail) - .ignore_then(just(Token::Once).ignored().or_not().map(|once| { - once.map(|_| OnTestFailure::SucceedImmediately) - .unwrap_or(OnTestFailure::SucceedEventually) - })) - .or_not(), - ) - .map_with_span(|name, span| (name, span)) - .then( - expr::sequence() - .or_not() - .delimited_by(just(Token::LeftBrace), just(Token::RightBrace)), - ) - .map_with_span(|((((name, arguments), fail), span_end), body), span| { - ast::UntypedDefinition::Benchmark(ast::Function { - arguments, - body: body.unwrap_or_else(|| UntypedExpr::todo(None, span)), - doc: None, - location: span_end, - end_position: span.end - 1, - name, - public: false, - return_annotation: None, - return_type: (), - on_test_failure: fail.unwrap_or(OnTestFailure::FailImmediately), - }) - }) -} - -pub fn via() -> impl Parser { - choice(( - select! {Token::DiscardName {name} => name}.map_with_span(|name, span| { - ast::ArgBy::ByName(ast::ArgName::Discarded { - label: name.clone(), - name, - location: span, - }) - }), - select! {Token::Name {name} => name}.map_with_span(|name, location| { - ast::ArgBy::ByName(ast::ArgName::Named { - label: name.clone(), - name, - location, - }) - }), - pattern().map(ast::ArgBy::ByPattern), - )) - .then(just(Token::Colon).ignore_then(annotation()).or_not()) - .map_with_span(|(arg_name, annotation), location| (arg_name, annotation, location)) - .then_ignore(just(Token::Via)) - .then(fuzzer()) - .map(|((by, annotation, location), via)| ast::ArgVia { - arg: ast::UntypedArg { - by, - annotation, - location, - doc: None, - is_validator_param: false, - }, - via, - }) -} - -pub fn fuzzer<'a>() -> impl Parser + 'a { - recursive(|expression| { - let chain = choice(( - tuple_index(), - field_access::parser(), - call(expression.clone()), - )); - - let int = || { - just(Token::Minus) - .to(ast::UnOp::Negate) - .map_with_span(|op, span| (op, span)) - .or_not() - .then(uint()) - .map(|(op, value)| match op { - None => value, - Some((op, location)) => UntypedExpr::UnOp { - op, - location, - value: Box::new(value), - }, - }) - }; - - choice(( - int(), - string(), - bytearray(), - tuple(expression.clone()), - list(expression.clone()), - var(), - )) - .then(chain.repeated()) - .foldl(|expr, chain| match chain { - Chain::Call(args, span) => expr.call(args, span), - Chain::FieldAccess(label, span) => expr.field_access(label, span), - Chain::TupleIndex(index, span) => expr.tuple_index(index, span), - }) - }) + crate::parser::definition::test_like::parser(Token::Benchmark) } #[cfg(test)] diff --git a/crates/aiken-lang/src/parser/definition/mod.rs b/crates/aiken-lang/src/parser/definition/mod.rs index 843d2cb6a..cf54b2464 100644 --- a/crates/aiken-lang/src/parser/definition/mod.rs +++ b/crates/aiken-lang/src/parser/definition/mod.rs @@ -6,6 +6,7 @@ mod data_type; mod function; pub mod import; mod test; +pub mod test_like; mod type_alias; mod validator; diff --git a/crates/aiken-lang/src/parser/definition/test.rs b/crates/aiken-lang/src/parser/definition/test.rs index 6e4cada87..2f3d1ec31 100644 --- a/crates/aiken-lang/src/parser/definition/test.rs +++ b/crates/aiken-lang/src/parser/definition/test.rs @@ -1,131 +1,14 @@ use crate::{ ast, - ast::OnTestFailure, - expr::UntypedExpr, - parser::{ - annotation, - chain::{call::parser as call, field_access, tuple_index::parser as tuple_index, Chain}, - error::ParseError, - expr::{self, bytearray, int as uint, list, string, tuple, var}, - pattern, - token::Token, - }, + parser::{error::ParseError, token::Token}, }; use chumsky::prelude::*; pub fn parser() -> impl Parser { - just(Token::Test) - .ignore_then(select! {Token::Name {name} => name}) - .then( - via() - .separated_by(just(Token::Comma)) - .allow_trailing() - .delimited_by(just(Token::LeftParen), just(Token::RightParen)), - ) - .then( - just(Token::Fail) - .ignore_then(just(Token::Once).ignored().or_not().map(|once| { - once.map(|_| OnTestFailure::SucceedImmediately) - .unwrap_or(OnTestFailure::SucceedEventually) - })) - .or_not(), - ) - .map_with_span(|name, span| (name, span)) - .then( - expr::sequence() - .or_not() - .delimited_by(just(Token::LeftBrace), just(Token::RightBrace)), - ) - .map_with_span(|((((name, arguments), fail), span_end), body), span| { - ast::UntypedDefinition::Test(ast::Function { - arguments, - body: body.unwrap_or_else(|| UntypedExpr::todo(None, span)), - doc: None, - location: span_end, - end_position: span.end - 1, - name, - public: false, - return_annotation: None, - return_type: (), - on_test_failure: fail.unwrap_or(OnTestFailure::FailImmediately), - }) - }) + crate::parser::definition::test_like::parser(Token::Test) } -pub fn via() -> impl Parser { - choice(( - select! {Token::DiscardName {name} => name}.map_with_span(|name, span| { - ast::ArgBy::ByName(ast::ArgName::Discarded { - label: name.clone(), - name, - location: span, - }) - }), - select! {Token::Name {name} => name}.map_with_span(|name, location| { - ast::ArgBy::ByName(ast::ArgName::Named { - label: name.clone(), - name, - location, - }) - }), - pattern().map(ast::ArgBy::ByPattern), - )) - .then(just(Token::Colon).ignore_then(annotation()).or_not()) - .map_with_span(|(arg_name, annotation), location| (arg_name, annotation, location)) - .then_ignore(just(Token::Via)) - .then(fuzzer()) - .map(|((by, annotation, location), via)| ast::ArgVia { - arg: ast::UntypedArg { - by, - annotation, - location, - doc: None, - is_validator_param: false, - }, - via, - }) -} - -pub fn fuzzer<'a>() -> impl Parser + 'a { - recursive(|expression| { - let chain = choice(( - tuple_index(), - field_access::parser(), - call(expression.clone()), - )); - let int = || { - just(Token::Minus) - .to(ast::UnOp::Negate) - .map_with_span(|op, span| (op, span)) - .or_not() - .then(uint()) - .map(|(op, value)| match op { - None => value, - Some((op, location)) => UntypedExpr::UnOp { - op, - location, - value: Box::new(value), - }, - }) - }; - - choice(( - int(), - string(), - bytearray(), - tuple(expression.clone()), - list(expression.clone()), - var(), - )) - .then(chain.repeated()) - .foldl(|expr, chain| match chain { - Chain::Call(args, span) => expr.call(args, span), - Chain::FieldAccess(label, span) => expr.field_access(label, span), - Chain::TupleIndex(index, span) => expr.tuple_index(index, span), - }) - }) -} #[cfg(test)] mod tests { diff --git a/crates/aiken-lang/src/parser/definition/test_like.rs b/crates/aiken-lang/src/parser/definition/test_like.rs new file mode 100644 index 000000000..0e8d3772a --- /dev/null +++ b/crates/aiken-lang/src/parser/definition/test_like.rs @@ -0,0 +1,143 @@ +use crate::{ + ast, + ast::OnTestFailure, + expr::UntypedExpr, + parser::{ + annotation, + chain::{call::parser as call, field_access, tuple_index::parser as tuple_index, Chain}, + error::ParseError, + expr::{self, bytearray, int as uint, list, string, tuple, var}, + pattern, + token::Token, + }, +}; +use chumsky::prelude::*; + +pub fn parser(keyword: Token) -> impl Parser { + just(keyword.clone()) + .ignore_then(select! {Token::Name {name} => name}) + .then( + via() + .separated_by(just(Token::Comma)) + .allow_trailing() + .delimited_by(just(Token::LeftParen), just(Token::RightParen)), + ) + .then( + just(Token::Fail) + .ignore_then(just(Token::Once).ignored().or_not().map(|once| { + once.map(|_| OnTestFailure::SucceedImmediately) + .unwrap_or(OnTestFailure::SucceedEventually) + })) + .or_not(), + ) + .map_with_span(|name, span| (name, span)) + .then( + expr::sequence() + .or_not() + .delimited_by(just(Token::LeftBrace), just(Token::RightBrace)), + ) + .map_with_span(move |((((name, arguments), fail), span_end), body), span| { + match keyword { + Token::Test => ast::UntypedDefinition::Test(ast::Function { + arguments, + body: body.unwrap_or_else(|| UntypedExpr::todo(None, span)), + doc: None, + location: span_end, + end_position: span.end - 1, + name, + public: false, + return_annotation: None, + return_type: (), + on_test_failure: fail.unwrap_or(OnTestFailure::FailImmediately), + }), + Token::Benchmark => ast::UntypedDefinition::Benchmark(ast::Function { + arguments, + body: body.unwrap_or_else(|| UntypedExpr::todo(None, span)), + doc: None, + location: span_end, + end_position: span.end - 1, + name, + public: false, + return_annotation: None, + return_type: (), + on_test_failure: fail.unwrap_or(OnTestFailure::FailImmediately), + }), + _ => unreachable!("Only Test and Benchmark tokens are supported"), + } + }) +} + +pub fn via() -> impl Parser { + choice(( + select! {Token::DiscardName {name} => name}.map_with_span(|name, span| { + ast::ArgBy::ByName(ast::ArgName::Discarded { + label: name.clone(), + name, + location: span, + }) + }), + select! {Token::Name {name} => name}.map_with_span(|name, location| { + ast::ArgBy::ByName(ast::ArgName::Named { + label: name.clone(), + name, + location, + }) + }), + pattern().map(ast::ArgBy::ByPattern), + )) + .then(just(Token::Colon).ignore_then(annotation()).or_not()) + .map_with_span(|(arg_name, annotation), location| (arg_name, annotation, location)) + .then_ignore(just(Token::Via)) + .then(fuzzer()) + .map(|((by, annotation, location), via)| ast::ArgVia { + arg: ast::UntypedArg { + by, + annotation, + location, + doc: None, + is_validator_param: false, + }, + via, + }) +} + +pub fn fuzzer<'a>() -> impl Parser + 'a { + recursive(|expression| { + let chain = choice(( + tuple_index(), + field_access::parser(), + call(expression.clone()), + )); + + let int = || { + just(Token::Minus) + .to(ast::UnOp::Negate) + .map_with_span(|op, span| (op, span)) + .or_not() + .then(uint()) + .map(|(op, value)| match op { + None => value, + Some((op, location)) => UntypedExpr::UnOp { + op, + location, + value: Box::new(value), + }, + }) + }; + + choice(( + int(), + string(), + bytearray(), + tuple(expression.clone()), + list(expression.clone()), + var(), + )) + .then(chain.repeated()) + .foldl(|expr, chain| match chain { + Chain::Call(args, span) => expr.call(args, span), + Chain::FieldAccess(label, span) => expr.field_access(label, span), + Chain::TupleIndex(index, span) => expr.tuple_index(index, span), + }) + }) +} \ No newline at end of file diff --git a/crates/aiken-lang/src/test_framework.rs b/crates/aiken-lang/src/test_framework.rs index 668a61d9c..85f9abae0 100644 --- a/crates/aiken-lang/src/test_framework.rs +++ b/crates/aiken-lang/src/test_framework.rs @@ -382,7 +382,7 @@ impl PropertyTest { let mut counterexample = Counterexample { value, choices: next_prng.choices(), - cache: Cache::new(move |choices| { + cache: Cache::new(|choices| { match Prng::from_choices(choices).sample(&self.fuzzer.program) { Err(..) => Status::Invalid, Ok(None) => Status::Invalid, @@ -442,6 +442,18 @@ impl PropertyTest { /// ----- Benchmark ----------------------------------------------------------------- +#[derive(Debug, Clone)] +pub struct Sampler { + pub program: Program, + + pub type_info: Rc, + + /// A version of the Fuzzer's type that has gotten rid of + /// all erasable opaque type. This is needed in order to + /// generate Plutus data with the appropriate shape. + pub stripped_type_info: Rc, +} + #[derive(Debug, Clone)] pub struct Benchmark { pub input_path: PathBuf, @@ -449,7 +461,7 @@ pub struct Benchmark { pub name: String, pub on_test_failure: OnTestFailure, pub program: Program, - pub sampler: Fuzzer, + pub sampler: Sampler, } unsafe impl Send for Benchmark {} @@ -458,14 +470,14 @@ impl Benchmark { pub fn benchmark( self, seed: u32, - n: usize, + max_iterations: usize, plutus_version: &PlutusVersion, ) -> Vec { - let mut results = Vec::with_capacity(n); + let mut results = Vec::with_capacity(max_iterations); let mut iteration = 0; let mut prng = Prng::from_seed(seed); - while n > iteration { + while max_iterations > iteration { let fuzzer = self .sampler .program diff --git a/crates/aiken-lang/src/tipo/error.rs b/crates/aiken-lang/src/tipo/error.rs index 44f365460..7d99ae040 100644 --- a/crates/aiken-lang/src/tipo/error.rs +++ b/crates/aiken-lang/src/tipo/error.rs @@ -1452,6 +1452,21 @@ fn suggest_unify( expected, given }, + Some(UnifyErrorSituation::SamplerAnnotationMismatch) => formatdoc! { + r#"While comparing the return annotation of a Sampler with its actual return type, I realized that both don't match. + + I am inferring the Sampler should return: + + {} + + but I found a conflicting annotation saying it returns: + + {} + + Either, fix (or remove) the annotation or adjust the Sampler to return the expected type."#, + expected, + given + }, None => formatdoc! { r#"I am inferring the following type: @@ -1869,6 +1884,8 @@ pub enum UnifyErrorSituation { Operator(BinOp), FuzzerAnnotationMismatch, + + SamplerAnnotationMismatch, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/crates/aiken-lang/src/tipo/infer.rs b/crates/aiken-lang/src/tipo/infer.rs index d901e1362..4431ff7e7 100644 --- a/crates/aiken-lang/src/tipo/infer.rs +++ b/crates/aiken-lang/src/tipo/infer.rs @@ -358,15 +358,12 @@ fn infer_definition( .map(|ann| hydrator.type_from_annotation(ann, environment)) .transpose()?; - let (inferred_annotation, inferred_inner_type) = match infer_fuzzer( + let (inferred_annotation, inferred_inner_type) = infer_fuzzer( environment, provided_inner_type.clone(), &typed_via.tipo(), &arg.via.location(), - ) { - Ok(result) => Ok(result), - Err(err) => Err(err), - }?; + )?; // Ensure that the annotation, if any, matches the type inferred from the // Fuzzer. @@ -494,15 +491,12 @@ fn infer_definition( .map(|ann| hydrator.type_from_annotation(ann, environment)) .transpose()?; - let (inferred_annotation, inferred_inner_type) = match infer_sampler( + let (inferred_annotation, inferred_inner_type) = infer_sampler( environment, provided_inner_type.clone(), &typed_via.tipo(), &arg.via.location(), - ) { - Ok(result) => Ok(result), - Err(err) => Err(err), - }?; + )?; // Ensure that the annotation, if any, matches the type inferred from the // Fuzzer. @@ -518,7 +512,7 @@ fn infer_definition( location: arg.arg.location, expected: inferred_inner_type.clone(), given: provided_inner_type.clone(), - situation: Some(UnifyErrorSituation::FuzzerAnnotationMismatch), + situation: Some(UnifyErrorSituation::SamplerAnnotationMismatch), rigid_type_names: hydrator.rigid_names(), }); } diff --git a/crates/aiken-project/src/lib.rs b/crates/aiken-project/src/lib.rs index 5a3dc2d30..dbf9999e0 100644 --- a/crates/aiken-project/src/lib.rs +++ b/crates/aiken-project/src/lib.rs @@ -304,7 +304,6 @@ where seed: u32, times_to_run: usize, env: Option, - output: PathBuf, ) -> Result<(), Vec> { let options = Options { tracing: Tracing::silent(), @@ -314,7 +313,6 @@ where exact_match, seed, times_to_run, - output, }, blueprint_path: self.blueprint_path(None), }; @@ -427,7 +425,7 @@ where seed, property_max_success, } => { - let tests = self.collect_tests(false, match_tests, exact_match, options.tracing)?; + let tests = self.collect_tests(verbose, match_tests, exact_match, options.tracing)?; if !tests.is_empty() { self.event_listener.handle_event(Event::RunningTests); @@ -471,9 +469,7 @@ where exact_match, seed, times_to_run, - output, } => { - // todo - collect benchmarks let tests = self.collect_benchmarks(false, match_tests, exact_match, options.tracing)?; @@ -496,51 +492,12 @@ where self.event_listener.handle_event(Event::FinishedBenchmarks { seed, - tests: tests.clone(), + tests, }); if !errors.is_empty() { Err(errors) } else { - // Write benchmark results to CSV - use std::fs::File; - use std::io::Write; - - let mut writer = File::create(&output).map_err(|error| { - vec![Error::FileIo { - error, - path: output.clone(), - }] - })?; - - // Write CSV header - writeln!(writer, "test_name,module,memory,cpu").map_err(|error| { - vec![Error::FileIo { - error, - path: output.clone(), - }] - })?; - - // Write benchmark results - for test in tests { - if let TestResult::Benchmark(result) = test { - writeln!( - writer, - "{},{},{},{}", - result.test.name, - result.test.module, - result.cost.mem, - result.cost.cpu - ) - .map_err(|error| { - vec![Error::FileIo { - error, - path: output.clone(), - }] - })?; - } - } - Ok(()) } } diff --git a/crates/aiken-project/src/options.rs b/crates/aiken-project/src/options.rs index 2ccf5fb2d..2afa69ed4 100644 --- a/crates/aiken-project/src/options.rs +++ b/crates/aiken-project/src/options.rs @@ -34,7 +34,6 @@ pub enum CodeGenMode { exact_match: bool, seed: u32, times_to_run: usize, - output: PathBuf, }, NoOp, } diff --git a/crates/aiken-project/src/telemetry.rs b/crates/aiken-project/src/telemetry.rs index f08034005..6e49b7c9e 100644 --- a/crates/aiken-project/src/telemetry.rs +++ b/crates/aiken-project/src/telemetry.rs @@ -135,7 +135,6 @@ pub(crate) fn find_max_execution_units(xs: &[TestResult]) -> (usize, us } } TestResult::Benchmark(..) => { - // todo riley - should this be reachable? unreachable!("property returned benchmark result ?!") } }); diff --git a/crates/aiken-project/src/telemetry/json.rs b/crates/aiken-project/src/telemetry/json.rs index 05ea57085..4b9f807a7 100644 --- a/crates/aiken-project/src/telemetry/json.rs +++ b/crates/aiken-project/src/telemetry/json.rs @@ -39,6 +39,30 @@ impl EventListener for Json { }); println!("{}", serde_json::to_string_pretty(&json_output).unwrap()); } + Event::FinishedBenchmarks { tests, seed } => { + let benchmark_results: Vec<_> = tests + .into_iter() + .filter_map(|test| { + if let TestResult::Benchmark(result) = test { + Some(serde_json::json!({ + "name": result.test.name, + "module": result.test.module, + "memory": result.cost.mem, + "cpu": result.cost.cpu + })) + } else { + None + } + }) + .collect(); + + let json = serde_json::json!({ + "benchmarks": benchmark_results, + "seed": seed, + }); + + println!("{}", serde_json::to_string_pretty(&json).unwrap()); + } _ => super::Terminal.handle_event(event), } } diff --git a/crates/aiken-project/src/telemetry/terminal.rs b/crates/aiken-project/src/telemetry/terminal.rs index bdb19dddb..c005b6b6d 100644 --- a/crates/aiken-project/src/telemetry/terminal.rs +++ b/crates/aiken-project/src/telemetry/terminal.rs @@ -224,14 +224,19 @@ impl EventListener for Terminal { "...".if_supports_color(Stderr, |s| s.bold()) ); } - Event::FinishedBenchmarks { seed: _, tests: _ } => { - eprintln!( - "{} {}", - " Complete" - .if_supports_color(Stderr, |s| s.bold()) - .if_supports_color(Stderr, |s| s.green()), - "benchmark results written to CSV".if_supports_color(Stderr, |s| s.bold()) - ); + Event::FinishedBenchmarks { tests, .. } => { + for test in tests { + if let TestResult::Benchmark(result) = test { + println!( + "{} {} ", + result.test.name.bold(), + "BENCH".blue(), + ); + println!(" Memory: {} bytes", result.cost.mem); + println!(" CPU: {} units", result.cost.cpu); + println!(); + } + } } } } diff --git a/crates/aiken-project/src/test_framework.rs b/crates/aiken-project/src/test_framework.rs index 38db882ae..bf5f744bb 100644 --- a/crates/aiken-project/src/test_framework.rs +++ b/crates/aiken-project/src/test_framework.rs @@ -289,7 +289,6 @@ mod test { result.labels ) } - // todo riley - should this be reachable? TestResult::Benchmark(..) => unreachable!("property returned benchmark result ?!"), } } diff --git a/crates/aiken/src/cmd/benchmark.rs b/crates/aiken/src/cmd/benchmark.rs index 1ba3506a2..65d7a60d7 100644 --- a/crates/aiken/src/cmd/benchmark.rs +++ b/crates/aiken/src/cmd/benchmark.rs @@ -34,10 +34,6 @@ pub struct Args { /// Environment to use for benchmarking env: Option, - - /// Output file for benchmark results - #[clap(short, long)] - output: PathBuf, } pub fn exec( @@ -48,7 +44,6 @@ pub fn exec( seed, times_to_run, env, - output, }: Args, ) -> miette::Result<()> { let mut rng = rand::thread_rng(); @@ -67,7 +62,6 @@ pub fn exec( seed, times_to_run, env.clone(), - output.clone(), ) }, ); diff --git a/crates/aiken/src/cmd/mod.rs b/crates/aiken/src/cmd/mod.rs index a46758197..0d99bdf50 100644 --- a/crates/aiken/src/cmd/mod.rs +++ b/crates/aiken/src/cmd/mod.rs @@ -36,7 +36,7 @@ pub enum Cmd { Docs(docs::Args), Add(packages::add::Args), - Benchmark(benchmark::Args), + Bench(benchmark::Args), #[clap(subcommand)] Blueprint(blueprint::Cmd), diff --git a/crates/aiken/src/main.rs b/crates/aiken/src/main.rs index 8e1fce359..c58b5c083 100644 --- a/crates/aiken/src/main.rs +++ b/crates/aiken/src/main.rs @@ -24,7 +24,7 @@ fn main() -> miette::Result<()> { Cmd::Build(args) => build::exec(args), Cmd::Address(args) => address::exec(args), Cmd::Check(args) => check::exec(args), - Cmd::Benchmark(args) => benchmark::exec(args), + Cmd::Bench(args) => benchmark::exec(args), Cmd::Docs(args) => docs::exec(args), Cmd::Add(args) => add::exec(args), Cmd::Blueprint(args) => blueprint::exec(args), diff --git a/examples/acceptance_tests/117/lib/tests.ak b/examples/acceptance_tests/117/lib/tests.ak index 822e49ad2..90c1b39f8 100644 --- a/examples/acceptance_tests/117/lib/tests.ak +++ b/examples/acceptance_tests/117/lib/tests.ak @@ -1,3 +1,4 @@ +// TODO: KtorZ/Riley - This should fail the type checker fn simple_sampler(): Sampler { fn(n: Int) { fn(prng: PRNG) {