Merge pull request #37 from HoneyPony/test-cleanup

Test cleanup

Author: HoneyPony

build/build_tests.rs

@@ -1,23 +1,37 @@
+use std::collections::HashMap;
 use std::fs::File;
 use std::io::Write as _;
 
-fn generate_test(file: &mut File, path: &str, test_name: &str) -> std::io::Result<()> {
-	writeln!(file, "#[test]")?;
-	writeln!(file, "fn {test_name}() {{")?;
+struct BuiltTests {
+	modules: HashMap<String, String>
+}
+
+fn generate_test(bt: &mut BuiltTests, path: &str, test_name: &str) {
+	let out = if let Some(out) = bt.modules.get_mut(path) {
+		out
+	}
+	else {
+		bt.modules.insert(path.to_string(), "".to_string());
+		bt.modules.get_mut(path).unwrap()
+	};
+
+	use std::fmt::Write;
+
+	writeln!(out, "\t#[test]").unwrap();
+	writeln!(out, "\tfn {test_name}() {{").unwrap();
 
 	// We let the generated test code actually do the concat!.
 	let exe_path = format!("concat!(env!(\"CARGO_TARGET_TMPDIR\"), \"/{test_name}\")");
 
-	writeln!(file, "\trun_integration_test(\"tests/poni/{path}{test_name}.poni\", {exe_path});")?;
-	writeln!(file, "}}")?;
-
-	Ok(())
+	writeln!(out, "\t\trun_integration_test(\"tests/{path}{test_name}.poni\", {exe_path});").unwrap();
+	writeln!(out, "\t}}").unwrap();
 }
 
 pub fn generate(tests_file: &mut File) {
 	let tests = [
 		("binary/", "binary_doubleblock"),
 		("binary/", "binary_bottom"),
+		("binary/", "binary_parens"),
 
 		// print_nested: the last line is 345 because it should re-print each of the inner print()s.
 		("print/", "print_nested"),
@@ -30,6 +44,9 @@ pub fn generate(tests_file: &mut File) {
 		("print/", "print_dif_funs"),
 		("print/", "print_inner_return"),
 
+		("print/", "err_print_empty"),
+
+
 		("globals/", "global_block"),
 		// TODO: Figure out precise float output format we want.
 		("globals/", "globals_3"),
@@ -38,16 +55,29 @@ pub fn generate(tests_file: &mut File) {
 		("globals/", "global_increment_indirect"),
 		("globals/", "global_mutate"),
 
+		("globals/", "err_global_fun_redefine"),
+		("globals/", "err_global_redefine"),
+		("globals/", "errTODO_bad_order"),
+
 		("typecheck/", "print_assume_int"),
 		("typecheck/", "promote_assumes_inside_block"),
 		("typecheck/", "promote_to_float_arithmetic"),
 		("typecheck/", "promote_to_float_assign"),
+		("typecheck/", "promote_to_float_block_assign_assumeint"),
+		("typecheck/", "promote_to_float_block_assign"),
+		("typecheck/", "promote_to_float_block_return"),
+		("typecheck/", "promote_to_float_return"),
 		("typecheck/", "str_types"),
 		("typecheck/", "return_block_return"), // Make sure this one at least compiles
-		// TODO: Add other tests when we get function calls
+
+		("typecheck/", "err_try_assign_float_for_int"),
+		("typecheck/", "err_try_return_float_for_int_short"),
+		("typecheck/", "err_try_return_float_for_int"),
+
 
 		("string/", "simple_str"),
 		("string/", "str_of_strbuf"),
+		("string/", "very_simple_str"),
 
 		("functions/", "parse_params"),
 		("functions/", "param_trailing_comma"),
@@ -57,20 +87,29 @@ pub fn generate(tests_file: &mut File) {
 		("functions/", "void_fun"),
 		("functions/", "fib"),
 
+		("functions/", "err_assign_to_fun"),
+
 		("scope/", "block_shadow"),
 
 		("if/", "basic_if_expr_ret"),
 		("if/", "basic_if_expr_var"),
 		("if/", "basic_if_expr"),
 		("if/", "basic_if"),
+		("if/", "if_extra_parens"),
 		("if/", "if_no_else"),
 		("if/", "if_no_else_in_print"),
 		("if/", "if_fun_calls"),
 
+		("if/", "err_if_bad_condition"),
+		("if/", "err_if_incompat_types"),
+		("if/", "err_if_no_else_bad_type"),
+
 		("comparison/", "compare_basic"),
 		("comparison/", "compare_constants"),
 		("comparison/", "compare_doubleblock"),
 
+		("lexer/", "err_unterminated_string"),
+
 		("logical/", "basic_and"),
 		("logical/", "basic_or"),
 		("logical/", "short_and"),
@@ -81,6 +120,17 @@ pub fn generate(tests_file: &mut File) {
 		("logical/", "short_or_expr_doubleblock"),
 		("logical/", "or_bottom"),
 
+		("misc/", "complex_return_in_binop"),
+		("misc/", "err_return_in_binop"),
+		("misc/", "err_top_level_return"),
+		("misc/", "noerr_return_in_binop"),
+		("misc/", "simple_var_exprs_and_infer"),
+		("misc/", "test_init"),
+		("misc/", "unused_expr"),
+
+		("parser/", "err_fun_missing_brace"),
+		("parser/", "err_missing_expr_paren"),
+
 		("call/", "call_captured_rev"),
 		("call/", "call_captured"),
 		("call/", "call_if_simple"),
@@ -98,6 +148,7 @@ pub fn generate(tests_file: &mut File) {
 		("call/", "uses_fun_with_class_retval"),
 		("call/", "uses_fun_with_class_param"),
 
+		("classes/", "basic_class"),
 		("classes/", "basic_new_inferred_get"),
 		("classes/", "basic_new_inferred_get_promote"),
 		("classes/", "basic_new_explicit_get"),
@@ -119,8 +170,24 @@ pub fn generate(tests_file: &mut File) {
 		("classes/", "basic_new_list"),
 		("classes/", "class_member_that_is_fun"),
 		("classes/", "class_member_function_capture"),
+		("classes/", "noout_data_and_fun_assign"),
+		("classes/", "noout_data_and_fun_read"),
+		("classes/", "noout_pure_data_complex"),
+		("classes/", "noout_pure_data_initializers"),
+		("classes/", "noout_pure_data"),
+
+		("classes/", "err_assign_to_class"),
+		("classes/", "err_get_nonexistent_member"),
+		("classes/", "err_new_unknown_property"),
+		("classes/", "err_new_wrong_ty_known"),
+		("classes/", "err_new_wrong_ty_unknown"),
+		("classes/", "err_pure_data_wrongty"),
+		("classes/", "err_try_to_read_class_in_initializer"),
+		("classes/", "err_weird_var"),
 
 		("get/", "get_string_length"),
+		("get/", "err_get_on_int"),
+		("get/", "err_get_on_string"),
 
 		("set/", "basic_set"),
 		("set/", "set_bottom"),
@@ -163,9 +230,21 @@ pub fn generate(tests_file: &mut File) {
 		("array/", "array_ref_semantics"),
 		("array/", "array_length"),
 		("array/", "array_complicated_signature"),
+		("array/", "err_empty_arr_and_var"),
 	];
 
+	let mut bt = BuiltTests {
+		modules: HashMap::new()
+	};
 	for (path, test) in tests {
-		generate_test(tests_file, path, test).unwrap();
+		generate_test(&mut bt, path, test);
+	}
+	for (k, v) in &bt.modules {
+		// get rid of slash
+		let substr = &k[..k.len() - 1];
+		writeln!(tests_file, "mod r#{substr} {{").unwrap();
+		writeln!(tests_file, "\tuse crate::run_integration_test;").unwrap();
+		writeln!(tests_file, "{v}").unwrap();
+		writeln!(tests_file, "}}").unwrap();
 	}
 }

out

@@ -137,6 +137,11 @@ impl<'db> Binder<'db> {
 			}
 		}
 
+		self.db.report_error(Error::simple(
+			format!("Unknown identifier '{}'", self.db.get(ident)),
+			&location
+		));
+
 		self.had_error = true;
 		None
 	}

src/binder.rs

@@ -110,6 +110,11 @@ pub struct Db {
 	/// Note that these do NOT include the newlines or carriage returns. Those
 	/// are assumed to already exist.
 	pub test_lines: Vec<String>,
+	/// Expected errors from the program.
+	/// 
+	/// For now, we do not expect any particular error string; if we have any
+	/// errors in the test_errors array, we simply expect the compilation to fail.
+	pub test_errors: Vec<String>,
 
 	/// Maps names of the form "scope.scope.Item" to ScopeEntries. Used to bind
 	/// names to specific objects.
@@ -197,6 +202,7 @@ impl Db {
 
 			test_mode: false,
 			test_lines: Vec::new(),
+			test_errors: Vec::new(),
 
 			sig_declare_code: String::new(),
 

src/db.rs

@@ -23,7 +23,7 @@ struct Note {
 }
 
 pub struct Error {
-	main_message: String,
+	pub main_message: String,
 	main_location: SourceLocation,
 
 	is_warning: bool,

src/error.rs

@@ -268,6 +268,7 @@ impl Lexer {
 		enum CommentKind {
 			None,
 			TestLine,
+			TestErr,
 		}
 
 		let mut kind = CommentKind::None;
@@ -279,6 +280,10 @@ impl Lexer {
 				// Start the buffer at the beginning of the line.
 				self.buffer.clear();
 			}
+			if self.advance_if('?')? {
+				kind = CommentKind::TestErr;
+				self.buffer.clear();
+			}
 		}
 
 		while !self.at_eof {
@@ -295,14 +300,30 @@ impl Lexer {
 				let line = self.buffer.trim();
 				db.test_lines.push(line.to_string());
 			}
+
+			CommentKind::TestErr => {
+				let line = self.buffer.trim();
+				db.test_errors.push(line.to_string());
+			}
 		}
 
 		Ok(())
 	}
 
+	fn mk_eof(&mut self, db: &mut Db) -> std::io::Result<Token> {
+		// When at the EOF, create a lexeme that looks like <EOF> so that
+		// we can nicely output it.
+		//
+		// This also helps with a problem where before we were creating a
+		// lexeme of \0, making it very difficult to test against it.
+		self.buffer.clear();
+		self.buffer.push_str("<EOF>");
+		return self.mk_token_res(db, Tok::Eof);
+	}
+
 	pub fn next_token(&mut self, db: &mut Db) -> std::io::Result<Token> {
 		if self.at_eof {
-			return self.mk_token_res(db, Tok::Eof);
+			return self.mk_eof(db);
 		}
 
 		let c = self.advance_past_whitespace()?;
@@ -376,7 +397,11 @@ impl Lexer {
 				}
 
 				// TODO: Do we want to introduce a separate "error token" here?
-				Tok::Eof
+
+				// Note that if we're in the self.at_eof == true case, we do
+				// want to return an eof, because we really are there. (Here
+				// we should be matching on the \0 that we generate above.)
+				return self.mk_eof(db);
 			}
 		};
 

src/lexer.rs

@@ -173,6 +173,41 @@ fn duration(prev: SystemTime, message: &str, duration_set: &mut Vec<&'static str
 }
 
 fn report_errors(db: &Db) {
+	if db.test_mode && !db.test_errors.is_empty() {
+		if db.errors.len() != db.test_errors.len() {
+			eprintln!("Test failure: Wrong number of errors.");
+			exit(15);
+		}
+
+		// For now, we simply expect every single error that the compiler generates.
+		// This does mean tests are brittle, but on the other hand... we don't update
+		// the error messages that often. And for the most part, changes to the error
+		// messages shouldn't require huge numbers of test updates (or where they do,
+		// the updates should be somewhat regexable).
+		for i in 0..db.errors.len() {
+			// TODO:
+			// For some reason, the err_fun_missing_brace test is failing even though
+			// the strings absolutely appear to be the same. Very strange...
+			let want_str = &db.test_errors[i];//.trim();
+			let got_str = &db.errors[i].main_message;//.trim();
+			if want_str != got_str {
+				eprintln!("Test failure: Error message mismatch (index {i}):\nExpected: [{}]\nGot:      [{}]",
+					want_str, got_str);
+				
+				let mut j = 0;
+				for (a, b) in want_str.chars().zip(got_str.chars()) {
+					if a != b {
+						eprintln!("Mismatch at index {j}: {a} vs {b}")
+					}
+					j += 1;
+				}
+				exit(15);
+			}
+		}
+
+		// The test was successful.
+		exit(0);
+	}
 	for error in &db.errors {
 		crate::error::show_error(&error, db);
 	}
@@ -204,7 +239,7 @@ fn main() {
 
 	if had_error {
 		report_errors(&db);
-		exit(1);
+		exit(2);
 	}
 
 	let timer = duration(timer, "binding", &mut duration_set);
@@ -214,7 +249,7 @@ fn main() {
 
 	if had_error {
 		report_errors(&db);
-		exit(2);
+		exit(3);
 	}
 
 	let timer = duration(timer, "type check", &mut duration_set);
@@ -264,6 +299,13 @@ fn main() {
 	// If we're in test mode, then we want to run the program and check its
 	// output.
 	if args.test_mode {
+		if !db.test_errors.is_empty() {
+			// In this case, we actually have an error: we expected the compilation
+			// to result in an error, but it didn't. So, report that to the test
+			// runner.
+			eprintln!("Test failure: Expected an error, but compilation suceeded.");
+			exit(15);
+		}
 		// We've already checked the output is an Exe, so just run it at
 		// that path.
 		match test_compiled(&args.output_path, &db) {

src/main.rs

@@ -477,6 +477,16 @@ impl<'a, 'b> Parser<'a, 'b> {
 		match self.peek_typ() {
 			Tok::LeftBrace => self.block(),
 
+			Tok::LeftParen => {
+				// Eat left paren
+				self.advance()?;
+				// Inner expression
+				let inner = self.expression()?;
+				// Expect right paren after expression
+				expected!(self, Tok::RightParen, "')' after parenthesized expression")?;
+				Ok(inner)
+			}
+
 			Tok::Identifier => self.expr_ident(),
 
 			Tok::If => self.expr_if(),
@@ -547,7 +557,7 @@ impl<'a, 'b> Parser<'a, 'b> {
 
 	fn expr_prefix(&mut self) -> Result<Expr> {
 		match self.peek_typ() {
-			Tok::LeftBrace | Tok::Identifier | Tok::If | Tok::Fun => {
+			Tok::LeftBrace | Tok::LeftParen | Tok::Identifier | Tok::If | Tok::Fun => {
 				let location = self.start();
 				let mut inner = self.expr_prefix_callable()?;