Merge patch series "test: Improvements to ut command and test-suite running"

Simon Glass <[email protected]> says:

The current method of running unit tests relies on subcommands of the
ut command. Only the code in each subcommand knows how to find the tests
related to that subcomand.

This is not ideal and we now have quite a few subcommands which do
nothing but locate the relevant tests in a linker list, then call a
common function to run them.

This series adds a list of test suites, so that these subcommands can be
removed.

An issue with 'ut all' is that it doesn't record how many tests failed
overall, so it is necessary to examine copious amounts of output to look
for failures. This series adds a new 'total' feature allow recording the
total number of failed tests.

To help with 'ut all' a new pytest is created which runs it (as well as
'ut info') and makes sure that all is well. Due to the 'ut all' failures
this does not pass, so the test is disabled for now. It is here because
it provides security against misnaming a test suite and causing it not
to run.

Future work may:
- get 'ut all' passing
- enable test_suite() in CL, to ensure that 'ut all' keeps passing
- record duration of each suite
- allow running the tests in random order to tease out dependencies
- tweak the output to remove common prefixes
- getting rid of bootstd, optee and seame 'ut' subcommands

Link: https://lore.kernel.org/r/[email protected]

5 files changed, 9 insertions, 37 deletions

diff --git a/test/boot/bootm.c b/test/boot/bootm.c
index 9455f44884c..7e0ccb0e23f 100644
--- a/test/boot/bootm.c
+++ b/test/boot/bootm.c
@@ -13,7 +13,7 @@
 
 DECLARE_GLOBAL_DATA_PTR;
 
-#define BOOTM_TEST(_name, _flags)	UNIT_TEST(_name, _flags, bootm_test)
+#define BOOTM_TEST(_name, _flags)	UNIT_TEST(_name, _flags, bootm)
 
 enum {
 	BUF_SIZE	= 1024,
@@ -249,12 +249,3 @@ static int bootm_test_subst_both(struct unit_test_state *uts)
 	return 0;
 }
 BOOTM_TEST(bootm_test_subst_both, 0);
-
-int do_ut_bootm(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(bootm_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(bootm_test);
-
-	return cmd_ut_category("bootm", "bootm_test_", tests, n_ents,
-			       argc, argv);
-}
diff --git a/test/boot/bootstd_common.c b/test/boot/bootstd_common.c
index ff8ed2303b3..724e3d9bdd2 100644
--- a/test/boot/bootstd_common.c
+++ b/test/boot/bootstd_common.c
@@ -94,10 +94,11 @@ void bootstd_reset_usb(void)
 	usb_started = false;
 }
 
-int do_ut_bootstd(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
+int do_ut_bootstd(struct unit_test_state *uts, struct cmd_tbl *cmdtp, int flag,
+		  int argc, char *const argv[])
 {
-	struct unit_test *tests = UNIT_TEST_SUITE_START(bootstd_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(bootstd_test);
+	struct unit_test *tests = UNIT_TEST_SUITE_START(bootstd);
+	const int n_ents = UNIT_TEST_SUITE_COUNT(bootstd);
 	int ret;
 
 	ret = bootstd_setup_for_tests();
@@ -106,6 +107,6 @@ int do_ut_bootstd(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
 		return CMD_RET_FAILURE;
 	}
 
-	return cmd_ut_category("bootstd", "bootstd_test_",
+	return cmd_ut_category(uts, "bootstd", "bootstd_",
 			       tests, n_ents, argc, argv);
 }
diff --git a/test/boot/bootstd_common.h b/test/boot/bootstd_common.h
index e29036c897c..ea3ecd1166c 100644
--- a/test/boot/bootstd_common.h
+++ b/test/boot/bootstd_common.h
@@ -12,8 +12,7 @@
 #include <version_string.h>
 
 /* Declare a new bootdev test */
-#define BOOTSTD_TEST(_name, _flags) \
-		UNIT_TEST(_name, _flags, bootstd_test)
+#define BOOTSTD_TEST(_name, _flags)	UNIT_TEST(_name, _flags, bootstd)
 
 #define NVDATA_START_BLK	((0x400 + 0x400) / MMC_MAX_BLOCK_LEN)
 #define VERSION_START_BLK	((0x400 + 0x800) / MMC_MAX_BLOCK_LEN)
diff --git a/test/boot/measurement.c b/test/boot/measurement.c
index 29be495412d..5a49c7a6b23 100644
--- a/test/boot/measurement.c
+++ b/test/boot/measurement.c
@@ -14,7 +14,7 @@
 #include <asm/io.h>
 
 #define MEASUREMENT_TEST(_name, _flags)	\
-	UNIT_TEST(_name, _flags, measurement_test)
+	UNIT_TEST(_name, _flags, measurement)
 
 static int measure(struct unit_test_state *uts)
 {
@@ -53,13 +53,3 @@ static int measure(struct unit_test_state *uts)
 	return 0;
 }
 MEASUREMENT_TEST(measure, 0);
-
-int do_ut_measurement(struct cmd_tbl *cmdtp, int flag, int argc,
-		      char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(measurement_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(measurement_test);
-
-	return cmd_ut_category("measurement", "measurement_test_", tests,
-			       n_ents, argc, argv);
-}
diff --git a/test/boot/upl.c b/test/boot/upl.c
index 99f02b7951b..aa58cdf083b 100644
--- a/test/boot/upl.c
+++ b/test/boot/upl.c
@@ -16,7 +16,7 @@
 #include "bootstd_common.h"
 
 /* Declare a new upl test */
-#define UPL_TEST(_name, _flags)	UNIT_TEST(_name, _flags, upl_test)
+#define UPL_TEST(_name, _flags)	UNIT_TEST(_name, _flags, upl)
 
 static int add_region(struct unit_test_state *uts, struct alist *lst,
 		      ulong base, ulong size)
@@ -426,12 +426,3 @@ static int upl_test_info_norun(struct unit_test_state *uts)
 	return 0;
 }
 UPL_TEST(upl_test_info_norun, UTF_CONSOLE | UTF_MANUAL);
-
-int do_ut_upl(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(upl_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(upl_test);
-
-	return cmd_ut_category("cmd_upl", "cmd_upl_", tests, n_ents, argc,
-			       argv);
-}