Merge patch series "test: Improvements to ut command and test-suite running"

Simon Glass <[email protected]> says:

The current method of running unit tests relies on subcommands of the
ut command. Only the code in each subcommand knows how to find the tests
related to that subcomand.

This is not ideal and we now have quite a few subcommands which do
nothing but locate the relevant tests in a linker list, then call a
common function to run them.

This series adds a list of test suites, so that these subcommands can be
removed.

An issue with 'ut all' is that it doesn't record how many tests failed
overall, so it is necessary to examine copious amounts of output to look
for failures. This series adds a new 'total' feature allow recording the
total number of failed tests.

To help with 'ut all' a new pytest is created which runs it (as well as
'ut info') and makes sure that all is well. Due to the 'ut all' failures
this does not pass, so the test is disabled for now. It is here because
it provides security against misnaming a test suite and causing it not
to run.

Future work may:
- get 'ut all' passing
- enable test_suite() in CL, to ensure that 'ut all' keeps passing
- record duration of each suite
- allow running the tests in random order to tease out dependencies
- tweak the output to remove common prefixes
- getting rid of bootstd, optee and seame 'ut' subcommands

Link: https://lore.kernel.org/r/[email protected]

15 files changed, 13 insertions, 141 deletions

diff --git a/test/cmd/Makefile b/test/cmd/Makefile
index 583e7c2eec4..d8a5e77402d 100644
--- a/test/cmd/Makefile
+++ b/test/cmd/Makefile
@@ -3,8 +3,6 @@
 # Copyright (c) 2013 Google, Inc
 # Copyright 2022-2023 Arm Limited and/or its affiliates <[email protected]>
 
-obj-y += cmd_ut_cmd.o
-
 obj-$(CONFIG_$(XPL_)CMDLINE) += command.o
 ifdef CONFIG_HUSH_PARSER
 obj-$(CONFIG_CONSOLE_RECORD) += test_echo.o
@@ -12,7 +10,7 @@ endif
 ifdef CONFIG_CONSOLE_RECORD
 obj-$(CONFIG_CMD_PAUSE) += test_pause.o
 endif
-obj-y += exit.o mem.o
+obj-y += exit.o
 obj-$(CONFIG_X86) += cpuid.o msr.o
 obj-$(CONFIG_CMD_ADDRMAP) += addrmap.o
 obj-$(CONFIG_CMD_BDI) += bdinfo.o
diff --git a/test/cmd/addrmap.c b/test/cmd/addrmap.c
index b34be895f5d..1f2deb15052 100644
--- a/test/cmd/addrmap.c
+++ b/test/cmd/addrmap.c
@@ -10,7 +10,7 @@
 #include <test/ut.h>
 
 /* Declare a new addrmap test */
-#define ADDRMAP_TEST(_name, _flags)	UNIT_TEST(_name, _flags, addrmap_test)
+#define ADDRMAP_TEST(_name, _flags)	UNIT_TEST(_name, _flags, addrmap)
 
 /* Test 'addrmap' command output */
 static int addrmap_test_basic(struct unit_test_state *uts)
@@ -24,12 +24,3 @@ static int addrmap_test_basic(struct unit_test_state *uts)
 	return 0;
 }
 ADDRMAP_TEST(addrmap_test_basic, UTF_CONSOLE);
-
-int do_ut_addrmap(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(addrmap_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(addrmap_test);
-
-	return cmd_ut_category("cmd_addrmap", "cmd_addrmap_", tests, n_ents,
-			       argc, argv);
-}
diff --git a/test/cmd/bdinfo.c b/test/cmd/bdinfo.c
index 76429485708..7408c271a30 100644
--- a/test/cmd/bdinfo.c
+++ b/test/cmd/bdinfo.c
@@ -26,7 +26,7 @@
 DECLARE_GLOBAL_DATA_PTR;
 
 /* Declare a new bdinfo test */
-#define BDINFO_TEST(_name, _flags)	UNIT_TEST(_name, _flags, bdinfo_test)
+#define BDINFO_TEST(_name, _flags)	UNIT_TEST(_name, _flags, bdinfo)
 
 static int test_num_l(struct unit_test_state *uts, const char *name,
 		      ulong value)
@@ -282,11 +282,3 @@ static int bdinfo_test_eth(struct unit_test_state *uts)
 	return 0;
 }
 BDINFO_TEST(bdinfo_test_eth, UTF_CONSOLE);
-
-int do_ut_bdinfo(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(bdinfo_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(bdinfo_test);
-
-	return cmd_ut_category("bdinfo", "bdinfo_test_", tests, n_ents, argc, argv);
-}
diff --git a/test/cmd/cmd_ut_cmd.c b/test/cmd/cmd_ut_cmd.c
deleted file mode 100644
index e77fa1c7f01..00000000000
--- a/test/cmd/cmd_ut_cmd.c
+++ /dev/null
@@ -1,20 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright 2023 Google LLC
- * Written by Simon Glass <[email protected]>
- *
- * Unit tests for command functions
- */
-
-#include <command.h>
-#include <test/cmd.h>
-#include <test/suites.h>
-#include <test/ut.h>
-
-int do_ut_cmd(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(cmd_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(cmd_test);
-
-	return cmd_ut_category("cmd", "cmd_test_", tests, n_ents, argc, argv);
-}
diff --git a/test/cmd/exit.c b/test/cmd/exit.c
index af58a57fca7..71c37edcdf6 100644
--- a/test/cmd/exit.c
+++ b/test/cmd/exit.c
@@ -14,7 +14,7 @@
 DECLARE_GLOBAL_DATA_PTR;
 
 /* Declare a new exit test */
-#define EXIT_TEST(_name, _flags)	UNIT_TEST(_name, _flags, exit_test)
+#define EXIT_TEST(_name, _flags)	UNIT_TEST(_name, _flags, exit)
 
 /* Test 'exit addr' getting/setting address */
 static int cmd_exit_test(struct unit_test_state *uts)
@@ -110,12 +110,3 @@ static int cmd_exit_test(struct unit_test_state *uts)
 	return 0;
 }
 EXIT_TEST(cmd_exit_test, UTF_CONSOLE);
-
-int do_ut_exit(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(exit_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(exit_test);
-
-	return cmd_ut_category("cmd_exit", "exit_test_", tests, n_ents,
-			       argc, argv);
-}
diff --git a/test/cmd/fdt.c b/test/cmd/fdt.c
index e64785101cd..ab6dbd45e54 100644
--- a/test/cmd/fdt.c
+++ b/test/cmd/fdt.c
@@ -23,7 +23,7 @@ DECLARE_GLOBAL_DATA_PTR;
  */
 
 /* Declare a new fdt test */
-#define FDT_TEST(_name, _flags)	UNIT_TEST(_name, _flags, fdt_test)
+#define FDT_TEST(_name, _flags)	UNIT_TEST(_name, _flags, fdt)
 
 /**
  * make_test_fdt() - Create an FDT with just a root node
@@ -1462,11 +1462,3 @@ static int fdt_test_apply(struct unit_test_state *uts)
 	return 0;
 }
 FDT_TEST(fdt_test_apply, UTF_CONSOLE);
-
-int do_ut_fdt(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(fdt_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(fdt_test);
-
-	return cmd_ut_category("fdt", "fdt_test_", tests, n_ents, argc, argv);
-}
diff --git a/test/cmd/font.c b/test/cmd/font.c
index 3335dd65bea..af88d1b5459 100644
--- a/test/cmd/font.c
+++ b/test/cmd/font.c
@@ -12,7 +12,7 @@
 #include <test/ut.h>
 
 /* Declare a new fdt test */
-#define FONT_TEST(_name, _flags)	UNIT_TEST(_name, _flags, font_test)
+#define FONT_TEST(_name, _flags)	UNIT_TEST(_name, _flags, font)
 
 /* Test 'fdt addr' resizing an fdt */
 static int font_test_base(struct unit_test_state *uts)
@@ -85,11 +85,3 @@ static int font_test_base(struct unit_test_state *uts)
 }
 FONT_TEST(font_test_base, UTF_SCAN_PDATA | UTF_SCAN_FDT | UTF_CONSOLE |
 	  UTF_DM);
-
-int do_ut_font(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(font_Test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(font_test);
-
-	return cmd_ut_category("font", "font_test_", tests, n_ents, argc, argv);
-}
diff --git a/test/cmd/loadm.c b/test/cmd/loadm.c
index dedb4f7683e..3c623aa655f 100644
--- a/test/cmd/loadm.c
+++ b/test/cmd/loadm.c
@@ -19,7 +19,7 @@
 
 #define BUF_SIZE 0x100
 
-#define LOADM_TEST(_name, _flags)	UNIT_TEST(_name, _flags, loadm_test)
+#define LOADM_TEST(_name, _flags)	UNIT_TEST(_name, _flags, loadm)
 
 static int loadm_test_params(struct unit_test_state *uts)
 {
@@ -58,12 +58,3 @@ static int loadm_test_load (struct unit_test_state *uts)
 	return 0;
 }
 LOADM_TEST(loadm_test_load, UTF_CONSOLE);
-
-int do_ut_loadm(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(loadm_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(loadm_test);
-
-	return cmd_ut_category("loadm", "loadm_test_", tests, n_ents, argc,
-			       argv);
-}
diff --git a/test/cmd/mbr.c b/test/cmd/mbr.c
index d137378a3be..45bab04923a 100644
--- a/test/cmd/mbr.c
+++ b/test/cmd/mbr.c
@@ -470,12 +470,4 @@ static int mbr_test_run(struct unit_test_state *uts)
 }
 
 /* Declare mbr test */
-UNIT_TEST(mbr_test_run, UTF_CONSOLE, mbr_test);
-
-int do_ut_mbr(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(mbr_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(mbr_test);
-
-	return cmd_ut_category("mbr", "mbr_test_", tests, n_ents, argc, argv);
-}
+UNIT_TEST(mbr_test_run, UTF_CONSOLE, mbr);
diff --git a/test/cmd/mem.c b/test/cmd/mem.c
deleted file mode 100644
index f1bbab6055b..00000000000
--- a/test/cmd/mem.c
+++ /dev/null
@@ -1,19 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/*
- * Executes tests for memory-related commands
- *
- * Copyright 2020 Google LLC
- */
-
-#include <command.h>
-#include <test/suites.h>
-#include <test/test.h>
-
-int do_ut_mem(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(mem_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(mem_test);
-
-	return cmd_ut_category("cmd_mem", "mem_test_", tests, n_ents, argc,
-			       argv);
-}
diff --git a/test/cmd/mem_copy.c b/test/cmd/mem_copy.c
index 67eca328777..3e904fc4e4b 100644
--- a/test/cmd/mem_copy.c
+++ b/test/cmd/mem_copy.c
@@ -12,7 +12,7 @@
 #define BUF_SIZE	256
 
 /* Declare a new mem test */
-#define MEM_TEST(_name)	UNIT_TEST(_name, 0, mem_test)
+#define MEM_TEST(_name)	UNIT_TEST(_name, 0, mem)
 
 struct param {
 	int d, s, count;
diff --git a/test/cmd/mem_search.c b/test/cmd/mem_search.c
index 3a031eed7ed..df8938bdb6c 100644
--- a/test/cmd/mem_search.c
+++ b/test/cmd/mem_search.c
@@ -14,7 +14,7 @@
 #define BUF_SIZE	0x100
 
 /* Declare a new mem test */
-#define MEM_TEST(_name, _flags)	UNIT_TEST(_name, _flags, mem_test)
+#define MEM_TEST(_name, _flags)	UNIT_TEST(_name, _flags, mem)
 
 /* Test 'ms' command with bytes */
 static int mem_test_ms_b(struct unit_test_state *uts)
diff --git a/test/cmd/pci_mps.c b/test/cmd/pci_mps.c
index a265105600c..8b3ea4a6134 100644
--- a/test/cmd/pci_mps.c
+++ b/test/cmd/pci_mps.c
@@ -11,7 +11,7 @@
 #include <test/suites.h>
 #include <test/ut.h>
 
-#define PCI_MPS_TEST(_name, _flags) UNIT_TEST(_name, _flags, pci_mps_test)
+#define PCI_MPS_TEST(_name, _flags) UNIT_TEST(_name, _flags, pci_mps)
 
 /* Test "pci_mps" command in safe "s" mode */
 static int test_pci_mps_safe(struct unit_test_state *uts)
@@ -28,13 +28,3 @@ static int test_pci_mps_safe(struct unit_test_state *uts)
 	return 0;
 }
 PCI_MPS_TEST(test_pci_mps_safe, UTF_CONSOLE);
-
-int do_ut_pci_mps(struct cmd_tbl *cmdtp, int flag, int argc,
-		  char * const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(pci_mps_test);
-	const int n = UNIT_TEST_SUITE_COUNT(pci_mps_test);
-
-	return cmd_ut_category("cmd_pci_mps", "pci_mps_test_", tests, n,
-			       argc, argv);
-}
diff --git a/test/cmd/seama.c b/test/cmd/seama.c
index 28d6b9ab517..1edc3fcac5a 100644
--- a/test/cmd/seama.c
+++ b/test/cmd/seama.c
@@ -11,7 +11,7 @@
 #include <test/test.h>
 #include <test/ut.h>
 
-#define SEAMA_TEST(_name, _flags)	UNIT_TEST(_name, _flags, seama_test)
+#define SEAMA_TEST(_name, _flags)	UNIT_TEST(_name, _flags, seama)
 
 static int seama_test_noargs(struct unit_test_state *uts)
 {
@@ -56,12 +56,3 @@ static int seama_test_index(struct unit_test_state *uts)
 	return 0;
 }
 SEAMA_TEST(seama_test_index, UTF_CONSOLE);
-
-int do_ut_seama(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(seama_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(seama_test);
-
-	return cmd_ut_category("seama", "seama_test_", tests, n_ents, argc,
-			       argv);
-}
diff --git a/test/cmd/setexpr.c b/test/cmd/setexpr.c
index 21a3268bd81..7f318a42ead 100644
--- a/test/cmd/setexpr.c
+++ b/test/cmd/setexpr.c
@@ -15,7 +15,7 @@
 #define BUF_SIZE	0x100
 
 /* Declare a new setexpr test */
-#define SETEXPR_TEST(_name, _flags)	UNIT_TEST(_name, _flags, setexpr_test)
+#define SETEXPR_TEST(_name, _flags)	UNIT_TEST(_name, _flags, setexpr)
 
 /* Test 'setexpr' command with simply setting integers */
 static int setexpr_test_int(struct unit_test_state *uts)
@@ -479,12 +479,3 @@ static int setexpr_test_fmt(struct unit_test_state *uts)
 }
 SETEXPR_TEST(setexpr_test_fmt, UTF_CONSOLE);
 #endif
-
-int do_ut_setexpr(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
-{
-	struct unit_test *tests = UNIT_TEST_SUITE_START(setexpr_test);
-	const int n_ents = UNIT_TEST_SUITE_COUNT(setexpr_test);
-
-	return cmd_ut_category("cmd_setexpr", "setexpr_test_", tests, n_ents,
-			       argc, argv);
-}