Merge patch series "test: Improvements to ut command and test-suite running"

Simon Glass <[email protected]> says:

The current method of running unit tests relies on subcommands of the
ut command. Only the code in each subcommand knows how to find the tests
related to that subcomand.

This is not ideal and we now have quite a few subcommands which do
nothing but locate the relevant tests in a linker list, then call a
common function to run them.

This series adds a list of test suites, so that these subcommands can be
removed.

An issue with 'ut all' is that it doesn't record how many tests failed
overall, so it is necessary to examine copious amounts of output to look
for failures. This series adds a new 'total' feature allow recording the
total number of failed tests.

To help with 'ut all' a new pytest is created which runs it (as well as
'ut info') and makes sure that all is well. Due to the 'ut all' failures
this does not pass, so the test is disabled for now. It is here because
it provides security against misnaming a test suite and causing it not
to run.

Future work may:
- get 'ut all' passing
- enable test_suite() in CL, to ensure that 'ut all' keeps passing
- record duration of each suite
- allow running the tests in random order to tease out dependencies
- tweak the output to remove common prefixes
- getting rid of bootstd, optee and seame 'ut' subcommands

Link: https://lore.kernel.org/r/[email protected]

13 files changed, 139 insertions, 60 deletions

diff --git a/include/dm/test.h b/include/dm/test.h
index 3cbf2c740d4..4aabb4603b9 100644
--- a/include/dm/test.h
+++ b/include/dm/test.h
@@ -143,7 +143,7 @@ extern struct unit_test_state global_dm_test_state;
 
 /* Declare a new driver model test */
 #define DM_TEST(_name, _flags) \
-	UNIT_TEST(_name, UTF_DM | UTF_CONSOLE | (_flags), dm_test)
+	UNIT_TEST(_name, UTF_DM | UTF_CONSOLE | (_flags), dm)
 
 /*
  * struct sandbox_sdl_plat - Platform data for the SDL video driver
diff --git a/include/linker_lists.h b/include/linker_lists.h
index f9a2ee0c762..0f4a2d686e2 100644
--- a/include/linker_lists.h
+++ b/include/linker_lists.h
@@ -187,6 +187,63 @@
 	})
 
 /**
+ * Declares a symbol that points to the start/end of the list.
+ *
+ * @_sym:	Arbitrary name for the symbol (to use later in the file)
+ * @_type:	Data type of the entry
+ * @_list:	Name of the list in which this entry is placed
+ *
+ * The name of the (new) symbol is arbitrary and can be anything that is not
+ * already declared in the file where it appears. It is provided in _sym and
+ * can then be used (later in the same file) within a data structure.
+ * The _type and _list arguments must match those passed to ll_entry_start/end()
+ *
+ * Example:
+ *
+ * Here we want to record the start of each sub-command in a list. We have two
+ * sub-commands, 'bob' and 'mary'.
+ *
+ * In bob.c::
+ *
+ *   ll_entry_declare(struct my_sub_cmd, bob_cmd, cmd_sub) = {...};
+ *
+ * In mary.c::
+ *
+ *   ll_entry_declare(struct my_sub_cmd, mary_cmd, cmd_sub) = {...};
+ *
+ * In a different file where we want a list the start of all sub-commands.
+ * It is not possible to use ll_entry_start() in a data structure, due to its
+ * use of code inside expressions - ({ ... }) - so this fails to compile:
+ *
+ * In sub_cmds.c::
+ *
+ *    struct cmd_sub *my_list[] = {
+ *      ll_entry_start(cmd_sub, bob),
+ *      ll_entry_start(cmd_sub, bob),
+ *    };
+ *
+ * Instead, we can use::
+ *
+ *    ll_start_decl(bob, struct my_sub_cmd, cmd_sub);
+ *    ll_start_decl(mary, struct my_sub_cmd, cmd_sub);
+ *
+ *     struct cmd_sub *my_list[] = {
+ *       bob,
+ *       mary,
+ *     };
+ *
+ * So 'bob' is declared as symbol, a struct my_list * which points to the
+ * start of the bob sub-commands. It is then used in my_list[]
+ */
+#define ll_start_decl(_sym, _type, _list)					\
+	static _type _sym[0] __aligned(CONFIG_LINKER_LIST_ALIGN)	\
+		__maybe_unused __section("__u_boot_list_2_" #_list "_1")
+
+#define ll_end_decl(_sym, _type, _list)					\
+	static _type _sym[0] __aligned(CONFIG_LINKER_LIST_ALIGN)	\
+		__maybe_unused __section("__u_boot_list_2_" #_list "_3")
+
+/**
  * ll_entry_get() - Retrieve entry from linker-generated array by name
  * @_type:	Data type of the entry
  * @_name:	Name of the entry
diff --git a/include/test/cmd.h b/include/test/cmd.h
index c200570e423..3d1e3e3bddb 100644
--- a/include/test/cmd.h
+++ b/include/test/cmd.h
@@ -10,6 +10,6 @@
 #include <test/test.h>
 
 /* Declare a new command test */
-#define CMD_TEST(_name, _flags) UNIT_TEST(_name, _flags, cmd_test)
+#define CMD_TEST(_name, _flags) UNIT_TEST(_name, _flags, cmd)
 
 #endif /* __TEST_CMD_H__ */
diff --git a/include/test/common.h b/include/test/common.h
index 81260d06ad6..d5a65d5b50b 100644
--- a/include/test/common.h
+++ b/include/test/common.h
@@ -10,6 +10,6 @@
 #include <test/test.h>
 
 /* Declare a new common function test */
-#define COMMON_TEST(_name, _flags) UNIT_TEST(_name, _flags, common_test)
+#define COMMON_TEST(_name, _flags) UNIT_TEST(_name, _flags, common)
 
 #endif /* __TEST_COMMON_H__ */
diff --git a/include/test/env.h b/include/test/env.h
index f45e33d71a4..6a63cc972e9 100644
--- a/include/test/env.h
+++ b/include/test/env.h
@@ -10,6 +10,6 @@
 #include <test/test.h>
 
 /* Declare a new environment test */
-#define ENV_TEST(_name, _flags)	UNIT_TEST(_name, _flags, env_test)
+#define ENV_TEST(_name, _flags)	UNIT_TEST(_name, _flags, env)
 
 #endif /* __TEST_ENV_H__ */
diff --git a/include/test/hush.h b/include/test/hush.h
index cca66544a06..e57bf13ea61 100644
--- a/include/test/hush.h
+++ b/include/test/hush.h
@@ -10,6 +10,6 @@
 #include <test/test.h>
 
 /* Declare a new environment test */
-#define HUSH_TEST(_name, _flags)	UNIT_TEST(_name, _flags, hush_test)
+#define HUSH_TEST(_name, _flags)	UNIT_TEST(_name, _flags, hush)
 
 #endif /* __TEST_HUSH_H__ */
diff --git a/include/test/lib.h b/include/test/lib.h
index 04b6241e54a..b19eb863a33 100644
--- a/include/test/lib.h
+++ b/include/test/lib.h
@@ -9,6 +9,6 @@
 #include <test/test.h>
 
 /* Declare a new library function test */
-#define LIB_TEST(_name, _flags)	UNIT_TEST(_name, _flags, lib_test)
+#define LIB_TEST(_name, _flags)	UNIT_TEST(_name, _flags, lib)
 
 #endif /* __TEST_LIB_H__ */
diff --git a/include/test/log.h b/include/test/log.h
index e3362b85e99..0921c0c1cbc 100644
--- a/include/test/log.h
+++ b/include/test/log.h
@@ -13,8 +13,8 @@
 #define LOGF_TEST (BIT(LOGF_FUNC) | BIT(LOGF_MSG))
 
 /* Declare a new logging test */
-#define LOG_TEST(_name) UNIT_TEST(_name, UTF_CONSOLE, log_test)
+#define LOG_TEST(_name) UNIT_TEST(_name, UTF_CONSOLE, log)
 #define LOG_TEST_FLAGS(_name, _flags) \
-		UNIT_TEST(_name, _flags | UTF_CONSOLE, log_test)
+		UNIT_TEST(_name, _flags | UTF_CONSOLE, log)
 
 #endif /* __TEST_LOG_H__ */
diff --git a/include/test/optee.h b/include/test/optee.h
index a8c6e6395f5..f4255b39ee3 100644
--- a/include/test/optee.h
+++ b/include/test/optee.h
@@ -9,6 +9,6 @@
 #include <test/test.h>
 
 /* Declare a new environment test */
-#define OPTEE_TEST(_name, _flags)	UNIT_TEST(_name, _flags, optee_test)
+#define OPTEE_TEST(_name, _flags)	UNIT_TEST(_name, _flags, optee)
 
 #endif /* __TEST_OPTEE_H__ */
diff --git a/include/test/overlay.h b/include/test/overlay.h
index c13f4d66e09..5dc98399ce7 100644
--- a/include/test/overlay.h
+++ b/include/test/overlay.h
@@ -10,6 +10,6 @@
 #include <test/test.h>
 
 /* Declare a new environment test */
-#define OVERLAY_TEST(_name, _flags)	UNIT_TEST(_name, _flags, overlay_test)
+#define OVERLAY_TEST(_name, _flags)	UNIT_TEST(_name, _flags, overlay)
 
 #endif /* __TEST_OVERLAY_H__ */
diff --git a/include/test/suites.h b/include/test/suites.h
index 2ceef577f7f..774dd893378 100644
--- a/include/test/suites.h
+++ b/include/test/suites.h
@@ -9,10 +9,18 @@
 
 struct cmd_tbl;
 struct unit_test;
+struct unit_test_state;
+
+/* 'command' functions normally called do_xxx where xxx is the command name */
+typedef int (*ut_cmd_func)(struct unit_test_state *uts, struct cmd_tbl *cmd,
+			   int flags, int argc, char *const argv[]);
 
 /**
  * cmd_ut_category() - Run a category of unit tests
  *
+ * @uts: Unit-test state, which must be ready for use, i.e. ut_init_state()
+ *	has been called. The caller is responsible for calling
+ *	ut_uninit_state() after this function returns
  * @name:	Category name
  * @prefix:	Prefix of test name
  * @tests:	List of tests to run
@@ -22,47 +30,14 @@ struct unit_test;
  * @argv:	Arguments: argv[1] is the test to run (if @argc >= 2)
  * Return: 0 if OK, CMD_RET_FAILURE on failure
  */
-int cmd_ut_category(const char *name, const char *prefix,
-		    struct unit_test *tests, int n_ents,
+int cmd_ut_category(struct unit_test_state *uts, const char *name,
+		    const char *prefix, struct unit_test *tests, int n_ents,
 		    int argc, char *const argv[]);
 
-int do_ut_addrmap(struct cmd_tbl *cmdtp, int flag, int argc,
-		  char *const argv[]);
-int do_ut_bdinfo(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_bootm(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_bootstd(struct cmd_tbl *cmdtp, int flag, int argc,
-		  char *const argv[]);
-int do_ut_bloblist(struct cmd_tbl *cmdtp, int flag, int argc,
-		   char *const argv[]);
-int do_ut_cmd(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_common(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_compression(struct cmd_tbl *cmdtp, int flag, int argc,
-		      char *const argv[]);
-int do_ut_dm(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_env(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_exit(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_fdt(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_font(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_hush(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_lib(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_loadm(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_log(struct cmd_tbl *cmdtp, int flag, int argc, char * const argv[]);
-int do_ut_mbr(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_measurement(struct cmd_tbl *cmdtp, int flag, int argc, char * const argv[]);
-int do_ut_mem(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
+int do_ut_bootstd(struct unit_test_state *uts, struct cmd_tbl *cmdtp, int flag,
+		  int argc, char *const argv[]);
 int do_ut_optee(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_overlay(struct cmd_tbl *cmdtp, int flag, int argc,
-		  char *const argv[]);
-int do_ut_pci_mps(struct cmd_tbl *cmdtp, int flag, int argc,
-		  char *const argv[]);
-int do_ut_print(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_seama(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_setexpr(struct cmd_tbl *cmdtp, int flag, int argc,
-		  char *const argv[]);
-int do_ut_str(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_time(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
-int do_ut_unicode(struct cmd_tbl *cmdtp, int flag, int argc,
-		  char *const argv[]);
-int do_ut_upl(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]);
+int do_ut_overlay(struct unit_test_state *uts, struct cmd_tbl *cmdtp, int flag,
+		  int argc, char *const argv[]);
 
 #endif /* __TEST_SUITES_H__ */
diff --git a/include/test/test.h b/include/test/test.h
index 21c0478befe..bac43c81d63 100644
--- a/include/test/test.h
+++ b/include/test/test.h
@@ -9,11 +9,26 @@
 #include <malloc.h>
 #include <linux/bitops.h>
 
-/*
- * struct unit_test_state - Entire state of test system
+/**
+ * struct ut_stats - Statistics about tests run
  *
  * @fail_count: Number of tests that failed
  * @skip_count: Number of tests that were skipped
+ * @test_count: Number of tests run. If a test is run muiltiple times, only one
+ *	is counted
+ */
+struct ut_stats {
+	int fail_count;
+	int skip_count;
+	int test_count;
+};
+
+/*
+ * struct unit_test_state - Entire state of test system
+ *
+ * @cur: Statistics for the current run
+ * @total: Statistics for all test runs
+ * @run_count: Number of times ut_run_list() has been called
  * @start: Store the starting mallinfo when doing leak test
  * @of_live: true to use livetree if available, false to use flattree
  * @of_root: Record of the livetree root node (used for setting up tests)
@@ -34,8 +49,9 @@
  * @actual_str: Temporary string used to hold actual string value
  */
 struct unit_test_state {
-	int fail_count;
-	int skip_count;
+	struct ut_stats cur;
+	struct ut_stats total;
+	int run_count;
 	struct mallinfo start;
 	struct device_node *of_root;
 	bool of_live;
diff --git a/include/test/ut.h b/include/test/ut.h
index c8838dad096..be5502e03a1 100644
--- a/include/test/ut.h
+++ b/include/test/ut.h
@@ -466,18 +466,37 @@ void ut_unsilence_console(struct unit_test_state *uts);
 void ut_set_skip_delays(struct unit_test_state *uts, bool skip_delays);
 
 /**
- * test_get_state() - Get the active test state
+ * ut_state_get() - Get the active test state
  *
  * Return: the currently active test state, or NULL if none
  */
-struct unit_test_state *test_get_state(void);
+struct unit_test_state *ut_get_state(void);
 
 /**
- * test_set_state() - Set the active test state
+ * ut_set_state() - Set the active test state
  *
  * @uts: Test state to use as currently active test state, or NULL if none
  */
-void test_set_state(struct unit_test_state *uts);
+void ut_set_state(struct unit_test_state *uts);
+
+/**
+ * ut_init_state() - Set up a new test state
+ *
+ * This must be called before using the test state with ut_run_tests()
+ *
+ * @uts: Test state to init
+ */
+void ut_init_state(struct unit_test_state *uts);
+
+/**
+ * ut_uninit_state() - Free memory used by test state
+ *
+ * This must be called before after the test state with ut_run_tests(). To later
+ * reuse the test state to run more tests, call test_state_init() first
+ *
+ * @uts: Test state to uninit
+ */
+void ut_uninit_state(struct unit_test_state *uts);
 
 /**
  * ut_run_tests() - Run a set of tests
@@ -485,6 +504,9 @@ void test_set_state(struct unit_test_state *uts);
  * This runs the test, handling any preparation and clean-up needed. It prints
  * the name of each test before running it.
  *
+ * @uts: Unit-test state, which must be ready for use, i.e. ut_init_state()
+ *	has been called. The caller is responsible for calling
+ *	ut_uninit_state() after this function returns
  * @category: Category of these tests. This is a string printed at the start to
  *	announce the the number of tests
  * @prefix: String prefix for the tests. Any tests that have this prefix will be
@@ -503,8 +525,17 @@ void test_set_state(struct unit_test_state *uts);
  * Pass NULL to disable this
  * Return: 0 if all tests passed, -1 if any failed
  */
-int ut_run_list(const char *name, const char *prefix, struct unit_test *tests,
-		int count, const char *select_name, int runs_per_test,
-		bool force_run, const char *test_insert);
+int ut_run_list(struct unit_test_state *uts, const char *category,
+		const char *prefix, struct unit_test *tests, int count,
+		const char *select_name, int runs_per_test, bool force_run,
+		const char *test_insert);
+
+/**
+ * ut_report() - Report stats on a test run
+ *
+ * @stats: Stats to show
+ * @run_count: Number of suites that were run
+ */
+void ut_report(struct ut_stats *stats, int run_count);
 
 #endif