From 5f6a59e03eff0f29295ce12b3807cbac7334e0aa Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 7 Feb 2025 11:30:35 -0700 Subject: test: Keep track of suite duration Show the time taken by each test suite with 'ut all' and the total time for all suites. Take care to remove any sandbox time-offset from the values. Fix the comment-format on timer_test_add_offset() while we are here. Signed-off-by: Simon Glass --- include/test/test.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/test') diff --git a/include/test/test.h b/include/test/test.h index bac43c81d63..25c7712d160 100644 --- a/include/test/test.h +++ b/include/test/test.h @@ -16,11 +16,15 @@ * @skip_count: Number of tests that were skipped * @test_count: Number of tests run. If a test is run muiltiple times, only one * is counted + * @start: Timer value when test started + * @duration_ms: Suite duration in milliseconds */ struct ut_stats { int fail_count; int skip_count; int test_count; + ulong start; + ulong duration_ms; }; /* -- cgit v1.2.3 From b85df267e1f9f6f53086875975b8e4b24570365d Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 7 Feb 2025 11:30:36 -0700 Subject: test: Show the average time per test Show the average duration of a test, so we can keep track of how it is trending. Report the suite with the longest average test to encourage people to improve it. Add a function to update the stats based on the results from a single suite and another to show the summary information. Make this optional, since sandbox's SPL tests do not have a timer driver and people may want to print results without times. Signed-off-by: Simon Glass --- include/test/test.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/test') diff --git a/include/test/test.h b/include/test/test.h index 25c7712d160..877fda8d5aa 100644 --- a/include/test/test.h +++ b/include/test/test.h @@ -33,6 +33,8 @@ struct ut_stats { * @cur: Statistics for the current run * @total: Statistics for all test runs * @run_count: Number of times ut_run_list() has been called + * @worst: Sute which had the first per-text run time + * @worst_ms: Time taken by that test * @start: Store the starting mallinfo when doing leak test * @of_live: true to use livetree if available, false to use flattree * @of_root: Record of the livetree root node (used for setting up tests) @@ -56,6 +58,8 @@ struct unit_test_state { struct ut_stats cur; struct ut_stats total; int run_count; + const struct suite *worst; + int worst_ms; struct mallinfo start; struct device_node *of_root; bool of_live; -- cgit v1.2.3 From c908ecb7b51e886469c4fc8eb5492bdbda55b871 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 7 Feb 2025 11:30:38 -0700 Subject: test: Support an init/uninit functions for test suites Some suites need things to be set up before they can run. Add a way to declare an init function using the UNIT_TEST_INIT() macro. The init function is just like any other test, but is always placed first so that it runs before all the other test functions in the suite. Add an uninit function as well, to clean up after the test. Signed-off-by: Simon Glass --- include/test/test.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'include/test') diff --git a/include/test/test.h b/include/test/test.h index 877fda8d5aa..0f2b68a5dee 100644 --- a/include/test/test.h +++ b/include/test/test.h @@ -100,6 +100,8 @@ enum ut_flags { UTF_ETH_BOOTDEV = BIT(9), /* enable Ethernet bootdevs */ UTF_SF_BOOTDEV = BIT(10), /* enable SPI flash bootdevs */ UFT_BLOBLIST = BIT(11), /* test changes gd->bloblist */ + UTF_INIT = BIT(12), /* test inits a suite */ + UTF_UNINIT = BIT(13), /* test uninits a suite */ }; /** @@ -147,6 +149,24 @@ struct unit_test { .func = _name, \ } +/* init function for unit-test suite (the 'A' makes it first) */ +#define UNIT_TEST_INIT(_name, _flags, _suite) \ + ll_entry_declare(struct unit_test, A ## _name, ut_ ## _suite) = { \ + .file = __FILE__, \ + .name = #_name, \ + .flags = (_flags) | UTF_INIT, \ + .func = _name, \ + } + +/* uninit function for unit-test suite (the 'aaa' makes it last) */ +#define UNIT_TEST_UNINIT(_name, _flags, _suite) \ + ll_entry_declare(struct unit_test, zzz ## _name, ut_ ## _suite) = { \ + .file = __FILE__, \ + .name = #_name, \ + .flags = (_flags) | UTF_UNINIT, \ + .func = _name, \ + } + /* Get the start of a list of unit tests for a particular suite */ #define UNIT_TEST_SUITE_START(_suite) \ ll_entry_start(struct unit_test, ut_ ## _suite) -- cgit v1.2.3 From ea29bad9cff2fd88d172276b58859f01649fe444 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 7 Feb 2025 11:30:39 -0700 Subject: test: Tweak FDT-overlay tests Use fdt_overlay consistently in the identifiers and file/dir names. Signed-off-by: Simon Glass --- include/test/fdt_overlay.h | 15 +++++++++++++++ include/test/overlay.h | 15 --------------- include/test/suites.h | 4 ++-- 3 files changed, 17 insertions(+), 17 deletions(-) create mode 100644 include/test/fdt_overlay.h delete mode 100644 include/test/overlay.h (limited to 'include/test') diff --git a/include/test/fdt_overlay.h b/include/test/fdt_overlay.h new file mode 100644 index 00000000000..34e8020a496 --- /dev/null +++ b/include/test/fdt_overlay.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright (c) 2016 NextThing Co + * Copyright (c) 2016 Free Electrons + */ + +#ifndef __TEST_OVERLAY_H__ +#define __TEST_OVERLAY_H__ + +#include + +/* Declare a new FDT-overlay test */ +#define FDT_OVERLAY_TEST(_name, _flags) UNIT_TEST(_name, _flags, fdt_overlay) + +#endif /* __TEST_OVERLAY_H__ */ diff --git a/include/test/overlay.h b/include/test/overlay.h deleted file mode 100644 index 5dc98399ce7..00000000000 --- a/include/test/overlay.h +++ /dev/null @@ -1,15 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ -/* - * Copyright (c) 2016 NextThing Co - * Copyright (c) 2016 Free Electrons - */ - -#ifndef __TEST_OVERLAY_H__ -#define __TEST_OVERLAY_H__ - -#include - -/* Declare a new environment test */ -#define OVERLAY_TEST(_name, _flags) UNIT_TEST(_name, _flags, overlay) - -#endif /* __TEST_OVERLAY_H__ */ diff --git a/include/test/suites.h b/include/test/suites.h index 774dd893378..dce720b4e78 100644 --- a/include/test/suites.h +++ b/include/test/suites.h @@ -36,8 +36,8 @@ int cmd_ut_category(struct unit_test_state *uts, const char *name, int do_ut_bootstd(struct unit_test_state *uts, struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); +int do_ut_fdt_overlay(struct unit_test_state *uts, struct cmd_tbl *cmdtp, + int flag, int argc, char *const argv[]); int do_ut_optee(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); -int do_ut_overlay(struct unit_test_state *uts, struct cmd_tbl *cmdtp, int flag, - int argc, char *const argv[]); #endif /* __TEST_SUITES_H__ */ -- cgit v1.2.3 From a7290bc4b7248118046f748d80088a40f9171576 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 7 Feb 2025 11:30:46 -0700 Subject: test: Update fdt_overlay to do init from tests Rather than having an init function and then running the tests, create a test-init function to do it. This will allow us to get rid of the command function. Signed-off-by: Simon Glass --- include/test/fdt_overlay.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/test') diff --git a/include/test/fdt_overlay.h b/include/test/fdt_overlay.h index 34e8020a496..251ad0ec97a 100644 --- a/include/test/fdt_overlay.h +++ b/include/test/fdt_overlay.h @@ -12,4 +12,8 @@ /* Declare a new FDT-overlay test */ #define FDT_OVERLAY_TEST(_name, _flags) UNIT_TEST(_name, _flags, fdt_overlay) +/* Declare init for FDT-overlay test */ +#define FDT_OVERLAY_TEST_INIT(_name, _flags) \ + UNIT_TEST_INIT(_name, _flags, fdt_overlay) + #endif /* __TEST_OVERLAY_H__ */ -- cgit v1.2.3 From ba1112839ace31b4aed85506cf5fe74256c597df Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 7 Feb 2025 11:30:47 -0700 Subject: test: Drop the function for running fdt_overlay tests Use the new suite-runner to run these tests instead. Signed-off-by: Simon Glass --- include/test/suites.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/test') diff --git a/include/test/suites.h b/include/test/suites.h index dce720b4e78..78f5cdb4bdf 100644 --- a/include/test/suites.h +++ b/include/test/suites.h @@ -36,8 +36,6 @@ int cmd_ut_category(struct unit_test_state *uts, const char *name, int do_ut_bootstd(struct unit_test_state *uts, struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); -int do_ut_fdt_overlay(struct unit_test_state *uts, struct cmd_tbl *cmdtp, - int flag, int argc, char *const argv[]); int do_ut_optee(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); #endif /* __TEST_SUITES_H__ */ -- cgit v1.2.3 From 7e1c6bd0778baffd90164e69f1d5f8ea8729a5ae Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 7 Feb 2025 11:30:49 -0700 Subject: test: Drop the function for running bootstd tests Use the new suite-runner to run these tests instead. Signed-off-by: Simon Glass --- include/test/suites.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/test') diff --git a/include/test/suites.h b/include/test/suites.h index 78f5cdb4bdf..692633dcaa4 100644 --- a/include/test/suites.h +++ b/include/test/suites.h @@ -34,8 +34,6 @@ int cmd_ut_category(struct unit_test_state *uts, const char *name, const char *prefix, struct unit_test *tests, int n_ents, int argc, char *const argv[]); -int do_ut_bootstd(struct unit_test_state *uts, struct cmd_tbl *cmdtp, int flag, - int argc, char *const argv[]); int do_ut_optee(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); #endif /* __TEST_SUITES_H__ */ -- cgit v1.2.3 From 1c05e2c0eb9cd449a743d49c89a777b6bc803a15 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 7 Feb 2025 11:30:50 -0700 Subject: test: Update optee to do init and uninit from tests Rather than having an init function and then running the tests, create a test-init function to do it. This will allow us to get rid of the command function. Fix the comment abotu 'environment' while we are here. Signed-off-by: Simon Glass --- include/test/optee.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'include/test') diff --git a/include/test/optee.h b/include/test/optee.h index f4255b39ee3..0a548a59e83 100644 --- a/include/test/optee.h +++ b/include/test/optee.h @@ -8,7 +8,9 @@ #include -/* Declare a new environment test */ +/* Declare a new optee test */ #define OPTEE_TEST(_name, _flags) UNIT_TEST(_name, _flags, optee) +#define OPTEE_TEST_INIT(_name, _flags) UNIT_TEST_INIT(_name, _flags, optee) +#define OPTEE_TEST_UNINIT(_name, _flags) UNIT_TEST_UNINIT(_name, _flags, optee) #endif /* __TEST_OPTEE_H__ */ -- cgit v1.2.3 From daf583a3174f1c0ea7a617e6a5bb1b83051dfbad Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 7 Feb 2025 11:30:52 -0700 Subject: test: Drop the function for running optee tests Use the new suite-runner to run these tests instead. Signed-off-by: Simon Glass --- include/test/suites.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/test') diff --git a/include/test/suites.h b/include/test/suites.h index 692633dcaa4..a2f982bf6fa 100644 --- a/include/test/suites.h +++ b/include/test/suites.h @@ -34,6 +34,4 @@ int cmd_ut_category(struct unit_test_state *uts, const char *name, const char *prefix, struct unit_test *tests, int n_ents, int argc, char *const argv[]); -int do_ut_optee(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[]); - #endif /* __TEST_SUITES_H__ */ -- cgit v1.2.3 From 59713c412aeb2f1cafd42a77c948ce92d387de44 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 7 Feb 2025 11:30:54 -0700 Subject: test: Drop support for test commands Now that everything is using the new test-suite features, drop support for running commands. Fix a missing closing-bracket while we are here. Signed-off-by: Simon Glass --- include/test/suites.h | 5 ----- 1 file changed, 5 deletions(-) (limited to 'include/test') diff --git a/include/test/suites.h b/include/test/suites.h index a2f982bf6fa..687a8a13e7f 100644 --- a/include/test/suites.h +++ b/include/test/suites.h @@ -7,14 +7,9 @@ #ifndef __TEST_SUITES_H__ #define __TEST_SUITES_H__ -struct cmd_tbl; struct unit_test; struct unit_test_state; -/* 'command' functions normally called do_xxx where xxx is the command name */ -typedef int (*ut_cmd_func)(struct unit_test_state *uts, struct cmd_tbl *cmd, - int flags, int argc, char *const argv[]); - /** * cmd_ut_category() - Run a category of unit tests * -- cgit v1.2.3 From 854225191af5527619fcc944d38f2e2fd5d3ced1 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 7 Feb 2025 11:30:55 -0700 Subject: test: Make cmd_ut_category() static This function is not used outside the cmd_ut file anymore, so make it static. Signed-off-by: Simon Glass --- include/test/suites.h | 22 ---------------------- 1 file changed, 22 deletions(-) (limited to 'include/test') diff --git a/include/test/suites.h b/include/test/suites.h index 687a8a13e7f..f817da747c2 100644 --- a/include/test/suites.h +++ b/include/test/suites.h @@ -7,26 +7,4 @@ #ifndef __TEST_SUITES_H__ #define __TEST_SUITES_H__ -struct unit_test; -struct unit_test_state; - -/** - * cmd_ut_category() - Run a category of unit tests - * - * @uts: Unit-test state, which must be ready for use, i.e. ut_init_state() - * has been called. The caller is responsible for calling - * ut_uninit_state() after this function returns - * @name: Category name - * @prefix: Prefix of test name - * @tests: List of tests to run - * @n_ents: Number of tests in @tests - * @argc: Argument count provided. Must be >= 1. If this is 1 then all - * tests are run, otherwise only the one named @argv[1] is run. - * @argv: Arguments: argv[1] is the test to run (if @argc >= 2) - * Return: 0 if OK, CMD_RET_FAILURE on failure - */ -int cmd_ut_category(struct unit_test_state *uts, const char *name, - const char *prefix, struct unit_test *tests, int n_ents, - int argc, char *const argv[]); - #endif /* __TEST_SUITES_H__ */ -- cgit v1.2.3 From 32aba887e3d7cbcdf8fd98c107aaa79f3bd44a16 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 7 Feb 2025 11:30:56 -0700 Subject: test: Drop suites.h This file is empty now. Remove it and its uses. Signed-off-by: Simon Glass --- include/test/suites.h | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 include/test/suites.h (limited to 'include/test') diff --git a/include/test/suites.h b/include/test/suites.h deleted file mode 100644 index f817da747c2..00000000000 --- a/include/test/suites.h +++ /dev/null @@ -1,10 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * (C) Copyright 2015 - * Joe Hershberger, National Instruments, joe.hershberger@ni.com - */ - -#ifndef __TEST_SUITES_H__ -#define __TEST_SUITES_H__ - -#endif /* __TEST_SUITES_H__ */ -- cgit v1.2.3