diff mbox series

[v2,1/4] kunit: Support skipped tests

Message ID 20210528075932.347154-1-davidgow@google.com
State Superseded
Headers show
Series [v2,1/4] kunit: Support skipped tests | expand

Commit Message

David Gow May 28, 2021, 7:59 a.m. UTC
The kunit_mark_skipped() macro marks the current test as "skipped", with
the provided reason. The kunit_skip() macro will mark the test as
skipped, and abort the test.

The TAP specification supports this "SKIP directive" as a comment after
the "ok" / "not ok" for a test. See the "Directives" section of the TAP
spec for details:
https://testanything.org/tap-specification.html#directives

The 'success' field for KUnit tests is replaced with a kunit_status
enum, which can be SUCCESS, FAILURE, or SKIPPED, combined with a
'status_comment' containing information on why a test was skipped.

A new 'kunit_status' test suite is added to test this.

Signed-off-by: David Gow <davidgow@google.com>
Tested-by: Marco Elver <elver@google.com>
Reviewed-by: Daniel Latypov <dlatypov@google.com>
---

Changes since v1:
https://lore.kernel.org/linux-kselftest/20210526081112.3652290-1-davidgow@google.com/
- Renamed kunit_status_to_string() to kunit_status_to_ok_not_ok
- Fixed incorrect printing of status comments on non-skipped tests.

Not changes:
- Still using kunit_log(KERN_INFO,...) instead of kunit_info() as it
  seems to be consistently used for printing results in the KUnit code.

 include/kunit/test.h   | 68 ++++++++++++++++++++++++++++++++++++++----
 lib/kunit/kunit-test.c | 42 +++++++++++++++++++++++++-
 lib/kunit/test.c       | 51 ++++++++++++++++++-------------
 3 files changed, 134 insertions(+), 27 deletions(-)

Comments

kernel test robot May 28, 2021, 9:46 a.m. UTC | #1
Hi David,

I love your patch! Perhaps something to improve:

[auto build test WARNING on linus/master]
[also build test WARNING on v5.13-rc3 next-20210527]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/David-Gow/kunit-Support-skipped-tests/20210528-160224
base:   https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git 97e5bf604b7a0d6e1b3e00fe31d5fd4b9bffeaae
config: i386-randconfig-s001-20210528 (attached as .config)
compiler: gcc-9 (Debian 9.3.0-22) 9.3.0
reproduce:
        # apt-get install sparse
        # sparse version: v0.6.3-341-g8af24329-dirty
        # https://github.com/0day-ci/linux/commit/a464519206cd4484f64540020093cb45ef8e272e
        git remote add linux-review https://github.com/0day-ci/linux
        git fetch --no-tags linux-review David-Gow/kunit-Support-skipped-tests/20210528-160224
        git checkout a464519206cd4484f64540020093cb45ef8e272e
        # save the attached .config to linux build tree
        make W=1 C=1 CF='-fdiagnostic-prefix -D__CHECK_ENDIAN__' W=1 ARCH=i386 

If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>

All warnings (new ones prefixed by >>):

   lib/kunit/debugfs.c:28:6: warning: no previous prototype for 'kunit_debugfs_cleanup' [-Wmissing-prototypes]
      28 | void kunit_debugfs_cleanup(void)
         |      ^~~~~~~~~~~~~~~~~~~~~
   lib/kunit/debugfs.c:33:6: warning: no previous prototype for 'kunit_debugfs_init' [-Wmissing-prototypes]
      33 | void kunit_debugfs_init(void)
         |      ^~~~~~~~~~~~~~~~~~
   lib/kunit/debugfs.c: In function 'debugfs_print_results':
   lib/kunit/debugfs.c:67:6: error: implicit declaration of function 'kunit_status_to_string'; did you mean 'kunit_status_to_ok_not_ok'? [-Werror=implicit-function-declaration]
      67 |      kunit_status_to_string(success), 1, suite->name);
         |      ^~~~~~~~~~~~~~~~~~~~~~
         |      kunit_status_to_ok_not_ok
>> lib/kunit/debugfs.c:66:20: warning: format '%s' expects argument of type 'char *', but argument 3 has type 'int' [-Wformat=]
      66 |  seq_printf(seq, "%s %d - %s\n",
         |                   ~^
         |                    |
         |                    char *
         |                   %d
      67 |      kunit_status_to_string(success), 1, suite->name);
         |      ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
         |      |
         |      int
   lib/kunit/debugfs.c: At top level:
   lib/kunit/debugfs.c:92:6: warning: no previous prototype for 'kunit_debugfs_create_suite' [-Wmissing-prototypes]
      92 | void kunit_debugfs_create_suite(struct kunit_suite *suite)
         |      ^~~~~~~~~~~~~~~~~~~~~~~~~~
   lib/kunit/debugfs.c:108:6: warning: no previous prototype for 'kunit_debugfs_destroy_suite' [-Wmissing-prototypes]
     108 | void kunit_debugfs_destroy_suite(struct kunit_suite *suite)
         |      ^~~~~~~~~~~~~~~~~~~~~~~~~~~
   cc1: some warnings being treated as errors


vim +66 lib/kunit/debugfs.c

e2219db280e3fe Alan Maguire 2020-03-26  48  
e2219db280e3fe Alan Maguire 2020-03-26  49  /*
e2219db280e3fe Alan Maguire 2020-03-26  50   * /sys/kernel/debug/kunit/<testsuite>/results shows all results for testsuite.
e2219db280e3fe Alan Maguire 2020-03-26  51   */
e2219db280e3fe Alan Maguire 2020-03-26  52  static int debugfs_print_results(struct seq_file *seq, void *v)
e2219db280e3fe Alan Maguire 2020-03-26  53  {
e2219db280e3fe Alan Maguire 2020-03-26  54  	struct kunit_suite *suite = (struct kunit_suite *)seq->private;
e2219db280e3fe Alan Maguire 2020-03-26  55  	bool success = kunit_suite_has_succeeded(suite);
e2219db280e3fe Alan Maguire 2020-03-26  56  	struct kunit_case *test_case;
e2219db280e3fe Alan Maguire 2020-03-26  57  
e2219db280e3fe Alan Maguire 2020-03-26  58  	if (!suite || !suite->log)
e2219db280e3fe Alan Maguire 2020-03-26  59  		return 0;
e2219db280e3fe Alan Maguire 2020-03-26  60  
e2219db280e3fe Alan Maguire 2020-03-26  61  	seq_printf(seq, "%s", suite->log);
e2219db280e3fe Alan Maguire 2020-03-26  62  
e2219db280e3fe Alan Maguire 2020-03-26  63  	kunit_suite_for_each_test_case(suite, test_case)
e2219db280e3fe Alan Maguire 2020-03-26  64  		debugfs_print_result(seq, suite, test_case);
e2219db280e3fe Alan Maguire 2020-03-26  65  
e2219db280e3fe Alan Maguire 2020-03-26 @66  	seq_printf(seq, "%s %d - %s\n",
e2219db280e3fe Alan Maguire 2020-03-26  67  		   kunit_status_to_string(success), 1, suite->name);
e2219db280e3fe Alan Maguire 2020-03-26  68  	return 0;
e2219db280e3fe Alan Maguire 2020-03-26  69  }
e2219db280e3fe Alan Maguire 2020-03-26  70  

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org
Brendan Higgins June 4, 2021, 9:04 p.m. UTC | #2
On Fri, May 28, 2021 at 12:59 AM David Gow <davidgow@google.com> wrote:
>

> The kunit_mark_skipped() macro marks the current test as "skipped", with

> the provided reason. The kunit_skip() macro will mark the test as

> skipped, and abort the test.

>

> The TAP specification supports this "SKIP directive" as a comment after

> the "ok" / "not ok" for a test. See the "Directives" section of the TAP

> spec for details:

> https://testanything.org/tap-specification.html#directives

>

> The 'success' field for KUnit tests is replaced with a kunit_status

> enum, which can be SUCCESS, FAILURE, or SKIPPED, combined with a

> 'status_comment' containing information on why a test was skipped.

>

> A new 'kunit_status' test suite is added to test this.

>

> Signed-off-by: David Gow <davidgow@google.com>

> Tested-by: Marco Elver <elver@google.com>

> Reviewed-by: Daniel Latypov <dlatypov@google.com>


One fairly minor nit below. Other than that, looks great!

Reviewed-by: Brendan Higgins <brendanhiggins@google.com>


> ---


[...]

> diff --git a/include/kunit/test.h b/include/kunit/test.h

> index b68c61348121..1401c620ac5e 100644

> --- a/include/kunit/test.h

> +++ b/include/kunit/test.h

> @@ -105,6 +105,18 @@ struct kunit;

>  #define KUNIT_SUBTEST_INDENT           "    "

>  #define KUNIT_SUBSUBTEST_INDENT                "        "

>

> +/**

> + * enum kunit_status - Type of result for a test or test suite

> + * @KUNIT_SUCCESS: Denotes the test suite has not failed nor been skipped

> + * @KUNIT_FAILURE: Denotes the test has failed.

> + * @KUNIT_SKIPPED: Denotes the test has been skipped.

> + */

> +enum kunit_status {

> +       KUNIT_SUCCESS,

> +       KUNIT_FAILURE,

> +       KUNIT_SKIPPED,

> +};

> +

>  /**

>   * struct kunit_case - represents an individual test case.

>   *

> @@ -148,13 +160,20 @@ struct kunit_case {

>         const void* (*generate_params)(const void *prev, char *desc);

>

>         /* private: internal use only. */

> -       bool success;

> +       enum kunit_status status;

>         char *log;

>  };

>

> -static inline char *kunit_status_to_string(bool status)

> +static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)

>  {

> -       return status ? "ok" : "not ok";

> +       switch (status) {

> +       case KUNIT_SKIPPED:

> +       case KUNIT_SUCCESS:

> +               return "ok";

> +       case KUNIT_FAILURE:

> +               return "not ok";

> +       }

> +       return "invalid";

>  }

>

>  /**

> @@ -212,6 +231,7 @@ struct kunit_suite {

>         struct kunit_case *test_cases;

>

>         /* private: internal use only */

> +       char status_comment[256];


nit: How about we make the 256 a constant since you use it in a number
of places?

If not, at least when you reference the struct, you might want to use
ARRAY_SIZE(...).

>         struct dentry *debugfs;

>         char *log;

>  };

[...]
diff mbox series

Patch

diff --git a/include/kunit/test.h b/include/kunit/test.h
index b68c61348121..1401c620ac5e 100644
--- a/include/kunit/test.h
+++ b/include/kunit/test.h
@@ -105,6 +105,18 @@  struct kunit;
 #define KUNIT_SUBTEST_INDENT		"    "
 #define KUNIT_SUBSUBTEST_INDENT		"        "
 
+/**
+ * enum kunit_status - Type of result for a test or test suite
+ * @KUNIT_SUCCESS: Denotes the test suite has not failed nor been skipped
+ * @KUNIT_FAILURE: Denotes the test has failed.
+ * @KUNIT_SKIPPED: Denotes the test has been skipped.
+ */
+enum kunit_status {
+	KUNIT_SUCCESS,
+	KUNIT_FAILURE,
+	KUNIT_SKIPPED,
+};
+
 /**
  * struct kunit_case - represents an individual test case.
  *
@@ -148,13 +160,20 @@  struct kunit_case {
 	const void* (*generate_params)(const void *prev, char *desc);
 
 	/* private: internal use only. */
-	bool success;
+	enum kunit_status status;
 	char *log;
 };
 
-static inline char *kunit_status_to_string(bool status)
+static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)
 {
-	return status ? "ok" : "not ok";
+	switch (status) {
+	case KUNIT_SKIPPED:
+	case KUNIT_SUCCESS:
+		return "ok";
+	case KUNIT_FAILURE:
+		return "not ok";
+	}
+	return "invalid";
 }
 
 /**
@@ -212,6 +231,7 @@  struct kunit_suite {
 	struct kunit_case *test_cases;
 
 	/* private: internal use only */
+	char status_comment[256];
 	struct dentry *debugfs;
 	char *log;
 };
@@ -245,19 +265,21 @@  struct kunit {
 	 * be read after the test case finishes once all threads associated
 	 * with the test case have terminated.
 	 */
-	bool success; /* Read only after test_case finishes! */
 	spinlock_t lock; /* Guards all mutable test state. */
+	enum kunit_status status; /* Read only after test_case finishes! */
 	/*
 	 * Because resources is a list that may be updated multiple times (with
 	 * new resources) from any thread associated with a test case, we must
 	 * protect it with some type of lock.
 	 */
 	struct list_head resources; /* Protected by lock. */
+
+	char status_comment[256];
 };
 
 static inline void kunit_set_failure(struct kunit *test)
 {
-	WRITE_ONCE(test->success, false);
+	WRITE_ONCE(test->status, KUNIT_FAILURE);
 }
 
 void kunit_init_test(struct kunit *test, const char *name, char *log);
@@ -348,7 +370,7 @@  static inline int kunit_run_all_tests(void)
 #define kunit_suite_for_each_test_case(suite, test_case)		\
 	for (test_case = suite->test_cases; test_case->run_case; test_case++)
 
-bool kunit_suite_has_succeeded(struct kunit_suite *suite);
+enum kunit_status kunit_suite_has_succeeded(struct kunit_suite *suite);
 
 /*
  * Like kunit_alloc_resource() below, but returns the struct kunit_resource
@@ -612,6 +634,40 @@  void kunit_cleanup(struct kunit *test);
 
 void kunit_log_append(char *log, const char *fmt, ...);
 
+/**
+ * kunit_mark_skipped() - Marks @test_or_suite as skipped
+ *
+ * @test_or_suite: The test context object.
+ * @fmt:  A printk() style format string.
+ *
+ * Marks the test as skipped. @fmt is given output as the test status
+ * comment, typically the reason the test was skipped.
+ *
+ * Test execution continues after kunit_mark_skipped() is called.
+ */
+#define kunit_mark_skipped(test_or_suite, fmt, ...)			\
+	do {								\
+		WRITE_ONCE((test_or_suite)->status, KUNIT_SKIPPED);	\
+		scnprintf((test_or_suite)->status_comment, 256, fmt, ##__VA_ARGS__); \
+	} while (0)
+
+/**
+ * kunit_skip() - Marks @test_or_suite as skipped
+ *
+ * @test_or_suite: The test context object.
+ * @fmt:  A printk() style format string.
+ *
+ * Skips the test. @fmt is given output as the test status
+ * comment, typically the reason the test was skipped.
+ *
+ * Test execution is halted after kunit_skip() is called.
+ */
+#define kunit_skip(test_or_suite, fmt, ...)				\
+	do {								\
+		kunit_mark_skipped((test_or_suite), fmt, ##__VA_ARGS__);\
+		kunit_try_catch_throw(&((test_or_suite)->try_catch));	\
+	} while (0)
+
 /*
  * printk and log to per-test or per-suite log buffer.  Logging only done
  * if CONFIG_KUNIT_DEBUGFS is 'y'; if it is 'n', no log is allocated/used.
diff --git a/lib/kunit/kunit-test.c b/lib/kunit/kunit-test.c
index 69f902440a0e..d69efcbed624 100644
--- a/lib/kunit/kunit-test.c
+++ b/lib/kunit/kunit-test.c
@@ -437,7 +437,47 @@  static void kunit_log_test(struct kunit *test)
 #endif
 }
 
+static void kunit_status_set_failure_test(struct kunit *test)
+{
+	struct kunit fake;
+
+	kunit_init_test(&fake, "fake test", NULL);
+
+	KUNIT_EXPECT_EQ(test, fake.status, (enum kunit_status)KUNIT_SUCCESS);
+	kunit_set_failure(&fake);
+	KUNIT_EXPECT_EQ(test, fake.status, (enum kunit_status)KUNIT_FAILURE);
+}
+
+static void kunit_status_mark_skipped_test(struct kunit *test)
+{
+	struct kunit fake;
+
+	kunit_init_test(&fake, "fake test", NULL);
+
+	/* Before: Should be SUCCESS with no comment. */
+	KUNIT_EXPECT_EQ(test, fake.status, KUNIT_SUCCESS);
+	KUNIT_EXPECT_STREQ(test, fake.status_comment, "");
+
+	/* Mark the test as skipped. */
+	kunit_mark_skipped(&fake, "Accepts format string: %s", "YES");
+
+	/* After: Should be SKIPPED with our comment. */
+	KUNIT_EXPECT_EQ(test, fake.status, (enum kunit_status)KUNIT_SKIPPED);
+	KUNIT_EXPECT_STREQ(test, fake.status_comment, "Accepts format string: YES");
+}
+
+static struct kunit_case kunit_status_test_cases[] = {
+	KUNIT_CASE(kunit_status_set_failure_test),
+	KUNIT_CASE(kunit_status_mark_skipped_test),
+	{}
+};
+
+static struct kunit_suite kunit_status_test_suite = {
+	.name = "kunit_status",
+	.test_cases = kunit_status_test_cases,
+};
+
 kunit_test_suites(&kunit_try_catch_test_suite, &kunit_resource_test_suite,
-		  &kunit_log_test_suite);
+		  &kunit_log_test_suite, &kunit_status_test_suite);
 
 MODULE_LICENSE("GPL v2");
diff --git a/lib/kunit/test.c b/lib/kunit/test.c
index 2f6cc0123232..8ce0c8fddb96 100644
--- a/lib/kunit/test.c
+++ b/lib/kunit/test.c
@@ -98,12 +98,14 @@  static void kunit_print_subtest_start(struct kunit_suite *suite)
 
 static void kunit_print_ok_not_ok(void *test_or_suite,
 				  bool is_test,
-				  bool is_ok,
+				  enum kunit_status status,
 				  size_t test_number,
-				  const char *description)
+				  const char *description,
+				  const char *directive)
 {
 	struct kunit_suite *suite = is_test ? NULL : test_or_suite;
 	struct kunit *test = is_test ? test_or_suite : NULL;
+	const char *directive_header = (status == KUNIT_SKIPPED) ? " # SKIP " : "";
 
 	/*
 	 * We do not log the test suite results as doing so would
@@ -114,25 +116,31 @@  static void kunit_print_ok_not_ok(void *test_or_suite,
 	 * representation.
 	 */
 	if (suite)
-		pr_info("%s %zd - %s\n",
-			kunit_status_to_string(is_ok),
-			test_number, description);
+		pr_info("%s %zd - %s%s%s\n",
+			kunit_status_to_ok_not_ok(status),
+			test_number, description, directive_header,
+			(status == KUNIT_SKIPPED) ? directive : "");
 	else
-		kunit_log(KERN_INFO, test, KUNIT_SUBTEST_INDENT "%s %zd - %s",
-			  kunit_status_to_string(is_ok),
-			  test_number, description);
+		kunit_log(KERN_INFO, test,
+			  KUNIT_SUBTEST_INDENT "%s %zd - %s%s%s",
+			  kunit_status_to_ok_not_ok(status),
+			  test_number, description, directive_header,
+			  (status == KUNIT_SKIPPED) ? directive : "");
 }
 
-bool kunit_suite_has_succeeded(struct kunit_suite *suite)
+enum kunit_status kunit_suite_has_succeeded(struct kunit_suite *suite)
 {
 	const struct kunit_case *test_case;
+	enum kunit_status status = KUNIT_SKIPPED;
 
 	kunit_suite_for_each_test_case(suite, test_case) {
-		if (!test_case->success)
-			return false;
+		if (test_case->status == KUNIT_FAILURE)
+			return KUNIT_FAILURE;
+		else if (test_case->status == KUNIT_SUCCESS)
+			status = KUNIT_SUCCESS;
 	}
 
-	return true;
+	return status;
 }
 EXPORT_SYMBOL_GPL(kunit_suite_has_succeeded);
 
@@ -143,7 +151,8 @@  static void kunit_print_subtest_end(struct kunit_suite *suite)
 	kunit_print_ok_not_ok((void *)suite, false,
 			      kunit_suite_has_succeeded(suite),
 			      kunit_suite_counter++,
-			      suite->name);
+			      suite->name,
+			      suite->status_comment);
 }
 
 unsigned int kunit_test_case_num(struct kunit_suite *suite,
@@ -252,7 +261,8 @@  void kunit_init_test(struct kunit *test, const char *name, char *log)
 	test->log = log;
 	if (test->log)
 		test->log[0] = '\0';
-	test->success = true;
+	test->status = KUNIT_SUCCESS;
+	test->status_comment[0] = '\0';
 }
 EXPORT_SYMBOL_GPL(kunit_init_test);
 
@@ -376,7 +386,8 @@  static void kunit_run_case_catch_errors(struct kunit_suite *suite,
 	context.test_case = test_case;
 	kunit_try_catch_run(try_catch, &context);
 
-	test_case->success = test->success;
+	test_case->status = test->status;
+
 }
 
 int kunit_run_tests(struct kunit_suite *suite)
@@ -388,7 +399,6 @@  int kunit_run_tests(struct kunit_suite *suite)
 
 	kunit_suite_for_each_test_case(suite, test_case) {
 		struct kunit test = { .param_value = NULL, .param_index = 0 };
-		bool test_success = true;
 
 		if (test_case->generate_params) {
 			/* Get initial param. */
@@ -398,7 +408,6 @@  int kunit_run_tests(struct kunit_suite *suite)
 
 		do {
 			kunit_run_case_catch_errors(suite, test_case, &test);
-			test_success &= test_case->success;
 
 			if (test_case->generate_params) {
 				if (param_desc[0] == '\0') {
@@ -410,7 +419,7 @@  int kunit_run_tests(struct kunit_suite *suite)
 					  KUNIT_SUBTEST_INDENT
 					  "# %s: %s %d - %s",
 					  test_case->name,
-					  kunit_status_to_string(test.success),
+					  kunit_status_to_ok_not_ok(test.status),
 					  test.param_index + 1, param_desc);
 
 				/* Get next param. */
@@ -420,9 +429,10 @@  int kunit_run_tests(struct kunit_suite *suite)
 			}
 		} while (test.param_value);
 
-		kunit_print_ok_not_ok(&test, true, test_success,
+		kunit_print_ok_not_ok(&test, true, test_case->status,
 				      kunit_test_case_num(suite, test_case),
-				      test_case->name);
+				      test_case->name,
+				      test.status_comment);
 	}
 
 	kunit_print_subtest_end(suite);
@@ -434,6 +444,7 @@  EXPORT_SYMBOL_GPL(kunit_run_tests);
 static void kunit_init_suite(struct kunit_suite *suite)
 {
 	kunit_debugfs_create_suite(suite);
+	suite->status_comment[0] = '\0';
 }
 
 int __kunit_test_suites_init(struct kunit_suite * const * const suites)