support: Add support for delayed test failure reporting
The new functions support_record_failure records a test failure,
but does not terminate the process. The macros TEST_VERIFY
and TEST_VERIFY_EXIT check that a condition is true.
2016-12-28 Florian Weimer <fweimer@redhat.com>
* support/Makefile (libsupport-routines): Add
support_test_verify_impl, support_record_failure, xfork, xwaitpid.
(tests): Add tst-support_record_failure.
(tests-special): tst-support_record_failure-2.
(tst-support_record_failure-2.out): Depend on
tst-support_record_failure-2.sh and tst-support_record_failure.
* support/check.h (TEST_VERIFY, TEST_VERIFY_EXIT): Define.
(support_test_verify_impl, support_record_failure)
(support_report_failure, support_report_failure_reset): Declare.
* support/support_test_main.c (adjust_exit_status): New function.
(support_test_main): Call it to incorporate record test failures.
* support/support_record_failure.c: New file.
* support/tst-support_record_failure.c: Likewise.
* support/tst-support_record_failure-2.sh: Likewise.
* support/xunistd.h: Likewise.
* support/xfork.c: Likewise.
* support/xwaitpid.c: Likewise.
@@ -30,11 +30,14 @@ libsupport-routines = \
ignore_stderr \
oom_error \
set_fortify_handler \
+ support_record_failure \
support_test_main \
+ support_test_verify_impl \
temp_file \
write_message \
xasprintf \
xcalloc \
+ xfork \
xmalloc \
xpthread_barrier_destroy \
xpthread_barrier_init \
@@ -51,6 +54,7 @@ libsupport-routines = \
xpthread_spin_lock \
xpthread_spin_unlock \
xrealloc \
+ xwaitpid \
libsupport-static-only-routines := $(libsupport-routines)
# Only build one variant of the library.
@@ -59,6 +63,18 @@ ifeq ($(build-shared),yes)
libsupport-inhibit-o += .o
endif
-tests = README-testing
+tests = \
+ README-testing \
+ tst-support_record_failure \
+
+tests-special = \
+ $(objpfx)tst-support_record_failure-2.out
+
+$(objpfx)tst-support_record_failure-2.out: tst-support_record_failure-2.sh \
+ $(objpfx)tst-support_record_failure
+ $(SHELL) $< $(common-objpfx) '$(test-program-prefix-before-env)' \
+ '$(run-program-env)' '$(test-program-prefix-after-env)' \
+ > $@; \
+ $(evaluate-test)
include ../Rules
@@ -1,4 +1,4 @@
-/* Macros for reporting test results.
+/* Functionality for reporting test results.
Copyright (C) 2016 Free Software Foundation, Inc.
This file is part of the GNU C Library.
@@ -35,6 +35,25 @@ __BEGIN_DECLS
#define FAIL_EXIT1(...) \
support_exit_failure_impl (1, __FILE__, __LINE__, __VA_ARGS__)
+/* Record a test failure (but continue executing) if EXPR evaluates to
+ false. */
+#define TEST_VERIFY(expr) \
+ ({ \
+ if (expr) \
+ ; \
+ else \
+ support_test_verify_impl (-1, __FILE__, __LINE__, #expr); \
+ })
+
+/* Record a test failure and exit if EXPR evaluates to false. */
+#define TEST_VERIFY_EXIT(expr) \
+ ({ \
+ if (expr) \
+ ; \
+ else \
+ support_test_verify_impl (1, __FILE__, __LINE__, #expr); \
+ })
+
int support_print_failure_impl (const char *file, int line,
const char *format, ...)
__attribute__ ((nonnull (1), format (printf, 3, 4)));
@@ -42,7 +61,24 @@ void support_exit_failure_impl (int exit_status,
const char *file, int line,
const char *format, ...)
__attribute__ ((noreturn, nonnull (2), format (printf, 4, 5)));
+void support_test_verify_impl (int status, const char *file, int line,
+ const char *expr);
+/* Record a test failure. This function returns and does not
+ terminate the process. The failure counter is stored in a shared
+ memory mapping, so that failures reported in child processes are
+ visible to the parent process and test driver. This function
+ depends on initialization by an ELF constructor, so it can only be
+ invoked after the test driver has run. Note that this function
+ does not support reporting failures from a DSO. */
+void support_record_failure (void);
+
+/* Internal function called by the test driver. */
+int support_report_failure (int status)
+ __attribute__ ((weak, warn_unused_result));
+
+/* Internal function used to test the failure recording framework. */
+void support_record_failure_reset (void);
__END_DECLS
new file mode 100644
@@ -0,0 +1,106 @@
+/* Global test failure counter.
+ Copyright (C) 2016 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include <support/check.h>
+#include <support/support.h>
+#include <support/test-driver.h>
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+/* This structure keeps track of test failures. The counter is
+ incremented on each failure. The failed member is set to true if a
+ failure is detected, so that even if the counter wraps around to
+ zero, the failure of a test can be detected.
+
+ The init constructor function below puts *state on a shared
+ annonymous mapping, so that failure reports from subprocesses
+ propagate to the parent process. */
+struct test_failures
+{
+ unsigned counter;
+ unsigned failed;
+};
+static struct test_failures *state;
+
+static __attribute__ ((constructor)) void
+init (void)
+{
+ void *ptr = mmap (NULL, sizeof (*state), PROT_READ | PROT_WRITE,
+ MAP_ANONYMOUS | MAP_SHARED, -1, 0);
+ if (ptr == MAP_FAILED)
+ {
+ printf ("error: could not map %zu bytes: %m\n", sizeof (*state));
+ exit (1);
+ }
+ /* Zero-initialization of the struct is sufficient. */
+ state = ptr;
+}
+
+void
+support_record_failure (void)
+{
+ if (state == NULL)
+ {
+ write_message
+ ("error: support_record_failure called without initialization\n");
+ _exit (1);
+ }
+ /* Relaxed MO is sufficient because we are only interested in the
+ values themselves, in isolation. */
+ __atomic_store_n (&state->failed, 1, __ATOMIC_RELEASE);
+ __atomic_add_fetch (&state->counter, 1, __ATOMIC_RELEASE);
+}
+
+int
+support_report_failure (int status)
+{
+ if (state == NULL)
+ {
+ write_message
+ ("error: support_report_failure called without initialization\n");
+ return 1;
+ }
+
+ /* Relaxed MO is sufficient because acquire test result reporting
+ assumes that exiting from the main thread happens before the
+ error reporting via support_record_failure, which requires some
+ form of external synchronization. */
+ bool failed = __atomic_load_n (&state->failed, __ATOMIC_RELAXED);
+ if (failed)
+ printf ("error: %u test failures\n",
+ __atomic_load_n (&state->counter, __ATOMIC_RELAXED));
+
+ if ((status == 0 || status == EXIT_UNSUPPORTED) && failed)
+ /* If we have a recorded failure, it overrides a non-failure
+ report from the test function. */
+ status = 1;
+ return status;
+}
+
+void
+support_record_failure_reset (void)
+{
+ /* Only used for testing the test framework, with external
+ synchronization, but use release MO for consistency. */
+ __atomic_store_n (&state->failed, 0, __ATOMIC_RELAXED);
+ __atomic_add_fetch (&state->counter, 0, __ATOMIC_RELAXED);
+}
@@ -17,6 +17,7 @@
<http://www.gnu.org/licenses/>. */
#include <support/test-driver.h>
+#include <support/check.h>
#include <support/temp_file-internal.h>
#include <assert.h>
@@ -164,6 +165,17 @@ static bool test_main_called;
const char *test_dir = NULL;
+
+/* If test failure reporting has been linked in, it may contribute
+ additional test failures. */
+static int
+adjust_exit_status (int status)
+{
+ if (support_report_failure != NULL)
+ return support_report_failure (status);
+ return status;
+}
+
int
support_test_main (int argc, char **argv, const struct test_config *config)
{
@@ -300,7 +312,7 @@ support_test_main (int argc, char **argv, const struct test_config *config)
/* If we are not expected to fork run the function immediately. */
if (direct)
- return run_test_function (argc, argv, config);
+ return adjust_exit_status (run_test_function (argc, argv, config));
/* Set up the test environment:
- prevent core dumps
@@ -363,8 +375,8 @@ support_test_main (int argc, char **argv, const struct test_config *config)
if (config->expected_status == 0)
{
if (config->expected_signal == 0)
- /* Simply exit with the return value of the test. */
- return WEXITSTATUS (status);
+ /* Exit with the return value of the test. */
+ return adjust_exit_status (WEXITSTATUS (status));
else
{
printf ("Expected signal '%s' from child, got none\n",
@@ -382,7 +394,7 @@ support_test_main (int argc, char **argv, const struct test_config *config)
exit (1);
}
}
- return 0;
+ return adjust_exit_status (0);
}
/* Process was killed by timer or other signal. */
else
@@ -401,6 +413,6 @@ support_test_main (int argc, char **argv, const struct test_config *config)
exit (1);
}
- return 0;
+ return adjust_exit_status (0);
}
}
new file mode 100644
@@ -0,0 +1,33 @@
+/* Implementation of the TEST_VERIFY and TEST_VERIFY_EXIT macros.
+ Copyright (C) 2016 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include <support/check.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+
+void
+support_test_verify_impl (int status, const char *file, int line,
+ const char *expr)
+{
+ support_record_failure ();
+ printf ("FAIL %s:%d: not true: %s\n", file, line, expr);
+ if (status >= 0)
+ exit (status);
+
+}
new file mode 100644
@@ -0,0 +1,66 @@
+#!/bin/sh
+# Test failure recording (with and without --direct).
+# Copyright (C) 2016 Free Software Foundation, Inc.
+# This file is part of the GNU C Library.
+
+# The GNU C Library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# The GNU C Library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with the GNU C Library; if not, see
+# <http://www.gnu.org/licenses/>. */
+
+set -e
+
+common_objpfx=$1; shift
+test_program_prefix_before_env=$1; shift
+run_program_env=$1; shift
+test_program_prefix_after_env=$1; shift
+
+run_test () {
+ expected_status="$1"
+ expected_output="$2"
+ shift 2
+ args="${common_objpfx}support/tst-support_record_failure $*"
+ echo "running: $args"
+ set +e
+ output="$(${test_program_prefix_before_env} \
+ ${run_program} ${test_program_prefix_after_env} $args)"
+ status=$?
+ set -e
+ echo " exit status: $status"
+ if test "$output" != "$expected_output" ; then
+ echo "error: unexpected ouput: $output"
+ exit 1
+ fi
+ if test "$status" -ne "$expected_status" ; then
+ echo "error: exit status $expected_status expected"
+ exit 1
+ fi
+}
+
+different_status () {
+ direct="$1"
+ run_test 1 "error: 1 test failures" $direct --status=0
+ run_test 1 "error: 1 test failures" $direct --status=1
+ run_test 2 "error: 1 test failures" $direct --status=2
+ run_test 1 "error: 1 test failures" $direct --status=77
+ run_test 2 "FAIL tst-support_record_failure.c:108: not true: false
+error: 1 test failures" $direct --test-verify
+}
+
+different_status
+different_status --direct
+
+run_test 1 "FAIL tst-support_record_failure.c:113: not true: false
+error: 1 test failures" --test-verify-exit
+# --direct does not print the summary error message if exit is called.
+run_test 1 "FAIL tst-support_record_failure.c:113: not true: false" \
+ --direct --test-verify-exit
new file mode 100644
@@ -0,0 +1,150 @@
+/* Test support_record_failure state sharing.
+ Copyright (C) 2016 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include <support/check.h>
+#include <support/support.h>
+#include <support/test-driver.h>
+#include <support/xunistd.h>
+
+#include <getopt.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+static int exit_status_with_failure = -1;
+static bool test_verify;
+static bool test_verify_exit;
+enum
+ {
+ OPT_STATUS = 10001,
+ OPT_TEST_VERIFY,
+ OPT_TEST_VERIFY_EXIT,
+ };
+#define CMDLINE_OPTIONS \
+ { "status", required_argument, NULL, OPT_STATUS }, \
+ { "test-verify", no_argument, NULL, OPT_TEST_VERIFY }, \
+ { "test-verify-exit", no_argument, NULL, OPT_TEST_VERIFY_EXIT },
+static void
+cmdline_process (int c)
+{
+ switch (c)
+ {
+ case OPT_STATUS:
+ exit_status_with_failure = atoi (optarg);
+ break;
+ case OPT_TEST_VERIFY:
+ test_verify = true;
+ break;
+ case OPT_TEST_VERIFY_EXIT:
+ test_verify_exit = true;
+ break;
+ }
+}
+#define CMDLINE_PROCESS cmdline_process
+
+static void
+check_failure_reporting (int phase, int zero, int unsupported)
+{
+ int status = support_report_failure (0);
+ if (status != zero)
+ {
+ printf ("real-error (phase %d): support_report_failure (0) == %d\n",
+ phase, status);
+ exit (1);
+ }
+ status = support_report_failure (1);
+ if (status != 1)
+ {
+ printf ("real-error (phase %d): support_report_failure (1) == %d\n",
+ phase, status);
+ exit (1);
+ }
+ status = support_report_failure (2);
+ if (status != 2)
+ {
+ printf ("real-error (phase %d): support_report_failure (2) == %d\n",
+ phase, status);
+ exit (1);
+ }
+ status = support_report_failure (EXIT_UNSUPPORTED);
+ if (status != unsupported)
+ {
+ printf ("real-error (phase %d): "
+ "support_report_failure (EXIT_UNSUPPORTED) == %d\n",
+ phase, status);
+ exit (1);
+ }
+}
+
+static int
+do_test (void)
+{
+ if (exit_status_with_failure >= 0)
+ {
+ /* External invocation with requested error status. Used by
+ tst-support_report_failure-2.sh. */
+ support_record_failure ();
+ return exit_status_with_failure;
+ }
+ TEST_VERIFY (true);
+ TEST_VERIFY_EXIT (true);
+ if (test_verify)
+ {
+ TEST_VERIFY (false);
+ return 2; /* Expected exit status. */
+ }
+ if (test_verify_exit)
+ {
+ TEST_VERIFY_EXIT (false);
+ return 3; /* Not reached. Expected exit status is 1. */
+ }
+
+ printf ("info: This test tests the test framework.\n"
+ "info: It reports some expected errors on stdout.\n");
+
+ /* Check that the status is passed through unchanged. */
+ check_failure_reporting (1, 0, EXIT_UNSUPPORTED);
+
+ /* Check state propagation from a subprocess. */
+ pid_t pid = xfork ();
+ if (pid == 0)
+ {
+ support_record_failure ();
+ _exit (0);
+ }
+ int status;
+ xwaitpid (pid, &status, 0);
+ if (status != 0)
+ {
+ printf ("real-error: incorrect status from subprocess: %d\n", status);
+ return 1;
+ }
+ check_failure_reporting (2, 1, 1);
+
+ /* Also test directly in the parent process. */
+ support_record_failure_reset ();
+ check_failure_reporting (3, 0, EXIT_UNSUPPORTED);
+ support_record_failure ();
+ check_failure_reporting (4, 1, 1);
+
+ /* We need to mask the failure above. */
+ support_record_failure_reset ();
+ return 0;
+}
+
+#include <support/test-driver.c>
new file mode 100644
@@ -0,0 +1,34 @@
+/* fork with error checking.
+ Copyright (C) 2016 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include <support/xunistd.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+
+pid_t
+xfork (void)
+{
+ pid_t result = fork ();
+ if (result < 0)
+ {
+ printf ("error: fork: %m\n");
+ exit (1);
+ }
+ return result;
+}
new file mode 100644
@@ -0,0 +1,35 @@
+/* POSIX-specific extra functions.
+ Copyright (C) 2016 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+/* These wrapper functions use POSIX types and therefore cannot be
+ declared in <support/support.h>. */
+
+#ifndef SUPPORT_XUNISTD_H
+#define SUPPORT_XUNISTD_H
+
+#include <unistd.h>
+#include <sys/cdefs.h>
+
+__BEGIN_DECLS
+
+pid_t xfork (void);
+pid_t xwaitpid (pid_t, int *status, int flags);
+
+__END_DECLS
+
+#endif /* SUPPORT_XUNISTD_H */
new file mode 100644
@@ -0,0 +1,35 @@
+/* waitpid with error checking.
+ Copyright (C) 2016 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include <support/xunistd.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/wait.h>
+
+int
+xwaitpid (int pid, int *status, int flags)
+{
+ pid_t result = waitpid (pid, status, flags);
+ if (result < 0)
+ {
+ printf ("error: waitpid: %m\n");
+ exit (1);
+ }
+ return result;
+}