From 3f37d14b8a3152441f36b6bc74000996679f0998 Mon Sep 17 00:00:00 2001 From: Rikard Falkeborn Date: Sun, 7 Jun 2020 23:57:15 +0200 Subject: [PATCH 1/4] kunit: kunit_config: Fix parsing of CONFIG options with space Commit 8b59cd81dc5e ("kbuild: ensure full rebuild when the compiler is updated") introduced a new CONFIG option CONFIG_CC_VERSION_TEXT. On my system, this is set to "gcc (GCC) 10.1.0" which breaks KUnit config parsing which did not like the spaces in the string. Fix this by updating the regex to allow strings containing spaces. Fixes: 8b59cd81dc5e ("kbuild: ensure full rebuild when the compiler is updated") Signed-off-by: Rikard Falkeborn Reviewed-by: Brendan Higgins Signed-off-by: Shuah Khan --- tools/testing/kunit/kunit_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/kunit/kunit_config.py b/tools/testing/kunit/kunit_config.py index e75063d603b5..02ffc3a3e5dc 100644 --- a/tools/testing/kunit/kunit_config.py +++ b/tools/testing/kunit/kunit_config.py @@ -10,7 +10,7 @@ import collections import re CONFIG_IS_NOT_SET_PATTERN = r'^# CONFIG_(\w+) is not set$' -CONFIG_PATTERN = r'^CONFIG_(\w+)=(\S+)$' +CONFIG_PATTERN = r'^CONFIG_(\w+)=(\S+|".*")$' KconfigEntryBase = collections.namedtuple('KconfigEntry', ['name', 'value']) From e173b8b8c419897eabe0e337a0125c6d87ef1644 Mon Sep 17 00:00:00 2001 From: Uriel Guajardo Date: Thu, 11 Jun 2020 21:05:45 +0000 Subject: [PATCH 2/4] kunit: show error if kunit results are not present Currently, if the kernel is configured incorrectly or if it crashes before any kunit tests are run, kunit finishes without error, reporting that 0 test cases were run. To fix this, an error is shown when the tap header is not found, which indicates that kunit was not able to run at all. Signed-off-by: Uriel Guajardo Reviewed-by: Brendan Higgins Signed-off-by: Shuah Khan --- tools/testing/kunit/kunit_parser.py | 8 ++++---- tools/testing/kunit/kunit_tool_test.py | 11 +++++++++++ .../kunit/test_data/test_insufficient_memory.log | 0 3 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 tools/testing/kunit/test_data/test_insufficient_memory.log diff --git a/tools/testing/kunit/kunit_parser.py b/tools/testing/kunit/kunit_parser.py index 64aac9dcd431..f13e0c0d6663 100644 --- a/tools/testing/kunit/kunit_parser.py +++ b/tools/testing/kunit/kunit_parser.py @@ -265,11 +265,9 @@ def bubble_up_suite_errors(test_suite_list: List[TestSuite]) -> TestStatus: return bubble_up_errors(lambda x: x.status, test_suite_list) def parse_test_result(lines: List[str]) -> TestResult: - if not lines: - return TestResult(TestStatus.NO_TESTS, [], lines) consume_non_diagnositic(lines) - if not parse_tap_header(lines): - return None + if not lines or not parse_tap_header(lines): + return TestResult(TestStatus.NO_TESTS, [], lines) test_suites = [] test_suite = parse_test_suite(lines) while test_suite: @@ -282,6 +280,8 @@ def parse_run_tests(kernel_output) -> TestResult: failed_tests = 0 crashed_tests = 0 test_result = parse_test_result(list(isolate_kunit_output(kernel_output))) + if test_result.status == TestStatus.NO_TESTS: + print_with_timestamp(red('[ERROR] ') + 'no kunit output detected') for test_suite in test_result.suites: if test_suite.status == TestStatus.SUCCESS: print_suite_divider(green('[PASSED] ') + test_suite.name) diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index 5bb7b118ebd9..f9eeaea94cad 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -170,6 +170,17 @@ class KUnitParserTest(unittest.TestCase): result.status) file.close() + def test_no_kunit_output(self): + crash_log = get_absolute_path( + 'test_data/test_insufficient_memory.log') + file = open(crash_log) + print_mock = mock.patch('builtins.print').start() + result = kunit_parser.parse_run_tests( + kunit_parser.isolate_kunit_output(file.readlines())) + print_mock.assert_any_call(StrContains("no kunit output detected")) + print_mock.stop() + file.close() + def test_crashed_test(self): crashed_log = get_absolute_path( 'test_data/test_is_test_passed-crash.log') diff --git a/tools/testing/kunit/test_data/test_insufficient_memory.log b/tools/testing/kunit/test_data/test_insufficient_memory.log new file mode 100644 index 000000000000..e69de29bb2d1 From ee61492ab92b108b7db66acfd2c5f794244466b9 Mon Sep 17 00:00:00 2001 From: David Gow Date: Mon, 15 Jun 2020 23:47:30 -0700 Subject: [PATCH 3/4] kunit: kunit_tool: Fix invalid result when build fails When separating out different phases of running tests[1] (build/exec/parse/etc), the format of the KunitResult tuple changed (adding an elapsed_time variable). This is not populated during a build failure, causing kunit.py to crash. This fixes [1] to probably populate the result variable, causing a failing build to be reported properly. [1]: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=45ba7a893ad89114e773b3dc32f6431354c465d6 Signed-off-by: David Gow Reviewed-by: Brendan Higgins Tested-by: Brendan Higgins Signed-off-by: Shuah Khan --- tools/testing/kunit/kunit.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py index 787b6d4ad716..f9b769f3437d 100755 --- a/tools/testing/kunit/kunit.py +++ b/tools/testing/kunit/kunit.py @@ -82,7 +82,9 @@ def build_tests(linux: kunit_kernel.LinuxSourceTree, request.make_options) build_end = time.time() if not success: - return KunitResult(KunitStatus.BUILD_FAILURE, 'could not build kernel') + return KunitResult(KunitStatus.BUILD_FAILURE, + 'could not build kernel', + build_end - build_start) if not success: return KunitResult(KunitStatus.BUILD_FAILURE, 'could not build kernel', From c63d2dd7e134ebddce4745c51f9572b3f0d92b26 Mon Sep 17 00:00:00 2001 From: David Gow Date: Tue, 2 Jun 2020 20:37:25 -0700 Subject: [PATCH 4/4] Documentation: kunit: Add some troubleshooting tips to the FAQ Add an FAQ entry to the KUnit documentation with some tips for troubleshooting KUnit and kunit_tool. These suggestions largely came from an email thread: https://lore.kernel.org/linux-kselftest/41db8bbd-3ba0-8bde-7352-083bf4b947ff@intel.com/T/#m23213d4e156db6d59b0b460a9014950f5ff6eb03 Signed-off-by: David Gow Reviewed-by: Alan Maguire Reviewed-by: Brendan Higgins Signed-off-by: Shuah Khan --- Documentation/dev-tools/kunit/faq.rst | 40 +++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/Documentation/dev-tools/kunit/faq.rst b/Documentation/dev-tools/kunit/faq.rst index ea55b2467653..1628862e7024 100644 --- a/Documentation/dev-tools/kunit/faq.rst +++ b/Documentation/dev-tools/kunit/faq.rst @@ -61,3 +61,43 @@ test, or an end-to-end test. kernel by installing a production configuration of the kernel on production hardware with a production userspace and then trying to exercise some behavior that depends on interactions between the hardware, the kernel, and userspace. + +KUnit isn't working, what should I do? +====================================== + +Unfortunately, there are a number of things which can break, but here are some +things to try. + +1. Try running ``./tools/testing/kunit/kunit.py run`` with the ``--raw_output`` + parameter. This might show details or error messages hidden by the kunit_tool + parser. +2. Instead of running ``kunit.py run``, try running ``kunit.py config``, + ``kunit.py build``, and ``kunit.py exec`` independently. This can help track + down where an issue is occurring. (If you think the parser is at fault, you + can run it manually against stdin or a file with ``kunit.py parse``.) +3. Running the UML kernel directly can often reveal issues or error messages + kunit_tool ignores. This should be as simple as running ``./vmlinux`` after + building the UML kernel (e.g., by using ``kunit.py build``). Note that UML + has some unusual requirements (such as the host having a tmpfs filesystem + mounted), and has had issues in the past when built statically and the host + has KASLR enabled. (On older host kernels, you may need to run ``setarch + `uname -m` -R ./vmlinux`` to disable KASLR.) +4. Make sure the kernel .config has ``CONFIG_KUNIT=y`` and at least one test + (e.g. ``CONFIG_KUNIT_EXAMPLE_TEST=y``). kunit_tool will keep its .config + around, so you can see what config was used after running ``kunit.py run``. + It also preserves any config changes you might make, so you can + enable/disable things with ``make ARCH=um menuconfig`` or similar, and then + re-run kunit_tool. +5. Try to run ``make ARCH=um defconfig`` before running ``kunit.py run``. This + may help clean up any residual config items which could be causing problems. +6. Finally, try running KUnit outside UML. KUnit and KUnit tests can run be + built into any kernel, or can be built as a module and loaded at runtime. + Doing so should allow you to determine if UML is causing the issue you're + seeing. When tests are built-in, they will execute when the kernel boots, and + modules will automatically execute associated tests when loaded. Test results + can be collected from ``/sys/kernel/debug/kunit//results``, and + can be parsed with ``kunit.py parse``. For more details, see "KUnit on + non-UML architectures" in :doc:`usage`. + +If none of the above tricks help, you are always welcome to email any issues to +kunit-dev@googlegroups.com.