[syslinux:firmware] tests: Regression testsuite
syslinux-bot for Matt Fleming
matt.fleming at intel.com
Tue Jul 23 07:18:06 PDT 2013
Commit-ID: 21f8596c58a9791e7ea1f4ba93be17eed60a05b6
Gitweb: http://www.syslinux.org/commit/21f8596c58a9791e7ea1f4ba93be17eed60a05b6
Author: Matt Fleming <matt.fleming at intel.com>
AuthorDate: Tue, 9 Jul 2013 13:09:31 +0100
Committer: Matt Fleming <matt.fleming at intel.com>
CommitDate: Tue, 23 Jul 2013 14:58:41 +0100
tests: Regression testsuite
A number of Syslinux releases have contained regressions when compared
with previous versions. Now that the size of the derivative grid has
exploded (PXELINUX, SYSLINUX,.... bios, efi32, efi64) we need to have
some means of ensuring we don't introduce regressions so easily.
Start building a suite of regression tests. Regression tests should test
end-to-end functionality, e.g. loading a Linux kernel. If a test fails,
that failure represents a regression in some behaviour, e.g. Syslinux
behaves differently than it previously did.
The idea is that when bugs are reported in Syslinux, we can create new
tests that reproduce the buggy behaviour. Once a fix has been committed
the new test should pass. From that point forward, the test provides a
means of ensuring we never reintroduce that bug.
The collection of tests illustrates the agreed upon behaviour of
Syslinux.
To execute the regression testsuite type,
make regression
Sample output might look like,
Executing regression tests for SYSLINUX
Running Linux kernel regression tests...
[+] empty passed
[+] kernelhello passed
Running COM32 module regression tests...
[+] chaindisk passed
Executing regression tests for PXELINUX
Running Linux kernel regression tests...
[!] empty failed
[+] kernelhello passed
[+] pxetest passed
Qemu is used to execute the regression tests in a virtual environment.
Because it takes time to load Qemu, we can forgive the regression
testsuite for not executing instantaneously. However, developers should
keep in mind the idea of "failing fast" in their regression tests to
minimize execution time - that is, if a test is going to fail, make it
fail as quickly as possible to reduce the testrun time.
It's possible to have derivative-specific tests, where the test only
runs for a specific Syslinux boot loader. For an example see pxetest in
tests/linux/Makefile.
Currently the regression testsuite requires root privileges to execute
and makes several calls to sudo(8). In future it would be nice to get
rid of this requirement.
Signed-off-by: Matt Fleming <matt.fleming at intel.com>
---
Makefile | 11 ++-
tests/Makefile | 79 +++++++++++++++++++++
tests/README | 15 ++++
tests/build-pxelinux | 51 +++++++++++++
tests/build-syslinux | 70 ++++++++++++++++++
tests/com32/Makefile | 29 ++++++++
tests/com32/chaindisk.cfg | 7 ++
tests/com32/chaindisk.results | 1 +
tests/linux/Makefile | 45 ++++++++++++
.../libpm.S => tests/linux/empty-vmlinuz | 0
tests/linux/empty.cfg | 3 +
tests/linux/empty.results | 2 +
tests/linux/kernelhello-vmlinuz | Bin 0 -> 6843584 bytes
tests/linux/kernelhello.cfg | 5 ++
tests/linux/kernelhello.results | 1 +
tests/linux/pxetest.cfg | 12 ++++
tests/linux/pxetest.results | 3 +
tests/recipes.mk | 71 ++++++++++++++++++
tests/test.cfg | 6 ++
19 files changed, 409 insertions(+), 2 deletions(-)
diff --git a/Makefile b/Makefile
index a9b515c..d90635a 100644
--- a/Makefile
+++ b/Makefile
@@ -72,7 +72,7 @@ include $(MAKEDIR)/syslinux.mk
-include $(OBJDIR)/version.mk
private-targets = prerel unprerel official release burn isolinux.iso \
- preupload upload
+ preupload upload test
ifeq ($(MAKECMDGOALS),)
MAKECMDGOALS += all
@@ -101,6 +101,13 @@ $(filter-out $(private-targets), $(MAKECMDGOALS)):
$(MAKE) -C $(OBJDIR) -f $(CURDIR)/Makefile SRC="$(topdir)" \
OBJ=$(OBJDIR) objdir=$(OBJDIR) $(MAKECMDGOALS)
+test:
+ printf "Executing unit tests\n"
+ $(MAKE) -C com32/lib/syslinux/tests all
+ $(MAKE) -C tests SRC="$(topdir)/tests" OBJ="$(topdir)/tests" \
+ objdir=$(OBJDIR) \
+ -f $(topdir)/tests/Makefile all
+
# Hook to add private Makefile targets for the maintainer.
-include $(topdir)/Makefile.private
@@ -206,7 +213,7 @@ NETINSTALLABLE = core/pxelinux.0 gpxe/gpxelinux.0 core/lpxelinux.0 \
endif # ifdef EFI_BUILD
-.PHONY: subdirs $(BSUBDIRS) $(ISUBDIRS)
+.PHONY: subdirs $(BSUBDIRS) $(ISUBDIRS) test
ifeq ($(HAVE_FIRMWARE),)
diff --git a/tests/Makefile b/tests/Makefile
new file mode 100644
index 0000000..99b1618
--- /dev/null
+++ b/tests/Makefile
@@ -0,0 +1,79 @@
+#
+# Copyright (C) 2013 Intel Corporation; author Matt Fleming
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+# Boston MA 02111-1307, USA; either version 2 of the License, or
+# (at your option) any later version; incorporated herein by reference.
+#
+
+#
+# See linux/Makefile for an example Makefile. When tests are invoked the
+# location of the Syslinux files, as used by qemu, is in $(MOUNTDIR).
+# Tests that copy files into $(MOUNTDIR) must be sure to delete them once
+# the test has completed, otherwise the qemu harddisk image will become
+# full.
+#
+
+ifeq ($(INITIAL),)
+
+INITIAL := 1
+
+HDD = $(OBJ)/hdd.img
+
+export QEMU QEMU_FLAGS INITIAL HDD
+
+SYSLINUX_output-str := $(shell sudo $(SRC)/build-syslinux start \
+ $(objdir)/bios $(HDD) test.cfg)
+SYSLINUX_MOUNT_DIR = $(firstword $(SYSLINUX_output-str))
+SYSLINUX_INSTALL_DIR = $(SYSLINUX_MOUNT_DIR)/boot/syslinux
+SYSLINUX_CONFIG_FILE = $(SYSLINUX_INSTALL_DIR)/syslinux.cfg
+SYSLINUX_LOOPDEV =$(lastword $(SYSLINUX_output-str))
+
+PXELINUX_output-str := $(shell $(SRC)/build-pxelinux start $(topdir) test.cfg)
+PXELINUX_INSTALL_DIR = $(firstword $(PXELINUX_output-str))/tftpboot
+PXELINUX_CONFIG_FILE = $(PXELINUX_INSTALL_DIR)/pxelinux.cfg/default
+
+QEMU = qemu-system-i386
+
+all: syslinux-tests pxelinux-tests
+syslinux-tests:
+ $(MAKE) SRC="$(SRC)" OBJ="$(OBJ)" objdir="$(objdir)" \
+ INITIAL="$(INITIAL)" INSTALL_DIR="$(SYSLINUX_INSTALL_DIR)" \
+ CONFIG_FILE="$(SYSLINUX_CONFIG_FILE)" \
+ DERIVATIVE=SYSLINUX
+ @sudo $(SRC)/build-syslinux stop $(SYSLINUX_MOUNT_DIR) \
+ $(SYSLINUX_LOOPDEV) test.cfg
+
+pxelinux-tests:
+ $(MAKE) SRC="$(SRC)" OBJ="$(OBJ)" objdir="$(objdir)" \
+ INITIAL="$(INITIAL)" INSTALL_DIR="$(PXELINUX_INSTALL_DIR)" \
+ CONFIG_FILE="$(PXELINUX_CONFIG_FILE)" \
+ DERIVATIVE=PXELINUX
+ $(SRC)/build-pxelinux stop $(PXELINUX_INSTALL_DIR) test.cfg
+else # ifeq ($(INITIAL),)
+
+export DERIVATIVE
+ifeq ($(DERIVATIVE),SYSLINUX)
+QEMU_FLAGS += -hda $(HDD) -no-reboot
+else
+QEMU_FLAGS = -boot n -net nic,model=virtio -net user,tftp="$(INSTALL_DIR)",bootfile=pxelinux.0 -no-reboot
+endif
+
+all: banner subdirs
+
+banner:
+ printf "Executing regression tests for $(DERIVATIVE)\n"
+
+SUBDIRS = linux com32
+.PHONY: subdirs $(SUBDIRS) fini
+subdirs: $(SUBDIRS)
+$(SUBDIRS):
+ mkdir -p $(OBJ)/$@
+ $(MAKE) -C $(OBJ)/$@ SRC="$(SRC)"/$@ OBJ="$(OBJ)"/$@/ \
+ -f $(SRC)/$@/Makefile objdir="$(objdir)/tests" \
+ INSTALL_DIR="$(INSTALL_DIR)" CONFIG_FILE="$(CONFIG_FILE)" \
+ DERIVATIVE="$(DERIVATIVE)"
+
+endif # ifeq ($(INITIAL),)
diff --git a/tests/README b/tests/README
new file mode 100644
index 0000000..44575bc
--- /dev/null
+++ b/tests/README
@@ -0,0 +1,15 @@
+Syslinux Regression Tests
+
+This directory contains acceptance or regression tests. Each test is
+intended to exercise a specific execution path through Syslinux, from
+user input to output. Unit tests are located in a module directory, e.g.
+com32/lib/syslinux/tests/, and test a specific "unit" of code such as a
+function or interface.
+
+The regression tests are intended to be run on the development
+environment. As such they must remain fast.
+
+Qemu must be installed and we grab OVMF from a predetermined location.
+
+If you're thinking of rewriting or refactoring a subsystem in a major
+way, please ensure there is a suitable test, and if not, write one.
diff --git a/tests/build-pxelinux b/tests/build-pxelinux
new file mode 100755
index 0000000..50c9de8
--- /dev/null
+++ b/tests/build-pxelinux
@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# Copyright (C) 2013 Intel Corporation; author Matt Fleming
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+# Boston MA 02111-1307, USA; either version 2 of the License, or
+# (at your option) any later version; incorporated herein by reference.
+#
+
+usage()
+{
+ echo "Usage: " $0 "[start|stop] <srcdir> <config>" > /dev/stderr
+}
+
+if [ $# -lt 3 ]; then
+ usage
+ exit 1
+fi
+
+start_func()
+{
+ tmpdir=`mktemp -d || exit 1`
+ echo $tmpdir
+
+ mkdir -p $tmpdir/tftpboot/pxelinux.cfg
+ cp $2 $tmpdir/tftpboot/pxelinux.cfg/default
+
+ # This doesn't work if we built with O=/tmp/foobar
+ #make -p -s -C $1 bios netinstall INSTALLROOT=$tmpdir > /dev/stderr
+ for f in `find $1/bios -name "*.c32" -o -name "*.0"`; do
+ cp $f $tmpdir/tftpboot/
+ done
+}
+
+stop_func()
+{
+ rm -fr $1
+}
+
+case "$1" in
+ start)
+ shift
+ start_func $*
+ ;;
+ stop)
+ shift
+ stop_func $*
+ ;;
+esac
diff --git a/tests/build-syslinux b/tests/build-syslinux
new file mode 100755
index 0000000..a8972c0
--- /dev/null
+++ b/tests/build-syslinux
@@ -0,0 +1,70 @@
+#!/bin/bash
+#
+# Copyright (C) 2013 Intel Corporation; author Matt Fleming
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+# Boston MA 02111-1307, USA; either version 2 of the License, or
+# (at your option) any later version; incorporated herein by reference.
+#
+
+usage()
+{
+ echo "Usage: " $0 "[start|stop] <objdir> <outputfile> <config>" > /dev/stderr
+}
+
+if [ $# -lt 4 ]; then
+ usage
+ exit 1
+fi
+
+if [ `id -u` -ne 0 ]; then
+ echo $0 "must be invoked as root" > /dev/stderr
+ exit 1
+fi
+
+start_func()
+{
+ dd if=/dev/zero of=$2 bs=512 count=$(((32 * 1024 * 1024)/512)) > /dev/null 2>&1
+
+ tmpdir=`mktemp -d || exit 1`
+
+ next=`losetup -f`
+ losetup $next $2
+
+ mkfs.vfat $next > /dev/null 2>&1
+
+ mount $next $tmpdir
+
+ mkdir -p $tmpdir/boot/syslinux
+ cp $3 $tmpdir/boot/syslinux/syslinux.cfg
+
+ for f in `find $1 -name "*.c32"`; do
+ cp $f $tmpdir/boot/syslinux
+ done
+
+ $1/linux/syslinux --directory /boot/syslinux --install $next > /dev/stderr
+ sync
+
+ echo $tmpdir $next
+}
+
+stop_func()
+{
+ umount $1
+ rm -fr $1
+
+ losetup -d $2
+}
+
+case "$1" in
+ start)
+ shift
+ start_func $*
+ ;;
+ stop)
+ shift
+ stop_func $*
+ ;;
+esac
diff --git a/tests/com32/Makefile b/tests/com32/Makefile
new file mode 100644
index 0000000..5a9a95b
--- /dev/null
+++ b/tests/com32/Makefile
@@ -0,0 +1,29 @@
+#
+# Copyright (C) 2013 Intel Corporation; author Matt Fleming
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+# Boston MA 02111-1307, USA; either version 2 of the License, or
+# (at your option) any later version; incorporated herein by reference.
+#
+
+include $(objdir)/recipes.mk
+
+all: tests
+
+ifeq ($(DERIVATIVE),SYSLINUX)
+chaindisk_cfg = chaindisk.cfg
+chaindisk_files = $(chaindisk_cfg)
+chaindisk_results = chaindisk.results
+chaindisk:
+ $(run-test)
+
+tests: banner chaindisk
+else
+
+tests:
+endif # ifeq ($(DERIVATIVE),SYSLINUX)
+
+banner:
+ printf " Running COM32 module regression tests...\n"
diff --git a/tests/com32/chaindisk.cfg b/tests/com32/chaindisk.cfg
new file mode 100644
index 0000000..037c3ec
--- /dev/null
+++ b/tests/com32/chaindisk.cfg
@@ -0,0 +1,7 @@
+#
+# Chainload ourselves from the boot drive
+#
+DEFAULT -
+LABEL -
+ COM32 chain.c32
+ APPEND boot fs nohand break
diff --git a/tests/com32/chaindisk.results b/tests/com32/chaindisk.results
new file mode 100644
index 0000000..f081e7f
--- /dev/null
+++ b/tests/com32/chaindisk.results
@@ -0,0 +1 @@
+Service-only run completed, exiting.
diff --git a/tests/linux/Makefile b/tests/linux/Makefile
new file mode 100644
index 0000000..bfa3ca7
--- /dev/null
+++ b/tests/linux/Makefile
@@ -0,0 +1,45 @@
+#
+# Copyright (C) 2013 Intel Corporation; author Matt Fleming
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+# Boston MA 02111-1307, USA; either version 2 of the License, or
+# (at your option) any later version; incorporated herein by reference.
+#
+# Regression tests for the Linux kernel loader code paths
+#
+
+include $(objdir)/recipes.mk
+
+ifeq ($(DERIVATIVE),PXELINUX)
+derivative-tests = pxetest
+
+pxetest_cfg = pxetest.cfg
+pxetest_files = $(pxetest_cfg) kernelhello-vmlinuz
+pxetest_results = pxetest.results
+
+endif
+
+all: tests
+
+empty_cfg = empty.cfg
+empty_files = $(empty_cfg) empty-vmlinuz
+empty_results = empty.results
+
+kernelhello_cfg = kernelhello.cfg
+kernelhello_files = $(kernelhello_cfg) kernelhello-vmlinuz
+kernelhello_results = kernelhello.results
+
+STANDARD_TESTS = kernelhello pxetest
+$(STANDARD_TESTS):
+ $(run-test)
+
+empty:
+ touch empty-vmlinuz
+ $(run-test)
+
+tests: banner empty kernelhello $(derivative-tests)
+
+banner:
+ printf " Running Linux kernel regression tests...\n"
diff --git a/gpxe/src/arch/i386/transitions/libpm.S b/tests/linux/empty-vmlinuz
similarity index 100%
copy from gpxe/src/arch/i386/transitions/libpm.S
copy to tests/linux/empty-vmlinuz
diff --git a/tests/linux/empty.cfg b/tests/linux/empty.cfg
new file mode 100644
index 0000000..12cca7c
--- /dev/null
+++ b/tests/linux/empty.cfg
@@ -0,0 +1,3 @@
+DEFAULT empty-linux
+LABEL empty-linux
+ KERNEL empty-vmlinuz
diff --git a/tests/linux/empty.results b/tests/linux/empty.results
new file mode 100644
index 0000000..fb512ec
--- /dev/null
+++ b/tests/linux/empty.results
@@ -0,0 +1,2 @@
+
+Loading empty-vmlinuz... failed: I/O error
diff --git a/tests/linux/kernelhello-vmlinuz b/tests/linux/kernelhello-vmlinuz
new file mode 100644
index 0000000..68ae7ef
Binary files /dev/null and b/tests/linux/kernelhello-vmlinuz differ
diff --git a/tests/linux/kernelhello.cfg b/tests/linux/kernelhello.cfg
new file mode 100644
index 0000000..3eb9e80
--- /dev/null
+++ b/tests/linux/kernelhello.cfg
@@ -0,0 +1,5 @@
+DEFAULT hello
+
+LABEL hello
+ KERNEL kernelhello-vmlinuz
+ APPEND console=ttyS0
diff --git a/tests/linux/kernelhello.results b/tests/linux/kernelhello.results
new file mode 100644
index 0000000..14c1f7d
--- /dev/null
+++ b/tests/linux/kernelhello.results
@@ -0,0 +1 @@
+Hello, World!
diff --git a/tests/linux/pxetest.cfg b/tests/linux/pxetest.cfg
new file mode 100644
index 0000000..db954eb
--- /dev/null
+++ b/tests/linux/pxetest.cfg
@@ -0,0 +1,12 @@
+DEFAULT dbg
+
+TIMEOUT 1
+ONTIMEOUT kernel
+
+LABEL dbg
+ COM32 debug.c32
+ APPEND -e pxelinux_scan_memory
+
+LABEL kernel
+ KERNEL kernelhello-vmlinuz
+ APPEND console=ttyS0 keeppxe
diff --git a/tests/linux/pxetest.results b/tests/linux/pxetest.results
new file mode 100644
index 0000000..f12bca6
--- /dev/null
+++ b/tests/linux/pxetest.results
@@ -0,0 +1,3 @@
+Hello, World!
+Marking PXE code region 0x9c630 - 0x9cee8 as SMT_TERMINAL
+Marking PXE data region 0x9cef0 - 0x9fbf8 as SMT_TERMINAL
diff --git a/tests/recipes.mk b/tests/recipes.mk
new file mode 100644
index 0000000..f01c1ad
--- /dev/null
+++ b/tests/recipes.mk
@@ -0,0 +1,71 @@
+#
+# Copyright (C) 2013 Intel Corporation; author Matt Fleming
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+# Boston MA 02111-1307, USA; either version 2 of the License, or
+# (at your option) any later version; incorporated herein by reference.
+#
+# Canned recipes
+
+#
+# copy-files - copy a config to the mounted filesystem
+#
+# Copies $(cfg) to the default Syslinux config location.
+# Usually this wants pairing with a use of $(remove-config).
+#
+# To use, your config filename must be the same as your target
+#
+define copy-files =
+ for f in $($@_files); do \
+ sudo cp $$f $(INSTALL_DIR) ;\
+ done
+ sync
+endef
+
+#
+# install-config
+#
+define install-config =
+ sudo sh -c 'echo INCLUDE $($@_cfg) >> $(CONFIG_FILE)'
+ sync
+endef
+
+#
+# remove-files - deletes files from the mounted filesystem
+#
+# Deletes $(mytest_files)
+#
+define remove-files =
+ for f in $($@_files); do \
+ sudo rm $(INSTALL_DIR)/$$f ;\
+ done
+endef
+
+#
+# delete-config - remove a test's config file from the master config
+#
+define delete-config =
+ sudo sed -i -e '/INCLUDE $($@_cfg)/d' $(CONFIG_FILE)
+endef
+
+#
+# run-test - begin executing the tests
+#
+define run-test =
+ $(copy-files)
+ $(install-config)
+
+ sudo $(QEMU) $(QEMU_FLAGS) -serial file:$@.log
+
+ $(delete-config)
+ $(remove-files)
+
+ sudo sort $@.log -o $@.log
+ if [ `comm -1 -3 $@.log $($@_results) | wc -l` -ne 0 ]; then \
+ printf " [!] $@ failed\n" ;\
+ else \
+ printf " [+] $@ passed\n" ;\
+ fi
+endef
diff --git a/tests/test.cfg b/tests/test.cfg
new file mode 100644
index 0000000..422603b
--- /dev/null
+++ b/tests/test.cfg
@@ -0,0 +1,6 @@
+#
+# The default config file
+SERIAL 0
+ONERROR reboot.c32
+TIMEOUT 1
+ONTIMEOUT reboot.c32
More information about the Syslinux-commits
mailing list