From 46f005d636ee3d5ad52c771065c9eb97aa743f10 Mon Sep 17 00:00:00 2001 From: michaelh Date: Sat, 5 May 2001 21:24:31 +0000 Subject: [PATCH] Imported the regression suite git-svn-id: https://sdcc.svn.sourceforge.net/svnroot/sdcc/trunk/sdcc@777 4a8a32a2-be11-0410-ad9d-d568d2c75423 --- support/regression/Makefile | 84 ++++++++++ support/regression/collate-results.py | 18 +++ support/regression/fwk/include/testfwk.h | 21 +++ support/regression/fwk/lib/testfwk.c | 80 ++++++++++ support/regression/generate-cases.py | 185 +++++++++++++++++++++++ support/regression/ports/host/spec.mk | 17 +++ support/regression/ports/host/support.c | 8 + support/regression/ports/z80/spec.mk | 39 +++++ support/regression/ports/z80/support.asm | 40 +++++ support/regression/tests/increment.c | 15 ++ support/regression/tests/muldiv.c | 40 +++++ 11 files changed, 547 insertions(+) create mode 100644 support/regression/Makefile create mode 100644 support/regression/collate-results.py create mode 100644 support/regression/fwk/include/testfwk.h create mode 100644 support/regression/fwk/lib/testfwk.c create mode 100644 support/regression/generate-cases.py create mode 100644 support/regression/ports/host/spec.mk create mode 100644 support/regression/ports/host/support.c create mode 100644 support/regression/ports/z80/spec.mk create mode 100644 support/regression/ports/z80/support.asm create mode 100644 support/regression/tests/increment.c create mode 100644 support/regression/tests/muldiv.c diff --git a/support/regression/Makefile b/support/regression/Makefile new file mode 100644 index 00000000..11c5de43 --- /dev/null +++ b/support/regression/Makefile @@ -0,0 +1,84 @@ +# Starting at the bottom +# Set of source test suites +# Each source suite is processesd producing multiple device specific test suites. +# Each device specific test suite is compiled. +# Each device specific test suite is run, and the output recorded. +# The output from each device specific test suite derrived from a source +# test suite are collated. + +.SILENT: + +CASES_DIR = cases +RESULTS_DIR = results +TESTS_DIR = tests +PORTS_DIR = ports +SUBRESULTS_DIR = subresults + +GENERATE_CASES = generate-cases.py + +ALL_PORTS = $(notdir $(wildcard $(PORTS_DIR)/*)) + +test-ports: + for i in $(ALL_PORTS); do $(MAKE) inter-port-clean test-port PORT=$$i; done + +ALL_TESTS = $(shell find $(TESTS_DIR) -name "*.c") + +PORT_CASES_DIR = $(CASES_DIR)/$(PORT) +PORT_RESULTS_DIR = $(RESULTS_DIR)/$(PORT) +PORT_SUBRESULTS_DIR = $(SUBRESULTS_DIR)/$(PORT) +PORT_RESULTS = $(ALL_TESTS:$(TESTS_DIR)/%.c=$(PORT_RESULTS_DIR)/%.out) + +# Defaults +SDCC = ../../bin/sdcc +SDCCFLAGS = -m$(PORT) +OBJEXT = .o +EXEEXT = .bin +DIREXT = + +ifdef PORT +include $(PORTS_DIR)/$(PORT)/spec.mk +endif + +.PRECIOUS: $(PORT_CASES_DIR)/% %$(OBJEXT) %$(EXEEXT) %.dir + +SDCCFLAGS += -Ifwk/include + +$(PORT_CASES_DIR)/%$(DIREXT): $(TESTS_DIR)/%.c $(GENERATE_CASES) + rm -rf $(CASES_DIR)/tests + mkdir -p $(CASES_DIR)/tests + mkdir -p $@ + python $(GENERATE_CASES) $< > /dev/null + cp $(CASES_DIR)/tests/*.c $@ + touch $@ + +$(PORT_RESULTS_DIR)/%.out: $(PORT_CASES_DIR)/%$(DIREXT) + $(MAKE) iterations PORT=$(PORT) CASES=$< + +port-results: port-dirs $(PORT_RESULTS) + echo Summary for \'$(PORT)\': `cat $(PORT_RESULTS) | python collate-results.py` + +port-dirs: + mkdir -p $(PORT_CASES_DIR) $(PORT_RESULTS_DIR) $(PORT_SUBRESULTS_DIR) + +test-port: port-results + +SUB_CASES = $(wildcard $(CASES)/*.c) +SUB_RESULTS = $(SUB_CASES:$(PORT_CASES_DIR)/%.c=$(PORT_SUBRESULTS_DIR)/%.out) +RESULTS = $(CASES:$(CASES_DIR)/%$(DIREXT)=$(RESULTS_DIR)/%.out) + +iterations: $(RESULTS) + +$(RESULTS): $(SUB_RESULTS) + cat $(SUB_RESULTS) > $@ + +#$(PORT_CASES_DIR)/%.bin: $(PORT_CASES_DIR)/%.c + +#$(PORT_CASES_DIR)/%.o: $(PORT_CASES_DIR)/%.c + +clean: + rm -rf $(CASES_DIR) $(RESULTS_DIR) $(SUBRESULTS_DIR) + +inter-port-clean: + rm -f fwk/lib/*.o + + diff --git a/support/regression/collate-results.py b/support/regression/collate-results.py new file mode 100644 index 00000000..ab5b03b3 --- /dev/null +++ b/support/regression/collate-results.py @@ -0,0 +1,18 @@ +import sys, re +import string + +lines = sys.stdin.readlines() + +failures = 0 +cases = 0 +tests = 0 + +for line in lines: + if (re.search(r'^--- Summary:', line)): + (summary, data, rest) = re.split(r':', line) + (nfailures, ntests, ncases) = re.split(r'/', data) + failures = failures + string.atof(nfailures) + tests = tests + string.atof(ntests) + cases = cases + string.atof(ncases) + +print "%.0f failues, %.0f tests, %.0f test cases" % (failures, tests, cases) diff --git a/support/regression/fwk/include/testfwk.h b/support/regression/fwk/include/testfwk.h new file mode 100644 index 00000000..2e3a721f --- /dev/null +++ b/support/regression/fwk/include/testfwk.h @@ -0,0 +1,21 @@ +#ifndef __TESTFWK_H +#define __TESTFWK_H 1 + +extern int __numTests; + +void __fail(const char *szMsg, const char *szCond, const char *szFile, int line); + +#define ASSERT(_a) (__numTests++, (_a) ? (void)0 : __fail("Assertion failed", #_a, __FILE__, __LINE__)) + +typedef void TESTFUN(void); + +// Provided by the suite +void ** +suite(void); + +const char * +getSuiteName(void); + +#define NULL 0 + +#endif diff --git a/support/regression/fwk/lib/testfwk.c b/support/regression/fwk/lib/testfwk.c new file mode 100644 index 00000000..4ebace85 --- /dev/null +++ b/support/regression/fwk/lib/testfwk.c @@ -0,0 +1,80 @@ +/** Test framework support functions. + */ +#include +#include + +//#include + +void _putchar(char c); + +static void _printn(int n) { + // PENDING + _putchar('0' + n); +} + +static void _printf(const char *szFormat, ...) +{ + va_list ap; + va_start(ap, szFormat); + + while (*szFormat) { + if (*szFormat == '%') { + switch (*++szFormat) { + case 's': { + const char *sz = va_arg(ap, const char *); + while (*sz) { + _putchar(*sz++); + } + break; + } + case 'u': { + int i = va_arg(ap, int); + _printn(i); + break; + } + default: + break; + } + } + else { + _putchar(*szFormat); + } + szFormat++; + } + va_end(ap); +} + +int __numTests; +int __numFailures; + +void +__fail(const char *szMsg, const char *szCond, const char *szFile, int line) +{ + _printf("--- FAIL: \"%s\" on %s at %s:%u\n", szMsg, szCond, szFile, line); + __numFailures++; +} + +int +main(void) +{ + TESTFUN **cases; + int numCases = 0; + + _printf("--- Running: %s\n", getSuiteName()); + + cases = (TESTFUN **)suite(); + + while (*cases) { + _printf("Running %u\n", numCases); + (*cases)(); + cases++; + numCases++; + } + + _printf("--- Summary: %u/%u/%u: %u failed of %u tests in %u cases.\n", + __numFailures, __numTests, numCases, + __numFailures, __numTests, numCases + ); + + return __numFailures; +} diff --git a/support/regression/generate-cases.py b/support/regression/generate-cases.py new file mode 100644 index 00000000..350e8088 --- /dev/null +++ b/support/regression/generate-cases.py @@ -0,0 +1,185 @@ +from HTMLgen import TemplateDocument +import sys, re, tempfile, os + +# Globals +# Directory that the generated files should be placed into +outdir = 'cases' + +# Start of the test function table definition +testfuntableheader = """ +static void (*const _tests[])(void) = { +""" + + +# End of the test function table definition +testfuntablefooter = """\tNULL +}; +""" + +# Code to generate the suite function +testfunsuite = """ +void ** +suite(void) +{ + return (void **)_tests; +} + +const char * +getSuiteName(void) +{ + return "{testcase}"; +} +""" + +# Utility functions +def trim(a): + """Removes all white space from the start and the end of a string. + Like java.lang.String.trim""" + ret = chomp(re.sub(r'^\s+', '', a)) + return ret + +def chomp(a): + """Removes all white space from the end of a string. + Like perl's chomp""" + return re.sub(r'\s+$', '', a) + +def createdir(path): + """Creates a directory if it doesn't exist""" + if not os.path.isdir(path): + os.mkdir(path) + +class InstanceGenerator: + """Test case iteration generator. + Takes the template given as the first argument, pulls out all the meta + iteration information, and generates an instance for each combination + of the names and types. + + See doc/test_suite_spec.tex for more information on the template file + format.""" + + def __init__(self, inname): + self.inname = inname + # Initalise the replacements hash. + # Map of name to values. + self.replacements = { } + # Initalise the function list hash. + self.functions = [] + # Emit the suite wrapper into a temporary file + self.tmpname = tempfile.mktemp() + (self.basename, self.ext) = re.split(r'\.', self.inname) + self.ext = '.' + self.ext + + def permute(self, basepath, keys, trans = {}): + """Permutes across all of the names. For each value, recursivly creates + a mangled form of the name, this value, and all the combinations of + the remaining values. At the tail of the recursion when one full + combination is built, generates an instance of the test case from + the template.""" + if len(keys) == 0: + # End of the recursion. + # Set the runtime substitutions. + trans['testcase'] = basepath + # Create the instance from the template + T = TemplateDocument(self.tmpname) + T.substitutions = trans + T.write(basepath + self.ext) + else: + # Pull off this key, then recursivly iterate through the rest. + key = keys[0] + for part in self.replacements[key]: + trans[key] = part + # Turn a empty string into something decent for a filename + if not part: + part = 'none' + # Remove any bad characters from the filename. + part = re.sub(r'\s+', r'_', part) + # The slice operator (keys[1:]) creates a copy of the list missing the + # first element. + # Can't use '-' as a seperator due to the mcs51 assembler. + self.permute(basepath + '_' + key + '_' + part, keys[1:], trans) + + def writetemplate(self): + """Given a template file and a temporary name writes out a verbatim copy + of the source file and adds the suite table and functions.""" + fout = open(self.tmpname, 'w') + + for line in self.lines: + fout.write(line) + + # Emmit the suite table + fout.write(testfuntableheader) + + for fun in self.functions: + # Turn the function definition into a pointer + fun = re.sub(r'\(\w+\)', '', fun) + fout.write("\t" + fun + ",\n") + + fout.write(testfuntablefooter) + fout.write(testfunsuite); + + fout.close() + + def readfile(self): + """Read in all of the input file.""" + fin = open(self.inname) + self.lines = fin.readlines() + fin.close() + + def parse(self): + # Start off in the header. + inheader = 1; + + # Iterate over the source file and pull out the meta data. + for line in self.lines: + line = trim(line) + + # If we are still in the header, see if this is a substitution line + if inheader: + # A substitution line has a ':' in it + if re.search(r':', line) != None: + # Split out the name from the values + (name, rawvalues) = re.split(r':', line) + # Split the values at the commas + values = re.split(r',', rawvalues) + + # Trim the name + name = trim(name) + # Trim all the values + values = map(trim, values) + + self.replacements[name] = values + elif re.search(r'\*/', line) != None: + # Hit the end of the comments + inheader = 0; + else: + # Do nothing. + None + else: + # Pull out any test function names + if re.search(r'^test\w+\(\w+\)', line) != None: + self.functions.append(line) + + def generate(self): + """Main function. Generates all of the instances.""" + self.readfile() + self.parse() + self.writetemplate() + + # Create the output directory if it doesn't exist + createdir(outdir) + + # Generate + self.permute(os.path.join(outdir, self.basename), self.replacements.keys()) + + # Remove the temporary file + os.remove(self.tmpname) + +# Check and parse the command line arguments +if len(sys.argv) < 2: + # PENDING: How to throw an error? + print "usage: generate-cases.py template.c" + +# Input name is the first arg. + +s = InstanceGenerator(sys.argv[1]) +s.generate() diff --git a/support/regression/ports/host/spec.mk b/support/regression/ports/host/spec.mk new file mode 100644 index 00000000..dc2449c6 --- /dev/null +++ b/support/regression/ports/host/spec.mk @@ -0,0 +1,17 @@ +SDCC = gcc +SDCCFLAGS = -Wall + +EXEEXT = .bin + +EXTRAS = fwk/lib/testfwk$(OBJEXT) ports/$(PORT)/support$(OBJEXT) + +$(PORT_SUBRESULTS_DIR)/%.out: $(PORT_CASES_DIR)/%$(EXEEXT) + mkdir -p `dirname $@` + -$< > $@ + if grep -q FAIL $@; then echo FAILURES in $@; fi + +%$(EXEEXT): %$(OBJEXT) $(EXTRAS) + $(SDCC) $(SDCCFLAGS) -o $@ $< $(EXTRAS) + +%$(OBJEXT): %.c fwk/include/*.h + $(SDCC) $(SDCCFLAGS) -c $< -o $@ diff --git a/support/regression/ports/host/support.c b/support/regression/ports/host/support.c new file mode 100644 index 00000000..756a6235 --- /dev/null +++ b/support/regression/ports/host/support.c @@ -0,0 +1,8 @@ +/** Host specific support routines. + */ +#include + +void _putchar(char c) +{ + putchar(c); +} diff --git a/support/regression/ports/z80/spec.mk b/support/regression/ports/z80/spec.mk new file mode 100644 index 00000000..78e6f699 --- /dev/null +++ b/support/regression/ports/z80/spec.mk @@ -0,0 +1,39 @@ +SDCCFLAGS += -I/home/michaelh/projects/gbdk-lib/include + +EXEEXT = .bin + +EXTRAS = fwk/lib/testfwk$(OBJEXT) ports/$(PORT)/support$(OBJEXT) \ + /home/michaelh/projects/gbdk-lib/libc/asm/z80/mul$(OBJEXT) \ + /home/michaelh/projects/gbdk-lib/libc/asm/z80/div$(OBJEXT) + +%$(EXEEXT): %.ihx + ../makebin/makebin -s 32768 < $< > $@ + +%.ihx: %$(OBJEXT) $(EXTRAS) + ../../bin/link-z80 -n -- -b_CODE=0x200 -b_DATA=0x8000 -i $@ $< $(EXTRAS) + +%$(OBJEXT): %.c fwk/include/*.h + $(SDCC) $(SDCCFLAGS) -c $< + +%$(OBJEXT): %.asm + ../../bin/as-z80 -plosgff $@ $< + +%$(OBJEXT): %.s + ../../bin/as-z80 -plosgff $@ $< + +$(PORT_SUBRESULTS_DIR)/%.out: $(PORT_CASES_DIR)/%$(EXEEXT) + mkdir -p `dirname $@` + java -cp /home/michaelh/projects/rose ConsoleZ80 $< > $@ + if grep -q FAIL $@; then echo FAILURES in $@; fi + +#$(PORT_SUBRESULTS_DIR)/%.out: $(PORT_CASES_DIR)/%$(EXEEXT) +# mkdir -p `dirname $@` +# -$< > $@ +# if grep -q FAIL $@; then echo FAILURES in $@; fi + + +#%$(EXEEXT): %$(OBJEXT) fwk/lib/testfwk$(OBJEXT) +# $(SDCC) $(SDCCFLAGS) -o $@ $< fwk/lib/testfwk$(OBJEXT) + +#%$(OBJEXT): %.c fwk/include/*.h +# $(SDCC) $(SDCCFLAGS) -c $< -o $@ diff --git a/support/regression/ports/z80/support.asm b/support/regression/ports/z80/support.asm new file mode 100644 index 00000000..6ba7686a --- /dev/null +++ b/support/regression/ports/z80/support.asm @@ -0,0 +1,40 @@ + ;; **************************************** + ;; Beginning of module + .title "Test runtime" + .module Runtime + + .globl _main + .STACK = 0xE000 + + .area _INIT (ABS) + .org 0x0 + jp 0x100 + + .org 0x100 +__init:: + ;; Beginning of the code + DI ; Disable interrupts + LD SP,#.STACK + ;; Call the main function + CALL _main + ld a, #0 + out (1), a + +__putchar:: + ld a,l + out (0xff),a + ret + + .org 0x200 + .area _HOME + .area _CODE + .area _OVERLAY + .area _ISEG + .area _BSEG + .area _XSEG + .area _GSINIT + .area _GSFINAL + .area _GSINIT + .area _CODE + + .area _DATA diff --git a/support/regression/tests/increment.c b/support/regression/tests/increment.c new file mode 100644 index 00000000..ff0f5ce3 --- /dev/null +++ b/support/regression/tests/increment.c @@ -0,0 +1,15 @@ +/** Simple test for increment + + type: signed char, int, long + storage: static, +*/ +#include + +static void +testIncrement(void) +{ + volatile {storage} {type} i; + i = 0; + i--; + ASSERT(i == -1); +} diff --git a/support/regression/tests/muldiv.c b/support/regression/tests/muldiv.c new file mode 100644 index 00000000..cb574213 --- /dev/null +++ b/support/regression/tests/muldiv.c @@ -0,0 +1,40 @@ +/** Simple test for increment + + type: int + storage: , +*/ +#include + +static void +testMul(void) +{ +#if SDCC +#else + volatile {storage} {type} i; + + i = 5; + ASSERT(i*5 == 25); + ASSERT(i*-4 == -20); + + i = -10; + ASSERT(i*12 == -120); + ASSERT(i*-3 == 30); +#endif +} + +static void +testDiv(void) +{ +#if SDCC +#else + volatile {storage} {type} i; + + i = 100; + ASSERT(i/5 == 20); + ASSERT(i/-4 == -25); + + i = -50; + ASSERT(i/25 == -2); + ASSERT(i/-12 == 4); +#endif +} -- 2.30.2