beginning of a test suite
[rxpd] / tests / test.sh
1 #!/bin/bash 
2 #  Copyright (C)         CinelerraCV
3 #    2007,               Christian Thaeter <ct@pipapo.org>
4 #                        Hermann Vosseler <Ichthyostega@web.de>
5 #
6 #  This program is free software; you can redistribute it and/or
7 #  modify it under the terms of the GNU General Public License as
8 #  published by the Free Software Foundation; either version 2 of the
9 #  License, or (at your option) any later version.
10 #
11 #  This program is distributed in the hope that it will be useful,
12 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
13 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14 #  GNU General Public License for more details.
15 #
16 #  You should have received a copy of the GNU General Public License
17 #  along with this program; if not, write to the Free Software
18 #  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19
20 # TESTMODE=FULL yet unimplemented
21 #   run all tests, PLANNED which fail count as error
22 #
23 # TESTMODE=FAST
24 #   run only tests which recently failed
25 #
26 # TESTMODE=FIRSTFAIL
27 #   stop testing on the first failure
28
29 export LC_ALL=C
30
31 arg0="$0"
32 srcdir=$(dirname "$arg0")
33
34 ulimit -S -t 1 -v 524288
35 valgrind=""
36
37 # gnu-pth & valgrind don't cooperate nicely
38
39 #if [ "$VALGRINDFLAGS" = 'DISABLE' ]; then
40 #    echo "valgrind explicit disabled"
41 #else
42 #    if [ "$(which valgrind)" ]; then
43 #        valgrind="$(which valgrind) --leak-check=yes --show-reachable=yes -q $VALGRINDFLAGS"
44 #       ulimit -S -t 10
45 #    else
46 #        echo "no valgrind found, go without it"
47 #    fi
48 #fi
49
50 echo
51 echo ================ ${0##*/} ================
52
53 TESTCNT=0
54 SKIPCNT=0
55 FAILCNT=0
56
57 # the old testlog if existing will be used to check for previous test states
58 if test -f ,testlog; then
59     mv ,testlog ,testlog.pre
60 else
61     touch ,testlog.pre
62 fi
63
64 date >,testlog
65
66 function TEST()
67 {
68         name="$1"
69         shift
70         rm -f ,send_stdin
71         rm -f ,expect_stdout
72         rm -f ,expect_stderr
73
74         while read -r line; do
75             cmd="${line%%:*}"
76             arg="${line#*: }"
77             expect_return=0
78
79             case $cmd in
80             'in')
81                 echo "$arg" >>,send_stdin
82                 ;;
83             'out')
84                 echo "$arg" >>,expect_stdout
85                 ;;
86             'err')
87                 echo "$arg" >>,expect_stderr
88                 ;;
89             'return')
90                 expect_return=$arg
91                 ;;
92             *)
93                 echo "UNKOWN TEST COMMAND '$cmd'" 1>&2
94                 exit
95                 ;;
96             esac
97         done
98         echo -n "TEST $name: "
99         echo -en "\nTEST $name: $* " >>,testlog
100
101         case $TESTMODE in
102         *FAST*)
103             if grep "^TEST $name: .* FAILED" ,testlog.pre >&/dev/null; then
104                 MSGOK=" (fixed)"
105                 MSGFAIL=" (still broken)"
106             elif grep "^TEST $name: .* \\(SKIPPED (ok)\\|OK\\)" ,testlog.pre >&/dev/null; then
107                 echo ".. SKIPPED (ok)"
108                 echo ".. SKIPPED (ok)" >>,testlog
109                 SKIPCNT=$(($SKIPCNT + 1))
110                 TESTCNT=$(($TESTCNT + 1))
111                 return
112             else
113                 MSGOK=" (new)"
114                 MSGFAIL=" (new)"
115             fi
116             ;;
117         *)
118             MSGOK=""
119             MSGFAIL=""
120             ;;
121         esac
122
123         TESTCNT=$(($TESTCNT + 1))
124
125         fails=0
126
127
128         if test -f ,send_stdin; then
129             cat ,send_stdin | $valgrind $TESTBIN "$@" 2>,stderr >,stdout
130         else
131             $valgrind $TESTBIN "$@" 2>,stderr >,stdout
132         fi &>/dev/null
133         return=$?
134
135         echo -n >,testtmp
136
137         if test -f ,expect_stdout; then
138             if ! cmp ,expect_stdout ,stdout &>/dev/null; then
139                 echo "unexpected data on stdout" >>,testtmp
140                 grep -v ': \(TRACE\|INFO\|NOTICE\|WARNING\|ERR\):' <,stdout >,tmp
141                 diff -ua ,expect_stdout ,tmp >>,testtmp
142                 rm ,tmp
143                 ((fails+=1))
144             fi
145         fi
146
147         if test -f ,expect_stderr; then
148             if ! cmp ,expect_stderr ,stderr &>/dev/null; then
149                 echo "unexpected data on stderr" >>,testtmp
150                 grep -v ': \(TRACE\|INFO\|NOTICE\|WARNING\|ERR\):' <,stderr >,tmp
151                 diff -ua ,expect_stderr ,tmp >>,testtmp
152                 rm ,tmp
153                 ((fails+=1))
154             fi
155         fi
156
157         if test $expect_return != $return; then
158             echo "unexpected return value $return" >>,testtmp
159             ((fails+=1))
160         fi
161
162         if test $fails -eq 0; then
163             echo ".. OK$MSGOK"
164             echo ".. OK$MSGOK" >>,testlog
165         else
166             echo ".. FAILED$MSGFAIL";
167             echo ".. FAILED$MSGFAIL" >>,testlog
168             cat ,testtmp >>,testlog
169             rm ,testtmp
170             echo "stderr was:" >>,testlog
171             cat ,stderr >>,testlog
172             echo END >>,testlog
173             FAILCNT=$(($FAILCNT + 1))
174             case $TESTMODE in
175             *FIRSTFAIL*)
176                 break 2
177                 ;;
178             esac
179         fi
180 }
181
182 function PLANNED()
183 {
184         echo -n "PLANNED $1: "
185         echo -en "\nPLANNED $* " >>,testlog
186         echo ".. SKIPPED (planned)"
187         echo ".. SKIPPED (planned)" >>,testlog
188         SKIPCNT=$(($SKIPCNT + 1))
189         TESTCNT=$(($TESTCNT + 1))
190 }
191
192 function RUNTESTS()
193 {
194     if test \( ! "${TESTSUITES/*,*/}" \) -a "$TESTSUITES"; then
195         TESTSUITES="{$TESTSUITES}"
196     fi
197     for t in $(eval echo $srcdir/*$TESTSUITES*.tests); do
198         echo "$t"
199     done | sort | uniq | {
200         while read i; do
201             if test -f $i; then
202                 source $i
203             fi
204         done
205         echo
206         if [ $FAILCNT = 0 ]; then
207             echo " ... PASSED $(($TESTCNT - $SKIPCNT)) TESTS, $SKIPCNT SKIPPED"
208             #rm ,testlog
209         else
210             echo " ... SUCCEDED $(($TESTCNT - $FAILCNT - $SKIPCNT)) TESTS"
211             echo " ... FAILED $FAILCNT TESTS"
212             echo " ... SKIPPED $SKIPCNT TESTS"
213             echo " see ',testlog' for details"
214             exit 1
215         fi
216     }
217 }
218
219 function TESTING()
220 {
221     echo
222     echo "$1 [${i##*/}]"
223     TESTBIN=$2
224 }
225
226 TESTSUITES="${TESTSUITES}${1:+${TESTSUITES:+,}$1}"
227
228 RUNTESTS