Skip to content

Commit e06e1f5

Browse files
steveristeveri
authored
Fix bad gold test (#12)
* embed failure path for testing * fixing a BAAAAAD typo * DIE if error during garnet build * improved log output maybe * Enhance vcompare with a prefilter to ignore comment lines :( * DO NOT compare garnet.v, compare genesis-created .sv files instead ! * tweak log output * Undo fault injection, should PASS now maybe * cleanup
1 parent 6459918 commit e06e1f5

1 file changed

Lines changed: 73 additions & 44 deletions

File tree

‎test/genesis-ci.sh‎

Lines changed: 73 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
#!/bin/bash
22

3+
# Uncomment to test failure path
4+
# TEST_FAILURE_PATH=true
5+
36
USAGE="
47
$0 <commit>
58
@@ -14,12 +17,12 @@ USAGE="
1417
if [ "$1" == "--help" ]; then echo "$USAGE"; exit; fi
1518

1619
# What does this script do?
17-
# - verifies that tmp-garnet.v[01] does not exist already
20+
# - verifies that compare-dirs tmp-gverif.d[01] do not exist already
1821
# - launches a container based on garnet:latest
1922
# -- pip-installs latest genesis2 (maybe not necessary no more?)
20-
# -- builds gold verilog tmp-garnet.v0 using master branch
21-
# -- builds target verilog tmp-garnet.v1 using target branch
22-
# -- compares the two
23+
# -- builds gold verilog files tmp-gverif.d0/*.sv using master branch
24+
# -- builds target verilog tmp-gverif.d1/*.sv using target branch
25+
# -- compares each target file *.sv against gold
2326

2427
# How to use it?
2528
# - I mean, it's container based...I think you just run
@@ -29,10 +32,8 @@ if [ "$1" == "--help" ]; then echo "$USAGE"; exit; fi
2932
# genesis-ci.sh origin/`git branch --show-current` |& tee genesis-ci.log | less -r
3033

3134
# Make sure the coast is clear
32-
f1=tmp-garnet.v0
33-
f2=tmp-garnet.v1
34-
if test -f $f1; then echo ERROR $f1 exists already; exit 13; fi
35-
if test -f $f2; then echo ERROR $f2 exists already; exit 13; fi
35+
d1=tmp-gverif.d0; if test -e $d1; then echo ERROR $d1 exists already; exit 13; fi
36+
d2=tmp-gverif.d1; if test -e $d2; then echo ERROR $d2 exists already; exit 13; fi
3637

3738
# Debugging
3839
# set -x
@@ -71,65 +72,93 @@ dexec "source /aha/bin/activate && pip uninstall -y genesis2 && pip install gene
7172
ENDGROUP
7273

7374
##############################################################################
74-
GROUP 'GOLD-BUILD (master) > tmp-garnet.v0'
75+
GROUP 'GOLD-BUILD (master) > tmp-gverif.d0/*.sv'
7576
printf "\nINFO Build gold-model verilog using Genesis2 branch 'master'"
76-
dexec "$build_garnet"
77-
docker cp ${container}:/aha/garnet/garnet.v tmp-garnet.v0
77+
dexec "$build_garnet" || exit 13
78+
docker cp ${container}:/aha/garnet/genesis_verif tmp-gverif.d0/
7879
dexec 'cd /aha/garnet; make clean' >& /dev/null # Clean up your mess, ignore errors :(
7980
ENDGROUP
8081

8182
# Can do this if want to accommodate commits like e.g. 'pull/9/head'
8283
# if $commit ~ '^pull/'; then git fetch origin $commit:TEST; commit=TEST; fi
8384

8485
##############################################################################
85-
GROUP "TEST-BUILD ($commit) > tmp-garnet.v1"
86+
GROUP "TEST-BUILD ($commit) > tmp-gverif.d1/*.sv"
8687
printf "\nINFO Build test-model verilog using Genesis2 branch '$commit'\n"
8788
REPO=/aha/lib/python3.8/site-packages/Genesis2-src
8889
dexec "set -x; cd $REPO; git pull; git checkout -fq $commit" || exit 13
89-
dexec "$build_garnet"
90-
docker cp ${container}:/aha/garnet/garnet.v tmp-garnet.v1
90+
if [ "$TEST_FAILURE_PATH" ]; then
91+
# Inject a fault in Genesis2.pl
92+
# FAULT INJECTION
93+
# FAULT INJECTION print("FOO Attempting fault injection")
94+
# FAULT INJECTION if os.path.isfile("/aha/garnet/genesis_verif/jtag.sv"):
95+
# FAULT INJECTION print("FOO injecting error in /aha/garnet/genesis_verif/jtag.sv")
96+
# FAULT INJECTION r2 = os.system("set -x; sed -i.bak 's/addr = 0/addr = 13/' /aha/garnet/genesis_verif/jtag.sv")
97+
# FAULT INJECTION print("FOO r2 = "); print (r2)
98+
# FAULT INJECTION if r2 != 0: sys.exit(13)
99+
# FAULT INJECTION print("FOO injection SUCCESSFULL...???")
100+
# FAULT INJECTION
101+
fault=$(egrep '^ # FAULT INJECTION' $0 | sed 's/.*INJECTION //' > /tmp/tmp$$)
102+
dexec "cp /aha/bin/Genesis2.pl /aha/bin/Genesis2.pl0"
103+
docker cp /tmp/tmp$$ $container:/tmp; # /bin/rm /tmp/tmp$$
104+
dexec "cat /tmp/tmp$$ >> /aha/bin/Genesis2.pl"
105+
dexec "diff /aha/bin/Genesis2.pl0 /aha/bin/Genesis2.pl"
106+
fi
107+
dexec "$build_garnet" || exit 13
108+
if [ "$TEST_FAILURE_PATH" ]; then
109+
set -x # Show injected fault
110+
dexec 'set -x; ls -l /aha/garnet/genesis_verif/'
111+
dexec 'set -x; diff /aha/garnet/genesis_verif/jtag.{sv.bak,sv}'
112+
set +x
113+
fi
114+
docker cp ${container}:/aha/garnet/genesis_verif tmp-gverif.d1/
91115
dexec 'cd /aha/garnet; make clean' >& /dev/null # Clean up your mess, ignore errors :(
92116
ENDGROUP
93117

118+
94119
##############################################################################
95120
# COMPARE "gold" and "test"; use vcompare utility from aha repo
96-
ls -l tmp-garnet.v[01]
121+
GROUP COMPARE gold and test models
122+
printf ".\nCOMPARE gold and test models\n.\n"
123+
echo .; ls -l tmp-gverif.d0/ | sed 's/^/ /'
97124
if ! test -e tmp-vcompare.sh; then
98125
docker cp ${container}:/aha/.buildkite/bin/vcompare.sh tmp-vcompare.sh
99126
fi
100-
function vcompare { ./tmp-vcompare.sh $*; }
127+
128+
# Enhance vcompare with a prefilter to ignore comment lines :(
129+
function delcomms { egrep -v '^//' $1; }
130+
function vcompare { ./tmp-vcompare.sh <(delcomms $1) <(delcomms $2); }
101131

102132
# docker kill $container # Don't need this anymore because of TRAP
103133

104134
set +x
105-
f1=tmp-garnet.v0
106-
f2=tmp-garnet.v1
107-
108-
# Copied from aha/.buildkite/bin/rtl-goldcheck.sh
109-
printf "\n"
110-
echo "Comparing `vcompare $f1 | wc -l` lines of $f1"
111-
echo "versus `vcompare $f2 | wc -l` lines of $f2"
112-
printf "\n"
113-
114-
echo "diff $f1 $f2"
115-
ndiffs=`vcompare $f1 $f2 | wc -l`
116-
117-
if [ "$ndiffs" != "0" ]; then
118-
# ------------------------------------------------------------------------
119-
# TEST FAILED
120-
printf ".\n.\n Test FAILED\n Test FAILED\n Test FAILED\n.\n"
121-
printf "Test FAILED with $ndiffs diffs\n"
122-
printf '(To update gold verilog, see $GARNET_REPO/bin/rtl-goldfetch.sh --help)'
123-
printf "\n"
124-
printf "Top 40 diffs:"
125-
vcompare $f1 $f2 | head -40
126-
printf ".\n.\n Test FAILED\n Test FAILED\n Test FAILED\n.\n"
127-
exit 13
128-
else
129-
# ------------------------------------------------------------------------
130-
# TEST PASSED
131-
printf ".\n.\n Test PASSED\n Test PASSED\n Test PASSED\n.\n"
132-
fi
135+
d1=tmp-gverif.d0
136+
d2=tmp-gverif.d1
137+
138+
echo .
139+
files=$(cd $d1; echo *.sv)
140+
for f in $files; do
141+
# E.g. f1=tmp-gverif.d0/jtag.sv, f2=tmp-gverif.d1/jtag.sv
142+
f1=$d1/$f; f2=$d2/$f
143+
echo " $f..."
144+
ndiffs=`vcompare $f1 $f2 | wc -l`
145+
if [ "$ndiffs" != "0" ]; then
146+
ENDGROUP
147+
# ------------------------------------------------------------------------
148+
# TEST FAILED
149+
printf ".\nTest of $f FAILED with $ndiffs diff lines\n"
150+
printf "\n"
151+
printf "Top 40 diff lines:"
152+
vcompare $f1 $f2 | head -40
153+
printf ".\n.\n Test FAILED\n Test FAILED\n Test FAILED\n.\n"
154+
exit 13
155+
fi
156+
done
157+
ENDGROUP
158+
# ------------------------------------------------------------------------
159+
# TEST PASSED
160+
printf ".\n Test PASSED\n Test PASSED\n Test PASSED\n.\n"
161+
133162

134163
# ...but what if master got corrupted and test-branch preserves that?
135164
# ...test will pass but answer is wrong??

0 commit comments

Comments
 (0)