[gpsd-dev] [PATCH] coverage: proof of concept

[Christian Gagneraud]

This patch is for illustration purpose only, it shows what have to be done to
get coverage data when running the regressions tests.
Basically, lcov has to be called after every single execution of a test
program. It looks to me that there's a need here to define a python test class
(or function) that will take care of calling lcov when in coverage mode.
The problem is that sometimes a single <blah>-regress involves calling
test programs more than once (aivdm and rtcm regress come to my mind).

I think this time I got it right, so can people gives feedback about the
results (see link below), especially, if you think that some files
should have a better coverage than what the report shows.

There's plenty of room here for improvment, please don't hesitate to
comment.

Usage:
  scons coveraging=True [...]
  scons cov-init
  scons check
  scons cov-report

An example report is available online:
  http://fjorgyn.chgans.co.nz/pub/gpsd/20131101

Signed-off-by: Christian Gagneraud <address@hidden>
---
 .gitignore |  5 +++++
 SConstruct | 51 ++++++++++++++++++++++++++++++++++++++++++---------
 2 files changed, 47 insertions(+), 9 deletions(-)

diff --git a/.gitignore b/.gitignore
index 93687b9..7f33295 100644
--- a/.gitignore
+++ b/.gitignore
@@ -135,3 +135,8 @@ devtools/.flocktest.ini
 # QT stuff
 libQgpsmm.prl
 libQgpsmm/binaries/*
+# Code coverage
+*.gcda
+*.gcno
+*.lcov
+coverage-report
diff --git a/SConstruct b/SConstruct
index 756616c..c6087ed 100644
--- a/SConstruct
+++ b/SConstruct
@@ -1451,7 +1451,8 @@ audit = env.Alias('audit',
 
 # Unit-test the bitfield extractor
 bits_regress = Utility('bits-regress', [test_bits], [
-    '$SRCDIR/test_bits --quiet'
+    '$SRCDIR/test_bits --quiet',
+    'lcov -c -d . -o gpsd_regress_bits.lcov'
     ])
 
 # Check that all Python modules compile properly 
@@ -1470,13 +1471,15 @@ else:
 if env['socket_export']:
     # Regression-test the daemon
     gps_regress = Utility("gps-regress", [gpsd, python_built_extensions],
-            '$SRCDIR/regress-driver %s test/daemon/*.log' 
%(regress_driver_options,))
+        ['$SRCDIR/regress-driver %s test/daemon/*.log' 
%(regress_driver_options,),
+         'lcov -c -d . -o gpsd_regress_gps.lcov'])
 
     # Test that super-raw mode works. Compare each logfile against itself
     # dumped through the daemon running in R=2 mode.  (This test is not
     # included in the normal regressions.)
     Utility("raw-regress", [gpsd, python_built_extensions],
-        '$SRCDIR/regress-driver %s test/daemon/*.log' 
%(regress_driver_options,))
+        ['$SRCDIR/regress-driver %s test/daemon/*.log' 
%(regress_driver_options,),
+        'lcov -c -d . -o gpsd_regress_raw.lcov'])
 
     # Build the regression tests for the daemon.
     # Note: You'll have to do this whenever the default leap second
@@ -1499,12 +1502,14 @@ rtcm_regress = Utility('rtcm-regress', [gpsdecode], [
         '$SRCDIR/gpsdecode -u -j <$${f} >$${TMPFILE}; '
         'diff -ub $${f}.chk $${TMPFILE}; '
         'rm -f $${TMPFILE}; '
+        'lcov -t "rtcm-regress" -c -d . -o gpsd_regress_rtcm_`basename 
$${f}`.lcov; '
     'done;',
     '@echo "Testing idempotency of JSON dump/decode for RTCM2"',
     '@TMPFILE=`mktemp -t gpsd-test-XXXXXXXXXXXXXX.chk`; '
     '$SRCDIR/gpsdecode -u -e -j <test/synthetic-rtcm2.json >$${TMPFILE}; '
         'grep -v "^#" test/synthetic-rtcm2.json | diff -ub - $${TMPFILE}; '
-        'rm -f $${TMPFILE}; ',
+        'rm -f $${TMPFILE}; '
+        'lcov -t "rtcm-regress" -c -d . -o 
gpsd_regress_rtcm_synthetic-rtcm2.json.lcov; ',
         ])
 
 # Rebuild the RTCM regression tests.
@@ -1523,6 +1528,7 @@ aivdm_regress = Utility('aivdm-regress', [gpsdecode], [
         '$SRCDIR/gpsdecode -u -c <$${f} >$${TMPFILE}; '
         'diff -ub $${f}.chk $${TMPFILE} || echo "Test FAILED!"; '
         'rm -f $${TMPFILE}; '
+        'lcov -t "aivdm-regress" -c -d . -o gpsd_regress_aivdm_csv_`basename 
$${f}`.lcov; '
     'done;',
     '@echo "Testing AIVDM decoding w/ JSON unscaled format..."',
     '@for f in $SRCDIR/test/*.aivdm; do '
@@ -1531,6 +1537,7 @@ aivdm_regress = Utility('aivdm-regress', [gpsdecode], [
         '$SRCDIR/gpsdecode -u -j <$${f} >$${TMPFILE}; '
         'diff -ub $${f}.ju.chk $${TMPFILE} || echo "Test FAILED!"; '
         'rm -f $${TMPFILE}; '
+        'lcov -t "aivdm-regress" -c -d . -o gpsd_regress_aivdm_jsu_`basename 
$${f}`.lcov; '
     'done;',
     '@echo "Testing AIVDM decoding w/ JSON scaled format..."',
     '@for f in $SRCDIR/test/*.aivdm; do '
@@ -1539,19 +1546,22 @@ aivdm_regress = Utility('aivdm-regress', [gpsdecode], [
         '$SRCDIR/gpsdecode -j <$${f} >$${TMPFILE}; '
         'diff -ub $${f}.js.chk $${TMPFILE} || echo "Test FAILED!"; '
         'rm -f $${TMPFILE}; '
+        'lcov -t "aivdm-regress" -c -d . -o gpsd_regress_aivdm_jss_`basename 
$${f}`.lcov; '
     'done;',
     '@echo "Testing idempotency of unscaled JSON dump/decode for AIS"',
     '@TMPFILE=`mktemp -t gpsd-test-XXXXXXXXXXXXXX.chk`; '
     '$SRCDIR/gpsdecode -u -e -j <$SRCDIR/test/sample.aivdm.ju.chk 
>$${TMPFILE}; '
         'grep -v "^#" $SRCDIR/test/sample.aivdm.ju.chk | diff -ub - 
$${TMPFILE}; '
-        'rm -f $${TMPFILE}; ',
+        'rm -f $${TMPFILE}; '
+        'lcov -t "aivdm-regress" -c -d . -o 
gpsd_regress_aivdm_jsui_sample.aivdm.ju.chk.lcov; ',
     # Parse the unscaled json reference, dump it as scaled json, 
     # and finally compare it with the scaled json reference
     '@echo "Testing idempotency of scaled JSON dump/decode for AIS"',
     '@TMPFILE=`mktemp -t gpsd-test-XXXXXXXXXXXXXX.chk`; '
     '$SRCDIR/gpsdecode -e -j <$SRCDIR/test/sample.aivdm.ju.chk >$${TMPFILE}; '
         'grep -v "^#" $SRCDIR/test/sample.aivdm.js.chk | diff -ub - 
$${TMPFILE}; '
-        'rm -f $${TMPFILE}; ',
+        'rm -f $${TMPFILE}; '
+        'lcov -t "aivdm-regress" -c -d . -o 
gpsd_regress_aivdm_jssi_sample.aivdm.ju.chk.lcov; ',
         ])
 
 # Rebuild the AIVDM regression tests.
@@ -1567,6 +1577,7 @@ Utility('aivdm-makeregress', [gpsdecode], [
 packet_regress = Utility('packet-regress', [test_packet], [
     '@echo "Testing detection of invalid packets..."',
     '$SRCDIR/test_packet | diff -u $SRCDIR/test/packet.test.chk -',
+    'lcov -t "packet-regress" -c -d . -o gpsd_regress_packet.lcov'
     ])
 
 # Rebuild the packet-getter regression test
@@ -1582,40 +1593,46 @@ Utility('geoid-makeregress', [test_geoid], [
 geoid_regress = Utility('geoid-regress', [test_geoid], [
     '@echo "Testing the geoid model..."',
     '$SRCDIR/test_geoid 37.371192 122.014965 | diff -u 
$SRCDIR/test/geoid.test.chk -',
+    'lcov -t "geoid-regress" -c -d . -o gpsd_regress_geoid.lcov'
     ])
 
 # Regression-test the Maidenhead Locator
 maidenhead_locator_regress = Utility('maidenhead-locator-regress', 
[python_built_extensions], [
     '@echo "Testing the Maidenhead Locator conversion..."',
     '$SRCDIR/test_maidenhead.py >/dev/null',
+    'lcov -t "maidenhead-locator-regress" -c -d . -o 
gpsd_regress_maidenhead.lcov'
     ])
 
 # Regression-test the calendar functions
 time_regress = Utility('time-regress', [test_mkgmtime], [
-    '$SRCDIR/test_mkgmtime'
+    '$SRCDIR/test_mkgmtime',
+    'lcov -t "time-regress" -c -d . -o gpsd_regress_time.lcov'
     ])
 
 # Regression test the unpacking code in libgps
 unpack_regress = Utility('unpack-regress', [test_libgps], [
     '@echo "Testing the client-library sentence decoder..."',
     '$SRCDIR/regress-driver %s -c $SRCDIR/test/clientlib/*.log' 
%(regress_driver_options,),
+    'lcov -t "unpack-regress" -c -d . -o gpsd_regress_unpack.lcov'
     ])
 
 # Build the regression test for the sentence unpacker
 Utility('unpack-makeregress', [test_libgps], [
     '@echo "Rebuilding the client sentence-unpacker tests..."',
-    '$SRCDIR/regress-driver %s -c -b $SRCDIR/test/clientlib/*.log' 
%(regress_driver_options,)
+    '$SRCDIR/regress-driver %s -c -b $SRCDIR/test/clientlib/*.log' 
%(regress_driver_options,),
     ])
 
 # Unit-test the JSON parsing
 json_regress = Utility('json-regress', [test_json], [
-    '$SRCDIR/test_json'
+    '$SRCDIR/test_json',
+    'lcov -t "json-regress" -c -d . -o gpsd_regress_json.lcov'
     ])
 
 # consistency-check the driver methods
 method_regress = Utility('packet-regress', [test_packet], [
     '@echo "Consistency-checking driver methods..."',
     '$SRCDIR/test_packet -c >/dev/null',
+    'lcov -t "packet-regress" -c -d . -o gpsd_regress_packet.lcov'
     ])
 
 # Run a valgrind audit on the daemon  - not in normal tests
@@ -1645,6 +1662,22 @@ check = env.Alias('check', [
 
 env.Alias('testregress', check)
 
+# Code coverage reporting
+#
+# Usage:
+#  scons coveraging=True [...]
+#  scons cov-init
+#  scons check
+#  scons cov-report
+covi = Utility("cov-init", [],
+               "lcov -c -i -d . -o gpsd_base.lcov")
+covc = Utility("cov-collect", [],
+               "lcov -a gpsd_base.lcov `ls -1 gpsd_regress_*.lcov | sed 's/^/ 
-a /g'` -o gpsd_all.lcov")
+covr = Utility("cov-report", [],
+               "genhtml -o coverage-report --show-details --legend 
gpsd_base.lcov gpsd_regress_*.lcov")
+covl = Utility("cov-clean", [],
+               "rm -f *.gc* *.lcov; rm -Rf coverage-report")
+
 # The website directory
 #
 # None of these productions are fired by default.
-- 
1.8.1.2