document the fault-coverage checker
authorChristian Thaeter <ct@pipapo.org>
Tue, 3 Aug 2010 03:11:16 +0000 (05:11 +0200)
committerChristian Thaeter <ct@pipapo.org>
Tue, 3 Aug 2010 03:11:16 +0000 (05:11 +0200)
doc/coverageintro.txt [new file with mode: 0644]
doc/nobug_manual.conf
src/nobug.h

diff --git a/doc/coverageintro.txt b/doc/coverageintro.txt
new file mode 100644 (file)
index 0000000..cb44110
--- /dev/null
@@ -0,0 +1,43 @@
+HEAD- Fault coverage checking;;
+
+CAUTION: Fault coverage checking is a experimental feature!
+
+Nobug can automatically inject faults at instrumented points and permute
+through all potential error paths of an application by restarting it with the
+state from the former run. It can be used to give answers to the question if
+any possible error is sufficiently handled in the application. Fault coverage
+checking is only available in ALPHA builds and optimized out otherwise.
+
+NOTE: This kind of testing is very expensive.
+
+HEAD~ How does automatic fault injection work;;
+
+First NoBug checks if the environment variable `NOBUG_COVERAGE` is set, if yes
+it must contain a space, comma or semicolon separated list of filename which
+are the logs from the previous run. This logs are then parsed for the states
+of the previous run, storing these in a lookup tree.
+
+After that the application proceeds as usual, when it then hits an
+instrumented coverage point it is checked against the already recorded states.
+Any so far unseen failure point injects a fault, the last seen but previously
+failed point will be pass now, all other fault injection points act like on
+their previous run. This ensures that each successive run of the application
+changes only one injection point and thus permutes through all possible code
+paths.
+
+Fault injection points are identified by a 64bit hash over the backtrace
+(return addresses on the stack) leading to it. This means each unique way to
+reach a injection point is recorded. Parameters and threads are intentionally
+not considered in this calculation.
+
+
+HEAD~ How to invoke fault-coverage checking;;
+
+Each fault-injection point emits a logging message about its identity and
+state, this logging uses the normal NoBug logging facilities. Thus one can
+control the logging with the `NOBUG_LOG` environment variable. Additionally
+fault coverage checking is only active when the `NOBUG_COVERAGE` environment
+variable was set to point some log files which are the results from the
+previous run. NoBug comes with a `tests/coverage.sh` example script which
+permutes through all possible error paths.
+
index 4af1a8d..22147a8 100644 (file)
@@ -42,6 +42,8 @@
 
 //=faultinjection
 
+//=coverageintro
+//=coverage
 
 //=resourcetracking
 //=resourcemacros
index 7edf816..fb1a257 100644 (file)
@@ -873,24 +873,20 @@ NOTREACHED      abort           abort           nothing
 #define NOBUG_INJECT_LEVEL LOG_NOTICE
 #endif
 
-/*
-  temporary disabling/enabling
-
-  sometimes fault injection yields false positives, errors which may never happen in real life
-  (and are possibly enforced with a ENSURE afterwards). For this cases coverage fault injection can be
-  disabled temporarly and reenabled later. disabling/enabling may nest.
- */
 
-#define NOBUG_COVERAGE_DISABLE                                  \
-  NOBUG_IF_ALPHA(unsigned nobug_coverage_disable_check =        \
-                 nobug_thread_get ()->coverage_disable_cnt;)    \
-  NOBUG_IF_ALPHA(++ nobug_thread_get ()->coverage_disable_cnt)
-
-#define NOBUG_COVERAGE_ENABLE                                   \
-  NOBUG_IF_ALPHA(-- nobug_thread_get ()->coverage_disable_cnt); \
-  ENSURE(nobug_coverage_disable_check ==                        \
-         nobug_thread_get ()->coverage_disable_cnt,             \
-         "COVERAGE ENABLE/DISABLE does not match")
+/*
+//coverage HEAD- Coverage checking Macros;;
+//coverage
+//coverage PARA COVERAGE_FAULT; COVERAGE_FAULT; coverage fault injection, statement
+//coverage  COVERAGE_FAULT(flag, ...)
+//coverage
+//coverage Injects the statement at `...` when simulating an failure. Only active in
+//coverage ALPHA builds.
+//coverage
+*/
+#define NOBUG_COVERAGE_FAULT(flag, ...)                                 \
+  NOBUG_IF_ALPHA(NOBUG_COVERAGE_FAULT_(flag, __VA_ARGS__))              \
+  NOBUG_IF_NOT_ALPHA(__VA_ARGS__)
 
 
 #define NOBUG_COVERAGE_FAULT_(flag, ...)                                \
@@ -910,10 +906,18 @@ NOTREACHED      abort           abort           nothing
   } while (0)
 
 
-#define NOBUG_COVERAGE_FAULT(flag, ...)                                 \
-  NOBUG_IF_ALPHA(NOBUG_COVERAGE_FAULT_(flag, __VA_ARGS__))              \
-  NOBUG_IF_NOT_ALPHA(__VA_ARGS__)
 
+/*
+//coverage PARA COVERAGE_GOODBAD; COVERAGE_GOODBAD; coverage fault injection, expression
+//coverage  COVERAGE_GOODBAD(flag, good, bad)
+//coverage
+//coverage Substitutes to an expression and injects `bad` when simulating an failure and `good`
+//coverage otherwise. Only active in ALPHA builds.
+//coverage
+*/
+#define NOBUG_COVERAGE_GOODBAD(flag, good, bad)                         \
+  NOBUG_IF_ALPHA(NOBUG_COVERAGE_GOODBAD_(flag, good, bad))              \
+  NOBUG_IF_NOT_ALPHA((good))
 
 #define NOBUG_COVERAGE_GOODBAD_(flag, good, bad)                        \
   ({                                                                    \
@@ -928,9 +932,6 @@ NOTREACHED      abort           abort           nothing
   })
 
 
-#define NOBUG_COVERAGE_GOODBAD(flag, good, bad)                         \
-  NOBUG_IF_ALPHA(NOBUG_COVERAGE_GOODBAD_(flag, good, bad))              \
-  NOBUG_IF_NOT_ALPHA((good))
 
 
 #define NOBUG_COVERAGE_LOG(flag, record)                \
@@ -940,12 +941,49 @@ NOTREACHED      abort           abort           nothing
               record.hash,                              \
               record.state == 'F'?"FAILURE":"PASS")
 
+
+/*
+//coverage PARA COVERAGE_LEVEL; COVERAGE_LEVEL; coverage fault injection log level
+//coverage  #define NOBUG_COVERAGE_LEVEL ...
+//coverage
+//coverage Logging level at what fault-coverage logging is emitted.
+//coverage
+//coverage NOTE: Supressing log output with this level will not supress fault injection,
+//coverage       actually the opposite is true since every new seen failure path gets injected.
+//coverage       This might be changed in future releases.
+//coverage
+// TODO see above, disable fault injection when not logged
+//       if (NOBUG_EXPECT_FALSE(NOBUG_COVERAGE_LEVEL <= flag->limits[NOBUG_TARGET_???]))
+*/
+
 #ifndef NOBUG_COVERAGE_LEVEL
 #define NOBUG_COVERAGE_LEVEL LOG_NOTICE
 #endif
 
 
 
+/*
+//coverage PARA Disabling and enabling fault-coverage checks; COVERAGE_DISABLE; handle false positives
+//coverage  NOBUG_COVERAGE_DISABLE
+//coverage  NOBUG_COVERAGE_ENABLE
+//coverage
+//coverage Sometimes fault injection yields false positives, errors which may never happen in real life
+//coverage (and are possibly enforced with a ENSURE afterwards). For this cases coverage fault injection can be
+//coverage disabled temporarly and reenabled later. Disabling/enabling may nest and must properly match.
+//coverage
+*/
+
+#define NOBUG_COVERAGE_DISABLE                                  \
+  NOBUG_IF_ALPHA(unsigned nobug_coverage_disable_check =        \
+                 nobug_thread_get ()->coverage_disable_cnt;)    \
+  NOBUG_IF_ALPHA(++ nobug_thread_get ()->coverage_disable_cnt)
+
+#define NOBUG_COVERAGE_ENABLE                                   \
+  NOBUG_IF_ALPHA(-- nobug_thread_get ()->coverage_disable_cnt); \
+  ENSURE(nobug_coverage_disable_check ==                        \
+         nobug_thread_get ()->coverage_disable_cnt,             \
+         "COVERAGE ENABLE/DISABLE does not match")
+
 
 /*
   Flag handling
@@ -2696,7 +2734,10 @@ nobug_section_cleaned (int* self)
 }
 
 
-//faultcoverage
+/*
+  fault coverage checking
+*/
+
 struct nobug_coverage_record
 {
   uint64_t hash;