# # # patch "tester-plaf.hh" # from [19ccb35a02a73e028ba8e80d027213b0eaf52523] # to [1fb912adce77493fcd98c6eb94eb3a5a2deab2b6] # # patch "tester.cc" # from [5d4f12816d5e6c83f0d96c39c0db03839b693765] # to [e6693333f1bbeeee26599a2a608fa5df3ea01150] # # patch "testlib.lua" # from [df3aee450ca03e7454704cde586448214cb6e2c5] # to [a688fbcf17fd30160c55237e1dfdb98b540ecd02] # # patch "unix/tester-plaf.cc" # from [c97649b0639fa5153f5386acc7910b23e364f06e] # to [9002cee809acea4b97baddb839acb021d2fa29c6] # # patch "win32/tester-plaf.cc" # from [636d02af62c13568df4a0e0886e7cc2639df0d51] # to [4baec6a135f718d97454be05fba7a7bc29cdd4b8] # ============================================================ --- tester-plaf.hh 19ccb35a02a73e028ba8e80d027213b0eaf52523 +++ tester-plaf.hh 1fb912adce77493fcd98c6eb94eb3a5a2deab2b6 @@ -13,10 +13,10 @@ // this describes functions to be found, alternatively, in win32/* or unix/* // directories and used only by the tester. -#include // time_t and pid_t +#include void make_accessible(std::string const & name); -time_t get_last_write_time(char const * name); +std::time_t get_last_write_time(char const * name); void do_copy_file(std::string const & from, std::string const & to); void set_env(char const * var, char const * val); void unset_env(char const * var); @@ -24,14 +24,71 @@ bool running_as_root(); char * make_temp_dir(); bool running_as_root(); +// This function has a decidedly awkward interface because (a) Windows +// doesn't have fork(), (b) platform files can't talk directly to the Lua +// interpreter. [Also it would be nice not to have to declare the full +// content of the callback functors, but that is not so bad.] +// +// next_test() is called repeatedly until it returns false. Each time it +// returns true, it shall fill in its test_to_run argument with the number +// and name of the next test to run. +// +// For each such test, *either* invoke() is called in a forked child +// process, *or* the program named in 'runner' is spawned with argument +// vector [runner, '-r', testfile, firstdir, test-name]. Either way, the +// child process is running in a just-created, empty directory which is +// exclusive to that test; its standard input is the null device; and its +// standard output and error are a logfile. If invoke() is called, it is +// expected not to return. +// +// After each per-test child process completes, cleanup() is called for that +// test. If cleanup() returns true, the per-test directory is deleted. + struct lua_State; -pid_t run_one_test_in_child(std::string const & testname, - std::string const & testdir, - lua_State * st, - std::string const & argv0, - std::string const & testfile, - std::string const & firstdir); +struct test_to_run +{ + int number; + std::string name; +}; + +struct test_enumerator +{ + lua_State * st; + int table_ref; + mutable int last_index; + mutable bool iteration_begun; + test_enumerator(lua_State *st, int t) + : st(st), table_ref(t), last_index(0), iteration_begun(false) {} + bool operator()(test_to_run & next_test) const; +}; + +struct test_invoker +{ + lua_State * st; + test_invoker(lua_State *st) : st(st) {} + void operator()(std::string const & testname) const; +}; + +struct test_cleaner +{ + lua_State * st; + int reporter_ref; + test_cleaner(lua_State *st, int r) : st(st), reporter_ref(r) {} + bool operator()(test_to_run const & test, int status) const; +}; + +void run_tests_in_children(test_enumerator const & next_test, + test_invoker const & invoke, + test_cleaner const & cleanup, + std::string const & run_dir, + std::string const & runner, + std::string const & testfile, + std::string const & firstdir); + +// These functions are actually in tester.cc but are used by tester-plaf.cc. +void do_remove_recursive(std::string const & dir); + // Local Variables: // mode: C++ // fill-column: 76 ============================================================ --- tester.cc 5d4f12816d5e6c83f0d96c39c0db03839b693765 +++ tester.cc e6693333f1bbeeee26599a2a608fa5df3ea01150 @@ -525,68 +525,134 @@ LUAEXT(require_not_root, ) } // run_tests_in_children (to_run, reporter) - +// // Run all of the tests in TO_RUN, each in its own isolated directory and // child process. As each exits, call REPORTER with the test number and // name, and the exit status. If REPORTER returns true, delete the test -// directory, otherwise leave it alone. The point of this exercise is to -// isolate in one C function all the black art involved with safe creation -// and destruction of child processes and their context. -LUAEXT(run_tests_in_children, ) +// directory, otherwise leave it alone. +// +// The meat of the work done by this function is so system-specific that it +// gets shoved off into tester-plaf.cc. However, all interaction with the +// Lua layer needs to remain in this file, so we have a mess of callback +// "closures" (or as close as C++ lets you get, anyway). + +// Iterate over the Lua table containing all the tests to run. +bool test_enumerator::operator()(test_to_run & next_test) const { - // lua arguments - const int to_run = 1; - const int reporter = 2; + int top = lua_gettop(st); + luaL_checkstack(st, 2, "preparing to retrieve next test"); - if (lua_gettop(L) != 2) - return luaL_error(L, "wrong number of arguments"); + lua_rawgeti(st, LUA_REGISTRYINDEX, table_ref); + if (iteration_begun) + lua_pushinteger(st, last_index); + else + lua_pushnil(st); - luaL_argcheck(L, lua_istable(L, 1), 1, "expected a table"); - luaL_argcheck(L, lua_isfunction(L, 2), 2, "expected a function"); + if (lua_next(st, -2) == 0) + { + lua_settop(st, top); + return false; + } + else + { + iteration_begun = true; + next_test.number = last_index = luaL_checkinteger(st, -2); + next_test.name = luaL_checkstring(st, -1); + lua_settop(st, top); + return true; + } +} - // iterate over to_run... - lua_pushnil(L); - while (lua_next(L, to_run) != 0) +// Invoke one test case in the child. This may be called by +// run_tests_in_children, or by main, because Windows doesn't have fork. + +void test_invoker::operator()(std::string const & testname) const +{ + int retcode; + try { - luaL_checkinteger(L, -2); - string testname = luaL_checkstring(L, -1); - int status; + luaL_checkstack(st, 2, "preparing call to run_one_test"); + lua_getglobal(st, "run_one_test"); + I(lua_isfunction(st, -1)); - string testdir = run_dir + "/" + testname; + lua_pushstring(st, testname.c_str()); + lua_call(st, 1, 1); - do_remove_recursive(testdir); - do_mkdir(testdir); + retcode = luaL_checkinteger(st, -1); + lua_remove(st, -1); + } + catch (informative_failure & e) + { + P(F("%s\n") % e.what()); + retcode = 1; + } + catch (std::logic_error & e) + { + P(F("Invariant failure: %s\n") % e.what()); + retcode = 3; + } + catch (std::exception & e) + { + P(F("Uncaught exception: %s") % e.what()); + retcode = 3; + } + catch (...) + { + P(F("Uncaught exception of unknown type")); + retcode = 3; + } - pid_t child = run_one_test_in_child(testname, testdir, - L, argv0, testfile, firstdir); + // This does not properly clean up global state, but none of it is + // process-external, so it's ok to let the OS obliterate it; and + // leaving this function any other way is not safe. + exit(retcode); +} - if (child == -1) - status = 127; // spawn failure - else - process_wait(child, &status); - - // Set up a call to REPORTER with the appropriate arguments ... - lua_pushvalue(L, reporter); // t_r r tno tna r - lua_insert(L, -2); // t_r r tno r tna - lua_pushvalue(L, -3); // t_r r tno r tna tno - lua_insert(L, -2); // t_r r tno r tno tna - lua_pushinteger(L, status); // t_r r tno r tno tna st - // ... call it ... - lua_call(L, 3, 1); +// Clean up after one child process. - // ... and if it returns true, delete testdir. - I(lua_isboolean(L, -1)); - if (lua_toboolean(L, -1)) - do_remove_recursive(testdir); +bool test_cleaner::operator()(test_to_run const & test, + int status) const +{ + // call reporter(testno, testname, status) + luaL_checkstack(st, 4, "preparing call to reporter"); - // pop return value, leaving to_run reporter testno, as expected by - // lua_next. - lua_remove(L, -1); - } + lua_rawgeti(st, LUA_REGISTRYINDEX, reporter_ref); + lua_pushinteger(st, test.number); + lua_pushstring(st, test.name.c_str()); + lua_pushinteger(st, status); + lua_call(st, 3, 1); + + // return is a boolean. There is, for no apparent reason, no + // luaL_checkboolean(). + I(lua_isboolean(st, -1)); + bool ret = lua_toboolean(st, -1); + lua_remove(st, -1); + return ret; +} + +LUAEXT(run_tests_in_children, ) +{ + if (lua_gettop(L) != 2) + return luaL_error(L, "wrong number of arguments"); + + luaL_argcheck(L, lua_istable(L, 1), 1, "expected a table"); + luaL_argcheck(L, lua_isfunction(L, 2), 2, "expected a function"); + + int reporter_ref = luaL_ref(L, LUA_REGISTRYINDEX); + int table_ref = luaL_ref(L, LUA_REGISTRYINDEX); + + run_tests_in_children(test_enumerator(L, table_ref), + test_invoker(L), + test_cleaner(L, reporter_ref), + run_dir, argv0, testfile, firstdir); + + luaL_unref(L, LUA_REGISTRYINDEX, table_ref); + luaL_unref(L, LUA_REGISTRYINDEX, reporter_ref); return 0; } + int main(int argc, char **argv) { int retcode = 2; ============================================================ --- testlib.lua df3aee450ca03e7454704cde586448214cb6e2c5 +++ testlib.lua a688fbcf17fd30160c55237e1dfdb98b540ecd02 @@ -928,8 +928,20 @@ function run_tests(debugging, list_only, local of_interest = {} local failed_testlogs = {} + -- exit codes which indicate failure at a point in the process-spawning + -- code where it is impossible to give more detailed diagnostics + local magic_exit_codes = { + [121] = "error creating test directory", + [122] = "error spawning test process", + [123] = "error entering test directory", + [124] = "error redirecting stdin", + [125] = "error redirecting stdout", + [126] = "error redirecting stderr", + [127] = "test did not exit as expected" + } + -- callback closure passed to run_tests_in_children - function report_one_test(tno, tname, status) + local function report_one_test(tno, tname, status) local tdir = run_dir .. "/" .. tname local test_header = string.format("%3d %-45s ", tno, tname) local what @@ -939,44 +951,47 @@ function run_tests(debugging, list_only, if status ~= 0 then if status < 0 then what = string.format("FAIL (signal %d)", -status) + elseif magic_exit_codes[status] ~= nil then + what = string.format("FAIL (%s)", magic_exit_codes[status]) else what = string.format("FAIL (exit %d)", status) end + else + local wfile, err = io.open(tdir .. "/STATUS", "r") + if wfile ~= nil then + what = wfile:read() + wfile:close() + else + what = string.format("FAIL (status file: %s)", err) + end + end + if what == "unexpected success" then + counts.noxfail = counts.noxfail + 1 + counts.of_interest = counts.of_interest + 1 + table.insert(of_interest, test_header .. "unexpected success") + can_delete = false + elseif what == "partial skip" or what == "ok" then + counts.success = counts.success + 1 + can_delete = true + elseif string.find(what, "skipped ") == 1 then + counts.skip = counts.skip + 1 + can_delete = true + elseif string.find(what, "expected failure ") == 1 then + counts.xfail = counts.xfail + 1 + can_delete = false + elseif string.find(what, "FAIL ") == 1 then counts.fail = counts.fail + 1 table.insert(of_interest, test_header .. what) table.insert(failed_testlogs, tdir .. "/tester.log") can_delete = false else - local wfile = io.open(tdir .. "/STATUS", "r") - what = wfile:read() - wfile:close() - if what == "unexpected success" then - counts.noxfail = counts.noxfail + 1 - counts.of_interest = counts.of_interest + 1 - table.insert(of_interest, test_header .. "unexpected success") - can_delete = false - elseif what == "partial skip" or what == "ok" then - counts.success = counts.success + 1 - can_delete = true - elseif string.find(what, "skipped ") == 1 then - counts.skip = counts.skip + 1 - can_delete = true - elseif string.find(what, "expected failure ") == 1 then - counts.xfail = counts.xfail + 1 - can_delete = false - elseif string.find(what, "FAIL ") == 1 then - counts.fail = counts.fail + 1 - table.insert(of_interest, test_header .. what) - table.insert(failed_testlogs, tdir .. "/tester.log") - can_delete = false - else - counts.fail = counts.fail + 1 - what = "FAIL (gobbledygook: " .. what .. ")" - table.insert(of_interest, test_header .. what) - table.insert(failed_testlogs, tdir .. "/tester.log") - can_delete = false - end + counts.fail = counts.fail + 1 + what = "FAIL (gobbledygook: " .. what .. ")" + table.insert(of_interest, test_header .. what) + table.insert(failed_testlogs, tdir .. "/tester.log") + can_delete = false end + counts.total = counts.total + 1 P(string.format("%s%s\n", test_header, what)) if debugging then ============================================================ --- unix/tester-plaf.cc c97649b0639fa5153f5386acc7910b23e364f06e +++ unix/tester-plaf.cc 9002cee809acea4b97baddb839acb021d2fa29c6 @@ -223,82 +223,78 @@ bool running_as_root() return !geteuid(); } -#include "lua/lua.h" -#include "lua/lualib.h" -#include "lua/lauxlib.h" +// General note: the magic numbers in this function are meaningful to +// testlib.lua. They indicate a number of failure scenarios in which +// more detailed diagnostics are not possible. -// Spawn a child to run the single test TESTNAME. Some args are not used on -// this platform. -pid_t -run_one_test_in_child(string const & testname, - string const & testdir, - lua_State * st, - string const & /* argv0 */, - string const & /* testfile */, - string const & /* firstdir */) +void run_tests_in_children(test_enumerator const & next_test, + test_invoker const & invoke, + test_cleaner const & cleanup, + std::string const & run_dir, + std::string const & /*runner*/, + std::string const & /*testfile*/, + std::string const & /*firstdir*/) { - // Make sure there is no pending buffered output before forking, or it - // may be doubled. - fflush(0); + test_to_run t; + string testdir; + while (next_test(t)) + { + // This must be done before we try to redirect stdout/err to a file + // within testdir. If we did it in the child, we would have to do it + // before it was safe to issue diagnostics. + try + { + testdir = run_dir + "/" + t.name; + do_remove_recursive(testdir); + do_mkdir(testdir); + } + catch (...) + { + cleanup(t, 121); + continue; + } - pid_t child = fork(); - if (child != 0) - return child; + // Make sure there is no pending buffered output before forking, or it + // may be doubled. + fflush(0); + pid_t child = fork(); - // From this point on we are in the child process. - // Until we have entered the test directory and re-opened fds 0-2 it is - // not safe to throw exceptions or call any of the diagnostic routines. - // Hence we use bare OS primitives and call _exit() if any of them fail. - // It is safe to assume that close() will not fail. - if (chdir(testdir.c_str()) != 0) _exit(127); - - close(0); - if (open("/dev/null", O_RDONLY) != 0) _exit(126); + if (child != 0) // parent + { + int status; + if (child == -1) + status = 122; // spawn failure + else + process_wait(child, &status); - close(1); - if (open("tester.log", O_WRONLY|O_CREAT|O_EXCL, 0666) != 1) _exit(125); - if (dup2(1, 2) == -1) _exit(124); + if (cleanup(t, status)) + do_remove_recursive(testdir); + } + else // child + { + // From this point on we are in the child process. Until we have + // entered the test directory and re-opened fds 0-2 it is not safe + // to throw exceptions or call any of the diagnostic routines. + // Hence we use bare OS primitives and call _exit(), if any of them + // fail. It is safe to assume that close() will not fail. + if (chdir(testdir.c_str()) != 0) + _exit(123); - // We can now safely use stdio, exceptions, and the normal diagnostic - // routines. However, as caller is not expecting it, we must not leave - // this function by any means save exit(), hence we duplicate all the - // outermost catch clauses from main(). + close(0); + if (open("/dev/null", O_RDONLY) != 0) + _exit(124); - int retcode; - try - { - lua_getglobal(st, "run_one_test"); - I(lua_isfunction(st, -1)); + close(1); + if (open("tester.log", O_WRONLY|O_CREAT|O_EXCL, 0666) != 1) + _exit(125); + if (dup2(1, 2) == -1) + _exit(126); - lua_pushstring(st, testname.c_str()); - lua_call(st, 1, 1); - I(lua_isnumber(st, -1)); - - retcode = lua_tointeger(st, -1); + invoke(t.name); + // If invoke() returns something has gone terribly wrong. + _exit(127); + } } - catch (informative_failure & e) - { - P(F("%s\n") % e.what()); - retcode = 1; - } - catch (std::logic_error & e) - { - P(F("Invariant failure: %s\n") % e.what()); - retcode = 3; - } - catch (std::exception & e) - { - P(F("Uncaught exception: %s") % e.what()); - retcode = 3; - } - catch (...) - { - P(F("Uncaught exception of unknown type")); - retcode = 3; - } - - lua_close(st); - exit(retcode); } // Local Variables: ============================================================ --- win32/tester-plaf.cc 636d02af62c13568df4a0e0886e7cc2639df0d51 +++ win32/tester-plaf.cc 4baec6a135f718d97454be05fba7a7bc29cdd4b8 @@ -103,30 +103,62 @@ bool running_as_root() return false; } +// General note: the magic numbers in this function are meaningful to +// testlib.lua. They indicate a number of failure scenarios in which +// more detailed diagnostics are not possible. +// The bulk of the work is done in main(), -r case, q.v. -pid_t run_one_test_in_child(string const & testname, - string const & testdir, - lua_State * /* st */, - string const & argv0, - string const & testfile, - string const & firstdir) +void run_tests_in_children(test_enumerator const & next_test, + test_invoker const & /*invoke*/, + test_cleaner const & cleanup, + std::string const & run_dir, + std::string const & runner, + std::string const & testfile, + std::string const & firstdir) { - // The bulk of the work is done in main(), -r case, q.v. char const * argv[6]; argv[0] = argv0.c_str(); argv[1] = "-r"; argv[2] = testfile.c_str(); argv[3] = firstdir.c_str(); - argv[4] = testname.c_str(); + argv[4] = 0; argv[5] = 0; - change_current_working_dir(testdir); - pid_t child = process_spawn_redirected("NUL:", - "tester.log", - "tester.log", - argv); - change_current_working_dir(run_dir); - return child; + test_to_run t; + string testdir; + while (next_test(t)) + { + // This must be done before we try to redirect stdout/err to a + // file within testdir. + try + { + testdir = run_dir + "/" + t.name; + do_remove_recursive(testdir); + do_mkdir(testdir); + } + catch (...) + { + cleanup(t, 121); + continue; + } + + change_current_working_dir(testdir); + argv[4] = t.name.c_str(); + pid_t child = process_spawn_redirected("NUL:", + "tester.log", + "tester.log", + argv); + change_current_working_dir(run_dir); + + int status; + if (child == -1) + status = 122; + else + process_wait(child, &status); + + if (cleanup(t, status)) + do_remove_recursive(testdir); + } }