# # patch "database.cc" # from [f43a456ebcb0b54a63a50dce5125ea8e6d61e28d] # to [51da17f0a125bd2f1f6775b6a83508d1b07d9b9d] # # patch "database.hh" # from [a6d58350ae742e2cdefceab07b7598f88749f8cb] # to [0103ae7924020613ab0eef601d6ca4930df8d9d4] # # patch "database_check.cc" # from [ddecb300ef2802d08947a15707ac1f14337253b8] # to [ef6fa8b0b5acf5297d64f4a16e8646f68dec40da] # # patch "tests/t_database_check.at" # from [f9e93071aa9dd8bdf4643550b9a54ee87aca2fda] # to [b034de4e58d344e3ac343a04f29bea851aa93717] # # patch "tests/t_database_check_minor.at" # from [c3716a80023cce652e10ebebb1df1e405a9d86ca] # to [fe8224086ed9df240a2ce9142c48e93ee95ace3f] # ======================================================================== --- database.cc f43a456ebcb0b54a63a50dce5125ea8e6d61e28d +++ database.cc 51da17f0a125bd2f1f6775b6a83508d1b07d9b9d @@ -1148,6 +1148,19 @@ return exists(id.inner(), "revisions"); } +bool +database::roster_exists_for_revision(revision_id const & rev_id) +{ + results res; + string query = ("SELECT id FROM rosters WHERE rev_id = ? " + "UNION " + "SELECT id FROM roster_deltas WHERE rev_id = ? "); + fetch(res, one_col, any_rows, query.c_str(), + rev_id.inner()().c_str(), rev_id.inner()().c_str()); + I((res.size() == 1) || (res.size() == 0)); + return res.size() == 1; +} + void database::get_file_ids(set & ids) { @@ -2511,7 +2524,6 @@ } } - void database::get_roster_id_for_revision(revision_id const & rev_id, hexenc & roster_id) @@ -2523,11 +2535,9 @@ } results res; - string data_table = "rosters"; - string delta_table = "roster_deltas"; - string query = ("SELECT id FROM " + data_table + " WHERE rev_id = ? " + string query = ("SELECT id FROM rosters WHERE rev_id = ? " "UNION " - "SELECT id FROM " + delta_table + " WHERE rev_id = ? "); + "SELECT id FROM roster_deltas WHERE rev_id = ? "); fetch(res, one_col, one_row, query.c_str(), rev_id.inner()().c_str(), ======================================================================== --- database.hh a6d58350ae742e2cdefceab07b7598f88749f8cb +++ database.hh 0103ae7924020613ab0eef601d6ca4930df8d9d4 @@ -229,6 +229,7 @@ bool file_version_exists(file_id const & id); bool manifest_version_exists(manifest_id const & id); bool revision_exists(revision_id const & id); + bool roster_exists_for_revision(revision_id const & id); void get_file_ids(std::set & ids); void get_manifest_ids(std::set & ids); ======================================================================== --- database_check.cc ddecb300ef2802d08947a15707ac1f14337253b8 +++ database_check.cc ef6fa8b0b5acf5297d64f4a16e8646f68dec40da @@ -86,6 +86,7 @@ size_t revision_refs; // number of references to this revision from other revisions size_t ancestry_parent_refs; // number of references to this revision by ancestry parent size_t ancestry_child_refs; // number of references to this revision by ancestry child + size_t marking_refs; // number of references to this revision by roster markings bool found_roster; // the roster for this revision exists bool manifest_mismatch; // manifest doesn't match the roster for this revision @@ -106,6 +107,7 @@ checked_revision(): found(false), revision_refs(0), ancestry_parent_refs(0), ancestry_child_refs(0), + marking_refs(0), found_roster(false), manifest_mismatch(false), incomplete_roster(false), missing_manifests(0), missing_revisions(0), cert_refs(0), parseable(false), normalized(false) {} @@ -237,27 +239,37 @@ { // lots of revisions that must exist marking_t mark = mm[n->first]; + checked_revisions[mark.birth_revision].marking_refs++; if (!checked_revisions[mark.birth_revision].found) checked_rosters[ros_id].missing_mark_revs++; for (set::const_iterator r = mark.parent_name.begin(); r != mark.parent_name.end(); r++) - if (!checked_revisions[*r].found) - checked_rosters[ros_id].missing_mark_revs++; + { + checked_revisions[*r].marking_refs++; + if (!checked_revisions[*r].found) + checked_rosters[ros_id].missing_mark_revs++; + } for (set::const_iterator r = mark.file_content.begin(); r != mark.file_content.end(); r++) - if (!checked_revisions[*r].found) - checked_rosters[ros_id].missing_mark_revs++; + { + checked_revisions[*r].marking_refs++; + if (!checked_revisions[*r].found) + checked_rosters[ros_id].missing_mark_revs++; + } for (map >::const_iterator attr = mark.attrs.begin(); attr != mark.attrs.end(); attr++) for (set::const_iterator r = attr->second.begin(); r != attr->second.end(); r++) - if (!checked_revisions[*r].found) - checked_rosters[ros_id].missing_mark_revs++; + { + checked_revisions[*r].marking_refs++; + if (!checked_revisions[*r].found) + checked_rosters[ros_id].missing_mark_revs++; + } } - ticks++; + ++ticks; } } @@ -304,11 +316,12 @@ checked_revisions[*i].normalized = true; // roster checks - hexenc roster_id; - app.db.get_roster_id_for_revision(*i, roster_id); - if (!null_id(roster_id)) + if (app.db.roster_exists_for_revision(*i)) { + hexenc roster_id; checked_revisions[*i].found_roster = true; + app.db.get_roster_id_for_revision(*i, roster_id); + I(checked_rosters[roster_id].found); checked_rosters[roster_id].revision_refs++; if (!(rev.new_manifest == checked_rosters[roster_id].man_id)) checked_revisions[*i].manifest_mismatch = true; @@ -524,7 +537,6 @@ static void report_rosters(std::map, checked_roster> const & checked_rosters, - size_t & missing_rosters, size_t & unreferenced_rosters, size_t & incomplete_rosters, size_t & non_parseable_rosters, @@ -535,13 +547,6 @@ { checked_roster roster = i->second; - if (!roster.found) - { - missing_rosters++; - P(F("roster %s missing (%d revision references)\n") - % i->first % roster.revision_refs); - } - if (roster.revision_refs == 0) { unreferenced_rosters++; @@ -597,8 +602,9 @@ if (!revision.found) { missing_revisions++; - P(F("revision %s missing (%d revision references; %d cert references)\n") - % i->first % revision.revision_refs % revision.cert_refs); + P(F("revision %s missing (%d revision references; %d cert references; %d parent references; %d child references; %d roster references)\n") + % i->first % revision.revision_refs % revision.cert_refs % revision.ancestry_parent_refs + % revision.ancestry_child_refs % revision.marking_refs); } if (revision.missing_manifests > 0) @@ -818,7 +824,7 @@ report_files(checked_files, missing_files, unreferenced_files); report_rosters(checked_rosters, - missing_rosters, unreferenced_rosters, + unreferenced_rosters, incomplete_rosters, non_parseable_rosters, non_normalized_rosters); @@ -841,8 +847,6 @@ if (unreferenced_files > 0) W(F("%d unreferenced files\n") % unreferenced_files); - if (missing_rosters > 0) - W(F("%d missing rosters\n") % missing_rosters); if (unreferenced_rosters > 0) W(F("%d unreferenced rosters\n") % unreferenced_rosters); if (incomplete_rosters > 0) @@ -882,7 +886,7 @@ W(F("%d bad signatures\n") % bad_sigs); size_t total = missing_files + unreferenced_files + - missing_rosters + unreferenced_rosters + incomplete_rosters + + unreferenced_rosters + incomplete_rosters + non_parseable_rosters + non_normalized_rosters + missing_revisions + incomplete_revisions + non_parseable_revisions + non_normalized_revisions + @@ -894,7 +898,7 @@ // unreferenced files and rosters and mismatched certs are not actually // serious errors; odd, but nothing will break. size_t serious = missing_files + - missing_rosters + incomplete_rosters + + incomplete_rosters + non_parseable_rosters + non_normalized_rosters + missing_revisions + incomplete_revisions + non_parseable_revisions + non_normalized_revisions + ======================================================================== --- tests/t_database_check.at f9e93071aa9dd8bdf4643550b9a54ee87aca2fda +++ tests/t_database_check.at b034de4e58d344e3ac343a04f29bea851aa93717 @@ -11,37 +11,16 @@ AT_DATA(file3, [file 3 ]) -# fileX is not referenced by any manifest we have - -AT_DATA(fileX, [@<:@fdata ff43181b2988e27db1d63605bb3e39d4f06148cb@:>@ -H4sIAAAAAAAAAFNQUEjLzElVUEpJTcpMzNMvKs1JLVbiSq1ITS4tAQqXFJWmKnEBADUsgmsm -AAAA -@<:@end@:>@ +AT_DATA(fileX, [coopers ]) -# manifestX below contains the following 6 files, none of which are -# present in the database -# -# bb724c73e9f7f0dda411aa83131c648519dc0413 AUTHORS -# 3e9d284b3c48a939e82407d5c3fa6c9e0bf5efcb COPYING -# 74772a499b672e55cb8367ef580a484709189513 ChangeLog -# a9891352474f767c8d2eeba834a121e46f725cb2 INSTALL -# 6909265d83b02ab3c0abd887f0a6ac96db42323b Makefile.am -# 80536d31e18d514e52983a5dbac8e19d66b564dd NEWS - -AT_DATA(manifestX, [@<:@mdata 1845ba581ec220b33fd14c7ca389576c5b97e959@:>@ -H4sIAAAAAAAAAxWPy05DMQwF9/cr8gUo8SO2lxVCUKm0iBYhlk7iFMTr/3eE/dGcmdYEqAuG -TZl5DKdS3BULll5JudjomQqmtHu5PJyez9uaDlBq2End0EKBsgzuOL12i9wmx+wtpdvT09v+ -eL8JiYCTWasCwdybYpWYrNlJSbIVNf7/uH33n2scfq+bm1pBBhKaUqXrgIi2xMgLlKA6BRYI -Utofz5fd4bBVywaVh2LL4Esvexuqq8qrd6ujESDg8nr0z5gfX3Hj35tmxjqwRNHBhYLBFJ1H -866x4mttXGmMlI53r+ftD+i9lrUvAQAA -@<:@end@:>@ +AT_DATA(fileY, [vitamin ]) AT_CHECK(MONOTONE add file1, [], [ignore], [ignore]) AT_CHECK(MONOTONE commit --branch=test --message='add file1', [], [ignore], [ignore]) REV1=`BASE_REVISION` -MAN1=`monotone --norc automate get_manifest_of | monotone --norc identify` +ROST1=`monotone --norc db execute "select id from rosters"| tail -1` AT_CHECK(MONOTONE add file2, [], [ignore], [ignore]) AT_CHECK(MONOTONE commit --branch=test --message='add file2', [], [ignore], [ignore]) @@ -50,12 +29,13 @@ AT_CHECK(MONOTONE add file3, [], [ignore], [ignore]) AT_CHECK(MONOTONE commit --branch=test --message='add file3', [], [ignore], [ignore]) +FILE3=`SHA1(file3)` REV3=`BASE_REVISION` AT_CHECK(MONOTONE db check --ticker=dot, [], [ignore], [stderr]) AT_CHECK(grep 'database is good' stderr, [], [ignore], [ignore]) -# remove file2 from the database invalidating manifest2 and manifest3 +# remove file2 from the database invalidating roster2 and roster3 # both of which include this file AT_CHECK(MONOTONE db execute "delete from files where id='$FILE2'", [], [ignore], [ignore]) @@ -64,21 +44,29 @@ AT_CHECK(grep 'database is good' stderr, [1], [ignore], [ignore]) AT_CHECK(grep 'problems detected' stderr, [0], [ignore], [ignore]) AT_CHECK(grep '1 missing file' stderr, [0], [ignore], [ignore]) -AT_CHECK(grep '2 incomplete manifest' stderr, [0], [ignore], [ignore]) +AT_CHECK(grep '2 incomplete roster' stderr, [0], [ignore], [ignore]) AT_CHECK(grep '2 incomplete revision' stderr, [0], [ignore], [ignore]) AT_CHECK(grep 'total problems detected: 5' stderr, [0], [ignore], [ignore]) AT_CHECK(grep '5 serious' stderr, [0], [ignore], [ignore]) -# add an unreferenced file, and an unreferenced manifest that -# references several files, none of which exist in the db -AT_CHECK(MONOTONE read < fileX, [], [ignore], [ignore]) -AT_CHECK(MONOTONE read < manifestX, [], [ignore], [ignore]) +# add an unreferenced file +AT_CHECK(MONOTONE fload < fileX, [], [ignore], [ignore]) +# create an unreferenced roster by deleting the revision. Note that this will increment +# the "missing revision" count by one for further checks +AT_CHECK(MONOTONE add fileY, [], [ignore], [ignore]) +AT_CHECK(MONOTONE commit --branch=test --message='to be removed', [], [ignore], [ignore]) +REV4=`BASE_REVISION` +AT_CHECK(MONOTONE db execute "delete from revisions where id='$REV4'", [0], [ignore], [ignore]) +# revert to the old working copy state +AT_CHECK(echo $REV3 > MT/revision) +# remove another file too +AT_CHECK(MONOTONE db execute "delete from files where id='$FILE3'", [], [ignore], [ignore]) AT_CHECK(MONOTONE db check --ticker=dot, [1], [ignore], [stderr]) AT_CHECK(grep '1 unreferenced file' stderr, [0], [ignore], [ignore]) -AT_CHECK(grep '1 unreferenced manifest' stderr, [0], [ignore], [ignore]) -AT_CHECK(grep '7 missing files' stderr, [0], [ignore], [ignore]) +AT_CHECK(grep '1 unreferenced roster' stderr, [0], [ignore], [ignore]) +AT_CHECK(grep '2 missing files' stderr, [0], [ignore], [ignore]) AT_CHECK(MONOTONE db execute "delete from revision_ancestry", [], [ignore], [ignore]) AT_CHECK(MONOTONE db check --ticker=dot, [1], [ignore], [stderr]) @@ -94,17 +82,18 @@ AT_CHECK(MONOTONE db check --ticker=dot, [1], [ignore], [stderr]) AT_CHECK(grep '3 mismatched parent' stderr, [0], [ignore], [ignore]) AT_CHECK(grep '3 mismatched child' stderr, [0], [ignore], [ignore]) -AT_CHECK(grep '2 missing revision' stderr, [0], [ignore], [ignore]) +AT_CHECK(grep '3 missing revision' stderr, [0], [ignore], [ignore]) -AT_CHECK(MONOTONE db execute "delete from manifest_deltas where id='$MAN1'", [], [ignore], [ignore]) +AT_CHECK(MONOTONE db execute "delete from roster_deltas where id='$ROST1'", [], [ignore], [ignore]) AT_CHECK(MONOTONE db check --ticker=dot, [1], [ignore], [stderr]) -AT_CHECK(grep '1 missing manifest' stderr, [0], [ignore], [ignore]) -AT_CHECK(grep '3 revisions with bad history' stderr, [0], [ignore], [ignore]) +# ROSTER TODO: need check_sane_history equivalent in db check +#AT_CHECK(grep '3 revisions with bad history' stderr, [0], [ignore], [ignore]) AT_CHECK(MONOTONE db execute "delete from revisions where id='$REV1'", [], [ignore], [ignore]) AT_CHECK(MONOTONE db check --ticker=dot, [1], [ignore], [stderr]) -AT_CHECK(grep '3 missing revision' stderr, [0], [ignore], [ignore]) -AT_CHECK(grep '2 revisions with bad history' stderr, [0], [ignore], [ignore]) +AT_CHECK(grep '4 missing revision' stderr, [0], [ignore], [ignore]) +# ROSTER TODO +#AT_CHECK(grep '2 revisions with bad history' stderr, [0], [ignore], [ignore]) echo "$REV2:comment:this is a test:address@hidden:bogus sig" | sha1sum | read HASH @@ -115,11 +104,11 @@ AT_CHECK(MONOTONE db execute "delete from revision_certs where name = 'date'", [], [ignore], [stderr]) AT_CHECK(MONOTONE db check --ticker=dot, [1], [ignore], [stderr]) AT_CHECK(grep '2 missing certs' stderr, [0], [ignore], [ignore]) -AT_CHECK(grep '2 mismatched certs' stderr, [0], [ignore], [ignore]) +AT_CHECK(grep '4 mismatched certs' stderr, [0], [ignore], [ignore]) AT_CHECK(MONOTONE db execute "delete from public_keys", [], [ignore], [stderr]) AT_CHECK(MONOTONE db check --ticker=dot, [1], [ignore], [stderr]) AT_CHECK(grep '1 missing key' stderr, [0], [ignore], [ignore]) -AT_CHECK(grep '10 unchecked signatures' stderr, [0], [ignore], [ignore]) +AT_CHECK(grep '13 unchecked signatures' stderr, [0], [ignore], [ignore]) AT_CLEANUP ======================================================================== --- tests/t_database_check_minor.at c3716a80023cce652e10ebebb1df1e405a9d86ca +++ tests/t_database_check_minor.at fe8224086ed9df240a2ce9142c48e93ee95ace3f @@ -8,18 +8,7 @@ ]) AT_DATA(fileY, [stuff stuff ]) -# manifestX is: -# -# e3f2b0427b57241225ba1ffc2b67fecd64d07613 fileX -# -# where e3f2 etc. is as above -AT_DATA(manifestX, [@<:@mdata 8e5d7e5ffca2393cfca5625ac9c1a19a46489f92@:>@ -H4sIAAAAAAAAAEs1TjNKMjAxMk8yNTcyMTQyMk1KNExLSzZKMjNPS01OMTNJMTA3MzRWUEjL -zEmN4AIA90gN9zAAAAA= -@<:@end@:>@ -]) - ADD_FILE(testfile, [more stuff ]) COMMIT(testbranch) @@ -27,13 +16,23 @@ AT_CHECK(MONOTONE cert $REV author extra_author, [], [ignore], [ignore]) +# if we drop the file, we'll have a roster that doesn't +# reference its own revision. +# we can then remove the revision to end up with a clean unreferenced roster. +AT_CHECK(MONOTONE drop testfile, [], [ignore], [ignore]) +AT_CHECK(MONOTONE commit -m "goingaway", [], [ignore], [ignore]) +DEL_REV=`BASE_REVISION` +AT_CHECK(MONOTONE db execute "delete from revisions where id = '$DEL_REV'", [], [ignore], [ignore]) +AT_CHECK(MONOTONE db execute "delete from revision_certs where id = '$DEL_REV'", [], [ignore], [ignore]) +AT_CHECK(MONOTONE db execute "delete from revision_ancestry where child = '$DEL_REV'", [], [ignore], [ignore]) + +# and also a few unused files shall float about AT_CHECK(MONOTONE fload < fileX, [], [ignore], [ignore]) AT_CHECK(MONOTONE fload < fileY, [], [ignore], [ignore]) -AT_CHECK(MONOTONE read < manifestX, [], [ignore], [ignore]) AT_CHECK(MONOTONE db check, [], [ignore], [stderr]) -AT_CHECK(QGREP('problems detected: 3' stderr)) +AT_CHECK(QGREP('problems detected: 4' stderr)) AT_CHECK(QGREP('0 serious' stderr)) AT_CHECK(QGREP('minor problems detected' stderr))