# # # patch "ChangeLog" # from [0955d776c91093b8260a4f2fd5fb4e1ca23c880a] # to [e661d96285a0c54c6dd981314e3728edb0687cf3] # # patch "database.cc" # from [b682d2155d6293546402decc6f96e723bd18d7dc] # to [db1989a440fae91e066e3168c776af078ab531e8] # # patch "database.hh" # from [bbf8e39a0adaa28a4694549d26602b3712f6109c] # to [df8da3f0b1c8995d1e4e64200a13b26200b1d767] # # patch "schema_migration.cc" # from [68292293578bfaa6f996002650d4ffe13a29d216] # to [e1ea1f91c0830784152e630beb42a16a601d2a52] # # patch "schema_migration.hh" # from [c6044b2d2db646eb8b2a89fabc01d0ef869cc92a] # to [c5297d15ee9dfa8b96cb01a96cd6a811d80ef97c] # ============================================================ --- ChangeLog 0955d776c91093b8260a4f2fd5fb4e1ca23c880a +++ ChangeLog e661d96285a0c54c6dd981314e3728edb0687cf3 @@ -1,5 +1,24 @@ 2007-01-09 Zack Weinberg + * schema_migration.cc: Prune headers and using-declarations. Make + various things static and/or wrapped in the anonymous namespace. + (struct migrator): Delete. + (migrator_cb): Move definition to just above first use. + (struct migration_event, migration_events, n_migration_events) + (schema_to_migration, diagnose_unrecognized_schema, check_sql_schema): + New functions and data. + (migrate_monotone_schema): Put all the logic that used to be in + migrator::migrate here instead. Use two loops instead of one loop + and a state boolean, for clarity. + * schema_migration.hh: Prune headers. Forward-declare system_path. + Declare check_sql_schema. Adjust spacing. + * database.cc (database::database): Delete initialization of "schema". + (database::check_schema, close_all_databases): Delete. + (database::sql): Call check_sql_schema instead of check_schema. + * database.hh (database): Remove "schema" data-member and + check_schema function-member. + (close_all_databases): Delete declaration. + * transforms.hh (xform): Make the generic template produce a compile error if instantiated. Use standard declarations of partial specializations for the usable cases of xform, not the g++ ============================================================ --- database.cc b682d2155d6293546402decc6f96e723bd18d7dc +++ database.cc db1989a440fae91e066e3168c776af078ab531e8 @@ -127,13 +127,7 @@ database::database(system_path const & f database::database(system_path const & fn) : filename(fn), - // nb. update this if you change the schema. unfortunately we are not - // using self-digesting schemas due to comment irregularities and - // non-alphabetic ordering of tables in sql source files. we could create - // a temporary db, write our intended schema into it, and read it back, - // but this seems like it would be too rude. possibly revisit this issue. __sql(NULL), - schema("48fd5d84f1e5a949ca093e87e5ac558da6e5956d"), transaction_level(0), roster_cache(constants::db_roster_cache_sz, roster_writeback_manager(*this)), @@ -151,19 +145,6 @@ void } void -database::check_schema() -{ - string db_schema_id; - calculate_schema_id (__sql, db_schema_id); - N (schema == db_schema_id, - F("layout of database %s doesn't match this version of monotone\n" - "wanted schema %s, got %s\n" - "try '%s db migrate' to upgrade\n" - "(this is irreversible; you may want to make a backup copy first)") - % filename % schema % db_schema_id % ui.prog_name); -} - -void database::check_is_not_rosterified() { results res; @@ -318,7 +299,7 @@ database::sql(bool init, bool migrating_ assert_sqlite3_ok(__sql); } - check_schema(); + check_sql_schema(__sql, filename); install_functions(__app); if (!migrating_format) @@ -3135,27 +3116,6 @@ transaction_guard::commit() committed = true; } - - -// called to avoid foo.db-journal files hanging around if we exit cleanly -// without unwinding the stack (happens with SIGINT & SIGTERM) -void -close_all_databases() -{ - L(FL("attempting to rollback and close %d databases") % sql_contexts.size()); - for (set::iterator i = sql_contexts.begin(); - i != sql_contexts.end(); i++) - { - // the ROLLBACK is required here, even though the sqlite docs - // imply that transactions are rolled back on database closure - int exec_err = sqlite3_exec(*i, "ROLLBACK", NULL, NULL, NULL); - int close_err = sqlite3_close(*i); - - L(FL("exec_err = %d, close_err = %d") % exec_err % close_err); - } - sql_contexts.clear(); -} - // Local Variables: // mode: C++ // fill-column: 76 ============================================================ --- database.hh bbf8e39a0adaa28a4694549d26602b3712f6109c +++ database.hh df8da3f0b1c8995d1e4e64200a13b26200b1d767 @@ -97,8 +97,6 @@ private: void open(); void close(); - std::string const schema; - void check_schema(); void check_format(); public: @@ -624,11 +622,6 @@ public: void commit(); }; - -void -close_all_databases(); - - // Local Variables: // mode: C++ // fill-column: 76 ============================================================ --- schema_migration.cc 68292293578bfaa6f996002650d4ffe13a29d216 +++ schema_migration.cc e1ea1f91c0830784152e630beb42a16a601d2a52 @@ -7,21 +7,12 @@ // implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR // PURPOSE. -#include -#include -#include -#include -#include -#include -#include - #include - #include +#include "botan/botan.h" #include "sanity.hh" #include "schema_migration.hh" -#include "botan/botan.h" #include "app_state.hh" #include "keys.hh" #include "transforms.hh" @@ -30,12 +21,8 @@ using std::map; using std::ctype; using std::locale; using std::map; -using std::pair; -using std::remove_if; -using std::string; using std::use_facet; using std::vector; - using boost::char_separator; // this file knows how to migrate schema databases. the general strategy is @@ -107,7 +94,7 @@ calculate_id(string const & in, ident = lowercase_facet(p.read_all_as_string()); } - +namespace { struct is_ws { @@ -116,6 +103,7 @@ is_ws return c == '\r' || c == '\n' || c == '\t' || c == ' '; } }; +} static void sqlite_sha1_fn(sqlite3_context *f, int nargs, sqlite3_value ** args) @@ -150,7 +138,7 @@ sqlite_sha1_fn(sqlite3_context *f, int n sqlite3_result_text(f,sha.c_str(),sha.size(),SQLITE_TRANSIENT); } -int +static int append_sql_stmt(void * vp, int ncols, char ** values, @@ -225,137 +213,6 @@ set_regime(upgrade_regime new_regime, up regime = std::min(new_regime, regime); } -typedef bool (*migrator_cb)(sqlite3 *, char **, app_state *, upgrade_regime &); - -struct -migrator -{ - vector< pair > migration_events; - app_state * __app; - - void set_app(app_state *app) - { - __app = app; - } - - void add(string schema_id, migrator_cb cb) - { - migration_events.push_back(make_pair(schema_id, cb)); - } - - void migrate(sqlite3 *sql, string target_id) - { - string init; - - I(sql != NULL); - - calculate_schema_id(sql, init); - - I(!sqlite3_create_function(sql, "sha1", -1, SQLITE_UTF8, NULL, - &sqlite_sha1_fn, NULL, NULL)); - - - P(F("calculating necessary migration steps")); - - upgrade_regime regime = upgrade_none; - - bool migrating = false; - for (vector< pair >::const_iterator i = migration_events.begin(); - i != migration_events.end(); ++i) - { - - if (i->first == init) - { - E(logged_sqlite3_exec(sql, "BEGIN EXCLUSIVE", NULL, NULL, NULL) == SQLITE_OK, - F("error at transaction BEGIN statement")); - P(F("migrating data")); - migrating = true; - } - - if (migrating) - { - // confirm that we are where we ought to be - string curr; - char *errmsg = NULL; - calculate_schema_id(sql, curr); - if (curr != i->first) - { - logged_sqlite3_exec(sql, "ROLLBACK", NULL, NULL, NULL); - I(false); - } - - if (i->second == NULL) - { - logged_sqlite3_exec(sql, "ROLLBACK", NULL, NULL, NULL); - I(false); - } - - // do this migration step - else if (! i->second(sql, &errmsg, __app, regime)) - { - logged_sqlite3_exec(sql, "ROLLBACK", NULL, NULL, NULL); - E(false, F("migration step failed: %s") - % (errmsg ? errmsg : "unknown error")); - } - } - } - - // confirm that our target schema was met - if (migrating) - { - string curr; - calculate_schema_id(sql, curr); - if (curr != target_id) - { - logged_sqlite3_exec(sql, "ROLLBACK", NULL, NULL, NULL); - E(false, F("mismatched result of migration, got %s, wanted %s") - % curr % target_id); - } - P(F("committing changes to database")); - E(logged_sqlite3_exec(sql, "COMMIT", NULL, NULL, NULL) == SQLITE_OK, - F("failure on COMMIT")); - - P(F("optimizing database")); - E(logged_sqlite3_exec(sql, "VACUUM", NULL, NULL, NULL) == SQLITE_OK, - F("error vacuuming after migration")); - - switch (regime) - { - case upgrade_changesetify: - case upgrade_rosterify: - { - string command_str = (regime == upgrade_changesetify - ? "changesetify" : "rosterify"); - P(F("NOTE: because this database was last used by a rather old version\n" - "of monotone, you're not done yet. If you're a project leader, then\n" - "see the file UPGRADE for instructions on running '%s db %s'") - % ui.prog_name % command_str); - } - break; - case upgrade_regen_caches: - P(F("NOTE: this upgrade cleared monotone's caches\n" - "you should now run '%s db regenerate_caches'") - % ui.prog_name); - break; - case upgrade_none: - break; - } - } - else - { - // if we didn't do anything, make sure that it's because we were - // already up to date. - E(init == target_id, - F("database schema %s is unknown; cannot perform migration") % init); - // We really want 'db migrate' on an up-to-date schema to be a no-op - // (no vacuum or anything, even), so that automated scripts can fire - // one off optimistically and not have to worry about getting their - // administrators to do it by hand. - P(F("no migration performed; database schema already up-to-date at %s") % init); - } - } -}; - static bool move_table(sqlite3 *sql, char **errmsg, char const * srcname, char const * dstname, @@ -1179,53 +1036,230 @@ migrate_add_heights(sqlite3 *sql, return true; } +typedef bool (*migrator_cb)(sqlite3 *, char **, app_state *, upgrade_regime &); +struct migration_event +{ + char const * id; + migrator_cb migrator; +}; + +// IMPORTANT: whenever you modify this to add a new schema version, you must +// also add a new migration test for the new schema version. See +// tests/schema_migration for details. + +const migration_event migration_events[] = { + { "edb5fa6cef65bcb7d0c612023d267c3aeaa1e57a", + migrate_client_merge_url_and_group }, + + { "f042f3c4d0a4f98f6658cbaf603d376acf88ff4b", + migrate_client_add_hashes_and_merkle_trees }, + + { "8929e54f40bf4d3b4aea8b037d2c9263e82abdf4", + migrate_client_to_revisions }, + + { "c1e86588e11ad07fa53e5d294edc043ce1d4005a", + migrate_client_to_epochs }, + + { "40369a7bda66463c5785d160819ab6398b9d44f4", + migrate_client_to_vars }, + + { "e372b508bea9b991816d1c74680f7ae10d2a6d94", + migrate_client_to_add_indexes }, + + { "1509fd75019aebef5ac3da3a5edf1312393b70e9", + migrate_client_to_external_privkeys }, + + { "bd86f9a90b5d552f0be1fa9aee847ea0f317778b", + migrate_client_to_add_rosters }, + + { "1db80c7cee8fa966913db1a463ed50bf1b0e5b0e", + migrate_files_BLOB }, + + { "9d2b5d7b86df00c30ac34fe87a3c20f1195bb2df", + migrate_rosters_no_hash }, + + { "ae196843d368d042f475e3dadfed11e9d7f9f01e", + migrate_add_heights }, + + // The last entry in this table should always be the current + // schema ID, with 0 for the migrator. + + { "48fd5d84f1e5a949ca093e87e5ac558da6e5956d", 0 } +}; +const size_t n_migration_events = (sizeof migration_events + / sizeof migration_events[0]); + +// Look through the migration_events table and return the index of the +// entry corresponding to schema ID, or -1 if it isn't there (i.e. if +// the database schema is not one we know). +static int +schema_to_migration(string const & id) +{ + int i; + for (i = n_migration_events - 1; i >= 0; i--) + if (migration_events[i].id == id) + break; + + return i; +} + +// Provide sensible diagnostics for a database schema whose hash we do not +// recognize. +static void NORETURN +diagnose_unrecognized_schema(sqlite3 * sql, system_path const & filename, + string const & id) +{ + // Give a special message for an utterly empty sqlite3 database, such as + // is created by "mtn db load < /dev/null", or by the sqlite3 command line + // utility if you don't give it anything to do. + N(id != "da39a3ee5e6b4b0d3255bfef95601890afd80709", + F("cannot use the empty sqlite database %s\n" + "(monotone databases must be created with '%s db init')") + % filename % ui.prog_name); + + // Do a sanity check to make sure we are actually looking at a monotone + // database, not some other sqlite3 database. Every version of the schema + // has included tables named 'files', 'file_deltas', 'manifests', and + // 'manifest_deltas'. + // ??? Use PRAGMA user_version to record an additional magic number in + // monotone databases. + string tmp; + logged_sqlite3_exec(sql, + "SELECT COUNT(*) FROM sqlite_master " + "WHERE type = 'table' AND sql IS NOT NULL " + "AND (name = 'files' OR name = 'file_deltas'" + " OR name = 'manifests'" + " OR name = 'manifest_deltas')", + append_sql_stmt, &tmp, 0); + N(tmp == "4\n", + F("%s does not appear to be a monotone database\n" + "(schema %s, core tables missing)") + % filename % id); + + N(false, + F("%s appears to be a monotone database, but this version of\n" + "monotone does not recognize its schema (%s).\n" + "you probably need a newer version of monotone.") + % filename % id); +} + +// check_sql_schema is called by database.cc on open, to determine whether +// the schema is up to date. If it returns at all, the schema is indeed +// up to date (otherwise it throws a diagnostic). void -migrate_monotone_schema(sqlite3 *sql, app_state *app) +check_sql_schema(sqlite3 * sql, system_path const & filename) { + I(sql != NULL); + + string id; + calculate_schema_id(sql, id); + + int migration = schema_to_migration(id); + + if (migration == -1) + diagnose_unrecognized_schema(sql, filename, id); + + N(migration_events[migration].migrator == 0, + F("database %s is laid out according to an old schema, %s\n" + "try '%s db migrate' to upgrade\n" + "(this is irreversible; you may want to make a backup copy first)") + % filename % id % ui.prog_name); +} - migrator m; - m.set_app(app); +void +migrate_monotone_schema(sqlite3 * sql, app_state * app) +{ + I(sql != NULL); + I(!sqlite3_create_function(sql, "sha1", -1, SQLITE_UTF8, NULL, + &sqlite_sha1_fn, NULL, NULL)); - m.add("edb5fa6cef65bcb7d0c612023d267c3aeaa1e57a", - &migrate_client_merge_url_and_group); + string init; + calculate_schema_id(sql, init); - m.add("f042f3c4d0a4f98f6658cbaf603d376acf88ff4b", - &migrate_client_add_hashes_and_merkle_trees); + P(F("calculating migration for schema %s") % init); - m.add("8929e54f40bf4d3b4aea8b037d2c9263e82abdf4", - &migrate_client_to_revisions); + int i = schema_to_migration(init); - m.add("c1e86588e11ad07fa53e5d294edc043ce1d4005a", - &migrate_client_to_epochs); + if (i == -1) + diagnose_unrecognized_schema(sql, app->db.get_filename(), init); - m.add("40369a7bda66463c5785d160819ab6398b9d44f4", - &migrate_client_to_vars); + // We really want 'db migrate' on an up-to-date schema to be a no-op + // (no vacuum or anything, even), so that automated scripts can fire + // one off optimistically and not have to worry about getting their + // administrators to do it by hand. + if (migration_events[i].migrator == 0) + { + P(F("no migration performed; database schema already up-to-date")); + return; + } - m.add("e372b508bea9b991816d1c74680f7ae10d2a6d94", - &migrate_client_to_add_indexes); + upgrade_regime regime = upgrade_none; + P(F("migrating data")); - m.add("1509fd75019aebef5ac3da3a5edf1312393b70e9", - &migrate_client_to_external_privkeys); + E(logged_sqlite3_exec(sql, "BEGIN EXCLUSIVE", NULL, NULL, NULL) == SQLITE_OK, + F("error at transaction BEGIN statement")); - m.add("bd86f9a90b5d552f0be1fa9aee847ea0f317778b", - &migrate_client_to_add_rosters); + for (;;) + { + // confirm that we are where we ought to be + string curr; + calculate_schema_id(sql, curr); + if (curr != migration_events[i].id) + { + logged_sqlite3_exec(sql, "ROLLBACK", NULL, NULL, NULL); + I(false); + } - m.add("1db80c7cee8fa966913db1a463ed50bf1b0e5b0e", - &migrate_files_BLOB); + if (migration_events[i].migrator == 0) + break; - m.add("9d2b5d7b86df00c30ac34fe87a3c20f1195bb2df", - &migrate_rosters_no_hash); + char * errmsg; + if (! migration_events[i].migrator(sql, &errmsg, app, regime)) + { + logged_sqlite3_exec(sql, "ROLLBACK", NULL, NULL, NULL); + E(false, F("migration step failed: %s") + % (errmsg ? errmsg : "unknown error")); + } - m.add("ae196843d368d042f475e3dadfed11e9d7f9f01e", - &migrate_add_heights); + i++; + if ((size_t)i >= n_migration_events) + { + logged_sqlite3_exec(sql, "ROLLBACK", NULL, NULL, NULL); + I(false); + } + } - // IMPORTANT: whenever you modify this to add a new schema version, you must - // also add a new migration test for the new schema version. See - // tests/t_migrate_schema.at for details. + P(F("committing changes to database")); + E(logged_sqlite3_exec(sql, "COMMIT", NULL, NULL, NULL) == SQLITE_OK, + F("failure on COMMIT")); - m.migrate(sql, "48fd5d84f1e5a949ca093e87e5ac558da6e5956d"); + P(F("optimizing database")); + logged_sqlite3_exec(sql, "VACUUM", NULL, NULL, NULL); + + switch (regime) + { + case upgrade_changesetify: + case upgrade_rosterify: + { + string command_str = (regime == upgrade_changesetify + ? "changesetify" : "rosterify"); + P(F("NOTE: because this database was last used by a rather old version\n" + "of monotone, you're not done yet. If you're a project leader, then\n" + "see the file UPGRADE for instructions on running '%s db %s'") + % ui.prog_name % command_str); + } + break; + case upgrade_regen_caches: + P(F("NOTE: this upgrade cleared monotone's caches\n" + "you should now run '%s db regenerate_caches'") + % ui.prog_name); + break; + case upgrade_none: + break; + } } + // Local Variables: // mode: C++ // fill-column: 76 ============================================================ --- schema_migration.hh c6044b2d2db646eb8b2a89fabc01d0ef869cc92a +++ schema_migration.hh c5297d15ee9dfa8b96cb01a96cd6a811d80ef97c @@ -11,7 +11,6 @@ // PURPOSE. #include -#include // this file knows how to migrate schema databases. the general strategy is // to hash each schema we ever use, and make a list of the SQL commands @@ -22,9 +21,11 @@ class app_state; struct sqlite3; class app_state; +class system_path; -void calculate_schema_id(sqlite3 *sql, std::string & id); -void migrate_monotone_schema(sqlite3 *sql, app_state *app); +void calculate_schema_id(sqlite3 * sql, std::string & id); +void migrate_monotone_schema(sqlite3 * sql, app_state * app); +void check_sql_schema(sqlite3 * sql, system_path const & filename); // Local Variables: // mode: C++