# # # patch "cmd_list.cc" # from [435da069e301c3791db07a6d9596257aa0d3fa25] # to [d673c46c744088510cfd0fc8b8b972f94708d52d] # # patch "database.cc" # from [3d56cf585bb8ed3bb3589e5b8fc5f548a8d374dd] # to [a25d2123c4050d3dbbbca0852cbc8d347cbad3c8] # # patch "database.hh" # from [cb2f020ae339ce2fccb1826b8b8b6553571b4354] # to [2ed0fd6ca1ba6c5269e446120df198dfedc970c2] # # patch "database_check.cc" # from [27daa8b1b77f291237f4ed84eacd9f2e354ac810] # to [4a77f59692f47b501a73ede9369b8a5923c161d7] # # patch "revision.cc" # from [47b8a2189742c9513c3234e6315551335c8a8697] # to [5cf8962282b67f85579d9ed30ce2b074f302f002] # # patch "update.cc" # from [5a40558d142d4ec20ff1ec6b7ef08e0be92d902f] # to [260394c808cc8f1c9c783a16274f60b8cbaed276] # ============================================================ --- cmd_list.cc 435da069e301c3791db07a6d9596257aa0d3fa25 +++ cmd_list.cc d673c46c744088510cfd0fc8b8b972f94708d52d @@ -9,7 +9,7 @@ #include "base.hh" #include -#include +#include "safe_map.hh" #include #include #include ============================================================ --- database.cc 3d56cf585bb8ed3bb3589e5b8fc5f548a8d374dd +++ database.cc a25d2123c4050d3dbbbca0852cbc8d347cbad3c8 @@ -48,6 +48,8 @@ #include "rev_height.hh" #include "vocab_hash.hh" #include "globish.hh" +#include "outdated_indicator.hh" +#include "lru_writeback_cache.hh" // defined in schema.c, generated from schema.sql: extern char const schema_constant[]; @@ -106,58 +108,336 @@ namespace }; return q; } -} -struct query + struct query + { + explicit query(string const & cmd) + : sql_cmd(cmd) + {} + + query() + {} + + query & operator %(query_param const & qp) + { + args.push_back(qp); + return *this; + } + + vector args; + string sql_cmd; + }; + + typedef vector< vector > results; + + struct statement + { + statement() : count(0), stmt(0, sqlite3_finalize) {} + int count; + cleanup_ptr stmt; + }; + + struct roster_size_estimator + { + unsigned long operator() (database::cached_roster const & cr) + { + I(cr.first); + I(cr.second); + // do estimate using a totally made up multiplier, probably wildly off + return (cr.first->all_nodes().size() + * constants::db_estimated_roster_node_sz); + } + }; + + struct datasz + { + unsigned long operator()(data const & t) { return t().size(); } + }; + + enum open_mode { normal_mode = 0, + schema_bypass_mode, + format_bypass_mode }; + + typedef hashmap::hash_map > parent_id_map; + typedef hashmap::hash_map height_map; + +} // anonymous namespace + +class database_impl { - explicit query(string const & cmd) - : sql_cmd(cmd) - {} + friend class database; - query() - {} + database_impl(system_path const & fn); + ~database_impl(); - query & operator %(query_param const & qp) + // + // --== Opening the database and schema checking ==-- + // + system_path filename; + struct sqlite3 * __sql; + + void install_functions(); + struct sqlite3 * sql(enum open_mode mode = normal_mode); + + void check_filename(); + void check_db_exists(); + void check_db_nonexistent(); + void open(); + void close(); + void check_format(); + + // + // --== Basic SQL interface and statement caching ==-- + // + map statement_cache; + + void fetch(results & res, + int const want_cols, int const want_rows, + query const & q); + void execute(query const & q); + + bool table_has_entry(string const & key, string const & column, + string const & table); + + // + // --== Generic database metadata gathering ==-- + // + string count(string const & table); + string space(string const & table, + string const & concatenated_columns, + u64 & total); + unsigned int page_size(); + unsigned int cache_size(); + + // + // --== Transactions ==-- + // + int transaction_level; + bool transaction_exclusive; + void begin_transaction(bool exclusive); + void commit_transaction(); + void rollback_transaction(); + friend class transaction_guard; + + struct roster_writeback_manager { - args.push_back(qp); - return *this; - } + database_impl & imp; + roster_writeback_manager(database_impl & imp) : imp(imp) {} + void writeout(revision_id const &, database::cached_roster const &); + }; + LRUWritebackCache + roster_cache; - vector args; - string sql_cmd; + bool have_delayed_file(file_id const & id); + void load_delayed_file(file_id const & id, file_data & dat); + void cancel_delayed_file(file_id const & id); + void drop_or_cancel_file(file_id const & id); + void schedule_delayed_file(file_id const & id, file_data const & dat); + + map delayed_files; + size_t delayed_writes_size; + + void flush_delayed_writes(); + void clear_delayed_writes(); + void write_delayed_file(file_id const & new_id, + file_data const & dat); + + void write_delayed_roster(revision_id const & new_id, + roster_t const & roster, + marking_map const & marking); + + // + // --== Reading/writing delta-compressed objects ==-- + // + + // "do we have any entry for 'ident' that is a base version" + bool roster_base_stored(revision_id const & ident); + bool roster_base_available(revision_id const & ident); + + // "do we have any entry for 'ident' that is a delta" + bool delta_exists(string const & ident, + string const & base, + string const & table); + + bool file_or_manifest_base_exists(hexenc const & ident, + std::string const & table); + + void get_file_or_manifest_base_unchecked(hexenc const & new_id, + data & dat, + string const & table); + void get_file_or_manifest_delta_unchecked(hexenc const & ident, + hexenc const & base, + delta & del, + string const & table); + void get_roster_base(string const & ident, + roster_t & roster, marking_map & marking); + void get_roster_delta(string const & ident, + string const & base, + roster_delta & del); + + friend struct file_and_manifest_reconstruction_graph; + friend struct roster_reconstruction_graph; + + LRUWritebackCache vcache; + + void get_version(hexenc const & ident, + data & dat, + string const & data_table, + string const & delta_table); + + void drop(string const & base, + string const & table); + void put_file_delta(file_id const & ident, + file_id const & base, + file_delta const & del); + + void put_roster_delta(revision_id const & ident, + revision_id const & base, + roster_delta const & del); + void put_version(hexenc const & old_id, + hexenc const & new_id, + delta const & del, + string const & data_table, + string const & delta_table); + + // + // --== The ancestry graph ==-- + // + void get_ids(string const & table, set< hexenc > & ids); + + // + // --== Rosters ==-- + // + struct extractor; + struct file_content_extractor; + struct markings_extractor; + void extract_from_deltas(revision_id const & id, extractor & x); + + height_map height_cache; + parent_id_map parent_cache; + + // + // --== Keys ==-- + // + void get_keys(string const & table, vector & keys); + + // + // --== Certs ==-- + // + // note: this section is ridiculous. please do something about it. + bool cert_exists(cert const & t, + string const & table); + void put_cert(cert const & t, string const & table); + void results_to_certs(results const & res, + vector & certs); + + void get_certs(vector & certs, + string const & table); + + void get_certs(hexenc const & ident, + vector & certs, + string const & table); + + void get_certs(cert_name const & name, + vector & certs, + string const & table); + + void get_certs(hexenc const & ident, + cert_name const & name, + vector & certs, + string const & table); + + void get_certs(hexenc const & ident, + cert_name const & name, + base64 const & val, + vector & certs, + string const & table); + + void get_certs(cert_name const & name, + base64 const & val, + vector & certs, + string const & table); + + outdated_indicator_factory cert_stamper; + }; -database::database(system_path const & fn) : +database_impl::database_impl(system_path const & fn) : filename(fn), __sql(NULL), transaction_level(0), roster_cache(constants::db_roster_cache_sz, roster_writeback_manager(*this)), - delayed_writes_size(0) + delayed_writes_size(0), + vcache(constants::db_version_cache_sz) {} +database_impl::~database_impl() +{ + L(FL("statement cache statistics")); + L(FL("prepared %d statements") % statement_cache.size()); + + for (map::const_iterator i = statement_cache.begin(); + i != statement_cache.end(); ++i) + L(FL("%d executions of %s") % i->second.count % i->first); + // trigger destructors to finalize cached statements + statement_cache.clear(); + + if (__sql) + close(); +} + +database::database(system_path const & fn) + : imp(new database_impl(fn)) +{} + +database::~database() +{ + delete imp; +} + +void +database::set_filename(system_path const & file) +{ + I(!imp->__sql); + imp->filename = file; +} + +system_path +database::get_filename() +{ + return imp->filename; +} + bool database::is_dbfile(any_path const & file) { - system_path fn(file);// why is this needed? - bool same = (filename.as_internal() == fn.as_internal()); + system_path fn(file); // canonicalize + bool same = (imp->filename == fn); if (same) L(FL("'%s' is the database file") % file); return same; } +bool +database::database_specified() +{ + return !imp->filename.empty(); +} + void database::check_is_not_rosterified() { results res; - string rosters_query = "SELECT 1 FROM rosters LIMIT 1"; - fetch(res, one_col, any_rows, query(rosters_query)); + imp->fetch(res, one_col, any_rows, + query("SELECT 1 FROM rosters LIMIT 1")); N(res.empty(), F("this database already contains rosters")); } void -database::check_format() +database_impl::check_format() { results res; @@ -222,14 +502,8 @@ sqlite3_gunzip_fn(sqlite3_context *f, in sqlite3_result_blob(f, unpacked().c_str(), unpacked().size(), SQLITE_TRANSIENT); } -void -database::set_app(app_state * app) -{ - __app = app; -} - struct sqlite3 * -database::sql(enum open_mode mode) +database_impl::sql(enum open_mode mode) { if (! __sql) { @@ -256,21 +530,23 @@ database::initialize() void database::initialize() { - check_filename(); - check_db_nonexistent(); - open(); + imp->check_filename(); + imp->check_db_nonexistent(); + imp->open(); - sqlite3_exec(__sql, schema_constant, NULL, NULL, NULL); - assert_sqlite3_ok(__sql); + sqlite3 *sql = imp->__sql; - sqlite3_exec(__sql, (FL("PRAGMA user_version = %u;") - % mtn_creator_code).str().c_str(), NULL, NULL, NULL); - assert_sqlite3_ok(__sql); + sqlite3_exec(sql, schema_constant, NULL, NULL, NULL); + assert_sqlite3_ok(sql); + sqlite3_exec(sql, (FL("PRAGMA user_version = %u;") + % mtn_creator_code).str().c_str(), NULL, NULL, NULL); + assert_sqlite3_ok(sql); + // make sure what we wanted is what we got - check_sql_schema(__sql, filename); - - close(); + check_sql_schema(sql, imp->filename); + + imp->close(); } struct @@ -393,7 +669,7 @@ database::dump(ostream & out) transaction_guard guard(*this); dump_request req; req.out = &out; - req.sql = sql(); + req.sql = imp->sql(); out << "BEGIN EXCLUSIVE;\n"; int res; res = sqlite3_exec(req.sql, @@ -424,13 +700,15 @@ database::load(istream & in) string line; string sql_stmt; - check_filename(); - check_db_nonexistent(); - open(); + imp->check_filename(); + imp->check_db_nonexistent(); + imp->open(); + sqlite3 * sql = imp->__sql; + // the page size can only be set before any other commands have been executed - sqlite3_exec(__sql, "PRAGMA page_size=8192", NULL, NULL, NULL); - assert_sqlite3_ok(__sql); + sqlite3_exec(sql, "PRAGMA page_size=8192", NULL, NULL, NULL); + assert_sqlite3_ok(sql); while(in) { @@ -439,13 +717,13 @@ database::load(istream & in) if (sqlite3_complete(sql_stmt.c_str())) { - sqlite3_exec(__sql, sql_stmt.c_str(), NULL, NULL, NULL); - assert_sqlite3_ok(__sql); + sqlite3_exec(sql, sql_stmt.c_str(), NULL, NULL, NULL); + assert_sqlite3_ok(sql); sql_stmt.clear(); } } - assert_sqlite3_ok(__sql); + assert_sqlite3_ok(sql); } @@ -455,7 +733,7 @@ database::debug(string const & sql, ostr ensure_open_for_maintenance(); results res; - fetch(res, any_cols, any_rows, query(sql)); + imp->fetch(res, any_cols, any_rows, query(sql)); out << '\'' << sql << "' -> " << res.size() << " rows\n\n"; for (size_t i = 0; i < res.size(); ++i) { @@ -545,31 +823,31 @@ database::info(ostream & out) // query of the user version because there's a bug in sqlite 3.3.10: // the routine that reads meta-values from the database header does not // check the file format. reported as sqlite bug #2182.) - sqlite3_exec(__sql, "SELECT 1 FROM sqlite_master LIMIT 0", 0, 0, 0); - assert_sqlite3_ok(__sql); + sqlite3_exec(imp->__sql, "SELECT 1 FROM sqlite_master LIMIT 0", 0, 0, 0); + assert_sqlite3_ok(imp->__sql); u32 ccode; { results res; - fetch(res, one_col, one_row, query("PRAGMA user_version")); + imp->fetch(res, one_col, one_row, query("PRAGMA user_version")); I(res.size() == 1); ccode = lexical_cast(res[0][0]); } vector counts; - counts.push_back(count("rosters")); - counts.push_back(count("roster_deltas")); - counts.push_back(count("files")); - counts.push_back(count("file_deltas")); - counts.push_back(count("revisions")); - counts.push_back(count("revision_ancestry")); - counts.push_back(count("revision_certs")); + counts.push_back(imp->count("rosters")); + counts.push_back(imp->count("roster_deltas")); + counts.push_back(imp->count("files")); + counts.push_back(imp->count("file_deltas")); + counts.push_back(imp->count("revisions")); + counts.push_back(imp->count("revision_ancestry")); + counts.push_back(imp->count("revision_certs")); { results res; try { - fetch(res, one_col, any_rows, + imp->fetch(res, one_col, any_rows, query("SELECT node FROM next_roster_node_number")); if (res.empty()) counts.push_back("0"); @@ -589,23 +867,23 @@ database::info(ostream & out) vector bytes; { u64 total = 0; - bytes.push_back(space("rosters", + bytes.push_back(imp->space("rosters", "length(id) + length(checksum) + length(data)", total)); - bytes.push_back(space("roster_deltas", + bytes.push_back(imp->space("roster_deltas", "length(id) + length(checksum)" "+ length(base) + length(delta)", total)); - bytes.push_back(space("files", "length(id) + length(data)", total)); - bytes.push_back(space("file_deltas", + bytes.push_back(imp->space("files", "length(id) + length(data)", total)); + bytes.push_back(imp->space("file_deltas", "length(id) + length(base) + length(delta)", total)); - bytes.push_back(space("revisions", "length(id) + length(data)", total)); - bytes.push_back(space("revision_ancestry", + bytes.push_back(imp->space("revisions", "length(id) + length(data)", total)); + bytes.push_back(imp->space("revision_ancestry", "length(parent) + length(child)", total)); - bytes.push_back(space("revision_certs", + bytes.push_back(imp->space("revision_certs", "length(hash) + length(id) + length(name)" "+ length(value) + length(keypair)" "+ length(signature)", total)); - bytes.push_back(space("heights", "length(revision) + length(height)", + bytes.push_back(imp->space("heights", "length(revision) + length(height)", total)); bytes.push_back((F("%u") % total).str()); } @@ -653,7 +931,7 @@ database::info(ostream & out) ); form = form % format_creator_code(ccode); - form = form % describe_sql_schema(__sql); + form = form % describe_sql_schema(imp->__sql); for (vector::iterator i = counts.begin(); i != counts.end(); i++) form = form % *i; @@ -661,8 +939,8 @@ database::info(ostream & out) for (vector::iterator i = bytes.begin(); i != bytes.end(); i++) form = form % *i; - form = form % page_size(); - form = form % cache_size(); + form = form % imp->page_size(); + form = form % imp->cache_size(); out << form.str() << '\n'; // final newline is kept out of the translation } @@ -671,7 +949,8 @@ database::version(ostream & out) database::version(ostream & out) { ensure_open_for_maintenance(); - out << (F("database schema version: %s") % describe_sql_schema(__sql)).str() + out << (F("database schema version: %s") + % describe_sql_schema(imp->__sql)).str() << '\n'; } @@ -679,61 +958,46 @@ database::migrate(key_store & keys) database::migrate(key_store & keys) { ensure_open_for_maintenance(); - migrate_sql_schema(__sql, get_filename(), keys); + migrate_sql_schema(imp->__sql, get_filename(), keys); } void database::test_migration_step(string const & schema, key_store & keys) { ensure_open_for_maintenance(); - ::test_migration_step(__sql, get_filename(), keys, schema); + ::test_migration_step(imp->__sql, get_filename(), keys, schema); } void database::ensure_open() { - sql(); + imp->sql(); } void database::ensure_open_for_format_changes() { - sql(format_bypass_mode); + imp->sql(format_bypass_mode); } void database::ensure_open_for_maintenance() { - sql(schema_bypass_mode); + imp->sql(schema_bypass_mode); } -database::~database() -{ - L(FL("statement cache statistics")); - L(FL("prepared %d statements") % statement_cache.size()); - - for (map::const_iterator i = statement_cache.begin(); - i != statement_cache.end(); ++i) - L(FL("%d executions of %s") % i->second.count % i->first); - // trigger destructors to finalize cached statements - statement_cache.clear(); - - if (__sql) - close(); -} - void -database::execute(query const & query) +database_impl::execute(query const & query) { results res; fetch(res, 0, 0, query); } void -database::fetch(results & res, - int const want_cols, - int const want_rows, - query const & query) +database_impl::fetch(results & res, + int const want_cols, + int const want_rows, + query const & query) { int nrow; int ncol; @@ -850,8 +1114,9 @@ bool } bool -database::table_has_entry(std::string const & key, std::string const & column, - std::string const & table) +database_impl::table_has_entry(string const & key, + string const & column, + string const & table) { results res; query q("SELECT 1 FROM " + table + " WHERE " + column + " = ? LIMIT 1"); @@ -862,21 +1127,8 @@ void // general application-level logic void -database::set_filename(system_path const & file) +database_impl::begin_transaction(bool exclusive) { - I(!__sql); - filename = file; -} - -system_path -database::get_filename() -{ - return filename; -} - -void -database::begin_transaction(bool exclusive) -{ if (transaction_level == 0) { I(delayed_files.empty()); @@ -897,27 +1149,27 @@ database::begin_transaction(bool exclusi } -size_t -database::size_delayed_file(file_id const & id, file_data const & dat) +static size_t +size_delayed_file(file_id const & id, file_data const & dat) { return id.inner()().size() + dat.inner()().size(); } bool -database::have_delayed_file(file_id const & id) +database_impl::have_delayed_file(file_id const & id) { return delayed_files.find(id) != delayed_files.end(); } void -database::load_delayed_file(file_id const & id, file_data & dat) +database_impl::load_delayed_file(file_id const & id, file_data & dat) { dat = safe_get(delayed_files, id); } // precondition: have_delayed_file(an_id) == true void -database::cancel_delayed_file(file_id const & an_id) +database_impl::cancel_delayed_file(file_id const & an_id) { file_data const & dat = safe_get(delayed_files, an_id); size_t cancel_size = size_delayed_file(an_id, dat); @@ -928,7 +1180,7 @@ void } void -database::drop_or_cancel_file(file_id const & id) +database_impl::drop_or_cancel_file(file_id const & id) { if (have_delayed_file(id)) cancel_delayed_file(id); @@ -937,8 +1189,8 @@ void } void -database::schedule_delayed_file(file_id const & an_id, - file_data const & dat) +database_impl::schedule_delayed_file(file_id const & an_id, + file_data const & dat) { if (!have_delayed_file(an_id)) { @@ -950,7 +1202,7 @@ void } void -database::flush_delayed_writes() +database_impl::flush_delayed_writes() { for (map::const_iterator i = delayed_files.begin(); i != delayed_files.end(); ++i) @@ -959,32 +1211,23 @@ void } void -database::clear_delayed_writes() +database_impl::clear_delayed_writes() { delayed_files.clear(); delayed_writes_size = 0; } void -database::roster_writeback_manager::writeout(revision_id const & id, - cached_roster const & cr) +database_impl::roster_writeback_manager::writeout(revision_id const & id, + database::cached_roster const & cr) { I(cr.first); I(cr.second); - db.write_delayed_roster(id, *(cr.first), *(cr.second)); + imp.write_delayed_roster(id, *(cr.first), *(cr.second)); } -unsigned long -database::roster_size_estimator::operator()(cached_roster const & cr) -{ - I(cr.first); - I(cr.second); - // do estimate using a totally made up multiplier, probably wildly off - return cr.first->all_nodes().size() * constants::db_estimated_roster_node_sz; -} - void -database::commit_transaction() +database_impl::commit_transaction() { if (transaction_level == 1) { @@ -996,7 +1239,7 @@ void } void -database::rollback_transaction() +database_impl::rollback_transaction() { if (transaction_level == 1) { @@ -1009,8 +1252,8 @@ bool bool -database::file_or_manifest_base_exists(hexenc const & ident, - std::string const & table) +database_impl::file_or_manifest_base_exists(hexenc const & ident, + string const & table) { // just check for a delayed file, since there are no delayed manifests if (have_delayed_file(file_id(ident))) @@ -1018,10 +1261,17 @@ database::file_or_manifest_base_exists(h return table_has_entry(ident(), "id", table); } +bool +database::file_or_manifest_base_exists(hexenc const & ident, + string const & table) +{ + return imp->file_or_manifest_base_exists(ident, table); +} + // returns true if we are currently storing (or planning to store) a // full-text for 'ident' bool -database::roster_base_stored(revision_id const & ident) +database_impl::roster_base_stored(revision_id const & ident) { if (roster_cache.exists(ident) && roster_cache.is_dirty(ident)) return true; @@ -1033,7 +1283,7 @@ bool // by calling roster_cache.insert_{clean,dirty}, because they can trigger // cache cleaning. bool -database::roster_base_available(revision_id const & ident) +database_impl::roster_base_available(revision_id const & ident) { if (roster_cache.exists(ident)) return true; @@ -1044,13 +1294,13 @@ database::delta_exists(string const & id database::delta_exists(string const & ident, string const & table) { - return table_has_entry(ident, "id", table); + return imp->table_has_entry(ident, "id", table); } bool -database::delta_exists(string const & ident, - string const & base, - string const & table) +database_impl::delta_exists(string const & ident, + string const & base, + string const & table) { results res; query q("SELECT 1 FROM " + table + " WHERE id = ? and base = ? LIMIT 1"); @@ -1059,7 +1309,7 @@ string } string -database::count(string const & table) +database_impl::count(string const & table) { try { @@ -1076,7 +1326,7 @@ string } string -database::space(string const & table, string const & rowspace, u64 & total) +database_impl::space(string const & table, string const & rowspace, u64 & total) { try { @@ -1095,7 +1345,7 @@ unsigned int } unsigned int -database::page_size() +database_impl::page_size() { results res; query q("PRAGMA page_size"); @@ -1104,7 +1354,7 @@ unsigned int } unsigned int -database::cache_size() +database_impl::cache_size() { // This returns the persistent (default) cache size. It's possible to // override this setting transiently at runtime by setting PRAGMA @@ -1116,7 +1366,7 @@ void } void -database::get_ids(string const & table, set< hexenc > & ids) +database_impl::get_ids(string const & table, set< hexenc > & ids) { results res; query q("SELECT id FROM " + table); @@ -1130,9 +1380,9 @@ void // for files and legacy manifest support void -database::get_file_or_manifest_base_unchecked(hexenc const & ident, - data & dat, - string const & table) +database_impl::get_file_or_manifest_base_unchecked(hexenc const & ident, + data & dat, + string const & table) { if (have_delayed_file(file_id(ident))) { @@ -1155,10 +1405,10 @@ void // for files and legacy manifest support void -database::get_file_or_manifest_delta_unchecked(hexenc const & ident, - hexenc const & base, - delta & del, - string const & table) +database_impl::get_file_or_manifest_delta_unchecked(hexenc const & ident, + hexenc const & base, + delta & del, + string const & table) { I(ident() != ""); I(base() != ""); @@ -1171,13 +1421,13 @@ void } void -database::get_roster_base(string const & ident_str, - roster_t & roster, marking_map & marking) +database_impl::get_roster_base(string const & ident_str, + roster_t & roster, marking_map & marking) { revision_id ident(ident_str); if (roster_cache.exists(ident)) { - cached_roster cr; + database::cached_roster cr; roster_cache.fetch(ident, cr); I(cr.first); roster = *(cr.first); @@ -1201,9 +1451,9 @@ void } void -database::get_roster_delta(string const & ident, - string const & base, - roster & del) +database_impl::get_roster_delta(string const & ident, + string const & base, + roster & del) { results res; query q("SELECT checksum, delta FROM roster_deltas WHERE id = ? AND base = ?"); @@ -1221,8 +1471,8 @@ void } void -database::write_delayed_file(file_id const & ident, - file_data const & dat) +database_impl::write_delayed_file(file_id const & ident, + file_data const & dat) { gzip dat_packed; encode_gzip(dat.inner(), dat_packed); @@ -1240,9 +1490,9 @@ void } void -database::write_delayed_roster(revision_id const & ident, - roster_t const & roster, - marking_map const & marking) +database_impl::write_delayed_roster(revision_id const & ident, + roster_t const & roster, + marking_map const & marking) { roster_data dat; write_roster_and_marking(roster, marking, dat); @@ -1272,17 +1522,16 @@ database::put_file_delta(file_id const & gzip del_packed; encode_gzip(del.inner(), del_packed); - query q("INSERT INTO file_deltas VALUES (?, ?, ?)"); - execute(q - % text(ident.inner()()) - % text(base.inner()()) - % blob(del_packed())); + imp->execute(query("INSERT INTO file_deltas VALUES (?, ?, ?)") + % text(ident.inner()()) + % text(base.inner()()) + % blob(del_packed())); } void -database::put_roster_delta(revision_id const & ident, - revision_id const & base, - roster_delta const & del) +database_impl::put_roster_delta(revision_id const & ident, + revision_id const & base, + roster_delta const & del) { gzip del_packed; encode_gzip(del.inner(), del_packed); @@ -1298,50 +1547,39 @@ database::put_roster_delta(revision_id c % blob(del_packed())); } - -// static ticker cache_hits("vcache hits", "h", 1); - -struct datasz -{ - unsigned long operator()(data const & t) { return t().size(); } -}; - -static LRUWritebackCache -vcache(constants::db_version_cache_sz); - struct file_and_manifest_reconstruction_graph : public reconstruction_graph { - database & db; + database_impl & imp; string const & data_table; string const & delta_table; - file_and_manifest_reconstruction_graph(database & db, + file_and_manifest_reconstruction_graph(database_impl & imp, string const & data_table, string const & delta_table) - : db(db), data_table(data_table), delta_table(delta_table) + : imp(imp), data_table(data_table), delta_table(delta_table) {} - virtual bool is_base(std::string const & node) const + virtual bool is_base(string const & node) const { - return vcache.exists(node) - || db.file_or_manifest_base_exists(hexenc(node), data_table); + return imp.vcache.exists(node) + || imp.file_or_manifest_base_exists(hexenc(node), data_table); } - virtual void get_next(std::string const & from, std::set & next) const + virtual void get_next(string const & from, set & next) const { next.clear(); - database::results res; + results res; query q("SELECT base FROM " + delta_table + " WHERE id = ?"); - db.fetch(res, one_col, any_rows, q % text(from)); - for (database::results::const_iterator i = res.begin(); i != res.end(); ++i) + imp.fetch(res, one_col, any_rows, q % text(from)); + for (results::const_iterator i = res.begin(); i != res.end(); ++i) next.insert((*i)[0]); } }; // used for files and legacy manifest migration void -database::get_version(hexenc const & ident, - data & dat, - string const & data_table, - string const & delta_table) +database_impl::get_version(hexenc const & ident, + data & dat, + string const & data_table, + string const & delta_table) { I(ident() != ""); @@ -1400,31 +1638,31 @@ struct roster_reconstruction_graph : pub struct roster_reconstruction_graph : public reconstruction_graph { - database & db; - roster_reconstruction_graph(database & db) : db(db) {} - virtual bool is_base(std::string const & node) const + database_impl & imp; + roster_reconstruction_graph(database_impl & imp) : imp(imp) {} + virtual bool is_base(string const & node) const { - return db.roster_base_available(revision_id(node)); + return imp.roster_base_available(revision_id(node)); } - virtual void get_next(std::string const & from, std::set & next) const + virtual void get_next(string const & from, set & next) const { next.clear(); - database::results res; + results res; query q("SELECT base FROM roster_deltas WHERE id = ?"); - db.fetch(res, one_col, any_rows, q % text(from)); - for (database::results::const_iterator i = res.begin(); i != res.end(); ++i) + imp.fetch(res, one_col, any_rows, q % text(from)); + for (results::const_iterator i = res.begin(); i != res.end(); ++i) next.insert((*i)[0]); } }; -struct database::extractor +struct database_impl::extractor { virtual bool look_at_delta(roster_delta const & del) = 0; virtual void look_at_roster(roster_t const & roster, marking_map const & mm) = 0; virtual ~extractor() {}; }; -struct database::markings_extractor : public database::extractor +struct database_impl::markings_extractor : public database_impl::extractor { private: node_id const & nid; @@ -1448,7 +1686,7 @@ public: } }; -struct database::file_content_extractor : database::extractor +struct database_impl::file_content_extractor : database_impl::extractor { private: node_id const & nid; @@ -1473,7 +1711,7 @@ void }; void -database::extract_from_deltas(revision_id const & id, extractor & x) +database_impl::extract_from_deltas(revision_id const & id, extractor & x) { reconstruction_path selected_path; { @@ -1539,8 +1777,8 @@ database::get_markings(revision_id const node_id const & nid, marking_t & markings) { - markings_extractor x(nid, markings); - extract_from_deltas(id, x); + database_impl::markings_extractor x(nid, markings); + imp->extract_from_deltas(id, x); } void @@ -1554,8 +1792,8 @@ database::get_file_content(revision_id c content = file_id(); return; } - file_content_extractor x(nid, content); - extract_from_deltas(id, x); + database_impl::file_content_extractor x(nid, content); + imp->extract_from_deltas(id, x); } void @@ -1563,15 +1801,15 @@ database::get_roster_version(revision_id cached_roster & cr) { // if we already have it, exit early - if (roster_cache.exists(id)) + if (imp->roster_cache.exists(id)) { - roster_cache.fetch(id, cr); + imp->roster_cache.fetch(id, cr); return; } reconstruction_path selected_path; { - roster_reconstruction_graph graph(*this); + roster_reconstruction_graph graph(*imp); get_reconstruction_path(id.inner()(), graph, selected_path); } @@ -1581,7 +1819,7 @@ database::get_roster_version(revision_id // above), so we should create new objects and spend time filling them in. shared_ptr roster(new roster_t); shared_ptr marking(new marking_map); - get_roster_base(curr, *roster, *marking); + imp->get_roster_base(curr, *roster, *marking); for (reconstruction_path::reverse_iterator i = selected_path.rbegin(); i != selected_path.rend(); ++i) @@ -1589,7 +1827,7 @@ database::get_roster_version(revision_id string const nxt = *i; L(FL("following delta %s -> %s") % curr % nxt); roster_delta del; - get_roster_delta(nxt, curr, del); + imp->get_roster_delta(nxt, curr, del); apply_roster_delta(del, *roster, *marking); curr = nxt; } @@ -1613,13 +1851,13 @@ database::get_roster_version(revision_id // const'ify the objects, to save them and pass them out cr.first = roster; cr.second = marking; - roster_cache.insert_clean(id, cr); + imp->roster_cache.insert_clean(id, cr); } void -database::drop(string const & ident, - string const & table) +database_impl::drop(string const & ident, + string const & table) { string drop = "DELETE FROM " + table + " WHERE id = ?"; execute(query(drop) % text(ident)); @@ -1642,7 +1880,7 @@ database::roster_version_exists(revision database::roster_version_exists(revision_id const & id) { return delta_exists(id.inner()(), "roster_deltas") - || roster_base_available(id); + || imp->roster_base_available(id); } bool @@ -1650,7 +1888,7 @@ database::revision_exists(revision_id co { results res; query q("SELECT id FROM revisions WHERE id = ?"); - fetch(res, one_col, any_rows, q % text(id.inner()())); + imp->fetch(res, one_col, any_rows, q % text(id.inner()())); I(res.size() <= 1); return res.size() == 1; } @@ -1660,8 +1898,8 @@ database::get_file_ids(set & id { ids.clear(); set< hexenc > tmp; - get_ids("files", tmp); - get_ids("file_deltas", tmp); + imp->get_ids("files", tmp); + imp->get_ids("file_deltas", tmp); add_decoration_to_container(tmp, ids); } @@ -1670,7 +1908,7 @@ database::get_revision_ids(set > tmp; - get_ids("revisions", tmp); + imp->get_ids("revisions", tmp); add_decoration_to_container(tmp, ids); } @@ -1679,9 +1917,9 @@ database::get_roster_ids(set > tmp; - get_ids("rosters", tmp); + imp->get_ids("rosters", tmp); add_decoration_to_container(tmp, ids); - get_ids("roster_deltas", tmp); + imp->get_ids("roster_deltas", tmp); add_decoration_to_container(tmp, ids); } @@ -1690,7 +1928,7 @@ database::get_file_version(file_id const file_data & dat) { data tmp; - get_version(id.inner(), tmp, "files", "file_deltas"); + imp->get_version(id.inner(), tmp, "files", "file_deltas"); dat = file_data(tmp); } @@ -1699,7 +1937,7 @@ database::get_manifest_version(manifest_ manifest_data & dat) { data tmp; - get_version(id.inner(), tmp, "manifests", "manifest_deltas"); + imp->get_version(id.inner(), tmp, "manifests", "manifest_deltas"); dat = manifest_data(tmp); } @@ -1710,7 +1948,7 @@ database::put_file(file_id const & id, if (file_version_exists(id)) L(FL("file version '%s' already exists in db") % id); else - schedule_delayed_file(id, dat); + imp->schedule_delayed_file(id, dat); } void @@ -1753,15 +1991,15 @@ database::put_file_version(file_id const { // descendent of a head version replaces the head, therefore old head // must be disposed of - drop_or_cancel_file(old_id); + imp->drop_or_cancel_file(old_id); } if (!file_or_manifest_base_exists(new_id.inner(), "files")) { - schedule_delayed_file(new_id, new_data); - drop(new_id.inner()(), "file_deltas"); + imp->schedule_delayed_file(new_id, new_data); + imp->drop(new_id.inner()(), "file_deltas"); } - if (!delta_exists(old_id.inner()(), new_id.inner()(), "file_deltas")) + if (!imp->delta_exists(old_id.inner()(), new_id.inner()(), "file_deltas")) { put_file_delta(old_id, new_id, reverse_delta); guard.commit(); @@ -1778,8 +2016,8 @@ database::get_arbitrary_file_delta(file_ results res; query q1("SELECT delta FROM file_deltas " "WHERE base = ? AND id = ?"); - fetch(res, one_col, any_rows, - q1 % text(src_id.inner()()) % text(dst_id.inner()())); + imp->fetch(res, one_col, any_rows, + q1 % text(src_id.inner()()) % text(dst_id.inner()())); if (!res.empty()) { @@ -1792,8 +2030,8 @@ database::get_arbitrary_file_delta(file_ query q2("SELECT delta FROM file_deltas " "WHERE base = ? AND id = ?"); - fetch(res, one_col, any_rows, - q2 % text(dst_id.inner()()) % text(src_id.inner()())); + imp->fetch(res, one_col, any_rows, + q2 % text(dst_id.inner()()) % text(src_id.inner()())); if (!res.empty()) { @@ -1826,8 +2064,8 @@ database::get_revision_ancestry(rev_ance results res; graph.clear(); - fetch(res, 2, any_rows, - query("SELECT parent,child FROM revision_ancestry")); + imp->fetch(res, 2, any_rows, + query("SELECT parent,child FROM revision_ancestry")); for (size_t i = 0; i < res.size(); ++i) graph.insert(make_pair(revision_id(res[i][0]), revision_id(res[i][1]))); @@ -1838,13 +2076,23 @@ database::get_revision_parents(revision_ set & parents) { I(!null_id(id)); - results res; - parents.clear(); - fetch(res, one_col, any_rows, - query("SELECT parent FROM revision_ancestry WHERE child = ?") - % text(id.inner()())); - for (size_t i = 0; i < res.size(); ++i) - parents.insert(revision_id(res[i][0])); + parent_id_map::iterator i = imp->parent_cache.find(id); + if (i == imp->parent_cache.end()) + { + results res; + parents.clear(); + imp->fetch(res, one_col, any_rows, + query("SELECT parent FROM revision_ancestry WHERE child = ?") + % text(id.inner()())); + for (size_t i = 0; i < res.size(); ++i) + parents.insert(revision_id(res[i][0])); + + imp->parent_cache.insert(make_pair(id, parents)); + } + else + { + parents = i->second; + } } void @@ -1853,8 +2101,8 @@ database::get_revision_children(revision { results res; children.clear(); - fetch(res, one_col, any_rows, - query("SELECT child FROM revision_ancestry WHERE parent = ?") + imp->fetch(res, one_col, any_rows, + query("SELECT child FROM revision_ancestry WHERE parent = ?") % text(id.inner()())); for (size_t i = 0; i < res.size(); ++i) children.insert(revision_id(res[i][0])); @@ -1865,11 +2113,11 @@ database::get_leaves(set & { results res; leaves.clear(); - fetch(res, one_col, any_rows, - query("SELECT revisions.id FROM revisions " - "LEFT JOIN revision_ancestry " - "ON revisions.id = revision_ancestry.parent " - "WHERE revision_ancestry.child IS null")); + imp->fetch(res, one_col, any_rows, + query("SELECT revisions.id FROM revisions " + "LEFT JOIN revision_ancestry " + "ON revisions.id = revision_ancestry.parent " + "WHERE revision_ancestry.child IS null")); for (size_t i = 0; i < res.size(); ++i) leaves.insert(revision_id(res[i][0])); } @@ -1899,9 +2147,9 @@ database::get_revision(revision_id const { I(!null_id(id)); results res; - fetch(res, one_col, one_row, - query("SELECT data FROM revisions WHERE id = ?") - % text(id.inner()())); + imp->fetch(res, one_col, one_row, + query("SELECT data FROM revisions WHERE id = ?") + % text(id.inner()())); gzip gzdata(res[0][0]); data rdat; @@ -1917,9 +2165,6 @@ database::get_revision(revision_id const dat = revision_data(rdat); } -typedef std::map height_map; -static height_map height_cache; - void database::get_rev_height(revision_id const & id, rev_height & height) @@ -1930,18 +2175,18 @@ database::get_rev_height(revision_id con return; } - height_map::const_iterator i = height_cache.find(id); - if (i == height_cache.end()) + height_map::const_iterator i = imp->height_cache.find(id); + if (i == imp->height_cache.end()) { results res; - fetch(res, one_col, one_row, - query("SELECT height FROM heights WHERE revision = ?") - % text(id.inner()())); + imp->fetch(res, one_col, one_row, + query("SELECT height FROM heights WHERE revision = ?") + % text(id.inner()())); I(res.size() == 1); height = rev_height(res[0][0]); - height_cache.insert(make_pair(id, height)); + imp->height_cache.insert(make_pair(id, height)); } else { @@ -1959,20 +2204,20 @@ database::put_rev_height(revision_id con I(revision_exists(id)); I(height.valid()); - height_cache.erase(id); + imp->height_cache.erase(id); - execute(query("INSERT INTO heights VALUES(?, ?)") - % text(id.inner()()) - % blob(height())); + imp->execute(query("INSERT INTO heights VALUES(?, ?)") + % text(id.inner()()) + % blob(height())); } bool database::has_rev_height(rev_height const & height) { results res; - fetch(res, one_col, any_rows, - query("SELECT height FROM heights WHERE height = ?") - % blob(height())); + imp->fetch(res, one_col, any_rows, + query("SELECT height FROM heights WHERE height = ?") + % blob(height())); I((res.size() == 1) || (res.size() == 0)); return res.size() == 1; } @@ -2005,8 +2250,8 @@ database::deltify_revision(revision_id c delta delt; diff(old_data.inner(), new_data.inner(), delt); file_delta del(delt); - drop_or_cancel_file(delta_entry_dst(j)); - drop(delta_entry_dst(j).inner()(), "file_deltas"); + imp->drop_or_cancel_file(delta_entry_dst(j)); + imp->drop(delta_entry_dst(j).inner()(), "file_deltas"); put_file_version(delta_entry_src(j), delta_entry_dst(j), del); } } @@ -2092,17 +2337,20 @@ database::put_revision(revision_id const write_revision(rev, d); gzip d_packed; encode_gzip(d.inner(), d_packed); - execute(query("INSERT INTO revisions VALUES(?, ?)") - % text(new_id.inner()()) - % blob(d_packed())); + imp->execute(query("INSERT INTO revisions VALUES(?, ?)") + % text(new_id.inner()()) + % blob(d_packed())); for (edge_map::const_iterator e = rev.edges.begin(); e != rev.edges.end(); ++e) { - execute(query("INSERT INTO revision_ancestry VALUES(?, ?)") - % text(edge_old_revision(e).inner()()) - % text(new_id.inner()())); + imp->execute(query("INSERT INTO revision_ancestry VALUES(?, ?)") + % text(edge_old_revision(e).inner()()) + % text(new_id.inner()())); } + // We don't have to clear out the child's entry in the parent_cache, + // because the child did not exist before this function was called, so + // it can't be in the parent_cache already. // Phase 3: Construct and write the roster (which also checks the manifest // id as it goes), but only if the roster does not already exist in the db @@ -2200,30 +2448,30 @@ database::delete_existing_revs_and_certs void database::delete_existing_revs_and_certs() { - execute(query("DELETE FROM revisions")); - execute(query("DELETE FROM revision_ancestry")); - execute(query("DELETE FROM revision_certs")); + imp->execute(query("DELETE FROM revisions")); + imp->execute(query("DELETE FROM revision_ancestry")); + imp->execute(query("DELETE FROM revision_certs")); } void database::delete_existing_manifests() { - execute(query("DELETE FROM manifests")); - execute(query("DELETE FROM manifest_deltas")); + imp->execute(query("DELETE FROM manifests")); + imp->execute(query("DELETE FROM manifest_deltas")); } void database::delete_existing_rosters() { - execute(query("DELETE FROM rosters")); - execute(query("DELETE FROM roster_deltas")); - execute(query("DELETE FROM next_roster_node_number")); + imp->execute(query("DELETE FROM rosters")); + imp->execute(query("DELETE FROM roster_deltas")); + imp->execute(query("DELETE FROM next_roster_node_number")); } void database::delete_existing_heights() { - execute(query("DELETE FROM heights")); + imp->execute(query("DELETE FROM heights")); } /// Deletes one revision from the local database. @@ -2243,18 +2491,18 @@ database::delete_existing_rev_and_certs( L(FL("Killing revision %s locally") % rid); // Kill the certs, ancestry, and revision. - execute(query("DELETE from revision_certs WHERE id = ?") - % text(rid.inner()())); - cert_stamper.note_change(); + imp->execute(query("DELETE from revision_certs WHERE id = ?") + % text(rid.inner()())); + imp->cert_stamper.note_change(); - execute(query("DELETE from revision_ancestry WHERE child = ?") - % text(rid.inner()())); + imp->execute(query("DELETE from revision_ancestry WHERE child = ?") + % text(rid.inner()())); - execute(query("DELETE from heights WHERE revision = ?") - % text(rid.inner()())); + imp->execute(query("DELETE from heights WHERE revision = ?") + % text(rid.inner()())); - execute(query("DELETE from revisions WHERE id = ?") - % text(rid.inner()())); + imp->execute(query("DELETE from revisions WHERE id = ?") + % text(rid.inner()())); guard.commit(); } @@ -2264,11 +2512,11 @@ database::delete_branch_named(cert_value database::delete_branch_named(cert_value const & branch) { L(FL("Deleting all references to branch %s") % branch); - execute(query("DELETE FROM revision_certs WHERE name='branch' AND value =?") - % blob(branch())); - cert_stamper.note_change(); - execute(query("DELETE FROM branch_epochs WHERE branch=?") - % blob(branch())); + imp->execute(query("DELETE FROM revision_certs WHERE name='branch' AND value =?") + % blob(branch())); + imp->cert_stamper.note_change(); + imp->execute(query("DELETE FROM branch_epochs WHERE branch=?") + % blob(branch())); } /// Deletes all certs referring to a particular tag. @@ -2276,9 +2524,9 @@ database::delete_tag_named(cert_value co database::delete_tag_named(cert_value const & tag) { L(FL("Deleting all references to tag %s") % tag); - execute(query("DELETE FROM revision_certs WHERE name='tag' AND value =?") - % blob(tag())); - cert_stamper.note_change(); + imp->execute(query("DELETE FROM revision_certs WHERE name='tag' AND value =?") + % blob(tag())); + imp->cert_stamper.note_change(); } // crypto key management @@ -2289,7 +2537,7 @@ database::get_key_ids(vectorfetch(res, one_col, any_rows, query("SELECT id FROM public_keys")); for (size_t i = 0; i < res.size(); ++i) pubkeys.push_back(rsa_keypair_id(res[i][0])); @@ -2302,7 +2550,7 @@ database::get_key_ids(globish const & pa pubkeys.clear(); results res; - fetch(res, one_col, any_rows, query("SELECT id FROM public_keys")); + imp->fetch(res, one_col, any_rows, query("SELECT id FROM public_keys")); for (size_t i = 0; i < res.size(); ++i) if (pattern.matches(res[i][0])) @@ -2310,7 +2558,7 @@ void } void -database::get_keys(string const & table, vector & keys) +database_impl::get_keys(string const & table, vector & keys) { keys.clear(); results res; @@ -2322,16 +2570,16 @@ database::get_public_keys(vector & keys) { - get_keys("public_keys", keys); + imp->get_keys("public_keys", keys); } bool database::public_key_exists(hexenc const & hash) { results res; - fetch(res, one_col, any_rows, - query("SELECT id FROM public_keys WHERE hash = ?") - % text(hash())); + imp->fetch(res, one_col, any_rows, + query("SELECT id FROM public_keys WHERE hash = ?") + % text(hash())); I((res.size() == 1) || (res.size() == 0)); if (res.size() == 1) return true; @@ -2342,9 +2590,9 @@ database::public_key_exists(rsa_keypair_ database::public_key_exists(rsa_keypair_id const & id) { results res; - fetch(res, one_col, any_rows, - query("SELECT id FROM public_keys WHERE id = ?") - % text(id())); + imp->fetch(res, one_col, any_rows, + query("SELECT id FROM public_keys WHERE id = ?") + % text(id())); I((res.size() == 1) || (res.size() == 0)); if (res.size() == 1) return true; @@ -2357,9 +2605,9 @@ database::get_pubkey(hexenc const & base64 & pub_encoded) { results res; - fetch(res, 2, one_row, - query("SELECT id, keydata FROM public_keys WHERE hash = ?") - % text(hash())); + imp->fetch(res, 2, one_row, + query("SELECT id, keydata FROM public_keys WHERE hash = ?") + % text(hash())); id = rsa_keypair_id(res[0][0]); encode_base64(rsa_pub_key(res[0][1]), pub_encoded); } @@ -2369,9 +2617,9 @@ database::get_key(rsa_keypair_id const & base64 & pub_encoded) { results res; - fetch(res, one_col, one_row, - query("SELECT keydata FROM public_keys WHERE id = ?") - % text(pub_id())); + imp->fetch(res, one_col, one_row, + query("SELECT keydata FROM public_keys WHERE id = ?") + % text(pub_id())); encode_base64(rsa_pub_key(res[0][0]), pub_encoded); } @@ -2397,10 +2645,10 @@ database::put_key(rsa_keypair_id const & rsa_pub_key pub_key; decode_base64(pub_encoded, pub_key); - execute(query("INSERT INTO public_keys VALUES(?, ?, ?)") - % text(thash()) - % text(pub_id()) - % blob(pub_key())); + imp->execute(query("INSERT INTO public_keys VALUES(?, ?, ?)") + % text(thash()) + % text(pub_id()) + % blob(pub_key())); return true; } @@ -2408,15 +2656,15 @@ database::delete_public_key(rsa_keypair_ void database::delete_public_key(rsa_keypair_id const & pub_id) { - execute(query("DELETE FROM public_keys WHERE id = ?") - % text(pub_id())); + imp->execute(query("DELETE FROM public_keys WHERE id = ?") + % text(pub_id())); } // cert management bool -database::cert_exists(cert const & t, - string const & table) +database_impl::cert_exists(cert const & t, + string const & table) { results res; cert_value value; @@ -2441,8 +2689,8 @@ void } void -database::put_cert(cert const & t, - string const & table) +database_impl::put_cert(cert const & t, + string const & table) { hexenc thash; cert_hash_code(t, thash); @@ -2463,8 +2711,8 @@ void } void -database::results_to_certs(results const & res, - vector & certs) +database_impl::results_to_certs(results const & res, + vector & certs) { certs.clear(); for (size_t i = 0; i < res.size(); ++i) @@ -2484,7 +2732,7 @@ void } void -database::install_functions() +database_impl::install_functions() { // register any functions we're going to use I(sqlite3_create_function(sql(), "gunzip", -1, @@ -2494,8 +2742,8 @@ void } void -database::get_certs(vector & certs, - string const & table) +database_impl::get_certs(vector & certs, + string const & table) { results res; query q("SELECT id, name, value, keypair, signature FROM " + table); @@ -2505,9 +2753,9 @@ void void -database::get_certs(hexenc const & ident, - vector & certs, - string const & table) +database_impl::get_certs(hexenc const & ident, + vector & certs, + string const & table) { results res; query q("SELECT id, name, value, keypair, signature FROM " + table + @@ -2519,9 +2767,9 @@ void void -database::get_certs(cert_name const & name, - vector & certs, - string const & table) +database_impl::get_certs(cert_name const & name, + vector & certs, + string const & table) { results res; query q("SELECT id, name, value, keypair, signature FROM " + table + @@ -2532,10 +2780,10 @@ void void -database::get_certs(hexenc const & ident, - cert_name const & name, - vector & certs, - string const & table) +database_impl::get_certs(hexenc const & ident, + cert_name const & name, + vector & certs, + string const & table) { results res; query q("SELECT id, name, value, keypair, signature FROM " + table + @@ -2548,10 +2796,10 @@ void } void -database::get_certs(cert_name const & name, - base64 const & val, - vector & certs, - string const & table) +database_impl::get_certs(cert_name const & name, + base64 const & val, + vector & certs, + string const & table) { results res; query q("SELECT id, name, value, keypair, signature FROM " + table + @@ -2567,11 +2815,11 @@ void void -database::get_certs(hexenc const & ident, - cert_name const & name, - base64 const & value, - vector & certs, - string const & table) +database_impl::get_certs(hexenc const & ident, + cert_name const & name, + base64 const & value, + vector & certs, + string const & table) { results res; query q("SELECT id, name, value, keypair, signature FROM " + table + @@ -2591,7 +2839,7 @@ database::revision_cert_exists(revision< bool database::revision_cert_exists(revision const & cert) { - return cert_exists(cert.inner(), "revision_certs"); + return imp->cert_exists(cert.inner(), "revision_certs"); } bool @@ -2612,8 +2860,8 @@ database::put_revision_cert(revisionput_cert(cert.inner(), "revision_certs"); + imp->cert_stamper.note_change(); return true; } @@ -2625,9 +2873,9 @@ database::get_revision_cert_nobranch_ind id::symtab id_syms; results res; - fetch(res, 3, any_rows, - query("SELECT hash, id, keypair " - "FROM 'revision_certs' WHERE name != 'branch'")); + imp->fetch(res, 3, any_rows, + query("SELECT hash, id, keypair " + "FROM 'revision_certs' WHERE name != 'branch'")); idx.clear(); idx.reserve(res.size()); @@ -2637,17 +2885,17 @@ database::get_revision_cert_nobranch_ind make_pair(revision_id((*i)[1]), rsa_keypair_id((*i)[2])))); } - return cert_stamper.get_indicator(); + return imp->cert_stamper.get_indicator(); } outdated_indicator database::get_revision_certs(vector< revision > & ts) { vector certs; - get_certs(certs, "revision_certs"); + imp->get_certs(certs, "revision_certs"); ts.clear(); add_decoration_to_container(certs, ts); - return cert_stamper.get_indicator(); + return imp->cert_stamper.get_indicator(); } outdated_indicator @@ -2655,10 +2903,10 @@ database::get_revision_certs(cert_name c vector< revision > & ts) { vector certs; - get_certs(name, certs, "revision_certs"); + imp->get_certs(name, certs, "revision_certs"); ts.clear(); add_decoration_to_container(certs, ts); - return cert_stamper.get_indicator(); + return imp->cert_stamper.get_indicator(); } outdated_indicator @@ -2667,10 +2915,10 @@ database::get_revision_certs(revision_id vector< revision > & ts) { vector certs; - get_certs(id.inner(), name, certs, "revision_certs"); + imp->get_certs(id.inner(), name, certs, "revision_certs"); ts.clear(); add_decoration_to_container(certs, ts); - return cert_stamper.get_indicator(); + return imp->cert_stamper.get_indicator(); } outdated_indicator @@ -2680,10 +2928,10 @@ database::get_revision_certs(revision_id vector< revision > & ts) { vector certs; - get_certs(id.inner(), name, val, certs, "revision_certs"); + imp->get_certs(id.inner(), name, val, certs, "revision_certs"); ts.clear(); add_decoration_to_container(certs, ts); - return cert_stamper.get_indicator(); + return imp->cert_stamper.get_indicator(); } outdated_indicator @@ -2696,10 +2944,10 @@ database::get_revisions_with_cert(cert_n query q("SELECT id FROM revision_certs WHERE name = ? AND value = ?"); cert_value binvalue; decode_base64(val, binvalue); - fetch(res, one_col, any_rows, q % text(name()) % blob(binvalue())); + imp->fetch(res, one_col, any_rows, q % text(name()) % blob(binvalue())); for (results::const_iterator i = res.begin(); i != res.end(); ++i) revisions.insert(revision_id((*i)[0])); - return cert_stamper.get_indicator(); + return imp->cert_stamper.get_indicator(); } outdated_indicator @@ -2708,10 +2956,10 @@ database::get_revision_certs(cert_name c vector< revision > & ts) { vector certs; - get_certs(name, val, certs, "revision_certs"); + imp->get_certs(name, val, certs, "revision_certs"); ts.clear(); add_decoration_to_container(certs, ts); - return cert_stamper.get_indicator(); + return imp->cert_stamper.get_indicator(); } outdated_indicator @@ -2719,10 +2967,10 @@ database::get_revision_certs(revision_id vector< revision > & ts) { vector certs; - get_certs(id.inner(), certs, "revision_certs"); + imp->get_certs(id.inner(), certs, "revision_certs"); ts.clear(); add_decoration_to_container(certs, ts); - return cert_stamper.get_indicator(); + return imp->cert_stamper.get_indicator(); } outdated_indicator @@ -2731,15 +2979,15 @@ database::get_revision_certs(revision_id { results res; vector certs; - fetch(res, one_col, any_rows, - query("SELECT hash " - "FROM revision_certs " - "WHERE id = ?") - % text(ident.inner()())); + imp->fetch(res, one_col, any_rows, + query("SELECT hash " + "FROM revision_certs " + "WHERE id = ?") + % text(ident.inner()())); ts.clear(); for (size_t i = 0; i < res.size(); ++i) ts.push_back(hexenc(res[i][0])); - return cert_stamper.get_indicator(); + return imp->cert_stamper.get_indicator(); } void @@ -2748,12 +2996,12 @@ database::get_revision_cert(hexenc c { results res; vector certs; - fetch(res, 5, one_row, - query("SELECT id, name, value, keypair, signature " - "FROM revision_certs " - "WHERE hash = ?") - % text(hash())); - results_to_certs(res, certs); + imp->fetch(res, 5, one_row, + query("SELECT id, name, value, keypair, signature " + "FROM revision_certs " + "WHERE hash = ?") + % text(hash())); + imp->results_to_certs(res, certs); I(certs.size() == 1); c = revision(certs[0]); } @@ -2763,11 +3011,11 @@ database::revision_cert_exists(hexenc certs; - fetch(res, one_col, any_rows, - query("SELECT id " - "FROM revision_certs " - "WHERE hash = ?") - % text(hash())); + imp->fetch(res, one_col, any_rows, + query("SELECT id " + "FROM revision_certs " + "WHERE hash = ?") + % text(hash())); I(res.size() == 0 || res.size() == 1); return (res.size() == 1); } @@ -2777,7 +3025,7 @@ database::get_manifest_certs(manifest_id vector< manifest > & ts) { vector certs; - get_certs(id.inner(), certs, "manifest_certs"); + imp->get_certs(id.inner(), certs, "manifest_certs"); ts.clear(); add_decoration_to_container(certs, ts); } @@ -2788,7 +3036,7 @@ database::get_manifest_certs(cert_name c vector< manifest > & ts) { vector certs; - get_certs(name, certs, "manifest_certs"); + imp->get_certs(name, certs, "manifest_certs"); ts.clear(); add_decoration_to_container(certs, ts); } @@ -2804,9 +3052,9 @@ database::complete(string const & partia string pattern = partial + "*"; - fetch(res, 1, any_rows, - query("SELECT id FROM revisions WHERE id GLOB ?") - % text(pattern)); + imp->fetch(res, 1, any_rows, + query("SELECT id FROM revisions WHERE id GLOB ?") + % text(pattern)); for (size_t i = 0; i < res.size(); ++i) completions.insert(revision_id(res[i][0])); @@ -2822,18 +3070,18 @@ database::complete(string const & partia string pattern = partial + "*"; - fetch(res, 1, any_rows, - query("SELECT id FROM files WHERE id GLOB ?") - % text(pattern)); + imp->fetch(res, 1, any_rows, + query("SELECT id FROM files WHERE id GLOB ?") + % text(pattern)); for (size_t i = 0; i < res.size(); ++i) completions.insert(file_id(res[i][0])); res.clear(); - fetch(res, 1, any_rows, - query("SELECT id FROM file_deltas WHERE id GLOB ?") - % text(pattern)); + imp->fetch(res, 1, any_rows, + query("SELECT id FROM file_deltas WHERE id GLOB ?") + % text(pattern)); for (size_t i = 0; i < res.size(); ++i) completions.insert(file_id(res[i][0])); @@ -2848,9 +3096,9 @@ database::complete(string const & partia string pattern = partial + "*"; - fetch(res, 2, any_rows, - query("SELECT hash, id FROM public_keys WHERE hash GLOB ?") - % text(pattern)); + imp->fetch(res, 2, any_rows, + query("SELECT hash, id FROM public_keys WHERE hash GLOB ?") + % text(pattern)); for (size_t i = 0; i < res.size(); ++i) completions.insert(make_pair(key_id(res[i][0]), utf8(res[i][1]))); @@ -2867,9 +3115,9 @@ database::select_parent(string const & p string pattern = partial + "*"; - fetch(res, 1, any_rows, - query("SELECT DISTINCT parent FROM revision_ancestry WHERE child GLOB ?") - % text(pattern)); + imp->fetch(res, 1, any_rows, + query("SELECT DISTINCT parent FROM revision_ancestry WHERE child GLOB ?") + % text(pattern)); for (size_t i = 0; i < res.size(); ++i) completions.insert(revision_id(res[i][0])); @@ -2882,9 +3130,9 @@ database::select_cert(string const & cer results res; completions.clear(); - fetch(res, 1, any_rows, - query("SELECT DISTINCT id FROM revision_certs WHERE name = ?") - % text(certname)); + imp->fetch(res, 1, any_rows, + query("SELECT DISTINCT id FROM revision_certs WHERE name = ?") + % text(certname)); for (size_t i = 0; i < res.size(); ++i) completions.insert(revision_id(res[i][0])); @@ -2897,10 +3145,10 @@ database::select_cert(string const & cer results res; completions.clear(); - fetch(res, 1, any_rows, - query("SELECT DISTINCT id FROM revision_certs" - " WHERE name = ? AND CAST(value AS TEXT) GLOB ?") - % text(certname) % text(certvalue)); + imp->fetch(res, 1, any_rows, + query("SELECT DISTINCT id FROM revision_certs" + " WHERE name = ? AND CAST(value AS TEXT) GLOB ?") + % text(certname) % text(certvalue)); for (size_t i = 0; i < res.size(); ++i) completions.insert(revision_id(res[i][0])); @@ -2915,12 +3163,12 @@ database::select_author_tag_or_branch(st string pattern = partial + "*"; - fetch(res, 1, any_rows, - query("SELECT DISTINCT id FROM revision_certs" - " WHERE (name=? OR name=? OR name=?)" - " AND CAST(value AS TEXT) GLOB ?") - % text(author_cert_name()) % text(tag_cert_name()) - % text(branch_cert_name()) % text(pattern)); + imp->fetch(res, 1, any_rows, + query("SELECT DISTINCT id FROM revision_certs" + " WHERE (name=? OR name=? OR name=?)" + " AND CAST(value AS TEXT) GLOB ?") + % text(author_cert_name()) % text(tag_cert_name()) + % text(branch_cert_name()) % text(pattern)); for (size_t i = 0; i < res.size(); ++i) completions.insert(revision_id(res[i][0])); @@ -2939,8 +3187,8 @@ database::select_date(string const & dat q.sql_cmd += comparison; q.sql_cmd += " ?"; - fetch(res, 1, any_rows, - q % text(date_cert_name()) % text(date)); + imp->fetch(res, 1, any_rows, + q % text(date_cert_name()) % text(date)); for (size_t i = 0; i < res.size(); ++i) completions.insert(revision_id(res[i][0])); } @@ -2952,7 +3200,7 @@ database::get_epochs(mapfetch(res, 2, any_rows, query("SELECT branch, epoch FROM branch_epochs")); for (results::const_iterator i = res.begin(); i != res.end(); ++i) { branch_name decoded(idx(*i, 0)); @@ -2967,10 +3215,10 @@ database::get_epoch(epoch_id const & eid { I(epoch_exists(eid)); results res; - fetch(res, 2, any_rows, - query("SELECT branch, epoch FROM branch_epochs" - " WHERE hash = ?") - % text(eid.inner()())); + imp->fetch(res, 2, any_rows, + query("SELECT branch, epoch FROM branch_epochs" + " WHERE hash = ?") + % text(eid.inner()())); I(res.size() == 1); branch = branch_name(idx(idx(res, 0), 0)); epo = epoch_data(idx(idx(res, 0), 1)); @@ -2980,9 +3228,9 @@ database::epoch_exists(epoch_id const & database::epoch_exists(epoch_id const & eid) { results res; - fetch(res, one_col, any_rows, - query("SELECT hash FROM branch_epochs WHERE hash = ?") - % text(eid.inner()())); + imp->fetch(res, one_col, any_rows, + query("SELECT hash FROM branch_epochs WHERE hash = ?") + % text(eid.inner()())); I(res.size() == 1 || res.size() == 0); return res.size() == 1; } @@ -2993,24 +3241,24 @@ database::set_epoch(branch_name const & epoch_id eid; epoch_hash_code(branch, epo, eid); I(epo.inner()().size() == constants::epochlen); - execute(query("INSERT OR REPLACE INTO branch_epochs VALUES(?, ?, ?)") - % text(eid.inner()()) - % blob(branch()) - % text(epo.inner()())); + imp->execute(query("INSERT OR REPLACE INTO branch_epochs VALUES(?, ?, ?)") + % text(eid.inner()()) + % blob(branch()) + % text(epo.inner()())); } void database::clear_epoch(branch_name const & branch) { - execute(query("DELETE FROM branch_epochs WHERE branch = ?") - % blob(branch())); + imp->execute(query("DELETE FROM branch_epochs WHERE branch = ?") + % blob(branch())); } bool database::check_integrity() { results res; - fetch(res, one_col, any_rows, query("PRAGMA integrity_check")); + imp->fetch(res, one_col, any_rows, query("PRAGMA integrity_check")); I(res.size() == 1); I(res[0].size() == 1); @@ -3024,7 +3272,7 @@ database::get_vars(mapfetch(res, 3, any_rows, query("SELECT domain, name, value FROM db_vars")); for (results::const_iterator i = res.begin(); i != res.end(); ++i) { var_domain domain(idx(*i, 0)); @@ -3059,18 +3307,18 @@ database::set_var(var_key const & key, v void database::set_var(var_key const & key, var_value const & value) { - execute(query("INSERT OR REPLACE INTO db_vars VALUES(?, ?, ?)") - % text(key.first()) - % blob(key.second()) - % blob(value())); + imp->execute(query("INSERT OR REPLACE INTO db_vars VALUES(?, ?, ?)") + % text(key.first()) + % blob(key.second()) + % blob(value())); } void database::clear_var(var_key const & key) { - execute(query("DELETE FROM db_vars WHERE domain = ? AND name = ?") - % text(key.first()) - % blob(key.second())); + imp->execute(query("DELETE FROM db_vars WHERE domain = ? AND name = ?") + % text(key.first()) + % blob(key.second())); } // branches @@ -3081,12 +3329,12 @@ database::get_branches(vector & results res; query q("SELECT DISTINCT value FROM revision_certs WHERE name = ?"); string cert_name = "branch"; - fetch(res, one_col, any_rows, q % text(cert_name)); + imp->fetch(res, one_col, any_rows, q % text(cert_name)); for (size_t i = 0; i < res.size(); ++i) { names.push_back(res[i][0]); } - return cert_stamper.get_indicator(); + return imp->cert_stamper.get_indicator(); } outdated_indicator @@ -3096,13 +3344,13 @@ database::get_branches(globish const & g results res; query q("SELECT DISTINCT value FROM revision_certs WHERE name = ?"); string cert_name = "branch"; - fetch(res, one_col, any_rows, q % text(cert_name)); + imp->fetch(res, one_col, any_rows, q % text(cert_name)); for (size_t i = 0; i < res.size(); ++i) { if (glob.matches(res[i][0])) names.push_back(res[i][0]); } - return cert_stamper.get_indicator(); + return imp->cert_stamper.get_indicator(); } void @@ -3125,7 +3373,7 @@ database::get_roster(revision_id const & return; } - cached_roster cr; + database::cached_roster cr; get_roster(rev_id, cr); roster = *cr.first; marking = *cr.second; @@ -3133,7 +3381,7 @@ database::get_roster(revision_id const & void database::get_roster(revision_id const & rev_id, - cached_roster & cr) + database::cached_roster & cr) { get_roster_version(rev_id, cr); I(cr.first); @@ -3154,7 +3402,7 @@ database::put_roster(revision_id const & // Our task is to add this roster, and deltify all the incoming edges (if // they aren't already). - roster_cache.insert_dirty(rev_id, make_pair(roster, marking)); + imp->roster_cache.insert_dirty(rev_id, make_pair(roster, marking)); set parents; get_revision_parents(rev_id, parents); @@ -3166,16 +3414,16 @@ database::put_roster(revision_id const & if (null_id(*i)) continue; revision_id old_rev = *i; - if (roster_base_stored(old_rev)) + if (imp->roster_base_stored(old_rev)) { - cached_roster cr; + database::cached_roster cr; get_roster_version(old_rev, cr); roster_delta reverse_delta; delta_rosters(*roster, *marking, *(cr.first), *(cr.second), reverse_delta); - if (roster_cache.exists(old_rev)) - roster_cache.mark_clean(old_rev); - drop(old_rev.inner()(), "rosters"); - put_roster_delta(old_rev, rev_id, reverse_delta); + if (imp->roster_cache.exists(old_rev)) + imp->roster_cache.mark_clean(old_rev); + imp->drop(old_rev.inner()(), "rosters"); + imp->put_roster_delta(old_rev, rev_id, reverse_delta); } } guard.commit(); @@ -3184,23 +3432,10 @@ struct rev_height_graph : rev_graph // for get_uncommon_ancestors struct rev_height_graph : rev_graph { - typedef hashmap::hash_map > parent_map; - typedef hashmap::hash_map height_map; - static parent_map parent_cache; - static height_map height_cache; rev_height_graph(database & db) : db(db) {} virtual void get_parents(revision_id const & rev, set & parents) const { - parent_map::iterator i = parent_cache.find(rev); - if (i == parent_cache.end()) - { - db.get_revision_parents(rev, parents); - parent_cache.insert(make_pair(rev, parents)); - } - else - { - parents = i->second; - } + db.get_revision_parents(rev, parents); } virtual void get_children(revision_id const & rev, set & parents) const { @@ -3209,22 +3444,11 @@ struct rev_height_graph : rev_graph } virtual void get_height(revision_id const & rev, rev_height & h) const { - height_map::iterator i = height_cache.find(rev); - if (i == height_cache.end()) - { - db.get_rev_height(rev, h); - height_cache.insert(make_pair(rev, h)); - } - else - { - h = i->second; - } + db.get_rev_height(rev, h); } database & db; }; -rev_height_graph::parent_map rev_height_graph::parent_cache; -rev_height_graph::height_map rev_height_graph::height_cache; void database::get_uncommon_ancestors(revision_id const & a, @@ -3237,53 +3461,42 @@ database::get_uncommon_ancestors(revisio ::get_uncommon_ancestors(a, b, graph, a_uncommon_ancs, b_uncommon_ancs); } - -u64 -database::next_id_from_table(string const & table) +node_id +database::next_node_id() { transaction_guard guard(*this); results res; // We implement this as a fixed db var. + imp->fetch(res, one_col, any_rows, + query("SELECT node FROM next_roster_node_number")); - fetch(res, one_col, any_rows, - query(string("SELECT node FROM ") + table)); - - u64 n; + u64 n = 1; if (res.empty()) { - n = 1; - execute (query(string("INSERT INTO ") + table + " VALUES(?)") - % text(lexical_cast(n))); + imp->execute(query("INSERT INTO next_roster_node_number VALUES(1)")); } else { I(res.size() == 1); n = lexical_cast(res[0][0]); ++n; - execute(query(string("UPDATE ") + table + " SET node = ?") - % text(lexical_cast(n))); - + imp->execute(query("UPDATE next_roster_node_number SET node = ?") + % text(lexical_cast(n))); } guard.commit(); - return n; + return static_cast(n); } -node_id -database::next_node_id() -{ - return static_cast(next_id_from_table("next_roster_node_number")); -} - void -database::check_filename() +database_impl::check_filename() { N(!filename.empty(), F("no database specified")); } void -database::check_db_exists() +database_impl::check_db_exists() { switch (get_path_status(filename)) { @@ -3318,7 +3531,7 @@ void } void -database::check_db_nonexistent() +database_impl::check_db_nonexistent() { require_path_is_nonexistent(filename, F("database %s already exists") @@ -3333,15 +3546,8 @@ database::check_db_nonexistent() } -bool -database::database_specified() -{ - return !filename.empty(); -} - - void -database::open() +database_impl::open() { I(!__sql); @@ -3353,7 +3559,7 @@ void } void -database::close() +database_impl::close() { I(__sql); @@ -3364,6 +3570,12 @@ database::close() } // FIXME: the quick hack lua link in functions +void +database::set_app(app_state * app) +{ + __app = app; +} + bool database::hook_get_manifest_cert_trust(set const & signers, hexenc const & id, cert_name const & name, cert_value const & val) @@ -3386,8 +3598,8 @@ bool } bool -database::hook_accept_testresult_change(std::map const & old_results, - std::map const & new_results) +database::hook_accept_testresult_change(map const & old_results, + map const & new_results) { return __app->lua.hook_accept_testresult_change(old_results, new_results); } @@ -3452,28 +3664,29 @@ transaction_guard::transaction_guard(dat transaction_guard::transaction_guard(database & d, bool exclusive, size_t checkpoint_batch_size, size_t checkpoint_batch_bytes) - : committed(false), db(d), exclusive(exclusive), + : imp(d.imp), checkpoint_batch_size(checkpoint_batch_size), checkpoint_batch_bytes(checkpoint_batch_bytes), checkpointed_calls(0), - checkpointed_bytes(0) + checkpointed_bytes(0), + committed(false), exclusive(exclusive) { - db.begin_transaction(exclusive); + imp->begin_transaction(exclusive); } transaction_guard::~transaction_guard() { if (committed) - db.commit_transaction(); + imp->commit_transaction(); else - db.rollback_transaction(); + imp->rollback_transaction(); } void transaction_guard::do_checkpoint() { - db.commit_transaction(); - db.begin_transaction(exclusive); + imp->commit_transaction(); + imp->begin_transaction(exclusive); checkpointed_calls = 0; checkpointed_bytes = 0; } ============================================================ --- database.hh cb2f020ae339ce2fccb1826b8b8b6553571b4354 +++ database.hh 2ed0fd6ca1ba6c5269e446120df198dfedc970c2 @@ -10,25 +10,23 @@ // implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR // PURPOSE. -struct sqlite3; -struct sqlite3_stmt; -struct cert; -int sqlite3_finalize(sqlite3_stmt *); - #include "vector.hh" #include #include -#include "numeric_vocab.hh" #include "vocab.hh" -#include "paths.hh" -#include "cleanup.hh" #include "roster.hh" #include "graph.hh" -// FIXME: would be better not to include this everywhere -#include "outdated_indicator.hh" -#include "lru_writeback_cache.hh" +class app_state; +struct cert; +struct date_t; +struct globish; +class key_store; +class outdated_indicator; +class rev_height; +struct revision_t; +class transaction_guard; // this file defines a public, typed interface to the database. // the database class encapsulates all knowledge about sqlite, @@ -69,97 +67,32 @@ int sqlite3_finalize(sqlite3_stmt *); // it makes the code confusing, I know. this is possibly the worst part of // the program. I don't know if there's any way to make it clearer. -class transaction_guard; -class app_state; -class key_store; -struct revision_t; -struct query; -class rev_height; -struct globish; -struct date_t; -struct project_t; +class database_impl; class database { // // --== Opening the database and schema checking ==-- // -private: - system_path filename; - app_state * __app; - struct sqlite3 * __sql; - - enum open_mode { normal_mode = 0, - schema_bypass_mode, - format_bypass_mode }; - - void install_functions(); - struct sqlite3 * sql(enum open_mode mode = normal_mode); - - void check_filename(); - void check_db_exists(); - void check_db_nonexistent(); - void open(); - void close(); - void check_format(); - public: database(system_path const & file); ~database(); - void set_app(app_state * app); - void set_filename(system_path const & file); system_path get_filename(); bool is_dbfile(any_path const & file); + bool database_specified(); + void check_is_not_rosterified(); + void ensure_open(); void ensure_open_for_format_changes(); private: void ensure_open_for_maintenance(); -public: - void check_is_not_rosterified(); - bool database_specified(); // - // --== Basic SQL interface and statement caching ==-- - // -private: - struct statement { - statement() : count(0), stmt(0, sqlite3_finalize) {} - int count; - cleanup_ptr stmt; - }; - std::map statement_cache; - - typedef std::vector< std::vector > results; - void fetch(results & res, - int const want_cols, int const want_rows, - query const & q); - void execute(query const & q); - - bool table_has_entry(std::string const & key, std::string const & column, - std::string const & table); - - // - // --== Generic database metadata gathering ==-- - // -private: - std::string count(std::string const & table); - std::string space(std::string const & table, - std::string const & concatenated_columns, - u64 & total); - unsigned int page_size(); - unsigned int cache_size(); - - // // --== Transactions ==-- // private: - int transaction_level; - bool transaction_exclusive; - void begin_transaction(bool exclusive); - void commit_transaction(); - void rollback_transaction(); friend class transaction_guard; // @@ -170,98 +103,11 @@ public: typedef boost::shared_ptr roster_t_cp; typedef boost::shared_ptr marking_map_cp; typedef std::pair cached_roster; -private: - struct roster_size_estimator - { - unsigned long operator() (cached_roster const &); - }; - struct roster_writeback_manager - { - database & db; - roster_writeback_manager(database & db) : db(db) {} - void writeout(revision_id const &, cached_roster const &); - }; - LRUWritebackCache - roster_cache; - size_t size_delayed_file(file_id const & id, file_data const & dat); - bool have_delayed_file(file_id const & id); - void load_delayed_file(file_id const & id, file_data & dat); - void cancel_delayed_file(file_id const & id); - void drop_or_cancel_file(file_id const & id); - void schedule_delayed_file(file_id const & id, file_data const & dat); - - std::map delayed_files; - size_t delayed_writes_size; - - void flush_delayed_writes(); - void clear_delayed_writes(); - void write_delayed_file(file_id const & new_id, - file_data const & dat); - - void write_delayed_roster(revision_id const & new_id, - roster_t const & roster, - marking_map const & marking); - // // --== Reading/writing delta-compressed objects ==-- // -private: - // "do we have any entry for 'ident' that is a base version" - bool file_or_manifest_base_exists(hexenc const & ident, - std::string const & table); - bool roster_base_stored(revision_id const & ident); - bool roster_base_available(revision_id const & ident); - - // "do we have any entry for 'ident' that is a delta" - bool delta_exists(std::string const & ident, - std::string const & table); - - bool delta_exists(std::string const & ident, - std::string const & base, - std::string const & table); - - void get_file_or_manifest_base_unchecked(hexenc const & new_id, - data & dat, - std::string const & table); - void get_file_or_manifest_delta_unchecked(hexenc const & ident, - hexenc const & base, - delta & del, - std::string const & table); - void get_roster_base(std::string const & ident, - roster_t & roster, marking_map & marking); - void get_roster_delta(std::string const & ident, - std::string const & base, - roster_delta & del); - friend struct file_and_manifest_reconstruction_graph; - friend struct roster_reconstruction_graph; - void get_version(hexenc const & ident, - data & dat, - std::string const & data_table, - std::string const & delta_table); - - void drop(std::string const & base, - std::string const & table); - void put_file_delta(file_id const & ident, - file_id const & base, - file_delta const & del); -private: - void put_roster_delta(revision_id const & ident, - revision_id const & base, - roster_delta const & del); - void put_version(hexenc const & old_id, - hexenc const & new_id, - delta const & del, - std::string const & data_table, - std::string const & delta_table); - - void put_roster(revision_id const & rev_id, - roster_t_cp const & roster, - marking_map_cp const & marking); - public: - bool file_version_exists(file_id const & ident); bool revision_exists(revision_id const & ident); bool roster_link_exists_for_revision(revision_id const & ident); @@ -291,6 +137,21 @@ public: void get_manifest_version(manifest_id const & ident, manifest_data & dat); +private: + bool file_or_manifest_base_exists(hexenc const & ident, + std::string const & table); + bool delta_exists(std::string const & ident, + std::string const & table); + void put_file_delta(file_id const & ident, + file_id const & base, + file_delta const & del); + + friend void rcs_put_raw_file_edge(hexenc const & old_id, + hexenc const & new_id, + delta const & del, + database & db); + + // // --== The ancestry graph ==-- // @@ -307,10 +168,6 @@ public: void get_revision_manifest(revision_id const & cid, manifest_id & mid); -private: - // helper - void get_ids(std::string const & table, std::set< hexenc > & ids); -public: void get_revision_ids(std::set & ids); // this is exposed for 'db check': void get_file_ids(std::set & ids); @@ -318,8 +175,6 @@ public: // // --== Revision reading/writing ==-- // -private: - void deltify_revision(revision_id const & rid); public: void get_revision(revision_id const & ident, revision_t & cs); @@ -333,12 +188,12 @@ public: bool put_revision(revision_id const & new_id, revision_data const & dat); +private: + void deltify_revision(revision_id const & rid); + // // --== Rosters ==-- // -private: - u64 next_id_from_table(std::string const & table); - void get_roster_version(revision_id const & ros_id, cached_roster & cr); public: node_id next_node_id(); @@ -364,17 +219,18 @@ public: void get_file_content(revision_id const & id, node_id const & nid, file_id & content); + private: - struct extractor; - struct file_content_extractor; - struct markings_extractor; - void extract_from_deltas(revision_id const & id, extractor & x); + void get_roster_version(revision_id const & ros_id, + cached_roster & cr); + void put_roster(revision_id const & rev_id, + roster_t_cp const & roster, + marking_map_cp const & marking); + // // --== Keys ==-- // -private: - void get_keys(std::string const & table, std::vector & keys); public: void get_key_ids(std::vector & pubkeys); void get_key_ids(globish const & pattern, @@ -401,43 +257,7 @@ public: // --== Certs ==-- // // note: this section is ridiculous. please do something about it. -private: - bool cert_exists(cert const & t, - std::string const & table); - void put_cert(cert const & t, std::string const & table); - void results_to_certs(results const & res, - std::vector & certs); - - void get_certs(std::vector< cert > & certs, - std::string const & table); - - void get_certs(hexenc const & ident, - std::vector< cert > & certs, - std::string const & table); - - void get_certs(cert_name const & name, - std::vector< cert > & certs, - std::string const & table); - - void get_certs(hexenc const & ident, - cert_name const & name, - std::vector< cert > & certs, - std::string const & table); - - void get_certs(hexenc const & ident, - cert_name const & name, - base64 const & val, - std::vector< cert > & certs, - std::string const & table); - - void get_certs(cert_name const & name, - base64 const & val, - std::vector & certs, - std::string const & table); - - outdated_indicator_factory cert_stamper; public: - bool revision_cert_exists(revision const & cert); bool revision_cert_exists(hexenc const & hash); @@ -566,14 +386,8 @@ public: // for kill_tag_locally: void delete_tag_named(cert_value const & tag); - // misc -private: - friend void rcs_put_raw_file_edge(hexenc const & old_id, - hexenc const & new_id, - delta const & del, - database & db); public: - // branches + // branches outdated_indicator get_branches(std::vector & names); outdated_indicator get_branches(globish const & glob, std::vector & names); @@ -609,6 +423,8 @@ public: revision_t const & rev); // FIXME: quick hack to make these hooks available via the database context + void set_app(app_state * app); + bool hook_get_manifest_cert_trust(std::set const & signers, hexenc const & id, cert_name const & name, cert_value const & val); bool hook_get_revision_cert_trust(std::set const & signers, @@ -628,6 +444,10 @@ public: void set_opt_branchname(branch_name const & branchname); key_store & get_key_store(); + +private: + database_impl *imp; + app_state * __app; }; // not a member function, defined in database_check.cc @@ -730,13 +550,13 @@ class transaction_guard class transaction_guard { - bool committed; - database & db; - bool exclusive; + database_impl * imp; size_t const checkpoint_batch_size; size_t const checkpoint_batch_bytes; size_t checkpointed_calls; size_t checkpointed_bytes; + bool committed; + bool const exclusive; public: transaction_guard(database & d, bool exclusive=true, size_t checkpoint_batch_size=1000, ============================================================ --- database_check.cc 27daa8b1b77f291237f4ed84eacd9f2e354ac810 +++ database_check.cc 4a77f59692f47b501a73ede9369b8a5923c161d7 @@ -20,6 +20,7 @@ #include "transforms.hh" #include "cert.hh" #include "rev_height.hh" +#include "outdated_indicator.hh" // the database has roughly the following structure // ============================================================ --- revision.cc 47b8a2189742c9513c3234e6315551335c8a8697 +++ revision.cc 5cf8962282b67f85579d9ed30ce2b074f302f002 @@ -40,6 +40,7 @@ #include "safe_map.hh" #include "legacy.hh" #include "rev_height.hh" +#include "outdated_indicator.hh" using std::back_inserter; using std::copy; ============================================================ --- update.cc 5a40558d142d4ec20ff1ec6b7ef08e0be92d902f +++ update.cc 260394c808cc8f1c9c783a16274f60b8cbaed276 @@ -9,7 +9,7 @@ #include "base.hh" #include -#include +#include "safe_map.hh" #include "vector.hh" #include "lexical_cast.hh"