# # # patch "automate.cc" # from [1c29dbf96103bd040cda530c7bba9a2d3fb2fbfa] # to [1bdd4c34b344b1c92f80a64f08e97b6a15bb9a7b] # # patch "cert.cc" # from [2fd05f14b5afc1de9f24b7eeca4c186b598ce0cd] # to [3a9169e5b974a447189069ee3ad9015947540544] # # patch "database.cc" # from [16e45849d18e0a542d0e05a0b743766f44e2b8cf] # to [c609818513960b330c4d833ba6abe267fa635c47] # # patch "database.hh" # from [5b08939400abfbf9a1889039bfec05cd78001fc2] # to [dbcf6a4985c8319fe05b5d012a4772316d00a07b] # # patch "key_store.cc" # from [647396cd291637e0c1d4c2465ca372f19a5762a3] # to [866ba8f4d52db4fef939d0e88a6a3edc9cc4b771] # # patch "key_store.hh" # from [d33d2d1c872225920d74afe7b9350edae71607a2] # to [40ecab081203aedf120cbd0147845222b4176282] # # patch "netsync.cc" # from [a4c703b9007b142478b1d8ecddf9d6c389e6d00c] # to [69251cce14272cc7bf5ff0b27fcf4f8e6cc434a4] # # patch "packet.cc" # from [1dd3e1b56ecdb676fc5bff8b948d3ccf79ad5900] # to [9fc7baf9b1b6e5daf0f947f2548af6ad0eb1f7e7] # # patch "rcs_import.cc" # from [93b52aa3c730c65e46c5e62f4566a3f6478c80bb] # to [901a0c2f1ed07d827db3bf85ac7817ecaa0339b5] # # patch "revision.cc" # from [4ff3ab0f710c886b388cad55b498cfa7189c260f] # to [94ffffda8ea5623d05fef8d062b2a4401599578d] # # patch "tests/automate_put_revision_unknown_file/__driver__.lua" # from [dc7ec109a51b1e8ccc29e876be4d40bad5363dbf] # to [2344d6e0926df64c127ed88a36130d93bdf91b11] # ============================================================ --- automate.cc 1c29dbf96103bd040cda530c7bba9a2d3fb2fbfa +++ automate.cc 1bdd4c34b344b1c92f80a64f08e97b6a15bb9a7b @@ -1630,8 +1630,7 @@ AUTOMATE(put_file, N_("[FILEID] CONTENTS file_data dat(idx(args, 0)()); calculate_ident(dat, sha1sum); - if (!app.db.file_version_exists(sha1sum)) - app.db.put_file(sha1sum, dat); + app.db.put_file(sha1sum, dat); } else if (args.size() == 2) { @@ -1640,19 +1639,17 @@ AUTOMATE(put_file, N_("[FILEID] CONTENTS file_id base_id(idx(args, 0)()); N(app.db.file_version_exists(base_id), F("no file version %s found in database") % base_id); - + + // put_file_version won't do anything if the target ID already exists, + // but we can save the delta calculation by checking here too if (!app.db.file_version_exists(sha1sum)) { file_data olddat; app.db.get_file_version(base_id, olddat); delta del; diff(olddat.inner(), dat.inner(), del); - L(FL("data size %d, delta size %d") % dat.inner()().size() % del().size()); - if (dat.inner()().size() <= del().size()) - // the data is smaller or of equal size to the patch - app.db.put_file(sha1sum, dat); - else - app.db.put_file_version(base_id, sha1sum, file_delta(del)); + + app.db.put_file_version(base_id, sha1sum, file_delta(del)); } } else I(false); @@ -1705,15 +1702,10 @@ AUTOMATE(put_revision, N_("REVISION-DATA revision_id id; calculate_ident(rev, id); - if (app.db.revision_exists(id)) - P(F("revision %s already present in the database, skipping") % id); - else - { - transaction_guard tr(app.db); - rev.made_for = made_for_database; - app.db.put_revision(id, rev); - tr.commit(); - } + // If the database refuses the revision, make sure this is because it's + // already there. + E(app.db.put_revision(id, rev) || app.db.revision_exists(id), + F("missing prerequisite for revision %s") % id); output << id << '\n'; } ============================================================ --- cert.cc 2fd05f14b5afc1de9f24b7eeca4c186b598ce0cd +++ cert.cc 3a9169e5b974a447189069ee3ad9015947540544 @@ -403,8 +403,7 @@ calculate_cert(app_state & app, cert & t cert_signable_text(t, signed_text); load_key_pair(app, t.key, kp); - if (!app.db.public_key_exists(t.key)) - app.db.put_key(t.key, kp.pub); + app.db.put_key(t.key, kp.pub); make_signature(app, t.key, kp.priv, signed_text, t.sig); } ============================================================ --- database.cc 16e45849d18e0a542d0e05a0b743766f44e2b8cf +++ database.cc c609818513960b330c4d833ba6abe267fa635c47 @@ -1681,7 +1681,10 @@ database::put_file(file_id const & id, database::put_file(file_id const & id, file_data const & dat) { - schedule_delayed_file(id, dat); + if (file_version_exists(id)) + L(FL("file version '%s' already exists in db") % id); + else + schedule_delayed_file(id, dat); } void @@ -1692,6 +1695,19 @@ database::put_file_version(file_id const file_data old_data, new_data; file_delta reverse_delta; + if (file_version_exists(new_id)) + { + L(FL("file version '%s' already exists in db") % new_id); + return; + } + + if (!file_version_exists(old_id)) + { + W(F("file preimage '%s' missing in db") % old_id); + W(F("dropping delta '%s' -> '%s'") % old_id % new_id); + return; + } + get_file_version(old_id, old_data); { data tmp; @@ -1939,7 +1955,7 @@ database::deltify_revision(revision_id c } -void +bool database::put_revision(revision_id const & new_id, revision_t const & rev) { @@ -1947,37 +1963,72 @@ database::put_revision(revision_id const MM(rev); I(!null_id(new_id)); - I(!revision_exists(new_id)); + if (revision_exists(new_id)) + { + L(FL("revision '%s' already exists in db") % new_id); + return false; + } + I(rev.made_for == made_for_database); rev.check_sane(); - revision_data d; - MM(d.inner()); - write_revision(rev, d); // Phase 1: confirm the revision makes sense, and we the required files // actually exist - { - revision_id tmp; - MM(tmp); - calculate_ident(d, tmp); - I(tmp == new_id); - for (edge_map::const_iterator e = rev.edges.begin(); e != rev.edges.end(); ++e) - { - cset const & cs = edge_changes(e); - for (map::const_iterator - i = cs.files_added.begin(); i != cs.files_added.end(); ++i) - I(file_version_exists(i->second)); - for (map >::const_iterator - i = cs.deltas_applied.begin(); i != cs.deltas_applied.end(); ++i) - I(file_version_exists(i->second.second)); - } - } + for (edge_map::const_iterator i = rev.edges.begin(); + i != rev.edges.end(); ++i) + { + if (!edge_old_revision(i).inner()().empty() + && !revision_exists(edge_old_revision(i))) + { + W(F("missing prerequisite revision '%s'") % edge_old_revision(i)); + W(F("dropping revision '%s'") % new_id); + return false; + } + for (map::const_iterator a + = edge_changes(i).files_added.begin(); + a != edge_changes(i).files_added.end(); ++a) + { + if (! file_version_exists(a->second)) + { + W(F("missing prerequisite file '%s'") % a->second); + W(F("dropping revision '%s'") % new_id); + return false; + } + } + + for (map >::const_iterator d + = edge_changes(i).deltas_applied.begin(); + d != edge_changes(i).deltas_applied.end(); ++d) + { + I(!delta_entry_src(d).inner()().empty()); + I(!delta_entry_dst(d).inner()().empty()); + + if (! file_version_exists(delta_entry_src(d))) + { + W(F("missing prerequisite file pre-delta '%s'") + % delta_entry_src(d)); + W(F("dropping revision '%s'") % new_id); + return false; + } + + if (! file_version_exists(delta_entry_dst(d))) + { + W(F("missing prerequisite file post-delta '%s'") + % delta_entry_dst(d)); + W(F("dropping revision '%s'") % new_id); + return false; + } + } + } + transaction_guard guard(*this); // Phase 2: Write the revision data (inside a transaction) + revision_data d; + write_revision(rev, d); gzip d_packed; encode_gzip(d.inner(), d_packed); execute(query("INSERT INTO revisions VALUES(?, ?)") @@ -2008,6 +2059,7 @@ database::put_revision(revision_id const // Finally, commit. guard.commit(); + return true; } void @@ -2064,13 +2116,13 @@ database::put_roster_for_revision(revisi put_roster(new_id, ros, mm); } -void +bool database::put_revision(revision_id const & new_id, revision_data const & dat) { revision_t rev; read_revision(dat, rev); - put_revision(new_id, rev); + return put_revision(new_id, rev); } @@ -2245,21 +2297,34 @@ database::get_key(rsa_keypair_id const & encode_base64(rsa_pub_key(res[0][0]), pub_encoded); } -void +bool database::put_key(rsa_keypair_id const & pub_id, base64 const & pub_encoded) { + if (public_key_exists(pub_id)) + { + base64 tmp; + get_key(pub_id, tmp); + if (!keys_match(pub_id, tmp, pub_id, pub_encoded)) + W(F("key '%s' is not equal to key '%s' in database") % pub_id % pub_id); + L(FL("skipping existing public key %s") % pub_id); + return false; + } + + L(FL("putting public key %s") % pub_id); + hexenc thash; key_hash_code(pub_id, pub_encoded, thash); I(!public_key_exists(thash)); - E(!public_key_exists(pub_id), - F("another key with name '%s' already exists") % pub_id); + rsa_pub_key pub_key; decode_base64(pub_encoded, pub_key); execute(query("INSERT INTO public_keys VALUES(?, ?, ?)") % text(thash()) % text(pub_id()) % blob(pub_key())); + + return true; } void @@ -2451,11 +2516,27 @@ database::revision_cert_exists(revision< return cert_exists(cert.inner(), "revision_certs"); } -void +bool database::put_revision_cert(revision const & cert) { + if (revision_cert_exists(cert)) + { + L(FL("revision cert on '%s' already exists in db") + % cert.inner().ident); + return false; + } + + if (!revision_exists(revision_id(cert.inner().ident))) + { + W(F("cert revision '%s' does not exist in db") + % cert.inner().ident); + W(F("dropping cert")); + return false; + } + put_cert(cert.inner(), "revision_certs"); cert_stamper.note_change(); + return true; } outdated_indicator ============================================================ --- database.hh 5b08939400abfbf9a1889039bfec05cd78001fc2 +++ database.hh dbcf6a4985c8319fe05b5d012a4772316d00a07b @@ -317,10 +317,10 @@ public: void get_revision(revision_id const & ident, revision_data & dat); - void put_revision(revision_id const & new_id, + bool put_revision(revision_id const & new_id, revision_t const & rev); - void put_revision(revision_id const & new_id, + bool put_revision(revision_id const & new_id, revision_data const & dat); // @@ -381,7 +381,7 @@ public: void get_key(rsa_keypair_id const & ident, base64 & pub_encoded); - void put_key(rsa_keypair_id const & ident, + bool put_key(rsa_keypair_id const & ident, base64 const & pub_encoded); void delete_public_key(rsa_keypair_id const & pub_id); @@ -430,7 +430,7 @@ public: bool revision_cert_exists(revision const & cert); bool revision_cert_exists(hexenc const & hash); - void put_revision_cert(revision const & cert); + bool put_revision_cert(revision const & cert); // this variant has to be rather coarse and fast, for netsync's use outdated_indicator get_revision_cert_nobranch_index(std::vector< std::pair, ============================================================ --- key_store.cc 647396cd291637e0c1d4c2465ca372f19a5762a3 +++ key_store.cc 866ba8f4d52db4fef939d0e88a6a3edc9cc4b771 @@ -104,15 +104,17 @@ key_store::ensure_in_database(rsa_keypai key_store::ensure_in_database(rsa_keypair_id const & ident) { maybe_read_key_dir(); - if (app->db.public_key_exists(ident)) + map::iterator i = keys.find(ident); + + // if this object does not have the key, the database had better. + if (i == keys.end()) { - L(FL("public key '%s' is already in db, not loading") % ident); + I(app->db.public_key_exists(ident)); return; } - map::iterator i = keys.find(ident); - I(i != keys.end()); - app->db.put_key(ident, i->second.pub); - L(FL("loaded public key '%s' into db") % ident); + + if (app->db.put_key(ident, i->second.pub)) + L(FL("loaded public key '%s' into db") % ident); } bool @@ -210,7 +212,7 @@ key_store::write_key(rsa_keypair_id cons write_data(file, dat, key_dir); } -void +bool key_store::put_key_pair(rsa_keypair_id const & ident, keypair const & kp) { @@ -224,13 +226,15 @@ key_store::put_key_pair(rsa_keypair_id c key_hash_code(ident, kp.pub, hash); I(hashes.insert(make_pair(hash, ident)).second); write_key(ident); + return true; } else { - E(/*keys_match(ident, res.first->second.priv, ident, kp.priv) - && */keys_match(ident, res.first->second.pub, ident, kp.pub), + E(keys_match(ident, res.first->second.pub, ident, kp.pub), F("Cannot store key '%s'; a different key by that name exists.") % ident); + L(FL("skipping existing key pair %s") % ident); + return false; } } ============================================================ --- key_store.hh d33d2d1c872225920d74afe7b9350edae71607a2 +++ key_store.hh 40ecab081203aedf120cbd0147845222b4176282 @@ -43,7 +43,7 @@ public: void get_key_pair(rsa_keypair_id const & ident, keypair & kp); - void put_key_pair(rsa_keypair_id const & ident, + bool put_key_pair(rsa_keypair_id const & ident, keypair const & kp); void delete_key(rsa_keypair_id const & ident); ============================================================ --- netsync.cc a4c703b9007b142478b1d8ecddf9d6c389e6d00c +++ netsync.cc 69251cce14272cc7bf5ff0b27fcf4f8e6cc434a4 @@ -1313,11 +1313,9 @@ session::process_hello_cmd(rsa_keypair_i "their key's fingerprint: %s") % peer_id % their_key_hash); app.db.set_var(their_key_key, var_value(their_key_hash())); } - if (!app.db.public_key_exists(their_key_hash)) - { - W(F("saving public key for %s to database") % their_keyname); - app.db.put_key(their_keyname, their_key_encoded); - } + if (app.db.put_key(their_keyname, their_key_encoded)) + W(F("saving public key for %s to database") % their_keyname); + { hexenc hnonce; encode_hexenc(nonce, hnonce); ============================================================ --- packet.cc 1dd3e1b56ecdb676fc5bff8b948d3ccf79ad5900 +++ packet.cc 9fc7baf9b1b6e5daf0f947f2548af6ad0eb1f7e7 @@ -76,12 +76,6 @@ packet_db_writer::consume_file_data(file packet_db_writer::consume_file_data(file_id const & ident, file_data const & dat) { - if (app.db.file_version_exists(ident)) - { - L(FL("file version '%s' already exists in db") % ident); - return; - } - transaction_guard guard(app.db); app.db.put_file(ident, dat); guard.commit(); @@ -93,22 +87,7 @@ packet_db_writer::consume_file_delta(fil file_delta const & del) { transaction_guard guard(app.db); - - if (app.db.file_version_exists(new_id)) - { - L(FL("file version '%s' already exists in db") % new_id); - return; - } - - if (!app.db.file_version_exists(old_id)) - { - W(F("file preimage '%s' missing in db") % old_id); - W(F("dropping delta '%s' -> '%s'") % old_id % new_id); - return; - } - app.db.put_file_version(old_id, new_id, del); - guard.commit(); } @@ -118,67 +97,9 @@ packet_db_writer::consume_revision_data( { MM(ident); transaction_guard guard(app.db); - if (app.db.revision_exists(ident)) - { - L(FL("revision '%s' already exists in db") % ident); - return; - } - - revision_t rev; - MM(rev); - read_revision(dat, rev); - - for (edge_map::const_iterator i = rev.edges.begin(); - i != rev.edges.end(); ++i) - { - if (!edge_old_revision(i).inner()().empty() - && !app.db.revision_exists(edge_old_revision(i))) - { - W(F("missing prerequisite revision '%s'") % edge_old_revision(i)); - W(F("dropping revision '%s'") % ident); - return; - } - - for (map::const_iterator a - = edge_changes(i).files_added.begin(); - a != edge_changes(i).files_added.end(); ++a) - { - if (! app.db.file_version_exists(a->second)) - { - W(F("missing prerequisite file '%s'") % a->second); - W(F("dropping revision '%s'") % ident); - return; - } - } - - for (map >::const_iterator d - = edge_changes(i).deltas_applied.begin(); - d != edge_changes(i).deltas_applied.end(); ++d) - { - I(!delta_entry_src(d).inner()().empty()); - I(!delta_entry_dst(d).inner()().empty()); - - if (! app.db.file_version_exists(delta_entry_src(d))) - { - W(F("missing prerequisite file pre-delta '%s'") - % delta_entry_src(d)); - W(F("dropping revision '%s'") % ident); - return; - } - - if (! app.db.file_version_exists(delta_entry_dst(d))) - { - W(F("missing prerequisite file post-delta '%s'") - % delta_entry_dst(d)); - W(F("dropping revision '%s'") % ident); - return; - } - } - } - - app.db.put_revision(ident, dat); - if (on_revision_written) - on_revision_written(ident); + if (app.db.put_revision(ident, dat)) + if (on_revision_written) + on_revision_written(ident); guard.commit(); } @@ -186,26 +107,9 @@ packet_db_writer::consume_revision_cert( packet_db_writer::consume_revision_cert(revision const & t) { transaction_guard guard(app.db); - - if (app.db.revision_cert_exists(t)) - { - L(FL("revision cert on '%s' already exists in db") - % t.inner().ident); - return; - } - - if (!app.db.revision_exists(revision_id(t.inner().ident))) - { - W(F("cert revision '%s' does not exist in db") - % t.inner().ident); - W(F("dropping cert")); - return; - } - - app.db.put_revision_cert(t); - if (on_cert_written) - on_cert_written(t.inner()); - + if (app.db.put_revision_cert(t)) + if (on_cert_written) + on_cert_written(t.inner()); guard.commit(); } @@ -215,22 +119,9 @@ packet_db_writer::consume_public_key(rsa base64< rsa_pub_key > const & k) { transaction_guard guard(app.db); - - if (app.db.public_key_exists(ident)) - { - base64 tmp; - app.db.get_key(ident, tmp); - if (!keys_match(ident, tmp, ident, k)) - W(F("key '%s' is not equal to key '%s' in database") % ident % ident); - L(FL("skipping existing public key %s") % ident); - return; - } - - L(FL("putting public key %s") % ident); - app.db.put_key(ident, k); - if (on_pubkey_written) - on_pubkey_written(ident); - + if (app.db.put_key(ident, k)) + if (on_pubkey_written) + on_pubkey_written(ident); guard.commit(); } @@ -239,17 +130,9 @@ packet_db_writer::consume_key_pair(rsa_k keypair const & kp) { transaction_guard guard(app.db); - - if (app.keys.key_pair_exists(ident)) - { - L(FL("skipping existing key pair %s") % ident); - return; - } - - app.keys.put_key_pair(ident, kp); - if (on_keypair_written) - on_keypair_written(ident); - + if (app.keys.put_key_pair(ident, kp)) + if (on_keypair_written) + on_keypair_written(ident); guard.commit(); } ============================================================ --- rcs_import.cc 93b52aa3c730c65e46c5e62f4566a3f6478c80bb +++ rcs_import.cc 901a0c2f1ed07d827db3bf85ac7817ecaa0339b5 @@ -685,12 +685,8 @@ import_rcs_file_with_cvs(string const & cvs.set_filename (filename, fid); cvs.index_branchpoint_symbols (r); + db.put_file(fid, file_data(dat)); - if (! db.file_version_exists (fid)) - { - db.put_file(fid, file_data(dat)); - } - { // create the head state in case it is a loner // cvs_key k; @@ -1347,16 +1343,11 @@ cluster_consumer::store_revisions() { for (vector::const_iterator i = preps.begin(); i != preps.end(); ++i) - { - if (! app.db.revision_exists(i->rid)) - { - data tmp; - write_revision(*(i->rev), tmp); - app.db.put_revision(i->rid, *(i->rev)); - store_auxiliary_certs(*i); - ++n_revisions; - } - } + if (app.db.put_revision(i->rid, *(i->rev))) + { + store_auxiliary_certs(*i); + ++n_revisions; + } } void ============================================================ --- revision.cc 4ff3ab0f710c886b388cad55b498cfa7189c260f +++ revision.cc 94ffffda8ea5623d05fef8d062b2a4401599578d @@ -925,11 +925,8 @@ void anc_graph::write_certs() cert new_cert; make_simple_cert(rev.inner(), name, val, app, new_cert); revision rcert(new_cert); - if (! app.db.revision_cert_exists(rcert)) - { - ++n_certs_out; - app.db.put_revision_cert(rcert); - } + if (app.db.put_revision_cert(rcert)) + ++n_certs_out; } } } @@ -1613,17 +1610,10 @@ anc_graph::construct_revisions_from_ance P(F("------------------------------------------------")); */ - if (!app.db.revision_exists (new_rid)) - { - L(FL("mapped node %d to revision %s") % child % new_rid); - app.db.put_revision(new_rid, rev); - ++n_revs_out; - } - else - { - L(FL("skipping already existing revision %s") % new_rid); - } - + L(FL("mapped node %d to revision %s") % child % new_rid); + if (app.db.put_revision(new_rid, rev)) + ++n_revs_out; + // Mark this child as done, hooray! safe_insert(done, child); ============================================================ --- tests/automate_put_revision_unknown_file/__driver__.lua dc7ec109a51b1e8ccc29e876be4d40bad5363dbf +++ tests/automate_put_revision_unknown_file/__driver__.lua 2344d6e0926df64c127ed88a36130d93bdf91b11 @@ -5,8 +5,13 @@ mtn_setup() mtn_setup() -- added files are checked -rev = "format_version \"1\"\n\nnew_manifest [0000000000000000000000000000000000000000]\n\nold_revision []\n\nadd_dir \"\"\n\nadd_file \"foo.txt\"\ncontent [1234567890123456789012345678901234567890]" -check(mtn("automate", "put_revision", rev), 3, false, false) +rev = ("format_version \"1\"\n\n".. + "new_manifest [0000000000000000000000000000000000000000]\n\n".. + "old_revision []\n\n".. + "add_dir \"\"\n\n".. + "add_file \"foo.txt\"\n".. + " content [1234567890123456789012345678901234567890]") +check(mtn("automate", "put_revision", rev), 1, false, false) addfile("foo", "asdf") commit() @@ -14,5 +19,10 @@ rhash = base_revision() rhash = base_revision() -- modified files are also checked +rev = ("format_version \"1\"\n\n".. + "new_manifest [0000000000000000000000000000000000000000]\n\n".. + "old_revision [" .. rhash .. "]\n\n".. + "patch \"foo\"\n".. + " from [" .. fhash .. "]\n".. + " to [0000000000000000000000000000000000000000]") +check(mtn("automate", "put_revision", rev), 1, false, false) -rev = "format_version \"1\"\n\nnew_manifest [0000000000000000000000000000000000000000]\n\nold_revision [" .. rhash .. "]]\n\npatch \"foo\" from [" .. fhash .. "] to [0000000000000000000000000000000000000000]" -check(mtn("automate", "put_revision", rev), 3, false, false)