# # # patch "cmd_scgi.cc" # from [b526e6a8d5d3e6f20689fb80511fd67c3ba30512] # to [799f752acb288807b8f0d73b3ae06b5aef7278fc] # # patch "gsync.cc" # from [d81cf073a1d92362cd99c270106b18f468bcfd6d] # to [4445f0113b5e93e4da95a4df906e80f5fd9f2a5c] # # patch "gsync.hh" # from [dff14231ff185742fdbd7941cdb44c0e94218edd] # to [6a96773b0c17214aab83a0540d9bb41f875263c4] # # patch "http_client.cc" # from [0d6e55b1421bcd7a4047fe5fc4b0b7d296dd2e30] # to [69ffc655e3c8de7de8eb4a5d3ca4c97ca583f0d5] # # patch "http_client.hh" # from [807bce3cae5c6ed4881393fddf658f4dc7020165] # to [fc8581a72087cf0924b2fa0dab5353756e07f560] # # patch "json_msgs.cc" # from [6c003e8759e0f22ae7b3e7f971937ed6da834165] # to [ed870e3e149d6039346590715486227ae139ed54] # # patch "json_msgs.hh" # from [86a66c961fb6eef48b00414a74865929d662d855] # to [ca18a9c15f3c5056ca002ed6a2bae3a7327ab9c2] # ============================================================ --- cmd_scgi.cc b526e6a8d5d3e6f20689fb80511fd67c3ba30512 +++ cmd_scgi.cc 799f752acb288807b8f0d73b3ae06b5aef7278fc @@ -169,6 +169,9 @@ do_cmd(database & db, json_io::json_obje { set request_revs; + revision_id rid; + revision_t rev; + if (decode_msg_inquire_request(cmd_obj, request_revs)) { L(FL("inquiring %d revisions") % request_revs.size()); @@ -202,6 +205,13 @@ do_cmd(database & db, json_io::json_obje toposort(db, response_set, response_revs); return encode_msg_descendants_response(response_revs); } + else if (decode_msg_put_rev_request(cmd_obj, rid, rev)) + { + revision_id check; + calculate_ident(rev, check); + I(rid == check); + return encode_msg_put_rev_response(); + } else { return encode_msg_error("unknown request"); ============================================================ --- gsync.cc d81cf073a1d92362cd99c270106b18f468bcfd6d +++ gsync.cc 4445f0113b5e93e4da95a4df906e80f5fd9f2a5c @@ -15,6 +15,7 @@ #include #include "constants.hh" +#include "cset.hh" #include "database.hh" #include "globish.hh" #include "graph.hh" @@ -187,7 +188,31 @@ push_revs(database & db, for (vector::const_iterator i = outbound_revs.begin(); i != outbound_revs.end(); ++i) { - ch.push_rev(*i); + revision_t rev; + db.get_revision(*i, rev); + + for (edge_map::const_iterator e = rev.edges.begin(); + e != rev.edges.end(); ++e) + { + cset const & cs = edge_changes(e); + for (map::const_iterator + f = cs.files_added.begin(); f != cs.files_added.end(); ++f) + { + file_data data; + db.get_file_version(f->second, data); + ch.push_file_data(f->second, data); + } + + for (map >::const_iterator + f = cs.deltas_applied.begin(); f != cs.deltas_applied.end(); ++f) + { + file_delta delta; + db.get_arbitrary_file_delta(f->second.first, f->second.second, delta); + ch.push_file_delta(f->second.first, f->second.second, delta); + } + } + + ch.push_rev(*i, rev); } } @@ -199,7 +224,31 @@ pull_revs(database & db, for (vector::const_iterator i = inbound_revs.begin(); i != inbound_revs.end(); ++i) { - ch.pull_rev(*i); + revision_t rev; + ch.pull_rev(*i, rev); + + for (edge_map::const_iterator e = rev.edges.begin(); + e != rev.edges.end(); ++e) + { + cset const & cs = edge_changes(e); + for (map::const_iterator + f = cs.files_added.begin(); f != cs.files_added.end(); ++f) + { + file_data data; + ch.pull_file_data(f->second, data); + //db.put_file(f->second, data); + } + + for (map >::const_iterator + f = cs.deltas_applied.begin(); f != cs.deltas_applied.end(); ++f) + { + file_delta delta; + ch.pull_file_delta(f->second.first, f->second.second, delta); + //db.put_file_version(f->second.first, f->second.second, delta); + } + } + + //db.put_revsion(*i, rev); } } @@ -257,7 +306,6 @@ run_gsync_protocol(lua_hooks & lua, data if (pulling) pull_revs(db, ch, inbound_revs); - } #ifdef BUILD_UNIT_TESTS @@ -270,22 +318,54 @@ public: public: test_channel(set & theirs) : theirs(theirs) - { }; + { }; void inquire_about_revs(set const & query_set, set & result) const - { - result.clear(); - for (set::const_iterator i = query_set.begin(); - i != query_set.end(); ++i) - if (theirs.find(*i) != theirs.end()) - result.insert(*i); - }; + { + result.clear(); + for (set::const_iterator i = query_set.begin(); + i != query_set.end(); ++i) + if (theirs.find(*i) != theirs.end()) + result.insert(*i); + }; - void push_rev(revision_id const & rid) const - { - I(false); - } + void get_descendants(std::set const & common_revs, + std::vector & inbound_revs) const + { + I(false); + } + + void push_file_data(file_id const & id, + file_data const & data) const + { + } + + void push_file_delta(file_id const & old_id, + file_id const & new_id, + file_delta const & delta) const + { + } + + void push_rev(revision_id const & rid, revision_t const & rev) const + { + } + + void pull_rev(revision_id const & rid, revision_t & rev) const + { + } + + void pull_file_data(file_id const & id, + file_data & data) const + { + } + + void pull_file_delta(file_id const & old_id, + file_id const & new_id, + file_delta & delta) const + { + } + }; UNIT_TEST(gsync, gsync_common_core) ============================================================ --- gsync.hh dff14231ff185742fdbd7941cdb44c0e94218edd +++ gsync.hh 6a96773b0c17214aab83a0540d9bb41f875263c4 @@ -20,6 +20,7 @@ class database; struct globish; class lua_hooks; class database; +class revision_t; class channel @@ -30,9 +31,21 @@ public: virtual void get_descendants(std::set const & common_revs, std::vector & inbound_revs) const = 0; - virtual void push_rev(revision_id const & rid) const = 0; - virtual void pull_rev(revision_id const & rid) const = 0; + virtual void push_file_data(file_id const & id, + file_data const & data) const = 0; + virtual void push_file_delta(file_id const & old_id, + file_id const & new_id, + file_delta const & delta) const = 0; + virtual void push_rev(revision_id const & rid, revision_t const & rev) const = 0; + virtual void pull_rev(revision_id const & rid, revision_t & rev) const = 0; + + virtual void pull_file_data(file_id const & id, + file_data & data) const = 0; + virtual void pull_file_delta(file_id const & old_id, + file_id const & new_id, + file_delta & delta) const = 0; + virtual ~channel() {} }; ============================================================ --- http_client.cc 0d6e55b1421bcd7a4047fe5fc4b0b7d296dd2e30 +++ http_client.cc 69ffc655e3c8de7de8eb4a5d3ca4c97ca583f0d5 @@ -107,10 +107,16 @@ http_client::transact_json(json_value_t io->flush(); L(FL("http_client: sent %d-byte body") % out.buf.size()); +// std::cerr << "json request" << std::endl +// << out.buf.data() << std::endl; + // Now read back the result string data; parse_http_response(data); +// std::cerr << "json response" << std::endl +// << data << std::endl; + json_io::input_source in(data, "scgi"); json_io::tokenizer tok(in); json_io::parser p(tok); @@ -219,15 +225,46 @@ void } void -http_channel::push_rev(revision_id const & rid) const +http_channel::push_file_data(file_id const & id, + file_data const & data) const { } void -http_channel::pull_rev(revision_id const & rid) const +http_channel::push_file_delta(file_id const & old_id, + file_id const & new_id, + file_delta const & delta) const { } +void +http_channel::push_rev(revision_id const & rid, revision_t const & rev) const +{ + json_value_t request = encode_msg_put_rev_request(rid, rev); + json_value_t response = client.transact_json(request); + E(decode_msg_put_rev_response(response), + F("received unexpected reply to 'put_rev_request' message")); +} + +void +http_channel::pull_rev(revision_id const & rid, revision_t & rev) const +{ +} + +void +http_channel::pull_file_data(file_id const & id, + file_data & data) const +{ +} + +void +http_channel::pull_file_delta(file_id const & old_id, + file_id const & new_id, + file_delta & delta) const +{ +} + + // Local Variables: // mode: C++ // fill-column: 76 ============================================================ --- http_client.hh 807bce3cae5c6ed4881393fddf658f4dc7020165 +++ http_client.hh fc8581a72087cf0924b2fa0dab5353756e07f560 @@ -67,8 +67,20 @@ public: virtual void get_descendants(std::set const & common_revs, std::vector & inbound_revs) const; - virtual void push_rev(revision_id const & rid) const; - virtual void pull_rev(revision_id const & rid) const; + virtual void push_file_data(file_id const & id, + file_data const & data) const; + virtual void push_file_delta(file_id const & old_id, + file_id const & new_id, + file_delta const & delta) const; + + virtual void push_rev(revision_id const & rid, revision_t const & rev) const; + virtual void pull_rev(revision_id const & rid, revision_t & rev) const; + + virtual void pull_file_data(file_id const & id, + file_data & data) const; + virtual void pull_file_delta(file_id const & old_id, + file_id const & new_id, + file_delta & delta) const; }; // Local Variables: ============================================================ --- json_msgs.cc 6c003e8759e0f22ae7b3e7f971937ed6da834165 +++ json_msgs.cc ed870e3e149d6039346590715486227ae139ed54 @@ -17,6 +17,9 @@ #include #include +#include + +using std::make_pair; using std::map; using std::set; using std::string; @@ -27,6 +30,7 @@ using json_io::json_object_t; using json_io::json_array_t; using json_io::json_object_t; +using boost::shared_ptr; namespace { @@ -46,6 +50,8 @@ namespace symbol const attr("attr"); symbol const value("value"); + symbol const changes("changes"); + // revision symbols symbol const old_revision("old_revision"); symbol const new_manifest("new_manifest"); @@ -64,19 +70,24 @@ namespace symbol const revs("revs"); symbol const error("error"); + // request/response pairs symbol const inquire_request("inquire_request"); symbol const inquire_response("inquire_response"); symbol const descendants_request("descendants_request"); symbol const descendants_response("descendants_response"); - symbol const get_rev("get_rev"); - symbol const get_full_rev("get_full_rev"); + symbol const put_rev_request("put_rev_request"); + symbol const put_rev_response("put_rev_response"); + symbol const get_rev_request("get_rev_request"); + symbol const get_rev_response("get_rev_request"); + + symbol const status("status"); + symbol const get_file_data("get_file_data"); symbol const get_file_delta("get_file_delta"); symbol const rev("rev"); - symbol const full_rev("full_rev"); symbol const file_data("file_data"); symbol const file_delta("file_delta"); } @@ -267,38 +278,38 @@ static void static void -cset_to_json(json_io::builder b, cset const & cs) +encode_cset(json_io::builder b, cset const & cs) { - for (set::const_iterator i = cs.nodes_deleted.begin(); - i != cs.nodes_deleted.end(); ++i) + for (set::const_iterator + i = cs.nodes_deleted.begin(); i != cs.nodes_deleted.end(); ++i) { b.add_obj()[syms::delete_node].str(i->as_internal()); } - for (map::const_iterator i = cs.nodes_renamed.begin(); - i != cs.nodes_renamed.end(); ++i) + for (map::const_iterator + i = cs.nodes_renamed.begin(); i != cs.nodes_renamed.end(); ++i) { json_io::builder tmp = b.add_obj(); tmp[syms::rename].str(i->first.as_internal()); tmp[syms::to].str(i->second.as_internal()); } - for (set::const_iterator i = cs.dirs_added.begin(); - i != cs.dirs_added.end(); ++i) + for (set::const_iterator + i = cs.dirs_added.begin(); i != cs.dirs_added.end(); ++i) { b.add_obj()[syms::add_dir].str(i->as_internal()); } - for (map::const_iterator i = cs.files_added.begin(); - i != cs.files_added.end(); ++i) + for (map::const_iterator + i = cs.files_added.begin(); i != cs.files_added.end(); ++i) { json_io::builder tmp = b.add_obj(); tmp[syms::add_file].str(i->first.as_internal()); tmp[syms::content].str(i->second.inner()()); } - for (map >::const_iterator i = cs.deltas_applied.begin(); - i != cs.deltas_applied.end(); ++i) + for (map >::const_iterator + i = cs.deltas_applied.begin(); i != cs.deltas_applied.end(); ++i) { json_io::builder tmp = b.add_obj(); tmp[syms::patch].str(i->first.as_internal()); @@ -306,16 +317,16 @@ cset_to_json(json_io::builder b, cset co tmp[syms::to].str(i->second.second.inner()()); } - for (set >::const_iterator i = cs.attrs_cleared.begin(); - i != cs.attrs_cleared.end(); ++i) + for (set >::const_iterator + i = cs.attrs_cleared.begin(); i != cs.attrs_cleared.end(); ++i) { json_io::builder tmp = b.add_obj(); tmp[syms::clear].str(i->first.as_internal()); tmp[syms::attr].str(i->second()); } - for (map, attr_value>::const_iterator i = cs.attrs_set.begin(); - i != cs.attrs_set.end(); ++i) + for (map, attr_value>::const_iterator + i = cs.attrs_set.begin(); i != cs.attrs_set.end(); ++i) { json_io::builder tmp = b.add_obj(); tmp[syms::set].str(i->first.first.as_internal()); @@ -324,11 +335,71 @@ cset_to_json(json_io::builder b, cset co } } -json_value_t -encode_msg_rev(revision_t const & rev) +static void +decode_cset(json_io::query q, cset & cs) { - json_io::builder b; - b[syms::type].str(syms::rev()); + size_t nargs = 0; + I(q.len(nargs)); + for (size_t i = 0; i < nargs; ++i) + { + json_io::query change = q[i]; + string path; + if (change[syms::delete_node].get(path)) + { + cs.nodes_deleted.insert(file_path_external(utf8(path))); + } + else if (change[syms::rename].get(path)) + { + string to; + I(change[syms::to].get(to)); + cs.nodes_renamed.insert(make_pair(file_path_external(utf8(path)), + file_path_external(utf8(to)))); + } + else if (change[syms::add_dir].get(path)) + { + cs.dirs_added.insert(file_path_external(utf8(path))); + } + else if (change[syms::add_file].get(path)) + { + string content; + I(change[syms::content].get(content)); + cs.files_added.insert(make_pair(file_path_external(utf8(path)), + file_id(content))); + } + else if (change[syms::patch].get(path)) + { + string from, to; + I(change[syms::from].get(from)); + I(change[syms::to].get(to)); + cs.deltas_applied.insert(make_pair(file_path_external(utf8(path)), + make_pair(file_id(from), + file_id(to)))); + } + else if (change[syms::clear].get(path)) + { + string key; + I(change[syms::attr].get(key)); + cs.attrs_cleared.insert(make_pair(file_path_external(utf8(path)), + attr_key(key))); + } + else if (change[syms::set].get(path)) + { + string key, val; + I(change[syms::attr].get(key)); + I(change[syms::value].get(val)); + cs.attrs_set.insert(make_pair(make_pair(file_path_external(utf8(path)), + attr_key(key)), + attr_value(val))); + } + else + I(false); + + } +} + +static void +encode_rev(json_io::builder b, revision_t const & rev) +{ b[syms::vers].str("1"); b[syms::new_manifest].str(rev.new_manifest.inner()()); json_io::builder edges = b[syms::edges].arr(); @@ -337,39 +408,89 @@ encode_msg_rev(revision_t const & rev) { json_io::builder edge = edges.add_obj(); edge[syms::old_revision].str(edge_old_revision(e).inner()()); - cset_to_json(edge, edge_changes(e)); + json_io::builder changes = edge[syms::changes].arr(); + encode_cset(changes, edge_changes(e)); } - return b.v; } +static void +decode_rev(json_io::query q, revision_t & rev) +{ + string new_manifest, vers; + I(q[syms::new_manifest].get(new_manifest)); + I(q[syms::vers].get(vers)); + I(vers == "1"); + rev.new_manifest = manifest_id(new_manifest); + size_t nargs = 0; + json_io::query edges = q[syms::edges]; + I(edges.len(nargs)); + + for (size_t i = 0; i < nargs; ++i) + { + json_io::query edge = edges[i]; + string old_revision; + I(edge[syms::old_revision].get(old_revision)); + json_io::query changes = edge[syms::changes]; + shared_ptr cs(new cset()); + decode_cset(changes, *cs); + rev.edges.insert(make_pair(revision_id(old_revision), cs)); + } +} + json_value_t -encode_msg_full_rev(revision_id const & rid, - revision_t const & rev, - set const & deltas, - set const & datas) +encode_msg_put_rev_request(revision_id const & rid, revision_t const & rev) { json_io::builder b; - b[syms::type].str(syms::full_rev()); + b[syms::type].str(syms::put_rev_request()); b[syms::vers].str("1"); - json_io::builder rev_builder = b[syms::rev]; + b[syms::id].str(rid.inner()()); + json_io::builder rb = b[syms::rev].obj(); + encode_rev(rb, rev); return b.v; } bool -decode_msg_full_rev(json_value_t val, - revision_id & rid, - revision_t & rev, - set & deltas, - set & datas) +decode_msg_put_rev_request(json_value_t val, revision_id & rid, revision_t & rev) { + string type, vers, id; + json_io::query q(val); + if (q[syms::type].get(type) && type == syms::put_rev_request() && + q[syms::vers].get(vers) && vers == "1" && + q[syms::id].get(id)) + { + rid = revision_id(id); + json_io::query rq = q[syms::rev]; + decode_rev(rq, rev); + return true; + } + return false; +} + +json_value_t +encode_msg_put_rev_response() +{ json_io::builder b; - b[syms::type].str(syms::full_rev()); + b[syms::type].str(syms::put_rev_response()); b[syms::vers].str("1"); - b[syms::rev] = encode_msg_rev(rev); + b[syms::status].str("received"); return b.v; } +bool +decode_msg_put_rev_response(json_value_t val) +{ + string type, vers, status; + json_io::query q(val); + if (q[syms::type].get(type) && type == syms::put_rev_response() && + q[syms::vers].get(vers) && vers == "1" && + q[syms::status].get(status)) + { + return true; + } + return false; +} + // Local Variables: // mode: C++ // fill-column: 76 ============================================================ --- json_msgs.hh 86a66c961fb6eef48b00414a74865929d662d855 +++ json_msgs.hh ca18a9c15f3c5056ca002ed6a2bae3a7327ab9c2 @@ -25,14 +25,17 @@ bool decode_msg_error(json_io::json_valu json_io::json_value_t encode_msg_error(std::string const & note); bool decode_msg_error(json_io::json_value_t val, std::string & note); +// inquire + json_io::json_value_t encode_msg_inquire_request(std::set const & revs); bool decode_msg_inquire_request(json_io::json_value_t val, std::set & revs); - json_io::json_value_t encode_msg_inquire_response(std::set const & revs); bool decode_msg_inquire_response(json_io::json_value_t val, std::set & revs); +// descendants + json_io::json_value_t encode_msg_descendants_request(std::set const & revs); bool decode_msg_descendants_request(json_io::json_value_t val, std::set & revs); @@ -40,8 +43,26 @@ bool decode_msg_descendants_response(jso bool decode_msg_descendants_response(json_io::json_value_t val, std::vector & revs); +// revs +json_io::json_value_t encode_msg_get_rev_request(revision_id const & rid); +bool decode_msg_get_rev_request(json_io::json_value_t val, + revision_id & rid); +json_io::json_value_t encode_msg_get_rev_response(revision_t const & rev); +bool decode_msg_get_rev_response(json_io::json_value_t val, + revision_t & rev); +json_io::json_value_t encode_msg_put_rev_request(revision_id const & rid, + revision_t const & rev); +bool decode_msg_put_rev_request(json_io::json_value_t val, + revision_id & rid, + revision_t & rev); +json_io::json_value_t encode_msg_put_rev_response(); +bool decode_msg_put_rev_response(json_io::json_value_t val); + + +// files + json_io::json_value_t encode_msg_get_file_data(file_id const & fid); bool decode_msg_get_file_data(json_io::json_value_t val, file_id & fid); @@ -51,46 +72,6 @@ bool decode_msg_get_file_delta(json_io:: file_id & src_id, file_id & dst_id); -json_io::json_value_t encode_msg_get_rev(revision_id const & rid); -bool decode_msg_get_rev(json_io::json_value_t val, revision_id & rid); - -json_io::json_value_t encode_msg_get_full_rev(revision_id const & rid); -bool decode_msg_get_full_rev(json_io::json_value_t val, revision_id & rid); - -struct -file_delta_record -{ - file_id src; - file_id dst; - file_delta del; -}; - -struct -file_data_record -{ - file_id id; - file_data dat; -}; - -json_io::json_value_t encode_msg_file_data(file_data_record const & fr); -bool decode_msg_file_data(json_io::json_value_t val, file_data_record & fr); - -json_io::json_value_t encode_msg_file_delta(file_delta_record const & fr); -bool decode_msg_file_delta(json_io::json_value_t val, file_delta_record & fr); - -json_io::json_value_t encode_msg_rev(revision_t const & rev); -bool decode_msg_rev(json_io::json_value_t val, revision_t & rev); - -json_io::json_value_t encode_msg_full_rev(revision_id const & rid, - revision_t const & rev, - std::set const & deltas, - std::set const & datas); -bool decode_msg_full_rev(json_io::json_value_t val, - revision_id & rid, - revision_t & rev, - std::set & deltas, - std::set & datas); - // Local Variables: // mode: C++ // fill-column: 76