# # # patch "cmd_scgi.cc" # from [9aea34024e0f9430dcfcc06e59bdca5f22d1feef] # to [73b6d790262a839810f04d79d886a56e9e12f917] # # patch "gsync.cc" # from [5f8a87f1a21027f3661fbd8b6fbcc9614ef0c201] # to [fa8fcd0b060bca33ac0ae1032f447b050ed381f7] # # patch "http_client.cc" # from [818d742b9a9e6e1ef0f51d07ccf9ff39136f591f] # to [6561236ce5d96d312751a03ce24d5de94f1d5664] # # patch "json_io.hh" # from [5d45274449db7c24ee44ef874a9d96d0cb93e2d0] # to [56962fa27b4da23043ab4cd1a5dbc4082a24bd37] # # patch "json_msgs.cc" # from [97ca800cb705214d2e8753312226a83e36288112] # to [59063a3a187dc0017d80f187b3ce5e05353a69a2] # # patch "json_msgs.hh" # from [8c5866e4e1cdc3aa6e27dca3d2f7210d537451c3] # to [5057e28480a4d7e2f673bd4111e8b10bd169782d] # # patch "work.cc" # from [84dc74091e3318424da00b914dd904d3e884ce5f] # to [cf12cd682028c7a4c8c9d972a6e86ea909493d61] # ============================================================ --- cmd_scgi.cc 9aea34024e0f9430dcfcc06e59bdca5f22d1feef +++ cmd_scgi.cc 73b6d790262a839810f04d79d886a56e9e12f917 @@ -55,7 +55,7 @@ using boost::lexical_cast; // // headers ::= header* // header ::= name NUL value NUL -// name ::= notnull+ +// name ::= notnull+ // value ::= notnull+ // notnull ::= <01> | <02> | <03> | ... | // NUL = <00> @@ -81,10 +81,10 @@ struct scgi_error }; // Consume string until null or EOF. Consumes trailing null. -static string +static string parse_str(istream & in) { - string s; + string s; while (in.good()) { char ch = static_cast(in.get()); @@ -95,7 +95,7 @@ parse_str(istream & in) return s; } -static inline bool +static inline bool eat(istream & in, char c) { if (!in.good()) @@ -104,7 +104,7 @@ eat(istream & in, char c) return c == static_cast(i); } -static bool +static bool parse_scgi(istream & in, string & data) { @@ -117,6 +117,15 @@ parse_scgi(istream & in, string & data) L(FL("scgi: netstring length: %d") % netstring_len); if (!eat(in, ':')) return false; + // although requirements of the scgi spec, + // the code below will allow requests that: + // - don't have CONTENT_LENGTH as the first header + // - don't have a CONTENT_LENGTH header at all + // - don't have an SCGI header + // perhaps this is just to "be liberal in what you accept" + // but we may want to tighten it up a bit. at the very least + // this is relying on the CONTENT_LENGTH header. + size_t content_length = 0; while (netstring_len > 0) { @@ -125,7 +134,7 @@ parse_scgi(istream & in, string & data) L(FL("scgi: got header: %s -> %s") % key % val); if (key == "CONTENT_LENGTH") - { + { content_length = lexical_cast(val); L(FL("scgi: content length: %d") % content_length); } @@ -157,18 +166,18 @@ do_cmd(database & db, json_io::json_obje static json_io::json_value_t do_cmd(database & db, json_io::json_object_t cmd_obj) { - set revs; + set request_revs; - if (decode_msg_inquire(cmd_obj, revs)) + if (decode_msg_inquire_request(cmd_obj, request_revs)) { - L(FL("inquiring %d revisions") % revs.size()); + L(FL("inquiring %d revisions") % request_revs.size()); db.ensure_open(); - set confirmed; - for (set::const_iterator i = revs.begin(); - i != revs.end(); ++i) + set response_revs; + for (set::const_iterator i = request_revs.begin(); + i != request_revs.end(); ++i) if (db.revision_exists(*i)) - confirmed.insert(*i); - return encode_msg_confirm(confirmed); + response_revs.insert(*i); + return encode_msg_inquire_response(response_revs); } else { @@ -196,7 +205,7 @@ process_scgi_transaction(database & db, json_io::parser p(tok); json_io::json_object_t obj = p.parse_object(); - if (static_cast(obj)) + if (static_cast(obj)) { transaction_guard guard(db); L(FL("read JSON object")); @@ -213,7 +222,7 @@ process_scgi_transaction(database & db, << "Content-Type: application/jsonrequest\r\n" << "\r\n"; - out.write(out_data.buf.data(), out_data.buf.size()); + out.write(out_data.buf.data(), out_data.buf.size()); out << "\n"; out.flush(); return; @@ -234,6 +243,7 @@ process_scgi_transaction(database & db, << "\r\n"; out.flush(); } + } @@ -244,9 +254,9 @@ CMD_NO_WORKSPACE(scgi, // C N_(""), // params N_("Serves SCGI+JSON connections"), // abstract "", // desc - options::opts::scgi_bind | + options::opts::scgi_bind | options::opts::pidfile | - options::opts::bind_stdio | + options::opts::bind_stdio | options::opts::no_transport_auth ) { @@ -276,7 +286,7 @@ CMD_NO_WORKSPACE(scgi, // C if (app.opts.bind_stdio) process_scgi_transaction(db, std::cin, std::cout); else - { + { #ifdef USE_IPV6 bool use_ipv6=true; @@ -288,8 +298,8 @@ CMD_NO_WORKSPACE(scgi, // C bool try_again=false; do - { - try + { + try { try_again = false; @@ -323,10 +333,10 @@ CMD_NO_WORKSPACE(scgi, // C } else break; - } + } } - // Possibly loop around if we get exceptions from Netxx and we're - // attempting to use ipv6, or have some other reason to try again. + // Possibly loop around if we get exceptions from Netxx and we're + // attempting to use ipv6, or have some other reason to try again. catch (Netxx::NetworkException &) { if (try_again) @@ -341,7 +351,7 @@ CMD_NO_WORKSPACE(scgi, // C else throw; } - } while (try_again); + } while (try_again); } } ============================================================ --- gsync.cc 5f8a87f1a21027f3661fbd8b6fbcc9614ef0c201 +++ gsync.cc fa8fcd0b060bca33ac0ae1032f447b050ed381f7 @@ -44,7 +44,7 @@ // // In the introduction step, the client asks the server to describe // its public key, branches, etc. such that the client knows what sort -// of material it can ask for in an authenticated fashion. +// of material it can ask for in an authenticated fashion. // // In the inquiry step, the client sends a set of revids to the server // and asks which of them the server has. The server responds with the @@ -59,7 +59,7 @@ // hash). This is a quasi-randomized-ish algorithm and it converges // very fast. Once the client determines a shared historical core DAG, // it calculates the graph frontier of that core. -// +// // Depending on the mode (push, pull, or sync) the playback phase // then involves one or both of the following: // @@ -121,8 +121,8 @@ determine_common_core(channel const & ch // Bite off a chunk of the remaining unknowns to ask about. set::const_iterator r = unknown_revs.begin(); - for (size_t i = 0; - i < constants::gsync_max_probe_set_size && r != unknown_revs.end(); + for (size_t i = 0; + i < constants::gsync_max_probe_set_size && r != unknown_revs.end(); ++i, ++r) { query_revs.insert(*r); @@ -136,16 +136,17 @@ determine_common_core(channel const & ch ch.inquire_about_revs(query_revs, revs_present); do_set_difference(query_revs, revs_present, revs_absent); - L(FL("pass #%d: inquired about %d revs, they have %d of them, missing %d of them") + L(FL("pass #%d: inquired about %d revs, they have %d of them, missing %d of them") % pass - % query_revs.size() + % query_revs.size() % revs_present.size() % revs_absent.size()); + // FIXME: "ancestors" is a misnomer; it's a graph-closure calculation... + get_all_ancestors(revs_present, child_to_parent_map, present_ancs); do_set_union(revs_present, present_ancs, present_closure); - // FIXME: "ancestors" is a misnomer; it's a graph-closure calculation... get_all_ancestors(revs_absent, parent_to_child_map, absent_descs); do_set_union(revs_absent, absent_descs, absent_closure); @@ -155,12 +156,12 @@ determine_common_core(channel const & ch do_set_difference(unknown_revs, present_closure, new_unknown); unknown_revs = new_unknown; - L(FL("pass #%d: unknown set after removing %d-entry present closure: %d nodes") + L(FL("pass #%d: unknown set after removing %d-entry present closure: %d nodes") % pass % present_closure.size() % unknown_revs.size()); do_set_difference(unknown_revs, absent_closure, new_unknown); unknown_revs = new_unknown; - L(FL("pass #%d: unknown set after removing %d-entry absent closure: %d nodes") + L(FL("pass #%d: unknown set after removing %d-entry absent closure: %d nodes") % pass % absent_closure.size() % unknown_revs.size()); // Update our total knowledge about them. @@ -181,7 +182,7 @@ do_missing_playback(database & db, static void do_missing_playback(database & db, channel const & ch, - set & core_frontier, + set & core_frontier, set & revs_to_push, rev_ancestry_map const & parent_to_child_map) { @@ -234,7 +235,7 @@ do_missing_playback(database & db, } -static void +static void request_missing_playback(database & db, channel const & ch, set const & core_frontier) @@ -254,7 +255,7 @@ run_gsync_protocol(lua_hooks & lua, data invert_ancestry(parent_to_child_map, child_to_parent_map); set our_revs; - for (rev_ancestry_map::const_iterator i = child_to_parent_map.begin(); + for (rev_ancestry_map::const_iterator i = child_to_parent_map.begin(); i != child_to_parent_map.end(); ++i) { if (!i->first.inner()().empty()) @@ -279,6 +280,7 @@ run_gsync_protocol(lua_hooks & lua, data if (pulling) request_missing_playback(db, ch, core_frontier); + } #ifdef BUILD_UNIT_TESTS ============================================================ --- http_client.cc 818d742b9a9e6e1ef0f51d07ccf9ff39136f591f +++ http_client.cc 6561236ce5d96d312751a03ce24d5de94f1d5664 @@ -107,7 +107,6 @@ http_client::transact_json(json_value_t io->flush(); L(FL("http_client: sent %d-byte body") % out.buf.size()); - // Now read back the result string data; parse_http_response(data); @@ -170,12 +169,15 @@ http_client::parse_http_response(std::st while (io->good() && content_length > 0) { - data += static_cast(io->get());; + data += static_cast(io->get()); content_length--; } io->flush(); + // if we keep the connection alive, and we're limited to a single active + // connection (as in the sample lighttpd.conf and required by the sqlite + // database locking scheme) this will probably block all other clients. if (!keepalive) { L(FL("http_client: closing connection")); @@ -198,10 +200,10 @@ http_channel::inquire_about_revs(set & theirs) const { theirs.clear(); - json_value_t query = encode_msg_inquire(query_set); - json_value_t response = client.transact_json(query); - E(decode_msg_confirm(response, theirs), - F("received unexpected reply to 'inquire' message")); + json_value_t request = encode_msg_inquire_request(query_set); + json_value_t response = client.transact_json(request); + E(decode_msg_inquire_response(response, theirs), + F("received unexpected reply to 'inquire_request' message")); } void ============================================================ --- json_io.hh 5d45274449db7c24ee44ef874a9d96d0cb93e2d0 +++ json_io.hh 56962fa27b4da23043ab4cd1a5dbc4082a24bd37 @@ -214,11 +214,11 @@ namespace json_io return json_io::TOK_SYMBOL; } - else if (in.lookahead == '"') + else if (in.lookahead == '"') // no static_cast here ... { in.advance(); mark(); - while (static_cast(in.lookahead) != '"') + while (static_cast(in.lookahead) != '"') // ... is there a real reason for one here? { if (UNLIKELY(in.lookahead == EOF)) in.err("input stream ended in string"); ============================================================ --- json_msgs.cc 97ca800cb705214d2e8753312226a83e36288112 +++ json_msgs.cc 59063a3a187dc0017d80f187b3ce5e05353a69a2 @@ -63,11 +63,11 @@ namespace symbol const revs("revs"); symbol const error("error"); - symbol const inquire("inquire"); - symbol const confirm("confirm"); + symbol const inquire_request("inquire_request"); + symbol const inquire_response("inquire_response"); - symbol const get_descendants("get_descendants"); - symbol const descendants("descendants"); + symbol const descendants_request("descendants_request"); + symbol const descendants_response("descendants_response"); symbol const get_rev("get_rev"); symbol const get_full_rev("get_full_rev"); @@ -82,10 +82,10 @@ namespace } ///////////////////////////////////////////////////////////////////// -// message type 'error' +// message type 'error' ///////////////////////////////////////////////////////////////////// -json_value_t +json_value_t encode_msg_error(string const & note) { json_io::builder b; @@ -93,8 +93,8 @@ encode_msg_error(string const & note) return b.v; } -bool -decode_msg_error(json_value_t val, +bool +decode_msg_error(json_value_t val, std::string & note) { json_io::query q(val); @@ -104,14 +104,14 @@ decode_msg_error(json_value_t val, ///////////////////////////////////////////////////////////////////// -// message type 'inquire' +// message type 'inquire_request' ///////////////////////////////////////////////////////////////////// -json_value_t -encode_msg_inquire(set const & revs) +json_value_t +encode_msg_inquire_request(set const & revs) { json_io::builder b; - b[syms::type].str(syms::inquire()); + b[syms::type].str(syms::inquire_request()); b[syms::vers].str("1"); json_io::builder r = b[syms::revs].arr(); for (set::const_iterator i = revs.begin(); i != revs.end(); ++i) @@ -119,23 +119,21 @@ encode_msg_inquire(set cons return b.v; } -bool -decode_msg_inquire(json_value_t val, - set & revs) +bool +decode_msg_inquire_request(json_value_t val, + set & revs) { string type, vers; - json_io::query q(val); - if (q[syms::type].get(type) && - type == syms::inquire() && - q[syms::vers].get(vers) && - vers == "1") - { + json_io::query q(val); + if (q[syms::type].get(type) && type == syms::inquire_request() && + q[syms::vers].get(vers) && vers == "1") + { size_t nargs = 0; - if (q[syms::revs].len(nargs)) + if (q[syms::revs].len(nargs)) { std::string s; for (size_t i = 0; i < nargs; ++i) - if (q[syms::revs][i].get(s)) + if (q[syms::revs][i].get(s)) revs.insert(revision_id(s)); return true; } @@ -145,14 +143,14 @@ decode_msg_inquire(json_value_t val, ///////////////////////////////////////////////////////////////////// -// message type 'confirm' +// message type 'inquire_response' ///////////////////////////////////////////////////////////////////// -json_value_t -encode_msg_confirm(set const & revs) +json_value_t +encode_msg_inquire_response(set const & revs) { json_io::builder b; - b[syms::type].str(syms::confirm()); + b[syms::type].str(syms::inquire_response()); b[syms::vers].str("1"); json_io::builder r = b[syms::revs].arr(); for (set::const_iterator i = revs.begin(); @@ -163,22 +161,20 @@ encode_msg_confirm(set cons return b.v; } -bool -decode_msg_confirm(json_value_t val, - set & revs) +bool +decode_msg_inquire_response(json_value_t val, + set & revs) { string type, vers; - json_io::query q(val); - if (q[syms::type].get(type) && - type == syms::confirm() && - q[syms::vers].get(vers) && - vers == "1") + json_io::query q(val); + if (q[syms::type].get(type) && type == syms::inquire_response() && + q[syms::vers].get(vers) && vers == "1") { size_t nrevs = 0; string tmp; json_io::query r = q[syms::revs]; if (r.len(nrevs)) - for (size_t i = 0; i < nrevs; ++i) + for (size_t i = 0; i < nrevs; ++i) if (r[i].get(tmp)) revs.insert(revision_id(tmp)); return true; @@ -187,29 +183,29 @@ decode_msg_confirm(json_value_t val, } ///////////////////////////////////////////////////////////////////// -// message type 'get_descendants' +// message type 'descendants_request' ///////////////////////////////////////////////////////////////////// -json_value_t -encode_msg_get_descendants(set const & revs); -bool -decode_msg_get_descendants(json_value_t val, - set & revs); +json_value_t +encode_msg_descendants_request(set const & revs); +bool +decode_msg_descendants_response(json_value_t val, + set & revs); ///////////////////////////////////////////////////////////////////// -// message type 'descendants' +// message type 'descendants_response' ///////////////////////////////////////////////////////////////////// -json_value_t -encode_msg_descendants(rev_ancestry_map const & parent_to_child_map); -bool -decode_msg_descendants(json_value_t val, - rev_ancestry_map & parent_to_child_map); +json_value_t +encode_descendants_response(rev_ancestry_map const & parent_to_child_map); +bool +decode_descendants_response(json_value_t val, + rev_ancestry_map & parent_to_child_map); ///////////////////////////////////////////////////////////////////// -// message type 'rev' +// message type 'rev' ///////////////////////////////////////////////////////////////////// @@ -274,12 +270,12 @@ encode_msg_rev(revision_t const & rev) json_value_t encode_msg_rev(revision_t const & rev) { - json_io::builder b; + json_io::builder b; b[syms::type].str(syms::rev()); b[syms::vers].str("1"); b[syms::new_manifest].str(rev.new_manifest.inner()()); json_io::builder edges = b[syms::edges].arr(); - for (edge_map::const_iterator e = rev.edges.begin(); + for (edge_map::const_iterator e = rev.edges.begin(); e != rev.edges.end(); ++e) { json_io::builder edge = edges.add_obj(); @@ -291,22 +287,21 @@ encode_msg_rev(revision_t const & rev) - -json_value_t +json_value_t encode_msg_full_rev(revision_id const & rid, revision_t const & rev, set const & deltas, set const & datas) { - json_io::builder b; + json_io::builder b; b[syms::type].str(syms::full_rev()); b[syms::vers].str("1"); - json_io::builder rev_builder = b[syms::rev]; + json_io::builder rev_builder = b[syms::rev]; return b.v; } -bool -decode_msg_full_rev(json_value_t val, +bool +decode_msg_full_rev(json_value_t val, revision_id & rid, revision_t & rev, set & deltas, @@ -319,9 +314,6 @@ decode_msg_full_rev(json_value_t val, return b.v; } - - - // Local Variables: // mode: C++ // fill-column: 76 ============================================================ --- json_msgs.hh 8c5866e4e1cdc3aa6e27dca3d2f7210d537451c3 +++ json_msgs.hh 5057e28480a4d7e2f673bd4111e8b10bd169782d @@ -25,28 +25,28 @@ bool decode_msg_error(json_io::json_valu json_io::json_value_t encode_msg_error(std::string const & note); bool decode_msg_error(json_io::json_value_t val, std::string & note); -json_io::json_value_t encode_msg_inquire(std::set const & revs); -bool decode_msg_inquire(json_io::json_value_t val, - std::set & revs); +json_io::json_value_t encode_msg_inquire_request(std::set const & revs); +bool decode_msg_inquire_request(json_io::json_value_t val, + std::set & revs); -json_io::json_value_t encode_msg_confirm(std::set const & revs); -bool decode_msg_confirm(json_io::json_value_t val, - std::set & revs); +json_io::json_value_t encode_msg_inquire_response(std::set const & revs); +bool decode_msg_inquire_response(json_io::json_value_t val, + std::set & revs); -json_io::json_value_t encode_msg_get_descendants(std::set const & start); -bool decode_msg_get_descendants(json_io::json_value_t val, +json_io::json_value_t encode_msg_descendants_request(std::set const & start); +bool decode_msg_descendants_request(json_io::json_value_t val, std::set & start); -json_io::json_value_t encode_msg_descendants(rev_ancestry_map const & parent_to_child_map); -bool decode_msg_descendants(json_io::json_value_t val, - rev_ancestry_map & parent_to_child_map); +json_io::json_value_t encode_msg_descendants_response(rev_ancestry_map const & parent_to_child_map); +bool decode_msg_descendants_response(json_io::json_value_t val, + rev_ancestry_map & parent_to_child_map); json_io::json_value_t encode_msg_get_file_data(file_id const & fid); bool decode_msg_get_file_data(json_io::json_value_t val, file_id & fid); json_io::json_value_t encode_msg_get_file_delta(file_id const & src_id, file_id const & dst_id); -bool decode_msg_get_file_delta(json_io::json_value_t val, +bool decode_msg_get_file_delta(json_io::json_value_t val, file_id & src_id, file_id & dst_id); @@ -64,7 +64,7 @@ file_delta_record file_delta del; }; -struct +struct file_data_record { file_id id; @@ -84,7 +84,7 @@ json_io::json_value_t encode_msg_full_re revision_t const & rev, std::set const & deltas, std::set const & datas); -bool decode_msg_full_rev(json_io::json_value_t val, +bool decode_msg_full_rev(json_io::json_value_t val, revision_id & rid, revision_t & rev, std::set & deltas, ============================================================ --- work.cc 84dc74091e3318424da00b914dd904d3e884ce5f +++ work.cc cf12cd682028c7a4c8c9d972a6e86ea909493d61 @@ -378,7 +378,7 @@ read_options_file(any_path const & optsp string opt, val; parser.sym(opt); parser.str(val); - + if (opt == "database") database_option = system_path(val); else if (opt == "branch")