#
#
# add_file "cmd_scgi.cc"
#  content [0ba8a300f548e224ba3c7f716de24fa4aac2279b]
# 
# add_file "gsync.cc"
#  content [5a5e7367916f54b940c3ff3db7f5e55da0c43a52]
# 
# add_file "http_client.cc"
#  content [52d1fd430adb95bae1d814a2bebbdb99128fa0f4]
# 
# add_file "http_client.hh"
#  content [1d37aad6de15a308b82eebb4943c387546f607b2]
# 
# add_file "json_io.cc"
#  content [a32899cbd5f29b85ee33e444f547ea3e4ca637a0]
# 
# add_file "json_io.hh"
#  content [07bdba5c2135920bb86262786926beb73f4e11c7]
# 
# patch "Makefile.am"
#  from [dfaa00d12056b5dc7a53ea779ba605d890669470]
#    to [e9e3d2c42df7879640a66e21854ad11a3f8eba30]
# 
# patch "cmd_netsync.cc"
#  from [39a96b273f26950ce8701dd041994f5c0b254877]
#    to [0571b71d680fc18500b38f25185ee7500402c31d]
# 
# patch "constants.hh"
#  from [35e5834105b0ab3572749d071313fbd106b81fe0]
#    to [26ea1fa80b9236854f37ac0fbdeaccf6236a5c90]
# 
# patch "database.cc"
#  from [331c15584601bfa5585b5169af3bc597196a0c47]
#    to [c3782890880b957df6e7ab1ab3a334915bad2324]
# 
# patch "graph.cc"
#  from [2d934cb225b5639919fec3eda0cbec615cf2f903]
#    to [ba666c56c19c7ca8d85f9b2898f479fe92be1da6]
# 
# patch "graph.hh"
#  from [6384f6bd01b8b43383ba2e6adf6ae10b6f725bac]
#    to [39241af3330a9cf3c42a9ba2cf7067017c781b6f]
# 
# patch "netxx/netbuf.h"
#  from [1a1f6f16e7946f9a477a31f52ae3515f8ad0e333]
#    to [3225d843e80c257bdd2ecfd718ff255c79a9cb0d]
# 
# patch "vocab.hh"
#  from [8c831a796771018eb1b2d28f56dc8e2f9b4d4e55]
#    to [fae7ea2ea7fd30a46e256af081e1efc716c26c39]
#
============================================================
--- cmd_scgi.cc	0ba8a300f548e224ba3c7f716de24fa4aac2279b
+++ cmd_scgi.cc	0ba8a300f548e224ba3c7f716de24fa4aac2279b
@@ -0,0 +1,280 @@
+// Copyright (C) 2007 Graydon Hoare <address@hidden>
+//
+// This program is made available under the GNU GPL version 2.0 or
+// greater. See the accompanying file COPYING for details.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the
+// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE.
+
+#include "base.hh"
+#include <iostream>
+#include <sstream>
+#include <map>
+
+#include "cmd.hh"
+#include "app_state.hh"
+#include "ui.hh"
+#include "lexical_cast.hh"
+#include "lua.hh"
+#include "lua_hooks.hh"
+#include "json_io.hh"
+
+using std::istream;
+using std::make_pair;
+using std::map;
+using std::ostream;
+using std::ostringstream;
+using std::pair;
+using std::set;
+using std::string;
+using std::vector;
+
+using boost::lexical_cast;
+
+// SCGI interface is pretty straightforward
+//
+// When we accept a connection, we read a netstring out of it covering the
+// header, and then a body consisting of a JSON object.
+//
+// The format of the headers is:
+//
+//        headers ::= header*
+//        header ::= name NUL value NUL
+//        name ::= notnull+                
+//        value ::= notnull+
+//        notnull ::= <01> | <02> | <03> | ... | <ff>
+//        NUL = <00>
+//
+// The format of the netstring containing the headers is:
+//
+// [len]":"[string]","
+//
+// Where [string] is the string and [len] is a nonempty sequence of ASCII
+// digits giving the length of [string] in decimal.
+//
+// The response is a sequence of CRLF-separated of HTTP headers, followed by
+// a bare CRLF, and a JSON object.
+//
+// This response format is not specified by the SCGI "spec".
+//
+
+
+// Consume string until null or EOF. Consumes trailing null.
+static string 
+parse_str(istream & in)
+{
+  string s; 
+  while (in.good())
+    {
+      char ch = static_cast<char>(in.get());
+      if (ch == '\0')
+        break;
+      s += ch;
+    }
+  return s;
+}
+
+static inline bool 
+eat(istream & in, char c)
+{
+  if (!in.good())
+    return false;
+  int i = in.get();
+  return c == static_cast<char>(i);
+}
+
+static bool 
+parse_scgi(istream & in, string & data)
+{
+  
+  if (!in.good()) return false;
+  
+  size_t netstring_len;
+  in >> netstring_len;
+  if (!in.good()) return false;
+
+  L(FL("scgi: netstring length: %d") % netstring_len);
+  if (!eat(in, ':')) return false;
+  
+  size_t content_length = 0;
+  while (netstring_len > 0)
+    {
+      string key = parse_str(in);
+      string val = parse_str(in);
+
+      L(FL("scgi: got header: %s -> %s") % key % val);
+      if (key == "CONTENT_LENGTH")
+        { 
+          content_length = lexical_cast<size_t,string>(val);
+          L(FL("scgi: content length: %d") % content_length);
+        }
+      else if (key == "SCGI" && val != "1")
+        return false;
+
+      netstring_len -= key.size();
+      netstring_len -= val.size();
+      netstring_len -= 2;
+    }
+
+  if(!eat(in, ',')) return false;
+  
+  data.clear();
+  data.reserve(content_length);
+  L(FL("reading %d bytes") % content_length);
+
+  while (in.good() && (content_length > 0))
+    {
+      data += static_cast<char>(in.get());
+      content_length--;
+    }
+
+  L(FL("read %d bytes, content_length now %d") % data.size() % content_length);
+
+  return (content_length == 0);
+}
+
+namespace syms
+{
+  symbol const status("status");
+  symbol const vers("vers");
+  symbol const cmd("cmd");
+  symbol const args("args");
+  symbol const inquire("inquire");
+  symbol const confirm("confirm");
+  symbol const revs("revs");
+  symbol const type("type");
+};
+
+static json_io::json_object_t
+bad_req()
+{
+  json_io::builder b;
+
+  b[syms::status].str("bad");
+
+  return b.as_obj();
+}
+
+static json_io::json_object_t
+do_cmd(app_state & app, json_io::json_object_t cmd_obj)
+{
+
+  string type, vers;
+  json_io::query q(cmd_obj);
+
+  if (! q[syms::type].get(type))
+    return bad_req();
+
+  L(FL("read JSON command type: %s") % type);
+
+  if (type == "ping" && 
+      q[syms::vers].get(vers) && 
+      vers == "1")
+    {
+      json_io::builder b;
+      json_io::builder args = b[syms::args].arr();
+      
+      size_t nargs = 0;
+      if (q[syms::args].len(nargs)) 
+        {
+          for (size_t i = 0; i < nargs; ++i)
+            args.add(q[syms::args][i].get());
+        }
+      return b.as_obj();
+    }
+  else if (type == syms::inquire() && 
+           q[syms::vers].get(vers) && 
+           vers == "1")
+    {
+      json_io::builder b;
+      b[syms::type].str(syms::confirm());
+      b[syms::vers].str("1");
+      json_io::builder revs = b[syms::revs].arr();
+      
+      size_t nargs = 0;
+      if (q[syms::revs].len(nargs)) 
+        {
+          app.db.ensure_open();
+          std::string s;
+          for (size_t i = 0; i < nargs; ++i)
+            {
+              if (q[syms::revs][i].get(s)) 
+                {
+                  if (app.db.revision_exists(revision_id(s)))
+                    revs.add_str(s);
+                }
+            }
+        }
+      return b.as_obj();
+    }
+  else
+    {
+      return bad_req();
+    }
+  return cmd_obj;
+}
+
+
+CMD_NO_WORKSPACE(scgi,             // C
+                 "scgi",           // name
+                 "",               // aliases
+                 CMD_REF(network), // parent
+                 N_(""),                              // params
+                 N_("Serves SCGI+JSON connections"),  // abstract
+                 "",                                  // desc
+                 options::opts::none                  // options
+                 )
+{
+  // FIXME: expand this function to take a pathname for a win32 named pipe
+  // or unix domain socket, for accept/read/dispatch loop.
+
+  N(args.size() == 0,
+    F("no arguments needed"));
+
+  string data;
+
+  if (parse_scgi(std::cin, data))
+    {
+      L(FL("read SCGI request: [[%s]]") % data);
+      
+      json_io::input_source in(data, "scgi");
+      json_io::tokenizer tok(in);
+      json_io::parser p(tok);
+      json_io::json_object_t obj = p.parse_object();
+      
+      if (static_cast<bool>(obj)) 
+        {
+          L(FL("read JSON object"));
+          
+          json_io::json_object_t res = do_cmd(app, obj);
+          if (static_cast<bool>(res))
+            {
+              L(FL("sending JSON response"));
+              json_io::printer out;
+              res->write(out);
+              
+              std::cout << "Status: 200 OK\r\n"
+                        << "Content-Length: " << (out.buf.size() + 1) << "\r\n"
+                        << "Content-Type: application/jsonrequest\r\n"
+                        << "\r\n";
+              
+              std::cout.write(out.buf.data(), out.buf.size());  
+              std::cout << "\n";
+              return;
+            }
+        }
+    }
+  
+  std::cout << "Status: 400 Bad request\r\n"
+            << "Content-Type: application/jsonrequest\r\n"
+            << "\r\n";
+}
+
+// Local Variables:
+// mode: C++
+// fill-column: 76
+// c-file-style: "gnu"
+// indent-tabs-mode: nil
+// End:
+// vim: et:sw=2:sts=2:ts=2:cino=>2s,{s,\:s,+s,t0,g0,^-2,e-2,n-2,p2s,(0,=s:
============================================================
--- gsync.cc	5a5e7367916f54b940c3ff3db7f5e55da0c43a52
+++ gsync.cc	5a5e7367916f54b940c3ff3db7f5e55da0c43a52
@@ -0,0 +1,403 @@
+// Copyright (C) 2007 Graydon Hoare <address@hidden>
+//
+// This program is made available under the GNU GPL version 2.0 or
+// greater. See the accompanying file COPYING for details.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the
+// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE.
+
+#include "base.hh"
+
+#include <algorithm>
+#include <map>
+#include <set>
+#include <string>
+
+#include "app_state.hh"
+#include "database.hh"
+#include "globish.hh"
+#include "http_client.hh"
+#include "json_io.hh"
+#include "revision.hh"
+#include "sanity.hh"
+#include "lexical_cast.hh"
+#include "uri.hh"
+
+//
+// Gsync is the "new new" synchronization system for monotone,
+// obsoleting netsync, the "old new" system that was developed back in
+// the bad old days of unstructured sets. With any luck it'll be the
+// last synchronization system. We'll see.
+//
+// The "g" in gsync stands for "graph". It is an algorithm quite
+// strongly based on DAGs. It does not have much to do with
+// unstructured sets. There are no merkle tries either. So long old
+// friends.
+//
+// It is also significantly simpler than netsync.
+//
+// The algorithm consists of three types of client-initiated exchange:
+// introduction, inquiry and playback. There is no coupling between
+// these steps. They can be done in any order, interrupted at any
+// time, repeated any number of times, etc. Like netsync, they are all
+// idempotent, informative actions.
+//
+// In the introduction step, the client asks the server to describe
+// its public key, branches, etc. such that the client knows what sort
+// of material it can ask for in an authenticated fashion. 
+//
+// In the inquiry step, the client sends a set of revids to the server
+// and asks which of them the server has. The server responds with the
+// subset that it has. The goal in this step is for the client to
+// figure out how much of history client and server have in
+// common. Crucially, the client does not need to enumerate all of its
+// revids this way: every time it learns of a rev that the server has,
+// it also knows that the server has all the ancestors of that rev;
+// and if it learns of a rev the server *doesn't* have, it also knows
+// that the server doesn't have any of the descendents of that rev. It
+// selects revids in essentially random order (lexicographic by
+// hash). This is a quasi-randomized-ish algorithm and it converges
+// very fast. Once the client determines a shared historical core DAG,
+// it calculates the graph frontier of that core.
+// 
+// Depending on the mode (push, pull, or sync) the playback phase
+// then involves one or both of the following:
+//
+//   - Sending a request to the server to play back from the frontier.
+//     The frontier to playback from is sent along with this
+//     request. It's stateless.
+//
+//   - Initiating and playing forward from the frontier on the client
+//     side. Similarly, these are stateless "submit" commands.
+//
+
+
+using std::make_pair;
+using std::map;
+using std::min;
+using std::set;
+using std::string;
+using std::pair;
+
+using json_io::json_value_t;
+using boost::lexical_cast;
+
+/////////////////////////////////////////////////////////////////////
+// monotone <-> json conversions
+/////////////////////////////////////////////////////////////////////
+
+
+namespace
+{
+  namespace syms
+  {
+    // cset symbols
+    symbol const delete_node("delete");
+    symbol const rename("rename");
+    symbol const content("content");
+    symbol const add_file("add_file");
+    symbol const add_dir("add_dir");
+    symbol const patch("patch");
+    symbol const from("from");
+    symbol const to("to");
+    symbol const clear("clear");
+    symbol const set("set");
+    symbol const attr("attr");
+    symbol const value("value");
+
+    // revision symbols
+    symbol const old_revision("old_revision");
+    symbol const new_manifest("new_manifest");
+    symbol const edges("edges");
+
+    // command symbols
+    symbol const type("type");
+    symbol const vers("vers");
+    symbol const revision("revision");
+    symbol const inquire("inquire");
+    symbol const confirm("confirm");
+    symbol const revs("revs");
+
+  }
+}
+
+static void
+cset_to_json(json_io::builder b, cset const & cs)
+{
+    for (set<file_path>::const_iterator i = cs.nodes_deleted.begin();
+       i != cs.nodes_deleted.end(); ++i)
+    {
+      b.add_obj()[syms::delete_node].str(i->as_internal());
+    }
+
+  for (map<file_path, file_path>::const_iterator i = cs.nodes_renamed.begin();
+       i != cs.nodes_renamed.end(); ++i)
+    {
+      json_io::builder tmp = b.add_obj();
+      tmp[syms::rename].str(i->first.as_internal());
+      tmp[syms::to].str(i->second.as_internal());
+    }
+
+  for (set<file_path>::const_iterator i = cs.dirs_added.begin();
+       i != cs.dirs_added.end(); ++i)
+    {
+      b.add_obj()[syms::add_dir].str(i->as_internal());
+    }
+
+  for (map<file_path, file_id>::const_iterator i = cs.files_added.begin();
+       i != cs.files_added.end(); ++i)
+    {
+      json_io::builder tmp = b.add_obj();
+      tmp[syms::add_file].str(i->first.as_internal());
+      tmp[syms::content].str(i->second.inner()());
+    }
+
+  for (map<file_path, pair<file_id, file_id> >::const_iterator i = cs.deltas_applied.begin();
+       i != cs.deltas_applied.end(); ++i)
+    {
+      json_io::builder tmp = b.add_obj();
+      tmp[syms::patch].str(i->first.as_internal());
+      tmp[syms::from].str(i->second.first.inner()());
+      tmp[syms::to].str(i->second.second.inner()());
+    }
+
+  for (set<pair<file_path, attr_key> >::const_iterator i = cs.attrs_cleared.begin();
+       i != cs.attrs_cleared.end(); ++i)
+    {
+      json_io::builder tmp = b.add_obj();
+      tmp[syms::clear].str(i->first.as_internal());
+      tmp[syms::attr].str(i->second());
+    }
+
+  for (map<pair<file_path, attr_key>, attr_value>::const_iterator i = cs.attrs_set.begin();
+       i != cs.attrs_set.end(); ++i)
+    {
+      json_io::builder tmp = b.add_obj();
+      tmp[syms::set].str(i->first.first.as_internal());
+      tmp[syms::attr].str(i->first.second());
+      tmp[syms::value].str(i->second());
+    }
+}
+
+static json_value_t
+revision_to_json(revision_t const & rev)
+{
+  json_io::builder b;
+  b[syms::type].str(syms::revision());
+  b[syms::vers].str("1");
+  b[syms::new_manifest].str(rev.new_manifest.inner()());
+  json_io::builder edges = b[syms::edges].arr();
+  for (edge_map::const_iterator e = rev.edges.begin(); 
+       e != rev.edges.end(); ++e)
+    {
+      json_io::builder edge = edges.add_obj();
+      edge[syms::old_revision].str(edge_old_revision(e).inner()());
+      cset_to_json(edge, edge_changes(e));
+    }
+  return b.v;
+}
+
+/////////////////////////////////////////////////////////////////////
+// core logic of gsync algorithm
+/////////////////////////////////////////////////////////////////////
+
+static inline void
+do_set_union(set<revision_id> const & a,
+	     set<revision_id> const & b,
+	     set<revision_id> & c)
+{
+  c.clear();
+  set_union(a.begin(), a.end(), b.begin(), b.end(), inserter(c, c.begin()));
+}
+
+static inline void
+do_set_difference(set<revision_id> const & a,
+		  set<revision_id> const & b,
+		  set<revision_id> & c)
+{
+  c.clear();
+  set_difference(a.begin(), a.end(), b.begin(), b.end(), inserter(c, c.begin()));
+}
+
+
+static void 
+inquire_about_revs(http_client & h,
+		   set<revision_id> const & query_set,
+		   set<revision_id> & theirs)
+{
+  theirs.clear();
+
+  json_io::builder b;
+  b[syms::type].str(syms::inquire());
+  b[syms::vers].str("1");
+  json_io::builder revs = b[syms::revs].arr();
+  for (set<revision_id>::const_iterator i = query_set.begin();
+       i != query_set.end(); ++i)
+    revs.add_str(i->inner()());
+  
+  json_value_t response = h.transact_json(b.v);
+  
+  string type, vers;
+  json_io::query q(response);
+
+  if (q[syms::type].get(type) &&
+      type == syms::confirm() &&
+      q[syms::vers].get(vers) &&
+      vers == "1")
+    {
+      size_t nrevs = 0;
+      string tmp;
+      json_io::query revs = q[syms::revs];
+      if (revs.len(nrevs))
+	for (size_t i = 0; i < nrevs; ++i) 
+	  if (revs[i].get(tmp))
+	    theirs.insert(revision_id(tmp));
+    }
+}
+
+static void
+determine_common_core(http_client & h,
+		      set<revision_id> const & our_revs,
+		      rev_ancestry_map const & child_to_parent_map,
+		      rev_ancestry_map const & parent_to_child_map,
+		      set<revision_id> & common_core)
+{
+  common_core.clear();
+  set<revision_id> unknown_revs = our_revs;
+  size_t pass = 0;
+
+  while (!unknown_revs.empty())
+    {
+      ++pass;
+      set<revision_id> query_revs;
+      
+      // Bite off a chunk of the remaining unknowns to ask about.
+      set<revision_id>::const_iterator r = unknown_revs.begin();
+      for (size_t i = 0; 
+	   i < constants::gsync_max_probe_set_size && r != unknown_revs.end(); 
+	   ++i, ++r)
+	{
+	  query_revs.insert(*r);
+	}
+
+      // Ask what they have of that chunk, form closures of the
+      // positive and negative sets on our side.
+      set<revision_id> revs_present, present_ancs, present_closure;
+      set<revision_id> revs_absent, absent_descs, absent_closure;
+
+      inquire_about_revs(h, query_revs, revs_present);
+      do_set_difference(query_revs, revs_present, revs_absent);
+
+      L(FL("pass #%d: inquired about %d revs, they have %d of them, missing %d of them") 
+	% pass
+	% query_revs.size() 
+	% revs_present.size()
+	% revs_absent.size());
+
+      get_all_ancestors(revs_present, child_to_parent_map, present_ancs);
+      do_set_union(revs_present, present_ancs, present_closure);
+
+      // FIXME: "ancestors" is a misnomer; it's a graph-closure calculation...
+      get_all_ancestors(revs_absent, parent_to_child_map, absent_descs);
+      do_set_union(revs_absent, absent_descs, absent_closure);
+
+      // Update the set we do not yet know about.
+      set<revision_id> new_unknown;
+      L(FL("pass #%d: unknown set initially: %d nodes") % pass % unknown_revs.size());
+
+      do_set_difference(unknown_revs, present_closure, new_unknown);
+      unknown_revs = new_unknown;
+      L(FL("pass #%d: unknown set after removing %d-entry present closure: %d nodes") 
+	% pass % present_closure.size() % unknown_revs.size());
+
+      do_set_difference(unknown_revs, absent_closure, new_unknown);
+      unknown_revs = new_unknown;
+      L(FL("pass #%d: unknown set after removing %d-entry absent closure: %d nodes") 
+	% pass % absent_closure.size() % unknown_revs.size());
+
+      // Update our total knowledge about them.
+      common_core.insert(present_closure.begin(), present_closure.end());
+    }
+}
+
+static void
+invert_ancestry(rev_ancestry_map const & in,
+		rev_ancestry_map & out)
+{
+  out.clear();
+  for (rev_ancestry_map::const_iterator i = in.begin();
+       i != in.end(); i++)
+    out.insert(make_pair(i->second, i->first));
+}
+
+static void
+do_missing_playback(http_client & h,
+		    app_state & app, 
+		    set<revision_id> const & core_frontier, 
+		    rev_ancestry_map const & child_to_parent_map)
+{
+  rev_ancestry_map parent_to_child_map;
+  invert_ancestry(child_to_parent_map, parent_to_child_map);
+}
+
+
+static void 
+request_missing_playback(http_client & h,
+			 app_state & app,
+			 set<revision_id> const & core_frontier)
+{
+  
+}
+
+void
+run_gsync_protocol(utf8 const & addr,
+		   globish const & include_pattern,
+		   globish const & exclude_pattern,
+		   app_state & app)
+{
+  uri u;
+  parse_uri(addr(), u);
+  http_client h(app, u, include_pattern, exclude_pattern);
+  
+  bool pushing = true, pulling = true;
+
+  rev_ancestry_map parent_to_child_map, child_to_parent_map;
+  app.db.get_revision_ancestry(parent_to_child_map);
+  invert_ancestry(parent_to_child_map, child_to_parent_map);
+
+  set<revision_id> our_revs;
+  for (rev_ancestry_map::const_iterator i = child_to_parent_map.begin(); 
+       i != child_to_parent_map.end(); ++i)
+    {
+      if (!i->first.inner()().empty())
+	our_revs.insert(i->first);
+      if (!i->second.inner()().empty())
+	our_revs.insert(i->second);
+    }
+
+  set<revision_id> common_core;
+  determine_common_core(h, our_revs, child_to_parent_map, parent_to_child_map, common_core);
+
+  set<revision_id> ours_alone;
+  do_set_difference(our_revs, common_core, ours_alone);
+  P(F("revs to send: %d") % ours_alone.size());
+
+  set<revision_id> core_frontier = common_core;
+  erase_ancestors(common_core, app);
+
+  if (pushing)
+    do_missing_playback(h, app, core_frontier, child_to_parent_map);
+
+  if (pulling)
+    request_missing_playback(h, app, core_frontier);
+}
+
+
+// Local Variables:
+// mode: C++
+// fill-column: 76
+// c-file-style: "gnu"
+// indent-tabs-mode: nil
+// End:
+// vim: et:sw=2:sts=2:ts=2:cino=>2s,{s,\:s,+s,t0,g0,^-2,e-2,n-2,p2s,(0,=s:
============================================================
--- http_client.cc	52d1fd430adb95bae1d814a2bebbdb99128fa0f4
+++ http_client.cc	52d1fd430adb95bae1d814a2bebbdb99128fa0f4
@@ -0,0 +1,229 @@
+// Copyright (C) 2007 Graydon Hoare <address@hidden>
+//
+// This program is made available under the GNU GPL version 2.0 or
+// greater. See the accompanying file COPYING for details.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the
+// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE.
+
+
+// micro http client implementation
+#include "base.hh"
+#include "app_state.hh"
+#include "globish.hh"
+#include "http_client.hh"
+#include "json_io.hh"
+#include "sanity.hh"
+#include "lexical_cast.hh"
+#include "constants.hh"
+#include "uri.hh"
+
+#include "netxx/address.h"
+#include "netxx/netbuf.h"
+#include "netxx/stream.h"
+#include "netxx/timeout.h"
+#include "netxx_pipe.hh"
+
+#include <vector>
+#include <string>
+
+#include <boost/shared_ptr.hpp>
+
+using json_io::json_value_t;
+using boost::shared_ptr;
+using boost::lexical_cast;
+using std::vector;
+using std::string;
+using std::iostream;
+
+
+using Netxx::Netbuf;
+using Netxx::Timeout;
+using Netxx::StreamBase;
+using Netxx::Stream;
+using Netxx::PipeStream;
+
+
+
+static shared_ptr<StreamBase>
+build_stream(app_state & app,
+	     uri const & u,
+	     globish const & include_pattern,
+	     globish const & exclude_pattern,	     
+	     Netxx::port_type default_port)
+{
+  Timeout timeout(static_cast<long>(constants::netsync_timeout_seconds)), 
+    instant(0,1);
+  vector<string> argv;    
+  if (app.lua.hook_get_netsync_connect_command(u,
+					       include_pattern,
+					       exclude_pattern,
+					       global_sanity.debug_p(),
+					       argv))
+    {
+      I(argv.size() > 0);
+      string cmd = argv[0];
+      argv.erase(argv.begin());
+      app.opts.use_transport_auth = app.lua.hook_use_transport_auth(u);
+      return shared_ptr<StreamBase>(new PipeStream(cmd, argv));
+    }
+  else
+    {
+#ifdef USE_IPV6
+      bool use_ipv6=true;
+#else
+      bool use_ipv6=false;
+#endif
+      Netxx::Address addr(u.host.c_str(), default_port, use_ipv6);
+      return shared_ptr<StreamBase>(new Stream(addr, timeout));
+    }   
+}
+
+
+http_client::http_client(app_state & app,
+			 uri const & u, 	      
+			 globish const & include_pattern,
+			 globish const & exclude_pattern)
+  : app(app), 
+    u(u), 
+    include_pattern(include_pattern), 
+    exclude_pattern(exclude_pattern), 
+    stream(build_stream(app, u, include_pattern, exclude_pattern, 
+			constants::default_http_port)), 
+    nb(new Netbuf<constants::bufsz>(*stream)), 
+    io(new iostream(&(*nb))),
+    open(true)
+{}
+
+json_value_t
+http_client::transact_json(json_value_t v)
+{
+  if (!open)
+    {
+      L(FL("reopening connection"));
+      stream = build_stream(app, u, include_pattern, exclude_pattern, 
+			    constants::default_http_port);
+      nb = shared_ptr< Netbuf<constants::bufsz> >(new Netbuf<constants::bufsz>(*stream));
+      io = shared_ptr<iostream>(new iostream(&(*nb)));
+      open = true;
+    }
+
+  I(stream);
+  I(nb);
+  I(io);
+  I(open);
+
+  json_io::printer out;
+  v->write(out);
+  string header = (F("POST %s HTTP/1.0\r\n"
+		     "Host: %s\r\n"
+		     "Content-Length: %s\r\n"
+		     "Content-Type: application/jsonrequest\r\n"
+		     "Accept: application/jsonrequest\r\n"
+		     "Accept-Encoding: identity\r\n"
+		     "Connection: Keep-Alive\r\n"
+		     "\r\n") 
+		   % (u.path.empty() ? "/" : u.path)
+		   % u.host 
+		   % lexical_cast<string>(out.buf.size())).str();
+  
+  L(FL("http_client: sending request [[POST %s HTTP/1.0]]") 
+    % (u.path.empty() ? "/" : u.path));
+  L(FL("http_client: to [[Host: %s]]") % u.host);
+  io->write(header.data(), header.size());
+  io->write(out.buf.data(), out.buf.length());
+  io->flush();
+
+  L(FL("http_client: sending %d-byte body") % out.buf.size());
+
+  // Now read back the result
+  string data;
+  parse_http_response(data);
+  json_io::input_source in(data, "scgi");
+  json_io::tokenizer tok(in);
+  json_io::parser p(tok);
+  return p.parse_object();
+}
+
+
+
+void 
+http_client::parse_http_status_line()
+{
+  // We're only interested in 200-series responses
+  string tmp;
+  string pat("HTTP/1.0 200");
+  while (tmp.empty())
+    std::getline(*io, tmp);
+  L(FL("http_client: response: [[%s]]") % tmp);
+  E(tmp.substr(0,pat.size()) == pat, F("HTTP status line: %s") % tmp);
+}
+
+void 
+http_client::parse_http_header_line(size_t & content_length, 
+				    bool & keepalive)
+{
+  string k, v, rest;
+  (*io) >> k >> v;
+  L(FL("http_client: header: [[%s %s]]") % k % v);
+  std::getline(*io, rest);
+  
+  if (k == "Content-Length:" 
+      || k == "Content-length:"
+      || k == "content-length:")
+    content_length = lexical_cast<size_t>(v);
+  else if (k == "Connection:" 
+	   || k == "connection:")
+    keepalive = (v == "Keep-Alive" 
+		 || v == "Keep-alive" 
+		 || v == "keep-alive");
+}
+
+void 
+http_client::crlf()
+{
+  E(io->get() == '\r', F("expected CR in HTTP response"));
+  E(io->get() == '\n', F("expected LF in HTTP response"));
+}
+
+void 
+http_client::parse_http_response(std::string & data)
+{
+  size_t content_length = 0;
+  bool keepalive = false;
+  data.clear();
+  parse_http_status_line();
+  while (io->peek() != '\r')
+    parse_http_header_line(content_length, keepalive);
+  crlf();
+
+  L(FL("http_client: receiving %d-byte body") % content_length);
+
+  while (io->good() && content_length > 0)
+    {
+      data += static_cast<char>(io->get());
+      content_length--;
+    }
+
+  io->flush();
+
+  if (!keepalive) 
+    {
+      L(FL("http_client: closing connection"));
+      stream->close();
+      io.reset();
+      nb.reset();
+      stream.reset();
+      open = false;
+    }
+}
+
+
+// Local Variables:
+// mode: C++
+// fill-column: 76
+// c-file-style: "gnu"
+// indent-tabs-mode: nil
+// End:
+// vim: et:sw=2:sts=2:ts=2:cino=>2s,{s,\:s,+s,t0,g0,^-2,e-2,n-2,p2s,(0,=s:
============================================================
--- http_client.hh	1d37aad6de15a308b82eebb4943c387546f607b2
+++ http_client.hh	1d37aad6de15a308b82eebb4943c387546f607b2
@@ -0,0 +1,63 @@
+#ifndef __HTTP_CLIENT__
+#define __HTTP_CLIENT__
+
+// Copyright (C) 2007 Graydon Hoare <address@hidden>
+//
+// This program is made available under the GNU GPL version 2.0 or
+// greater. See the accompanying file COPYING for details.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the
+// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE.
+
+#include "netxx/netbuf.h"
+#include "netxx/streambase.h"
+
+#include "base.hh"
+#include "constants.hh"
+#include "json_io.hh"
+
+#include <iostream>
+
+#include <boost/shared_ptr.hpp>
+
+struct app_state;
+struct uri;
+struct globish;
+
+
+struct
+http_client
+{
+  app_state & app;
+  uri const & u;
+  globish const & include_pattern;
+  globish const & exclude_pattern;
+
+  boost::shared_ptr<Netxx::StreamBase> stream;
+  boost::shared_ptr< Netxx::Netbuf<constants::bufsz> > nb;
+  boost::shared_ptr<std::iostream> io;
+  bool open;
+
+  http_client(app_state & app,
+	      uri const & u, 	      
+	      globish const & include_pattern,
+	      globish const & exclude_pattern); 
+
+  json_io::json_value_t transact_json(json_io::json_value_t v);
+  void parse_http_status_line();
+  void parse_http_header_line(size_t & content_length,
+			      bool & keepalive);
+  void parse_http_response(std::string & data);
+  void crlf();  
+};
+
+// Local Variables:
+// mode: C++
+// fill-column: 76
+// c-file-style: "gnu"
+// indent-tabs-mode: nil
+// End:
+// vim: et:sw=2:sts=2:ts=2:cino=>2s,{s,\:s,+s,t0,g0,^-2,e-2,n-2,p2s,(0,=s:
+
+#endif // __HTTP_CLIENT__
============================================================
--- json_io.cc	a32899cbd5f29b85ee33e444f547ea3e4ca637a0
+++ json_io.cc	a32899cbd5f29b85ee33e444f547ea3e4ca637a0
@@ -0,0 +1,204 @@
+// Copyright (C) 2007 Graydon Hoare <address@hidden>
+//
+// This program is made available under the GNU GPL version 2.0 or
+// greater. See the accompanying file COPYING for details.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the
+// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE.
+
+#include "base.hh"
+#include <sstream>
+#include <cctype>
+#include <cstdlib>
+
+#include "json_io.hh"
+#include "sanity.hh"
+#include "vocab.hh"
+
+using std::logic_error;
+using std::make_pair;
+using std::pair;
+using std::string;
+using std::vector;
+
+
+void 
+json_io::input_source::err(string const & s)
+{
+  E(false,
+    F("parsing a %s at %d:%d:E: %s") % name % line % col % s);
+}
+
+
+void 
+json_io::tokenizer::err(string const & s)
+{
+  in.err(s);
+}
+
+string json_io::printer::buf;
+size_t json_io::printer::count;
+
+
+json_io::printer::printer()
+{  
+  I(count == 0);
+  count++;
+  indent = 0;
+  buf.clear();
+}
+
+json_io::printer::~printer()
+{
+  count--;
+}
+
+string
+json_io::escape(string const & s)
+{
+  string escaped;
+  escaped.reserve(s.size() + 8);
+
+  escaped += "\"";
+
+  for (string::const_iterator i = s.begin(); i != s.end(); ++i)
+    {
+      switch (*i)
+        {
+        case '"':
+          escaped += "\\\"";
+          break;
+        case '\\':
+          escaped += "\\\\";
+          break;
+        case '/':
+          escaped += "\\/";
+          break;
+        case '\b':
+          escaped += "\\b";
+          break;
+        case '\f':
+          escaped += "\\f";
+          break;
+        case '\n':
+          escaped += "\\n";
+          break;
+        case '\r':
+          escaped += "\\r";
+          break;
+        case '\t':
+          escaped += "\\t";
+          break;          
+        default:
+          escaped += *i;
+        }
+    }
+
+  escaped += "\"";
+
+  return escaped;
+}
+
+
+
+void json_io::parser::err(string const & s)
+{
+  tok.err(s);
+}
+
+string json_io::parser::tt2str(token_type tt)
+{
+  switch (tt)
+    {
+    case json_io::TOK_STRING:
+      return "TOK_STRING";
+    case json_io::TOK_SYMBOL:
+      return "TOK_SYMBOL";
+    case json_io::TOK_LBRACE:
+      return "TOK_LBRACE";
+    case json_io::TOK_RBRACE:
+      return "TOK_RBRACE";
+    case json_io::TOK_LBRACKET:
+      return "TOK_LBRACKET";
+    case json_io::TOK_RBRACKET:
+      return "TOK_RBRACKET";
+    case json_io::TOK_COMMA:
+      return "TOK_COMMA";
+    case json_io::TOK_COLON:
+      return "TOK_COLON";
+    case json_io::TOK_NONE:
+      return "TOK_NONE";
+    }
+  return "TOK_UNKNOWN";
+}
+
+
+void
+json_io::json_string::write(printer &pr)
+{
+  pr.append(escape(data));
+}
+
+void
+json_io::json_object::write(printer &pr)
+{
+  bool first = true;
+  pr.append("{\n");
+  pr.indent++;
+  for (std::map<std::string, json_value_t>::const_iterator 
+         i = fields.begin(); i != fields.end(); ++i) 
+    {
+      if (!first)
+        pr.append(",\n");
+      pr.append_indent();
+      pr.append(escape(i->first));
+      pr.append(": ");
+      i->second->write(pr);
+      first = false;
+    }
+  pr.indent--;
+  pr.append("\n");
+  pr.append_indent();
+  pr.append("}");
+}
+  
+void 
+json_io::json_array::write(printer &pr)
+{
+  bool first = true;
+  pr.append("[\n");
+  pr.indent++;
+  for (std::vector<json_value_t>::const_iterator 
+         i = fields.begin(); i != fields.end(); ++i) 
+    {
+      if (!first)
+        pr.append(",\n");
+      pr.append_indent();
+      (*i)->write(pr);
+      first = false;
+    }
+  pr.indent--;
+  pr.append("\n");
+  pr.append_indent();
+  pr.append("]");
+}
+
+
+
+#ifdef BUILD_UNIT_TESTS
+#include "unit_tests.hh"
+
+UNIT_TEST(json_io, binary_transparency)
+{
+}
+
+#endif // BUILD_UNIT_TESTS
+
+// Local Variables:
+// mode: C++
+// fill-column: 76
+// c-file-style: "gnu"
+// indent-tabs-mode: nil
+// End:
+// vim: et:sw=2:sts=2:ts=2:cino=>2s,{s,\:s,+s,t0,g0,^-2,e-2,n-2,p2s,(0,=s:
============================================================
--- json_io.hh	07bdba5c2135920bb86262786926beb73f4e11c7
+++ json_io.hh	07bdba5c2135920bb86262786926beb73f4e11c7
@@ -0,0 +1,641 @@
+#ifndef __JSON_IO_HH__
+#define __JSON_IO_HH__
+
+// Copyright (C) 2007 Graydon Hoare <address@hidden>
+//
+// This program is made available under the GNU GPL version 2.0 or
+// greater. See the accompanying file COPYING for details.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the
+// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+// PURPOSE.
+
+
+#include "vector.hh"
+#include <map>
+
+#include <boost/shared_ptr.hpp>
+
+#include "paths.hh"
+#include "sanity.hh"
+#include "vocab.hh"
+#include "numeric_vocab.hh"
+#include "safe_map.hh"
+
+namespace json_io
+{
+
+  ///////////////////////////////////////////////////////////
+  // vocabulary
+  ///////////////////////////////////////////////////////////
+
+  struct printer;
+
+  struct json_value 
+  {
+    virtual void write(printer &pr) = 0;
+    virtual ~json_value() {}
+  };
+
+  typedef boost::shared_ptr<json_value> json_value_t;
+
+  struct json_object 
+    : public json_value
+  {
+    std::map<std::string, json_value_t> fields;
+    void add(std::string const &str, json_value_t v) 
+    { fields.insert(std::make_pair(str, v)); }
+    virtual void write(printer &pr);
+    virtual ~json_object() {}
+  };
+
+  typedef boost::shared_ptr<json_object> json_object_t;
+  
+  struct json_array 
+    : public json_value
+  {
+    std::vector<json_value_t> fields;
+    void add(json_value_t v) 
+    { fields.push_back(v); }
+    virtual void write(printer &pr);        
+    virtual ~json_array() {}
+  };
+
+  typedef boost::shared_ptr<json_array> json_array_t;
+
+  struct json_string  
+    : public json_value
+  {
+    json_string(std::string const &s) : data(s) {}
+    std::string data;
+    virtual void write(printer &pr);
+    virtual ~json_string() {}
+  };
+
+  typedef boost::shared_ptr<json_string> json_string_t;
+
+  ///////////////////////////////////////////////////////////
+  // lexing
+  ///////////////////////////////////////////////////////////
+    
+  typedef enum
+    {
+      TOK_SYMBOL,
+      TOK_STRING,
+      TOK_LBRACE,
+      TOK_RBRACE,
+      TOK_LBRACKET,
+      TOK_RBRACKET,
+      TOK_COMMA,
+      TOK_COLON,
+      TOK_NONE
+    } token_type;
+
+  struct
+  input_source
+  {
+    size_t line, col;
+    std::string const & in;
+    std::string::const_iterator curr;
+    std::string name;
+    int lookahead;
+    char c;
+    input_source(std::string const & in, std::string const & nm)
+      : line(1), col(1), in(in), curr(in.begin()),
+	name(nm), lookahead(0), c('\0')
+    {}
+
+    inline void peek()
+    {
+      if (LIKELY(curr != in.end()))
+        // we do want to distinguish between EOF and '\xff', 
+        // so we translate '\xff' to 255u
+	lookahead = widen<unsigned int,char>(*curr);
+      else
+	lookahead = EOF;
+    }
+
+    inline void advance()
+    {
+      if (LIKELY(curr != in.end()))
+        {
+          c = *curr;
+          ++curr;
+          ++col;
+          if (c == '\n')
+            {
+              col = 1;
+              ++line;
+            }
+        }
+      peek();
+    }
+    void err(std::string const & s);
+  };
+
+  struct
+  tokenizer
+  {
+    input_source & in;
+    std::string::const_iterator begin;
+    std::string::const_iterator end;
+
+    tokenizer(input_source & i) : in(i), begin(in.curr), end(in.curr)
+    {}
+
+    inline void mark()
+    {
+      begin = in.curr;
+      end = begin;
+    }
+
+    inline void advance()
+    {
+      in.advance();
+      end = in.curr;
+    }
+
+    inline void store(std::string & val)
+    {
+      val.assign(begin, end);
+    }
+
+    inline void
+    read_escape(std::string & val, char c)
+    {
+      switch (c) 
+        {
+        case '/':
+        case '\\':
+        case '"':
+          val += c;
+          break;
+        case 'b':
+          val += '\b';
+          break;
+        case 'f':
+          val += '\f';
+          break;
+        case 'n':
+          val += '\n';
+          break;
+        case 'r':
+          val += '\r';
+          break;
+        case 't':
+          val += '\t';
+          break;
+        default:
+          in.err("unrecognized character escape");
+          break;
+        }
+    }
+
+    inline token_type get_token(std::string & val)
+    {
+      in.peek();
+
+      while (true)
+        {
+          if (UNLIKELY(in.lookahead == EOF))
+            return TOK_NONE;
+          if (!is_space(in.lookahead))
+            break;
+          in.advance();
+        }
+
+      if (is_alpha(in.lookahead))
+	{
+	  mark();
+	  while (is_alnum(in.lookahead) || in.lookahead == '_')
+	    advance();
+	  store(val);
+	  return json_io::TOK_SYMBOL;
+	}
+
+      else if (in.lookahead == '"')
+	{
+	  in.advance();
+	  mark();
+	  while (static_cast<char>(in.lookahead) != '"')
+	    {
+	      if (UNLIKELY(in.lookahead == EOF))
+		in.err("input stream ended in string");
+	      if (UNLIKELY(static_cast<char>(in.lookahead) == '\\'))
+		{
+                  // When we hit an escape, we switch from doing mark/store
+                  // to a slower per-character append loop, until the end
+                  // of the token.
+
+                  // So first, store what we have *before* the escape.
+                  store(val);
+
+                  // Then skip over the escape backslash.
+		  in.advance();
+
+                  // Handle the escaped char.
+                  read_escape(val, static_cast<char>(in.lookahead));
+
+                  // Advance past the escaped char.
+                  in.advance();
+
+                  // Now enter special slow loop for remainder.
+                  while (static_cast<char>(in.lookahead) != '"')
+                    {
+                      if (UNLIKELY(in.lookahead == EOF))
+                        in.err("input stream ended in string");
+                      if (UNLIKELY(static_cast<char>(in.lookahead) == '\\'))
+                        {
+                          in.advance();
+                          read_escape(val, static_cast<char>(in.lookahead));
+                          in.advance();
+                        }
+                      else
+                        {
+                          in.advance();
+                          val += in.c;
+                        }
+                    }
+                  // When slow loop completes, return early.
+                  if (static_cast<char>(in.lookahead) != '"')
+                    in.err("string did not end with '\"'");
+                  in.advance();
+
+                  return json_io::TOK_STRING;
+		}
+	      advance();
+	    }
+	
+	  store(val);
+
+	  if (UNLIKELY(static_cast<char>(in.lookahead) != '"'))
+	    in.err("string did not end with '\"'");
+	  in.advance();
+	
+	  return json_io::TOK_STRING;
+	}
+      else if (in.lookahead == '[')
+	{
+	  in.advance();
+          return json_io::TOK_LBRACKET;
+        }
+      else if (in.lookahead == ']')
+        {
+	  in.advance();
+          return json_io::TOK_RBRACKET;
+        }
+      else if (in.lookahead == '{')
+        {
+	  in.advance();
+          return json_io::TOK_LBRACE;
+        }
+      else if (in.lookahead == '}')
+        {
+	  in.advance();
+          return json_io::TOK_RBRACE;
+        }
+      else if (in.lookahead == ':')
+        {
+	  in.advance();
+          return json_io::TOK_COLON;
+        }
+      else if (in.lookahead == ',')
+        {
+	  in.advance();
+          return json_io::TOK_COMMA;
+        }
+      else
+	return json_io::TOK_NONE;
+    }
+   void err(std::string const & s);
+  };
+
+
+  ///////////////////////////////////////////////////////////
+  // parsing
+  ///////////////////////////////////////////////////////////
+  
+  struct
+  parser
+  {
+    tokenizer & tok;
+    parser(tokenizer & t) : tok(t)
+    {
+      token.reserve(128);
+      advance();
+    }
+
+    std::string token;
+    token_type ttype;
+
+    void err(std::string const & s);
+    std::string tt2str(token_type tt);
+
+    inline void advance()
+    {
+      ttype = tok.get_token(token);
+    }
+
+    inline void eat(token_type want)
+    {
+      if (ttype != want)
+        err("wanted "
+            + tt2str(want)
+            + ", got "
+            + tt2str(ttype)
+            + (token.empty()
+               ? std::string("")
+               : (std::string(" with value ") + token)));
+      advance();
+    }
+
+
+    inline json_object_t
+    parse_object() 
+    {
+      bool first = true;
+      json_object_t obj(new json_object());
+      lbrace();
+      while (ttype != TOK_RBRACE) 
+        {
+          if (!first)
+            comma();
+          first = false;
+          std::string key;
+          str(key);
+          colon();
+          json_value_t val = parse_value();
+          safe_insert(obj->fields, std::make_pair(key, val));
+        }
+      rbrace();
+      return obj;
+    }
+
+    inline json_array_t
+    parse_array() 
+    {
+      bool first = true;
+      json_array_t arr(new json_array());
+      lbracket();
+      while (ttype != TOK_RBRACKET) 
+        {
+          if (!first)
+            comma();
+          first = false;
+          json_value_t val = parse_value();
+          arr->add(val);
+        }
+      rbracket();
+      return arr;
+    }
+
+    inline json_string_t
+    parse_string()
+    {
+      json_string_t s(new json_string(""));
+      str(s->data);
+      return s;
+    }
+
+    inline json_value_t
+    parse_value()
+    {
+      if (ttype == TOK_LBRACE)
+        return parse_object();
+      else if (ttype == TOK_LBRACKET)
+        return parse_array();
+      else if (ttype == TOK_STRING)
+        return parse_string();
+      else 
+        return json_value_t();
+    }
+    
+    inline void str() { eat(json_io::TOK_STRING); }
+    inline void sym() { eat(json_io::TOK_SYMBOL); }
+    inline void colon() { eat(json_io::TOK_COLON); }
+    inline void comma() { eat(json_io::TOK_COMMA); }
+    inline void lbrace() { eat(json_io::TOK_LBRACE); }
+    inline void rbrace() { eat(json_io::TOK_RBRACE); }
+    inline void lbracket() { eat(json_io::TOK_LBRACKET); }
+    inline void rbracket() { eat(json_io::TOK_RBRACKET); }
+
+    inline void str(std::string & v) { v = token; str(); }
+    inline void sym(std::string & v) { v = token; sym(); }
+  };
+
+
+  ///////////////////////////////////////////////////////////
+  // printing
+  ///////////////////////////////////////////////////////////
+
+  std::string escape(std::string const & s);
+
+  // Note: printer uses a static buffer; thus only one buffer 
+  // may be referenced (globally). An invariant will be triggered
+  // if more than one json_io::printer is instantiated.
+  struct
+  printer
+  {
+    static std::string buf;
+    static size_t count;
+    size_t indent;
+    printer();
+    ~printer();
+    void append(std::string const &s)
+    {
+      buf.append(s);
+    }
+    void append_indent() 
+    {
+      for (size_t i = 0; i < indent; ++i)
+        buf += '\t';
+    }
+  };
+
+
+  ///////////////////////////////////////////////////////////
+  /////////////////////// building //////////////////////////
+  ///////////////////////////////////////////////////////////
+  
+  struct builder
+  {
+    json_value_t v;
+    std::string key;
+    builder(json_value_t v, symbol const &k) : v(v), key(k()) {}
+    builder(json_value_t v) : v(v), key("") {}
+    builder() : v(new json_object()), key("") {}
+
+    json_object_t as_obj() 
+    {
+      json_object_t ob = boost::dynamic_pointer_cast<json_object, json_value>(v);
+      I(static_cast<bool>(ob));
+      return ob;
+    }
+
+    json_array_t as_arr() 
+    {
+      json_array_t a = boost::dynamic_pointer_cast<json_array, json_value>(v);
+      I(static_cast<bool>(a));
+      return a;
+    }
+
+    json_string_t as_str() 
+    {
+      json_string_t s = boost::dynamic_pointer_cast<json_string, json_value>(v);
+      I(static_cast<bool>(s));
+      return s;
+    }
+
+    builder bad()
+    {
+      return builder(json_value_t());
+    }
+
+    builder operator[](symbol const &k)
+    {
+      I(key.empty());
+      return builder(as_obj(), k);
+    }
+
+    void add_str(std::string const &s)
+    {      
+      I(key.empty());
+      as_arr()->add(json_string_t(new json_string(s)));
+    }
+
+    builder add_obj()
+    {
+      I(key.empty());
+      json_array_t a = as_arr();
+      json_object_t ob(new json_object());
+      a->add(ob);
+      return builder(ob);
+    }
+
+    builder add_arr()
+    {
+      I(key.empty()); 
+      json_array_t a = as_arr();
+      json_array_t a2(new json_array());
+      a->add(a2);
+      return builder(a2);
+    }
+
+    void add(json_value_t val)
+    {
+      I(key.empty());
+      as_arr()->add(val);
+    }
+
+    void set(json_value_t v) 
+    {
+      I(!key.empty());
+      as_obj()->add(key, v);
+    }
+
+    void str(std::string const &s) 
+    {
+      set(json_string_t(new json_string(s)));
+    }
+
+    builder obj() 
+    {
+      json_object_t ob(new json_object());
+      set(ob);
+      return builder(ob);
+    }
+
+    builder arr() 
+    {
+      json_array_t a(new json_array());
+      set(a);
+      return builder(a);
+    }
+  };
+
+  ///////////////////////////////////////////////////////////
+  ///////////////////////   query  //////////////////////////
+  ///////////////////////////////////////////////////////////
+
+  struct 
+  query
+  {
+    json_value_t v;
+    query(json_value_t v) : v(v) {}
+
+    json_object_t as_obj() 
+    {
+      return boost::dynamic_pointer_cast<json_object, json_value>(v);
+    }
+
+    json_array_t as_arr() 
+    {
+      return boost::dynamic_pointer_cast<json_array, json_value>(v);
+    }
+
+    json_string_t as_str() 
+    {
+      return boost::dynamic_pointer_cast<json_string, json_value>(v);
+    }
+
+    query bad()
+    {
+      return query(json_value_t());
+    }
+
+
+    query operator[](symbol const &key) 
+    {
+      json_object_t ob = as_obj();
+      if (static_cast<bool>(ob))
+        {
+          std::map<std::string, json_value_t>::const_iterator i = ob->fields.find(key());
+          if (i != ob->fields.end())
+            return query(i->second);
+        }
+      return bad();
+    }
+
+    query operator[](size_t &idx) 
+    {
+      json_array_t a = as_arr();
+      if (static_cast<bool>(a) && idx < a->fields.size()) 
+        return query(a->fields.at(idx));
+      return bad();
+    }
+
+    bool len(size_t & length) { 
+      json_array_t a = as_arr();
+      if (static_cast<bool>(a)) {
+        length = a->fields.size();
+        return true;
+      }
+      return false;
+    }
+
+    json_value_t get() 
+    {
+      return v;
+    }
+
+    bool get(std::string & str) { 
+      json_string_t s = as_str();
+      if (static_cast<bool>(s)) 
+        {
+          str = s->data;
+          return true;
+        }
+      return false;
+    }    
+  };
+}
+
+// Local Variables:
+// mode: C++
+// fill-column: 76
+// c-file-style: "gnu"
+// indent-tabs-mode: nil
+// End:
+// vim: et:sw=2:sts=2:ts=2:cino=>2s,{s,\:s,+s,t0,g0,^-2,e-2,n-2,p2s,(0,=s:
+
+#endif // __JSON_IO_HH__
============================================================
--- Makefile.am	dfaa00d12056b5dc7a53ea779ba605d890669470
+++ Makefile.am	e9e3d2c42df7879640a66e21854ad11a3f8eba30
@@ -1,10 +1,10 @@ CMD_SOURCES =								\
 AUTOMAKE_OPTIONS=subdir-objects 1.7.1
 ACLOCAL_AMFLAGS = -I m4
 
 CMD_SOURCES =								\
 	cmd.hh cmd_netsync.cc cmd_list.cc cmd_packet.cc	cmd_key_cert.cc \
 	cmd_merging.cc cmd_db.cc cmd_diff_log.cc cmd_ws_commit.cc 	\
-	cmd_othervcs.cc cmd_automate.cc cmd_files.cc
+	cmd_othervcs.cc cmd_automate.cc cmd_files.cc cmd_scgi.cc
 
 SANITY_CORE_SOURCES =							\
 	sanity.cc sanity.hh quick_alloc.hh vector.hh base.hh		\
@@ -19,7 +19,8 @@ LUAEXT_SOURCES = 							\
 	luaext_mkstemp.cc luaext_parse_basic_io.cc			\
 	luaext_guess_binary.cc luaext_platform.cc luaext_globish.cc	\
 	lua.cc lua.hh mkstemp.cc mkstemp.hh file_io.cc file_io.hh	\
-	globish.cc globish.hh basic_io.cc basic_io.hh
+	globish.cc globish.hh basic_io.cc basic_io.hh json_io.cc        \
+        json_io.hh
 
 MOST_SOURCES = 								\
 	$(SANITY_CORE_SOURCES) $(LUAEXT_SOURCES) platform-wrapped.hh	\
@@ -45,6 +46,7 @@ MOST_SOURCES = 								\
 	refiner.cc refiner.hh						\
 	enumerator.cc enumerator.hh					\
 	netsync.cc            						\
+	gsync.cc            						\
 	netxx_pipe.cc netxx_pipe.hh					\
 	netcmd.cc netcmd.hh						\
 	merkle_tree.cc merkle_tree.hh 					\
@@ -74,6 +76,7 @@ MOST_SOURCES = 								\
 	rev_height.cc rev_height.hh					\
 	asciik.cc asciik.hh						\
 	dates.cc dates.hh						\
+	http_client.cc http_client.hh                                   \
 									\
 	lru_writeback_cache.hh hybrid_map.hh				\
 									\
@@ -308,7 +311,7 @@ UNIT_TEST_SOURCES =							\
 	packet.cc paths.cc refiner.cc restrictions.cc rev_height.cc	\
 	revision.cc roster.cc roster_merge.cc simplestring_xform.cc	\
 	string_queue.cc transforms.cc unit_tests.cc uri.cc vocab.cc	\
-	xdelta.cc
+	xdelta.cc json_io.cc
 
 # these files do not contain unit tests, but are required for unit testing
 # and must be recompiled for that purpose
============================================================
--- cmd_netsync.cc	39a96b273f26950ce8701dd041994f5c0b254877
+++ cmd_netsync.cc	0571b71d680fc18500b38f25185ee7500402c31d
@@ -452,6 +452,30 @@ CMD_NO_WORKSPACE(serve, "serve", "", CMD
                        globish("*"), globish(""), app);
 }
 
+void 
+run_gsync_protocol(utf8 const & addr,
+		   globish const & include_pattern,
+		   globish const & exclude_pattern,
+		   app_state & app);
+
+CMD(gsync, "gsync", "", CMD_REF(network),
+    N_("[ADDRESS[:PORTNUMBER] [PATTERN ...]]"),
+    N_("Synchronizes branches with a netsync server"),
+    N_("This synchronizes branches that match the pattern given in PATTERN "
+       "with the gsync server at the address ADDRESS."),
+    options::opts::set_default | options::opts::exclude |
+    options::opts::key_to_push)
+{
+  utf8 addr;
+  globish include_pattern, exclude_pattern;
+  extract_address(args, addr, app);
+  extract_patterns(args, include_pattern, exclude_pattern, app);
+  find_key_if_needed(addr, include_pattern, exclude_pattern, app);
+
+  run_gsync_protocol(addr, include_pattern, exclude_pattern, app);
+}
+
+
 // Local Variables:
 // mode: C++
 // fill-column: 76
============================================================
--- constants.hh	35e5834105b0ab3572749d071313fbd106b81fe0
+++ constants.hh	26ea1fa80b9236854f37ac0fbdeaccf6236a5c90
@@ -98,6 +98,12 @@ namespace constants
   // all the ASCII characters (bytes) which can occur in key names
   extern char const legal_key_name_bytes[];
 
+  // maximum number nodes in a randomized gsync probe set
+  std::size_t const gsync_max_probe_set_size = 128;
+
+  // standard HTTP port number
+  std::size_t const default_http_port = 80;
+
   // remaining constants are related to netsync protocol
 
   // number of bytes in the hash used in netsync
============================================================
--- database.cc	331c15584601bfa5585b5169af3bc597196a0c47
+++ database.cc	c3782890880b957df6e7ab1ab3a334915bad2324
@@ -1188,10 +1188,12 @@ database::get_roster_base(string const &
   query q("SELECT checksum, data FROM rosters WHERE id = ?");
   fetch(res, 2, one_row, q % text(ident_str));
 
-  hexenc<id> checksum(res[0][0]);
-  hexenc<id> calculated;
-  calculate_ident(data(res[0][1]), calculated);
-  I(calculated == checksum);
+  /*
+    hexenc<id> checksum(res[0][0]);
+    hexenc<id> calculated;
+    calculate_ident(data(res[0][1]), calculated);
+    I(calculated == checksum);
+  */
 
   gzip<data> dat_packed(res[0][1]);
   data dat;
@@ -1208,10 +1210,12 @@ database::get_roster_delta(string const 
   query q("SELECT checksum, delta FROM roster_deltas WHERE id = ? AND base = ?");
   fetch(res, 2, one_row, q % text(ident) % text(base));
 
-  hexenc<id> checksum(res[0][0]);
-  hexenc<id> calculated;
-  calculate_ident(data(res[0][1]), calculated);
-  I(calculated == checksum);
+  /*
+    hexenc<id> checksum(res[0][0]);
+    hexenc<id> calculated;
+    calculate_ident(data(res[0][1]), calculated);
+    I(calculated == checksum);
+  */
 
   gzip<delta> del_packed(res[0][1]);
   delta tmp;
@@ -1227,12 +1231,14 @@ database::write_delayed_file(file_id con
   encode_gzip(dat.inner(), dat_packed);
 
   // ident is a hash, which we should check
-  I(!null_id(ident));
-  file_id tid;
-  calculate_ident(dat, tid);
-  MM(ident);
-  MM(tid);
-  I(tid == ident);
+  /*
+    I(!null_id(ident));
+    file_id tid;
+    calculate_ident(dat, tid);
+    MM(ident);
+    MM(tid);
+    I(tid == ident);
+  */
   // and then write things to the db
   query q("INSERT INTO files (id, data) VALUES (?, ?)");
   execute(q % text(ident.inner()()) % blob(dat_packed()));
@@ -1389,9 +1395,11 @@ database::get_version(hexenc<id> const &
   appl->finish(tmp);
   dat = data(tmp);
 
-  hexenc<id> final;
-  calculate_ident(dat, final);
-  I(final == ident);
+  /*
+    hexenc<id> final;
+    calculate_ident(dat, final);
+    I(final == ident);
+  */
 
   if (!vcache.exists(ident()))
     vcache.insert_clean(ident(), dat);
@@ -1603,11 +1611,14 @@ database::get_roster_version(revision_id
   // delta reconstruction code; if there is a bug where we put something
   // into the database and then later get something different back out, then
   // this is the only thing that can catch it.
+
+  /*
   roster->check_sane_against(*marking);
   manifest_id expected_mid, actual_mid;
   get_revision_manifest(id, expected_mid);
   calculate_ident(*roster, actual_mid);
   I(expected_mid == actual_mid);
+  */
 
   // const'ify the objects, to save them and pass them out
   cr.first = roster;
@@ -1907,11 +1918,13 @@ database::get_revision(revision_id const
   decode_gzip(gzdata,rdat);
 
   // verify that we got a revision with the right id
+  /*
   {
     revision_id tmp;
     calculate_ident(revision_data(rdat), tmp);
     I(id == tmp);
   }
+  */
 
   dat = revision_data(rdat);
 }
@@ -2178,8 +2191,10 @@ database::put_roster_for_revision(revisi
   manifest_id roster_manifest_id;
   MM(roster_manifest_id);
   make_roster_for_revision(rev, new_id, *ros_writeable, *mm_writeable, *__app);
-  calculate_ident(*ros_writeable, roster_manifest_id);
-  I(rev.new_manifest == roster_manifest_id);
+  /*
+    calculate_ident(*ros_writeable, roster_manifest_id);
+    I(rev.new_manifest == roster_manifest_id);
+  */
   // const'ify the objects, suitable for caching etc.
   roster_t_cp ros = ros_writeable;
   marking_map_cp mm = mm_writeable;
============================================================
--- graph.cc	2d934cb225b5639919fec3eda0cbec615cf2f903
+++ graph.cc	ba666c56c19c7ca8d85f9b2898f479fe92be1da6
@@ -388,21 +388,13 @@ get_uncommon_ancestors(revision_id const
   }  
 }
 
-#ifdef BUILD_UNIT_TESTS
-
-#include <map>
-#include "unit_tests.hh"
-#include "randomizer.hh"
-#include "roster.hh"
-
-
-static void
-get_all_ancestors(revision_id const & start, rev_ancestry_map const & child_to_parent_map,
+void
+get_all_ancestors(set<revision_id> const & start, 
+                  rev_ancestry_map const & child_to_parent_map,
                   set<revision_id> & ancestors)
 {
   ancestors.clear();
-  vector<revision_id> frontier;
-  frontier.push_back(start);
+  vector<revision_id> frontier(start.begin(), start.end());
   while (!frontier.empty())
     {
       revision_id rid = frontier.back();
@@ -417,6 +409,23 @@ get_all_ancestors(revision_id const & st
     }
 }
 
+#ifdef BUILD_UNIT_TESTS
+
+#include <map>
+#include "unit_tests.hh"
+#include "randomizer.hh"
+#include "roster.hh"
+
+
+static void
+get_all_ancestors(revision_id const & start, rev_ancestry_map const & child_to_parent_map,
+                  set<revision_id> & ancestors)
+{
+  set<revision_id> start_set;
+  start_set.insert(start);
+  get_all_ancestors(start, child_to_parent_map, ancestors);
+}
+
 struct mock_rev_graph : rev_graph
 {
   mock_rev_graph(rev_ancestry_map const & child_to_parent_map)
============================================================
--- graph.hh	6384f6bd01b8b43383ba2e6adf6ae10b6f725bac
+++ graph.hh	39241af3330a9cf3c42a9ba2cf7067017c781b6f
@@ -57,6 +57,11 @@ get_uncommon_ancestors(revision_id const
                        rev_graph const & hg,
                        std::set<revision_id> & a_uncommon_ancs,
                        std::set<revision_id> & b_uncommon_ancs);
+
+void
+get_all_ancestors(std::set<revision_id> const & start, 
+                  rev_ancestry_map const & child_to_parent_map,
+                  std::set<revision_id> & ancestors);
                        
 
 
============================================================
--- netxx/netbuf.h	1a1f6f16e7946f9a477a31f52ae3515f8ad0e333
+++ netxx/netbuf.h	3225d843e80c257bdd2ecfd718ff255c79a9cb0d
@@ -92,7 +92,7 @@ protected:
     ~Netbuf (void);
 protected:
     // TODO streamsize xsputn (const char_type *s, streamsize n);
-    int_type overflow (int_type c=traits_type::eof());
+    int_type overflow (int_type c=traits::eof());
     int sync (void);
 
     int_type underflow (void);
@@ -127,11 +127,11 @@ typename Netbuf<bufsize, charT, traits>:
 template<std::streamsize bufsize, class charT, class traits>
 typename Netbuf<bufsize, charT, traits>::int_type Netbuf<bufsize, charT, traits>::overflow (int_type c) {
     if (buffer_out() < 0) {
-	return traits_type::eof();
-    } else if (!traits_type::eq_int_type(c, traits_type::eof())) {
+	return traits::eof();
+    } else if (!traits::eq_int_type(c, traits::eof())) {
 	return sputc(c);
     } else {
-	return traits_type::not_eof(c);
+	return traits::not_eof(c);
     }
 }
 //#############################################################################
@@ -142,47 +142,47 @@ int Netbuf<bufsize, charT, traits>::buff
 //#############################################################################
 template<std::streamsize bufsize, class charT, class traits>
 int Netbuf<bufsize, charT, traits>::buffer_out (void) {
-    int length = pptr() - pbase();
+    int length = this->pptr() - this->pbase();
     int rc = stream_.write(putbuf_, length);
-    pbump(-length);
+    this->pbump(-length);
     return rc;
 }
 //#############################################################################
 template<std::streamsize bufsize, class charT, class traits>
 typename Netbuf<bufsize, charT, traits>::int_type Netbuf<bufsize, charT, traits>::underflow (void) {
-    if (gptr() < egptr()) return traits_type::to_int_type(*gptr());
-    if (buffer_in() < 0) return traits_type::eof();
-    else return traits_type::to_int_type(*gptr());
+    if (this->gptr() < this->egptr()) return traits::to_int_type(*(this->gptr()));
+    if (buffer_in() < 0) return traits::eof();
+    else return traits::to_int_type(*(this->gptr()));
 }
 //#############################################################################
 template<std::streamsize bufsize, class charT, class traits>
 typename Netbuf<bufsize, charT, traits>::int_type Netbuf<bufsize, charT, traits>::pbackfail(int_type c) {
-    if (gptr() != eback()) {
-	gbump(-1);
+    if (this->gptr() != this->eback()) {
+	this->gbump(-1);
 
-	if (!traits_type::eq_int_type(c, traits_type::eof())) {
-	    *(gptr()) = traits_type::to_char_type(c);
+	if (!traits::eq_int_type(c, traits::eof())) {
+	    *(this->gptr()) = traits::to_char_type(c);
 	}
 
-	return traits_type::not_eof(c);
+	return traits::not_eof(c);
     } else {
-	return traits_type::eof();
+	return traits::eof();
     }
 }
 //#############################################################################
 template<std::streamsize bufsize, class charT, class traits>
 int Netbuf<bufsize, charT, traits>::buffer_in (void) {
-    std::streamsize number_putbacks = std::min(gptr() - eback(), PUTBACK_SIZE);
+    std::streamsize number_putbacks = std::min(this->gptr() - this->eback(), PUTBACK_SIZE);
     std::memcpy(getbuf_ + (PUTBACK_SIZE - number_putbacks) * sizeof(char_type),
-	    gptr() - number_putbacks * sizeof(char_type), number_putbacks * sizeof(char_type));
+	    this->gptr() - number_putbacks * sizeof(char_type), number_putbacks * sizeof(char_type));
 
     int rc = stream_.read(getbuf_ + PUTBACK_SIZE * sizeof(char_type), bufsize - PUTBACK_SIZE);
     
     if (rc <= 0) {
-	setg(0, 0, 0);
+	this->setg(0, 0, 0);
 	return -1;
     } else {
-	setg(getbuf_ + PUTBACK_SIZE - number_putbacks, getbuf_ + PUTBACK_SIZE, getbuf_ + PUTBACK_SIZE + rc);
+	this->setg(getbuf_ + PUTBACK_SIZE - number_putbacks, getbuf_ + PUTBACK_SIZE, getbuf_ + PUTBACK_SIZE + rc);
 	return rc;
     }
 }
============================================================
--- vocab.hh	8c831a796771018eb1b2d28f56dc8e2f9b4d4e55
+++ vocab.hh	fae7ea2ea7fd30a46e256af081e1efc716c26c39
@@ -75,6 +75,11 @@ inline bool is_alnum(char x)
           || (x >= 'A' && x <= 'Z'));
 }
 
+inline bool is_num(char x)
+{
+  return (x >= '0' && x <= '9');
+}
+
 inline bool is_space(char x)
 {
   return (x == ' ')