The unified diff between revisions [2449c78b..] and [ebdccab0..] is displayed below. It can also be downloaded as a raw diff.

#
#
# patch "automate.cc"
#  from [68d579fc833b4a0ec52a0c8e5b130af7517ab7ed]
#    to [d86c347f85ee7ce7a59f699c8e51feaa1fa57bfb]
#
# patch "commands.cc"
#  from [a10d3cb2745a4e0abf8281668b57204b333e03a6]
#    to [0fc12a37debd7db7812e2670afcb99d787f869e0]
#
# patch "database.cc"
#  from [1cd429afd5808a6b11ca51f037108d31f290873a]
#    to [fd5eff861239cd1408cca007c0df653f8a0f7d3f]
#
# patch "database.hh"
#  from [ad0aa7e49ff8c91790d1134a7b91664123a58085]
#    to [4ea836110ca59b61655724a3eb2dcd1c9a5a34cb]
#
# patch "enumerator.cc"
#  from [9cd3afa7ef779cb7b0c53a053a9567b57b59f660]
#    to [33a06c9ebfb955efb197f5eafb2e8d9beaee8a1d]
#
# patch "enumerator.hh"
#  from [6e6ecbf59d43a8e4bf4de07008a8795bd18cd960]
#    to [1880dae7a6147b0ddd8064bfc4679334b520aeab]
#
# patch "netsync.cc"
#  from [ff048e5f5c432920b0ddbffbe0fb7133b8eb9f46]
#    to [0edebb049f6f44132b6c97be4663efc113efeee8]
#
# patch "revision.cc"
#  from [6640272c7839f8489f1fe23813d7c520afc87d02]
#    to [ca74a6db1747f8c8cf213167ec226df0e30c0b84]
#
# patch "revision.hh"
#  from [1031cee8580d55528f5fb80e754fbf6e43dedb25]
#    to [ee41c7c61df986d551aebd2445a6232150a9c8b0]
#
============================================================
--- automate.cc	68d579fc833b4a0ec52a0c8e5b130af7517ab7ed
+++ automate.cc	d86c347f85ee7ce7a59f699c8e51feaa1fa57bfb
@@ -283,7 +283,7 @@ automate_toposort(std::vector<utf8> args
       revs.insert(rid);
     }
   std::vector<revision_id> sorted;
-  toposort(revs, sorted, app);
+  toposort(revs, sorted, app.db);
   for (std::vector<revision_id>::const_iterator i = sorted.begin();
        i != sorted.end(); ++i)
     output << (*i).inner()() << std::endl;
@@ -329,7 +329,7 @@ automate_ancestry_difference(std::vector
   ancestry_difference(a, bs, ancestors, app);

   std::vector<revision_id> sorted;
-  toposort(ancestors, sorted, app);
+  toposort(ancestors, sorted, app.db);
   for (std::vector<revision_id>::const_iterator i = sorted.begin();
        i != sorted.end(); ++i)
     output << (*i).inner()() << std::endl;
============================================================
--- commands.cc	a10d3cb2745a4e0abf8281668b57204b333e03a6
+++ commands.cc	0fc12a37debd7db7812e2670afcb99d787f869e0
@@ -2222,6 +2222,11 @@ CMD(db, N_("database"),
         build_changesets_from_manifest_ancestry(app);
       else if (idx(args, 0)() == "rosterify")
         build_roster_style_revs_from_manifest_style_revs(app);
+      else if (idx(args, 0)() == "flip")
+        {
+          app.db.make_fwd_deltas("files", "file_deltas");
+          app.db.make_fwd_deltas("rosters", "roster_deltas");
+        }
       else
         throw usage(name);
     }
============================================================
--- database.cc	1cd429afd5808a6b11ca51f037108d31f290873a
+++ database.cc	fd5eff861239cd1408cca007c0df653f8a0f7d3f
@@ -794,6 +794,17 @@ database::delta_exists(hexenc<id> const
   return res.size() > 0;
 }

+bool
+database::delta_exists(hexenc<id> const & ident,
+                       hexenc<id> const & base,
+                       string const & table)
+{
+  results res;
+  query q("SELECT id FROM " + table + " WHERE id = ? AND base = ?");
+  fetch(res, one_col, any_rows, q % text(ident()) % text(base()));
+  return res.size() > 0;
+}
+
 unsigned long
 database::count(string const & table)
 {
@@ -1170,23 +1181,10 @@ database::put_version(hexenc<id> const &
                       string const & data_table,
                       string const & delta_table)
 {
+  // TODO: add an invariant or something perhaps

-  data old_data, new_data;
-  delta reverse_delta;
-
-  get_version(old_id, old_data, data_table, delta_table);
-  patch(old_data, del, new_data);
-  diff(new_data, old_data, reverse_delta);
-
   transaction_guard guard(*this);
-  if (exists(old_id, data_table))
-    {
-      // descendent of a head version replaces the head, therefore old head
-      // must be disposed of
-      drop(old_id, data_table);
-    }
-  put(new_id, new_data, data_table);
-  put_delta(old_id, new_id, reverse_delta, delta_table);
+  put_delta(new_id, old_id, del, delta_table);
   guard.commit();
 }

@@ -1376,6 +1374,29 @@ database::get_file_version(file_id const
   dat = tmp;
 }

+void
+database::get_file_delta(file_id const & id,
+                         file_id const & base,
+                         file_delta & del)
+{
+  if (delta_exists(id.inner(), base.inner(), "file_deltas"))
+    {
+      delta d;
+      get_delta(id.inner(), base.inner(), d, "file_deltas");
+      del = file_delta(d);
+    }
+  else
+    {
+      L(FL("get_file_delta failed for '%s'->'%s'") % base % id);
+      file_data dat, base_dat;
+      get_file_version(base, base_dat);
+      get_file_version(id, dat);
+      delta d;
+      diff(base_dat.inner(), dat.inner(), d);
+      del = file_delta(d);
+    }
+}
+
 void
 database::get_manifest_version(manifest_id const & id,
                                manifest_data & dat)
@@ -1482,6 +1503,81 @@ database::get_revision(revision_id const
   dat = rdat;
 }

+void
+database::make_fwd_deltas(string const & data_table, string const & delta_table)
+{
+  transaction_guard guard(*this);
+
+  set< hexenc<id> > del_bases, all_ids;
+  set< pair< hexenc<id>, hexenc<id> > > dels;
+
+  // id, base
+  get_ids(data_table, all_ids);
+    {
+      results res;
+      fetch(res, 2, any_rows, query("SELECT id, base FROM " + delta_table));
+
+      for (size_t i = 0; i < res.size(); ++i)
+        {
+          all_ids.insert(hexenc<id>(res[i][0]));
+          del_bases.insert(hexenc<id>(res[i][1]));
+          dels.insert( make_pair(res[i][0], res[i][1]) );
+        }
+    }
+
+  set< hexenc<id> > new_bases;
+
+  set_difference(all_ids.begin(), all_ids.end(), del_bases.begin(), del_bases.end(),
+                 inserter(new_bases, new_bases.begin()));
+
+  // create some empty temporary tables
+  string tmp_data_table("tmp_" + data_table);
+  string tmp_delta_table("tmp_" + delta_table);
+
+  execute(query("CREATE TABLE " + tmp_data_table + " AS SELECT * FROM " + data_table + " WHERE 1=0"));
+  execute(query("CREATE TABLE " + tmp_delta_table + " AS SELECT * FROM " + delta_table + " WHERE 1=0"));
+
+  ticker full("full", "g", 1);
+  for ( set< hexenc<id> >::const_iterator i = new_bases.begin(); i != new_bases.end(); i++)
+    {
+      MM(*i);
+      data dat;
+      get_version(*i, dat, data_table, delta_table);
+      put(*i, dat, tmp_data_table);
+      ++full;
+    }
+
+  ticker flips("flips", "f", 1);
+
+  for ( set< pair< hexenc<id>, hexenc<id> > >::const_iterator i = dels.begin(); i != dels.end(); i++)
+    {
+      ++flips;
+      hexenc<id> new_id = i->second;
+      hexenc<id> new_base = i->first;
+      MM(new_id);
+      MM(new_base);
+
+      data base, dat;
+      get_version(new_id, dat, data_table, delta_table);
+      get_version(new_base, base, data_table, delta_table);
+      delta del;
+      diff(base, dat, del);
+
+      put_delta(new_id, new_base, del, tmp_delta_table);
+    }
+
+  execute(query("DELETE FROM " + data_table));
+  execute(query("DELETE FROM " + delta_table));
+
+  execute(query("INSERT INTO " + data_table + " SELECT * FROM " + tmp_data_table));
+  execute(query("INSERT INTO " + delta_table + " SELECT * FROM " + tmp_delta_table));
+
+  execute(query("DROP TABLE " + tmp_data_table));
+  execute(query("DROP TABLE " + tmp_delta_table));
+
+  guard.commit();
+}
+
 void
 database::deltify_revision(revision_id const & rid)
 {
@@ -1574,7 +1670,8 @@ database::put_revision(revision_id const
               % text(new_id.inner()()));
     }

-  deltify_revision(new_id);
+  // TODO: some different version?
+  // deltify_revision(new_id);

   // Phase 4: write the roster data and commit
   put_roster(new_id, ros, mm);
@@ -2639,7 +2736,6 @@ database::get_roster(revision_id const &
   read_roster_and_marking(dat, roster, marks);
 }

-
 void
 database::put_roster(revision_id const & rev_id,
                      roster_t & roster,
@@ -2647,7 +2743,6 @@ database::put_roster(revision_id const &
 {
   MM(rev_id);
   data old_data, new_data;
-  delta reverse_delta;
   hexenc<id> old_id, new_id;

   write_roster_and_marking(roster, marks, new_data);
@@ -2676,13 +2771,13 @@ database::put_roster(revision_id const &
   // Else we have a new roster the database hasn't seen yet; our task is to
   // add it, and deltify all the incoming edges (if they aren't already).

-  put(new_id, new_data, data_table);

   std::set<revision_id> parents;
   get_revision_parents(rev_id, parents);

   // Now do what deltify would do if we bothered (we have the
   // roster written now, so might as well do it here).
+  bool delta_written = false;
   for (std::set<revision_id>::const_iterator i = parents.begin();
        i != parents.end(); ++i)
     {
@@ -2690,14 +2785,18 @@ database::put_roster(revision_id const &
         continue;
       revision_id old_rev = *i;
       get_roster_id_for_revision(old_rev, old_id);
-      if (exists(new_id, data_table))
-        {
-          get_version(old_id, old_data, data_table, delta_table);
-          diff(new_data, old_data, reverse_delta);
-          drop(old_id, data_table);
-          put_delta(old_id, new_id, reverse_delta, delta_table);
-        }
+      get_version(old_id, old_data, data_table, delta_table);
+      delta del;
+      diff(old_data, new_data, del);
+      put_delta(new_id, old_id, del, delta_table);
+      delta_written = true;
+      break;
     }
+
+  // in case the parents were all null/didn't exist
+  if (!delta_written)
+    put(new_id, new_data, data_table);
+
   guard.commit();
 }

============================================================
--- database.hh	ad0aa7e49ff8c91790d1134a7b91664123a58085
+++ database.hh	4ea836110ca59b61655724a3eb2dcd1c9a5a34cb
@@ -110,6 +110,9 @@ class database
               std::string const & table);
   bool delta_exists(hexenc<id> const & ident,
                     std::string const & table);
+  bool database::delta_exists(hexenc<id> const & ident,
+                              hexenc<id> const & base,
+                              std::string const & table);

   unsigned long count(std::string const & table);
   unsigned long space_usage(std::string const & table,
@@ -244,6 +247,11 @@ public:
                         file_id const & new_id,
                         file_delta const & del);

+
+  void get_file_delta(file_id const & id,
+                      file_id const & base,
+                      file_delta & del);
+
   // get plain version if it exists, or reconstruct version
   // from deltas (if they exist).
   void get_manifest_version(manifest_id const & id,
@@ -260,6 +268,8 @@ public:
   void get_revision_manifest(revision_id const & cid,
                              manifest_id & mid);

+  void make_fwd_deltas(std::string const & data_table, std::string const & delta_table);
+
   void deltify_revision(revision_id const & rid);

   void get_revision(revision_id const & id,
============================================================
--- enumerator.cc	9cd3afa7ef779cb7b0c53a053a9567b57b59f660
+++ enumerator.cc	33a06c9ebfb955efb197f5eafb2e8d9beaee8a1d
@@ -27,10 +27,14 @@ revision_enumerator::revision_enumerator
                                          set<revision_id> const & terminal)
   : cb(cb), app(app), terminal_nodes(terminal)
 {
+  // TODO: needs sorting out with the toposort stuff.
+  I(false);
+  /*
   for (set<revision_id>::const_iterator i = initial.begin();
        i != initial.end(); ++i)
     revs.push_back(*i);
-  load_graphs();
+    */
+  load_revs();
 }

 revision_enumerator::revision_enumerator(enumerator_callbacks & cb,
@@ -38,226 +42,188 @@ revision_enumerator::revision_enumerator
   : cb(cb), app(app)
 {
   revision_id root;
-  revs.push_back(root);
-  load_graphs();
+  load_revs();
 }

 void
-revision_enumerator::load_graphs()
+revision_enumerator::load_revs()
 {
-  app.db.get_revision_ancestry(graph);
-  for (multimap<revision_id, revision_id>::const_iterator i = graph.begin();
-       i != graph.end(); ++i)
+  // TODO: this totally disrespects the initial and terminal stuff.
+  // It should probably be integrated into the toposort code itself.
+  vector<revision_id> topo_vec;
+  toposort(topo_vec, app.db);
+  for (vector<revision_id>::const_iterator r = topo_vec.begin();
+       r != topo_vec.end(); r++)
     {
-      inverse_graph.insert(make_pair(i->second, i->first));
+      if (null_id(*r))
+        continue;
+      topo_revs.push_back(*r);
     }
 }

-bool
-revision_enumerator::all_parents_enumerated(revision_id const & child)
-{
-  typedef multimap<revision_id, revision_id>::const_iterator ci;
-  pair<ci,ci> range = inverse_graph.equal_range(child);
-  for (ci i = range.first; i != range.second; ++i)
-    {
-      if (i->first == child)
-        {
-          if (enumerated_nodes.find(i->second) == enumerated_nodes.end())
-            return false;
-        }
-    }
-  return true;
-}
-
 bool
 revision_enumerator::done()
 {
-  return revs.empty() && items.empty();
+  return topo_revs.empty() && items.empty();
 }

 void
-revision_enumerator::files_for_revision(revision_id const & r,
-                                        set<file_id> & full_files,
-                                        set<pair<file_id,file_id> > & del_files)
+revision_enumerator::process_bunch()
 {
-  // when we're sending a merge, we have to be careful if we
-  // want to send as little data as possible. see bug #15846
-  //
-  // njs's solution: "when sending the files for a revision,
-  // look at both csets. If a given hash is not listed as new
-  // in _both_ csets, throw it out. Now, for everything left
-  // over, if one side says "add" and the other says "delta",
-  // do a delta. If both sides say "add", do a data."
+  // we build up a set of files and revs to send
+  vector<revision_id> bunch_revs;
+  set<file_id> bunch_files;
+  multimap<file_id, file_id> bunch_file_deltas;

-  set<file_id> file_adds;
-  // map<dst, src>.  src is arbitrary.
-  map<file_id, file_id> file_deltas;
-  map<file_id, size_t> file_edge_counts;
+  // files that should be sent first
+  set<file_id> top_files;
+  // files that will be sent either as full files are as the dst of deltas
+  set<file_id> dst_files;

-  revision_set rs;
-  MM(rs);
-  app.db.get_revision(r, rs);
+  while (bunch_revs.size() < 100 && !topo_revs.empty())
+    {
+      revision_id r = topo_revs.front();
+      topo_revs.pop_front();
+      if (!cb.process_this_rev(r))
+        continue;

-  for (edge_map::const_iterator i = rs.edges.begin();
-       i != rs.edges.end(); ++i)
-    {
-      set<file_id> file_dsts;
-      cset const & cs = edge_changes(i);
-
-      // Queue up all the file-adds
-      for (map<split_path, file_id>::const_iterator fa = cs.files_added.begin();
-           fa != cs.files_added.end(); ++fa)
+      bunch_revs.push_back(r);
+
+      revision_set rs;
+      app.db.get_revision(r, rs);
+      for (edge_map::const_iterator i = rs.edges.begin();
+           i != rs.edges.end(); ++i)
         {
-          file_adds.insert(fa->second);
-          file_dsts.insert(fa->second);
-        }
-
-      // Queue up all the file-deltas
-      for (map<split_path, std::pair<file_id, file_id> >::const_iterator fd
-             = cs.deltas_applied.begin();
-           fd != cs.deltas_applied.end(); ++fd)
-        {
-          file_deltas[fd->second.second] = fd->second.first;
-          file_dsts.insert(fd->second.second);
-        }
+          cset const & cs = edge_changes(i);
+
+          // Queue up all the file-adds
+          for (map<split_path, file_id>::const_iterator fa = cs.files_added.begin();
+               fa != cs.files_added.end(); ++fa)
+            {
+              if (cb.queue_this_file(fa->second.inner()))
+                {
+                  dst_files.insert(fa->second);
+                  top_files.insert(fa->second);
+                  bunch_files.insert(fa->second);
+                }
+            }
+
+          // Queue up all the file-deltas
+          for (map<split_path, std::pair<file_id, file_id> >::const_iterator fd
+                 = cs.deltas_applied.begin();
+               fd != cs.deltas_applied.end(); ++fd)
+            {
+              if (cb.queue_this_file(fd->second.second.inner()))
+                {
+                  if (dst_files.find(fd->second.first) == dst_files.end())
+                    {
+                      top_files.insert(fd->second.first);
+                    }

-      // we don't want to be counting files twice in a single edge
-      for (set<file_id>::const_iterator i = file_dsts.begin();
-           i != file_dsts.end(); i++)
-        file_edge_counts[*i]++;
+                  bunch_file_deltas.insert(make_pair(fd->second.first,
+                                                     fd->second.second));
+                  dst_files.insert(fd->second.second);
+                }
+            }
+        }
     }

-  del_files.clear();
-  full_files.clear();
-  size_t num_edges = rs.edges.size();
+  // XXX required?
+  set<file_id> sent_files;

-  for (map<file_id, size_t>::const_iterator i = file_edge_counts.begin();
-       i != file_edge_counts.end(); i++)
+  // now we can queue up the file items in order.
+  for (set<file_id>::const_iterator t = top_files.begin();
+       t != top_files.end(); t++)
     {
-      MM(i->first);
-      if (i->second < num_edges)
-        continue;
-
-      // first preference is to send as a delta...
-      map<file_id, file_id>::const_iterator fd = file_deltas.find(i->first);
-      if (fd != file_deltas.end())
+      if (sent_files.find(*t) != sent_files.end())
         {
-          del_files.insert(make_pair(fd->second, fd->first));
+          L(FL("already sent top_file %s") % t->inner()());
           continue;
         }

-      // ... otherwise as a full file.
-      set<file_id>::const_iterator f = file_adds.find(i->first);
-      if (f != file_adds.end())
+      L(FL("top_file %s") % t->inner()());
+
+      if (bunch_files.find(*t) != bunch_files.end())
         {
-          full_files.insert(*f);
-          continue;
+          // a full file to send.
+          enumerator_item item;
+          item.tag = enumerator_item::fdata;
+          item.ident_a = t->inner();
+          items.push_back(item);
+          sent_files.insert(*t);
+          L(FL("send full_file %s") % t->inner()());
         }

-      I(false);
-    }
-}
+      // XXX: doing BFS now, should try DFS too.
+      deque<file_id> frontier;
+      frontier.push_back(*t);

-void
-revision_enumerator::step()
-{
-  while (!done())
-    {
-      if (items.empty() && !revs.empty())
+      while (!frontier.empty())
         {
-          revision_id r = revs.front();
-          revs.pop_front();
+          file_id f = frontier.front();
+          frontier.pop_front();

-          // It's possible we've enumerated this node elsewhere since last
-          // time around. Cull rather than reprocess.
-          if (enumerated_nodes.find(r) != enumerated_nodes.end())
-            continue;
-
-          if (!all_parents_enumerated(r))
+          L(FL("frontier %s") % f.inner()());
+
+          for (multimap<file_id,file_id>::const_iterator
+               d = bunch_file_deltas.lower_bound(f);
+               d != bunch_file_deltas.upper_bound(f);
+               d++)
             {
-              revs.push_back(r);
-              continue;
-            }
-
-          if (terminal_nodes.find(r) == terminal_nodes.end())
-            {
-              typedef multimap<revision_id, revision_id>::const_iterator ci;
-              pair<ci,ci> range = graph.equal_range(r);
-              for (ci i = range.first; i != range.second; ++i)
+              if (sent_files.find(d->second) != sent_files.end())
                 {
-                  if (i->first == r)
-                    if (enumerated_nodes.find(i->first) == enumerated_nodes.end())
-                      revs.push_back(i->second);
+                  L(FL("already sent delta %s") % d->second.inner()());
+                  continue;
                 }
-            }
+              sent_files.insert(d->second);
+              frontier.push_back(d->second);

-          enumerated_nodes.insert(r);
+              enumerator_item item;
+              item.tag = enumerator_item::fdelta;
+              item.ident_a = d->first.inner();
+              item.ident_b = d->second.inner();
+              items.push_back(item);
+              L(FL("file_delta %s->%s") % d->first.inner()() % d->second.inner()());
+            }
+        }
+    }

-          if (null_id(r))
-            continue;
+  // and the revs
+  for (vector<revision_id>::const_iterator r = bunch_revs.begin();
+       r != bunch_revs.end(); r++)
+    {
+      enumerator_item item;
+      item.tag = enumerator_item::rev;
+      item.ident_a = r->inner();
+      items.push_back(item);

-          if (cb.process_this_rev(r))
+      // Queue up some or all of the rev's certs
+      vector<hexenc<id> > hashes;
+      app.db.get_revision_certs(*r, hashes);
+      for (vector<hexenc<id> >::const_iterator i = hashes.begin();
+           i != hashes.end(); ++i)
+        {
+          if (cb.queue_this_cert(*i))
             {
-              L(FL("revision_enumerator::step expanding "
-                  "contents of rev '%d'\n") % r);
-
-              // The rev's files and fdeltas
-              {
-                set<file_id> full_files;
-                set<pair<file_id, file_id> > del_files;
-                files_for_revision(r, full_files, del_files);
-
-                for (set<file_id>::const_iterator f = full_files.begin();
-                     f != full_files.end(); f++)
-                  {
-                    if (cb.queue_this_file(f->inner()))
-                      {
-                        enumerator_item item;
-                        item.tag = enumerator_item::fdata;
-                        item.ident_a = f->inner();
-                        items.push_back(item);
-                      }
-                  }
-
-                for (set<pair<file_id, file_id> >::const_iterator fd = del_files.begin();
-                     fd != del_files.end(); fd++)
-                  {
-                    if (cb.queue_this_file(fd->second.inner()))
-                      {
-                        enumerator_item item;
-                        item.tag = enumerator_item::fdelta;
-                        item.ident_a = fd->first.inner();
-                        item.ident_b = fd->second.inner();
-                        items.push_back(item);
-                      }
-                  }
-              }
-
-              // Queue up the rev itself
-              {
-                enumerator_item item;
-                item.tag = enumerator_item::rev;
-                item.ident_a = r.inner();
-                items.push_back(item);
-              }
+              enumerator_item item;
+              item.tag = enumerator_item::cert;
+              item.ident_a = *i;
+              items.push_back(item);
             }
-
-          // Queue up some or all of the rev's certs
-          vector<hexenc<id> > hashes;
-          app.db.get_revision_certs(r, hashes);
-          for (vector<hexenc<id> >::const_iterator i = hashes.begin();
-               i != hashes.end(); ++i)
-            {
-              if (cb.queue_this_cert(*i))
-                {
-                  enumerator_item item;
-                  item.tag = enumerator_item::cert;
-                  item.ident_a = *i;
-                  items.push_back(item);
-                }
-            }
         }
+    }
+}

+void
+revision_enumerator::step()
+{
+  while (!done())
+    {
+      if (items.empty() && !topo_revs.empty())
+        {
+          process_bunch();
+        }
+
       if (!items.empty())
         {
           L(FL("revision_enumerator::step extracting item\n"));
============================================================
--- enumerator.hh	6e6ecbf59d43a8e4bf4de07008a8795bd18cd960
+++ enumerator.hh	1880dae7a6147b0ddd8064bfc4679334b520aeab
@@ -50,10 +50,8 @@ revision_enumerator
   app_state & app;
   std::set<revision_id> terminal_nodes;
   std::set<revision_id> enumerated_nodes;
-  std::deque<revision_id> revs;
   std::deque<enumerator_item> items;
-  std::multimap<revision_id, revision_id> graph;
-  std::multimap<revision_id, revision_id> inverse_graph;
+  std::deque<revision_id> topo_revs;

   revision_enumerator(enumerator_callbacks & cb,
                       app_state & app,
@@ -61,11 +59,8 @@ revision_enumerator
                       std::set<revision_id> const & terminal);
   revision_enumerator(enumerator_callbacks & cb,
                       app_state & app);
-  void load_graphs();
-  bool all_parents_enumerated(revision_id const & child);
-  void files_for_revision(revision_id const & r,
-                          std::set<file_id> & full_files,
-                          std::set<std::pair<file_id,file_id> > & del_files);
+  void load_revs();
+  void process_bunch();
   void step();
   bool done();
 };
============================================================
--- netsync.cc	ff048e5f5c432920b0ddbffbe0fb7133b8eb9f46
+++ netsync.cc	0edebb049f6f44132b6c97be4663efc113efeee8
@@ -591,15 +591,12 @@ session::note_file_delta(file_id const &
 {
   if (role == sink_role)
     return;
-  file_data fd1, fd2;
-  delta del;
-  id fid1, fid2;
-  decode_hexenc(src.inner(), fid1);
-  decode_hexenc(dst.inner(), fid2);
-  app.db.get_file_version(src, fd1);
-  app.db.get_file_version(dst, fd2);
-  diff(fd1.inner(), fd2.inner(), del);
-  queue_delta_cmd(file_item, fid1, fid2, del);
+  file_delta del;
+  app.db.get_file_delta(dst, src, del);
+  id src_id, dst_id;
+  decode_hexenc(src.inner(), src_id);
+  decode_hexenc(dst.inner(), dst_id);
+  queue_delta_cmd(file_item, src_id, dst_id, del.inner());
   file_items_sent.insert(dst);
 }

============================================================
--- revision.cc	6640272c7839f8489f1fe23813d7c520afc87d02
+++ revision.cc	ca74a6db1747f8c8cf213167ec226df0e30c0b84
@@ -361,17 +361,15 @@ void
 // passed in set.  if anyone ever needs to toposort the whole graph, then,
 // this function would be a good thing to generalize...
 void
-toposort(std::set<revision_id> const & revisions,
-         std::vector<revision_id> & sorted,
-         app_state & app)
+toposort(std::vector<revision_id> & sorted, database & db)
 {
   sorted.clear();
   typedef std::multimap<revision_id, revision_id>::iterator gi;
   typedef std::map<revision_id, int>::iterator pi;
   std::multimap<revision_id, revision_id> graph;
-  app.db.get_revision_ancestry(graph);
+  db.get_revision_ancestry(graph);
   std::set<revision_id> leaves;
-  app.db.get_revision_ids(leaves);
+  db.get_revision_ids(leaves);
   std::map<revision_id, int> pcount;
   for (gi i = graph.begin(); i != graph.end(); ++i)
     pcount.insert(std::make_pair(i->first, 0));
@@ -387,8 +385,7 @@ toposort(std::set<revision_id> const & r
       // now stick them in our ordering (if wanted) and remove them from the
       // graph, calculating the new roots as we go
       L(FL("new root: %s\n") % (roots.front()));
-      if (revisions.find(roots.front()) != revisions.end())
-        sorted.push_back(roots.front());
+      sorted.push_back(roots.front());
       for(gi i = graph.lower_bound(roots.front());
           i != graph.upper_bound(roots.front()); i++)
         if(--(pcount[i->second]) == 0)
@@ -402,6 +399,21 @@ toposort(std::set<revision_id> const & r
        i != leaves.end(); ++i)
     {
       L(FL("new leaf: %s\n") % (*i));
+      sorted.push_back(*i);
+    }
+}
+
+void
+toposort(std::set<revision_id> const & revisions,
+         std::vector<revision_id> & sorted,
+         database & db)
+{
+  std::vector<revision_id> all;
+  toposort(all, db);
+  sorted.clear();
+  for (std::vector<revision_id>::const_iterator i = all.begin();
+       i != all.end(); i++)
+    {
       if (revisions.find(*i) != revisions.end())
         sorted.push_back(*i);
     }
============================================================
--- revision.hh	1031cee8580d55528f5fb80e754fbf6e43dedb25
+++ revision.hh	ee41c7c61df986d551aebd2445a6232150a9c8b0
@@ -125,9 +125,13 @@ toposort(std::set<revision_id> const & r
 void
 toposort(std::set<revision_id> const & revisions,
          std::vector<revision_id> & sorted,
-         app_state & app);
+         database & db);

 void
+toposort(std::vector<revision_id> & sorted,
+         database & db);
+
+void
 erase_ancestors(std::set<revision_id> & revisions, app_state & app);

 void