[Sks-devel] Re: incremental dumps

[Peter Palfrader]

On Thu, 11 Mar 2004, Yaron M. Minsky wrote:

>> does SKS keep track of when it last updated a key?  Because I would
>> really like to see an option to dump keys that were changed after a
>> certain date.  Do you think this is possible?
>
> It's certainly possible.  SKS keeps a log of every update that is made.
> In particular, it keeps a log of every time a key is added or deleted
> (and updating a key counts as a deletion of the old version and an
> addition of the new.)  This would be fairly easy to do, although I don't
> have time to do it right now.

This implementation sucks, but it is a start and seems to work.  Please
comment.


diff -Nur sks/build-tree/sks-1.0.6/Makefile sks-dump/sks-1.0.6/Makefile
--- sks/build-tree/sks-1.0.6/Makefile   Mon Mar  8 21:35:17 2004
+++ sks-dump/sks-1.0.6/Makefile Fri Mar 12 06:03:18 2004
@@ -85,7 +85,7 @@
         index.cmo mRindex.cmo pTreeDB.cmo \
        sendmail.cmo recvmail.cmo mailsync.cmo stats.cmo \
        clean_keydb.cmo build.cmo fastbuild.cmo pbuild.cmo merge_keyfiles.cmo \
-       sksdump.cmo dbserver.cmo reconComm.cmo recoverList.cmo catchup.cmo \
+       sksdump.cmo sksincdump.cmo dbserver.cmo reconComm.cmo recoverList.cmo 
catchup.cmo \
        reconserver.cmo update_subkeys.cmo sks_do.cmo
 
 OBJS=$(OBJS.bc:.cmo=.cmx)
diff -Nur sks/build-tree/sks-1.0.6/sks.ml sks-dump/sks-1.0.6/sks.ml
--- sks/build-tree/sks-1.0.6/sks.ml     Sat Nov 29 15:25:51 2003
+++ sks-dump/sks-1.0.6/sks.ml   Fri Mar 12 05:44:37 2004
@@ -96,6 +96,15 @@
              M.run ()
           )
   };
+  { name = "incdump";
+    usage = "timestamp [filename]";
+    desc = "Create a raw dump of the keys in the database that got" ^
+           "updated after timestamp";
+    func = (fun () -> 
+             let module M = Sksincdump.F(struct end) in
+             M.run ()
+          )
+  };
   { name = "merge";
     usage = "";
     desc = "Adds key from key files to existing database";
diff -Nur sks/build-tree/sks-1.0.6/sksincdump.ml 
sks-dump/sks-1.0.6/sksincdump.ml
--- sks/build-tree/sks-1.0.6/sksincdump.ml      Thu Jan  1 01:00:00 1970
+++ sks-dump/sks-1.0.6/sksincdump.ml    Fri Mar 12 13:54:17 2004
@@ -0,0 +1,117 @@
+(************************************************************************)
+(* This file is part of SKS.  SKS is free software; you can
+   redistribute it and/or modify it under the terms of the GNU General
+   Public License as published by the Free Software Foundation; either
+   version 2 of the License, or (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+   USA *)
+
+(* Copyright 2002, 2003, 2004 Yaron M. Minsky *)
+(* Copyright 2004 Peter Palfrader *)
+(***********************************************************************)
+
+(** takes content of SKS keyserver and creates key dump files from that *)
+
+module F(M:sig end) = 
+struct
+  open StdLabels
+  open MoreLabels
+  open Printf
+  open Common
+  open Packet
+
+  let settings = {
+    Keydb.withtxn = !Settings.transactions;
+    Keydb.cache_bytes = !Settings.cache_bytes;
+    Keydb.pagesize = !Settings.pagesize;
+    Keydb.dbdir = !Settings.dbdir;
+    Keydb.dumpdir = !Settings.dumpdir;
+  }
+
+  module Keydb = Keydb.Unsafe
+
+  let dump_database timestamp fname =
+    let maxsize = 250000 in
+    let log = Keydb.reverse_logquery ~maxsize timestamp in
+    if List.length log = 0 then
+      printf "No changes since timestamp\n"
+    else
+      let file = open_out fname in
+      let run () =
+       printf "Analyzing log."; flush stdout;
+       let adds = List.filter log ~f:(function (time,Add hash) -> true;
+                                             | (time,Delete hash) -> false ) in
+       let adds = List.map adds ~f:(fun (time, Add hash) -> hash) in
+       printf "."; flush stdout;
+       (*
+       let dels = List.filter log ~f:(function (time,Delete hash) -> true
+                                              | (time,Add hash) -> false ) in
+       let dels = List.map dels ~f:(fun (time, Delete hash) -> hash) in
+       printf "."; flush stdout;
+       (* O( n^2 ) alert.  How expensive is requesting a hash that isn't there?
+          can we do better, should we?  or just drop that whole filter
+        *)
+        let dump = List.filter adds ~f:(fun (hashadd) ->
+                   List.exists dels ~f:(fun (hashdel) -> compare hashadd 
hashdel == 0;) == false ) in
+       *)
+
+     (* printf "h1: %s;  h2: %s;  %b\n"  (KeyHash.hexify hashadd) 
(KeyHash.hexify hashdel) ((compare hashadd hashdel) == 0);
+        List.iter log ~f:(function (time,Add hash) ->  printf "Add %s\n" 
(KeyHash.hexify hash)
+                                | (time,Delete hash) ->  printf "Del %s\n" 
(KeyHash.hexify hash) );
+        List.iter adds ~f:(function (hash) ->  printf "ADD Add %s\n" 
(KeyHash.hexify hash) );
+        List.iter dels ~f:(function (hash) ->  printf "DEL Del %s\n" 
(KeyHash.hexify hash) );
+        List.iter dump ~f:(function (hash) ->  printf "dump %s\n" 
(KeyHash.hexify hash) );
+       printf "Adds %d keys.\n" (List.length adds);
+       printf "Dels %d keys.\n" (List.length dels); *)
+
+       (*
+       printf "Dumping %d keys.\n" (List.length dump);
+        List.iter dump ~f:(fun (hash) ->
+       *)
+       printf "%d hashes updated in log.  Some of them are probably gone 
again.\n" (List.length adds);
+        List.iter adds ~f:(fun (hash) ->
+               try
+                 flush stdout;
+                 let keystring = Keydb.get_keystring_by_hash hash in
+                 output_string file keystring;
+                 (* printf "Dumped key %s\n" (KeyHash.hexify hash); *)
+               with
+                 e -> printf ""
+                       (*printf "Failed attempt to extract key %s\n" 
(KeyHash.hexify hash); *)
+                       (*signore (eplerror 2 e
+                            "Failed attempt to extract key %s" (KeyHash.hexify 
hash)) *)
+               )
+      in
+      protect ~f:run ~finally:(fun () -> close_out file)
+
+  exception Argument_error
+
+  let run () = 
+    try (
+      match !Settings.anonlist with
+       | timestamp::tl ->
+           let name = match tl with
+             | [] -> "inc-sks-dump.pgp"
+             | [name] -> name
+             | _ -> raise Argument_error
+           in
+           set_logfile ".dump";
+           Keydb.open_dbs settings;
+           let timestamp = float_of_string timestamp in
+           dump_database timestamp name
+       | _ -> 
+           raise Argument_error
+    ) with Argument_error -> 
+      eprintf "wrong number of arguments\n";
+      eprintf "usage: sksdump timestamp(seconds since 1970) [dumpname]\n";
+      flush stderr;
+      exit (-1)
+end



Peter
-- 
 PGP signed and encrypted  |  .''`.  ** Debian GNU/Linux **
    messages preferred.    | : :' :      The  universal
                           | `. `'      Operating System
 http://www.palfrader.org/ |   `-    http://www.debian.org/

signature.asc
Description: Digital signature