[Sks-devel] Re: incremental dumps

[Yaron M. Minsky]

This patch actually looks quite good.  There are some issues to fix, but
not too many.  I will get this added to CVS in the next few days.

y

On Fri, 2004-03-12 at 07:59, Peter Palfrader wrote:
> On Thu, 11 Mar 2004, Yaron M. Minsky wrote:
> 
> >> does SKS keep track of when it last updated a key?  Because I would
> >> really like to see an option to dump keys that were changed after a
> >> certain date.  Do you think this is possible?
> >
> > It's certainly possible.  SKS keeps a log of every update that is made.
> > In particular, it keeps a log of every time a key is added or deleted
> > (and updating a key counts as a deletion of the old version and an
> > addition of the new.)  This would be fairly easy to do, although I don't
> > have time to do it right now.
> 
> This implementation sucks, but it is a start and seems to work.  Please
> comment.
> 
> 
> diff -Nur sks/build-tree/sks-1.0.6/Makefile sks-dump/sks-1.0.6/Makefile
> --- sks/build-tree/sks-1.0.6/Makefile Mon Mar  8 21:35:17 2004
> +++ sks-dump/sks-1.0.6/Makefile       Fri Mar 12 06:03:18 2004
> @@ -85,7 +85,7 @@
>          index.cmo mRindex.cmo pTreeDB.cmo \
>       sendmail.cmo recvmail.cmo mailsync.cmo stats.cmo \
>       clean_keydb.cmo build.cmo fastbuild.cmo pbuild.cmo merge_keyfiles.cmo \
> -     sksdump.cmo dbserver.cmo reconComm.cmo recoverList.cmo catchup.cmo \
> +     sksdump.cmo sksincdump.cmo dbserver.cmo reconComm.cmo recoverList.cmo 
> catchup.cmo \
>       reconserver.cmo update_subkeys.cmo sks_do.cmo
>  
>  OBJS=$(OBJS.bc:.cmo=.cmx)
> diff -Nur sks/build-tree/sks-1.0.6/sks.ml sks-dump/sks-1.0.6/sks.ml
> --- sks/build-tree/sks-1.0.6/sks.ml   Sat Nov 29 15:25:51 2003
> +++ sks-dump/sks-1.0.6/sks.ml Fri Mar 12 05:44:37 2004
> @@ -96,6 +96,15 @@
>             M.run ()
>          )
>    };
> +  { name = "incdump";
> +    usage = "timestamp [filename]";
> +    desc = "Create a raw dump of the keys in the database that got" ^
> +           "updated after timestamp";
> +    func = (fun () -> 
> +           let module M = Sksincdump.F(struct end) in
> +           M.run ()
> +        )
> +  };
>    { name = "merge";
>      usage = "";
>      desc = "Adds key from key files to existing database";
> diff -Nur sks/build-tree/sks-1.0.6/sksincdump.ml 
> sks-dump/sks-1.0.6/sksincdump.ml
> --- sks/build-tree/sks-1.0.6/sksincdump.ml    Thu Jan  1 01:00:00 1970
> +++ sks-dump/sks-1.0.6/sksincdump.ml  Fri Mar 12 13:54:17 2004
> @@ -0,0 +1,117 @@
> +(************************************************************************)
> +(* This file is part of SKS.  SKS is free software; you can
> +   redistribute it and/or modify it under the terms of the GNU General
> +   Public License as published by the Free Software Foundation; either
> +   version 2 of the License, or (at your option) any later version.
> +
> +   This program is distributed in the hope that it will be useful, but
> +   WITHOUT ANY WARRANTY; without even the implied warranty of
> +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +   General Public License for more details.
> +
> +   You should have received a copy of the GNU General Public License
> +   along with this program; if not, write to the Free Software
> +   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
> +   USA *)
> +
> +(* Copyright 2002, 2003, 2004 Yaron M. Minsky *)
> +(* Copyright 2004 Peter Palfrader *)
> +(***********************************************************************)
> +
> +(** takes content of SKS keyserver and creates key dump files from that *)
> +
> +module F(M:sig end) = 
> +struct
> +  open StdLabels
> +  open MoreLabels
> +  open Printf
> +  open Common
> +  open Packet
> +
> +  let settings = {
> +    Keydb.withtxn = !Settings.transactions;
> +    Keydb.cache_bytes = !Settings.cache_bytes;
> +    Keydb.pagesize = !Settings.pagesize;
> +    Keydb.dbdir = !Settings.dbdir;
> +    Keydb.dumpdir = !Settings.dumpdir;
> +  }
> +
> +  module Keydb = Keydb.Unsafe
> +
> +  let dump_database timestamp fname =
> +    let maxsize = 250000 in
> +    let log = Keydb.reverse_logquery ~maxsize timestamp in
> +    if List.length log = 0 then
> +      printf "No changes since timestamp\n"
> +    else
> +      let file = open_out fname in
> +      let run () =
> +     printf "Analyzing log."; flush stdout;
> +     let adds = List.filter log ~f:(function (time,Add hash) -> true;
> +                                           | (time,Delete hash) -> false ) in
> +     let adds = List.map adds ~f:(fun (time, Add hash) -> hash) in
> +     printf "."; flush stdout;
> +     (*
> +     let dels = List.filter log ~f:(function (time,Delete hash) -> true
> +                                            | (time,Add hash) -> false ) in
> +     let dels = List.map dels ~f:(fun (time, Delete hash) -> hash) in
> +     printf "."; flush stdout;
> +     (* O( n^2 ) alert.  How expensive is requesting a hash that isn't there?
> +        can we do better, should we?  or just drop that whole filter
> +      *)
> +        let dump = List.filter adds ~f:(fun (hashadd) ->
> +                 List.exists dels ~f:(fun (hashdel) -> compare hashadd 
> hashdel == 0;) == false ) in
> +     *)
> +
> +     (* printf "h1: %s;  h2: %s;  %b\n"  (KeyHash.hexify hashadd) 
> (KeyHash.hexify hashdel) ((compare hashadd hashdel) == 0);
> +        List.iter log ~f:(function (time,Add hash) ->  printf "Add %s\n" 
> (KeyHash.hexify hash)
> +                              | (time,Delete hash) ->  printf "Del %s\n" 
> (KeyHash.hexify hash) );
> +        List.iter adds ~f:(function (hash) ->  printf "ADD Add %s\n" 
> (KeyHash.hexify hash) );
> +        List.iter dels ~f:(function (hash) ->  printf "DEL Del %s\n" 
> (KeyHash.hexify hash) );
> +        List.iter dump ~f:(function (hash) ->  printf "dump %s\n" 
> (KeyHash.hexify hash) );
> +     printf "Adds %d keys.\n" (List.length adds);
> +     printf "Dels %d keys.\n" (List.length dels); *)
> +
> +     (*
> +     printf "Dumping %d keys.\n" (List.length dump);
> +        List.iter dump ~f:(fun (hash) ->
> +     *)
> +     printf "%d hashes updated in log.  Some of them are probably gone 
> again.\n" (List.length adds);
> +        List.iter adds ~f:(fun (hash) ->
> +             try
> +               flush stdout;
> +               let keystring = Keydb.get_keystring_by_hash hash in
> +               output_string file keystring;
> +               (* printf "Dumped key %s\n" (KeyHash.hexify hash); *)
> +             with
> +               e -> printf ""
> +                     (*printf "Failed attempt to extract key %s\n" 
> (KeyHash.hexify hash); *)
> +                     (*signore (eplerror 2 e
> +                          "Failed attempt to extract key %s" (KeyHash.hexify 
> hash)) *)
> +             )
> +      in
> +      protect ~f:run ~finally:(fun () -> close_out file)
> +
> +  exception Argument_error
> +
> +  let run () = 
> +    try (
> +      match !Settings.anonlist with
> +     | timestamp::tl ->
> +         let name = match tl with
> +           | [] -> "inc-sks-dump.pgp"
> +           | [name] -> name
> +           | _ -> raise Argument_error
> +         in
> +         set_logfile ".dump";
> +         Keydb.open_dbs settings;
> +         let timestamp = float_of_string timestamp in
> +         dump_database timestamp name
> +     | _ -> 
> +         raise Argument_error
> +    ) with Argument_error -> 
> +      eprintf "wrong number of arguments\n";
> +      eprintf "usage: sksdump timestamp(seconds since 1970) [dumpname]\n";
> +      flush stderr;
> +      exit (-1)
> +end
> 
> 
> 
> Peter
-- 
|--------/            Yaron M. Minsky              \--------|
|--------\ http://www.cs.cornell.edu/home/yminsky/ /--------|

Open PGP --- KeyID B1FFD916
Fingerprint: 5BF6 83E1 0CE3 1043 95D8 F8D5 9F12 B3A9 B1FF D916