Re: [Sks-devel] Re: incremental dumps

[Yaron Minsky]

A version of this patch is now in CVS.  Take a look at it and tell me what
you think.

y

Yaron M. Minsky said:
> This patch actually looks quite good.  There are some issues to fix, but
> not too many.  I will get this added to CVS in the next few days.
>
> y
>
> On Fri, 2004-03-12 at 07:59, Peter Palfrader wrote:
>> On Thu, 11 Mar 2004, Yaron M. Minsky wrote:
>>
>> >> does SKS keep track of when it last updated a key?  Because I would
>> >> really like to see an option to dump keys that were changed after a
>> >> certain date.  Do you think this is possible?
>> >
>> > It's certainly possible.  SKS keeps a log of every update that is
>> made.
>> > In particular, it keeps a log of every time a key is added or deleted
>> > (and updating a key counts as a deletion of the old version and an
>> > addition of the new.)  This would be fairly easy to do, although I
>> don't
>> > have time to do it right now.
>>
>> This implementation sucks, but it is a start and seems to work.  Please
>> comment.
>>
>>
>> diff -Nur sks/build-tree/sks-1.0.6/Makefile sks-dump/sks-1.0.6/Makefile
>> --- sks/build-tree/sks-1.0.6/Makefile        Mon Mar  8 21:35:17 2004
>> +++ sks-dump/sks-1.0.6/Makefile      Fri Mar 12 06:03:18 2004
>> @@ -85,7 +85,7 @@
>>          index.cmo mRindex.cmo pTreeDB.cmo \
>>      sendmail.cmo recvmail.cmo mailsync.cmo stats.cmo \
>>      clean_keydb.cmo build.cmo fastbuild.cmo pbuild.cmo merge_keyfiles.cmo
>> \
>> -    sksdump.cmo dbserver.cmo reconComm.cmo recoverList.cmo catchup.cmo \
>> +    sksdump.cmo sksincdump.cmo dbserver.cmo reconComm.cmo recoverList.cmo
>> catchup.cmo \
>>      reconserver.cmo update_subkeys.cmo sks_do.cmo
>>
>>  OBJS=$(OBJS.bc:.cmo=.cmx)
>> diff -Nur sks/build-tree/sks-1.0.6/sks.ml sks-dump/sks-1.0.6/sks.ml
>> --- sks/build-tree/sks-1.0.6/sks.ml  Sat Nov 29 15:25:51 2003
>> +++ sks-dump/sks-1.0.6/sks.ml        Fri Mar 12 05:44:37 2004
>> @@ -96,6 +96,15 @@
>>            M.run ()
>>         )
>>    };
>> +  { name = "incdump";
>> +    usage = "timestamp [filename]";
>> +    desc = "Create a raw dump of the keys in the database that got" ^
>> +           "updated after timestamp";
>> +    func = (fun () ->
>> +          let module M = Sksincdump.F(struct end) in
>> +          M.run ()
>> +       )
>> +  };
>>    { name = "merge";
>>      usage = "";
>>      desc = "Adds key from key files to existing database";
>> diff -Nur sks/build-tree/sks-1.0.6/sksincdump.ml
>> sks-dump/sks-1.0.6/sksincdump.ml
>> --- sks/build-tree/sks-1.0.6/sksincdump.ml   Thu Jan  1 01:00:00 1970
>> +++ sks-dump/sks-1.0.6/sksincdump.ml Fri Mar 12 13:54:17 2004
>> @@ -0,0 +1,117 @@
>> +(************************************************************************)
>> +(* This file is part of SKS.  SKS is free software; you can
>> +   redistribute it and/or modify it under the terms of the GNU General
>> +   Public License as published by the Free Software Foundation; either
>> +   version 2 of the License, or (at your option) any later version.
>> +
>> +   This program is distributed in the hope that it will be useful, but
>> +   WITHOUT ANY WARRANTY; without even the implied warranty of
>> +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
>> +   General Public License for more details.
>> +
>> +   You should have received a copy of the GNU General Public License
>> +   along with this program; if not, write to the Free Software
>> +   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
>> +   USA *)
>> +
>> +(* Copyright 2002, 2003, 2004 Yaron M. Minsky *)
>> +(* Copyright 2004 Peter Palfrader *)
>> +(***********************************************************************)
>> +
>> +(** takes content of SKS keyserver and creates key dump files from that
>> *)
>> +
>> +module F(M:sig end) =
>> +struct
>> +  open StdLabels
>> +  open MoreLabels
>> +  open Printf
>> +  open Common
>> +  open Packet
>> +
>> +  let settings = {
>> +    Keydb.withtxn = !Settings.transactions;
>> +    Keydb.cache_bytes = !Settings.cache_bytes;
>> +    Keydb.pagesize = !Settings.pagesize;
>> +    Keydb.dbdir = !Settings.dbdir;
>> +    Keydb.dumpdir = !Settings.dumpdir;
>> +  }
>> +
>> +  module Keydb = Keydb.Unsafe
>> +
>> +  let dump_database timestamp fname =
>> +    let maxsize = 250000 in
>> +    let log = Keydb.reverse_logquery ~maxsize timestamp in
>> +    if List.length log = 0 then
>> +      printf "No changes since timestamp\n"
>> +    else
>> +      let file = open_out fname in
>> +      let run () =
>> +    printf "Analyzing log."; flush stdout;
>> +    let adds = List.filter log ~f:(function (time,Add hash) -> true;
>> +                                          | (time,Delete hash) -> false )
>> in
>> +    let adds = List.map adds ~f:(fun (time, Add hash) -> hash) in
>> +    printf "."; flush stdout;
>> +    (*
>> +    let dels = List.filter log ~f:(function (time,Delete hash) -> true
>> +                                           | (time,Add hash) -> false ) in
>> +    let dels = List.map dels ~f:(fun (time, Delete hash) -> hash) in
>> +    printf "."; flush stdout;
>> +    (* O( n^2 ) alert.  How expensive is requesting a hash that isn't
>> there?
>> +       can we do better, should we?  or just drop that whole filter
>> +     *)
>> +        let dump = List.filter adds ~f:(fun (hashadd) ->
>> +                List.exists dels ~f:(fun (hashdel) -> compare hashadd 
>> hashdel ==
>> 0;) == false ) in
>> +    *)
>> +
>> +     (* printf "h1: %s;  h2: %s;  %b\n"  (KeyHash.hexify hashadd)
>> (KeyHash.hexify hashdel) ((compare hashadd hashdel) == 0);
>> +        List.iter log ~f:(function (time,Add hash) ->  printf "Add
>> %s\n" (KeyHash.hexify hash)
>> +                             | (time,Delete hash) ->  printf "Del %s\n"
>> (KeyHash.hexify hash) );
>> +        List.iter adds ~f:(function (hash) ->  printf "ADD Add %s\n"
>> (KeyHash.hexify hash) );
>> +        List.iter dels ~f:(function (hash) ->  printf "DEL Del %s\n"
>> (KeyHash.hexify hash) );
>> +        List.iter dump ~f:(function (hash) ->  printf "dump %s\n"
>> (KeyHash.hexify hash) );
>> +    printf "Adds %d keys.\n" (List.length adds);
>> +    printf "Dels %d keys.\n" (List.length dels); *)
>> +
>> +    (*
>> +    printf "Dumping %d keys.\n" (List.length dump);
>> +        List.iter dump ~f:(fun (hash) ->
>> +    *)
>> +    printf "%d hashes updated in log.  Some of them are probably gone
>> again.\n" (List.length adds);
>> +        List.iter adds ~f:(fun (hash) ->
>> +            try
>> +              flush stdout;
>> +              let keystring = Keydb.get_keystring_by_hash hash in
>> +              output_string file keystring;
>> +              (* printf "Dumped key %s\n" (KeyHash.hexify hash); *)
>> +            with
>> +              e -> printf ""
>> +                    (*printf "Failed attempt to extract key %s\n" 
>> (KeyHash.hexify hash);
>> *)
>> +                    (*signore (eplerror 2 e
>> +                         "Failed attempt to extract key %s" (KeyHash.hexify 
>> hash)) *)
>> +            )
>> +      in
>> +      protect ~f:run ~finally:(fun () -> close_out file)
>> +
>> +  exception Argument_error
>> +
>> +  let run () =
>> +    try (
>> +      match !Settings.anonlist with
>> +    | timestamp::tl ->
>> +        let name = match tl with
>> +          | [] -> "inc-sks-dump.pgp"
>> +          | [name] -> name
>> +          | _ -> raise Argument_error
>> +        in
>> +        set_logfile ".dump";
>> +        Keydb.open_dbs settings;
>> +        let timestamp = float_of_string timestamp in
>> +        dump_database timestamp name
>> +    | _ ->
>> +        raise Argument_error
>> +    ) with Argument_error ->
>> +      eprintf "wrong number of arguments\n";
>> +      eprintf "usage: sksdump timestamp(seconds since 1970)
>> [dumpname]\n";
>> +      flush stderr;
>> +      exit (-1)
>> +end
>>
>>
>>
>> Peter
> --
> |--------/            Yaron M. Minsky              \--------|
> |--------\ http://www.cs.cornell.edu/home/yminsky/ /--------|
>
> Open PGP --- KeyID B1FFD916
> Fingerprint: 5BF6 83E1 0CE3 1043 95D8 F8D5 9F12 B3A9 B1FF D916
>
>
>
>
> _______________________________________________
> Sks-devel mailing list
> address@hidden
> http://mail.nongnu.org/mailman/listinfo/sks-devel
>


|--------/            Yaron M. Minsky              \--------|
|--------\ http://www.cs.cornell.edu/home/yminsky/ /--------|

Open PGP --- KeyID B1FFD916
Fingerprint: 5BF6 83E1 0CE3 1043 95D8 F8D5 9F12 B3A9 B1FF D916