Re: [Sks-devel] Re: [PATCH] Proper case handling for words index

[Kim Minh Kaplan]

This should correct the word index.

    $ ./fix_word_index 
    2009-08-08 13:35:41 Fixing word index
    2009-08-08 13:38:04 Fixed 17247 words

I even believe you can run it without stopping the SKS db server but I
am not certain, feedback on this part would be very appreciated as I am
still investigating SKS's safety with regard to transactions.

Kim Minh.

add fix_word_index to repare the word database without rebuilding the whole 
database.

diff -r 912355056ea4 Makefile
--- a/Makefile  Sat Aug 08 11:51:09 2009 +0200
+++ b/Makefile  Sat Aug 08 13:25:01 2009 +0200
@@ -153,6 +153,9 @@
 sks.8: sks.pod
        pod2man -c "SKS OpenPGP Key server" --section 8 -r 0.1 -name sks 
sks.pod sks.8
 
+fix_word_index: $(LIBS) $(ALLOBJS) fix_word_index.cmx
+       $(OCAMLOPT) -o fix_word_index $(OCAMLOPTFLAGS) $(ALLOBJS) 
fix_word_index.cmx
+
 spider: $(LIBS) $(ALLOBJS) spider.cmx
        $(OCAMLOPT) -o spider $(OCAMLOPTFLAGS) $(ALLOBJS) spider.cmx
 
diff -r 912355056ea4 fix_word_index.ml
--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
+++ b/fix_word_index.ml Sat Aug 08 13:25:01 2009 +0200
@@ -0,0 +1,118 @@
+open Common
+open Bdb
+open Keydb
+
+let env = ref None
+let db_word = ref None
+
+let settings = {
+  Keydb.withtxn = !Settings.transactions;
+  Keydb.cache_bytes = !Settings.cache_bytes;
+  Keydb.pagesize = !Settings.pagesize;
+  Keydb.dbdir = Lazy.force Settings.dbdir;
+  Keydb.dumpdir = Lazy.force Settings.dumpdir;
+}
+
+let get_env settings =
+  match !env with
+    Some e -> e
+  | None ->
+      let e = Dbenv.create () in
+      try
+       (
+        match settings.cache_bytes with
+          None -> ()
+        | Some cache_bytes ->
+            Dbenv.set_cachesize e ~gbytes:0 ~bytes:cache_bytes ~ncache:0);
+       Dbenv.dopen e settings.dbdir 
+         ([Dbenv.INIT_MPOOL; Dbenv.CREATE ]
+          @ (if settings.withtxn then [ Dbenv.INIT_TXN ]
+          else [] ) )
+         0o600;
+       env := Some e;
+       e
+      with
+       ex ->
+         if !env = None then Dbenv.close e;
+         raise ex
+
+let get_db settings =
+  plerror 5 "Opening word index";
+  match !db_word with
+    Some db -> db
+  | None ->
+      let dbenv = get_env settings in
+      let openflags = (
+       if settings.withtxn then [Db.CREATE; Db.AUTO_COMMIT]
+       else [Db.CREATE])
+      in
+      let db = Db.sopen ~dbenv "word" Db.BTREE
+         ~moreflags:[Db.DUPSORT] openflags 0o600 
+      in
+      db_word := Some db;
+      db
+
+let close_db () =
+  plerror 5 "Closing word index";
+  (match !db_word with
+    Some db -> Db.close db
+  | None -> ());
+  db_word := None;
+  (match !env with
+    Some e -> Dbenv.close e
+  | None -> ());
+  env := None
+
+let fix_words_loop ~txn db c =
+  let rec loop n =
+    match (try Some (Cursor.get c Cursor.NEXT []) with Not_found -> None)
+    with
+      None -> n
+    | Some (key,fp) ->
+       let canon_key = String.lowercase key in
+       if key = canon_key
+       then loop n
+       else (
+         plerror 5 "Fix 0x%s %s -> %s" (Utils.hexstring fp) key canon_key;
+         (try
+           match txn with
+             None -> Db.put db canon_key fp [Db.NODUPDATA];
+           | Some txn -> Db.put db ~txn  ~key:canon_key ~data:fp [Db.NODUPDATA]
+         with
+           Bdb.Key_exists -> ());
+         Cursor.del c;
+         loop (n+1))
+  in
+  loop 0
+  
+let fix_words ?txn db settings =
+  plerror 0 "Fixing word index";
+  let txn =
+    ref (if settings.withtxn
+    then Some(Txn.txn_begin (get_env settings) txn [])
+    else None)
+  in
+  let count = protect
+      ~f:(fun () ->
+       let c =
+         match !txn with
+           None -> Cursor.create db
+         | Some txn -> Cursor.create ~txn db
+       in
+       let count = protect ~f:(fun () -> fix_words_loop !txn db c)
+            ~finally:(fun () -> Cursor.close c);
+       in begin
+         match !txn with None -> () | Some t -> Txn.commit t [];
+       end;
+       txn := None;
+       count)
+      ~finally:(fun () ->
+       match !txn with
+         Some tx -> Txn.abort tx
+       | None -> ())
+  in
+  plerror 0 "Fixed %d words\n" count
+
+let () =
+  let f () = fix_words (get_db settings) settings in
+  protect ~f ~finally:close_db