[Gzz-commits] gzz/lava gzz/storm/impl/AbstractLocalPool.java ...

[Benja Fallenstein]

CVSROOT:        /cvsroot/gzz
Module name:    gzz
Changes by:     Benja Fallenstein <address@hidden>      03/01/13 18:21:44

Modified files:
        lava/gzz/storm/impl: AbstractLocalPool.java AbstractPool.java 
                             DirPool.java TransientPool.java 
                             ZipPool.java 
Added files:
        lava/test/gzz/storm/impl: DirPoolIndexing.test 

Log message:
        DirPool supports some indexing (actually: all, but not too reliable yet)

CVSWeb URLs:
http://savannah.gnu.org/cgi-bin/viewcvs/gzz/gzz/lava/gzz/storm/impl/AbstractLocalPool.java.diff?tr1=1.5&tr2=1.6&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/gzz/gzz/lava/gzz/storm/impl/AbstractPool.java.diff?tr1=1.9&tr2=1.10&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/gzz/gzz/lava/gzz/storm/impl/DirPool.java.diff?tr1=1.16&tr2=1.17&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/gzz/gzz/lava/gzz/storm/impl/TransientPool.java.diff?tr1=1.22&tr2=1.23&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/gzz/gzz/lava/gzz/storm/impl/ZipPool.java.diff?tr1=1.10&tr2=1.11&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/gzz/gzz/lava/test/gzz/storm/impl/DirPoolIndexing.test?rev=1.1

Patches:
Index: gzz/lava/gzz/storm/impl/AbstractLocalPool.java
diff -u gzz/lava/gzz/storm/impl/AbstractLocalPool.java:1.5 
gzz/lava/gzz/storm/impl/AbstractLocalPool.java:1.6
--- gzz/lava/gzz/storm/impl/AbstractLocalPool.java:1.5  Sat Jan 11 13:01:38 2003
+++ gzz/lava/gzz/storm/impl/AbstractLocalPool.java      Mon Jan 13 18:21:44 2003
@@ -31,7 +31,7 @@
  */
 public abstract class AbstractLocalPool extends AbstractPool {
 
-    public AbstractLocalPool(Set indexTypes) {
+    public AbstractLocalPool(Set indexTypes) throws IOException {
        super(indexTypes);
     }
 
@@ -42,10 +42,5 @@
     public Block request(BlockId id, BlockListener listener) 
                                                    throws IOException {
        return get(id);
-    }
-
-    // XXX temporary, until subclasses have implementations
-    protected DB getDB(String typeURI) {
-       return null;
     }
 }
Index: gzz/lava/gzz/storm/impl/AbstractPool.java
diff -u gzz/lava/gzz/storm/impl/AbstractPool.java:1.9 
gzz/lava/gzz/storm/impl/AbstractPool.java:1.10
--- gzz/lava/gzz/storm/impl/AbstractPool.java:1.9       Sat Jan 11 13:01:38 2003
+++ gzz/lava/gzz/storm/impl/AbstractPool.java   Mon Jan 13 18:21:44 2003
@@ -41,9 +41,9 @@
      *  or removed (given that there's a corresponding
      *  index type in <code>indexTypes</code>).
      */
-    protected abstract DB getDB(String typeURI);
+    protected abstract DB getDB(String typeURI) throws IOException;
 
-    public AbstractPool(Set indexTypes) {
+    public AbstractPool(Set indexTypes) throws IOException {
        Map indices = new HashMap();
 
        for(Iterator i=indexTypes.iterator(); i.hasNext();) {
Index: gzz/lava/gzz/storm/impl/DirPool.java
diff -u gzz/lava/gzz/storm/impl/DirPool.java:1.16 
gzz/lava/gzz/storm/impl/DirPool.java:1.17
--- gzz/lava/gzz/storm/impl/DirPool.java:1.16   Mon Dec 30 08:48:23 2002
+++ gzz/lava/gzz/storm/impl/DirPool.java        Mon Jan 13 18:21:44 2003
@@ -23,6 +23,7 @@
 package gzz.storm.impl;
 import gzz.storm.*;
 import gzz.storm.headers.*;
+import gzz.util.*;
 import java.io.*;
 import java.util.*;
 
@@ -35,6 +36,10 @@
      */
     File dir;
 
+    /** The DB objects by index type URI.
+     */
+    protected Map dbs;
+
     /** Get the File object for the block
      *  corresponding to a given id.
      *  This is needed in many places and wrapping it in a convenience
@@ -81,6 +86,7 @@
            }
 
            block = new FileBlock(id);
+           added(block);
        }
     }
 
@@ -98,13 +104,68 @@
        }
     }
 
+    protected class DirDB implements DB {
+       protected File dbDir;
+
+       protected DirDB(String indexTypeURI) throws IOException {
+           byte[] ascii = indexTypeURI.getBytes("US-ASCII");
+           String hex = HexUtil.byteArrToHex(ascii);
+           dbDir = new File(dir, "idx_"+hex);
+
+           if(!dbDir.exists()) {
+               dbDir.mkdir();
+               
+               OutputStream os = new FileOutputStream(new File(dbDir, 
"index_type"));
+               os.write(ascii); os.write((byte)'\n');
+               os.close();
+           }
+       }
+
+       protected File getKeyFile(byte[] key) throws IOException {
+           return new File(dbDir, "key_"+HexUtil.byteArrToHex(key));
+       }
+
+       public Collector get(byte[] key) throws IOException {
+           Set result = new HashSet();
+           File f = getKeyFile(key);
+
+           if(f.exists()) {
+               InputStream in = new FileInputStream(getKeyFile(key));
+               Reader ir = new InputStreamReader(in, "US-ASCII");
+               BufferedReader r = new BufferedReader(ir);
+               
+               String line = r.readLine();
+               while(line != null && !line.equals("")) {
+                   int i = line.indexOf(' ');
+                   BlockId block = new BlockId(line.substring(0, i));
+                   byte[] value = HexUtil.hexToByteArr(line.substring(i+1));
+                   
+                   result.add(new Mapping(block, key, value));
+
+                   line = r.readLine();
+               }
+           }
+
+           return new SimpleSetCollector(result);
+       }
+
+       protected void add(Mapping m) throws IOException {
+           OutputStream os = new FileOutputStream(getKeyFile(m.key), true);
+           os.write(m.block.getURI().getBytes("US-ASCII"));
+           os.write((byte)' ');
+           os.write(HexUtil.byteArrToHex(m.value).getBytes("US-ASCII"));
+           os.write((byte)'\n');
+           os.close();
+       }
+    }
+
     /** Create a new DirPool.
      *  @param dir The directory blocks are stored in.
      *             Must already exist.
      *  @throws IllegalArgumentException if the file isn't a directory
      *                                   or does not exist yet.
      */
-    public DirPool(File dir, Set indexTypes) {
+    public DirPool(File dir, Set indexTypes) throws IOException {
        super(indexTypes);
        this.dir = dir;
     }
@@ -124,6 +185,8 @@
 
        if(!temp.renameTo(getFile(id)))
            throw new IOException("Could not rename temporary file");
+       
+       added(get(id));
     }
     public void delete(Block b) throws IOException {
        getFile(b.getId()).delete();
@@ -149,5 +212,29 @@
                                                          throws IOException {
        File tempFile = gzz.util.TempFileUtil.tmpFile(dir);
        return new FileBlockOutputStream(new VerbatimHeader822(hdr), tempFile);
+    }
+
+    protected DB getDB(String typeURI) throws IOException {
+       if(dbs == null) dbs = new HashMap();
+       DB db = (DB)dbs.get(typeURI);
+       if(db == null) {
+           db = new DirDB(typeURI);
+           dbs.put(typeURI, db);
+       }
+       return db;
+    }
+
+    protected void added(Block block) throws IOException {
+       for(Iterator i = indexTypes.iterator(); i.hasNext();) {
+           IndexType it = (IndexType)i.next();
+           Set mappings = it.getMappings(block);
+           for(Iterator j = mappings.iterator(); j.hasNext();) {
+               Mapping m = (Mapping)j.next();
+               if(!m.block.equals(block.getId()))
+                   throw new Error("Wrong block in mapping: "+m);
+
+               ((DirDB)getDB(it.getIndexTypeURI())).add(m);
+           }
+       }
     }
 }
Index: gzz/lava/gzz/storm/impl/TransientPool.java
diff -u gzz/lava/gzz/storm/impl/TransientPool.java:1.22 
gzz/lava/gzz/storm/impl/TransientPool.java:1.23
--- gzz/lava/gzz/storm/impl/TransientPool.java:1.22     Sat Jan 11 13:01:38 2003
+++ gzz/lava/gzz/storm/impl/TransientPool.java  Mon Jan 13 18:21:44 2003
@@ -98,7 +98,7 @@
        }
     }
 
-    public TransientPool(Set indexTypes) {
+    public TransientPool(Set indexTypes) throws IOException {
        super(indexTypes);
     }
 
Index: gzz/lava/gzz/storm/impl/ZipPool.java
diff -u gzz/lava/gzz/storm/impl/ZipPool.java:1.10 
gzz/lava/gzz/storm/impl/ZipPool.java:1.11
--- gzz/lava/gzz/storm/impl/ZipPool.java:1.10   Mon Dec 30 08:48:23 2002
+++ gzz/lava/gzz/storm/impl/ZipPool.java        Mon Jan 13 18:21:44 2003
@@ -7,6 +7,9 @@
 import java.util.*;
 import java.util.zip.*;
 
+/** A Pool storing blocks in a zip file.
+ *  Attention: This does not support indexing yet!
+ */
 public class ZipPool extends AbstractLocalPool {
 
        protected Map blocks = new HashMap();
@@ -56,7 +59,7 @@
           }
        }
 
-       public ZipPool(ZipFile file, Set indexTypes) {
+       public ZipPool(ZipFile file, Set indexTypes) throws IOException {
            super(indexTypes);
            this.file = file;
        }
@@ -160,4 +163,9 @@
 IOException{
            return new ZipBlockOutputStream(new VerbatimHeader822(hdr));
         }
+
+    // XXX temporary, until we have an implementation
+    protected DB getDB(String typeURI) {
+       throw new UnsupportedOperationException("Not implemented");
+    }
 }