[GNUnet-SVN] r421 - in GNUnet/src: applications/fs/ecrs applications/fs/lib applications/fs/module include

[durner]

Author: durner
Date: 2005-03-09 13:51:39 -0800 (Wed, 09 Mar 2005)
New Revision: 421

Modified:
   GNUnet/src/applications/fs/ecrs/unindex.c
   GNUnet/src/applications/fs/ecrs/upload.c
   GNUnet/src/applications/fs/lib/fslib.c
   GNUnet/src/applications/fs/module/fs.c
   GNUnet/src/applications/fs/module/ondemand.c
   GNUnet/src/applications/fs/module/ondemand.h
   GNUnet/src/include/fs.h
   GNUnet/src/include/gnunet_protocols.h
Log:
Always symlink original file when indexing content

Modified: GNUnet/src/applications/fs/ecrs/unindex.c
===================================================================
--- GNUnet/src/applications/fs/ecrs/unindex.c   2005-03-09 21:33:19 UTC (rev 
420)
+++ GNUnet/src/applications/fs/ecrs/unindex.c   2005-03-09 21:51:39 UTC (rev 
421)
@@ -111,10 +111,9 @@
 
 
 /**
- * Undo sym-linking operation (if allowed by config): 
+ * Undo sym-linking operation: 
  * a) check if we have a symlink
  * b) delete symbolic link
- * c) copy file, if fails restore symlink
  */
 static int undoSymlinking(const char * fn,
                          const HashCode512 * fileId,
@@ -130,7 +129,7 @@
 #ifndef S_ISLNK
   return OK; /* symlinks do not exist? */
 #endif
-  if (0 != lstat(fn,
+  if (0 != LSTAT(fn,
                 &buf)) {
     LOG_FILE_STRERROR(LOG_ERROR, "stat", fn);
     return SYSERR;
@@ -156,61 +155,14 @@
           &enc);
   strcat(serverFN,
         (char*)&enc);  
-  tmpName = MALLOC(strlen(serverFN) + 5);
-  ret = readlink(fn,
-                tmpName,
-                strlen(serverFN) + 4);
-  if (ret == -1) {
-    LOG_FILE_STRERROR(LOG_ERROR, "readlink", fn);
-    FREE(tmpName);
-    FREE(serverFN);
-    return SYSERR;
-  }
-  if ( (ret == strlen(serverFN) + 4) ||
-       (0 != strcmp(tmpName,
-                   serverFN)) ) {
-    FREE(tmpName);
-    FREE(serverFN);
-    return OK; /* symlink elsewhere... */
-  }
-  FREE(tmpName);
-  if (OK != getFileHash(serverFN,
-                       &serverFileId)) {
-    FREE(serverFN);
-    return SYSERR;
-  }
-  if (! equalsHashCode512(&serverFileId,
-                         fileId)) {
-    FREE(serverFN);
-    BREAK(); /* rather odd... */
-    return SYSERR;
-  }
-  tmpName = MALLOC(strlen(fn) + 4);
-  strcpy(tmpName, fn);
-  strcat(tmpName, "_");
-#ifdef HAVE_LINK
-  if (0 != link(serverFN, tmpName)) 
-#endif
-    if (OK != copyFile(serverFN, tmpName)) {
-      FREE(serverFN);
-      FREE(tmpName);
-      return SYSERR;
-    }
 
-  if (0 != UNLINK(fn)) {
+  if (0 != UNLINK(serverFN)) {
     FREE(serverFN);
     FREE(tmpName);
     LOG_FILE_STRERROR(LOG_ERROR, "unlink", tmpName);
     return SYSERR;
   }
   
-  if (0 != RENAME(tmpName,
-                 fn)) {
-    LOG_FILE_STRERROR(LOG_ERROR, "rename", tmpName);
-    FREE(tmpName);
-    FREE(serverFN);
-    return SYSERR;
-  }
   FREE(tmpName);
   FREE(serverFN);
   return OK;

Modified: GNUnet/src/applications/fs/ecrs/upload.c
===================================================================
--- GNUnet/src/applications/fs/ecrs/upload.c    2005-03-09 21:33:19 UTC (rev 
420)
+++ GNUnet/src/applications/fs/ecrs/upload.c    2005-03-09 21:51:39 UTC (rev 
421)
@@ -110,91 +110,6 @@
 }
 
 /**
- * sym-linking operation (if allowed by config): 
- * a) check if hash matches,
- * b) rename old file (fn)
- * c) symlink filename to target, if fails,
- *    undo renaming (and abort)
- * d) delete old file
- */
-static void trySymlinking(const char * fn,
-                         const HashCode512 * fileId,
-                         GNUNET_TCP_SOCKET * sock) {
-  EncName enc;
-  char * serverDir;
-  char * serverFN;
-  char * tmpName;
-  HashCode512 serverFileId;
-
-  if (testConfigurationString("FS",
-                             "DISABLE-SYMLINKING",
-                             "YES"))
-    return;
-  serverDir 
-    = getConfigurationOptionValue(sock,
-                                 "FS",
-                                 "INDEX-DIRECTORY");
-  if (serverDir == NULL)
-    return;
-  serverFN = MALLOC(strlen(serverDir) + 2 + sizeof(EncName));
-  strcpy(serverFN,
-        serverDir);
-  FREE(serverDir);
-  strcat(serverFN,
-        DIR_SEPARATOR_STR);
-  hash2enc(fileId,
-          &enc);
-  strcat(serverFN,
-        (char*)&enc);
-  if (OK != getFileHash(serverFN,
-                       &serverFileId)) {
-    FREE(serverFN);
-    return;
-  }
-  if (! equalsHashCode512(&serverFileId,
-                         fileId)) {
-    BREAK(); /* rather odd... */
-    return;
-  }
-  tmpName = MALLOC(strlen(fn) + 4);
-  strcpy(tmpName, fn);
-  strcat(tmpName, "_");
-  if (0 != RENAME(fn,
-                 tmpName)) {
-    LOG_FILE_STRERROR(LOG_ERROR, "rename", fn);
-    FREE(tmpName);
-    FREE(serverFN);
-    return;
-  }
-  if (0 != SYMLINK(serverFN,
-                  fn)) {
-    LOG_FILE_STRERROR(LOG_ERROR, "symlink", fn);
-    if (0 != RENAME(tmpName,
-                   fn)) {
-      /* oops, error recovery failed, how can this happen??? 
-        Well, at least let's give the user some good warning... */
-      LOG_FILE_STRERROR(LOG_ERROR, "rename", fn);
-      LOG(LOG_NOTHING,
-         _("RISK OF DATA LOSS, READ THIS: "
-           "I failed to symlink a file and then failed to"
-           " rename your original file back to its original name.  "
-           "You should find your file '%s' under the new name '%s'."),
-         fn,
-         tmpName);
-      BREAK();
-    }
-    FREE(tmpName);
-    FREE(serverFN);
-    return;
-  }
-  if (0 != UNLINK(tmpName)) 
-    LOG_FILE_STRERROR(LOG_ERROR, "unlink", tmpName);
-  FREE(tmpName);
-  FREE(serverFN);
-}
-
-
-/**
  * Index or insert a file.
  *
  * @param priority what is the priority for OUR node to
@@ -279,6 +194,10 @@
     LOG_FILE_STRERROR(LOG_WARNING, "OPEN", filename);
     return SYSERR;
   }
+
+  if (FS_initIndex(sock, &fileId, filename) == SYSERR)
+    return SYSERR;
+
   dblock = MALLOC(sizeof(Datastore_Value) + DBLOCK_SIZE + sizeof(DBlock));
   dblock->size = htonl(sizeof(Datastore_Value) + DBLOCK_SIZE + sizeof(DBlock));
   dblock->anonymityLevel = htonl(anonymityLevel);
@@ -297,7 +216,7 @@
     iblocks[i]->expirationTime = htonll(expirationTime);
     ((DBlock*) &iblocks[i][1])->type = htonl(D_BLOCK);
   }
-
+  
   pos = 0;
   while (pos < filesize) {
     if (upcb != NULL)
@@ -422,11 +341,6 @@
     FREE(iblocks[i]);
     iblocks[i] = NULL;
   }
-  if (doIndex) {
-    trySymlinking(filename,
-                 &fileId,
-                 sock);
-  }
   IFLOG(LOG_DEBUG,
        hash2enc(&chk.query,
                 &enc));

Modified: GNUnet/src/applications/fs/lib/fslib.c
===================================================================
--- GNUnet/src/applications/fs/lib/fslib.c      2005-03-09 21:33:19 UTC (rev 
420)
+++ GNUnet/src/applications/fs/lib/fslib.c      2005-03-09 21:51:39 UTC (rev 
421)
@@ -293,6 +293,40 @@
 }
 
 /**
+ * Initialize to index a file
+ */
+int FS_initIndex(GNUNET_TCP_SOCKET * sock,
+       const HashCode512 * fileHc,
+       const char *fn) {
+  int ret;
+  RequestInitIndex *ri;
+  unsigned int size, fnSize;
+  
+  fnSize = strlen(fn);
+  size = sizeof(RequestIndex) + fnSize;
+  ri = MALLOC(size);
+  ri->header.size = htons(size);
+  ri->header.type = htons(AFS_CS_PROTO_INIT_INDEX);
+  ri->fileId = *fileHc;
+  memcpy(&ri[1], fn, fnSize);
+  
+  LOG(LOG_DEBUG,
+      "Sending index initialization request to gnunetd\n");
+  if (OK != writeToSocket(sock,
+        &ri->header)) {
+    FREE(ri);
+    return SYSERR; 
+  }
+  FREE(ri);
+  LOG(LOG_DEBUG,
+      "Waiting for confirmation of index initialization request by gnunetd\n");
+  if (OK != readTCPResult(sock,
+        &ret))
+    return SYSERR;
+  return ret;
+}
+
+/**
  * Index a block. 
  * 
  * @param fileHc the hash of the entire file

Modified: GNUnet/src/applications/fs/module/fs.c
===================================================================
--- GNUnet/src/applications/fs/module/fs.c      2005-03-09 21:33:19 UTC (rev 
420)
+++ GNUnet/src/applications/fs/module/fs.c      2005-03-09 21:51:39 UTC (rev 
421)
@@ -365,6 +365,28 @@
 }
 
 /**
+ * Process a request to symlink a file
+ */
+static int csHandleRequestInitIndex(ClientHandle sock,
+        const CS_HEADER * req) {
+  int ret;
+  
+  if (ntohs(req->size) < sizeof(RequestInitIndex)) {
+    BREAK();
+    return SYSERR;
+  }
+  
+  ret = ONDEMAND_initIndex(&((RequestInitIndex *)req)->fileId,
+          (const char *) &((RequestInitIndex *)req)[1]);
+
+  LOG(LOG_DEBUG,
+      "Sending confirmation (%s) of index initialization request to client\n",
+      ret == OK ? "success" : "failure");
+  return coreAPI->sendValueToClient(sock,
+            ret);
+}
+
+/**
  * Process a request to index content from the client.
  *
  * @return SYSERR if the TCP connection should be closed, otherwise OK
@@ -979,12 +1001,13 @@
   } 
 
   LOG(LOG_DEBUG,
-      _("'%s' registering client handlers %d %d %d %d %d %d %d %d %d\n"),
+      _("'%s' registering client handlers %d %d %d %d %d %d %d %d %d %d\n"),
       "fs",
       AFS_CS_PROTO_QUERY_START,
       AFS_CS_PROTO_QUERY_STOP,
       AFS_CS_PROTO_RESULT,
       AFS_CS_PROTO_INSERT,
+      AFS_CS_PROTO_INIT_INDEX,
       AFS_CS_PROTO_INDEX,
       AFS_CS_PROTO_DELETE,
       AFS_CS_PROTO_UNINDEX,
@@ -999,6 +1022,8 @@
                                                      &csHandleRequestInsert));
   GNUNET_ASSERT(SYSERR != capi->registerClientHandler(AFS_CS_PROTO_INDEX,
                                                      &csHandleRequestIndex));
+  GNUNET_ASSERT(SYSERR != 
coreAPI->registerClientHandler(AFS_CS_PROTO_INIT_INDEX,
+                 &csHandleRequestInitIndex));
   GNUNET_ASSERT(SYSERR != capi->registerClientHandler(AFS_CS_PROTO_DELETE,
                                                      &csHandleRequestDelete));
   GNUNET_ASSERT(SYSERR != capi->registerClientHandler(AFS_CS_PROTO_UNINDEX,
@@ -1030,6 +1055,8 @@
                                                           
&csHandleRequestInsert));
   GNUNET_ASSERT(SYSERR != coreAPI->unregisterClientHandler(AFS_CS_PROTO_INDEX,
                                                           
&csHandleRequestIndex));
+  GNUNET_ASSERT(SYSERR != 
coreAPI->unregisterClientHandler(AFS_CS_PROTO_INIT_INDEX,
+                 &csHandleRequestInitIndex));
   GNUNET_ASSERT(SYSERR != coreAPI->unregisterClientHandler(AFS_CS_PROTO_DELETE,
                                                           
&csHandleRequestDelete));
   GNUNET_ASSERT(SYSERR != 
coreAPI->unregisterClientHandler(AFS_CS_PROTO_UNINDEX,

Modified: GNUnet/src/applications/fs/module/ondemand.c
===================================================================
--- GNUnet/src/applications/fs/module/ondemand.c        2005-03-09 21:33:19 UTC 
(rev 420)
+++ GNUnet/src/applications/fs/module/ondemand.c        2005-03-09 21:51:39 UTC 
(rev 421)
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     (C) 2001, 2002, 2004 Christian Grothoff (and other contributing authors)
+     (C) 2001, 2002, 2004, 2005 Christian Grothoff (and other contributing 
authors)
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -101,6 +101,64 @@
 }
 
 /**
+ * Creates a symlink to the given file in the shared directory
+ * @param fn the file that was indexed
+ * @param fileId the file's hash code
+ * @return SYSERR on error, YES on success
+ */
+int ONDEMAND_initIndex(const HashCode512 * fileId,
+      const char *fn) {
+  EncName enc;
+  char * serverDir;
+  char * serverFN;
+  char unavail_key[_MAX_PATH + 1];
+
+  serverDir 
+    = getConfigurationString(
+          "FS",
+          "INDEX-DIRECTORY");
+  if (!serverDir) {
+    serverDir = getConfigurationString(
+      "",
+      "GNUNETD_HOME");
+    if (!serverDir)
+      return SYSERR;
+      
+    serverDir = REALLOC(serverDir, strlen(serverDir) + 14);
+    strcat(serverDir, "/data/shared/");
+  }
+  
+  serverFN = MALLOC(strlen(serverDir) + 2 + sizeof(EncName));
+  strcpy(serverFN,
+   serverDir);
+  
+  /* Just in case... */
+  mkdirp(serverDir);
+   
+  FREE(serverDir);
+  strcat(serverFN,
+   DIR_SEPARATOR_STR);
+  hash2enc(fileId,
+     &enc);
+  strcat(serverFN,
+   (char*)&enc);
+  if (0 != SYMLINK(fn, serverFN)) {
+    LOG_FILE_STRERROR(LOG_ERROR, "symlink", fn);
+
+    FREE(serverFN);
+    return SYSERR;
+  }
+  
+  strcpy(unavail_key, "FIRST_UNAVAILABLE-");
+  strcat(unavail_key, (char*)&enc);
+  stateUnlinkFromDB(unavail_key);
+  
+  FREE(serverFN);
+  
+  return YES;
+}
+
+/**
  * Writes the given content to the file at the specified offset
  * and stores an OnDemandBlock into the datastore.
  *
@@ -117,8 +175,6 @@
                   const HashCode512 * fileId,
                   unsigned int size,
                   const DBlock * content) {
-  char * fn;
-  int fd;
   int ret;
   OnDemandBlock odb;
   HashCode512 key;
@@ -128,38 +184,6 @@
     BREAK();
     return SYSERR;
   }
-  fn = getOnDemandFile(fileId);
-  LOG(LOG_DEBUG,
-      "Storing on-demand encoded data in '%s'.\n",
-      fn);
-  fd = OPEN(fn, 
-#ifdef O_LARGEFILE
-           O_CREAT|O_WRONLY|O_LARGEFILE,
-#else
-           O_CREAT|O_WRONLY,
-#endif
-           S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH); /* 644 */
-  if(fd == -1) {    
-    LOG_FILE_STRERROR(LOG_ERROR, "open", fn);
-    FREE(fn);    
-    return SYSERR;
-  }  
-  lseek(fd, 
-       fileOffset,
-       SEEK_SET);
-  ret = WRITE(fd,
-             &content[1],
-             size - sizeof(DBlock));
-  if (ret == size - sizeof(DBlock)) {
-    ret = OK;
-  } else {
-    LOG_FILE_STRERROR(LOG_ERROR, "write", fn);
-    ret = SYSERR;
-  }
-  CLOSE(fd);
-  FREE(fn);
-  if (ret == SYSERR)
-    return ret;
 
   odb.header.size = htonl(sizeof(OnDemandBlock));
   odb.header.type = htonl(ONDEMAND_BLOCK);
@@ -241,7 +265,47 @@
   fileHandle = OPEN(fn, O_RDONLY, 0);
 #endif
   if (fileHandle == -1) {
+    char unavail_key[_MAX_PATH + 1];
+    EncName enc;
+    cron_t *first_unavail;
+    struct stat linkStat;
+       
     LOG_FILE_STRERROR(LOG_ERROR, "open", fn);
+    
+    /* Is the symlink there? */
+    if (LSTAT(fn, &linkStat) == -1) {
+      /* No, we have deleted it previously.
+         Now delete the query that still references the unavailable file. */
+      datastore->del(query, dbv);
+    }
+    else {
+      /* For how long has the file been unavailable? */
+      hash2enc(&odb->fileId,
+        &enc);
+      strcpy(unavail_key, "FIRST_UNVAILABLE-");
+      strcat(unavail_key, (char *) &enc);
+      if (stateReadContent(unavail_key, (void *) &first_unavail) == SYSERR) {
+        unsigned long long now = htonll(cronTime(NULL));
+        stateWriteContent(unavail_key, sizeof(cron_t), (void *) &now);
+      }
+      else {
+        /* Delete it after 3 days */
+        if (*first_unavail - cronTime(NULL) > 259200 * cronSECONDS) {
+          char ofn[_MAX_PATH + 1];
+          
+          if (READLINK(fn, ofn, _MAX_PATH) != -1)
+            LOG(LOG_ERROR, _("Because the file %s has been unavailable for 3 
days"
+              " it got removed from your share. Please unindex files before "
+              " deleting them as the index now contains invalid references!"),
+              ofn);
+          
+          datastore->del(query, dbv);
+          stateUnlinkFromDB(unavail_key);
+          UNLINK(fn);
+        }
+      }
+    }
+    
     FREE(fn);
     return SYSERR;
   }
@@ -382,6 +446,7 @@
   unsigned long long delta;
   DBlock * block;
   EncName enc;
+  char unavail_key[_MAX_PATH + 1];
 
   fn = getOnDemandFile(fileId);
   LOG(LOG_DEBUG,
@@ -451,6 +516,12 @@
   FREE(block);
   CLOSE(fd);
   UNLINK(fn);
+  
+  /* Remove information about unavailability */
+  strcpy(unavail_key, "FIRST_UNAVAILABLE-");
+  strcat(unavail_key, (char*)&enc);
+  stateUnlinkFromDB(unavail_key);
+  
   FREE(fn);
   return OK;
 }

Modified: GNUnet/src/applications/fs/module/ondemand.h
===================================================================
--- GNUnet/src/applications/fs/module/ondemand.h        2005-03-09 21:33:19 UTC 
(rev 420)
+++ GNUnet/src/applications/fs/module/ondemand.h        2005-03-09 21:51:39 UTC 
(rev 421)
@@ -30,6 +30,12 @@
 #include "gnunet_datastore_service.h"
 
 /**
+ * Creates a symlink to the given file in the shared directory
+ */
+int ONDEMAND_initIndex(const HashCode512 * fileId,
+      const char *fn);
+
+/**
  * @return NO if already present, YES on success,
  *  SYSERR on other error (i.e. datastore full)
  */

Modified: GNUnet/src/include/fs.h
===================================================================
--- GNUnet/src/include/fs.h     2005-03-09 21:33:19 UTC (rev 420)
+++ GNUnet/src/include/fs.h     2005-03-09 21:51:39 UTC (rev 421)
@@ -105,6 +105,22 @@
 } RequestInsert;
 
 /**
+ * Client to server: initialize to index content
+ * (for on-demand encoding).  This struct is followed
+ * by the filename to index.
+ */
+typedef struct {
+  CS_HEADER header;
+
+  /**
+   * What is the hash of the file that contains
+   * this block?
+   */
+  HashCode512 fileId;
+  
+} RequestInitIndex;
+
+/**
  * Client to server: index content (for on-demand
  * encoding).  This struct is followed by a variable
  * number of bytes of content.

Modified: GNUnet/src/include/gnunet_protocols.h
===================================================================
--- GNUnet/src/include/gnunet_protocols.h       2005-03-09 21:33:19 UTC (rev 
420)
+++ GNUnet/src/include/gnunet_protocols.h       2005-03-09 21:51:39 UTC (rev 
421)
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     (C) 2001, 2002, 2003, 2004 Christian Grothoff (and other contributing 
authors)
+     (C) 2001, 2002, 2003, 2004, 2005 Christian Grothoff (and other 
contributing authors)
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -159,6 +159,10 @@
  */
 #define AFS_CS_PROTO_GET_AVG_PRIORITY 15
 
+/**
+ * client to gnunetd: initialize to index file
+ */
+#define AFS_CS_PROTO_INIT_INDEX 16
 
 /* *********** messages for traffic module ************* */