[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[GNUnet-SVN] r9959 - in gnunet: . m4 src/fs src/include src/util
|
From: |
gnunet |
|
Subject: |
[GNUnet-SVN] r9959 - in gnunet: . m4 src/fs src/include src/util |
|
Date: |
Sun, 3 Jan 2010 22:42:52 +0100 |
Author: grothoff
Date: 2010-01-03 22:42:52 +0100 (Sun, 03 Jan 2010)
New Revision: 9959
Removed:
gnunet/m4/libltdl-external.m4
gnunet/src/include/gnunet_directories.h
Modified:
gnunet/bootstrap
gnunet/configure.ac
gnunet/src/fs/fs_directory.c
gnunet/src/fs/fs_file_information.c
gnunet/src/fs/fs_getopt.c
gnunet/src/fs/fs_publish.c
gnunet/src/fs/fs_uri.c
gnunet/src/fs/gnunet-directory.c
gnunet/src/fs/gnunet-publish.c
gnunet/src/fs/gnunet-search.c
gnunet/src/include/gnunet_container_lib.h
gnunet/src/util/bio.c
gnunet/src/util/container_meta_data.c
gnunet/src/util/pseudonym.c
Log:
adapting main code base to work with libextractor 0.6 API; testcases will
follow later
Modified: gnunet/bootstrap
===================================================================
--- gnunet/bootstrap 2010-01-03 00:26:34 UTC (rev 9958)
+++ gnunet/bootstrap 2010-01-03 21:42:52 UTC (rev 9959)
@@ -1,2 +1,5 @@
#!/bin/sh
-autoreconf -fiv -I m4
+libtoolize --automake --copy --force
+aclocal
+automake
+autoreconf -f -i
Modified: gnunet/configure.ac
===================================================================
--- gnunet/configure.ac 2010-01-03 00:26:34 UTC (rev 9958)
+++ gnunet/configure.ac 2010-01-03 21:42:52 UTC (rev 9959)
@@ -1,5 +1,5 @@
# This file is part of GNUnet.
-# (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Christian Grothoff
(and other contributing authors)
+# (C) 2001--2010 Christian Grothoff (and other contributing authors)
#
# GNUnet is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published
@@ -38,7 +38,6 @@
AC_PROG_LN_S
AC_PROG_MAKE_SET
AM_PROG_CC_C_O
-LT_CONFIG_LTDL_DIR([libltdl])
AC_CANONICAL_HOST
LT_INIT([disable-static dlopen win32-dll])
LTDL_INIT
@@ -141,7 +140,7 @@
AC_CHECK_LIB(intl, gettext)
LDFLAGS="$LDFLAGS -no-undefined -Wl,--export-all-symbols"
LIBS="$LIBS -lws2_32 -lplibc"
- CFLAGS="-mms-bitfields -I../../libltdl $CFLAGS"
+ CFLAGS="-mms-bitfields $CFLAGS"
build_target="mingw"
AC_PROG_CXX
LIBPREFIX=lib
@@ -212,14 +211,14 @@
;;
yes)
AC_CHECK_HEADERS(extractor.h,
- AC_CHECK_LIB([extractor], [EXTRACTOR_loadDefaultLibraries],
+ AC_CHECK_LIB([extractor], [EXTRACTOR_plugin_add_defaults],
extractor=1))
;;
*)
LDFLAGS="-L$with_extractor/lib $LDFLAGS"
CPPFLAGS="-I$with_extractor/include $CPPFLAGS"
AC_CHECK_HEADERS(extractor.h,
- AC_CHECK_LIB([extractor], [EXTRACTOR_loadDefaultLibraries],
+ AC_CHECK_LIB([extractor], [EXTRACTOR_plugin_add_defaults],
EXT_LIB_PATH="-L$with_extractor/lib $EXT_LIB_PATH"
extractor=1))
;;
@@ -227,7 +226,7 @@
],
[AC_MSG_RESULT([--with-extractor not specified])
AC_CHECK_HEADERS(extractor.h,
- AC_CHECK_LIB([extractor], [EXTRACTOR_loadDefaultLibraries],
+ AC_CHECK_LIB([extractor], [EXTRACTOR_plugin_add_defaults],
extractor=1))])
if test "$extractor" != 1
then
Deleted: gnunet/m4/libltdl-external.m4
===================================================================
--- gnunet/m4/libltdl-external.m4 2010-01-03 00:26:34 UTC (rev 9958)
+++ gnunet/m4/libltdl-external.m4 2010-01-03 21:42:52 UTC (rev 9959)
@@ -1,75 +0,0 @@
-dnl Autoconf macro for an always external libltdl
-dnl Copyright (C) 2009 Heikki Lindholm
-dnl
-dnl This file is free software; as a special exception the author gives
-dnl unlimited permission to copy and/or distribute it, with or without
-dnl modifications, as long as this notice is preserved.
-dnl
-dnl This file is distributed in the hope that it will be useful, but
-dnl WITHOUT ANY WARRANTY, to the extent permitted by law; without even the
-dnl implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-
-dnl AM_PATH_LIBLTDL(
-dnl [CHECK-SYMBOLS, [ACTION-IF-FOUND [, ACTION-IF-NOT-FOUND ]]])
-dnl
-AC_DEFUN([AM_PATH_LIBLTDL],
-[ AC_ARG_WITH(libltdl-prefix,
- AC_HELP_STRING([--with-libltdl-prefix=PFX],
- [prefix where libltdl is installed (optional)]),
- libltdl_prefix="$withval", libltdl_prefix="")
- ltdl_save_CPPFLAGS="$CPPFLAGS"
- ltdl_save_LDFLAGS="$LDFLAGS"
- if test x$libltdl_prefix != x ; then
- CPPFLAGS="-I$libltdl_prefix/include $CPPFLAGS"
- LDFLAGS="-L$libltdl_prefix/lib -lltdl $LDFLAGS"
- else
- if test x"$LIBLTDL" = x ; then
- LIBLTDL="-lltdl"
- fi
- CPPFLAGS="$LTDLINCL $CPPFLAGS"
- LDFLAGS="$LIBLTDL $LDFLAGS"
- fi
-
- symbols_to_check=ifelse([$1], ,"ltdl_dlopen","$1")
- ltdl_found=yes
- AC_CHECK_HEADER([ltdl.h],
- [
- for sym in $symbols_to_check
- do
- AC_CHECK_DECL([$sym],
- [AC_LINK_IFELSE(AC_LANG_CALL([], [$sym]),
- [ltdl_found=yes],
- [ltdl_found=no])],
- [ltdl_found=no],
- [AC_INCLUDES_DEFAULT
- #include <ltdl.h>])
- done
- ],
- [ltdl_found=no],
- [AC_INCLUDES_DEFAULT]
- )
-
- if test x$libltdl_prefix != x ; then
- LTDLINCL="-I$libltdl_prefix/include"
- LIBLTDL="-L$libltdl_prefix/lib -lltdl"
- else
- if test x"$LIBLTDL" = x ; then
- LIBLTDL="-lltdl"
- fi
- fi
- CPPFLAGS="$ltdl_save_CPPFLAGS"
- LDFLAGS="$ltdl_save_LDFLAGS"
-
- AC_MSG_CHECKING(for libltdl with symbols $symbols_to_check)
- if test $ltdl_found = yes; then
- AC_MSG_RESULT(yes)
- ifelse([$2], , :, [$2])
- else
- LTDLINCL=""
- LIBLTDL=""
- AC_MSG_RESULT(no)
- ifelse([$3], , :, [$3])
- fi
- AC_SUBST(LTDLINCL)
- AC_SUBST(LIBLTDL)
-])
Modified: gnunet/src/fs/fs_directory.c
===================================================================
--- gnunet/src/fs/fs_directory.c 2010-01-03 00:26:34 UTC (rev 9958)
+++ gnunet/src/fs/fs_directory.c 2010-01-03 21:42:52 UTC (rev 9959)
@@ -36,10 +36,6 @@
#include "gnunet_fs_service.h"
#include "fs.h"
-#ifndef EXTRACTOR_GNUNET_FULL_DATA
-#define EXTRACTOR_GNUNET_FULL_DATA 137
-#endif
-
/**
* String that is used to indicate that a file
* is a GNUnet directory.
@@ -60,7 +56,7 @@
char *mime;
int ret;
- mime = GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_MIMETYPE);
+ mime = GNUNET_CONTAINER_meta_data_get_by_type (md,
EXTRACTOR_METATYPE_MIMETYPE);
if (mime == NULL)
return GNUNET_SYSERR;
ret = (0 == strcmp (mime, GNUNET_FS_DIRECTORY_MIME)) ? GNUNET_YES :
GNUNET_NO;
@@ -80,7 +76,7 @@
{
char *mime;
- mime = GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_MIMETYPE);
+ mime = GNUNET_CONTAINER_meta_data_get_by_type (md,
EXTRACTOR_METATYPE_MIMETYPE);
if (mime != NULL)
{
GNUNET_break (0 == strcmp (mime,
@@ -89,8 +85,12 @@
return;
}
GNUNET_CONTAINER_meta_data_insert (md,
- EXTRACTOR_MIMETYPE,
- GNUNET_FS_DIRECTORY_MIME);
+ "<gnunet>",
+ EXTRACTOR_METATYPE_MIMETYPE,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ GNUNET_FS_DIRECTORY_MIME,
+ strlen (GNUNET_FS_DIRECTORY_MIME)+1);
}
@@ -225,9 +225,9 @@
}
pos += mdSize;
filename = GNUNET_CONTAINER_meta_data_get_by_type (md,
- EXTRACTOR_FILENAME);
+
EXTRACTOR_METATYPE_FILENAME);
file_data = GNUNET_CONTAINER_meta_data_get_by_type (md,
-
EXTRACTOR_GNUNET_FULL_DATA);
+
EXTRACTOR_METATYPE_GNUNET_FULL_DATA);
if (dep != NULL)
dep (dep_cls,
filename,
@@ -322,6 +322,7 @@
size_t mdxs;
char *uris;
char *ser;
+ char *sptr;
size_t slen;
struct GNUNET_CONTAINER_MetaData *meta;
const struct GNUNET_CONTAINER_MetaData *meta_use;
@@ -340,25 +341,24 @@
fsize = 0; /* not given */
if (fsize > MAX_INLINE_SIZE)
fsize = 0; /* too large */
- if ( (NULL == data) ||
- (NULL != memchr (data, 0, fsize)) )
- fsize = 0; /* must not have 0's in data! */
uris = GNUNET_FS_uri_to_string (uri);
slen = strlen (uris) + 1;
mds =
- GNUNET_CONTAINER_meta_data_get_serialized_size (md,
-
GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL);
+ GNUNET_CONTAINER_meta_data_get_serialized_size (md);
meta_use = md;
meta = NULL;
if (fsize > 0)
{
meta = GNUNET_CONTAINER_meta_data_duplicate (md);
GNUNET_CONTAINER_meta_data_insert (meta,
- EXTRACTOR_GNUNET_FULL_DATA,
- data);
+ "<gnunet>",
+ EXTRACTOR_METATYPE_GNUNET_FULL_DATA,
+ EXTRACTOR_METAFORMAT_BINARY,
+ NULL,
+ data,
+ fsize);
mdxs =
- GNUNET_CONTAINER_meta_data_get_serialized_size (meta,
-
GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL);
+ GNUNET_CONTAINER_meta_data_get_serialized_size (meta);
if ( (slen + sizeof (uint32_t) + mdxs - 1) / DBLOCK_SIZE ==
(slen + sizeof (uint32_t) + mds - 1) / DBLOCK_SIZE)
{
@@ -376,8 +376,9 @@
ser = (char*) &e[1];
memcpy (ser, uris, slen);
GNUNET_free (uris);
+ sptr = &ser[slen + sizeof(uint32_t)];
ret = GNUNET_CONTAINER_meta_data_serialize (meta_use,
- &ser[slen + sizeof(uint32_t)],
+ &sptr,
mds,
GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
if (NULL != meta)
@@ -503,6 +504,7 @@
void **rdata)
{
char *data;
+ char *sptr;
size_t *sizes;
unsigned int *perm;
unsigned int i;
@@ -516,8 +518,7 @@
uint32_t big;
size = 8 + sizeof (uint32_t);
- size += GNUNET_CONTAINER_meta_data_get_serialized_size (bld->meta,
-
GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL);
+ size += GNUNET_CONTAINER_meta_data_get_serialized_size (bld->meta);
sizes = NULL;
perm = NULL;
bes = NULL;
@@ -560,9 +561,9 @@
memcpy (data, GNUNET_DIRECTORY_MAGIC, 8);
off = 8;
+ sptr = &data[off + sizeof (uint32_t)];
ret = GNUNET_CONTAINER_meta_data_serialize (bld->meta,
- &data[off +
- sizeof (uint32_t)],
+ &sptr,
size - off - sizeof (uint32_t),
GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL);
GNUNET_assert (ret != -1);
Modified: gnunet/src/fs/fs_file_information.c
===================================================================
--- gnunet/src/fs/fs_file_information.c 2010-01-03 00:26:34 UTC (rev 9958)
+++ gnunet/src/fs/fs_file_information.c 2010-01-03 21:42:52 UTC (rev 9959)
@@ -364,7 +364,7 @@
/**
* Metadata extractors to use.
*/
- struct EXTRACTOR_Extractor *extractors;
+ struct EXTRACTOR_PluginList *extractors;
/**
* Function to call on each directory entry.
@@ -518,7 +518,7 @@
void *proc_cls,
char **emsg)
{
- struct EXTRACTOR_Extractor *ex = cls;
+ struct EXTRACTOR_PluginList *ex = cls;
struct DirScanCls dsc;
dsc.extractors = ex;
Modified: gnunet/src/fs/fs_getopt.c
===================================================================
--- gnunet/src/fs/fs_getopt.c 2010-01-03 00:26:34 UTC (rev 9958)
+++ gnunet/src/fs/fs_getopt.c 2010-01-03 21:42:52 UTC (rev 9959)
@@ -131,7 +131,7 @@
{
struct GNUNET_CONTAINER_MetaData **mm = scls;
- EXTRACTOR_KeywordType type;
+ enum EXTRACTOR_MetaType type;
const char *typename;
const char *typename_i18n;
struct GNUNET_CONTAINER_MetaData *meta;
@@ -151,17 +151,23 @@
"utf-8"
#endif
);
- type = EXTRACTOR_getHighestKeywordTypeNumber ();
+ type = EXTRACTOR_metatype_get_max ();
while (type > 0)
{
type--;
- typename = EXTRACTOR_getKeywordTypeAsString (type);
+ typename = EXTRACTOR_metatype_to_string (type);
typename_i18n = dgettext (LIBEXTRACTOR_GETTEXT_DOMAIN, typename);
if ((strlen (tmp) >= strlen (typename) + 1) &&
(tmp[strlen (typename)] == ':') &&
(0 == strncmp (typename, tmp, strlen (typename))))
{
- GNUNET_CONTAINER_meta_data_insert (meta, type, &tmp[strlen
(typename) + 1]);
+ GNUNET_CONTAINER_meta_data_insert (meta,
+ "<gnunet>",
+ type,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ &tmp[strlen (typename) + 1],
+ strlen (&tmp[strlen (typename) +
1])+1);
GNUNET_free (tmp);
tmp = NULL;
break;
@@ -170,8 +176,13 @@
(tmp[strlen (typename_i18n)] == ':') &&
(0 == strncmp (typename_i18n, tmp, strlen (typename_i18n))))
{
- GNUNET_CONTAINER_meta_data_insert (meta, type,
- &tmp[strlen (typename_i18n) + 1]);
+ GNUNET_CONTAINER_meta_data_insert (meta,
+ "<gnunet>",
+ type,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ &tmp[strlen (typename_i18n) + 1],
+ strlen (&tmp[strlen
(typename_i18n) + 1]) + 1);
GNUNET_free (tmp);
tmp = NULL;
break;
@@ -179,7 +190,13 @@
}
if (tmp != NULL)
{
- GNUNET_CONTAINER_meta_data_insert (meta, EXTRACTOR_UNKNOWN, tmp);
+ GNUNET_CONTAINER_meta_data_insert (meta,
+ "<gnunet>",
+ EXTRACTOR_METATYPE_UNKNOWN,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ tmp,
+ strlen(tmp) + 1);
GNUNET_free (tmp);
printf (_
("Unknown metadata type in metadata option `%s'. Using metadata
type `unknown' instead.\n"),
Modified: gnunet/src/fs/fs_publish.c
===================================================================
--- gnunet/src/fs/fs_publish.c 2010-01-03 00:26:34 UTC (rev 9958)
+++ gnunet/src/fs/fs_publish.c 2010-01-03 21:42:52 UTC (rev 9959)
@@ -842,7 +842,7 @@
while (NULL != p->dir)
{
fn = GNUNET_CONTAINER_meta_data_get_by_type (p->meta,
- EXTRACTOR_FILENAME);
+
EXTRACTOR_METATYPE_FILENAME);
p = p->dir;
GNUNET_asprintf (&p->emsg,
_("Recursive upload failed at `%s'"),
@@ -1305,6 +1305,7 @@
char *uris;
size_t size;
char *kbe;
+ char *sptr;
pkc = GNUNET_malloc (sizeof (struct PublishKskContext));
pkc->h = h;
@@ -1324,8 +1325,7 @@
return;
}
}
- pkc->mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta,
-
GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
+ pkc->mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta);
GNUNET_assert (pkc->mdsize >= 0);
uris = GNUNET_FS_uri_to_string (uri);
pkc->slen = strlen (uris) + 1;
@@ -1339,8 +1339,9 @@
kbe = (char *) &pkc->kb[1];
memcpy (kbe, uris, pkc->slen);
GNUNET_free (uris);
+ sptr = &kbe[pkc->slen];
pkc->mdsize = GNUNET_CONTAINER_meta_data_serialize (meta,
- &kbe[pkc->slen],
+ &sptr,
pkc->mdsize,
GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
if (pkc->mdsize == -1)
@@ -1480,8 +1481,7 @@
if (update == NULL)
update = "";
nidlen = strlen (update) + 1;
- mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta,
-
GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
+ mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta);
size = sizeof (struct SBlock) + slen + nidlen + mdsize;
if (size > MAX_SBLOCK_SIZE)
@@ -1496,7 +1496,7 @@
memcpy (dest, uris, slen);
dest += slen;
mdsize = GNUNET_CONTAINER_meta_data_serialize (meta,
- dest,
+ &dest,
mdsize,
GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
if (mdsize == -1)
Modified: gnunet/src/fs/fs_uri.c
===================================================================
--- gnunet/src/fs/fs_uri.c 2010-01-03 00:26:34 UTC (rev 9958)
+++ gnunet/src/fs/fs_uri.c 2010-01-03 21:42:52 UTC (rev 9959)
@@ -1469,19 +1469,34 @@
* Adds it to the URI.
*
* @param cls URI to update
- * @param type type of the meta data
- * @param data value of the meta data
- * @return GNUNET_OK (always)
+ * @param plugin_name name of the plugin that produced this value;
+ * special values can be used (i.e. '<zlib>' for zlib being
+ * used in the main libextractor library and yielding
+ * meta data).
+ * @param type libextractor-type describing the meta data
+ * @param format basic format information about data
+ * @param data_mime_type mime-type of data (not of the original file);
+ * can be NULL (if mime-type is not known)
+ * @param data actual meta-data found
+ * @param data_len number of bytes in data
+ * @return 0 (always)
*/
static int
gather_uri_data (void *cls,
- EXTRACTOR_KeywordType type,
- const char *data)
+ const char *plugin_name,
+ enum EXTRACTOR_MetaType type,
+ enum EXTRACTOR_MetaFormat format,
+ const char *data_mime_type,
+ const char *data,
+ size_t data_len)
{
struct GNUNET_FS_Uri *uri = cls;
char *nkword;
int j;
+ if ( (format != EXTRACTOR_METAFORMAT_UTF8) &&
+ (format != EXTRACTOR_METAFORMAT_C_STRING) )
+ return 0;
for (j = uri->data.ksk.keywordCount - 1; j >= 0; j--)
if (0 == strcmp (&uri->data.ksk.keywords[j][1], data))
return GNUNET_OK;
@@ -1489,7 +1504,7 @@
strcpy (nkword, " "); /* not mandatory */
strcat (nkword, data);
uri->data.ksk.keywords[uri->data.ksk.keywordCount++] = nkword;
- return GNUNET_OK;
+ return 0;
}
@@ -1514,8 +1529,8 @@
ret->data.ksk.keywords = NULL;
ret->data.ksk.keywords
= GNUNET_malloc (sizeof (char *) *
- GNUNET_CONTAINER_meta_data_get_contents (md, NULL, NULL));
- GNUNET_CONTAINER_meta_data_get_contents (md, &gather_uri_data, ret);
+ GNUNET_CONTAINER_meta_data_iterate (md, NULL, NULL));
+ GNUNET_CONTAINER_meta_data_iterate (md, &gather_uri_data, ret);
return ret;
}
Modified: gnunet/src/fs/gnunet-directory.c
===================================================================
--- gnunet/src/fs/gnunet-directory.c 2010-01-03 00:26:34 UTC (rev 9958)
+++ gnunet/src/fs/gnunet-directory.c 2010-01-03 21:42:52 UTC (rev 9959)
@@ -36,13 +36,20 @@
*/
static int
item_printer (void *cls,
- EXTRACTOR_KeywordType type,
- const char *data)
+ const char *plugin_name,
+ enum EXTRACTOR_MetaType type,
+ enum EXTRACTOR_MetaFormat format,
+ const char *data_mime_type,
+ const char *data,
+ size_t data_size)
{
+ if ( (format != EXTRACTOR_METAFORMAT_UTF8) &&
+ (format != EXTRACTOR_METAFORMAT_C_STRING) )
+ return 0;
printf ("\t%20s: %s\n",
dgettext (LIBEXTRACTOR_GETTEXT_DOMAIN,
- EXTRACTOR_getKeywordTypeAsString (type)), data);
- return GNUNET_OK;
+ EXTRACTOR_metatype_to_string (type)), data);
+ return 0;
}
@@ -75,9 +82,9 @@
string = GNUNET_FS_uri_to_string (uri);
printf ("%s:\n", string);
GNUNET_free (string);
- GNUNET_CONTAINER_meta_data_get_contents (meta,
- &item_printer,
- NULL);
+ GNUNET_CONTAINER_meta_data_iterate (meta,
+ &item_printer,
+ NULL);
}
Modified: gnunet/src/fs/gnunet-publish.c
===================================================================
--- gnunet/src/fs/gnunet-publish.c 2010-01-03 00:26:34 UTC (rev 9958)
+++ gnunet/src/fs/gnunet-publish.c 2010-01-03 21:42:52 UTC (rev 9959)
@@ -140,43 +140,57 @@
* @param cls closure
* @param type type of the meta data
* @param data value of the meta data
- * @return GNUNET_OK to continue to iterate, GNUNET_SYSERR to abort
+ * @return always 0
*/
static int
meta_printer (void *cls,
- EXTRACTOR_KeywordType type,
- const char *data)
+ const char *plugin_name,
+ enum EXTRACTOR_MetaType type,
+ enum EXTRACTOR_MetaFormat format,
+ const char *data_mime_type,
+ const char *data,
+ size_t data_size)
{
- if ( (type == EXTRACTOR_FILENAME) ||
- (EXTRACTOR_isBinaryType (type)) )
- return GNUNET_OK;
+ if ( (format != EXTRACTOR_METAFORMAT_UTF8) &&
+ (format != EXTRACTOR_METAFORMAT_C_STRING) )
+ return 0;
+ if (type == EXTRACTOR_METATYPE_FILENAME)
+ return 0;
fprintf (stdout,
"%s - %s",
- EXTRACTOR_getKeywordTypeAsString (type),
+ EXTRACTOR_metatype_to_string (type),
data);
- return GNUNET_OK;
+ return 0;
}
/**
- * Merge metadata entries (except binary
- * metadata).
+ * Merge metadata entries.
*
* @param cls closure, target metadata structure
* @param type type of the meta data
* @param data value of the meta data
- * @return GNUNET_OK to continue to iterate, GNUNET_SYSERR to abort
+ * @return always 0
*/
static int
meta_merger (void *cls,
- EXTRACTOR_KeywordType type,
- const char *data)
+ const char *plugin_name,
+ enum EXTRACTOR_MetaType type,
+ enum EXTRACTOR_MetaFormat format,
+ const char *data_mime_type,
+ const char *data,
+ size_t data_size)
{
struct GNUNET_CONTAINER_MetaData *m = cls;
+
GNUNET_CONTAINER_meta_data_insert (m,
+ plugin_name,
type,
- data);
- return GNUNET_OK;
+ format,
+ data_mime_type,
+ data,
+ data_size);
+ return 0;
}
@@ -227,16 +241,16 @@
}
if (NULL != meta)
{
- GNUNET_CONTAINER_meta_data_get_contents (meta,
- &meta_merger,
- m);
+ GNUNET_CONTAINER_meta_data_iterate (meta,
+ &meta_merger,
+ m);
GNUNET_CONTAINER_meta_data_destroy (meta);
meta = NULL;
}
if (extract_only)
{
fn = GNUNET_CONTAINER_meta_data_get_by_type (meta,
- EXTRACTOR_FILENAME);
+ EXTRACTOR_METATYPE_FILENAME);
fs = GNUNET_STRINGS_byte_size_fancy (length);
fprintf (stdout,
_("Keywords for file `%s' (%s)\n"),
@@ -244,9 +258,9 @@
fs);
GNUNET_free (fn);
GNUNET_free (fs);
- GNUNET_CONTAINER_meta_data_get_contents (meta,
- &meta_printer,
- NULL);
+ GNUNET_CONTAINER_meta_data_iterate (meta,
+ &meta_printer,
+ NULL);
fprintf (stdout, "\n");
}
if (GNUNET_FS_meta_data_test_for_directory (meta))
@@ -275,7 +289,7 @@
{
struct GNUNET_FS_FileInformation *fi;
struct GNUNET_FS_Namespace *namespace;
- EXTRACTOR_ExtractorList *l;
+ struct EXTRACTOR_PluginList *l;
char *ex;
char *emsg;
@@ -377,13 +391,13 @@
l = NULL;
if (! disable_extractor)
{
- l = EXTRACTOR_loadDefaultLibraries ();
+ l = EXTRACTOR_plugin_add_defaults (EXTRACTOR_OPTION_DEFAULT_POLICY);
if (GNUNET_OK ==
GNUNET_CONFIGURATION_get_value_string (cfg, "FS", "EXTRACTORS",
&ex))
{
if (strlen (ex) > 0)
- l = EXTRACTOR_loadConfigLibraries (l, ex);
+ l = EXTRACTOR_plugin_add_config (l, ex,
EXTRACTOR_OPTION_DEFAULT_POLICY);
GNUNET_free (ex);
}
}
@@ -396,7 +410,7 @@
priority,
GNUNET_TIME_relative_to_absolute (DEFAULT_EXPIRATION),
&emsg);
- EXTRACTOR_removeAll (l);
+ EXTRACTOR_plugin_remove_all (l);
if (fi == NULL)
{
fprintf (stderr,
Modified: gnunet/src/fs/gnunet-search.c
===================================================================
--- gnunet/src/fs/gnunet-search.c 2010-01-03 00:26:34 UTC (rev 9958)
+++ gnunet/src/fs/gnunet-search.c 2010-01-03 21:42:52 UTC (rev 9959)
@@ -45,12 +45,19 @@
static int
item_printer (void *cls,
- EXTRACTOR_KeywordType type,
- const char *data)
+ const char *plugin_name,
+ enum EXTRACTOR_MetaType type,
+ enum EXTRACTOR_MetaFormat format,
+ const char *data_mime_type,
+ const char *data,
+ size_t data_size)
{
+ if ( (format != EXTRACTOR_METAFORMAT_UTF8) &&
+ (format != EXTRACTOR_METAFORMAT_C_STRING) )
+ return 0;
printf ("\t%20s: %s\n",
dgettext (LIBEXTRACTOR_GETTEXT_DOMAIN,
- EXTRACTOR_getKeywordTypeAsString (type)),
+ EXTRACTOR_metatype_to_string (type)),
data);
return GNUNET_OK;
}
@@ -86,7 +93,7 @@
printf ("%s:\n", uri);
filename =
GNUNET_CONTAINER_meta_data_get_by_type
(info->value.search.specifics.result.meta,
- EXTRACTOR_FILENAME);
+ EXTRACTOR_METATYPE_FILENAME);
if (filename != NULL)
{
while (NULL != (dotdot = strstr (filename, "..")))
@@ -98,9 +105,9 @@
else
printf ("gnunet-download %s\n", uri);
if (verbose)
- GNUNET_CONTAINER_meta_data_get_contents
(info->value.search.specifics.result.meta,
- &item_printer,
- NULL);
+ GNUNET_CONTAINER_meta_data_iterate
(info->value.search.specifics.result.meta,
+ &item_printer,
+ NULL);
printf ("\n");
fflush(stdout);
GNUNET_free_non_null (filename);
Modified: gnunet/src/include/gnunet_container_lib.h
===================================================================
--- gnunet/src/include/gnunet_container_lib.h 2010-01-03 00:26:34 UTC (rev
9958)
+++ gnunet/src/include/gnunet_container_lib.h 2010-01-03 21:42:52 UTC (rev
9959)
@@ -1,6 +1,6 @@
/*
This file is part of GNUnet.
- (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009 Christian Grothoff
(and other contributing authors)
+ (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009, 2010 Christian
Grothoff (and other contributing authors)
GNUnet is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published
@@ -196,23 +196,12 @@
struct GNUNET_CONTAINER_MetaData;
/**
- * Iterator over meta data.
- *
- * @param cls closure
- * @param type type of the meta data
- * @param data value of the meta data
- * @return GNUNET_OK to continue to iterate, GNUNET_SYSERR to abort
- */
-typedef int (*GNUNET_CONTAINER_MetaDataProcessor) (void *cls,
- EXTRACTOR_KeywordType type,
- const char *data);
-
-/**
* Create a fresh MetaData token.
*
* @return empty meta-data container
*/
-struct GNUNET_CONTAINER_MetaData *GNUNET_CONTAINER_meta_data_create (void);
+struct GNUNET_CONTAINER_MetaData *
+GNUNET_CONTAINER_meta_data_create (void);
/**
* Duplicate a MetaData token.
@@ -220,44 +209,62 @@
* @param md what to duplicate
* @return duplicate meta-data container
*/
-struct GNUNET_CONTAINER_MetaData *GNUNET_CONTAINER_meta_data_duplicate (const
- struct
-
GNUNET_CONTAINER_MetaData
- *md);
+struct GNUNET_CONTAINER_MetaData *
+GNUNET_CONTAINER_meta_data_duplicate (const struct
+ GNUNET_CONTAINER_MetaData *md);
/**
* Free meta data.
*
* @param md what to free
*/
-void GNUNET_CONTAINER_meta_data_destroy (struct GNUNET_CONTAINER_MetaData
- *md);
+void
+GNUNET_CONTAINER_meta_data_destroy (struct GNUNET_CONTAINER_MetaData *md);
/**
- * Test if two MDs are equal.
+ * Test if two MDs are equal. We consider them equal if
+ * the meta types, formats and content match (we do not
+ * include the mime types and plugins names in this
+ * consideration).
*
* @param md1 first value to check
* @param md2 other value to check
* @return GNUNET_YES if they are equal
*/
-int GNUNET_CONTAINER_meta_data_test_equal (const struct
- GNUNET_CONTAINER_MetaData *md1,
- const struct
- GNUNET_CONTAINER_MetaData *md2);
+int
+GNUNET_CONTAINER_meta_data_test_equal (const struct
+ GNUNET_CONTAINER_MetaData *md1,
+ const struct
+ GNUNET_CONTAINER_MetaData *md2);
/**
* Extend metadata.
*
* @param md metadata to extend
- * @param type type of the new entry
- * @param data value for the entry
+ * @param plugin_name name of the plugin that produced this value;
+ * special values can be used (i.e. '<zlib>' for zlib being
+ * used in the main libextractor library and yielding
+ * meta data).
+ * @param type libextractor-type describing the meta data
+ * @param format basic format information about data
+ * @param data_mime_type mime-type of data (not of the original file);
+ * can be NULL (if mime-type is not known)
+ * @param data actual meta-data found
+ * @param data_len number of bytes in data
* @return GNUNET_OK on success, GNUNET_SYSERR if this entry already exists
+ * data_mime_type and plugin_name are not considered for "exists"
checks
*/
-int GNUNET_CONTAINER_meta_data_insert (struct GNUNET_CONTAINER_MetaData *md,
- EXTRACTOR_KeywordType type,
- const char *data);
+int
+GNUNET_CONTAINER_meta_data_insert (struct GNUNET_CONTAINER_MetaData *md,
+ const char *plugin_name,
+ enum EXTRACTOR_MetaType type,
+ enum EXTRACTOR_MetaFormat format,
+ const char *data_mime_type,
+ const char *data,
+ size_t data_len);
+
/**
* Remove an item.
*
@@ -265,70 +272,86 @@
* @param type type of the item to remove
* @param data specific value to remove, NULL to remove all
* entries of the given type
+ * @param data_len number of bytes in data
* @return GNUNET_OK on success, GNUNET_SYSERR if the item does not exist in md
*/
-int GNUNET_CONTAINER_meta_data_delete (struct GNUNET_CONTAINER_MetaData *md,
- EXTRACTOR_KeywordType type,
- const char *data);
+int
+GNUNET_CONTAINER_meta_data_delete (struct GNUNET_CONTAINER_MetaData *md,
+ enum EXTRACTOR_MetaType type,
+ const char *data,
+ size_t data_len);
+
/**
* Add the current time as the publication date
* to the meta-data.
*
* @param md metadata to modify
*/
-void GNUNET_CONTAINER_meta_data_add_publication_date (struct
- GNUNET_CONTAINER_MetaData
- *md);
+void
+GNUNET_CONTAINER_meta_data_add_publication_date (struct
+ GNUNET_CONTAINER_MetaData
+ *md);
+
/**
- * Iterate over MD entries, excluding thumbnails.
+ * Iterate over MD entries.
*
* @param md metadata to inspect
* @param iter function to call on each entry
* @param iter_cls closure for iterator
* @return number of entries
*/
-int GNUNET_CONTAINER_meta_data_get_contents (const struct
- GNUNET_CONTAINER_MetaData *md,
- GNUNET_CONTAINER_MetaDataProcessor
- iter, void *iter_cls);
+int GNUNET_CONTAINER_meta_data_iterate (const struct
+ GNUNET_CONTAINER_MetaData *md,
+ EXTRACTOR_MetaDataProcessor
+ iter, void *iter_cls);
/**
- * Get the first MD entry of the given type.
+ * Get the first MD entry of the given type. Caller
+ * is responsible for freeing the return value.
+ * Also, only meta data items that are strings (0-terminated)
+ * are returned by this function.
*
* @param md metadata to inspect
* @param type type to look for
- * @return NULL if we do not have any such entry,
- * otherwise client is responsible for freeing the value!
+ * @return NULL if no entry was found
*/
-char *GNUNET_CONTAINER_meta_data_get_by_type (const struct
- GNUNET_CONTAINER_MetaData *md,
- EXTRACTOR_KeywordType type);
+char *
+GNUNET_CONTAINER_meta_data_get_by_type (const struct
+ GNUNET_CONTAINER_MetaData *md,
+ enum EXTRACTOR_MetaType type);
+
/**
- * Get the first matching MD entry of the given types.
+ * Get the first matching MD entry of the given types. Caller is
+ * responsible for freeing the return value. Also, only meta data
+ * items that are strings (0-terminated) are returned by this
+ * function.
*
* @param md metadata to inspect
* @param ... -1-terminated list of types
* @return NULL if we do not have any such entry,
* otherwise client is responsible for freeing the value!
*/
-char *GNUNET_CONTAINER_meta_data_get_first_by_types (const struct
- GNUNET_CONTAINER_MetaData
- *md, ...);
+char *
+GNUNET_CONTAINER_meta_data_get_first_by_types (const struct
+ GNUNET_CONTAINER_MetaData
+ *md, ...);
/**
- * Get a thumbnail from the meta-data (if present).
+ * Get a thumbnail from the meta-data (if present). Only matches meta
+ * data with mime type "image" and binary format.
*
* @param md metadata to inspect
* @param thumb will be set to the thumbnail data. Must be
* freed by the caller!
* @return number of bytes in thumbnail, 0 if not available
*/
-size_t GNUNET_CONTAINER_meta_data_get_thumbnail (const struct
- GNUNET_CONTAINER_MetaData
- *md, unsigned char **thumb);
+size_t
+GNUNET_CONTAINER_meta_data_get_thumbnail (const struct
+ GNUNET_CONTAINER_MetaData
+ *md, unsigned char **thumb);
/**
* Extract meta-data from a file.
@@ -339,11 +362,12 @@
* @return GNUNET_SYSERR on error, otherwise the number
* of meta-data items obtained
*/
-int GNUNET_CONTAINER_meta_data_extract_from_file (struct
- GNUNET_CONTAINER_MetaData
- *md, const char *filename,
- EXTRACTOR_ExtractorList *
- extractors);
+int
+GNUNET_CONTAINER_meta_data_extract_from_file (struct
+ GNUNET_CONTAINER_MetaData
+ *md, const char *filename,
+ struct EXTRACTOR_PluginList *
+ extractors);
/**
@@ -373,7 +397,8 @@
* Serialize meta-data to target.
*
* @param md metadata to serialize
- * @param target where to write the serialized metadata
+ * @param target where to write the serialized metadata;
+ * *target can be NULL, in which case memory is allocated
* @param max maximum number of bytes available
* @param opt is it ok to just write SOME of the
* meta-data to match the size constraint,
@@ -384,7 +409,7 @@
*/
ssize_t GNUNET_CONTAINER_meta_data_serialize (const struct
GNUNET_CONTAINER_MetaData *md,
- char *target,
+ char **target,
size_t max,
enum
GNUNET_CONTAINER_MetaDataSerializationOptions
@@ -392,23 +417,16 @@
/**
- * Estimate (!) the size of the meta-data in
- * serialized form. The estimate MAY be higher
- * than what is strictly needed.
+ * Get the size of the full meta-data in serialized form.
*
* @param md metadata to inspect
- * @param opt is it ok to just write SOME of the
- * meta-data to match the size constraint,
- * possibly discarding some data?
* @return number of bytes needed for serialization, -1 on error
*/
ssize_t GNUNET_CONTAINER_meta_data_get_serialized_size (const struct
GNUNET_CONTAINER_MetaData
- *md,
- enum
-
GNUNET_CONTAINER_MetaDataSerializationOptions
- opt);
+ *md);
+
/**
* Deserialize meta-data. Initializes md.
*
Deleted: gnunet/src/include/gnunet_directories.h
===================================================================
--- gnunet/src/include/gnunet_directories.h 2010-01-03 00:26:34 UTC (rev
9958)
+++ gnunet/src/include/gnunet_directories.h 2010-01-03 21:42:52 UTC (rev
9959)
@@ -1,34 +0,0 @@
-/*
- This file is part of GNUnet.
- (C) 2001, 2002, 2003, 2004, 2005, 2006 Christian Grothoff (and other
contributing authors)
-
- GNUnet is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published
- by the Free Software Foundation; either version 2, or (at your
- option) any later version.
-
- GNUnet is distributed in the hope that it will be useful, but
- WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with GNUnet; see the file COPYING. If not, write to the
- Free Software Foundation, Inc., 59 Temple Place - Suite 330,
- Boston, MA 02111-1307, USA.
-*/
-
-/**
- * @file include/gnunet_directories.h
- * @brief directories and files in GNUnet (default locations)
- *
- * @author Christian Grothoff
- */
-
-#ifndef GNUNET_DIRECTORIES
-#define GNUNET_DIRECTORIES
-
-#define GNUNET_DEFAULT_CLIENT_CONFIG_FILE "~/.gnunet/gnunet.conf"
-#define GNUNET_DEFAULT_DAEMON_CONFIG_FILE "/etc/gnunetd.conf"
-
-#endif
Modified: gnunet/src/util/bio.c
===================================================================
--- gnunet/src/util/bio.c 2010-01-03 00:26:34 UTC (rev 9958)
+++ gnunet/src/util/bio.c 2010-01-03 21:42:52 UTC (rev 9959)
@@ -434,21 +434,11 @@
ssize_t size;
char *buf;
- size = GNUNET_CONTAINER_meta_data_get_serialized_size (m,
-
GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL
- |
-
GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS);
- if (size == -1)
- return GNUNET_SYSERR;
- if (size > MAX_META_DATA)
- size = MAX_META_DATA;
- buf = GNUNET_malloc (size);
+ buf = NULL;
size = GNUNET_CONTAINER_meta_data_serialize (m,
- buf,
- size,
-
GNUNET_CONTAINER_META_DATA_SERIALIZE_PART
- |
-
GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS);
+ &buf,
+ MAX_META_DATA,
+
GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
if (size == -1)
{
GNUNET_free (buf);
Modified: gnunet/src/util/container_meta_data.c
===================================================================
--- gnunet/src/util/container_meta_data.c 2010-01-03 00:26:34 UTC (rev
9958)
+++ gnunet/src/util/container_meta_data.c 2010-01-03 21:42:52 UTC (rev
9959)
@@ -1,6 +1,6 @@
/*
This file is part of GNUnet.
- (C) 2003, 2004, 2005, 2006, 2008, 2009 Christian Grothoff (and other
contributing authors)
+ (C) 2003, 2004, 2005, 2006, 2008, 2009, 2010 Christian Grothoff (and
other contributing authors)
GNUnet is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published
@@ -32,12 +32,46 @@
#include <extractor.h>
#include <zlib.h>
-#define EXTRA_CHECKS ALLOW_EXTRA_CHECKS
+/**
+ * Meta data item.
+ */
+struct MetaItem
+{
+ /**
+ * This is a linked list.
+ */
+ struct MetaItem *next;
-struct Item
-{
- EXTRACTOR_KeywordType type;
+ /**
+ * Name of the extracting plugin.
+ */
+ char *plugin_name;
+
+ /**
+ * Mime-type of data.
+ */
+ char *mime_type;
+
+ /**
+ * The actual meta data.
+ */
char *data;
+
+ /**
+ * Number of bytes in 'data'.
+ */
+ size_t data_size;
+
+ /**
+ * Type of the meta data.
+ */
+ enum EXTRACTOR_MetaType type;
+
+ /**
+ * Format of the meta data.
+ */
+ enum EXTRACTOR_MetaFormat format;
+
};
/**
@@ -45,86 +79,224 @@
*/
struct GNUNET_CONTAINER_MetaData
{
- uint32_t itemCount;
- struct Item *items;
+ /**
+ * Linked list of the meta data items.
+ */
+ struct MetaItem *items;
+
+ /**
+ * Complete serialized and compressed buffer of the items.
+ * NULL if we have not computed that buffer yet.
+ */
+ char *sbuf;
+
+ /**
+ * Number of bytes in 'sbuf'. 0 if the buffer is stale.
+ */
+ size_t sbuf_size;
+
+ /**
+ * Number of items in the linked list.
+ */
+ unsigned int item_count;
+
};
+
/**
* Create a fresh struct CONTAINER_MetaData token.
+ *
+ * @return empty meta-data container
*/
struct GNUNET_CONTAINER_MetaData *
GNUNET_CONTAINER_meta_data_create ()
{
- struct GNUNET_CONTAINER_MetaData *ret;
- ret = GNUNET_malloc (sizeof (struct GNUNET_CONTAINER_MetaData));
- ret->items = NULL;
- ret->itemCount = 0;
- return ret;
+ return GNUNET_malloc (sizeof (struct GNUNET_CONTAINER_MetaData));
}
+
/**
+ * Free meta data item.
+ *
+ * @param item item to free
+ */
+static void
+meta_item_free (struct MetaItem *item)
+{
+ GNUNET_free_non_null (item->plugin_name);
+ GNUNET_free_non_null (item->mime_type);
+ GNUNET_free_non_null (item->data);
+ GNUNET_free (item);
+}
+
+
+/**
+ * The meta data has changed, invalidate its serialization
+ * buffer.
+ *
+ * @param md meta data that changed
+ */
+static void
+invalidate_sbuf (struct GNUNET_CONTAINER_MetaData *md)
+{
+ if (md->sbuf == NULL)
+ return;
+ GNUNET_free (md->sbuf);
+ md->sbuf = NULL;
+ md->sbuf_size = 0;
+}
+
+
+/**
* Free meta data.
+ *
+ * @param md what to free
*/
void
GNUNET_CONTAINER_meta_data_destroy (struct GNUNET_CONTAINER_MetaData *md)
{
- int i;
+ struct MetaItem *item;
if (md == NULL)
return;
- for (i = 0; i < md->itemCount; i++)
- GNUNET_free (md->items[i].data);
- GNUNET_array_grow (md->items, md->itemCount, 0);
+ while (NULL != (item = md->items))
+ {
+ md->items = item->next;
+ meta_item_free (item);
+ }
+ GNUNET_free_non_null (md->sbuf);
GNUNET_free (md);
}
+
/**
- * Add the current time as the publication date
- * to the meta-data.
+ * Test if two MDs are equal. We consider them equal if
+ * the meta types, formats and content match (we do not
+ * include the mime types and plugins names in this
+ * consideration).
+ *
+ * @param md1 first value to check
+ * @param md2 other value to check
+ * @return GNUNET_YES if they are equal
*/
-void
-GNUNET_CONTAINER_meta_data_add_publication_date (struct
- GNUNET_CONTAINER_MetaData
- *md)
+int
+GNUNET_CONTAINER_meta_data_test_equal (const struct GNUNET_CONTAINER_MetaData
+ *md1,
+ const struct GNUNET_CONTAINER_MetaData
+ *md2)
{
- char *dat;
- struct GNUNET_TIME_Absolute t;
+ struct MetaItem *i;
+ struct MetaItem *j;
+ int found;
- t = GNUNET_TIME_absolute_get ();
- GNUNET_CONTAINER_meta_data_delete (md, EXTRACTOR_PUBLICATION_DATE, NULL);
- dat = GNUNET_STRINGS_absolute_time_to_string (t);
- GNUNET_CONTAINER_meta_data_insert (md, EXTRACTOR_PUBLICATION_DATE, dat);
- GNUNET_free (dat);
+ if (md1 == md2)
+ return GNUNET_YES;
+ if (md1->item_count != md2->item_count)
+ return GNUNET_NO;
+
+ i = md1->items;
+ while (NULL != i)
+ {
+ found = GNUNET_NO;
+ j = md2->items;
+ while (NULL != j)
+ {
+ if ( (i->type == j->type) &&
+ (i->format == j->format) &&
+ (i->data_size == j->data_size) &&
+ (0 == memcmp (i->data,
+ j->data,
+ i->data_size)))
+ {
+ found = GNUNET_YES;
+ break;
+ }
+ j = j->next;
+ }
+ if (found == GNUNET_NO)
+ return GNUNET_NO;
+ i = i->next;
+ }
+ return GNUNET_YES;
}
+
/**
- * Extend metadata.
+ * Extend metadata. Note that the list of meta data items is
+ * sorted by size (largest first).
+ *
+ * @param md metadata to extend
+ * @param plugin_name name of the plugin that produced this value;
+ * special values can be used (i.e. '<zlib>' for zlib being
+ * used in the main libextractor library and yielding
+ * meta data).
+ * @param type libextractor-type describing the meta data
+ * @param format basic format information about data
+ * @param data_mime_type mime-type of data (not of the original file);
+ * can be NULL (if mime-type is not known)
+ * @param data actual meta-data found
+ * @param data_len number of bytes in data
* @return GNUNET_OK on success, GNUNET_SYSERR if this entry already exists
+ * data_mime_type and plugin_name are not considered for "exists"
checks
*/
int
GNUNET_CONTAINER_meta_data_insert (struct GNUNET_CONTAINER_MetaData *md,
- EXTRACTOR_KeywordType type,
- const char *data)
+ const char *plugin_name,
+ enum EXTRACTOR_MetaType type,
+ enum EXTRACTOR_MetaFormat format,
+ const char *data_mime_type,
+ const char *data,
+ size_t data_len)
{
- uint32_t idx;
+ struct MetaItem *prev;
+ struct MetaItem *pos;
+ struct MetaItem *i;
char *p;
- GNUNET_assert (data != NULL);
- for (idx = 0; idx < md->itemCount; idx++)
+ prev = NULL;
+ pos = md->items;
+ while (NULL != pos)
{
- if ((md->items[idx].type == type) &&
- (0 == strcmp (md->items[idx].data, data)))
- return GNUNET_SYSERR;
+ if (pos->data_size < data_len)
+ break;
+ if ( (pos->type == type) &&
+ (pos->format == format) &&
+ (pos->data_size == data_len) &&
+ (0 == memcmp (pos->data,
+ data,
+ data_len)))
+ {
+ if ( (pos->mime_type == NULL) &&
+ (data_mime_type != NULL) )
+ {
+ pos->mime_type = GNUNET_strdup (data_mime_type);
+ invalidate_sbuf (md);
+ }
+ return GNUNET_SYSERR;
+ }
+ prev = pos;
+ pos = pos->next;
}
- idx = md->itemCount;
- GNUNET_array_grow (md->items, md->itemCount, md->itemCount + 1);
- md->items[idx].type = type;
- md->items[idx].data = p = GNUNET_strdup (data);
-
+ md->item_count++;
+ i = GNUNET_malloc (sizeof (struct MetaItem));
+ i->type = type;
+ i->format = format;
+ i->data_size = data_len;
+ i->next = pos;
+ if (prev == NULL)
+ md->items = i;
+ else
+ prev->next = i;
+ i->mime_type = (data_mime_type == NULL) ? NULL : GNUNET_strdup
(data_mime_type);
+ i->plugin_name = (plugin_name == NULL) ? NULL : GNUNET_strdup (plugin_name);
+ i->data = GNUNET_malloc (data_len);
+ memcpy (i->data, data, data_len);
/* change OS native dir separators to unix '/' and others to '_' */
- if (type == EXTRACTOR_FILENAME)
+ if (type == EXTRACTOR_METATYPE_FILENAME)
{
- while (*p != '\0')
+ p = i->data;
+ while ( (*p != '\0') &&
+ (p < i->data + data_len) )
{
if (*p == DIR_SEPARATOR)
*p = '/';
@@ -133,10 +305,11 @@
p++;
}
}
-
+ invalidate_sbuf (md);
return GNUNET_OK;
}
+
/**
* Remove an item.
*
@@ -144,88 +317,150 @@
* @param type type of the item to remove
* @param data specific value to remove, NULL to remove all
* entries of the given type
+ * @param data_len number of bytes in data
* @return GNUNET_OK on success, GNUNET_SYSERR if the item does not exist in md
*/
int
GNUNET_CONTAINER_meta_data_delete (struct GNUNET_CONTAINER_MetaData *md,
- EXTRACTOR_KeywordType type,
- const char *data)
+ enum EXTRACTOR_MetaType type,
+ const char *data,
+ size_t data_len)
{
- uint32_t idx;
- int ret = GNUNET_SYSERR;
- for (idx = 0; idx < md->itemCount; idx++)
+ struct MetaItem *pos;
+ struct MetaItem *prev;
+
+ prev = NULL;
+ pos = md->items;
+ while (NULL != pos)
{
- if ((md->items[idx].type == type) &&
- ((data == NULL) || (0 == strcmp (md->items[idx].data, data))))
- {
- GNUNET_free (md->items[idx].data);
- md->items[idx] = md->items[md->itemCount - 1];
- GNUNET_array_grow (md->items, md->itemCount, md->itemCount - 1);
- if (data == NULL)
- {
- ret = GNUNET_OK;
- continue;
- }
- return GNUNET_OK;
- }
+ if ( (pos->type == type) &&
+ ( (data == NULL) ||
+ ( (pos->data_size == data_len) &&
+ (0 == memcmp (pos->data,
+ data,
+ data_len))) ) )
+ {
+ if (prev == NULL)
+ md->items = pos->next;
+ else
+ prev->next = pos->next;
+ meta_item_free (pos);
+ md->item_count--;
+ invalidate_sbuf (md);
+ return GNUNET_OK;
+ }
+ prev = pos;
+ pos = pos->next;
}
- return ret;
+ return GNUNET_SYSERR;
}
+
/**
- * Iterate over MD entries, excluding thumbnails.
+ * Add the current time as the publication date
+ * to the meta-data.
*
+ * @param md metadata to modify
+ */
+void
+GNUNET_CONTAINER_meta_data_add_publication_date (struct
+ GNUNET_CONTAINER_MetaData
+ *md)
+{
+ char *dat;
+ struct GNUNET_TIME_Absolute t;
+
+ t = GNUNET_TIME_absolute_get ();
+ GNUNET_CONTAINER_meta_data_delete (md,
+ EXTRACTOR_METATYPE_PUBLICATION_DATE,
+ NULL,
+ 0);
+ dat = GNUNET_STRINGS_absolute_time_to_string (t);
+ GNUNET_CONTAINER_meta_data_insert (md,
+ "<gnunet>",
+ EXTRACTOR_METATYPE_PUBLICATION_DATE,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ dat,
+ strlen(dat)+1);
+ GNUNET_free (dat);
+}
+
+
+/**
+ * Iterate over MD entries.
+ *
* @param md metadata to inspect
* @param iter function to call on each entry
* @param iter_cls closure for iterator
* @return number of entries
*/
int
-GNUNET_CONTAINER_meta_data_get_contents (const struct
- GNUNET_CONTAINER_MetaData *md,
- GNUNET_CONTAINER_MetaDataProcessor
- iter, void *iter_cls)
+GNUNET_CONTAINER_meta_data_iterate (const struct
+ GNUNET_CONTAINER_MetaData *md,
+ EXTRACTOR_MetaDataProcessor
+ iter, void *iter_cls)
{
- uint32_t i;
- uint32_t sub;
+ struct MetaItem *pos;
- sub = 0;
- for (i = 0; i < md->itemCount; i++)
+ if (iter == NULL)
+ return md->item_count;
+ pos = md->items;
+ while (NULL != pos)
{
- if (!EXTRACTOR_isBinaryType (md->items[i].type))
- {
- if ((iter != NULL) &&
- (GNUNET_OK != iter (iter_cls,
- md->items[i].type, md->items[i].data)))
- return GNUNET_SYSERR;
- }
- else
- sub++;
+ if (0 != iter (iter_cls,
+ pos->plugin_name,
+ pos->type,
+ pos->format,
+ pos->mime_type,
+ pos->data,
+ pos->data_size))
+ return md->item_count;
+ pos = pos->next;
}
- return (int) (md->itemCount - sub);
+ return md->item_count;
}
+
/**
- * Iterate over MD entries
+ * Get the first MD entry of the given type. Caller
+ * is responsible for freeing the return value.
+ * Also, only meta data items that are strings (0-terminated)
+ * are returned by this function.
*
- * @return number of entries
+ * @param md metadata to inspect
+ * @param type type to look for
+ * @return NULL if no entry was found
*/
char *
GNUNET_CONTAINER_meta_data_get_by_type (const struct GNUNET_CONTAINER_MetaData
- *md, EXTRACTOR_KeywordType type)
+ *md, enum EXTRACTOR_MetaType type)
{
- uint32_t i;
+ struct MetaItem *pos;
- for (i = 0; i < md->itemCount; i++)
- if (type == md->items[i].type)
- return GNUNET_strdup (md->items[i].data);
+ pos = md->items;
+ while (NULL != pos)
+ {
+ if ( (type == pos->type) &&
+ ( (pos->format == EXTRACTOR_METAFORMAT_UTF8) ||
+ (pos->format == EXTRACTOR_METAFORMAT_C_STRING) ) )
+ return GNUNET_strdup (pos->data);
+ pos = pos->next;
+ }
return NULL;
}
+
/**
- * Iterate over MD entries
+ * Get the first matching MD entry of the given types. Caller is
+ * responsible for freeing the return value. Also, only meta data
+ * items that are strings (0-terminated) are returned by this
+ * function.
*
- * @return number of entries
+ * @param md metadata to inspect
+ * @param ... -1-terminated list of types
+ * @return NULL if we do not have any such entry,
+ * otherwise client is responsible for freeing the value!
*/
char *
GNUNET_CONTAINER_meta_data_get_first_by_types (const struct
@@ -234,13 +469,13 @@
{
char *ret;
va_list args;
- EXTRACTOR_KeywordType type;
+ enum EXTRACTOR_MetaType type;
ret = NULL;
va_start (args, md);
while (1)
{
- type = va_arg (args, EXTRACTOR_KeywordType);
+ type = va_arg (args, enum EXTRACTOR_MetaType);
if (type == -1)
break;
ret = GNUNET_CONTAINER_meta_data_get_by_type (md, type);
@@ -251,6 +486,7 @@
return ret;
}
+
/**
* Get a thumbnail from the meta-data (if present).
*
@@ -264,27 +500,33 @@
GNUNET_CONTAINER_MetaData * md,
unsigned char **thumb)
{
- char *encoded;
- int ret;
- size_t size;
+ struct MetaItem *pos;
+ struct MetaItem *match;
- encoded =
- GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_THUMBNAIL_DATA);
- if (encoded == NULL)
- return 0;
- if (strlen (encoded) == 0)
+ match = NULL;
+ pos = md->items;
+ while (NULL != pos)
{
- GNUNET_free (encoded);
- return 0; /* invalid */
+ if ( (0 == strncasecmp ("image/", pos->mime_type,
+ strlen("image/"))) &&
+ (pos->format == EXTRACTOR_METAFORMAT_BINARY) )
+ {
+ if (match == NULL)
+ match = pos;
+ else if ( (match->type != EXTRACTOR_METATYPE_THUMBNAIL) &&
+ (pos->type == EXTRACTOR_METATYPE_THUMBNAIL) )
+ match = pos;
+ }
+ pos = pos->next;
}
- *thumb = NULL;
- ret = EXTRACTOR_binaryDecode (encoded, thumb, &size);
- GNUNET_free (encoded);
- if (ret != 0)
+ if (match == NULL)
return 0;
- return size;
+ *thumb = GNUNET_malloc (match->data_size);
+ memcpy (*thumb, match->data, match->data_size);
+ return match->data_size;
}
+
/**
* Duplicate struct GNUNET_CONTAINER_MetaData.
*
@@ -295,19 +537,67 @@
GNUNET_CONTAINER_meta_data_duplicate (const struct GNUNET_CONTAINER_MetaData
*md)
{
- uint32_t i;
struct GNUNET_CONTAINER_MetaData *ret;
+ struct MetaItem *pos;
if (md == NULL)
return NULL;
ret = GNUNET_CONTAINER_meta_data_create ();
- for (i = 0; i < md->itemCount; i++)
- GNUNET_CONTAINER_meta_data_insert (ret, md->items[i].type,
- md->items[i].data);
+ pos = md->items;
+ while (NULL != pos)
+ {
+ GNUNET_CONTAINER_meta_data_insert (ret,
+ pos->plugin_name,
+ pos->type,
+ pos->format,
+ pos->mime_type,
+ pos->data,
+ pos->data_size);
+ pos = pos->next;
+ }
return ret;
}
+
/**
+ * Add meta data that libextractor finds to our meta data
+ * container.
+ *
+ * @param cls closure, our meta data container
+ * @param plugin_name name of the plugin that produced this value;
+ * special values can be used (i.e. '<zlib>' for zlib being
+ * used in the main libextractor library and yielding
+ * meta data).
+ * @param type libextractor-type describing the meta data
+ * @param format basic format information about data
+ * @param data_mime_type mime-type of data (not of the original file);
+ * can be NULL (if mime-type is not known)
+ * @param data actual meta-data found
+ * @param data_len number of bytes in data
+ * @return always 0 to continue extracting
+ */
+static int
+add_to_md(void *cls,
+ const char *plugin_name,
+ enum EXTRACTOR_MetaType type,
+ enum EXTRACTOR_MetaFormat format,
+ const char *data_mime_type,
+ const char *data,
+ size_t data_len)
+{
+ struct GNUNET_CONTAINER_MetaData *md = cls;
+ (void) GNUNET_CONTAINER_meta_data_insert (md,
+ plugin_name,
+ type,
+ format,
+ data_mime_type,
+ data,
+ data_len);
+ return 0;
+}
+
+
+/**
* Extract meta-data from a file.
*
* @return GNUNET_SYSERR on error, otherwise the number
@@ -316,37 +606,43 @@
int
GNUNET_CONTAINER_meta_data_extract_from_file (struct GNUNET_CONTAINER_MetaData
*md, const char *filename,
- EXTRACTOR_ExtractorList *
+ struct EXTRACTOR_PluginList *
extractors)
{
- EXTRACTOR_KeywordList *head;
- EXTRACTOR_KeywordList *pos;
- int ret;
+ unsigned int old;
if (filename == NULL)
return GNUNET_SYSERR;
if (extractors == NULL)
return 0;
- head = EXTRACTOR_getKeywords (extractors, filename);
- head = EXTRACTOR_removeDuplicateKeywords (head,
-
EXTRACTOR_DUPLICATES_REMOVE_UNKNOWN);
- pos = head;
- ret = 0;
- while (pos != NULL)
- {
- if (GNUNET_OK ==
- GNUNET_CONTAINER_meta_data_insert (md, pos->keywordType,
- pos->keyword))
- ret++;
- pos = pos->next;
- }
- EXTRACTOR_freeKeywords (head);
- return ret;
+ old = md->item_count;
+ EXTRACTOR_extract (extractors,
+ filename,
+ NULL, 0,
+ &add_to_md,
+ md);
+ return (int) (md->item_count - old);
}
-static unsigned int
-tryCompression (char *data, unsigned int oldSize)
+/**
+ * Try to compress the given block of data.
+ *
+ * @param data block to compress; if compression
+ * resulted in a smaller block, the first
+ * bytes of data are updated to the compressed
+ * data
+ * @param oldSize number of bytes in data
+ * @param result set to the compressed data
+ * @param newSize set to size of result
+ * @return GNUNET_YES if compression reduce the size,
+ * GNUNET_NO if compression did not help
+ */
+static int
+try_compression (const char *data,
+ size_t oldSize,
+ char **result,
+ size_t *newSize)
{
char *tmp;
uLongf dlen;
@@ -364,62 +660,40 @@
{
if (dlen < oldSize)
{
- memcpy (data, tmp, dlen);
- GNUNET_free (tmp);
- return dlen;
+ *result = tmp;
+ *newSize = dlen;
+ return GNUNET_YES;
}
}
GNUNET_free (tmp);
- return oldSize;
+ return GNUNET_NO;
}
-/**
- * Decompress input, return the decompressed data
- * as output, set outputSize to the number of bytes
- * that were found.
- *
- * @return NULL on error
- */
-static char *
-decompress (const char *input,
- unsigned int inputSize, unsigned int outputSize)
-{
- char *output;
- uLongf olen;
- olen = outputSize;
- output = GNUNET_malloc (olen);
- if (Z_OK == uncompress ((Bytef *) output,
- &olen, (const Bytef *) input, inputSize))
- {
- return output;
- }
- else
- {
- GNUNET_free (output);
- return NULL;
- }
-}
-
/**
* Flag in 'version' that indicates compressed meta-data.
*/
#define HEADER_COMPRESSED 0x80000000
+
/**
* Bits in 'version' that give the version number.
*/
#define HEADER_VERSION_MASK 0x7FFFFFFF
+
+/**
+ * Header for serialized meta data.
+ */
struct MetaDataHeader
{
/**
- * The version of the MD serialization.
- * The highest bit is used to indicate
- * compression.
+ * The version of the MD serialization. The highest bit is used to
+ * indicate compression.
*
- * Version 0 is the current version;
- * Version is 1 for a NULL pointer.
+ * Version 0 is traditional (pre-0.9) meta data (unsupported)
+ * Version is 1 for a NULL pointer
+ * Version 2 is for 0.9.x (and possibly higher)
* Other version numbers are not yet defined.
*/
uint32_t version;
@@ -430,24 +704,57 @@
uint32_t entries;
/**
- * Size of the MD (decompressed)
+ * Size of the decompressed meta data.
*/
uint32_t size;
/**
- * This is followed by 'entries' values of type 'uint32_t' that
- * correspond to EXTRACTOR_KeywordTypes. After that, the meta-data
- * keywords follow (0-terminated). The MD block always ends with
- * 0-termination, padding with 0 until a multiple of 8 bytes.
+ * This is followed by 'entries' values of type 'struct MetaDataEntry'
+ * and then by 'entry' plugin names, mime-types and data blocks
+ * as specified in those meta data entries.
*/
+};
+
+/**
+ * Entry of serialized meta data.
+ */
+struct MetaDataEntry
+{
+ /**
+ * Meta data type. Corresponds to an 'enum EXTRACTOR_MetaType'
+ */
+ uint32_t type;
+
+ /**
+ * Meta data format. Corresponds to an 'enum EXTRACTOR_MetaFormat'
+ */
+ uint32_t format;
+
+ /**
+ * Number of bytes of meta data.
+ */
+ uint32_t data_size;
+
+ /**
+ * Number of bytes in the plugin name including 0-terminator. 0 for NULL.
+ */
+ uint32_t plugin_name_len;
+
+ /**
+ * Number of bytes in the mime type including 0-terminator. 0 for NULL.
+ */
+ uint32_t mime_type_len;
+
};
+
/**
* Serialize meta-data to target.
*
* @param md metadata to serialize
- * @param target where to write the serialized metadata
+ * @param target where to write the serialized metadata;
+ * *target can be NULL, in which case memory is allocated
* @param max maximum number of bytes available in target
* @param opt is it ok to just write SOME of the
* meta-data to match the size constraint,
@@ -458,149 +765,273 @@
*/
ssize_t
GNUNET_CONTAINER_meta_data_serialize (const struct GNUNET_CONTAINER_MetaData
- *md, char *target, size_t max,
+ *md, char **target, size_t max,
enum
GNUNET_CONTAINER_MetaDataSerializationOptions
opt)
{
- struct MetaDataHeader *hdr;
+ struct GNUNET_CONTAINER_MetaData *vmd;
+ struct MetaItem *pos;
+ struct MetaDataHeader *hdr;
+ struct MetaDataEntry *ent;
+ unsigned int i;
+ uint64_t msize;
+ size_t off;
+ char *mdata;
+ char *cdata;
+ size_t mlen;
+ size_t plen;
size_t size;
- size_t pos;
- uint32_t i;
- size_t len;
- uint32_t ic;
+ size_t left;
+ size_t clen;
+ int comp;
if (max < sizeof (struct MetaDataHeader))
return GNUNET_SYSERR; /* far too small */
- ic = md ? md->itemCount : 0;
- hdr = NULL;
- while (1)
+ if (md == NULL)
+ return 0;
+
+ if (md->sbuf != NULL)
{
- size = sizeof (struct MetaDataHeader);
- size += sizeof (uint32_t) * ic;
- for (i = 0; i < ic; i++)
- size += 1 + strlen (md->items[i].data);
- while (size % 8 != 0)
- size++;
- hdr = GNUNET_malloc (size);
- hdr->version = htonl (md == NULL ? 1 : 0);
- hdr->entries = htonl (ic);
- for (i = 0; i < ic; i++)
- ((uint32_t *) & hdr[1])[i] = htonl ((uint32_t) md->items[i].type);
- pos = sizeof (struct MetaDataHeader);
- pos += sizeof (uint32_t) * ic;
- for (i = 0; i < ic; i++)
- {
- len = strlen (md->items[i].data) + 1;
- memcpy (&((char *) hdr)[pos], md->items[i].data, len);
- pos += len;
- }
+ /* try to use serialization cache */
+ if (md->sbuf_size < max)
+ {
+ if (NULL == *target)
+ *target = GNUNET_malloc (md->sbuf_size);
+ memcpy (*target,
+ md->sbuf,
+ md->sbuf_size);
+ return md->sbuf_size;
+ }
+ if (0 == (opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_PART))
+ return GNUNET_SYSERR; /* can say that this will fail */
+ /* need to compute a partial serialization, sbuf useless ... */
+ }
- hdr->size = htonl (size);
- if ((opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS) == 0)
- {
- pos = tryCompression ((char *) &hdr[1],
- size - sizeof (struct MetaDataHeader));
- }
+
+ msize = 0;
+ pos = md->items;
+ while (NULL != pos)
+ {
+ msize += sizeof (struct MetaDataEntry);
+ msize += pos->data_size;
+ if (pos->plugin_name != NULL)
+ msize += strlen (pos->plugin_name) + 1;
+ if (pos->mime_type != NULL)
+ msize += strlen (pos->mime_type) + 1;
+ pos = pos->next;
+ }
+ size = (size_t) msize;
+ if (size != msize)
+ {
+ GNUNET_break (0); /* integer overflow */
+ return GNUNET_SYSERR;
+ }
+ if (size >= GNUNET_MAX_MALLOC_CHECKED)
+ {
+ /* too large to be processed */
+ return GNUNET_SYSERR;
+ }
+ ent = GNUNET_malloc (size);
+ mdata = (char *) &ent[md->item_count];
+ off = size - (md->item_count * sizeof(struct MetaDataEntry));
+ i = 0;
+ pos = md->items;
+ while (NULL != pos)
+ {
+ ent[i].type = htonl ((uint32_t) pos->type);
+ ent[i].format = htonl ((uint32_t) pos->format);
+ ent[i].data_size = htonl ((uint32_t) pos->data_size);
+ if (pos->plugin_name == NULL)
+ plen = 0;
else
- {
- pos = size - sizeof (struct MetaDataHeader);
- }
- if (pos < size - sizeof (struct MetaDataHeader))
- {
- hdr->version = htonl (HEADER_COMPRESSED);
- size = pos + sizeof (struct MetaDataHeader);
- }
- if (size <= max)
- break;
- GNUNET_free (hdr);
- hdr = NULL;
+ plen = strlen (pos->plugin_name) + 1;
+ ent[i].plugin_name_len = htonl ( (uint32_t) plen);
+ if (pos->mime_type == NULL)
+ mlen = 0;
+ else
+ mlen = strlen (pos->mime_type) + 1;
+ ent[i].mime_type_len = htonl ((uint32_t) mlen);
+ off -= pos->data_size;
+ memcpy (&mdata[off], pos->data, pos->data_size);
+ off -= plen;
+ memcpy (&mdata[off], pos->plugin_name, plen);
+ off -= mlen;
+ memcpy (&mdata[off], pos->mime_type, mlen);
+ i++;
+ pos = pos->next;
+ }
+ GNUNET_assert (off == 0);
- if ((opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_PART) == 0)
- {
- return GNUNET_SYSERR; /* does not fit! */
+ left = size;
+ for (i=0;i<md->item_count;i++)
+ {
+ comp = GNUNET_NO;
+ if (0 == (opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS))
+ comp = try_compression ((const char*) &ent[i],
+ left,
+ &cdata,
+ &clen);
+
+ if ( (md->sbuf == NULL) &&
+ (i == 0) )
+ {
+ /* fill 'sbuf'; this "modifies" md, but since this is only
+ an internal cache we will cast away the 'const' instead
+ of making the API look strange. */
+ vmd = (struct GNUNET_CONTAINER_MetaData*) md;
+ hdr = GNUNET_malloc (left + sizeof (struct MetaDataHeader));
+ hdr->entries = htonl (md->item_count);
+ if (GNUNET_YES == comp)
+ {
+ hdr->size = htonl (clen);
+ hdr->version = htonl (2 | HEADER_COMPRESSED);
+ memcpy (&hdr[1],
+ cdata,
+ clen);
+ vmd->sbuf_size = clen + sizeof (struct MetaDataHeader);
+ }
+ else
+ {
+ hdr->size = htonl (left);
+ hdr->version = htonl (2);
+ memcpy (&hdr[1],
+ &ent[0],
+ left);
+ vmd->sbuf_size = left + sizeof (struct MetaDataHeader);
+ }
+ vmd->sbuf = (char*) hdr;
+ }
+
+ if ( ( (left + sizeof (struct MetaDataHeader)) <= max) ||
+ ( (comp == GNUNET_YES) &&
+ (clen <= max)) )
+ {
+ /* success, this now fits! */
+ if (GNUNET_YES == comp)
+ {
+ hdr = (struct MetaDataHeader*) *target;
+ if (hdr == NULL)
+ {
+ hdr = GNUNET_malloc (clen + sizeof (struct MetaDataHeader));
+ *target = (char*) hdr;
+ }
+ hdr->version = htonl (2 | HEADER_COMPRESSED);
+ hdr->entries = htonl (md->item_count - i);
+ hdr->size = htonl (left);
+ memcpy (&target[sizeof(struct MetaDataHeader)],
+ cdata,
+ clen);
+ GNUNET_free (cdata);
+ GNUNET_free (ent);
+ return clen + sizeof (struct MetaDataHeader);
+ }
+ else
+ {
+ hdr = (struct MetaDataHeader*) target;
+ if (hdr == NULL)
+ {
+ hdr = GNUNET_malloc (left + sizeof (struct MetaDataHeader));
+ *target = (char*) hdr;
+ }
+ hdr->version = htonl (2);
+ hdr->entries = htonl (md->item_count - i);
+ hdr->size = htonl (left);
+ memcpy (&target[sizeof(struct MetaDataHeader)],
+ &ent[i],
+ left);
+ GNUNET_free (ent);
+ return left + sizeof (struct MetaDataHeader);
+ }
+ }
+
+ if (0 == (opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_PART))
+ {
+ /* does not fit! */
+ GNUNET_free (ent);
+ return GNUNET_SYSERR;
}
- /* partial serialization ok, try again with less meta-data */
- if (size > 2 * max)
- ic = ic * 2 / 3; /* still far too big, make big reductions */
- else
- ic--; /* small steps, we're close */
+
+ /* next iteration: ignore the corresponding meta data at the
+ end and try again without it */
+ left -= sizeof (struct MetaDataEntry);
+ left -= pos->data_size;
+ if (pos->plugin_name != NULL)
+ left -= strlen (pos->plugin_name) + 1;
+ if (pos->mime_type != NULL)
+ left -= strlen (pos->mime_type) + 1;
}
- GNUNET_assert (size <= max);
- memcpy (target, hdr, size);
- GNUNET_free (hdr);
- /* extra check: deserialize! */
-#if EXTRA_CHECKS
- {
- struct GNUNET_CONTAINER_MetaData *mdx;
- mdx = GNUNET_CONTAINER_meta_data_deserialize (target, size);
- GNUNET_assert (NULL != mdx);
- GNUNET_CONTAINER_meta_data_destroy (mdx);
- }
-#endif
- return size;
+ GNUNET_free (ent);
+
+ /* nothing fit, only write header! */
+ hdr = (struct MetaDataHeader*) target;
+ if (hdr == NULL)
+ {
+ hdr = GNUNET_malloc (sizeof (struct MetaDataHeader));
+ *target = (char*) hdr;
+ }
+ hdr->version = htonl (2);
+ hdr->entries = htonl (0);
+ hdr->size = htonl (0);
+ return sizeof (struct MetaDataHeader);
}
+
/**
- * Estimate (!) the size of the meta-data in
- * serialized form. The estimate MAY be higher
- * than what is strictly needed.
+ * Get the size of the full meta-data in serialized form.
*
* @param md metadata to inspect
- * @param opt is it ok to just write SOME of the
- * meta-data to match the size constraint,
- * possibly discarding some data?
* @return number of bytes needed for serialization, -1 on error
*/
ssize_t
-GNUNET_CONTAINER_meta_data_get_serialized_size (const struct
- GNUNET_CONTAINER_MetaData *
- md,
- enum
-
GNUNET_CONTAINER_MetaDataSerializationOptions
- opt)
+GNUNET_CONTAINER_meta_data_get_serialized_size (const struct
GNUNET_CONTAINER_MetaData *md)
{
- struct MetaDataHeader *hdr;
- size_t size;
- size_t pos;
- uint32_t i;
- size_t len;
- uint32_t ic;
+ ssize_t ret;
+ char *ptr;
+
+ if (md->sbuf != NULL)
+ return md->sbuf_size;
+ ptr = NULL;
+ ret = GNUNET_CONTAINER_meta_data_serialize (md,
+ &ptr,
+ GNUNET_MAX_MALLOC_CHECKED,
+
GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL);
+ if (ret != -1)
+ GNUNET_free (ptr);
+ return ret;
+}
- ic = md ? md->itemCount : 0;
- size = sizeof (struct MetaDataHeader);
- size += sizeof (uint32_t) * ic;
- for (i = 0; i < ic; i++)
- size += 1 + strlen (md->items[i].data);
- while (size % 8 != 0)
- size++;
- hdr = GNUNET_malloc (size);
- hdr->version = htonl (md == NULL ? 1 : 0);
- hdr->entries = htonl (ic);
- for (i = 0; i < ic; i++)
- ((uint32_t *) & hdr[1])[i] = htonl ((uint32_t) md->items[i].type);
- pos = sizeof (struct MetaDataHeader);
- pos += sizeof (uint32_t) * ic;
- for (i = 0; i < ic; i++)
+
+/**
+ * Decompress input, return the decompressed data
+ * as output, set outputSize to the number of bytes
+ * that were found.
+ *
+ * @param input compressed data
+ * @param inputSize number of bytes in input
+ * @param outputSize expected size of the output
+ * @return NULL on error
+ */
+static char *
+decompress (const char *input,
+ size_t inputSize,
+ size_t outputSize)
+{
+ char *output;
+ uLongf olen;
+
+ olen = outputSize;
+ output = GNUNET_malloc (olen);
+ if (Z_OK == uncompress ((Bytef *) output,
+ &olen, (const Bytef *) input, inputSize))
{
- len = strlen (md->items[i].data) + 1;
- memcpy (&((char *) hdr)[pos], md->items[i].data, len);
- pos += len;
+ return output;
}
- if ((opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS) == 0)
- {
- pos =
- tryCompression ((char *) &hdr[1],
- size - sizeof (struct MetaDataHeader));
- }
else
{
- pos = size - sizeof (struct MetaDataHeader);
+ GNUNET_free (output);
+ return NULL;
}
- if (pos < size - sizeof (struct MetaDataHeader))
- size = pos + sizeof (struct MetaDataHeader);
- GNUNET_free (hdr);
- return size;
}
@@ -616,41 +1047,57 @@
GNUNET_CONTAINER_meta_data_deserialize (const char *input, size_t size)
{
struct GNUNET_CONTAINER_MetaData *md;
- const struct MetaDataHeader *hdr;
+ struct MetaDataHeader hdr;
+ struct MetaDataEntry ent;
uint32_t ic;
+ uint32_t i;
char *data;
const char *cdata;
+ uint32_t version;
uint32_t dataSize;
int compressed;
- uint32_t i;
- size_t pos;
- size_t len;
- uint32_t version;
+ size_t left;
+ uint32_t mlen;
+ uint32_t plen;
+ uint32_t dlen;
+ const char *mdata;
+ const char *meta_data;
+ const char *plugin_name;
+ const char *mime_type;
+ enum EXTRACTOR_MetaFormat format;
if (size < sizeof (struct MetaDataHeader))
return NULL;
- hdr = (const struct MetaDataHeader *) input;
- version = ntohl (MAKE_UNALIGNED (hdr->version)) & HEADER_VERSION_MASK;
+ memcpy (&hdr,
+ input,
+ sizeof (struct MetaDataHeader));
+ version = ntohl (hdr.version) & HEADER_VERSION_MASK;
+ compressed = (ntohl (hdr.version) & HEADER_COMPRESSED) != 0;
+
if (version == 1)
- return NULL; /* null pointer */
- if (version != 0)
+ return NULL; /* null pointer */
+ if (version != 2)
{
GNUNET_break_op (0); /* unsupported version */
return NULL;
}
- ic = ntohl (MAKE_UNALIGNED (hdr->entries));
- compressed =
- (ntohl (MAKE_UNALIGNED (hdr->version)) & HEADER_COMPRESSED) != 0;
+
+ ic = ntohl (hdr.entries);
+ dataSize = ntohl (hdr.size);
+ if ((sizeof (struct MetaDataEntry) * ic) > dataSize)
+ {
+ GNUNET_break_op (0);
+ return NULL;
+ }
+
if (compressed)
{
- dataSize =
- ntohl (MAKE_UNALIGNED (hdr->size)) - sizeof (struct MetaDataHeader);
- if (dataSize > 2 * 1042 * 1024)
+ if (dataSize >= GNUNET_MAX_MALLOC_CHECKED)
{
- GNUNET_break (0);
- return NULL; /* only 2 MB allowed [to make sure we don't
blow
- our memory limit because of a mal-formed
- message... ] */
+ /* make sure we don't blow our memory limit because of a mal-formed
+ message... */
+ GNUNET_break_op (0);
+ return NULL;
}
data =
decompress ((const char *) &input[sizeof (struct MetaDataHeader)],
@@ -665,85 +1112,94 @@
else
{
data = NULL;
- cdata = (const char *) &hdr[1];
- dataSize = size - sizeof (struct MetaDataHeader);
- if (size != ntohl (MAKE_UNALIGNED (hdr->size)))
+ cdata = (const char *) &input[sizeof (struct MetaDataHeader)];
+ if (dataSize != size - sizeof (struct MetaDataHeader))
{
- GNUNET_break (0);
+ GNUNET_break_op (0);
return NULL;
}
}
- if ((sizeof (uint32_t) * ic + ic) > dataSize)
+ md = GNUNET_CONTAINER_meta_data_create ();
+ left = dataSize - ic * sizeof (struct MetaDataEntry);
+ mdata = &cdata[ic * sizeof (struct MetaDataEntry)];
+ for (i=0;i<ic;i++)
{
- GNUNET_break (0);
- goto FAILURE;
- }
- if ((ic > 0) && (cdata[dataSize - 1] != '\0'))
- {
- GNUNET_break (0);
- goto FAILURE;
- }
+ memcpy (&ent,
+ &cdata[i * sizeof(struct MetaDataEntry)],
+ sizeof (struct MetaDataEntry));
+ format = (enum EXTRACTOR_MetaFormat) ntohl (ent.format);
+ if ( (format != EXTRACTOR_METAFORMAT_UTF8) &&
+ (format != EXTRACTOR_METAFORMAT_C_STRING) &&
+ (format != EXTRACTOR_METAFORMAT_BINARY) )
+ {
+ GNUNET_break_op (0);
+ break;
+ }
+ dlen = ntohl (ent.data_size);
+ plen = ntohl (ent.plugin_name_len);
+ mlen = ntohl (ent.mime_type_len);
+ if (dlen > left)
+ {
+ GNUNET_break_op (0);
+ break;
+ }
+ left -= dlen;
+ meta_data = &mdata[left];
+ if ( (format == EXTRACTOR_METAFORMAT_UTF8) ||
+ (format == EXTRACTOR_METAFORMAT_C_STRING) )
+ {
+ if ( (dlen == 0) ||
+ (mdata[left + dlen - 1] != '\0') )
+ {
+ GNUNET_break_op (0);
+ break;
+ }
+ }
+ if (plen > left)
+ {
+ GNUNET_break_op (0);
+ break;
+ }
+ left -= plen;
+ if ( (plen > 0) &&
+ (mdata[left + plen - 1] != '\0') )
+ {
+ GNUNET_break_op (0);
+ break;
+ }
+ if (plen == 0)
+ plugin_name = NULL;
+ else
+ plugin_name = &mdata[left];
- md = GNUNET_CONTAINER_meta_data_create ();
- GNUNET_array_grow (md->items, md->itemCount, ic);
- i = 0;
- pos = sizeof (uint32_t) * ic;
- while ((pos < dataSize) && (i < ic))
- {
- len = strlen (&cdata[pos]) + 1;
- md->items[i].type = (EXTRACTOR_KeywordType)
- ntohl (MAKE_UNALIGNED (((const uint32_t *) cdata)[i]));
- md->items[i].data = GNUNET_strdup (&cdata[pos]);
- pos += len;
- i++;
+ if (mlen > left)
+ {
+ GNUNET_break_op (0);
+ break;
+ }
+ left -= mlen;
+ if ( (mlen > 0) &&
+ (mdata[left + mlen - 1] != '\0') )
+ {
+ GNUNET_break_op (0);
+ break;
+ }
+ if (mlen == 0)
+ mime_type = NULL;
+ else
+ mime_type = &mdata[left];
+ GNUNET_CONTAINER_meta_data_insert (md,
+ plugin_name,
+ (enum EXTRACTOR_MetaType) ntohl
(ent.type),
+ format,
+ mime_type,
+ meta_data,
+ dlen);
}
- if (i < ic)
- { /* oops */
- GNUNET_CONTAINER_meta_data_destroy (md);
- goto FAILURE;
- }
- GNUNET_free_non_null (data);
+ GNUNET_free_non_null (data);
return md;
-FAILURE:
- GNUNET_free_non_null (data);
- return NULL; /* size too small */
}
-/**
- * Test if two MDs are equal.
- *
- * @param md1 first value to check
- * @param md2 other value to check
- * @return GNUNET_YES if they are equal
- */
-int
-GNUNET_CONTAINER_meta_data_test_equal (const struct GNUNET_CONTAINER_MetaData
- *md1,
- const struct GNUNET_CONTAINER_MetaData
- *md2)
-{
- uint32_t i;
- uint32_t j;
- int found;
- if (md1->itemCount != md2->itemCount)
- return GNUNET_NO;
- for (i = 0; i < md1->itemCount; i++)
- {
- found = GNUNET_NO;
- for (j = 0; j < md2->itemCount; j++)
- if ((md1->items[i].type == md2->items[j].type) &&
- (0 == strcmp (md1->items[i].data, md2->items[j].data)))
- {
- found = GNUNET_YES;
- break;
- }
- if (found == GNUNET_NO)
- return GNUNET_NO;
- }
- return GNUNET_YES;
-}
-
-
/* end of container_meta_data.c */
Modified: gnunet/src/util/pseudonym.c
===================================================================
--- gnunet/src/util/pseudonym.c 2010-01-03 00:26:34 UTC (rev 9958)
+++ gnunet/src/util/pseudonym.c 2010-01-03 21:42:52 UTC (rev 9959)
@@ -300,15 +300,14 @@
{
if ((meta != NULL) && (name == NULL))
name = GNUNET_CONTAINER_meta_data_get_first_by_types (meta,
- EXTRACTOR_TITLE,
-
EXTRACTOR_FILENAME,
-
EXTRACTOR_DESCRIPTION,
-
EXTRACTOR_SUBJECT,
-
EXTRACTOR_PUBLISHER,
- EXTRACTOR_AUTHOR,
-
EXTRACTOR_COMMENT,
-
EXTRACTOR_SUMMARY,
- EXTRACTOR_OWNER,
+
EXTRACTOR_METATYPE_TITLE,
+
EXTRACTOR_METATYPE_FILENAME,
+
EXTRACTOR_METATYPE_DESCRIPTION,
+
EXTRACTOR_METATYPE_SUBJECT,
+
EXTRACTOR_METATYPE_PUBLISHER,
+
EXTRACTOR_METATYPE_AUTHOR_NAME,
+
EXTRACTOR_METATYPE_COMMENT,
+
EXTRACTOR_METATYPE_SUMMARY,
-1);
if (meta != NULL)
{
@@ -545,11 +544,21 @@
* @param data value of entry to insert
*/
static int
-merge_meta_helper (void *cls, EXTRACTOR_KeywordType type, const char *data)
+merge_meta_helper (void *cls,
+ const char *plugin_name,
+ enum EXTRACTOR_MetaType type,
+ enum EXTRACTOR_MetaFormat format,
+ const char *data_mime_type,
+ const char *data,
+ size_t data_len)
{
struct GNUNET_CONTAINER_MetaData *meta = cls;
- GNUNET_CONTAINER_meta_data_insert (meta, type, data);
- return GNUNET_OK;
+
+ (void) GNUNET_CONTAINER_meta_data_insert (meta, plugin_name,
+ type, format,
+ data_mime_type,
+ data, data_len);
+ return 0;
}
@@ -581,7 +590,7 @@
if ((0 == STAT (fn, &sbuf)) &&
(GNUNET_OK == read_info (cfg, id, &old, &ranking, &name)))
{
- GNUNET_CONTAINER_meta_data_get_contents (meta, &merge_meta_helper, old);
+ GNUNET_CONTAINER_meta_data_iterate (meta, &merge_meta_helper, old);
write_pseudonym_info (cfg, id, old, ranking, name);
GNUNET_CONTAINER_meta_data_destroy (old);
GNUNET_free_non_null (name);
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- [GNUnet-SVN] r9959 - in gnunet: . m4 src/fs src/include src/util,
gnunet <=