[gnunet] branch master updated: - fix record publishing; allow multiline

[gnunet]

This is an automated email from the git hooks/post-receive script.

martin-schanzenbach pushed a commit to branch master
in repository gnunet.

The following commit(s) were added to refs/heads/master by this push:
     new e1bf9b2b2 - fix record publishing; allow multiline
e1bf9b2b2 is described below

commit e1bf9b2b29bbc8112656e0cfbd36b69ab5e3ab49
Author: Martin Schanzenbach <schanzen@gnunet.org>
AuthorDate: Thu Oct 20 09:25:57 2022 +0900

    - fix record publishing; allow multiline
---
 src/namestore/example_zonefile            |   7 +-
 src/namestore/gnunet-namestore-zonefile.c | 313 +++++++++++++++++++-----------
 2 files changed, 208 insertions(+), 112 deletions(-)

diff --git a/src/namestore/example_zonefile b/src/namestore/example_zonefile
index 3a2324eec..d75adcbde 100644
--- a/src/namestore/example_zonefile
+++ b/src/namestore/example_zonefile
@@ -1,6 +1,10 @@
 $ORIGIN example.com.     ; designates the start of this zone file in the 
namespace
 $TTL 3600                ; default expiration time (in seconds) of all RRs 
without their own TTL value
-example.com.    IN  SOA   ns.example.com. username.example.com. ( 2020091025 
7200 3600 1209600 3600 )
+example.com.    IN  SOA   ns.example.com. username.example.com. ( 2020091025 ; 
A comment
+                                                                  7200 ; 
Comment
+                                                                  3600
+                                                                  1209600
+                                                                  3600 )
 example.com.    IN  NS    ns                    ; ns.example.com is a 
nameserver for example.com
 example.com.    IN  NS    ns.somewhere.example. ; ns.somewhere.example is a 
backup nameserver for example.com
 example.com.    IN  MX    10 mail.example.com.  ; mail.example.com is the 
mailserver for example.com
@@ -15,3 +19,4 @@ wwwtest         IN  CNAME www                   ; 
wwwtest.example.com is another
 mail            IN  A     192.0.2.3             ; IPv4 address for 
mail.example.com
 mail2           IN  A     192.0.2.4             ; IPv4 address for 
mail2.example.com
 mail3           IN  A     192.0.2.5             ; IPv4 address for 
mail3.example.com
+mail3           IN  TXT   "This is ; quoted"    ; A quoted comment separator
diff --git a/src/namestore/gnunet-namestore-zonefile.c 
b/src/namestore/gnunet-namestore-zonefile.c
index f2eb17297..22ae4a9f5 100644
--- a/src/namestore/gnunet-namestore-zonefile.c
+++ b/src/namestore/gnunet-namestore-zonefile.c
@@ -37,6 +37,12 @@ static int ret = 0;
  */
 static char *ego_name = NULL;
 
+/**
+ * Currently read line or NULL on EOF
+ */
+static char *res;
+
+
 /**
  * Handle to identity lookup.
  */
@@ -113,19 +119,58 @@ add_continuation (void *cls, int32_t success, const char 
*emsg)
   GNUNET_SCHEDULER_add_now (&parse, NULL);
 }
 
+static char*
+trim (char *line)
+{
+  char *ltrimmed = line;
+  int ltrimmed_len;
+  int quoted = 0;
+
+  // Trim all whitespace to the left
+  while (*ltrimmed == ' ')
+    ltrimmed++;
+  ltrimmed_len = strlen (ltrimmed);
+  // Find the first occurence of an unqoted ';', which is our comment
+  for (int i = 0; i < ltrimmed_len; i++)
+  {
+    if (ltrimmed[i] == '"')
+      quoted = ! quoted;
+    if ((ltrimmed[i] != ';') || quoted)
+      continue;
+    ltrimmed[i] = '\0';
+  }
+  ltrimmed_len = strlen (ltrimmed);
+  // Remove trailing whitespace
+  for (int i = ltrimmed_len; i > 0; i--)
+  {
+    if (ltrimmed[i - 1] != ' ')
+      break;
+    ltrimmed[i - 1] = '\0';
+  }
+  ltrimmed_len = strlen (ltrimmed);
+  if (ltrimmed[ltrimmed_len - 1] == '\n')
+    ltrimmed[ltrimmed_len - 1] = ' ';
+  return ltrimmed;
+}
+
+static char*
+next_token (char *token)
+{
+  char *next = token;
+  while (*next == ' ')
+    next++;
+  return next;
+}
+
 /**
  * Main function that will be run.
  *
  * TODO:
- *  - We need to actually create and store the records with in begin/commit
- *  - We need to get as argument for what zone to import
  *  - We must assume that names are not repeated later in the zonefile because
  *    our _store APIs are replacing. No sure if that is common in zonefiles.
  *  - We must only actually store a record set when the name to store changes 
or
  *    the end of the file is reached.
  *    that way we can group them and add (see above).
- *  - We currently do not allow multiline payloads which seem to be common
- *  - We currently do not sanitize payloads (e.g. `()')
  *  - We need to hope our string formats are compatible, but seems ok.
  *
  * @param cls closure
@@ -138,8 +183,10 @@ parse (void *cls)
 {
   static struct GNUNET_GNSRECORD_Data rd[50]; // Let's hope we do not need more
   char buf[5000];   /* buffer to hold entire line (adjust MAXC as needed) */
+  char payload[5000];
   char *next;
   char *token;
+  char *payload_pos;
   char origin[255];
   static char lastname[255];
   char newname[255];
@@ -151,162 +198,197 @@ parse (void *cls)
   int type;
   static unsigned int rd_count = 0;
   uint32_t ttl_tmp;
-  int publish_rd = GNUNET_NO;
+  int name_changed = 0;
+  int bracket_unclosed = 0;
+  int quoted = 0;
+  static unsigned int published_sets = 0;
+  static unsigned int published_records = 0;
 
-/* use filename provided as 1st argument (stdin by default) */
+  /* use filename provided as 1st argument (stdin by default) */
   int i = 0;
-  while (fgets (buf, 5000, stdin))                     /* read each line of 
input */
+  while (res = fgets (buf, 5000, stdin))                     /* read each line 
of input */
   {
     i++;
     origin_line = 0;
     ttl_line = 0;
-    /* Find space */
-    next = strchr (buf, ' ');
-    if (NULL == next)
-    {
-      fprintf (stderr, "Error at line %u: %s\n", i, buf);
-      break;
-    }
-    next[0] = '\0';
-    next++;
-    if (0 == (strcmp (buf, "$ORIGIN")))
-      origin_line = 1;
-    else if (0 == (strcmp (buf, "$TTL")))
-    {
-      ttl_line = 1;
-    }
-    else
+    token = trim (buf);
+    printf ("Trimmed line (bracket %s): `%s'\n",
+            (bracket_unclosed > 0) ? "unclosed" : "closed",
+            token);
+    if (bracket_unclosed == 0)
     {
-      if (0 == strlen (buf)) // Inherit name from before
+      /* Payload is already parsed */
+      payload_pos = payload;
+      /* Find space */
+      next = strchr (token, ' ');
+      if (NULL == next)
       {
-        printf ("Old name: %s\n", lastname);
-        strcpy (newname, lastname);
+        fprintf (stderr, "Error at line %u: %s\n", i, token);
+        break;
       }
-      else if (buf[strlen (buf) - 1] != '.') // no fqdn
+      next[0] = '\0';
+      next++;
+      if (0 == (strcmp (token, "$ORIGIN")))
       {
-        printf ("New name: %s\n", buf);
-        strcpy (newname, buf);
+        origin_line = 1;
+        token = next_token (next);
       }
-      else if (0 == strcmp (buf, origin))
+      else if (0 == (strcmp (token, "$TTL")))
       {
-        printf ("New name: @\n");
-        strcpy (newname, "@");
+        ttl_line = 1;
+        token = next_token (next);
       }
       else
       {
-        if (strlen (buf) < strlen (origin))
+        printf ("TOKEN: %s\n", token);
+        if (0 == strcmp (token, "IN")) // Inherit name from before
+        {
+          printf ("Old name: %s\n", lastname);
+          strcpy (newname, lastname);
+          token[strlen (token)] = ' ';
+        }
+        else if (token[strlen (token) - 1] != '.') // no fqdn
+        {
+          printf ("New name: %s\n", token);
+          strcpy (newname, token);
+          token = next_token (next);
+        }
+        else if (0 == strcmp (token, origin))
         {
-          fprintf (stderr, "Wrong origin: %s (expected %s)\n", buf, origin);
-          break; // FIXME error?
+          printf ("New name: @\n");
+          strcpy (newname, "@");
+          token = next_token (next);
         }
-        if (0 != strcmp (buf + (strlen (buf) - strlen (origin)), origin))
+        else
         {
-          fprintf (stderr, "Wrong origin: %s (expected %s)\n", buf, origin);
+          if (strlen (token) < strlen (origin))
+          {
+            fprintf (stderr, "Wrong origin: %s (expected %s)\n", token, 
origin);
+            break; // FIXME error?
+          }
+          if (0 != strcmp (token + (strlen (token) - strlen (origin)), origin))
+          {
+            fprintf (stderr, "Wrong origin: %s (expected %s)\n", token, 
origin);
+            break;
+          }
+          token[strlen (token) - strlen (origin) - 1] = '\0';
+          printf ("New name: %s\n", token);
+          strcpy (newname, token);
+          token = next_token (next);
+        }
+        if (0 != strcmp (newname, lastname) &&
+            (0 < rd_count))
+        {
+          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                      "Name changed %s->%s, storing record set of %u 
elements\n",
+                      lastname, newname,
+                      rd_count);
+          name_changed = 1;
+        }
+        else {
+          name_changed = 0;
+          strcpy (lastname, newname);
+        }
+      }
+
+      if (ttl_line)
+      {
+        next = strchr (token, ';');
+        if (NULL != next)
+          next[0] = '\0';
+        next = strchr (token, ' ');
+        if (NULL != next)
+          next[0] = '\0';
+        if (1 != sscanf (token, "%u", &ttl_tmp))
+        {
+          fprintf (stderr, "Unable to parse TTL `%s'\n", token);
           break;
         }
-        buf[strlen (buf) - strlen (origin) - 1] = '\0';
-        printf ("New name: %s\n", buf);
-        strcpy (newname, buf);
+        printf ("TTL is: %u\n", ttl_tmp);
+        ttl.rel_value_us = ttl_tmp * 1000 * 1000;
+        continue;
       }
-      if (0 != strcmp (newname, lastname) &&
-          (0 < rd_count))
+      if (origin_line)
       {
-        GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                    "Name changed %s->%s, storing record set of %u elements\n",
-                    lastname, newname,
-                    rd_count);
-        publish_rd = GNUNET_YES;
+        next = strchr (token, ';');
+        if (NULL != next)
+          next[0] = '\0';
+        next = strchr (token, ' ');
+        if (NULL != next)
+          next[0] = '\0';
+        strcpy (origin, token);
+        printf ("Origin is: %s\n", origin);
+        continue;
       }
-      strcpy (lastname, newname);
-    }
-    while (*next == ' ')
-      next++;
-    token = next;
-
-    if (ttl_line)
-    {
-      next = strchr (token, ';');
-      if (NULL != next)
-        next[0] = '\0';
+      // This is a record, let's go
+      rd[rd_count].flags = GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION;
+      rd[rd_count].expiration_time = ttl.rel_value_us;
       next = strchr (token, ' ');
-      if (NULL != next)
-        next[0] = '\0';
-      if (1 != sscanf (token, "%u", &ttl_tmp))
+      if (NULL == next)
       {
-        fprintf (stderr, "Unable to parse TTL `%s'\n", token);
+        fprintf (stderr, "Error, last token: %s\n", token);
         break;
       }
-      printf ("TTL is: %u\n", ttl_tmp);
-      ttl.rel_value_us = ttl_tmp * 1000 * 1000;
-      continue;
-    }
-    if (origin_line)
-    {
-      next = strchr (token, ';');
-      if (NULL != next)
-        next[0] = '\0';
+      next[0] = '\0';
+      next++;
+      printf ("class is: %s\n", token);
+      while (*next == ' ')
+        next++;
+      token = next;
       next = strchr (token, ' ');
-      if (NULL != next)
-        next[0] = '\0';
-      strcpy (origin, token);
-      printf ("Origin is: %s\n", origin);
-      continue;
+      if (NULL == next)
+      {
+        fprintf (stderr, "Error\n");
+        break;
+      }
+      next[0] = '\0';
+      next++;
+      printf ("type is: %s\n", token);
+      type = GNUNET_GNSRECORD_typename_to_number (token);
+      rd[rd_count].record_type = type;
+      while (*next == ' ')
+        next++;
+      token = next;
     }
-    // This is a record, let's go
-    rd[rd_count].flags = GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION;
-    rd[rd_count].expiration_time = ttl.rel_value_us;
-    next = strchr (token, ' ');
-    if (NULL == next)
+    for (int i = 0; i < strlen (token); i++)
     {
-      fprintf (stderr, "Error, last token: %s\n", token);
-      break;
+      if (token[i] == '"')
+        quoted = ! quoted;
+      if ((token[i] == '(') && ! quoted)
+        bracket_unclosed++;
+      if ((token[i] == ')') && ! quoted)
+        bracket_unclosed--;
     }
-    next[0] = '\0';
-    next++;
-    printf ("class is: %s\n", token);
-    while (*next == ' ')
-      next++;
-    token = next;
-    next = strchr (token, ' ');
-    if (NULL == next)
+    memcpy (payload_pos, token, strlen (token));
+    payload_pos += strlen (token);
+    if (bracket_unclosed > 0)
     {
-      fprintf (stderr, "Error\n");
-      break;
+      *payload_pos = ' ';
+      payload_pos++;
+      continue;
     }
-    next[0] = '\0';
-    next++;
-    printf ("type is: %s\n", token);
-    type = GNUNET_GNSRECORD_typename_to_number (token);
-    rd[rd_count].record_type = type;
-    while (*next == ' ')
-      next++;
-    token = next;
-    next = strchr (token, ';');
-    if (NULL != next)
-      next[0] = '\0';
-    while (token[strlen (token) - 1] == ' ')
-      token[strlen (token) - 1] = '\0';
-    printf ("data is: %s\n\n", token);
+    *payload_pos = '\0';
+    printf ("data is: %s\n\n", payload);
     if (GNUNET_OK !=
-        GNUNET_GNSRECORD_string_to_value (type, token,
+        GNUNET_GNSRECORD_string_to_value (type, payload,
                                           &data,
                                           &data_size))
     {
       // FIXME free rd
       fprintf (stderr,
                _ ("Data `%s' invalid\n"),
-               token);
+               payload);
       ret = 1;
       GNUNET_SCHEDULER_shutdown ();
       return;
     }
     rd[rd_count].data = data;
     rd[rd_count].data_size = data_size;
-    if (GNUNET_YES == publish_rd)
+    if (name_changed)
       break;
     rd_count++;
   }
-  if (GNUNET_YES == publish_rd)
+  if (rd_count > 0)
   {
     ns_qe = GNUNET_NAMESTORE_records_store (ns,
                                             &zone_pkey,
@@ -315,11 +397,20 @@ parse (void *cls)
                                             rd,
                                             &add_continuation,
                                             NULL);
+    published_sets++;
+    published_records += rd_count;
     // FIXME cleanup rd
+    if (name_changed)
+    {
     rd[0] = rd[rd_count]; // recover last rd parsed.
     rd_count = 1;
+    strcpy (lastname, newname);
+    } else
+      rd_count = 0;
     return;
   }
+  printf ("Published %u records sets with total %u records\n",
+          published_sets, published_records);
   ns_qe = GNUNET_NAMESTORE_transaction_commit (ns,
                                                &tx_end,
                                                NULL);

-- 
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.