[GNUnet-SVN] [ascension] 34/45: updated gnsmigrator for multiline, added multiline tests

[gnunet]

This is an automated email from the git hooks/post-receive script.

rexxnor pushed a commit to branch master
in repository ascension.

commit a057889a86c15c3bb8b3262b7a49caa204de041e
Author: rexxnor <address@hidden>
AuthorDate: Tue Dec 11 13:23:51 2018 +0100

    updated gnsmigrator for multiline, added multiline tests
---
 gnsmigrator/gnsmigrator.py                  | 133 +++++++++++++++++++++++-----
 gnsmigrator/test/test_gnunet_multiple.sh    |  99 ---------------------
 gnsmigrator/test/test_namestore_multiple.sh | 108 ++++++++++++++++++++++
 3 files changed, 219 insertions(+), 121 deletions(-)

diff --git a/gnsmigrator/gnsmigrator.py b/gnsmigrator/gnsmigrator.py
index d8031b7..0c13fa5 100644
--- a/gnsmigrator/gnsmigrator.py
+++ b/gnsmigrator/gnsmigrator.py
@@ -191,7 +191,7 @@ class GNSMigrator():
             sys.exit(1)
 
     @classmethod
-    def add_records_to_gns(cls):
+    def add_records_to_gns(cls, flags="n"):
         """
         Extracts records from zone and adds them to GNS
         """
@@ -203,19 +203,35 @@ class GNSMigrator():
         # Defining worker
         def worker():
             while True:
-                record = taskqueue.get()
-                if record is None:
+                # define recordline
+                recordline = []
+                label = ""
+
+                rdataset = taskqueue.get()
+                if rdataset is None:
                     break
                 # execute thing to run on item
-                _, _, authns = record
-                if str(authns)[:-1] == ".":
-                    authns = str(authns)[:-1]
-                else:
-                    authns = "%s.%s" % (authns, zonename)
-
-                # building gns record struct
-                # GNUnetGNSRecordData()
-                cls.add_record_to_gns(record, zonename, cls.domain)
+                label, rdata = rdataset
+                for record in rdata:
+                    rdtype = dns.rdatatype.to_text(record.rdtype)
+                    ttl = rdata.ttl
+                    value = str(record)
+
+                    # modify value to fit gns syntax
+                    rdtype, value = cls.transform_to_gns_format(record,
+                                                                rdtype,
+                                                                zonename,
+                                                                cls.domain,
+                                                                label)
+
+                    # build recordline
+                    recordline.append("-R")
+                    recordline.append("%d %s %s %s" %
+                                      (int(ttl), rdtype, flags, value))
+
+                #cls.add_record_to_gns(record, zonename, cls.domain)
+                # add recordline to gns
+                cls.add_recordline_to_gns(recordline, zonename, label)
 
                 taskqueue.task_done()
 
@@ -223,27 +239,98 @@ class GNSMigrator():
         thread = threading.Thread(target=worker)
         thread.start()
 
-        # Add glue records to zone
-        for gluerecord in cls.zone.iterate_rdatas(rdtype=dns.rdatatype.A):
-            taskqueue.put(gluerecord)
-        for gluerecord in cls.zone.iterate_rdatas(rdtype=dns.rdatatype.AAAA):
-            taskqueue.put(gluerecord)
-        # Add NS records to zone
-        for nsrecord in cls.zone.iterate_rdatas(rdtype=dns.rdatatype.NS):
-            taskqueue.put(nsrecord)
+        ## Add glue and rest of A/AAAA records to zone
+        #for gluerecord in cls.zone.iterate_rdatasets(rdtype=dns.rdatatype.A):
+        #    taskqueue.put(gluerecord)
+        #for gluerecord in 
cls.zone.iterate_rdatasets(rdtype=dns.rdatatype.AAAA):
+        #    taskqueue.put(gluerecord)
+        ## Add NS records to zone
+        #for nsrecord in cls.zone.iterate_rdatasets(rdtype=dns.rdatatype.NS):
+        #    taskqueue.put(nsrecord)
+        # Add remaining records to zone
+        for remaining in cls.zone.iterate_rdatasets():
+            taskqueue.put(remaining)
 
         # Block until all tasks are done
         taskqueue.join()
 
-        # Stop workers
+        # Stop workers and threads
         taskqueue.put(None)
         thread.join()
 
         # Add soa record to GNS once completed (updates the previous one)
         soa = cls.get_zone_soa(cls.zone)
-        cls.add_record_to_gns(soa, zonename, cls.domain)
+        cls.add_soa_record_to_gns(soa, zonename, cls.domain)
         logging.info("All records have been added!")
 
+    @staticmethod
+    def add_recordline_to_gns(recordline, zonename, label):
+        """
+        Replaces records in zone or adds them if not
+        :param recordline: records to replace
+        :param zonename: zonename of zone to add records to
+        :param label: label under which to add the records
+        """
+        logging.info("adding %d records with name %s", len(recordline)/2, 
label)
+        ret = sp.run([GNUNET_NAMESTORE_COMMAND,
+                      '-z', zonename,
+                      '-n', str(label),
+                      ] + recordline)
+        logging.info("executed command: %s", " ".join(ret.args))
+
+    @staticmethod
+    def transform_to_gns_format(record, rdtype, zonename, domain, label):
+        """
+        Teansforms value of record to gnunet compatible format
+        :param record: record to transform
+        :param rdtype: record value to transform
+        :param zonename: name of the zone to add to
+        :param domain: domain of the zone to add
+        :param domain: label under which the record is stored
+        """
+        value = str(record)
+        if rdtype == 'SOA':
+            zonetuple = str(value).split(' ')
+            domain = str(".".join(domain.split('.')[:-1]))
+            authns, owner, serial, refresh, retry, expiry, irefresh = zonetuple
+            if authns[-1] == '.':
+                authns = authns[:-1]
+            if owner[-1] == '.':
+                owner = owner[:-1]
+            value = "rname=%s.%s mname=%s.%s %d,%d,%d,%d,%d" % (
+                authns, domain, owner, domain,
+                int(serial), int(refresh), int(retry),
+                int(expiry), int(irefresh)
+            )
+        elif rdtype in ['TXT', 'CNAME']:
+            if value[-1] == ".":
+                value = value[:-1]
+        elif rdtype == 'NS' and label != "@":
+            nameserver = str(record)
+            if value[-1] == ".":
+                value = value[:-1]
+            else:
+                value = "%s.%s" % (value, zonename)
+            if nameserver[-1] == ".":
+                dnsresolver = nameserver[:-1]
+            else:
+                dnsresolver = "%s.%s" % (nameserver, domain)
+            value = 'address@hidden' % (str(label), zonename, dnsresolver)
+            rdtype = 'GNS2DNS'
+            logging.info("transformed %s record to GNS2DNS format", rdtype)
+        elif rdtype == 'MX':
+            priority, mailserver = str(value).split(' ')
+            if mailserver[-1] == ".":
+                mailserver = mailserver[:-1]
+            value = '%s,%s' % (priority, mailserver)
+            logging.info("transformed %s record to GNS format", rdtype)
+        # TODO add SRV records
+        #elif rdtype == 'SRV':
+        #    GNSMigrator.add_srv_record_to_gns(record, zonename)
+        else:
+            logging.info("Did not transform record of type: %s", rdtype)
+        return (rdtype, value)
+
     @staticmethod
     def add_record_to_gns(record, zonename, domain):
         """
@@ -272,6 +359,8 @@ class GNSMigrator():
             else:
                 logging.warning("Unsupported record type: %s", rtype_str)
 
+
+
     @staticmethod
     def get_zone_serial(zonename):
         """
diff --git a/gnsmigrator/test/test_gnunet_multiple.sh 
b/gnsmigrator/test/test_gnunet_multiple.sh
deleted file mode 100644
index 8d78567..0000000
--- a/gnsmigrator/test/test_gnunet_multiple.sh
+++ /dev/null
@@ -1,99 +0,0 @@
-#!/bin/bash
-
-# Check for required packages
-if ! [ -x "$(command -v gnunet-namestore)" ]; then
-    echo 'bind/named is not installed' >&2
-    exit 1
-fi
-
-# Check if gnunet is running
-gnunet-arm -I 2&>1 /dev/null
-ret=$?
-if [ 0 -ne $ret ]; then
-    echo 'gnunet services are not running'
-    exit 1
-fi
-
-## GNUNET part
-# Check if identity exists and delets and readds it to get rid of entries in 
zone
-gnunet-identity -d | grep randomtestingid 2>&1 /dev/null
-ret=$?
-
-if [ 0 -ne $ret ]; then
-    gnunet-identity -D randomtestingid
-    gnunet-identity -C randomtestingid
-fi
-
-function minimize_ttl {
-    ttl=10000000
-    arr=$1
-    # parse each element and update ttl to smallest one
-    for i in "address@hidden"
-    do
-        currttl=$(echo -n "$i" | cut -d' ' -f1)
-        if [ "$currttl"  -lt "$ttl" ]
-        then
-            ttl=$currttl
-        fi
-
-    done
-    echo "$ttl"
-}
-
-function get_record_type {
-    arr=$1
-    typ=$(echo -n "${arr[0]}" | cut -d' ' -f2)
-    echo "$typ"
-}
-
-function get_value {
-    arr=$1
-    val=$(echo -n "${arr[0]}" | cut -d' ' -f4)
-    echo "$val"
-}
-
-function testing {
-    label=$1
-    records=$2
-    recordstring=""
-    typ=$(get_record_type "address@hidden")
-    for i in "address@hidden"
-    do
-        recordstring+="-R $i"
-    done
-    #echo "$recordstring"
-    gnunet-namestore -z randomtestingid -n "$label" "$recordstring" 2>&1  
/dev/null
-    if [ 0 -ne $ret ]; then
-        echo "failed to add record $label: $recordstring"
-    fi
-    gnunet-gns -t "$typ" -u foo2.randomtestingid 2>&1 /dev/null
-    if [ 0 -ne $ret ]; then
-        echo "record $label could not be found"
-    fi
-}
-
-# TEST CASES
-# 1
-echo "Testing adding of single A record with -R"
-declare -a arr=('1200 A n 127.0.0.1')
-testing test1 "address@hidden"
-# 2
-echo "Testing adding of multiple A records with -R"
-declare -a arr=('1200 A n 127.0.0.1' '2400 A n 127.0.0.2')
-testing test2 "address@hidden"
-# 3
-echo "Testing adding of multiple different records with -R"
-declare -a arr=('1200 A n 127.0.0.1' '2400 AAAA n 2002::')
-testing test3 "address@hidden"
-# 4
-echo "Testing adding of single GNS2DNS record with -R"
-declare -a arr=('86400 GNS2DNS n address@hidden')
-testing test4 "address@hidden"
-# 5
-echo "Testing adding of single GNS2DNS shadow record with -R"
-declare -a arr=('86409 GNS2DNS s address@hidden')
-testing test5 "address@hidden"
-# 6
-echo "Testing adding of multiple GNS2DNS record with -R"
-declare -a arr=('86400 GNS2DNS s address@hidden' '86400 GNS2DNS s 
address@hidden')
-testing test6 "address@hidden"
diff --git a/gnsmigrator/test/test_namestore_multiple.sh 
b/gnsmigrator/test/test_namestore_multiple.sh
new file mode 100644
index 0000000..f238de1
--- /dev/null
+++ b/gnsmigrator/test/test_namestore_multiple.sh
@@ -0,0 +1,108 @@
+#!/bin/bash
+# This file is in the public domain.
+#trap "gnunet-arm -e -c test_gns_lookup.conf" SIGINT
+#
+#LOCATION=$(command -v gnunet-config)
+#if [ -z "$LOCATION" ]
+#then
+#    LOCATION="gnunet-config"
+#fi
+#$LOCATION --version 1> /dev/null
+#if test $? != 0
+#then
+#    echo "GNUnet command line tools cannot be found, check environmental 
variables PATH and GNUNET_PREFIX"
+#    exit 77
+#fi
+#
+#rm -rf "$(gnunet-config -c test_gns_lookup.conf -s PATHS -o GNUNET_HOME -f)"
+#command -v timeout &> /dev/null && DO_TIMEOUT="timeout 30"
+#
+## VARS
+MYEGO=myego
+#command -v timeout &> /dev/null && DO_TIMEOUT="timeout 15"
+#
+#gnunet-arm -s -c test_gns_lookup.conf
+gnunet-identity -C myego
+
+# HELPERS
+function get_record_type {
+    arr=$1
+    typ=$(echo -n "${arr[0]}" | cut -d' ' -f2)
+    echo "$typ"
+}
+
+function get_value {
+    arr=$1
+    val=$(echo -n "${arr[0]}" | cut -d' ' -f4-)
+    echo "$val"
+}
+
+function testing {
+    label=$1
+    records=$2
+    recordstring=""
+    for i in "address@hidden"
+    do
+        recordstring+="-R $i"
+    done
+    gnunet-namestore -z "$MYEGO" -n "$label" "$recordstring"
+    if [ 0 -ne $? ]; then
+        echo "failed to add record $label: $recordstring"
+    fi
+    ret=$(gnunet-namestore -D -z "$MYEGO" -n "$label")
+    for i in "address@hidden"
+    do
+        value=$(get_value "$i")
+        if [[ $ret == *"$value"* ]]; then
+            echo "Value(s) added successfully!"
+            return 0
+        else
+            exit 1
+        fi
+    done
+}
+
+# TEST CASES
+# 1
+echo "Testing adding of single A record with -R"
+declare -a arr=('1200 A n 127.0.0.1')
+testing test1 "address@hidden"
+# 2
+echo "Testing adding of multiple A records with -R"
+declare -a arr=('1200 A n 127.0.0.1' '2400 A n 127.0.0.2')
+testing test2 "address@hidden"
+# 3
+echo "Testing adding of multiple different records with -R"
+declare -a arr=('1200 A n 127.0.0.1' '2400 AAAA n 2002::')
+testing test3 "address@hidden"
+# 4
+echo "Testing adding of single GNS2DNS record with -R"
+declare -a arr=('86400 GNS2DNS n address@hidden')
+testing test4 "address@hidden"
+# 5
+echo "Testing adding of single GNS2DNS shadow record with -R"
+declare -a arr=('86409 GNS2DNS s address@hidden')
+testing test5 "address@hidden"
+# 6
+echo "Testing adding of multiple GNS2DNS record with -R"
+declare -a arr=('1 GNS2DNS n address@hidden' '3600 GNS2DNS s address@hidden')
+testing test6 "address@hidden"
+# 7
+echo "Testing adding MX record with -R"
+declare -a arr=('3600 MX n 10,mail')
+testing test7 "address@hidden"
+# 8
+echo "Testing adding TXT record with -R"
+declare -a arr=('3600 TXT n Pretty_Unicorns')
+testing test8 "address@hidden"
+# 8
+echo "Testing adding TXT record with -R"
+declare -a arr=('3600 SRV n _autodiscover_old._tcp.bfh.ch.')
+testing test8 "address@hidden"
+# 9
+echo "Testing adding many A records with -R"
+declare -a arr=('3600 A n 127.0.0.1' '3600 A n 127.0.0.2' '3600 A n 127.0.0.3' 
'3600 A n 127.0.0.4' '3600 A n 127.0.0.5')
+testing test9 "address@hidden"
+
+# CLEANUP
+gnunet-identity -D "$MYEGO"

-- 
To stop receiving notification emails like this one, please contact
address@hidden