[GNUnet-SVN] r18427 - gnunet-update/gnunet_update

[gnunet]

Author: harsha
Date: 2011-12-04 21:54:48 +0100 (Sun, 04 Dec 2011)
New Revision: 18427

Added:
   gnunet-update/gnunet_update/file.py
Modified:
   gnunet-update/gnunet_update/install.py
   gnunet-update/gnunet_update/metadata.py
   gnunet-update/gnunet_update/package.py
   gnunet-update/gnunet_update/util.py
Log:
Modified classes for BinaryObject and Dependency; added hash collecting and 
checking for all files to be distributed

Added: gnunet-update/gnunet_update/file.py
===================================================================
--- gnunet-update/gnunet_update/file.py                         (rev 0)
+++ gnunet-update/gnunet_update/file.py 2011-12-04 20:54:48 UTC (rev 18427)
@@ -0,0 +1,123 @@
+# This file is part of GNUnet.
+# (C) 2001--2011 Christian Grothoff (and other contributing authors)
+#
+# GNUnet is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published
+# by the Free Software Foundation; either version 2, or (at your
+# option) any later version.
+#
+# GNUnet is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNUnet; see the file COPYING.  If not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+#
+#File:     gnunet_update/file_entity.py
+#Author:   Sree Harsha Totakura
+
+"""File holdin the FileEntity Classes."""
+
+import re
+
+import util
+
+class FileObject(object):
+    """Class for holding data for a file entity"""
+    name = None
+    path = None
+    hash = None
+
+    def __init__(self, name, path=None, hash=None):
+        """
+        Creates a new FileObject object with name and path
+
+        name: The name of the file represented by this FileObject
+        path: The path on the hosts file system where this file is located. If
+            not None then the hash for file is also created
+        """
+        self.name = name
+        self.path = path
+        self.hash = hash
+
+    def __eq__(self, other):
+        """Compares two dependency objects. Returns True if both have same name
+        and path, false otherwise.
+        """
+        return (self.name == other.name)
+
+    def __hash__(self):
+        """Calculates the hash of the instance."""
+        return hash(self.name)
+
+class ExecutableFileObject(FileObject):
+    """Class representing an executable file entity."""
+    deps = None
+
+    def __init__(self, name, path=None, hash=None):
+        """Creates a new ExecutableFileObject."""
+        super(ExecutableFileObject, self).__init__(name, path, hash)
+        self.deps = list()
+
+    def add_dependency(self, dep):
+         """
+         Adds dep object to the list of dependencies.
+         dep: Dependency file entity
+         
+         """
+         self.deps.append(dep)
+    
+    def __eq__(self, other):
+        """The equality relation."""
+        if not (self.name == other.name and self.path == other.path):
+            return False
+        for dep in other.deps:
+            if dep not in self.deps:
+                return False
+        return True
+
+class DependencyFileObject(FileObject):
+    """Class representing a dependency (library object)."""
+    _major = None
+    _minor = None
+    _rel = None
+    _realname = None
+    _regex = 
re.compile(".*\.so\.(?P<major>\d+)(?:\.(?P<minor>\d+))?(?:\.(?P<rev>\d+))?$")
+    
+    def __init__(self, name, path=None, hash=None):
+        """Creates a new DependencyFileObject instance."""
+        super(DependencyFileObject, self).__init__(name, path, hash)
+        match = self._regex.match(name)
+        if match:
+            (self._major, self._minor, self._rel) = match.groups()
+
+    @property
+    def realname(self):
+        """Getter for realname"""
+        return self._realname
+
+    @realname.setter
+    def realname(self, realname_value):
+        """Sets realname and tries to parse major, minor and release numbers"""
+        self._realname = realname_value
+        match = self._regex.match(self._realname)
+        if match:
+            (self._major, self._minor, self._rel) = match.groups()
+    
+    @property
+    def major(self):
+        """Getter for major"""
+        return self._major
+    
+    @property
+    def minor(self):
+        """Getter for minor."""
+        return self._minor
+    
+    @property
+    def rel(self):
+        """Getter for rel."""
+        return self._rel

Modified: gnunet-update/gnunet_update/install.py
===================================================================
--- gnunet-update/gnunet_update/install.py      2011-12-02 07:38:24 UTC (rev 
18426)
+++ gnunet-update/gnunet_update/install.py      2011-12-04 20:54:48 UTC (rev 
18427)
@@ -31,7 +31,6 @@
 
 import util
 from metadata import Metadata
-from dependency import Dependency
 from config import GnunetUpdateConfig
 
 
@@ -107,7 +106,10 @@
                                                    available_libs)
 
     # Extract the package's main software contents
-    util.extract_install_prefix(package_tarfile, install_dir, installed_files)
+    util.extract_install_prefix(metadata.binary_objects + 
metadata.other_objects,
+                                package_tarfile,
+                                install_dir,
+                                installed_files)
     
     # Install the needed dependencies from tarfile
     dep_dir = os.path.join(install_dir, "lib/gnunet-deps")
@@ -126,7 +128,7 @@
         package_tarfile.extract(dep_tarinfo, "./")
         installed_files.append(os.path.join("lib/gnunet-deps/", 
dep_tarinfo.name))
         # Check the hash of the extracted file
-        if util.sha512_hexdigest(dep_tarinfo.name) != dep.hash:
+        if util.hexdigest(dep_tarinfo.name) != dep.hash:
             print (dep_tarinfo.name + 
                    " not matched with the expected hash " + dep.hash)
             print "Given package contains code not signed by trusted packager"
@@ -134,7 +136,7 @@
             package_tarfile.close()
             os.chdir(orig_working_dir)
             shutil.rmtree(install_dir)
-            sys.exit(0)
+            sys.exit(-1)
         
         # Generate symbolic link from dep.name to dep.realname
         # NOTE: Available only on Unix type systems!

Modified: gnunet-update/gnunet_update/metadata.py
===================================================================
--- gnunet-update/gnunet_update/metadata.py     2011-12-02 07:38:24 UTC (rev 
18426)
+++ gnunet-update/gnunet_update/metadata.py     2011-12-04 20:54:48 UTC (rev 
18427)
@@ -22,15 +22,16 @@
 #To handle metadata files.
 
 import tempfile
-from dependency import Dependency, BinaryObject
+from file import FileObject, ExecutableFileObject, DependencyFileObject
 
-class Metadata:
+class Metadata(object):
     """Class for holding metadata information."""
     machine = None
     system = None
     pkey = None
     release = None
     binary_objects = None
+    other_objects = None
     dependencies = None
 
     def __init__(self, machine=None, system=None, pkey=None,
@@ -74,14 +75,17 @@
         #write the metadata body
         writeln_("%%") #section seperator
         for binary_object in self.binary_objects:
-            f.writelines(binary_object.dependency_listlines())
+            # f.writelines(binary_object.dependency_listlines())
+            for dep in binary_object.deps:
+                writeln_(binary_object.name + ";" + dep.name)
         
-        #write the signatures of binary objects
+        #write the signatures of files in the package
         writeln_("%%") #section seperator
-        for binary_object in self.binary_objects:
-            writeln_(binary_object.name + ";" + binary_object.hash)
+        for file_object in self.binary_objects + self.other_objects:
+            writeln_(file_object.name + ";" + file_object.hash)
 
         # write the signatures of dependencies
+        writeln_("%%") #section seperator
         for dep in self.dependencies:
             writeln_(dep.name + ";" + dep.realname + ";" + dep.hash)
 
@@ -124,8 +128,9 @@
 
         # read until next `%%'
         seen_bin_object = None
-        self.binary_objects = list()
+        _binary_objects = dict()
         self.dependencies = dict()
+        self.other_objects = list()
         while True:
             read_line = f.readline()
             if len(read_line) == 0:
@@ -137,34 +142,34 @@
             bin_name = tokens[0]
             dep_name = tokens[1][:-1] # last character will be `\n'
             if seen_bin_object is None or seen_bin_object.name != bin_name:
-                seen_bin_object = BinaryObject(name=bin_name)
-                self.binary_objects.append(seen_bin_object)
-            dep = Dependency(dep_name)
+                seen_bin_object = ExecutableFileObject(name=bin_name)
+                #_binary_objects.append(seen_bin_object)
+                _binary_objects[bin_name] = seen_bin_object
+            dep = DependencyFileObject(dep_name)
+            # try to use the existing dependency object
             if dep not in self.dependencies:
                 self.dependencies[dep] = dep
-            else:
-                # use the one which is already existing
-                seen_bin_object.add_dependency(self.dependencies[dep])
+            seen_bin_object.add_dependency(self.dependencies[dep])
             
         # read until next `%%'
-        # read the hashes from signatures
-        for binary in self.binary_objects:
+        # read the hashes
+        # The lines in this section are hashes of both binary and other objects
+        while True:
             read_line = f.readline()
             if len(read_line) == 0:
                 print "Unrecognized metadata file"
                 error_exit()
+            if "%%" == read_line[:2]:
+                break
             tokens = read_line.split(';')
-            bin_name = tokens[0]
+            file_name = tokens[0]
             hash = tokens[1][:-1] # last character will be `\n'
-            # FIXME: We are reading based on the assumption that the order of
-            # binary objects in the metadata body and signatures is
-            # similar. However this may not be the case according to the
-            # documentation.
-            if binary.name != bin_name:
-                print "Unrecognized file format"
-                error_exit()
-            binary.hash = hash
-
+            if file_name in _binary_objects: 
+                _binary_objects[file_name].hash = hash
+            else:
+                other_object = FileObject(file_name, hash=hash)
+                self.other_objects.append(other_object)
+            
         # read the dependency name, file name(real name) and its hash
         while True:
             read_line = f.readline()
@@ -174,12 +179,13 @@
             dep_name = tokens[0]
             dep_realname = tokens[1]
             dep_hash = tokens[2][:-1] # last character is `\n'
-            dep = Dependency(dep_name)
+            dep = DependencyFileObject(dep_name)
             # dep should already be in self.dependencies
             # if we get a keyerror here, then something went wrong
             dep = self.dependencies[dep] # gets the correct object
             dep.name = dep_name
-            dep.set_realname(dep_realname)
+            dep.realname = dep_realname
             dep.hash = dep_hash
-
         f.close()
+        # Set the binary_objects to the items from _binary_objects dictionary
+        self.binary_objects = _binary_objects.values()

Modified: gnunet-update/gnunet_update/package.py
===================================================================
--- gnunet-update/gnunet_update/package.py      2011-12-02 07:38:24 UTC (rev 
18426)
+++ gnunet-update/gnunet_update/package.py      2011-12-04 20:54:48 UTC (rev 
18427)
@@ -38,7 +38,7 @@
 import platform
 
 import util
-from dependency import Dependency, BinaryObject
+from file import FileObject, ExecutableFileObject, DependencyFileObject
 from metadata import Metadata
 from config import GnunetUpdateConfig
 
@@ -49,6 +49,7 @@
 prefix_given = False
 config_options = list()
 binary_objects = list()
+other_objects = list()
 #dependency cache
 dependencies = dict()
 
@@ -123,18 +124,33 @@
             if (os.path.islink(file_path) and
                 len(os.path.commonprefix([file_path, root])) >= 
                 len(install_dir)):
+                other_objects.append(FileObject(
+                        os.path.join(root[len(install_dir) + 1:],
+                                     file),
+                        file_path,
+                        "-LINK"))
                 continue
             
             proc = subprocess.Popen(["ldd", file_path],
                                     bufsize=-1, stdout=subprocess.PIPE)
             (proc_stdout, proc_stderr) = proc.communicate()
             proc.stdout.close()
+            # FIXME: We will be processing symbolic links twice, once for the
+            # symbolic link and again for its target. Also if the symbolic link
+            # points to something outside of the install directory then it may
+            # cause issues during installation
             if 0 != proc.returncode:
+                other_object = FileObject(os.path.join(root[len(install_dir) + 
1:],
+                                                           file),
+                                          file_path,
+                                          util.hexdigest(file_path))
+                other_objects.append(other_object)
                 continue
-            #create a new BinaryObject instance and collect its dependencies
-            bin_object = BinaryObject(root[len(install_dir) + 1:] + '/' + file,
-                                      file_path)
-            
+
+            #create a new ExecutableFileObject instance and collect its 
dependencies
+            bin_object = ExecutableFileObject(root[len(install_dir) + 1:] + 
'/' + file,
+                                              file_path,
+                                              hash=util.hexdigest(file_path))
             for dep_data in util.parse_ldd_output(proc_stdout):
                 #we cannot find a library without its location
                 if dep_data[-1][0] == '(':
@@ -143,10 +159,13 @@
                 if dep_data[-1].startswith(os.path.abspath(install_dir)):
                     continue
                 #create a new dependency object and add it to the set
-                dep = Dependency(os.path.basename(dep_data[0]), dep_data[-1])
+                dep = DependencyFileObject(os.path.basename(dep_data[0]), 
+                                           dep_data[-1])
                 #check in cache if we already saw this dependency
                 if dep not in dependencies:
                     dependencies[dep] = dep
+                    dep.hash = util.hexdigest(dep_data[-1])
+                    dep.realname = 
os.path.basename(os.path.realpath(dep_data[-1]))
                 else:
                     dep = dependencies[dep]
                 bin_object.add_dependency(dep)
@@ -157,8 +176,8 @@
     """Function to check whether we are collecting dependencies correctly."""
     for bin_object in binary_objects:
         print bin_object.name
-        deps = bin_object.get_dependencies()
-        for dep in deps:
+        
+        for dep in bin_object.deps:
             print "|--" + dep.name + " (" + dep.path + ")"
             
 def run(action):
@@ -182,11 +201,15 @@
                         pkey=config.get('SECURITY', 'PGP_SIGN_KEY'),
                         release="0")                       
     metadata.binary_objects = binary_objects
+    metadata.other_objects = other_objects
     metadata.dependencies = dependencies
 
     #package the installed files
     tar_file = tarfile.open(package_file + ".tgz", 'w:gz')
-    tar_file.add(install_prefix, "install-prefix")
+    #tar_file.add(install_prefix, "install-prefix")
+    for file_object in other_objects + binary_objects:
+        tar_file.add(file_object.path, 
+                     os.path.join("install-prefix", file_object.name))
     
     #generate the metadata file and add it to tar
     metadata_file = metadata.write_to_file(package_file + ".meta")
@@ -255,7 +278,7 @@
                 #check if the user has given an installation prefix
                 if(value.strip()[0:9] == "--prefix="):
                     prefix_given = True
-                    install_prefix = value.strip()[9:] 
+                    install_prefix = os.path.normpath(value.strip()[9:])
             elif option == "-i":
                 action = "extract_deps"  
         if len(args) != 2:

Modified: gnunet-update/gnunet_update/util.py
===================================================================
--- gnunet-update/gnunet_update/util.py 2011-12-02 07:38:24 UTC (rev 18426)
+++ gnunet-update/gnunet_update/util.py 2011-12-04 20:54:48 UTC (rev 18427)
@@ -24,6 +24,7 @@
 from hashlib import sha512
 import gpgme
 import os
+import sys
 import tempfile
 import tarfile
 import shutil
@@ -31,7 +32,7 @@
 import subprocess
 
 from metadata import Metadata
-from dependency import Dependency
+from file import DependencyFileObject
 
 def parse_ldd_output(ldd_output, splitted_input=False):
     """Parses ldd output.
@@ -76,7 +77,7 @@
                parse_ldd_output(ldconfig_output.splitlines()[1:],
                                 splitted_input=True))
 
-def sha512_hexdigest(path):
+def hexdigest(path):
     """Return the sha512 hexdigest of the file at path."""
 
     hash_obj = sha512()
@@ -205,8 +206,11 @@
     else:
         parsed_dep_list = parse_ldconfig_output(_test_input)
     def create_dependency(parsed_dep_line):
-        dep = Dependency(parsed_dep_line[0])
-        dep.set_path(parsed_dep_line[-1]) # sets path and realname
+        dep = DependencyFileObject(parsed_dep_line[0], # Dependency name
+                                   parsed_dep_line[-1]) # Dependency path
+        #  Set the realname - The major, minor, release numbers are auto
+        #  generated from this
+        dep.realname = os.path.basename(os.path.realpath(parsed_dep_line[-1]))
         return dep
     return map(create_dependency, parsed_dep_list)
 
@@ -259,21 +263,41 @@
         ndep_cnt += 1;
     return to_be_installed_deps;
 
-def extract_install_prefix(package_tarfile, install_dir, installed_files=None):
+def extract_install_prefix(objects, 
+                           package_tarfile, 
+                           install_dir, 
+                           installed_files=None):
     """
-    Extracts the tarfile objects contents into install_dir
+    Extracts objects from tarfile into install_dir
 
     package_tarfile: the tarfile object on which extraction is done
     install_dir: the directory where the extracted objects are to be placed
+    objects: list of objects to be extracted
     installed_files: If not None, it should be a list object. It can be used to
         have a list of files extracted into the install_dir
     """
-
-    #FIXME: Security warning! Perhaps we should examin the contents of tarfile
-    #before extracting
-    for member in package_tarfile.getmembers():
-        if member.name.startswith("install-prefix/"):
-            # Remove the `install-prefix' directory
-            member.name = member.name.replace("install-prefix/","",1)
-            package_tarfile.extract(member, install_dir)
-            if installed_files is not None: installed_files.append(member.name)
+    for member_obj in objects:
+        member = None
+        try:
+            member = package_tarfile.getmember("install-prefix/" + 
member_obj.name)
+        except KeyError:
+            print "Broken/Bad package - Cannot find " + member_obj.name
+            sys.exit(-1)
+        
+        member.name = member.name.replace("install-prefix/","",1)
+        package_tarfile.extract(member, install_dir)
+        # Check the hash of the extracted file if it is not a symbolic link
+        if not member.issym():
+            if hexdigest(os.path.join(install_dir, 
+                                      member.name)) != member_obj.hash:
+                print (member_obj.name + 
+                       " not matched with the expected hash " + 
member_obj.hash)
+                print "Given package contains code not signed by trusted 
packager"
+                print "Installation failed due to security reasons."
+                sys.exit(-1) 
+        # If the file is a symbolic link, then its hash should be -LINK
+        elif member_obj.hash != "-LINK":
+            print "Malicious code detected. Stopping installation"
+            print member_obj.name + "<-->" + member_obj.hash
+            sys.exit(-1)
+        if installed_files is not None: installed_files.append(member.name)