Add --checksum support to disk-image-create

We have been using DIB_CHECKSUM for a few weeks now, and things
appear to be working well. Now switch to passing --checksum via
disk-iamge-create and properly clean up the checksum files when we
delete our images.

Change-Id: Ia849aa179750648be13455d1f38a6a01423bd8c0
Signed-off-by: Paul Belanger <pabelanger@redhat.com>

nodepool/builder.py

@@ -52,7 +52,9 @@ class DibImageFile(object):
         self.image_id = image_id
         self.extension = extension
         self.md5 = None
+        self.md5_file = None
         self.sha256 = None
+        self.sha256_file = None
 
     @staticmethod
     def from_path(path):
@@ -83,21 +85,24 @@ class DibImageFile(object):
                 )
             my_path += '.' + self.extension
 
-        md5 = self._checksum(my_path, 'md5')
+        md5_path = '%s.%s' % (my_path, 'md5')
+        md5 = self._checksum(md5_path)
         if md5:
+            self.md5_file = md5_path
             self.md5 = md5[0:32]
 
-        sha256 = self._checksum(my_path, 'sha256')
+        sha256_path = '%s.%s' % (my_path, 'sha256')
+        sha256 = self._checksum(sha256_path)
         if sha256:
+            self.sha256_file = sha256_path
             self.sha256 = sha256[0:64]
 
         return my_path
 
-    def _checksum(self, filename, hash_name):
-        checksum = '%s.%s' % (filename, hash_name)
-        if not os.path.isfile(checksum):
+    def _checksum(self, filename):
+        if not os.path.isfile(filename):
             return None
-        with open(checksum, 'r') as f:
+        with open(filename, 'r') as f:
             data = f.read()
         return data
 
@@ -213,6 +218,16 @@ class CleanupWorker(BaseWorker):
             return True
         return False
 
+    def _removeDibItem(self, filename):
+        if filename is None:
+            return
+        try:
+            os.remove(filename)
+            self.log.info("Removed DIB file %s" % filename)
+        except OSError as e:
+            if e.errno != 2:    # No such file or directory
+                raise e
+
     def _deleteLocalBuild(self, image, build_id):
         '''
         Remove expired image build from local disk.
@@ -236,13 +251,7 @@ class CleanupWorker(BaseWorker):
             if not manifest_dir:
                 path, ext = filename.rsplit('.', 1)
                 manifest_dir = path + ".d"
-
-            try:
-                os.remove(filename)
-                self.log.info("Removed DIB file %s" % filename)
-            except OSError as e:
-                if e.errno != 2:    # No such file or directory
-                    raise e
+            map(self._removeDibItem, [filename, f.md5_file, f.sha256_file])
 
         try:
             shutil.rmtree(manifest_dir)
@@ -647,7 +656,7 @@ class BuildWorker(BaseWorker):
         else:
             dib_cmd = 'disk-image-create'
 
-        cmd = ('%s -x -t %s --no-tmpfs %s -o %s %s' %
+        cmd = ('%s -x -t %s --checksum --no-tmpfs %s -o %s %s' %
                (dib_cmd, img_types, qemu_img_options, filename, img_elements))
 
         log = logging.getLogger("nodepool.image.build.%s" %

nodepool/tests/__init__.py

@@ -16,6 +16,7 @@
 """Common utilities used in testing"""
 
 import errno
+import glob
 import logging
 import os
 import pymysql
@@ -507,6 +508,10 @@ class DBTestCase(BaseTestCase):
             files = builder.DibImageFile.from_image_id(
                 self._config_images_dir.path, base)
             if not files:
+                # Now, check the disk to ensure we didn't leak any files.
+                matches = glob.glob('%s/%s.*' % (self._config_images_dir.path,
+                                                 base))
+                self.assertEqual(matches, [])
                 break
             time.sleep(1)
 

nodepool/tests/fake-image-create

@@ -41,7 +41,7 @@ fi
 outfile=
 outtypes=("qcow2")
 
-TEMP=$(getopt -o xo:t: --long qemu-img-options:,no-tmpfs -- "$@")
+TEMP=$(getopt -o xo:t: --long qemu-img-options:,no-tmpfs,checksum -- "$@")
 if [ $? -ne 0 ]; then
     echo "Invalid option"
     exit 1
@@ -49,6 +49,8 @@ fi
 eval set -- "$TEMP"
 while true ; do
     case "$1" in
+        --checksum)
+            echo " -> set --checksum"; shift 1;;
         --no-tmpfs)
             echo " -> set --no-tmpfs"; shift 1;;
         --qemu-img-options)
@@ -67,6 +69,8 @@ if [ -z "$outfile" ]; then
 else
     for outtype in ${outtypes[@]} ; do
         echo "fake-data" > $outfile.$outtype
+        echo "10da41d43d4bd6d67db763616c18b72f" > $outfile.$outtype.md5
+        echo "0033e9d444953d11689b5fa6a6dba32bf901582f62b0825bc35f593190b1f7dc" > $outfile.$outtype.sha256
     done
 fi
 

requirements.txt

@@ -16,7 +16,7 @@ PrettyTable>=0.6,<0.8
 six>=1.7.0
 os-client-config>=1.2.0
 shade>=1.12.0
-diskimage-builder
+diskimage-builder>=1.21.0
 voluptuous
 kazoo
 Paste