Block layer patches for 2.10.0-rc1

-----BEGIN PGP SIGNATURE-----
 
 iQIcBAABAgAGBQJZgKgMAAoJEH8JsnLIjy/W8FQP/i8f6lf3kpmTyqfyG/d9rU+V
 jizaa3uHBmmctQ9Ib0k1woFudSc1Rxt1kXtG3Cj45nYjHXYzWZOA2p3LF0bpRVtX
 keq3yXTLE+Il0ED6WiklKMS+BjpzxlnnHJpqnP3axl2TWZQtlQFIqO/f51RYtrVe
 SOaNdXxqEFQgV3uJIwtPdo38BBZvbIwA99+gCHM1YURkCESUnmy5h8plpnovJtMT
 Ta8sF+LXAjHtusYzfghNJ/p0Rpg3DkUmvHgo5QTA1F54AAPa3TePewqxssHaNK1T
 cDfDocvq9/gEMCMca2uyWNFxeqDZzaExoNUo2EVYoCPXKWr+vPaEgs9+RjMv6XUw
 d6lXZ6F0Rpm1zdtYQI8R1/ZpYcf29oo13q6fW0EEDx8y+9LMRKZP7pRtWA+MpNzT
 9iRMVm3y0G4FOaoWD9W9cMVfD9aJknz8j3pggIY8nUhvh7BqkEmbgoaO230AmDYc
 dVDDmGL8544g0x/v0USqe2ed/XdBkZSScOeKVeRpuS/r2E4UCBhhJNSefxzvn2+p
 GYj+M6HLZ+biyKBkK3gwyk1fT74vOMpOBzysbBpIN8kg9ySDSkFhvj7qE24YJMKT
 6yuWQE7WzPmXMiUz6hpn/m5TSjtVAXvL3BDc2lMz0HW6tHWJ5asv9zJjku+Ze9P4
 FGBPYzwJ585Wxq/Z6y5b
 =DIex
 -----END PGP SIGNATURE-----

Merge remote-tracking branch 'remotes/kevin/tags/for-upstream' into staging

Block layer patches for 2.10.0-rc1

# gpg: Signature made Tue 01 Aug 2017 17:10:52 BST
# gpg:                using RSA key 0x7F09B272C88F2FD6
# gpg: Good signature from "Kevin Wolf <kwolf@redhat.com>"
# Primary key fingerprint: DC3D EB15 9A9A F95D 3D74  56FE 7F09 B272 C88F 2FD6

* remotes/kevin/tags/for-upstream:
  block/qapi: Remove redundant NULL check to silence Coverity
  qemu-iotests/059: Fix leaked image files
  qemu-iotests/063: Fix leaked image
  qemu-iotests/162: Fix leaked temporary files
  qemu-iotests/153: Fix leaked scratch images
  qemu-iotests/141: Fix image cleanup
  qemu-iotests: Remove blkdebug.conf after tests
  qemu-iotests/041: Fix leaked scratch images
  block: fix leaks in bdrv_open_driver()
  block: fix dangling bs->explicit_options in block.c
  iotests: Add test of recent fix to 'qemu-img measure'
  iotests: Check dirty bitmap statistics in 124
  iotests: Redirect stderr to stdout in 186
  iotests: Fix test 156

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

block.c

@@ -1119,20 +1119,19 @@ static int bdrv_open_driver(BlockDriverState *bs, BlockDriver *drv,
         } else {
             error_setg_errno(errp, -ret, "Could not open image");
         }
-        goto free_and_fail;
+        goto open_failed;
     }
 
     ret = refresh_total_sectors(bs, bs->total_sectors);
     if (ret < 0) {
         error_setg_errno(errp, -ret, "Could not refresh total sector count");
-        goto free_and_fail;
+        return ret;
     }
 
     bdrv_refresh_limits(bs, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
-        ret = -EINVAL;
+        return -EINVAL;
-        goto free_and_fail;
     }
 
     assert(bdrv_opt_mem_align(bs) != 0);
@@ -1140,12 +1139,14 @@ static int bdrv_open_driver(BlockDriverState *bs, BlockDriver *drv,
     assert(is_power_of_2(bs->bl.request_alignment));
 
     return 0;
-
+open_failed:
-free_and_fail:
+    bs->drv = NULL;
-    /* FIXME Close bs first if already opened*/
+    if (bs->file != NULL) {
+        bdrv_unref_child(bs, bs->file);
+        bs->file = NULL;
+    }
     g_free(bs->opaque);
     bs->opaque = NULL;
-    bs->drv = NULL;
     return ret;
 }
 
@@ -1166,7 +1167,9 @@ BlockDriverState *bdrv_new_open_driver(BlockDriver *drv, const char *node_name,
     ret = bdrv_open_driver(bs, drv, node_name, bs->options, flags, errp);
     if (ret < 0) {
         QDECREF(bs->explicit_options);
+        bs->explicit_options = NULL;
         QDECREF(bs->options);
+        bs->options = NULL;
         bdrv_unref(bs);
         return NULL;
     }
@@ -2600,14 +2603,12 @@ static BlockDriverState *bdrv_open_inherit(const char *filename,
 
 fail:
     blk_unref(file);
-    if (bs->file != NULL) {
-        bdrv_unref_child(bs, bs->file);
-    }
     QDECREF(snapshot_options);
     QDECREF(bs->explicit_options);
     QDECREF(bs->options);
     QDECREF(options);
     bs->options = NULL;
+    bs->explicit_options = NULL;
     bdrv_unref(bs);
     error_propagate(errp, local_err);
     return NULL;
@@ -3087,6 +3088,7 @@ static void bdrv_close(BlockDriverState *bs)
         QDECREF(bs->options);
         QDECREF(bs->explicit_options);
         bs->options = NULL;
+        bs->explicit_options = NULL;
         QDECREF(bs->full_open_options);
         bs->full_open_options = NULL;
     }

block/qapi.c

@@ -145,8 +145,9 @@ BlockDeviceInfo *bdrv_block_device_info(BlockBackend *blk,
 
         /* Skip automatically inserted nodes that the user isn't aware of for
          * query-block (blk != NULL), but not for query-named-block-nodes */
-        while (blk && bs0 && bs0->drv && bs0->implicit) {
+        while (blk && bs0->drv && bs0->implicit) {
             bs0 = backing_bs(bs0);
+            assert(bs0);
         }
     }
 

tests/qemu-iotests/041

@@ -418,6 +418,7 @@ new_state = "1"
     def tearDown(self):
         self.vm.shutdown()
         os.remove(test_img)
+        os.remove(target_img)
         os.remove(backing_img)
         os.remove(self.blkdebug_file)
 
@@ -568,6 +569,7 @@ new_state = "1"
     def tearDown(self):
         self.vm.shutdown()
         os.remove(test_img)
+        os.remove(target_img)
         os.remove(backing_img)
         os.remove(self.blkdebug_file)
 
@@ -821,7 +823,7 @@ class TestRepairQuorum(iotests.QMPTestCase):
 
     def tearDown(self):
         self.vm.shutdown()
-        for i in self.IMAGES + [ quorum_repair_img ]:
+        for i in self.IMAGES + [ quorum_repair_img, quorum_snapshot_file ]:
             # Do a try/except because the test may have deleted some images
             try:
                 os.remove(i)

tests/qemu-iotests/059

@@ -29,7 +29,8 @@ status=1	# failure is the default!
 
 _cleanup()
 {
-	_cleanup_test_img
+    _cleanup_test_img
+    rm -f "$TEST_IMG.qcow2"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -72,15 +73,18 @@ echo
 echo "=== Testing monolithicFlat creation and opening ==="
 IMGOPTS="subformat=monolithicFlat" _make_test_img 2G
 _img_info
+_cleanup_test_img
 
 echo
 echo "=== Testing monolithicFlat with zeroed_grain ==="
 IMGOPTS="subformat=monolithicFlat,zeroed_grain=on" _make_test_img 2G
+_cleanup_test_img
 
 echo
 echo "=== Testing big twoGbMaxExtentFlat ==="
 IMGOPTS="subformat=twoGbMaxExtentFlat" _make_test_img 1000G
 $QEMU_IMG info $TEST_IMG | _filter_testdir | sed -e 's/cid: [0-9]*/cid: XXXXXXXX/'
+_cleanup_test_img
 
 echo
 echo "=== Testing malformed VMFS extent description line ==="
@@ -114,6 +118,7 @@ echo "=== Testing monolithicFlat with internally generated JSON file name ==="
 IMGOPTS="subformat=monolithicFlat" _make_test_img 64M
 $QEMU_IO -c "open -o driver=$IMGFMT,file.driver=blkdebug,file.image.filename=$TEST_IMG,file.inject-error.0.event=read_aio" 2>&1 \
     | _filter_testdir | _filter_imgfmt
+_cleanup_test_img
 
 echo
 echo "=== Testing version 3 ==="
@@ -123,6 +128,7 @@ for i in {0..99}; do
     $QEMU_IO -r -c "read -P $(( i % 10 + 0x30 )) $(( i * 64 * 1024 * 10 + i * 512 )) 512" $TEST_IMG \
         | _filter_qemu_io
 done
+_cleanup_test_img
 
 echo
 echo "=== Testing 4TB monolithicFlat creation and IO ==="
@@ -130,6 +136,7 @@ IMGOPTS="subformat=monolithicFlat" _make_test_img 4T
 _img_info
 $QEMU_IO -c "write -P 0xa 900G 512" "$TEST_IMG" | _filter_qemu_io
 $QEMU_IO -c "read -v 900G 1024" "$TEST_IMG" | _filter_qemu_io
+_cleanup_test_img
 
 echo
 echo "=== Testing qemu-img map on extents ==="
@@ -139,12 +146,14 @@ for fmt in monolithicSparse twoGbMaxExtentSparse; do
     $QEMU_IO -c "write 2147483136 1k" "$TEST_IMG" | _filter_qemu_io
     $QEMU_IO -c "write 5G 1k" "$TEST_IMG" | _filter_qemu_io
     $QEMU_IMG map "$TEST_IMG" | _filter_testdir
+    _cleanup_test_img
 done
 
 echo
 echo "=== Testing afl image with a very large capacity ==="
 _use_sample_img afl9.vmdk.bz2
 _img_info
+_cleanup_test_img
 
 # success, all done
 echo "*** done"

tests/qemu-iotests/059.out

@@ -2259,8 +2259,8 @@ read 512/512 bytes at offset 64931328
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 === Testing 4TB monolithicFlat creation and IO ===
-Formatting 'TEST_DIR/iotest-version3.IMGFMT', fmt=IMGFMT size=4398046511104 subformat=monolithicFlat
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4398046511104 subformat=monolithicFlat
-image: TEST_DIR/iotest-version3.IMGFMT
+image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
 virtual size: 4.0T (4398046511104 bytes)
 wrote 512/512 bytes at offset 966367641600
@@ -2333,7 +2333,7 @@ read 1024/1024 bytes at offset 966367641600
 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 === Testing qemu-img map on extents ===
-Formatting 'TEST_DIR/iotest-version3.IMGFMT', fmt=IMGFMT size=33285996544 subformat=monolithicSparse
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=33285996544 subformat=monolithicSparse
 wrote 1024/1024 bytes at offset 65024
 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 1024/1024 bytes at offset 2147483136
@@ -2341,10 +2341,10 @@ wrote 1024/1024 bytes at offset 2147483136
 wrote 1024/1024 bytes at offset 5368709120
 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 Offset          Length          Mapped to       File
-0               0x20000         0x3f0000        TEST_DIR/iotest-version3.vmdk
+0               0x20000         0x3f0000        TEST_DIR/t.vmdk
-0x7fff0000      0x20000         0x410000        TEST_DIR/iotest-version3.vmdk
+0x7fff0000      0x20000         0x410000        TEST_DIR/t.vmdk
-0x140000000     0x10000         0x430000        TEST_DIR/iotest-version3.vmdk
+0x140000000     0x10000         0x430000        TEST_DIR/t.vmdk
-Formatting 'TEST_DIR/iotest-version3.IMGFMT', fmt=IMGFMT size=33285996544 subformat=twoGbMaxExtentSparse
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=33285996544 subformat=twoGbMaxExtentSparse
 wrote 1024/1024 bytes at offset 65024
 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 1024/1024 bytes at offset 2147483136
@@ -2352,10 +2352,10 @@ wrote 1024/1024 bytes at offset 2147483136
 wrote 1024/1024 bytes at offset 5368709120
 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 Offset          Length          Mapped to       File
-0               0x20000         0x50000         TEST_DIR/iotest-version3-s001.vmdk
+0               0x20000         0x50000         TEST_DIR/t-s001.vmdk
-0x7fff0000      0x10000         0x70000         TEST_DIR/iotest-version3-s001.vmdk
+0x7fff0000      0x10000         0x70000         TEST_DIR/t-s001.vmdk
-0x80000000      0x10000         0x50000         TEST_DIR/iotest-version3-s002.vmdk
+0x80000000      0x10000         0x50000         TEST_DIR/t-s002.vmdk
-0x140000000     0x10000         0x50000         TEST_DIR/iotest-version3-s003.vmdk
+0x140000000     0x10000         0x50000         TEST_DIR/t-s003.vmdk
 
 === Testing afl image with a very large capacity ===
 qemu-img: Can't get image size 'TEST_DIR/afl9.IMGFMT': File too large

tests/qemu-iotests/063

@@ -31,7 +31,7 @@ status=1	# failure is the default!
 _cleanup()
 {
 	_cleanup_test_img
-	rm -f "$TEST_IMG.orig" "$TEST_IMG.raw" "$TEST_IMG.raw2"
+	rm -f "$TEST_IMG.orig" "$TEST_IMG.raw1" "$TEST_IMG.raw2"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -91,8 +91,6 @@ if $QEMU_IMG convert -f $IMGFMT -O $IMGFMT -n "$TEST_IMG.orig" "$TEST_IMG" >/dev
     exit 1
 fi
 
-rm -f "$TEST_IMG.orig" "$TEST_IMG.raw" "$TEST_IMG.raw2"
-
 echo "*** done"
 rm -f $seq.full
 status=0

tests/qemu-iotests/074

@@ -32,6 +32,7 @@ _cleanup()
     echo "Cleanup"
     _cleanup_test_img
     rm "${TEST_IMG2}"
+    rm -f "$TEST_DIR/blkdebug.conf"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 

tests/qemu-iotests/124

@@ -336,7 +336,12 @@ class TestIncrementalBackup(TestIncrementalBackupBase):
                            (('0xab', 0, 512),
                             ('0xfe', '16M', '256k'),
                             ('0x64', '32736k', '64k')))
-
+        # Check the dirty bitmap stats
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/dirty-bitmaps[0]/name', 'bitmap0')
+        self.assert_qmp(result, 'return[0]/dirty-bitmaps[0]/count', 458752)
+        self.assert_qmp(result, 'return[0]/dirty-bitmaps[0]/granularity', 65536)
+        self.assert_qmp(result, 'return[0]/dirty-bitmaps[0]/status', 'active')
 
         # Prepare a cluster_size=128k backup target without a backing file.
         (target, _) = bitmap0.new_target()

tests/qemu-iotests/141

@@ -31,7 +31,7 @@ _cleanup()
 {
     _cleanup_qemu
     _cleanup_test_img
-    rm -f "$TEST_DIR/{b,m,o}.$IMGFMT"
+    rm -f "$TEST_DIR"/{b,m,o}.$IMGFMT
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 

tests/qemu-iotests/153

@@ -35,6 +35,7 @@ _cleanup()
     rm -f "${TEST_IMG}.convert"
     rm -f "${TEST_IMG}.a"
     rm -f "${TEST_IMG}.b"
+    rm -f "${TEST_IMG}.c"
     rm -f "${TEST_IMG}.lnk"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15

tests/qemu-iotests/156

@@ -38,7 +38,7 @@ status=1	# failure is the default!
 _cleanup()
 {
     _cleanup_qemu
-    rm -f "$TEST_IMG{,.target}{,.backing,.overlay}"
+    rm -f "$TEST_IMG"{,.target}{,.backing,.overlay}
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -83,7 +83,7 @@ _send_qemu_cmd $QEMU_HANDLE \
     'return'
 
 # Create target image
-TEST_IMG="$TEST_IMG.target.overlay" _make_test_img -b "$TEST_IMG.target" 1M
+TEST_IMG="$TEST_IMG.target.overlay" _make_test_img -u -b "$TEST_IMG.target" 1M
 
 # Mirror snapshot
 _send_qemu_cmd $QEMU_HANDLE \

tests/qemu-iotests/162

@@ -28,6 +28,13 @@ echo "QA output created by $seq"
 here="$PWD"
 status=1	# failure is the default!
 
+_cleanup()
+{
+    rm -f "${TEST_DIR}/qemu-nbd.pid"
+    rm -f 42
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
 # get standard environment, filters and checks
 . ./common.rc
 . ./common.filter

tests/qemu-iotests/179

@@ -30,6 +30,7 @@ status=1	# failure is the default!
 _cleanup()
 {
 	_cleanup_test_img
+    rm -f "$TEST_DIR/blkdebug.conf"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 

tests/qemu-iotests/186

@@ -56,7 +56,7 @@ function do_run_qemu()
             done
         fi
         echo quit
-    ) | $QEMU -S -nodefaults -display none -device virtio-scsi-pci -monitor stdio "$@"
+    ) | $QEMU -S -nodefaults -display none -device virtio-scsi-pci -monitor stdio "$@" 2>&1
     echo
 }
 

tests/qemu-iotests/186.out

@@ -442,28 +442,28 @@ ide0-cd0 (NODE_NAME): null-co:// (null-co, read-only)
     Cache mode:       writeback
 (qemu) quit
 
-qemu-system-x86_64: -drive if=scsi,driver=null-co: warning: bus=0,unit=0 is deprecated with this machine type
 Testing: -drive if=scsi,driver=null-co
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) info block
+(qemu) QEMU_PROG: -drive if=scsi,driver=null-co: warning: bus=0,unit=0 is deprecated with this machine type
+info block
 scsi0-hd0 (NODE_NAME): null-co:// (null-co)
     Attached to:      /machine/unattached/device[27]/scsi.0/legacy[0]
     Cache mode:       writeback
 (qemu) quit
 
-qemu-system-x86_64: -drive if=scsi,media=cdrom: warning: bus=0,unit=0 is deprecated with this machine type
 Testing: -drive if=scsi,media=cdrom
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) info block
+(qemu) QEMU_PROG: -drive if=scsi,media=cdrom: warning: bus=0,unit=0 is deprecated with this machine type
+info block
 scsi0-cd0: [not inserted]
     Attached to:      /machine/unattached/device[27]/scsi.0/legacy[0]
     Removable device: not locked, tray closed
 (qemu) quit
 
-qemu-system-x86_64: -drive if=scsi,driver=null-co,media=cdrom: warning: bus=0,unit=0 is deprecated with this machine type
 Testing: -drive if=scsi,driver=null-co,media=cdrom
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) info block
+(qemu) QEMU_PROG: -drive if=scsi,driver=null-co,media=cdrom: warning: bus=0,unit=0 is deprecated with this machine type
+info block
 scsi0-cd0 (NODE_NAME): null-co:// (null-co, read-only)
     Attached to:      /machine/unattached/device[27]/scsi.0/legacy[0]
     Removable device: not locked, tray closed

tests/qemu-iotests/190

@@ -0,0 +1,59 @@
+#!/bin/bash
+#
+# qemu-img measure sub-command tests on huge qcow2 files
+#
+# Copyright (C) 2017 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=eblake@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+status=1    # failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+    rm -f "$TEST_IMG.converted"
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.pattern
+
+# See 178 for more extensive tests across more formats
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+echo "== Huge file =="
+echo
+
+IMGOPTS='cluster_size=2M' _make_test_img 2T
+
+$QEMU_IMG measure -O raw -f qcow2 "$TEST_IMG"
+$QEMU_IMG measure -O qcow2 -o cluster_size=64k -f qcow2 "$TEST_IMG"
+$QEMU_IMG measure -O qcow2 -o cluster_size=2M -f qcow2 "$TEST_IMG"
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0

tests/qemu-iotests/190.out

@@ -0,0 +1,11 @@
+QA output created by 190
+== Huge file ==
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2199023255552
+required size: 2199023255552
+fully allocated size: 2199023255552
+required size: 335806464
+fully allocated size: 2199359062016
+required size: 18874368
+fully allocated size: 2199042129920
+*** done

tests/qemu-iotests/common.rc

@@ -92,6 +92,7 @@ else
         TEST_IMG=$IMGPROTO:$TEST_DIR/t.$IMGFMT
     fi
 fi
+ORIG_TEST_IMG="$TEST_IMG"
 
 _optstr_add()
 {
@@ -228,6 +229,8 @@ _cleanup_test_img()
             if [ -n "$SAMPLE_IMG_FILE" ]
             then
                 rm -f "$TEST_DIR/$SAMPLE_IMG_FILE"
+                SAMPLE_IMG_FILE=
+                TEST_IMG="$ORIG_TEST_IMG"
             fi
             ;;
 

tests/qemu-iotests/group

@@ -184,3 +184,4 @@
 186 rw auto
 188 rw auto quick
 189 rw auto
+190 rw auto quick