[Devel,RHEL7,COMMIT] ms/vxlan: fix use-after-free on deletion

Submitted by Konstantin Khorenko on June 20, 2017, 5:02 p.m.

Details

Message ID 201706201702.v5KH27Nx003754@finist_cl7.x64_64.work.ct
State New
Series "ms/vxlan: fix use-after-free on deletion"
Headers show

Commit Message

Konstantin Khorenko June 20, 2017, 5:02 p.m. 
The commit is pushed to "branch-rh7-3.10.0-514.16.1.vz7.32.x-ovz" and will appear at https://src.openvz.org/scm/ovz/vzkernel.git
after rh7-3.10.0-514.16.1.vz7.32.10
------>
commit 75ef002ea6ccedf78482e2e0972cdee8edad2da5
Author: Mark Bloch <markb@mellanox.com>
Date:   Tue Jun 20 21:02:07 2017 +0400

    ms/vxlan: fix use-after-free on deletion
    
    Reproduced on debug kernel with KASAN:
    
    [20903.934764] BUG: KASan: use after free in vxlan_dellink+0x5b2/0x5f0 [vxlan] at addr ffff880014180018
    [20903.939483] Write of size 8 by task dockerd/392675
    [20903.943568] page:ffffea0000506000 count:0 mapcount:-127 mapping:          (null) index:0x0
    [20903.948279] page flags: 0x1fffff00000000()
    [20903.952357] page dumped because: kasan: bad access detected
    [20903.956635] CPU: 2 PID: 392675 Comm: dockerd ve: 54366ea9-3f08-446c-a794-02b8a9176a72 Not tainted 3.10.0-514.16.1.vz7.32.7.debug #1 32.7
    [20903.965876] Hardware name: Virtuozzo KVM, BIOS 1.9.1-5.3.2.vz7.6 04/01/2014
    [20903.970907]  ffff880014180018 00000000f359ad59 ffff8801a63d7438 ffffffff8255cce0
    [20903.976184]  ffff8801a63d74b8 ffffffff8162352f ffff8801a63d74a0 ffffffff8133cc11
    [20903.981393]  0000000000000297 0000000000000297 ffffffffa0f4a4f2 ffff8801bbf14ce8
    [20903.986748] Call Trace:
    [20903.991106]  [<ffffffff8255cce0>] dump_stack+0x1e/0x20
    [20903.995961]  [<ffffffff8162352f>] kasan_report+0x4ff/0x540
    [20904.000804]  [<ffffffff8133cc11>] ? lock_acquired+0x331/0xfd0
    [20904.005656]  [<ffffffffa0f4a4f2>] ? vxlan_dellink+0x5b2/0x5f0 [vxlan]
    [20904.010856]  [<ffffffff8162368c>] __asan_report_store8_noabort+0x1c/0x20
    [20904.015990]  [<ffffffffa0f4a4f2>] vxlan_dellink+0x5b2/0x5f0 [vxlan]
    [20904.021161]  [<ffffffffa0f49fd1>] ? vxlan_dellink+0x91/0x5f0 [vxlan]
    [20904.026385]  [<ffffffffa0f49f40>] ? vxlan_fdb_dump+0x600/0x600 [vxlan]
    [20904.031598]  [<ffffffff8217f9b8>] rtnl_delete_link+0xd8/0x150
    [20904.036637]  [<ffffffff8217f8e0>] ? __rtnl_link_unregister+0x260/0x260
    [20904.041786]  [<ffffffff81a4b8e2>] ? nla_parse+0x1f2/0x290
    [20904.046826]  [<ffffffff82182d54>] rtnl_dellink+0x244/0x300
    [20904.051865]  [<ffffffff82182b10>] ? rtnl_dump_all+0x3a0/0x3a0
    [20904.057010]  [<ffffffff811b992d>] ? ns_capable+0xbd/0x100
    [20904.062182]  [<ffffffff821ea0d2>] ? __netlink_ns_capable+0xe2/0x130
    [20904.067412]  [<ffffffff82182b10>] ? rtnl_dump_all+0x3a0/0x3a0
    [20904.072582]  [<ffffffff8218314a>] rtnetlink_rcv_msg+0x27a/0x770
    [20904.077802]  [<ffffffff82182ed0>] ? rtnetlink_rcv+0x40/0x40
    [20904.082916]  [<ffffffff82182eb0>] ? rtnetlink_rcv+0x20/0x40
    [20904.087978]  [<ffffffff821f2190>] ? netlink_connect+0x500/0x500
    [20904.093052]  [<ffffffff825660f0>] ? mutex_lock_interruptible_nested+0xfc0/0xfc0
    [20904.098449]  [<ffffffff821fca15>] netlink_rcv_skb+0x2a5/0x3a0
    [20904.103574]  [<ffffffff82182ed0>] ? rtnetlink_rcv+0x40/0x40
    [20904.108663]  [<ffffffff82182ebf>] rtnetlink_rcv+0x2f/0x40
    [20904.113659]  [<ffffffff821fa630>] netlink_unicast+0x420/0x4e0
    [20904.118731]  [<ffffffff821fa210>] ? netlink_attachskb+0x680/0x680
    [20904.123814]  [<ffffffff81622844>] ? kasan_check_write+0x14/0x20
    [20904.128912]  [<ffffffff81a03a3b>] ? memcpy_fromiovec+0x11b/0x170
    [20904.133906]  [<ffffffff821fb288>] netlink_sendmsg+0xb98/0x1bd0
    [20904.138832]  [<ffffffff81345c79>] ? lock_acquire+0x169/0x460
    [20904.143832]  [<ffffffff81282043>] ? rcu_read_lock+0x43/0xd0
    [20904.148765]  [<ffffffff821fa6f0>] ? netlink_unicast+0x4e0/0x4e0
    [20904.153705]  [<ffffffff81342d58>] ? __lock_acquire+0x6f8/0x2b40
    [20904.158676]  [<ffffffff820df2dd>] sock_sendmsg+0x13d/0x1e0
    [20904.163561]  [<ffffffff81342660>] ? debug_check_no_locks_freed+0x320/0x320
    [20904.168663]  [<ffffffff820df1a0>] ? sockfd_lookup+0x160/0x160
    [20904.173451]  [<ffffffff81221326>] ? __wake_up_bit+0xc6/0x110
    [20904.178162]  [<ffffffff81221260>] ? wait_woken+0x1d0/0x1d0
    [20904.182809]  [<ffffffff820e443e>] SYSC_sendto+0x22e/0x380
    [20904.187373]  [<ffffffff820e4210>] ? SYSC_connect+0x2e0/0x2e0
    [20904.191979]  [<ffffffff8168e79e>] ? fput+0x16e/0x1a0
    [20904.196333]  [<ffffffff815a3a10>] ? copy_page_range+0x980/0x980
    [20904.200689]  [<ffffffff81345c79>] ? lock_acquire+0x169/0x460
    [20904.205067]  [<ffffffff8122faa4>] ? up_read+0x24/0x40
    [20904.209250]  [<ffffffff8257e469>] ? __do_page_fault+0x1e9/0xb30
    [20904.213532]  [<ffffffff81348efa>] ? lockdep_sys_exit+0x4a/0x100
    [20904.217749]  [<ffffffff819f80c4>] ? lockdep_sys_exit_thunk+0x16/0x18
    [20904.221985]  [<ffffffff81333d65>] ? trace_hardirqs_off_caller+0x1e5/0x2c0
    [20904.226264]  [<ffffffff820e8625>] SyS_sendto+0x45/0x60
    [20904.230307]  [<ffffffff8258f149>] system_call_fastpath+0x16/0x1b
    [20904.234467] Memory state around the buggy address:
    [20904.238463]  ffff88001417ff00: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc
    [20904.242913]  ffff88001417ff80: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc
    [20904.247317] >ffff880014180000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff
    [20904.251683]                             ^
    [20904.255539]  ffff880014180080: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff
    [20904.260011]  ffff880014180100: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff
    [20904.264473] ==================================================================
    
    In vxlan_dellink->hlist_del_rcu->__hlist_del: vxlan->hlist->pprev points to freed
    memory (0xb88 is offset of pprev relative to struct net_device):
    
    0xffffffffa0f4a17e <vxlan_dellink+574>: mov    0xb88(%r12),%r13
    /usr/src/debug/kernel-3.10.0-514.16.1.el7/linux-3.10.0-514.16.1.vz7.32.7/drivers/net/vxlan.c: 3105
    0xffffffffa0f4a186 <vxlan_dellink+582>: test   %r13,%r13
    0xffffffffa0f4a189 <vxlan_dellink+585>: je     0xffffffffa0f4a23d <vxlan_dellink+765>
    /usr/src/debug/kernel-3.10.0-514.16.1.el7/linux-3.10.0-514.16.1.vz7.32.7/drivers/net/vxlan.c: 3106
    0xffffffffa0f4a18f <vxlan_dellink+591>: callq  0xffffffff813e20a0 <__sanitizer_cov_trace_pc>
    /usr/src/debug/kernel-3.10.0-514.16.1.el7/linux-3.10.0-514.16.1.vz7.32.7/include/linux/list.h: 626
    0xffffffffa0f4a194 <vxlan_dellink+596>: lea    0xb80(%r12),%rdi
    0xffffffffa0f4a19c <vxlan_dellink+604>: movabs $0xdffffc0000000000,%rax
    0xffffffffa0f4a1a6 <vxlan_dellink+614>: mov    %rdi,%rdx
    0xffffffffa0f4a1a9 <vxlan_dellink+617>: shr    $0x3,%rdx
    0xffffffffa0f4a1ad <vxlan_dellink+621>: cmpb   $0x0,(%rdx,%rax,1)
    0xffffffffa0f4a1b1 <vxlan_dellink+625>: jne    0xffffffffa0f4a4f7 <vxlan_dellink+1463>
    /usr/src/debug/kernel-3.10.0-514.16.1.el7/linux-3.10.0-514.16.1.vz7.32.7/include/linux/list.h: 628
    0xffffffffa0f4a1b7 <vxlan_dellink+631>: mov    %r13,%rdx
    0xffffffffa0f4a1ba <vxlan_dellink+634>: movabs $0xdffffc0000000000,%rax
    /usr/src/debug/kernel-3.10.0-514.16.1.el7/linux-3.10.0-514.16.1.vz7.32.7/include/linux/list.h: 626
    0xffffffffa0f4a1c4 <vxlan_dellink+644>: mov    0xb80(%r12),%r15
    /usr/src/debug/kernel-3.10.0-514.16.1.el7/linux-3.10.0-514.16.1.vz7.32.7/include/linux/list.h: 628
    0xffffffffa0f4a1cc <vxlan_dellink+652>: shr    $0x3,%rdx
    0xffffffffa0f4a1d0 <vxlan_dellink+656>: cmpb   $0x0,(%rdx,%rax,1)
    0xffffffffa0f4a1d4 <vxlan_dellink+660>: jne    0xffffffffa0f4a4ea <vxlan_dellink+1450>
    
    0xffffffffa0f4a4ea <vxlan_dellink+1450>:        mov    %r13,%rdi
    0xffffffffa0f4a4ed <vxlan_dellink+1453>:        callq  0xffffffff81623670 <__asan_report_store8_noabort>
    0xffffffffa0f4a4f2 <vxlan_dellink+1458>:        jmpq   0xffffffffa0f4a1da <vxlan_dellink+666>
    
    https://jira.sw.ru/browse/PSBM-67263
    
    Adding a vxlan interface to a socket isn't symmetrical, while adding
    is done in vxlan_open() the deletion is done in vxlan_dellink().
    This can cause a use-after-free error when we close the vxlan
    interface before deleting it.
    
    We add vxlan_vs_del_dev() to match vxlan_vs_add_dev() and call
    it from vxlan_stop() to match the call from vxlan_open().
    
    Fixes: 56ef9c909b40 ("vxlan: Move socket initialization to within rtnl scope")
    Acked-by: Jiri Benc <jbenc@redhat.com>
    Tested-by: Roi Dayan <roid@mellanox.com>
    Signed-off-by: Mark Bloch <markb@mellanox.com>
    Acked-by: Roopa Prabhu <roopa@cumulusnetworks.com>
    Signed-off-by: David S. Miller <davem@davemloft.net>
    Signed-off-by: Pavel Tikhomirov <ptikhomirov@virtuozzo.com>
---
 drivers/net/vxlan.c | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

Patch hide | download patch | download mbox 

diff --git a/drivers/net/vxlan.c b/drivers/net/vxlan.c
index 5670d15..ec3a83b 100644
--- a/drivers/net/vxlan.c
+++ b/drivers/net/vxlan.c
@@ -77,6 +77,8 @@  static const u8 all_zeros_mac[ETH_ALEN + 2];
 
 static int vxlan_sock_add(struct vxlan_dev *vxlan);
 
+static void vxlan_vs_del_dev(struct vxlan_dev *vxlan);
+
 /* per-network namespace private data for this module */
 struct vxlan_net {
 	struct list_head  vxlan_list;
@@ -1065,6 +1067,8 @@  static void vxlan_sock_release(struct vxlan_dev *vxlan)
 
 	synchronize_net();
 
+	vxlan_vs_del_dev(vxlan);
+
 	if (ipv4) {
 		udp_tunnel_sock_release(vxlan->vn4_sock->sock);
 		kfree(vxlan->vn4_sock);
@@ -2307,6 +2311,15 @@  static void vxlan_cleanup(unsigned long arg)
 	mod_timer(&vxlan->age_timer, next_timer);
 }
 
+static void vxlan_vs_del_dev(struct vxlan_dev *vxlan)
+{
+	struct vxlan_net *vn = net_generic(vxlan->net, vxlan_net_id);
+
+	spin_lock(&vn->sock_lock);
+	hlist_del_init_rcu(&vxlan->hlist);
+	spin_unlock(&vn->sock_lock);
+}
+
 static void vxlan_vs_add_dev(struct vxlan_sock *vs, struct vxlan_dev *vxlan)
 {
 	struct vxlan_net *vn = net_generic(vxlan->net, vxlan_net_id);
@@ -3099,12 +3112,6 @@  static int vxlan_newlink(struct net *src_net, struct net_device *dev,
 static void vxlan_dellink(struct net_device *dev, struct list_head *head)
 {
 	struct vxlan_dev *vxlan = netdev_priv(dev);
-	struct vxlan_net *vn = net_generic(vxlan->net, vxlan_net_id);
-
-	spin_lock(&vn->sock_lock);
-	if (!hlist_unhashed(&vxlan->hlist))
-		hlist_del_rcu(&vxlan->hlist);
-	spin_unlock(&vn->sock_lock);
 
 	gro_cells_destroy(&vxlan->gro_cells);
 	list_del(&vxlan->next);