One of the 'features' of overlayfs is that depending on whether a file
is on the upper or lower dir you get back a different device from stat.
That breaks our lxc_rmdir_onedev.

So at lxc_rmdir_ondev check the device of the directory being deleted.
If it is overlayfs, then skip the device check.

Note this is unrelated to overlayfs snapshots - in those cases when you
delete a container, /var/lib/lxc/$container/ does not actually have an
overlayfs under it.  Rather, to reproduce this you would

sudo mkdir /opt/{lower,upper,workdir}
sudo mount -t overlayfs -o lower=/opt/lower,upper=/opt/upper,workdir=/opt/workdir \
	lxc /var/lib/lxc
sudo lxc-create -t download -n c1 -- -d ubuntu -r trusty -a amd64
sudo lxc-destroy -n c1

Signed-off-by: Serge Hallyn <serge.hallyn@ubuntu.com>
---
 src/lxc/utils.c | 39 ++++++++++++++++++++++++++++++++-------
 1 file changed, 32 insertions(+), 7 deletions(-)

--- a/src/lxc/utils.c
+++ b/src/lxc/utils.c
@@ -29,6 +29,7 @@
 #include <stddef.h>
 #include <string.h>
 #include <sys/types.h>
+#include <sys/vfs.h>
 #include <sys/stat.h>
 #include <sys/mman.h>
 #include <sys/param.h>
@@ -68,8 +69,8 @@
 
 lxc_log_define(lxc_utils, lxc);
 
-static int _recursive_rmdir_onedev(char *dirname, dev_t pdev,
-				   const char *exclude, int level)
+static int _recursive_rmdir(char *dirname, dev_t pdev,
+			    const char *exclude, int level, bool onedev)
 {
 	struct dirent dirent, *direntp;
 	DIR *dir;
@@ -106,7 +107,7 @@ static int _recursive_rmdir_onedev(char
 			if (ret < 0) {
 				switch(errno) {
 				case ENOTEMPTY:
-					INFO("Not deleting snapshots");
+					INFO("Not deleting snapshot %s", pathname);
 					hadexclude = true;
 					break;
 				case ENOTDIR:
@@ -129,14 +130,14 @@ static int _recursive_rmdir_onedev(char
 			failed=1;
 			continue;
 		}
-		if (mystat.st_dev != pdev)
+		if (onedev && mystat.st_dev != pdev)
 			continue;
 		if (S_ISDIR(mystat.st_mode)) {
-			if (_recursive_rmdir_onedev(pathname, pdev, exclude, level+1) < 0)
+			if (_recursive_rmdir(pathname, pdev, exclude, level+1, onedev) < 0)
 				failed=1;
 		} else {
 			if (unlink(pathname) < 0) {
-				ERROR("%s: failed to delete %s", __func__, pathname);
+				SYSERROR("%s: failed to delete %s", __func__, pathname);
 				failed=1;
 			}
 		}
@@ -158,17 +159,41 @@ static int _recursive_rmdir_onedev(char
 	return failed ? -1 : 0;
 }
 
+/* we have two different magic values for overlayfs, yay */
+#define OVERLAYFS_SUPER_MAGIC 0x794c764f
+#define OVERLAY_SUPER_MAGIC 0x794c7630
+/*
+ * In overlayfs, st_dev is unreliable.  so on overlayfs we don't do
+ * the lxc_rmdir_onedev()
+ */
+static bool is_native_overlayfs(const char *path)
+{
+	struct statfs sb;
+
+	if (statfs(path, &sb) < 0)
+		return false;
+	if (sb.f_type == OVERLAYFS_SUPER_MAGIC ||
+			sb.f_type == OVERLAY_SUPER_MAGIC)
+		return true;
+	return false;
+}
+
 /* returns 0 on success, -1 if there were any failures */
 extern int lxc_rmdir_onedev(char *path, const char *exclude)
 {
 	struct stat mystat;
+	bool onedev = true;
+
+	if (is_native_overlayfs(path)) {
+		onedev = false;
+	}
 
 	if (lstat(path, &mystat) < 0) {
 		ERROR("%s: failed to stat %s", __func__, path);
 		return -1;
 	}
 
-	return _recursive_rmdir_onedev(path, mystat.st_dev, exclude, 0);
+	return _recursive_rmdir(path, mystat.st_dev, exclude, 0, onedev);
 }
 
 static int mount_fs(const char *source, const char *target, const char *type)