squashfs-tools: fix segfault

Unsquashfs was segfaulting.  When examining in gdb the stack was corrupt.
I found that converting the variable length arrays to malloc caused the
stack corruption to not happen and the segfault went away.  This is due to
the musl pthread stack size being 80k by default.  So the chance of a stack
overflow is high.

Signed-off-by: Colby Whitney <colby.whitney@luxul.com>

utils/squashfs-tools/Makefile

@@ -9,7 +9,7 @@ include $(TOPDIR)/rules.mk
 
 PKG_NAME:=squashfs-tools
 PKG_VERSION:=4.3
-PKG_RELEASE:=4
+PKG_RELEASE:=5
 PKG_LICENSE:=GPL-2.0
 PKG_LICENSE_FILES:=COPYING
 

utils/squashfs-tools/patches/0004-vla-to-malloc.patch

@@ -0,0 +1,474 @@
+diff -aurp a/squashfs-tools/unsquash-1.c b/squashfs-tools/unsquash-1.c
+--- a/squashfs-tools/unsquash-1.c	2014-09-18 20:16:18.000000000 -0600
++++ b/squashfs-tools/unsquash-1.c	2017-08-29 13:18:14.403020644 -0600
+@@ -332,17 +332,19 @@ int read_uids_guids_1()
+ 	guid_table = uid_table + sBlk.no_uids;
+ 
+ 	if(swap) {
+-		unsigned int suid_table[sBlk.no_uids + sBlk.no_guids];
++		unsigned int* suid_table = malloc((sBlk.no_uids + sBlk.no_guids) * sizeof(unsigned int));
+ 
+ 		res = read_fs_bytes(fd, sBlk.uid_start, (sBlk.no_uids +
+ 			sBlk.no_guids) * sizeof(unsigned int), suid_table);
+ 		if(res == FALSE) {
++			free(suid_table);
+ 			ERROR("read_uids_guids: failed to read uid/gid table"
+ 				"\n");
+ 			return FALSE;
+ 		}
+ 		SQUASHFS_SWAP_INTS_3(uid_table, suid_table,
+ 			sBlk.no_uids + sBlk.no_guids);
++		free(suid_table);
+ 	} else {
+ 		res = read_fs_bytes(fd, sBlk.uid_start, (sBlk.no_uids +
+ 			sBlk.no_guids) * sizeof(unsigned int), uid_table);
+diff -aurp a/squashfs-tools/unsquash-2.c b/squashfs-tools/unsquash-2.c
+--- a/squashfs-tools/unsquash-2.c	2014-09-18 20:16:18.000000000 -0600
++++ b/squashfs-tools/unsquash-2.c	2017-08-29 13:23:48.111321548 -0600
+@@ -32,7 +32,7 @@ void read_block_list_2(unsigned int *blo
+ 	TRACE("read_block_list: blocks %d\n", blocks);
+ 
+ 	if(swap) {
+-		unsigned int sblock_list[blocks];
++		unsigned int* sblock_list = malloc(blocks*sizeof(unsigned int));
+ 		memcpy(sblock_list, block_ptr, blocks * sizeof(unsigned int));
+ 		SQUASHFS_SWAP_INTS_3(block_list, sblock_list, blocks);
+ 	} else
+@@ -45,7 +45,7 @@ int read_fragment_table_2(long long *dir
+ 	int res, i;
+ 	int bytes = SQUASHFS_FRAGMENT_BYTES_2(sBlk.s.fragments);
+ 	int indexes = SQUASHFS_FRAGMENT_INDEXES_2(sBlk.s.fragments);
+-	unsigned int fragment_table_index[indexes];
++	unsigned int* fragment_table_index = malloc(indexes * sizeof(unsigned int));
+ 
+ 	TRACE("read_fragment_table: %d fragments, reading %d fragment indexes "
+ 		"from 0x%llx\n", sBlk.s.fragments, indexes,
+@@ -53,6 +53,7 @@ int read_fragment_table_2(long long *dir
+ 
+ 	if(sBlk.s.fragments == 0) {
+ 		*directory_table_end = sBlk.s.fragment_table_start;
++		free(fragment_table_index);
+ 		return TRUE;
+ 	}
+ 
+@@ -62,7 +63,7 @@ int read_fragment_table_2(long long *dir
+ 			"fragment table\n");
+ 
+ 	if(swap) {
+-		 unsigned int sfragment_table_index[indexes];
++		 unsigned int* sfragment_table_index = malloc(indexes * sizeof(unsigned int));
+ 
+ 		 res = read_fs_bytes(fd, sBlk.s.fragment_table_start,
+ 			SQUASHFS_FRAGMENT_INDEX_BYTES_2(sBlk.s.fragments),
+@@ -70,10 +71,14 @@ int read_fragment_table_2(long long *dir
+ 		if(res == FALSE) {
+ 			ERROR("read_fragment_table: failed to read fragment "
+ 				"table index\n");
++			free(sfragment_table_index);
++			free(fragment_table_index);
+ 			return FALSE;
+ 		}
+ 		SQUASHFS_SWAP_FRAGMENT_INDEXES_2(fragment_table_index,
+ 			sfragment_table_index, indexes);
++		
++		free(sfragment_table_index);
+ 	} else {
+ 		res = read_fs_bytes(fd, sBlk.s.fragment_table_start,
+ 			SQUASHFS_FRAGMENT_INDEX_BYTES_2(sBlk.s.fragments),
+@@ -81,6 +86,7 @@ int read_fragment_table_2(long long *dir
+ 		if(res == FALSE) {
+ 			ERROR("read_fragment_table: failed to read fragment "
+ 				"table index\n");
++			free(fragment_table_index);
+ 			return FALSE;
+ 		}
+ 	}
+@@ -96,6 +102,7 @@ int read_fragment_table_2(long long *dir
+ 		if(length == FALSE) {
+ 			ERROR("read_fragment_table: failed to read fragment "
+ 				"table block\n");
++			free(fragment_table_index);
+ 			return FALSE;
+ 		}
+ 	}
+@@ -111,6 +118,7 @@ int read_fragment_table_2(long long *dir
+ 	}
+ 
+ 	*directory_table_end = fragment_table_index[0];
++	free(fragment_table_index);
+ 	return TRUE;
+ }
+ 
+diff -aurp a/squashfs-tools/unsquash-3.c b/squashfs-tools/unsquash-3.c
+--- a/squashfs-tools/unsquash-3.c	2014-09-18 20:16:18.000000000 -0600
++++ b/squashfs-tools/unsquash-3.c	2017-08-29 14:43:17.016089289 -0600
+@@ -32,7 +32,7 @@ int read_fragment_table_3(long long *dir
+ 	int res, i;
+ 	int bytes = SQUASHFS_FRAGMENT_BYTES_3(sBlk.s.fragments);
+ 	int indexes = SQUASHFS_FRAGMENT_INDEXES_3(sBlk.s.fragments);
+-	long long fragment_table_index[indexes];
++	long long* fragment_table_index = malloc(indexes * sizeof(long long));
+ 
+ 	TRACE("read_fragment_table: %d fragments, reading %d fragment indexes "
+ 		"from 0x%llx\n", sBlk.s.fragments, indexes,
+@@ -40,6 +40,7 @@ int read_fragment_table_3(long long *dir
+ 
+ 	if(sBlk.s.fragments == 0) {
+ 		*directory_table_end = sBlk.s.fragment_table_start;
++		free(fragment_table_index);
+ 		return TRUE;
+ 	}
+ 
+@@ -49,7 +50,7 @@ int read_fragment_table_3(long long *dir
+ 			"fragment table\n");
+ 
+ 	if(swap) {
+-		long long sfragment_table_index[indexes];
++		long long* sfragment_table_index = malloc(indexes * sizeof(long long));
+ 
+ 		res = read_fs_bytes(fd, sBlk.s.fragment_table_start,
+ 			SQUASHFS_FRAGMENT_INDEX_BYTES_3(sBlk.s.fragments),
+@@ -57,10 +58,13 @@ int read_fragment_table_3(long long *dir
+ 		if(res == FALSE) {
+ 			ERROR("read_fragment_table: failed to read fragment "
+ 				"table index\n");       
++			free(fragment_table_index);
++			free(sfragment_table_index);
+ 			return FALSE;
+ 		}
+ 		SQUASHFS_SWAP_FRAGMENT_INDEXES_3(fragment_table_index,
+ 			sfragment_table_index, indexes);
++		free(sfragment_table_index);
+ 	} else {
+ 		res = read_fs_bytes(fd, sBlk.s.fragment_table_start,
+ 			SQUASHFS_FRAGMENT_INDEX_BYTES_3(sBlk.s.fragments),
+@@ -68,6 +72,7 @@ int read_fragment_table_3(long long *dir
+ 		if(res == FALSE) {
+ 			ERROR("read_fragment_table: failed to read fragment "
+ 				"table index\n");       
++			free(fragment_table_index);
+ 			return FALSE;
+ 		}
+ 	}
+@@ -83,6 +88,7 @@ int read_fragment_table_3(long long *dir
+ 		if(length == FALSE) {
+ 			ERROR("read_fragment_table: failed to read fragment "
+ 				"table block\n");       
++			free(fragment_table_index);
+ 			return FALSE;
+ 		}
+ 	}
+@@ -98,6 +104,7 @@ int read_fragment_table_3(long long *dir
+ 	}
+ 
+ 	*directory_table_end = fragment_table_index[0];
++	free(fragment_table_index);
+ 	return TRUE;
+ }
+ 
+diff -aurp a/squashfs-tools/unsquash-4.c b/squashfs-tools/unsquash-4.c
+--- a/squashfs-tools/unsquash-4.c	2014-09-18 20:16:18.000000000 -0600
++++ b/squashfs-tools/unsquash-4.c	2017-08-29 14:49:01.424441708 -0600
+@@ -33,7 +33,7 @@ int read_fragment_table_4(long long *dir
+ 	int res, i;
+ 	int bytes = SQUASHFS_FRAGMENT_BYTES(sBlk.s.fragments);
+ 	int  indexes = SQUASHFS_FRAGMENT_INDEXES(sBlk.s.fragments);
+-	long long fragment_table_index[indexes];
++	long long* fragment_table_index = malloc(indexes * sizeof(long long));
+ 
+ 	TRACE("read_fragment_table: %d fragments, reading %d fragment indexes "
+ 		"from 0x%llx\n", sBlk.s.fragments, indexes,
+@@ -41,6 +41,7 @@ int read_fragment_table_4(long long *dir
+ 
+ 	if(sBlk.s.fragments == 0) {
+ 		*directory_table_end = sBlk.s.fragment_table_start;
++		free(fragment_table_index);
+ 		return TRUE;
+ 	}
+ 
+@@ -55,6 +56,7 @@ int read_fragment_table_4(long long *dir
+ 	if(res == FALSE) {
+ 		ERROR("read_fragment_table: failed to read fragment table "
+ 			"index\n");
++		free(fragment_table_index);
+ 		return FALSE;
+ 	}
+ 	SQUASHFS_INSWAP_FRAGMENT_INDEXES(fragment_table_index, indexes);
+@@ -70,6 +72,7 @@ int read_fragment_table_4(long long *dir
+ 		if(length == FALSE) {
+ 			ERROR("read_fragment_table: failed to read fragment "
+ 				"table index\n");
++			free(fragment_table_index);
+ 			return FALSE;
+ 		}
+ 	}
+@@ -78,6 +81,7 @@ int read_fragment_table_4(long long *dir
+ 		SQUASHFS_INSWAP_FRAGMENT_ENTRY(&fragment_table[i]);
+ 
+ 	*directory_table_end = fragment_table_index[0];
++	free(fragment_table_index);
+ 	return TRUE;
+ }
+ 
+@@ -356,13 +360,14 @@ int read_uids_guids_4()
+ 	int res, i;
+ 	int bytes = SQUASHFS_ID_BYTES(sBlk.s.no_ids);
+ 	int indexes = SQUASHFS_ID_BLOCKS(sBlk.s.no_ids);
+-	long long id_index_table[indexes];
++	long long* id_index_table = malloc(indexes * sizeof(long long));
+ 
+ 	TRACE("read_uids_guids: no_ids %d\n", sBlk.s.no_ids);
+ 
+ 	id_table = malloc(bytes);
+ 	if(id_table == NULL) {
+ 		ERROR("read_uids_guids: failed to allocate id table\n");
++		free(id_index_table);
+ 		return FALSE;
+ 	}
+ 
+@@ -370,6 +375,7 @@ int read_uids_guids_4()
+ 		SQUASHFS_ID_BLOCK_BYTES(sBlk.s.no_ids), id_index_table);
+ 	if(res == FALSE) {
+ 		ERROR("read_uids_guids: failed to read id index table\n");
++		free(id_index_table);
+ 		return FALSE;
+ 	}
+ 	SQUASHFS_INSWAP_ID_BLOCKS(id_index_table, indexes);
+@@ -382,11 +388,13 @@ int read_uids_guids_4()
+ 		if(res == FALSE) {
+ 			ERROR("read_uids_guids: failed to read id table block"
+ 				"\n");
++			free(id_index_table);
+ 			return FALSE;
+ 		}
+ 	}
+ 
+ 	SQUASHFS_INSWAP_INTS(id_table, sBlk.s.no_ids);
+ 
++	free(id_index_table);
+ 	return TRUE;
+ }
+diff -aurp a/squashfs-tools/unsquashfs.c b/squashfs-tools/unsquashfs.c
+--- a/squashfs-tools/unsquashfs.c	2017-08-29 14:58:51.917037533 -0600
++++ b/squashfs-tools/unsquashfs.c	2017-08-29 13:14:03.082818149 -0600
+@@ -691,7 +691,7 @@ int read_block(int fd, long long start,
+ 		return 0;
+ 
+ 	if(compressed) {
+-		char buffer[c_byte];
++		char* buffer = malloc(c_byte);
+ 		int error;
+ 
+ 		res = read_fs_bytes(fd, start + offset, c_byte, buffer);
+@@ -704,8 +704,10 @@ int read_block(int fd, long long start,
+ 		if(res == -1) {
+ 			ERROR("%s uncompress failed with error code %d\n",
+ 				comp->name, error);
++			free(buffer);
+ 			goto failed;
+ 		}
++		free(buffer);
+ 	} else {
+ 		res = read_fs_bytes(fd, start + offset, c_byte, block);
+ 		if(res == FALSE)
+@@ -2097,7 +2099,7 @@ void *writer(void *arg)
+  */
+ void *inflator(void *arg)
+ {
+-	char tmp[block_size];
++	char* tmp = malloc(block_size);
+ 
+ 	while(1) {
+ 		struct cache_entry *entry = queue_get(to_inflate);
+@@ -2120,6 +2122,7 @@ void *inflator(void *arg)
+  		 */ 
+ 		cache_block_ready(entry, res == -1);
+ 	}
++	free(tmp);
+ }
+ 
+ 
+diff -aurp a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c
+--- a/squashfs-tools/mksquashfs.c	2017-09-05 15:09:19.090937121 -0600
++++ b/squashfs-tools/mksquashfs.c	2017-09-01 09:58:11.274529037 -0600
+@@ -652,7 +652,7 @@ long long write_directories()
+ long long write_id_table()
+ {
+ 	unsigned int id_bytes = SQUASHFS_ID_BYTES(id_count);
+-	unsigned int p[id_count];
++	unsigned int* p = malloc(id_count * sizeof(unsigned int));
+ 	int i;
+ 
+ 	TRACE("write_id_table: ids %d, id_bytes %d\n", id_count, id_bytes);
+@@ -655,6 +655,9 @@ long long write_id_table()
+ 	unsigned int* p = malloc(id_count * sizeof(unsigned int));
+ 	int i;
+ 
++	if(p == NULL)
++		MEM_ERROR();
++
+ 	TRACE("write_id_table: ids %d, id_bytes %d\n", id_count, id_bytes);
+ 	for(i = 0; i < id_count; i++) {
+ 		TRACE("write_id_table: id index %d, id %d", i, id_table[i]->id);
+
+@@ -661,6 +661,7 @@ long long write_id_table()
+ 		SQUASHFS_SWAP_INTS(&id_table[i]->id, p + i, 1);
+ 	}
+ 
++	free(p);
+ 	return generic_write_table(id_bytes, p, 0, NULL, noI);
+ }
+ 
+diff -aurp a/squashfs-tools/read_fs.c b/squashfs-tools/read_fs.c
+--- a/squashfs-tools/read_fs.c	2014-09-18 20:16:18.000000000 -0600
++++ b/squashfs-tools/read_fs.c	2017-09-05 15:35:19.328547536 -0600
+@@ -77,18 +77,24 @@ int read_block(int fd, long long start,
+ 		return 0;
+ 
+ 	if(compressed) {
+-		char buffer[c_byte];
++		char* buffer = malloc(c_byte);
+ 		int error;
+ 
++		if(buffer == NULL)
++			MEM_ERROR();
++
+ 		res = read_fs_bytes(fd, start + 2, c_byte, buffer);
+-		if(res == 0)
++		if(res == 0) {
++			free(buffer);
+ 			return 0;
++		}
+ 
+ 		res = compressor_uncompress(comp, block, buffer, c_byte,
+ 			outlen, &error);
+ 		if(res == -1) {
+ 			ERROR("%s uncompress failed with error code %d\n",
+ 				comp->name, error);
++			free(buffer);
+ 			return 0;
+ 		}
+ 	} else {
+@@ -699,7 +705,7 @@ all_done:
+ unsigned int *read_id_table(int fd, struct squashfs_super_block *sBlk)
+ {
+ 	int indexes = SQUASHFS_ID_BLOCKS(sBlk->no_ids);
+-	long long index[indexes];
++	long long* index;
+ 	int bytes = SQUASHFS_ID_BYTES(sBlk->no_ids);
+ 	unsigned int *id_table;
+ 	int res, i;
+@@ -708,12 +714,17 @@ unsigned int *read_id_table(int fd, stru
+ 	if(id_table == NULL)
+ 		MEM_ERROR();
+ 
++	index = malloc(indexes * sizeof(long long));
++	if(index == NULL)
++		MEM_ERROR();
++
+ 	res = read_fs_bytes(fd, sBlk->id_table_start,
+ 		SQUASHFS_ID_BLOCK_BYTES(sBlk->no_ids), index);
+ 	if(res == 0) {
+ 		ERROR("Failed to read id table index\n");
+ 		ERROR("Filesystem corrupted?\n");
+ 		free(id_table);
++		free(index);
+ 		return NULL;
+ 	}
+ 
+@@ -732,6 +743,7 @@ unsigned int *read_id_table(int fd, stru
+ 				"length %d\n", i, index[i], length);
+ 			ERROR("Filesystem corrupted?\n");
+ 			free(id_table);
++			free(index);
+ 			return NULL;
+ 		}
+ 	}
+@@ -753,14 +765,19 @@ int read_fragment_table(int fd, struct s
+ 	int res, i;
+ 	int bytes = SQUASHFS_FRAGMENT_BYTES(sBlk->fragments);
+ 	int indexes = SQUASHFS_FRAGMENT_INDEXES(sBlk->fragments);
+-	long long fragment_table_index[indexes];
++	long long* fragment_table_index = malloc(indexes * sizeof(long long));
++
++	if(fragment_table_index == NULL)
++		MEM_ERROR();
+ 
+ 	TRACE("read_fragment_table: %d fragments, reading %d fragment indexes "
+ 		"from 0x%llx\n", sBlk->fragments, indexes,
+ 		sBlk->fragment_table_start);
+ 
+-	if(sBlk->fragments == 0)
++	if(sBlk->fragments == 0) {
++		free(fragment_table_index);
+ 		return 1;
++	}
+ 
+ 	*fragment_table = malloc(bytes);
+ 	if(*fragment_table == NULL)
+@@ -773,6 +790,7 @@ int read_fragment_table(int fd, struct s
+ 		ERROR("Failed to read fragment table index\n");
+ 		ERROR("Filesystem corrupted?\n");
+ 		free(*fragment_table);
++		free(fragment_table_index);
+ 		return 0;
+ 	}
+ 
+@@ -792,6 +810,7 @@ int read_fragment_table(int fd, struct s
+ 				fragment_table_index[i], length);
+ 			ERROR("Filesystem corrupted?\n");
+ 			free(*fragment_table);
++			free(fragment_table_index);
+ 			return 0;
+ 		}
+ 	}
+@@ -799,6 +818,7 @@ int read_fragment_table(int fd, struct s
+ 	for(i = 0; i < sBlk->fragments; i++)
+ 		SQUASHFS_INSWAP_FRAGMENT_ENTRY(&(*fragment_table)[i]);
+ 
++	free(fragment_table_index);
+ 	return 1;
+ }
+ 
+@@ -808,11 +828,16 @@ int read_inode_lookup_table(int fd, stru
+ {
+ 	int lookup_bytes = SQUASHFS_LOOKUP_BYTES(sBlk->inodes);
+ 	int indexes = SQUASHFS_LOOKUP_BLOCKS(sBlk->inodes);
+-	long long index[indexes];
++	long long* index = malloc(indexes * sizeof(long long));
+ 	int res, i;
+ 
+-	if(sBlk->lookup_table_start == SQUASHFS_INVALID_BLK)
++	if(index == NULL)
++		MEM_ERROR();
++
++	if(sBlk->lookup_table_start == SQUASHFS_INVALID_BLK) {
++		free(index);
+ 		return 1;
++	}
+ 
+ 	*inode_lookup_table = malloc(lookup_bytes);
+ 	if(*inode_lookup_table == NULL)
+@@ -824,6 +849,7 @@ int read_inode_lookup_table(int fd, stru
+ 		ERROR("Failed to read inode lookup table index\n");
+ 		ERROR("Filesystem corrupted?\n");
+ 		free(*inode_lookup_table);
++		free(index);
+ 		return 0;
+ 	}
+ 
+@@ -843,12 +869,14 @@ int read_inode_lookup_table(int fd, stru
+ 				length);
+ 			ERROR("Filesystem corrupted?\n");
+ 			free(*inode_lookup_table);
++			free(index);
+ 			return 0;
+ 		}
+ 	}
+ 
+ 	SQUASHFS_INSWAP_LONG_LONGS(*inode_lookup_table, sBlk->inodes);
+ 
++	free(index);
+ 	return 1;
+ }
+