Upgrade bundled libmagic to 5.31

ext/fileinfo/libmagic/apprentice.c

@@ -34,7 +34,7 @@
 #include "file.h"
 
 #ifndef	lint
-FILE_RCSID("@(#)$File: apprentice.c,v 1.255 2016/10/24 18:02:17 christos Exp $")
+FILE_RCSID("@(#)$File: apprentice.c,v 1.260 2017/04/28 16:27:58 christos Exp $")
 #endif	/* lint */
 
 #include "magic.h"
@@ -2235,7 +2235,7 @@ parse_ext(struct magic_set *ms, struct magic_entry *me, const char *line)
 
 	return parse_extra(ms, me, line,
 	    CAST(off_t, offsetof(struct magic, ext)),
-	    sizeof(m->ext), "EXTENSION", ",!+-/", 0);
+	    sizeof(m->ext), "EXTENSION", ",!+-/@", 0);
 }
 
 /*
@@ -2296,6 +2296,8 @@ check_format_type(const char *ptr, int type, const char **estr)
 			ptr++;
 		if (*ptr == '.')
 			ptr++;
+		if (*ptr == '#')
+			ptr++;
 #define CHECKLEN() do { \
 	for (len = cnt = 0; isdigit((unsigned char)*ptr); ptr++, cnt++) \
 		len = len * 10 + (*ptr - '0'); \
@@ -3289,22 +3291,35 @@ file_pstring_get_length(const struct magic *m, const char *ss)
 {
 	size_t len = 0;
 	const unsigned char *s = (const unsigned char *)ss;
+	unsigned int s3, s2, s1, s0;
 
 	switch (m->str_flags & PSTRING_LEN) {
 	case PSTRING_1_LE:
 		len = *s;
 		break;
 	case PSTRING_2_LE:
-		len = (s[1] << 8) | s[0];
+		s0 = s[0];
+		s1 = s[1];
+		len = (s1 << 8) | s0;
 		break;
 	case PSTRING_2_BE:
-		len = (s[0] << 8) | s[1];
+		s0 = s[0];
+		s1 = s[1];
+		len = (s0 << 8) | s1;
 		break;
 	case PSTRING_4_LE:
-		len = (s[3] << 24) | (s[2] << 16) | (s[1] << 8) | s[0];
+		s0 = s[0];
+		s1 = s[1];
+		s2 = s[2];
+		s3 = s[3];
+		len = (s3 << 24) | (s2 << 16) | (s1 << 8) | s0;
 		break;
 	case PSTRING_4_BE:
-		len = (s[0] << 24) | (s[1] << 16) | (s[2] << 8) | s[3];
+		s0 = s[0];
+		s1 = s[1];
+		s2 = s[2];
+		s3 = s[3];
+		len = (s0 << 24) | (s1 << 16) | (s2 << 8) | s3;
 		break;
 	default:
 		abort();	/* Impossible */

ext/fileinfo/libmagic/cdf.c

@@ -35,7 +35,7 @@
 #include "file.h"
 
 #ifndef lint
-FILE_RCSID("@(#)$File: cdf.c,v 1.85 2016/10/24 18:02:17 christos Exp $")
+FILE_RCSID("@(#)$File: cdf.c,v 1.106 2017/04/30 17:05:02 christos Exp $")
 #endif
 
 #include <assert.h>
@@ -90,6 +90,31 @@ static union {
 			    CDF_TOLE8(CAST(uint64_t, x))))
 #define CDF_GETUINT32(x, y)	cdf_getuint32(x, y)
 
+#define CDF_MALLOC(n) cdf_malloc(__FILE__, __LINE__, (n))
+#define CDF_REALLOC(p, n) cdf_realloc(__FILE__, __LINE__, (p), (n))
+#define CDF_CALLOC(n, u) cdf_calloc(__FILE__, __LINE__, (n), (u))
+
+
+static void *
+cdf_malloc(const char *file, size_t line, size_t n)
+{
+	DPRINTF(("%s,%zu: %s %zu\n", file, line, __func__, n));
+	return malloc(n);
+}
+
+static void *
+cdf_realloc(const char *file, size_t line, void *p, size_t n)
+{
+	DPRINTF(("%s,%zu: %s %zu\n", file, line, __func__, n));
+	return realloc(p, n);
+}
+
+static void *
+cdf_calloc(const char *file, size_t line, size_t n, size_t u)
+{
+	DPRINTF(("%s,%zu: %s %zu %zu\n", file, line, __func__, n, u));
+	return calloc(n, u);
+}
 
 /*
  * swap a short
@@ -354,18 +379,18 @@ cdf_read_header(const cdf_info_t *info, cdf_header_t *h)
 	cdf_unpack_header(h, buf);
 	cdf_swap_header(h);
 	if (h->h_magic != CDF_MAGIC) {
-		DPRINTF(("Bad magic 0x%" INT64_T_FORMAT "x != 0x%"
+		DPRINTF(("Bad magic %#" INT64_T_FORMAT "x != %#"
 		    INT64_T_FORMAT "x\n",
 		    (unsigned long long)h->h_magic,
 		    (unsigned long long)CDF_MAGIC));
 		goto out;
 	}
 	if (h->h_sec_size_p2 > 20) {
-		DPRINTF(("Bad sector size 0x%u\n", h->h_sec_size_p2));
+		DPRINTF(("Bad sector size %hu\n", h->h_sec_size_p2));
 		goto out;
 	}
 	if (h->h_short_sec_size_p2 > 20) {
-		DPRINTF(("Bad short sector size 0x%u\n",
+		DPRINTF(("Bad short sector size %hu\n",
 		    h->h_short_sec_size_p2));
 		goto out;
 	}
@@ -422,7 +447,7 @@ cdf_read_sat(const cdf_info_t *info, cdf_header_t *h, cdf_sat_t *sat)
 		if (h->h_master_sat[i] == CDF_SECID_FREE)
 			break;
 
-#define CDF_SEC_LIMIT (UINT32_MAX / (4 * ss))
+#define CDF_SEC_LIMIT (UINT32_MAX / (8 * ss))
 	if ((nsatpersec > 0 &&
 	    h->h_num_sectors_in_master_sat > CDF_SEC_LIMIT / nsatpersec) ||
 	    i > CDF_SEC_LIMIT) {
@@ -435,7 +460,7 @@ cdf_read_sat(const cdf_info_t *info, cdf_header_t *h, cdf_sat_t *sat)
 	sat->sat_len = h->h_num_sectors_in_master_sat * nsatpersec + i;
 	DPRINTF(("sat_len = %" SIZE_T_FORMAT "u ss = %" SIZE_T_FORMAT "u\n",
 	    sat->sat_len, ss));
-	if ((sat->sat_tab = CAST(cdf_secid_t *, calloc(sat->sat_len, ss)))
+	if ((sat->sat_tab = CAST(cdf_secid_t *, CDF_CALLOC(sat->sat_len, ss)))
 	    == NULL)
 		return -1;
 
@@ -449,7 +474,7 @@ cdf_read_sat(const cdf_info_t *info, cdf_header_t *h, cdf_sat_t *sat)
 		}
 	}
 
-	if ((msa = CAST(cdf_secid_t *, calloc(1, ss))) == NULL)
+	if ((msa = CAST(cdf_secid_t *, CDF_CALLOC(1, ss))) == NULL)
 		goto out1;
 
 	mid = h->h_secid_first_sector_in_master_sat;
@@ -541,13 +566,16 @@ cdf_read_long_sector_chain(const cdf_info_t *info, const cdf_header_t *h,
 	ssize_t nr;
 	scn->sst_tab = NULL;
 	scn->sst_len = cdf_count_chain(sat, sid, ss);
-	scn->sst_dirlen = len;
+	scn->sst_dirlen = MAX(h->h_min_size_standard_stream, len);
 	scn->sst_ss = ss;
 
+	if (sid == CDF_SECID_END_OF_CHAIN || len == 0)
+		return cdf_zero_stream(scn);
+
 	if (scn->sst_len == (size_t)-1)
 		goto out;
 
-	scn->sst_tab = calloc(scn->sst_len, ss);
+	scn->sst_tab = CDF_CALLOC(scn->sst_len, ss);
 	if (scn->sst_tab == NULL)
 		return cdf_zero_stream(scn);
 
@@ -593,7 +621,7 @@ cdf_read_short_sector_chain(const cdf_header_t *h,
 	if (scn->sst_len == (size_t)-1)
 		goto out;
 
-	scn->sst_tab = calloc(scn->sst_len, ss);
+	scn->sst_tab = CDF_CALLOC(scn->sst_len, ss);
 	if (scn->sst_tab == NULL)
 		return cdf_zero_stream(scn);
 
@@ -651,11 +679,11 @@ cdf_read_dir(const cdf_info_t *info, const cdf_header_t *h,
 
 	dir->dir_len = ns * nd;
 	dir->dir_tab = CAST(cdf_directory_t *,
-	    calloc(dir->dir_len, sizeof(dir->dir_tab[0])));
+	    CDF_CALLOC(dir->dir_len, sizeof(dir->dir_tab[0])));
 	if (dir->dir_tab == NULL)
 		return -1;
 
-	if ((buf = CAST(char *, malloc(ss))) == NULL) {
+	if ((buf = CAST(char *, CDF_MALLOC(ss))) == NULL) {
 		free(dir->dir_tab);
 		return -1;
 	}
@@ -701,7 +729,7 @@ cdf_read_ssat(const cdf_info_t *info, const cdf_header_t *h,
 	if (ssat->sat_len == (size_t)-1)
 		goto out;
 
-	ssat->sat_tab = CAST(cdf_secid_t *, calloc(ssat->sat_len, ss));
+	ssat->sat_tab = CAST(cdf_secid_t *, CDF_CALLOC(ssat->sat_len, ss));
 	if (ssat->sat_tab == NULL)
 		goto out1;
 
@@ -822,13 +850,107 @@ cdf_find_stream(const cdf_dir_t *dir, const char *name, int type)
 		    == 0)
 			break;
 	if (i > 0)
-		return i;
+		return CAST(int, i);
 
 	DPRINTF(("Cannot find type %d `%s'\n", type, name));
 	errno = ESRCH;
 	return 0;
 }
 
+#define CDF_SHLEN_LIMIT (UINT32_MAX / 8)
+#define CDF_PROP_LIMIT (UINT32_MAX / (8 * sizeof(cdf_property_info_t)))
+
+static const void *
+cdf_offset(const void *p, size_t l)
+{
+	return CAST(const void *, CAST(const uint8_t *, p) + l);
+}
+
+static const uint8_t *
+cdf_get_property_info_pos(const cdf_stream_t *sst, const cdf_header_t *h, 
+    const uint8_t *p, const uint8_t *e, size_t i)
+{
+	size_t tail = (i << 1) + 1;
+	size_t ofs;
+	const uint8_t *q;
+
+	if (p >= e) {
+		DPRINTF(("Past end %p < %p\n", e, p));
+		return NULL;
+	}
+	if (cdf_check_stream_offset(sst, h, p, (tail + 1) * sizeof(uint32_t),
+	    __LINE__) == -1)
+		return NULL;
+	ofs = CDF_GETUINT32(p, tail);
+	q = CAST(const uint8_t *, cdf_offset(CAST(const void *, p), 
+	    ofs - 2 * sizeof(uint32_t)));
+
+	if (q < p) {
+		DPRINTF(("Wrapped around %p < %p\n", q, p));
+		return NULL;
+	}
+
+	if (q >= e) {
+		DPRINTF(("Ran off the end %p >= %p\n", q, e));
+		return NULL;
+	}
+	return q;
+}
+
+static cdf_property_info_t *
+cdf_grow_info(cdf_property_info_t **info, size_t *maxcount, size_t incr)
+{
+	cdf_property_info_t *inp;
+	size_t newcount = *maxcount + incr;
+
+	if (newcount > CDF_PROP_LIMIT) {
+		DPRINTF(("exceeded property limit %zu > %zu\n", 
+		    newcount, CDF_PROP_LIMIT));
+		goto out;
+	}
+	inp = CAST(cdf_property_info_t *,
+	    CDF_REALLOC(*info, newcount * sizeof(*inp)));
+	if (inp == NULL)
+		goto out;
+
+	*info = inp;
+	*maxcount = newcount;
+	return inp;
+out:
+	free(*info);
+	*maxcount = 0;
+	*info = NULL;
+	return NULL;
+}
+
+static int
+cdf_copy_info(cdf_property_info_t *inp, const void *p, const void *e,
+    size_t len)
+{
+	if (inp->pi_type & CDF_VECTOR)
+		return 0;
+
+	if ((size_t)(CAST(const char *, e) - CAST(const char *, p)) < len)
+		return 0;
+
+	(void)memcpy(&inp->pi_val, p, len);
+
+	switch (len) {
+	case 2:
+		inp->pi_u16 = CDF_TOLE2(inp->pi_u16);
+		break;
+	case 4:
+		inp->pi_u32 = CDF_TOLE4(inp->pi_u32);
+		break;
+	case 8:
+		inp->pi_u64 = CDF_TOLE8(inp->pi_u64);
+		break;
+	default:
+		abort();
+	}
+	return 1;
+}
+
 int
 cdf_read_property_info(const cdf_stream_t *sst, const cdf_header_t *h,
     uint32_t offs, cdf_property_info_t **info, size_t *count, size_t *maxcount)
@@ -836,92 +958,69 @@ cdf_read_property_info(const cdf_stream_t *sst, const cdf_header_t *h,
 	const cdf_section_header_t *shp;
 	cdf_section_header_t sh;
 	const uint8_t *p, *q, *e;
-	int16_t s16;
+	size_t i, o4, nelements, j, slen, left;
-	int32_t s32;
-	uint32_t u32;
-	int64_t s64;
-	uint64_t u64;
-	cdf_timestamp_t tp;
-	size_t i, o, o4, nelements, j;
 	cdf_property_info_t *inp;
 
 	if (offs > UINT32_MAX / 4) {
 		errno = EFTYPE;
 		goto out;
 	}
-	shp = CAST(const cdf_section_header_t *, (const void *)
+	shp = CAST(const cdf_section_header_t *,
-	    ((const char *)sst->sst_tab + offs));
+	    cdf_offset(sst->sst_tab, offs));
 	if (cdf_check_stream_offset(sst, h, shp, sizeof(*shp), __LINE__) == -1)
 		goto out;
 	sh.sh_len = CDF_TOLE4(shp->sh_len);
-#define CDF_SHLEN_LIMIT (UINT32_MAX / 8)
 	if (sh.sh_len > CDF_SHLEN_LIMIT) {
 		errno = EFTYPE;
 		goto out;
 	}
-	sh.sh_properties = CDF_TOLE4(shp->sh_properties);
+
-#define CDF_PROP_LIMIT (UINT32_MAX / (4 * sizeof(*inp)))
+	if (cdf_check_stream_offset(sst, h, shp, sh.sh_len, __LINE__) == -1)
-	if (sh.sh_properties > CDF_PROP_LIMIT)
 		goto out;
+
+	sh.sh_properties = CDF_TOLE4(shp->sh_properties);
 	DPRINTF(("section len: %u properties %u\n", sh.sh_len,
 	    sh.sh_properties));
-	if (*maxcount) {
+	if (sh.sh_properties > CDF_PROP_LIMIT)
-		if (*maxcount > CDF_PROP_LIMIT)
+		goto out;
-			goto out;
+	inp = cdf_grow_info(info, maxcount, sh.sh_properties);
-		*maxcount += sh.sh_properties;
-		inp = CAST(cdf_property_info_t *,
-		    realloc(*info, *maxcount * sizeof(*inp)));
-	} else {
-		*maxcount = sh.sh_properties;
-		inp = CAST(cdf_property_info_t *,
-		    malloc(*maxcount * sizeof(*inp)));
-	}
 	if (inp == NULL)
-		goto out1;
+		goto out;
-	*info = inp;
 	inp += *count;
 	*count += sh.sh_properties;
-	p = CAST(const uint8_t *, (const void *)
+	p = CAST(const uint8_t *, cdf_offset(sst->sst_tab, offs + sizeof(sh)));
-	    ((const char *)(const void *)sst->sst_tab +
+	e = CAST(const uint8_t *, cdf_offset(shp, sh.sh_len));
-	    offs + sizeof(sh)));
+	if (p >= e || cdf_check_stream_offset(sst, h, e, 0, __LINE__) == -1)
-	e = CAST(const uint8_t *, (const void *)
-	    (((const char *)(const void *)shp) + sh.sh_len));
-	if (cdf_check_stream_offset(sst, h, e, 0, __LINE__) == -1)
 		goto out;
+
 	for (i = 0; i < sh.sh_properties; i++) {
-		size_t tail = (i << 1) + 1;
+		if ((q = cdf_get_property_info_pos(sst, h, p, e, i)) == NULL)
-		size_t ofs;
-		if (cdf_check_stream_offset(sst, h, p, tail * sizeof(uint32_t),
-		    __LINE__) == -1)
 			goto out;
-		ofs = CDF_GETUINT32(p, tail);
-		q = (const uint8_t *)(const void *)
-		    ((const char *)(const void *)p + ofs
-		    - 2 * sizeof(uint32_t));
-		if (q < p) {
-			DPRINTF(("Wrapped around %p < %p\n", q, p));
-			goto out;
-		}
-		if (q > e) {
-			DPRINTF(("Ran of the end %p > %p\n", q, e));
-			goto out;
-		}
 		inp[i].pi_id = CDF_GETUINT32(p, i << 1);
+		left = CAST(size_t, e - q);
+		if (left < sizeof(uint32_t)) {
+			DPRINTF(("short info (no type)_\n"));
+			goto out;
+		}
 		inp[i].pi_type = CDF_GETUINT32(q, 0);
-		DPRINTF(("%" SIZE_T_FORMAT "u) id=%x type=%x offs=0x%tx,0x%x\n",
+		DPRINTF(("%" SIZE_T_FORMAT "u) id=%#x type=%#x offs=%#tx,%#x\n",
 		    i, inp[i].pi_id, inp[i].pi_type, q - p, offs));
 		if (inp[i].pi_type & CDF_VECTOR) {
+			if (left < sizeof(uint32_t) * 2) {
+				DPRINTF(("missing CDF_VECTOR length\n"));
+				goto out;
+			}
 			nelements = CDF_GETUINT32(q, 1);
 			if (nelements == 0) {
 				DPRINTF(("CDF_VECTOR with nelements == 0\n"));
 				goto out;
 			}
-			o = 2;
+			slen = 2;
 		} else {
 			nelements = 1;
-			o = 1;
+			slen = 1;
 		}
-		o4 = o * sizeof(uint32_t);
+		o4 = slen * sizeof(uint32_t);
 		if (inp[i].pi_type & (CDF_ARRAY|CDF_BYREF|CDF_RESERVED))
 			goto unknown;
 		switch (inp[i].pi_type & CDF_TYPEMASK) {
@@ -929,109 +1028,83 @@ cdf_read_property_info(const cdf_stream_t *sst, const cdf_header_t *h,
 		case CDF_EMPTY:
 			break;
 		case CDF_SIGNED16:
-			if (inp[i].pi_type & CDF_VECTOR)
+			if (!cdf_copy_info(&inp[i], &q[o4], e, sizeof(int16_t)))
 				goto unknown;
-			(void)memcpy(&s16, &q[o4], sizeof(s16));
-			inp[i].pi_s16 = CDF_TOLE2(s16);
 			break;
 		case CDF_SIGNED32:
-			if (inp[i].pi_type & CDF_VECTOR)
-				goto unknown;
-			(void)memcpy(&s32, &q[o4], sizeof(s32));
-			inp[i].pi_s32 = CDF_TOLE4((uint32_t)s32);
-			break;
 		case CDF_BOOL:
 		case CDF_UNSIGNED32:
-			if (inp[i].pi_type & CDF_VECTOR)
+		case CDF_FLOAT:
+			if (!cdf_copy_info(&inp[i], &q[o4], e, sizeof(int32_t)))
 				goto unknown;
-			(void)memcpy(&u32, &q[o4], sizeof(u32));
-			inp[i].pi_u32 = CDF_TOLE4(u32);
 			break;
 		case CDF_SIGNED64:
-			if (inp[i].pi_type & CDF_VECTOR)
-				goto unknown;
-			(void)memcpy(&s64, &q[o4], sizeof(s64));
-			inp[i].pi_s64 = CDF_TOLE8((uint64_t)s64);
-			break;
 		case CDF_UNSIGNED64:
-			if (inp[i].pi_type & CDF_VECTOR)
-				goto unknown;
-			(void)memcpy(&u64, &q[o4], sizeof(u64));
-			inp[i].pi_u64 = CDF_TOLE8((uint64_t)u64);
-			break;
-		case CDF_FLOAT:
-			if (inp[i].pi_type & CDF_VECTOR)
-				goto unknown;
-			(void)memcpy(&u32, &q[o4], sizeof(u32));
-			u32 = CDF_TOLE4(u32);
-			memcpy(&inp[i].pi_f, &u32, sizeof(inp[i].pi_f));
-			break;
 		case CDF_DOUBLE:
-			if (inp[i].pi_type & CDF_VECTOR)
+		case CDF_FILETIME:
+			if (!cdf_copy_info(&inp[i], &q[o4], e, sizeof(int64_t)))
 				goto unknown;
-			(void)memcpy(&u64, &q[o4], sizeof(u64));
-			u64 = CDF_TOLE8((uint64_t)u64);
-			memcpy(&inp[i].pi_d, &u64, sizeof(inp[i].pi_d));
 			break;
 		case CDF_LENGTH32_STRING:
 		case CDF_LENGTH32_WSTRING:
 			if (nelements > 1) {
 				size_t nelem = inp - *info;
-				if (*maxcount > CDF_PROP_LIMIT
+				inp = cdf_grow_info(info, maxcount, nelements);
-				    || nelements > CDF_PROP_LIMIT)
-					goto out;
-				*maxcount += nelements;
-				inp = CAST(cdf_property_info_t *,
-				    realloc(*info, *maxcount * sizeof(*inp)));
 				if (inp == NULL)
-					goto out1;
+					goto out;
-				*info = inp;
+				inp += nelem;
-				inp = *info + nelem;
 			}
 			DPRINTF(("nelements = %" SIZE_T_FORMAT "u\n",
 			    nelements));
 			for (j = 0; j < nelements && i < sh.sh_properties;
 			    j++, i++)
 			{
-				uint32_t l = CDF_GETUINT32(q, o);
+				uint32_t l;
+
+				if (o4 + sizeof(uint32_t) > left)
+					goto out;
+
+				l = CDF_GETUINT32(q, slen);
+				o4 += sizeof(uint32_t);
+				if (o4 + l > left)
+					goto out;
+
 				inp[i].pi_str.s_len = l;
-				inp[i].pi_str.s_buf = (const char *)
+				inp[i].pi_str.s_buf = CAST(const char *,
-				    (const void *)(&q[o4 + sizeof(l)]);
+				    CAST(const void *, &q[o4]));
-				DPRINTF(("l = %d, r = %" SIZE_T_FORMAT
+
-				    "u, s = %s\n", l,
+				DPRINTF(("o=%zu l=%d(%" SIZE_T_FORMAT
-				    CDF_ROUND(l, sizeof(l)),
+				    "u), t=%zu s=%s\n", o4, l,
+				    CDF_ROUND(l, sizeof(l)), left,
 				    inp[i].pi_str.s_buf));
+
 				if (l & 1)
 					l++;
-				o += l >> 1;
+
-				if (q + o >= e)
+				slen += l >> 1;
-					goto out;
+				o4 = slen * sizeof(uint32_t);
-				o4 = o * sizeof(uint32_t);
 			}
 			i--;
 			break;
-		case CDF_FILETIME:
-			if (inp[i].pi_type & CDF_VECTOR)
-				goto unknown;
-			(void)memcpy(&tp, &q[o4], sizeof(tp));
-			inp[i].pi_tp = CDF_TOLE8((uint64_t)tp);
-			break;
 		case CDF_CLIPBOARD:
 			if (inp[i].pi_type & CDF_VECTOR)
 				goto unknown;
 			break;
 		default:
 		unknown:
-			DPRINTF(("Don't know how to deal with %x\n",
+			memset(&inp[i].pi_val, 0, sizeof(inp[i].pi_val));
+			DPRINTF(("Don't know how to deal with %#x\n",
 			    inp[i].pi_type));
 			break;
 		}
 	}
 	return 0;
 out:
-	errno = EFTYPE;
-out1:
 	free(*info);
+	*info = NULL;
+	*count = 0;
+	*maxcount = 0;
+	errno = EFTYPE;
 	return -1;
 }
 
@@ -1079,7 +1152,7 @@ cdf_unpack_catalog(const cdf_header_t *h, const cdf_stream_t *sst,
 {
 	size_t ss = cdf_check_stream(sst, h);
 	const char *b = CAST(const char *, sst->sst_tab);
-	const char *eb = b + ss * sst->sst_len;
+	const char *nb, *eb = b + ss * sst->sst_len;
 	size_t nr, i, j, k;
 	cdf_catalog_entry_t *ce;
 	uint16_t reclen;
@@ -1098,7 +1171,7 @@ cdf_unpack_catalog(const cdf_header_t *h, const cdf_stream_t *sst,
 		return -1;
 	nr--;
 	*cat = CAST(cdf_catalog_t *,
-	    malloc(sizeof(cdf_catalog_t) + nr * sizeof(*ce)));
+	    CDF_MALLOC(sizeof(cdf_catalog_t) + nr * sizeof(*ce)));
 	if (*cat == NULL)
 		return -1;
 	ce = (*cat)->cat_e;
@@ -1124,7 +1197,9 @@ cdf_unpack_catalog(const cdf_header_t *h, const cdf_stream_t *sst,
 			cep->ce_namlen = rlen;
 
 		np = CAST(const uint16_t *, CAST(const void *, (b + 16)));
-		if (RCAST(const char *, np + cep->ce_namlen) > eb) {
+		nb = CAST(const char *, CAST(const void *,
+		    (np + cep->ce_namlen)));
+		if (nb > eb) {
 			cep->ce_namlen = 0;
 			break;
 		}
@@ -1183,7 +1258,7 @@ cdf_print_property_name(char *buf, size_t bufsiz, uint32_t p)
 	for (i = 0; i < __arraycount(vn); i++)
 		if (vn[i].v == p)
 			return snprintf(buf, bufsiz, "%s", vn[i].n);
-	return snprintf(buf, bufsiz, "0x%x", p);
+	return snprintf(buf, bufsiz, "%#x", p);
 }
 
 int
@@ -1242,7 +1317,7 @@ cdf_dump_header(const cdf_header_t *h)
     h->h_ ## b, 1 << h->h_ ## b)
 	DUMP("%d", revision);
 	DUMP("%d", version);
-	DUMP("0x%x", byte_order);
+	DUMP("%#x", byte_order);
 	DUMP2("%d", sec_size_p2);
 	DUMP2("%d", short_sec_size_p2);
 	DUMP("%d", num_sectors_in_sat);
@@ -1336,7 +1411,7 @@ cdf_dump_dir(const cdf_info_t *info, const cdf_header_t *h,
 		    d->d_color ? "black" : "red");
 		(void)fprintf(stderr, "Left child: %d\n", d->d_left_child);
 		(void)fprintf(stderr, "Right child: %d\n", d->d_right_child);
-		(void)fprintf(stderr, "Flags: 0x%x\n", d->d_flags);
+		(void)fprintf(stderr, "Flags: %#x\n", d->d_flags);
 		cdf_timestamp_to_timespec(&ts, d->d_created);
 		(void)fprintf(stderr, "Created %s", cdf_ctime(&ts.tv_sec, buf));
 		cdf_timestamp_to_timespec(&ts, d->d_modified);
@@ -1429,7 +1504,7 @@ cdf_dump_property_info(const cdf_property_info_t *info, size_t count)
 			(void)fprintf(stderr, "CLIPBOARD %u\n", info[i].pi_u32);
 			break;
 		default:
-			DPRINTF(("Don't know how to deal with %x\n",
+			DPRINTF(("Don't know how to deal with %#x\n",
 			    info[i].pi_type));
 			break;
 		}
@@ -1448,7 +1523,7 @@ cdf_dump_summary_info(const cdf_header_t *h, const cdf_stream_t *sst)
 	(void)&h;
 	if (cdf_unpack_summary_info(sst, h, &ssi, &info, &count) == -1)
 		return;
-	(void)fprintf(stderr, "Endian: %x\n", ssi.si_byte_order);
+	(void)fprintf(stderr, "Endian: %#x\n", ssi.si_byte_order);
 	(void)fprintf(stderr, "Os Version %d.%d\n", ssi.si_os_version & 0xff,
 	    ssi.si_os_version >> 8);
 	(void)fprintf(stderr, "Os %d\n", ssi.si_os);

ext/fileinfo/libmagic/cdf.h

@@ -129,9 +129,9 @@ typedef struct {
 
 typedef struct {
 	void *sst_tab;
-	size_t sst_len;
+	size_t sst_len;		/* Number of sectors */
-	size_t sst_dirlen;
+	size_t sst_dirlen;	/* Directory sector size */
-	size_t sst_ss;
+	size_t sst_ss;		/* Sector size */
 } cdf_stream_t;
 
 typedef struct {

ext/fileinfo/libmagic/cdf_time.c

@@ -27,7 +27,7 @@
 #include "file.h"
 
 #ifndef lint
-FILE_RCSID("@(#)$File: cdf_time.c,v 1.15 2014/05/14 23:15:42 christos Exp $")
+FILE_RCSID("@(#)$File: cdf_time.c,v 1.16 2017/03/29 15:57:48 christos Exp $")
 #endif
 
 #include <time.h>
@@ -172,7 +172,7 @@ cdf_ctime(const time_t *sec, char *buf)
 	char *ptr = ctime_r(sec, buf);
 	if (ptr != NULL)
 		return buf;
-	(void)snprintf(buf, 26, "*Bad* 0x%16.16" INT64_T_FORMAT "x\n",
+	(void)snprintf(buf, 26, "*Bad* %#16.16" INT64_T_FORMAT "x\n",
 	    (long long)*sec);
 	return buf;
 }

ext/fileinfo/libmagic/compress.c

@@ -35,7 +35,7 @@
 #include "file.h"
 
 #ifndef lint
-FILE_RCSID("@(#)$File: compress.c,v 1.100 2016/10/24 18:02:17 christos Exp $")
+FILE_RCSID("@(#)$File: compress.c,v 1.104 2017/03/29 15:57:48 christos Exp $")
 #endif
 
 #include "magic.h"
@@ -458,7 +458,7 @@ uncompresszlib(const unsigned char *old, unsigned char **newch,
 	z.next_in = CCAST(Bytef *, old);
 	z.avail_in = CAST(uint32_t, *n);
 	z.next_out = *newch;
-	z.avail_out = bytes_max;
+	z.avail_out = CAST(unsigned int, bytes_max);
 	z.zalloc = Z_NULL;
 	z.zfree = Z_NULL;
 	z.opaque = Z_NULL;
@@ -593,7 +593,7 @@ filter_error(unsigned char *ubuf, ssize_t n)
 		while (isspace((unsigned char)*p))
 			p++;
 		n = strlen(p);
-		memmove(ubuf, p, n + 1);
+		memmove(ubuf, p, CAST(size_t, n + 1));
 	}
 	DPRINTF("Filter error after[[[%s]]]\n", (char *)ubuf);
 	if (islower(*ubuf))
@@ -649,7 +649,7 @@ uncompressbuf(int fd, size_t bytes_max, size_t method, const unsigned char *old,
 		}
 		
 		for (i = 0; i < __arraycount(fdp); i++)
-			copydesc(i, fdp[i]);
+			copydesc(CAST(int, i), fdp[i]);
 
 		(void)execvp(compr[method].argv[0],
 		    (char *const *)(intptr_t)compr[method].argv);

ext/fileinfo/libmagic/der.c

@@ -35,7 +35,7 @@
 #include "file.h"
 
 #ifndef lint
-FILE_RCSID("@(#)$File: der.c,v 1.10 2016/10/24 18:02:17 christos Exp $")
+FILE_RCSID("@(#)$File: der.c,v 1.12 2017/02/10 18:14:01 christos Exp $")
 #endif
 #endif
 
@@ -161,31 +161,49 @@ gettag(const uint8_t *c, size_t *p, size_t l)
 	return tag;
 }
 
+/*
+ * Read the length of a DER tag from the input.
+ *
+ * `c` is the input, `p` is an output parameter that specifies how much of the
+ * input we consumed, and `l` is the maximum input length.
+ *
+ * Returns the length, or DER_BAD if the end of the input is reached or the
+ * length exceeds the remaining input.
+ */
 static uint32_t
 getlength(const uint8_t *c, size_t *p, size_t l)
 {
 	uint8_t digits, i;
 	size_t len;
+	int is_onebyte_result;
 
 	if (*p >= l)
 		return DER_BAD;
 
-	digits = c[(*p)++];
+	/*
-
+	 * Digits can either be 0b0 followed by the result, or 0b1
-        if ((digits & 0x80) == 0)
+	 * followed by the number of digits of the result. In either case,
-		return digits;
+	 * we verify that we can read so many bytes from the input.
-
+	 */
-        digits &= 0x7f;
+	is_onebyte_result = (c[*p] & 0x80) == 0;
-	len = 0;
+	digits = c[(*p)++] & 0x7f;
-
 	if (*p + digits >= l)
 		return DER_BAD;
 
+	if (is_onebyte_result)
+		return digits;
+
+	/*
+	 * Decode len. We've already verified that we're allowed to read
+	 * `digits` bytes.
+	 */
+	len = 0;
 	for (i = 0; i < digits; i++)
 		len = (len << 8) | c[(*p)++];
+
 	if (*p + len >= l)
 		return DER_BAD;
-        return len;
+	return CAST(uint32_t, len);
 }
 
 static const char *
@@ -245,12 +263,12 @@ der_offs(struct magic_set *ms, struct magic *m, size_t nbytes)
 #endif
 	if (m->cont_level != 0) {
 		if (offs + tlen > nbytes)
-			return DER_BAD;
+			return -1;
-		ms->c.li[m->cont_level - 1].off = offs + tlen;
+		ms->c.li[m->cont_level - 1].off = CAST(int, offs + tlen);
 		DPRINTF(("cont_level[%u] = %u\n", m->cont_level - 1,
 		    ms->c.li[m->cont_level - 1].off));
 	}
-	return offs;
+	return CAST(int32_t, offs);
 }
 
 int

ext/fileinfo/libmagic/file.h

@@ -27,7 +27,7 @@
  */
 /*
  * file.h - definitions for file(1) program
- * @(#)$File: file.h,v 1.180 2016/07/20 11:27:08 christos Exp $
+ * @(#)$File: file.h,v 1.182 2017/04/07 19:46:44 christos Exp $
  */
 
 #ifndef __file_h__
@@ -553,7 +553,11 @@ static const char *rcsid(const char *p) { \
 #endif
 
 #ifdef PHP_WIN32
+#ifdef _WIN64
+#define FINFO_LSEEK_FUNC _lseeki64
+#else
 #define FINFO_LSEEK_FUNC _lseek
+#endif
 #define FINFO_READ_FUNC _read
 #else
 #define FINFO_LSEEK_FUNC lseek

ext/fileinfo/libmagic/funcs.c

@@ -27,7 +27,7 @@
 #include "file.h"
 
 #ifndef	lint
-FILE_RCSID("@(#)$File: funcs.c,v 1.90 2016/10/19 20:51:17 christos Exp $")
+FILE_RCSID("@(#)$File: funcs.c,v 1.92 2017/04/07 20:10:24 christos Exp $")
 #endif	/* lint */
 
 #include "magic.h"
@@ -61,7 +61,6 @@ extern public void convert_libmagic_pattern(zval *pattern, char *val, int len, i
 protected int
 file_printf(struct magic_set *ms, const char *fmt, ...)
 {
-	int rv;
 	va_list ap;
 	int len;
 	char *buf = NULL, *newstr;

ext/fileinfo/libmagic/is_tar.c

@@ -40,7 +40,7 @@
 #include "file.h"
 
 #ifndef lint
-FILE_RCSID("@(#)$File: is_tar.c,v 1.38 2015/04/09 20:01:41 christos Exp $")
+FILE_RCSID("@(#)$File: is_tar.c,v 1.39 2017/03/17 20:45:01 christos Exp $")
 #endif
 
 #include "magic.h"
@@ -51,7 +51,7 @@ FILE_RCSID("@(#)$File: is_tar.c,v 1.38 2015/04/09 20:01:41 christos Exp $")
 #define	isodigit(c)	( ((c) >= '0') && ((c) <= '7') )
 
 private int is_tar(const unsigned char *, size_t);
-private int from_oct(int, const char *);	/* Decode octal number */
+private int from_oct(const char *, size_t);	/* Decode octal number */
 
 static const char tartype[][32] = {
 	"tar archive",
@@ -93,31 +93,35 @@ private int
 is_tar(const unsigned char *buf, size_t nbytes)
 {
 	const union record *header = (const union record *)(const void *)buf;
-	int	i;
+	size_t i;
-	int	sum, recsum;
+	int sum, recsum;
-	const unsigned char	*p;
+	const unsigned char *p, *ep;
 
-	if (nbytes < sizeof(union record))
+	if (nbytes < sizeof(*header))
 		return 0;
 
-	recsum = from_oct(8,  header->header.chksum);
+	recsum = from_oct(header->header.chksum, sizeof(header->header.chksum));
 
 	sum = 0;
 	p = header->charptr;
-	for (i = sizeof(union record); --i >= 0;)
+	ep = header->charptr + sizeof(*header);
+	while (p < ep)
 		sum += *p++;
 
 	/* Adjust checksum to count the "chksum" field as blanks. */
-	for (i = sizeof(header->header.chksum); --i >= 0;)
+	for (i = 0; i < sizeof(header->header.chksum); i++)
 		sum -= header->header.chksum[i];
-	sum += ' ' * sizeof header->header.chksum;
+	sum += ' ' * sizeof(header->header.chksum);
 
 	if (sum != recsum)
 		return 0;	/* Not a tar archive */
 
-	if (strcmp(header->header.magic, GNUTMAGIC) == 0)
+	if (strncmp(header->header.magic, GNUTMAGIC,
+	    sizeof(header->header.magic)) == 0)
 		return 3;		/* GNU Unix Standard tar archive */
-	if (strcmp(header->header.magic, TMAGIC) == 0)
+
+	if (strncmp(header->header.magic, TMAGIC,
+	    sizeof(header->header.magic)) == 0)
 		return 2;		/* Unix Standard tar archive */
 
 	return 1;			/* Old fashioned tar archive */
@@ -130,19 +134,22 @@ is_tar(const unsigned char *buf, size_t nbytes)
  * Result is -1 if the field is invalid (all blank, or non-octal).
  */
 private int
-from_oct(int digs, const char *where)
+from_oct(const char *where, size_t digs)
 {
 	int	value;
 
+	if (digs == 0)
+		return -1;
+
 	while (isspace((unsigned char)*where)) {	/* Skip spaces */
 		where++;
-		if (--digs <= 0)
+		if (digs-- == 0)
 			return -1;		/* All blank field */
 	}
 	value = 0;
 	while (digs > 0 && isodigit(*where)) {	/* Scan til non-octal */
 		value = (value << 3) | (*where++ - '0');
-		--digs;
+		digs--;
 	}
 
 	if (digs > 0 && *where && !isspace((unsigned char)*where))

ext/fileinfo/libmagic/magic.c

@@ -40,11 +40,7 @@ FILE_RCSID("@(#)$File: magic.c,v 1.100 2016/07/18 11:43:05 christos Exp $")
 #include <unistd.h>
 #endif
 #include <string.h>
-#ifdef PHP_WIN32
+#include "config.h"
-# include "config.w32.h"
-#else
-# include "php_config.h"
-#endif
 
 #ifdef PHP_WIN32
 #include <shlwapi.h>

ext/fileinfo/libmagic/magic.h

@@ -80,7 +80,7 @@
 #define	MAGIC_NO_CHECK_FORTRAN	0x000000 /* Don't check ascii/fortran */
 #define	MAGIC_NO_CHECK_TROFF	0x000000 /* Don't check ascii/troff */
 
-#define MAGIC_VERSION		529	/* This implementation */
+#define MAGIC_VERSION		531	/* This implementation */
 
 
 #ifdef __cplusplus

ext/fileinfo/libmagic/patchlevel.h

@@ -1,11 +1,15 @@
 #define	FILE_VERSION_MAJOR	5
-#define	patchlevel		29
+#define	patchlevel		31
 
 /*
  * Patchlevel file for Ian Darwin's MAGIC command.
  * $File: patchlevel.h,v 1.68 2008/03/22 21:39:43 christos Exp $
  *
  * $Log$
+ * Revision 2.0  2017/10/11 22:27:12 ab
+ * Update libmagic to 5.31
+ *
+ * $Log$
  * Revision 1.9  2016/11/24 22:27:12 ab
  * Update libmagic to 5.29
  *

ext/fileinfo/libmagic/print.c

@@ -34,7 +34,7 @@
 #include "file.h"
 
 #ifndef lint
-FILE_RCSID("@(#)$File: print.c,v 1.81 2016/01/19 15:09:03 christos Exp $")
+FILE_RCSID("@(#)$File: print.c,v 1.82 2017/02/10 18:14:01 christos Exp $")
 #endif  /* lint */
 
 #include <string.h>

ext/fileinfo/libmagic/readcdf.c

@@ -26,7 +26,7 @@
 #include "file.h"
 
 #ifndef lint
-FILE_RCSID("@(#)$File: readcdf.c,v 1.63 2016/10/18 22:25:42 christos Exp $")
+FILE_RCSID("@(#)$File: readcdf.c,v 1.65 2017/04/08 20:58:03 christos Exp $")
 #endif
 
 #include <assert.h>
@@ -140,7 +140,7 @@ cdf_file_property_info(struct magic_set *ms, const cdf_property_info_t *info,
         struct timeval ts;
         char buf[64];
         const char *str = NULL;
-        const char *s;
+        const char *s, *e;
         int len;
 
 	memset(&ts, 0, sizeof(ts));
@@ -189,7 +189,9 @@ cdf_file_property_info(struct magic_set *ms, const cdf_property_info_t *info,
                                 if (info[i].pi_type == CDF_LENGTH32_WSTRING)
                                     k++;
                                 s = info[i].pi_str.s_buf;
-                                for (j = 0; j < sizeof(vbuf) && len--; s += k) {
+				e = info[i].pi_str.s_buf + len;
+                                for (j = 0; s < e && j < sizeof(vbuf)
+				    && len--; s += k) {
                                         if (*s == '\0')
                                                 break;
                                         if (isprint((unsigned char)*s))
@@ -596,7 +598,7 @@ file_trycdf(struct magic_set *ms, int fd, const unsigned char *buf,
 	if ((i = cdf_read_user_stream(&info, &h, &sat, &ssat, &sst, &dir,
 	    "FileHeader", &scn)) != -1) {
 #define HWP5_SIGNATURE "HWP Document File"
-		if (scn.sst_dirlen >= sizeof(HWP5_SIGNATURE) - 1
+		if (scn.sst_len * scn.sst_ss >= sizeof(HWP5_SIGNATURE) - 1
 		    && memcmp(scn.sst_tab, HWP5_SIGNATURE,
 		    sizeof(HWP5_SIGNATURE) - 1) == 0) {
 		    if (NOTMIME(ms)) {

ext/fileinfo/libmagic/softmagic.c

@@ -32,7 +32,7 @@
 #include "file.h"
 
 #ifndef	lint
-FILE_RCSID("@(#)$File: softmagic.c,v 1.238 2016/10/24 18:02:17 christos Exp $")
+FILE_RCSID("@(#)$File: softmagic.c,v 1.248 2017/04/21 16:54:57 christos Exp $")
 #endif	/* lint */
 
 #include "magic.h"
@@ -198,6 +198,7 @@ flush:
 			while (magindex < nmagic - 1 &&
 			    magic[magindex + 1].cont_level != 0)
 				magindex++;
+			cont_level = 0;
 			continue; /* Skip to next top-level test*/
 		}
 
@@ -376,6 +377,7 @@ flush:
 				case -1:
 				case 0:
 					flush = 1;
+					cont_level--;
 					break;
 				default:
 					break;
@@ -1015,9 +1017,8 @@ private int
 mconvert(struct magic_set *ms, struct magic *m, int flip)
 {
 	union VALUETYPE *p = &ms->ms_value;
-	uint8_t type;
 
-	switch (type = cvt_flip(m->type, flip)) {
+	switch (cvt_flip(m->type, flip)) {
 	case FILE_BYTE:
 		if (cvt_8(p, m) == -1)
 			goto out;
@@ -1182,7 +1183,7 @@ mcopy(struct magic_set *ms, union VALUETYPE *p, int type, int indir,
 		case FILE_DER:
 		case FILE_SEARCH:
 			if (offset > nbytes)
-				offset = nbytes;
+				offset = CAST(uint32_t, nbytes);
 			ms->search.s = RCAST(const char *, s) + offset;
 			ms->search.s_len = nbytes - offset;
 			ms->search.offset = offset;
@@ -1265,7 +1266,8 @@ mcopy(struct magic_set *ms, union VALUETYPE *p, int type, int indir,
 				if (*dst == '\0') {
 					if (type == FILE_BESTRING16 ?
 					    *(src - 1) != '\0' :
-					    *(src + 1) != '\0')
+					    ((src + 1 < esrc) &&
+					    *(src + 1) != '\0'))
 						*dst = ' ';
 				}
 			}
@@ -1370,7 +1372,7 @@ mget(struct magic_set *ms, const unsigned char *s, struct magic *m,
 		return -1;
 
 	if ((ms->flags & MAGIC_DEBUG) != 0) {
-		fprintf(stderr, "mget(type=%d, flag=%x, offset=%u, o=%"
+		fprintf(stderr, "mget(type=%d, flag=%#x, offset=%u, o=%"
 		    SIZE_T_FORMAT "u, " "nbytes=%" SIZE_T_FORMAT
 		    "u, il=%hu, nc=%hu)\n",
 		    m->type, m->flag, offset, o, nbytes,
@@ -1631,6 +1633,7 @@ file_strncmp(const char *s1, const char *s2, size_t len, uint32_t flags)
 	 */
 	const unsigned char *a = (const unsigned char *)s1;
 	const unsigned char *b = (const unsigned char *)s2;
+	const unsigned char *eb = b + len;
 	uint64_t v;
 
 	/*
@@ -1645,6 +1648,10 @@ file_strncmp(const char *s1, const char *s2, size_t len, uint32_t flags)
 	}
 	else { /* combine the others */
 		while (len-- > 0) {
+			if (b >= eb) {
+				v = 1;
+				break;
+			}
 			if ((flags & STRING_IGNORE_LOWERCASE) &&
 			    islower(*a)) {
 				if ((v = tolower(*b++) - *a++) != '\0')
@@ -1659,14 +1666,9 @@ file_strncmp(const char *s1, const char *s2, size_t len, uint32_t flags)
 			    isspace(*a)) {
 				a++;
 				if (isspace(*b++)) {
-					if (!isspace(*a)) {
+					if (!isspace(*a))
-						/* Limit to the remaining len only, otherwise
+						while (b < eb && isspace(*b))
-							we risk to cause CVE-2014-3538. This
-							might be done better though, not sure. */
-						size_t remaining_len = len;
-						while (isspace(*b) && remaining_len-- > 0)
 							b++;
-					}
 				}
 				else {
 					v = 1;
@@ -1675,10 +1677,8 @@ file_strncmp(const char *s1, const char *s2, size_t len, uint32_t flags)
 			}
 			else if ((flags & STRING_COMPACT_OPTIONAL_WHITESPACE) &&
 			    isspace(*a)) {
-				/* Same as the comment in the previous elif clause. */
-				size_t remaining_len = len;
 				a++;
-				while (isspace(*b)&& remaining_len-- > 0)
+				while (b < eb && isspace(*b))
 					b++;
 			}
 			else {
@@ -1884,13 +1884,13 @@ magiccheck(struct magic_set *ms, struct magic *m)
 
 		for (idx = 0; m->str_range == 0 || idx < m->str_range; idx++) {
 			if (slen + idx > ms->search.s_len)
-				break;
+				return 0;
 
 			v = file_strncmp(m->value.s, ms->search.s + idx, slen,
 			    m->str_flags);
 			if (v == 0) {	/* found match */
 				ms->search.offset += idx;
-				ms->search.rm_len = m->str_range - idx;
+				ms->search.rm_len = ms->search.s_len - idx;
 				break;
 			}
 		}

ext/fileinfo/magicdata.patch

@@ -1,9 +1,6 @@
-Patches applied to file 5.17 sources tree before generating magic.mgc
+diff -u file-5.31.orig/magic/Magdir/cafebabe file-5.31/magic/Magdir/cafebabe
-and before running create_data_file.php to create data_file.c.
+--- file-5.31.orig/magic/Magdir/cafebabe	2017-03-17 21:35:28.000000000 +0000
-
++++ file-5.31/magic/Magdir/cafebabe	2017-10-11 14:30:57.476077825 +0000
-diff -u magic/Magdir.orig.0/cafebabe magic/Magdir/cafebabe
---- magic/Magdir.orig.0/cafebabe	2015-10-16 00:54:18.000000000 +0000
-+++ magic/Magdir/cafebabe	2016-11-24 15:59:25.553060242 +0000
 @@ -15,22 +15,6 @@
  # might add another one or two as time goes by...
  #
@@ -60,29 +57,41 @@ diff -u magic/Magdir.orig.0/cafebabe magic/Magdir/cafebabe
  >>>48	use		mach-o		\b
  >>4	belong		4
  >>>68	use		mach-o		\b
-diff -u magic/Magdir.orig.0/commands magic/Magdir/commands
+diff -u file-5.31.orig/magic/Magdir/commands file-5.31/magic/Magdir/commands
---- magic/Magdir.orig.0/commands	2016-07-14 19:01:12.000000000 +0000
+--- file-5.31.orig/magic/Magdir/commands	2017-04-04 20:34:24.000000000 +0000
-+++ magic/Magdir/commands	2016-11-24 15:59:25.557060441 +0000
++++ file-5.31/magic/Magdir/commands	2017-10-11 14:29:57.653552987 +0000
 @@ -56,7 +56,7 @@
  !:mime	text/x-awk
  0	string/wt	#!\ /usr/bin/awk	awk script text executable
  !:mime	text/x-awk
--0	regex/4096	=^\\s{0,100}BEGIN\\s{0,100}[{]	awk or perl script text
+-0	regex/4096	=^[A-Za-z0-9_]{0,100}BEGIN[A-Za-z0-9_]{0,100}[{]	awk or perl script text
-+0	regex/4096	=^\\s{0,100}BEGIN\\s{0,100}[{]	awk script text
++0	regex/4096	=^[\040\t\f\r\n]{0,100}BEGIN[\040\t\f\r\n]{0,100}[{]	awk or perl script text
  
  # AT&T Bell Labs' Plan 9 shell
  0	string/wt	#!\ /bin/rc	Plan 9 rc shell script text executable
-diff -u magic/Magdir.orig.0/filesystems magic/Magdir/filesystems
+diff -u file-5.31.orig/magic/Magdir/dump file-5.31/magic/Magdir/dump
---- magic/Magdir.orig.0/filesystems	2016-09-05 08:34:25.000000000 +0000
+--- file-5.31.orig/magic/Magdir/dump	2017-03-17 21:35:28.000000000 +0000
-+++ magic/Magdir/filesystems	2016-11-24 15:59:25.553060242 +0000
++++ file-5.31/magic/Magdir/dump	2017-10-11 14:30:09.622057792 +0000
+@@ -69,6 +69,8 @@
+ >0	use	old-dump-be
+ 
+ 24	lelong	60012		new-fs dump file (little endian),
++# to correctly recognize '*.mo' GNU message catalog (little endian)
++!:strength - 15
+ >0	use	\^new-dump-be
+ 
+ 24	lelong	60011		old-fs dump file (little endian),
+diff -u file-5.31.orig/magic/Magdir/filesystems file-5.31/magic/Magdir/filesystems
+--- file-5.31.orig/magic/Magdir/filesystems	2017-03-24 19:29:26.000000000 +0000
++++ file-5.31/magic/Magdir/filesystems	2017-10-11 14:30:57.476077825 +0000
 @@ -254,7 +254,7 @@
- 30		search/481	\x55\xAA	
+ 30		search/481	\x55\xAA
  # to display DOS/MBR boot sector (40) before old one (strength=50+21),Syslinux bootloader (71),SYSLINUX MBR (37+36),NetBSD mbr (110),AdvanceMAME mbr (111)
  # DOS BPB information (70) and after DOS floppy (120) like in previous file version
 -!:strength +65
 +!:strength +0
  # for sector sizes < 512 Bytes
- >11		uleshort	<512		
+ >11		uleshort	<512
  >>(11.s-2)	uleshort	0xAA55		DOS/MBR boot sector
 @@ -266,7 +266,7 @@
  0x1FE          leshort         0xAA55         	DOS/MBR boot sector
@@ -93,191 +102,74 @@ diff -u magic/Magdir.orig.0/filesystems magic/Magdir/filesystems
  >2		string		OSBS		OS/BS MBR
  # added by Joerg Jenderek at Feb 2013 according to http://thestarman.pcministry.com/asm/mbr/
  # and http://en.wikipedia.org/wiki/Master_Boot_Record
-diff -u magic/Magdir.orig.0/msdos magic/Magdir/msdos
+Only in file-5.31/magic/Magdir/: gconv
---- magic/Magdir.orig.0/msdos	2016-09-14 01:26:26.000000000 +0000
+diff -u file-5.31.orig/magic/Magdir/images file-5.31/magic/Magdir/images
-+++ magic/Magdir/msdos	2016-11-24 15:58:56.327609010 +0000
+--- file-5.31.orig/magic/Magdir/images	2017-04-04 20:34:24.000000000 +0000
-@@ -404,7 +404,7 @@
++++ file-5.31/magic/Magdir/images	2017-10-11 14:29:57.653552987 +0000
- # GRR: line below too general as it catches also 
+@@ -175,42 +175,42 @@
- # rt.lib DYADISKS.PIC and many more
+ >>>&0	regex		=[0-9]{1,50}			\b %s
- # start with assembler instruction MOV
--0	ubyte		0x8c		
-+#0	ubyte		0x8c		
- # skip "AppleWorks word processor data" like ARTICLE.1 ./apple
- >4	string			!O====	
- # skip some unknown basic binaries like RocketRnger.SHR
-@@ -428,17 +428,17 @@
- # updated by Joerg Jenderek at Oct 2008
- 0	ulelong		0xffff10eb	DR-DOS executable (COM)
- # byte 0xeb conflicts with "sequent" magic leshort 0xn2eb
--0	ubeshort&0xeb8d	>0xeb00		
-+#0	ubeshort&0xeb8d	>0xeb00		
- # DR-DOS STACKER.COM SCREATE.SYS missed
  
- 0       name    msdos-com
+ 0	search/1	P1
- >0  byte        x               DOS executable (COM)
+->0	regex/4		P1[A-Za-z0-9_]
- >6	string		SFX\ of\ LHarc	\b, %s
++>0	regex/4		P1[\040\t\f\r\n]
-->0x1FE leshort	0xAA55		    \b, boot code
+ >>0	use		netpbm
-->85	string		UPX		        \b, UPX compressed
+ >>>0	string		x	\b, bitmap
-->4	string		\ $ARX		    \b, ARX self-extracting archive
+ !:strength + 45
-->4	string		\ $LHarc	    \b, LHarc self-extracting archive
+ !:mime	image/x-portable-bitmap
-->0x20e string	SFX\ by\ LARC	\b, LARC self-extracting archive
-+#>0x1FE leshort	0xAA55		    \b, boot code
-+#>85	string		UPX		        \b, UPX compressed
-+#>4	string		\ $ARX		    \b, ARX self-extracting archive
-+#>4	string		\ $LHarc	    \b, LHarc self-extracting archive
-+#>0x20e string	SFX\ by\ LARC	\b, LARC self-extracting archive
  
- # JMP 8bit
+ 0	search/1	P2
- 0	        byte	0xeb
+->0	regex/4		P2[A-Za-z0-9_]
-@@ -508,27 +508,27 @@
++>0	regex/4		P2[\040\t\f\r\n]
- # GRR search is not working
+ >>0	use		netpbm
- #2	search/28	\xcd\x21	COM executable for MS-DOS
+ >>>0	string		x	\b, greymap
- #WHICHFAT.cOM
+ !:strength + 45
--2	string	\xcd\x21		COM executable for DOS
+ !:mime	image/x-portable-greymap
-+#2	string	\xcd\x21		COM executable for DOS
- #DELTREE.cOM DELTREE2.cOM
--4	string	\xcd\x21		COM executable for DOS
-+#4	string	\xcd\x21		COM executable for DOS
- #IFMEMDSK.cOM ASSIGN.cOM COMP.cOM
--5	string	\xcd\x21		COM executable for DOS
-+#5	string	\xcd\x21		COM executable for DOS
- #DELTMP.COm HASFAT32.cOM
--7	string	\xcd\x21		
-->0	byte	!0xb8			COM executable for DOS
-+#7	string	\xcd\x21		
-+#>0	byte	!0xb8			COM executable for DOS
- #COMP.cOM MORE.COm
--10	string	\xcd\x21		
-->5	string	!\xcd\x21		COM executable for DOS
-+#10	string	\xcd\x21		
-+#>5	string	!\xcd\x21		COM executable for DOS
- #comecho.com
--13	string	\xcd\x21		COM executable for DOS
-+#13	string	\xcd\x21		COM executable for DOS
- #HELP.COm EDIT.coM
--18	string	\xcd\x21		COM executable for MS-DOS
-+#18	string	\xcd\x21		COM executable for MS-DOS
- #NWRPLTRM.COm
--23	string	\xcd\x21		COM executable for MS-DOS
-+#23	string	\xcd\x21		COM executable for MS-DOS
- #LOADFIX.cOm LOADFIX.cOm
--30	string	\xcd\x21		COM executable for MS-DOS
-+#30	string	\xcd\x21		COM executable for MS-DOS
- #syslinux.com 3.11
--70	string	\xcd\x21		COM executable for DOS
-+#70	string	\xcd\x21		COM executable for DOS
- # many compressed/converted COMs start with a copy loop instead of a jump
- 0x6	search/0xa	\xfc\x57\xf3\xa5\xc3	COM executable for MS-DOS
- 0x6	search/0xa	\xfc\x57\xf3\xa4\xc3	COM executable for DOS
-diff -u magic/Magdir.orig.0/perl magic/Magdir/perl
---- magic/Magdir.orig.0/perl	2016-06-12 16:50:16.000000000 +0000
-+++ magic/Magdir/perl	2016-11-24 15:59:25.553060242 +0000
-@@ -7,40 +7,54 @@
- # Send additions to <perl5-porters@perl.org>
- 0	search/1024	eval\ "exec\ perl		Perl script text
- !:mime	text/x-perl
-+!:strength + 30
- 0	search/1024	eval\ "exec\ /bin/perl		Perl script text
- !:mime	text/x-perl
-+!:strength + 30
- 0	search/1024	eval\ "exec\ /usr/bin/perl	Perl script text
- !:mime	text/x-perl
-+!:strength + 30
- 0	search/1024	eval\ "exec\ /usr/local/bin/perl	Perl script text
- !:mime	text/x-perl
-+!:strength + 30
- 0	search/1024	eval\ 'exec\ perl		Perl script text
- !:mime	text/x-perl
-+!:strength + 30
- 0	search/1024	eval\ 'exec\ /bin/perl		Perl script text
- !:mime	text/x-perl
-+!:strength + 30
- 0	search/1024	eval\ 'exec\ /usr/bin/perl	Perl script text
- !:mime	text/x-perl
-+!:strength + 30
- 0	search/1024	eval\ 'exec\ /usr/local/bin/perl	Perl script text
- !:mime	text/x-perl
-+!:strength + 30
- 0	search/1024	eval\ '(exit\ $?0)'\ &&\ eval\ 'exec	Perl script text
- !:mime	text/x-perl
-+!:strength + 1
- 0	string	#!/usr/bin/env\ perl	Perl script text executable
- !:mime	text/x-perl
-+!:strength + 30
- 0	string	#!\ /usr/bin/env\ perl	Perl script text executable
- !:mime	text/x-perl
-+!:strength + 30
- 0	string	#!
- >0	regex	\^#!.*/bin/perl([[:space:]].*)*$	Perl script text executable
- !:mime	text/x-perl
-+!:strength + 30
  
- # by Dmitry V. Levin and Alexey Tourbin
+ 0	search/1	P3
- # check the first line
+->0	regex/4		P3[A-Za-z0-9_]
- 0	search/1024	package
++>0	regex/4		P3[\040\t\f\r\n]
- >0	regex		\^package[\ \t]+[0-9A-Za-z_:]+\ *;	Perl5 module source text
+ >>0	use		netpbm
-+!:mime  text/x-perl
+ >>>0	string		x	\b, pixmap
- !:strength + 10
+ !:strength + 45
- # not 'p', check other lines
+ !:mime	image/x-portable-pixmap
- 0	search/1024	!p
- >0	regex		\^package[\ \t]+[0-9A-Za-z_:]+\ *;
- >>0	regex		\^1\ *;|\^(use|sub|my)\ .*[(;{=]	Perl5 module source text
--!:strength + 10
-+!:mime  text/x-perl
-+!:strength + 40
  
- # Perl POD documents
+ 0	string		P4
- # From: Tom Hukins <tom@eborcom.com>
+->0	regex/4		P4[A-Za-z0-9_]
-diff -u magic/Magdir.orig.0/python magic/Magdir/python
++>0	regex/4		P4[\040\t\f\r\n]
---- magic/Magdir.orig.0/python	2016-09-16 12:06:13.000000000 +0000
+ >>0	use		netpbm
-+++ magic/Magdir/python	2016-11-24 15:59:25.549060044 +0000
+ >>>0	string		x	\b, rawbits, bitmap
-@@ -43,20 +43,24 @@
+ !:strength + 45
- # from module.submodule import func1, func2
+ !:mime	image/x-portable-bitmap
- 0	regex	\^from\\s+(\\w|\\.)+\\s+import.*$	Python script text executable
- !:mime text/x-python
-+!:strength + 15
  
- # def __init__ (self, ...):
+ 0	string		P5
- 0	search/4096	def\ __init__
+->0	regex/4		P5[A-Za-z0-9_]
- >&0	search/64 self	Python script text executable
++>0	regex/4		P5[\040\t\f\r\n]
- !:mime text/x-python
+ >>0	use		netpbm
-+!:strength + 15
+ >>>0	string		x	\b, rawbits, greymap
+ !:strength + 45
+ !:mime	image/x-portable-greymap
  
- # comments
+ 0	string		P6
--#0	search/4096	'''
+->0	regex/4		P6[A-Za-z0-9_]
--#>&0	regex	.*'''$	Python script text executable
++>0	regex/4		P6[\040\t\f\r\n]
--#!:mime text/x-python
+ >>0	use		netpbm
-+0	search/4096	'''
+ >>>0	string		x	\b, rawbits, pixmap
-+>&0	regex	.*'''$	Python script text executable
+ !:strength + 45
-+!:mime text/x-python
+diff -u file-5.31.orig/magic/Magdir/python file-5.31/magic/Magdir/python
-+!:strength + 15
+--- file-5.31.orig/magic/Magdir/python	2017-05-08 18:10:13.000000000 +0000
- 
++++ file-5.31/magic/Magdir/python	2017-10-11 14:29:57.653552987 +0000
--#0	search/4096	"""
+@@ -64,7 +64,7 @@
--#>&0	regex	.*"""$	Python script text executable
--#!:mime text/x-python
-+0	search/4096	"""
-+>&0	regex	.*"""$	Python script text executable
-+!:mime text/x-python
-+!:strength + 15
- 
- # try:
  # except: or finally:
-@@ -66,8 +70,10 @@
+ # block
+ 0	search/4096	try:
+->&0	regex	\^[A-Za-z0-9_]*except.*:	Python script text executable
++>&0	regex	\^[\040\t\f\r\n]*except.*:	Python script text executable
+ !:strength + 15
  !:mime text/x-python
  >&0	search/4096	finally:	Python script text executable
- !:mime text/x-python
+diff -u file-5.31.orig/magic/Magdir/rpm file-5.31/magic/Magdir/rpm
-+!:strength + 15
+--- file-5.31.orig/magic/Magdir/rpm	2017-03-17 21:34:26.000000000 +0000
- 
++++ file-5.31/magic/Magdir/rpm	2017-10-11 14:30:57.480078009 +0000
- # def name(args, args):
--0	regex	 \^(\ |\\t){0,50}def\ {1,50}[a-zA-Z]{1,100}
-->&0	regex	\ {0,50}\\(([a-zA-Z]|,|\ ){1,255}\\):$ Python script text executable
-+0	regex	 \^(\ |\\t)*def\ +[a-zA-Z]+
-+>&0	regex	\ *\\(([a-zA-Z]|,|\ )*\\):$ Python script text executable
- !:mime text/x-python
-+!:strength + 15
-diff -u magic/Magdir.orig.0/rpm magic/Magdir/rpm
---- magic/Magdir.orig.0/rpm	2014-09-11 15:03:07.000000000 +0000
-+++ magic/Magdir/rpm	2016-11-24 15:58:28.458225125 +0000
 @@ -29,6 +29,7 @@
  >>8	beshort		17		SuperH
  >>8	beshort		18		Xtensa
@@ -286,18 +178,76 @@ diff -u magic/Magdir.orig.0/rpm magic/Magdir/rpm
  
  #delta RPM    Daniel Novotny (dnovotny@redhat.com)
  0	string		drpm		Delta RPM
-diff -u magic/Magdir.orig.0/securitycerts magic/Magdir/securitycerts
+diff -u file-5.31.orig/magic/Magdir/ruby file-5.31/magic/Magdir/ruby
---- magic/Magdir.orig.0/securitycerts	2014-09-11 15:03:07.000000000 +0000
+--- file-5.31.orig/magic/Magdir/ruby	2017-03-17 21:34:26.000000000 +0000
-+++ magic/Magdir/securitycerts	2016-11-24 15:59:25.549060044 +0000
++++ file-5.31/magic/Magdir/ruby	2017-10-11 14:30:24.410682267 +0000
+@@ -6,27 +6,46 @@
+ # From: Reuben Thomas <rrt@sc3d.org>
+ 
+ # Ruby scripts
+-0	search/1/w	#!\ /usr/bin/ruby	Ruby script text executable
++0	search/1/w	#!\ /usr/bin/ruby				Ruby script text executable
+ !:strength + 15
+ !:mime text/x-ruby
+ 0	search/1/w	#!\ /usr/local/bin/ruby	Ruby script text executable
+ !:strength + 15
+ !:mime text/x-ruby
+-0	search/1	#!/usr/bin/env\ ruby	Ruby script text executable
++0	search/1	#!/usr/bin/env\ ruby				Ruby script text executable
+ !:strength + 15
+ !:mime text/x-ruby
+-0	search/1	#!\ /usr/bin/env\ ruby	Ruby script text executable
++0	search/1	#!\ /usr/bin/env\ ruby			Ruby script text executable
+ !:strength + 15
+ !:mime text/x-ruby
+ 
+ # What looks like ruby, but does not have a shebang
+ # (modules and such)
+ # From: Lubomir Rintel <lkundrak@v3.sk>
+-0	regex		\^[\ \t]*require[\ \t]'[A-Za-z_/]+'
+->0	regex		include\ [A-Z]|def\ [a-z]|\ do$
+->>0	regex		\^[\ \t]*end([\ \t]*[;#].*)?$		Ruby script text
++0	regex		\^[[:space:]]*require[[:space:]]'[A-Za-z_/]+'
++>0	regex		def\ [a-z]|\ do$
++>>&0	regex		\^[[:space:]]*end([[:space:]]+[;#].*)?$		Ruby script text
++!:strength + 30
+ !:mime	text/x-ruby
+-0	regex		\^[\ \t]*(class|module)[\ \t][A-Z]
++0	regex		\^[[:space:]]*(class|module)[[:space:]][A-Z]
+ >0	regex		(modul|includ)e\ [A-Z]|def\ [a-z]
+->>0	regex		\^[\ \t]*end([\ \t]*[;#].*)?$		Ruby module source text
++>>&0	regex		\^[[:space:]]*end([[:space:]]+[;#].*)?$		Ruby script text
++!:strength + 30
++!:mime	text/x-ruby
++# Classes with no modules or defs, beats simple ASCII
++0	regex		\^[[:space:]]*(class|module)[[:space:]][A-Z]
++>&0	regex	\^[[:space:]]*end([[:space:]]+[;#if].*)?$		Ruby script text
++!:strength + 10
++!:mime	text/x-ruby
++# Looks for function definiton to balance python magic
++# def name (args)
++# end
++0	regex		\^[[:space:]]*def\ [a-z]|def\ [[:alpha:]]+::[a-z]
++>&0	regex		\^[[:space:]]*end([[:space:]]+[;#].*)?$		Ruby script text
++!:strength + 10
++!:mime	text/x-ruby
++
++0	regex		\^[[:space:]]*require[[:space:]]'[A-Za-z_/]+'	Ruby script text
++!:mime	text/x-ruby
++0 regex 	\^[[:space:]]*include\ ([A-Z]+[a-z]*(::))+	Ruby script text
+ !:mime	text/x-ruby
+diff -u file-5.31.orig/magic/Magdir/securitycerts file-5.31/magic/Magdir/securitycerts
+--- file-5.31.orig/magic/Magdir/securitycerts	2017-03-17 21:34:26.000000000 +0000
++++ file-5.31/magic/Magdir/securitycerts	2017-10-11 14:30:57.480078009 +0000
 @@ -4,3 +4,5 @@
  0	search/1		-----BEGIN\ CERTIFICATE------	RFC1421 Security Certificate text
  0	search/1		-----BEGIN\ NEW\ CERTIFICATE	RFC1421 Security Certificate Signing Request text
  0	belong	0xedfeedfe	Sun 'jks' Java Keystore File data
 +
 +0	string \0volume_key	volume_key escrow packet
-diff -u magic/Magdir.orig.0/varied.script magic/Magdir/varied.script
+diff -u file-5.31.orig/magic/Magdir/varied.script file-5.31/magic/Magdir/varied.script
---- magic/Magdir.orig.0/varied.script	2015-03-27 17:59:39.000000000 +0000
+--- file-5.31.orig/magic/Magdir/varied.script	2017-03-17 21:34:26.000000000 +0000
-+++ magic/Magdir/varied.script	2016-11-24 15:59:25.557060441 +0000
++++ file-5.31/magic/Magdir/varied.script	2017-10-11 14:30:57.480078009 +0000
 @@ -4,27 +4,35 @@
  
  0	string/t		#!\ /			a