archive/php-src
1
0
Fork 0
mirror of https://github.com/php/php-src.git synced 2025-08-15 21:48:51 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Fix mbstring support for CP1252 encoding

Browse source 
It's a bit surprising how much was broken here.

- Identify filter was utterly and completely wrong.
- Instead of handling invalid CP1252 bytes as specified by
  `mb_substitute_character`, it would convert them to Unicode 0xFFFD
  (generic replacement character).
- When converting ISO-8859-1 to CP1252, invalid ISO-8859-1 bytes would
  be passed through silently.
- Unicode codepoints from 0x80-0x9F were converted to CP1252 bytes 0x80-0x9F,
  which is wrong.
- Unicode codepoint 0xFFFD was converted to CP1252 0x9F, which is very wrong.

Also clean up some unneeded code, and make the conversion table consistent with
others by using zero as a 'invalid' marker, rather than 0xFFFD.
This commit is contained in:
Alex Dowad 2020-10-18 14:56:32 +02:00
parent 2ce15d9c8d
commit b5ff87ca71
2 changed files with 17 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

43
ext/mbstring/libmbfl/filters/mbfilter_cp1252.c
View file

@ -74,32 +74,22 @@ const struct mbfl_convert_vtbl vtbl_wchar_cp1252 = {
#define CK(statement) do { if ((statement) < 0) return (-1); } while (0)
/*
* wchar => cp1252
*/
int mbfl_filt_conv_wchar_cp1252(int c, mbfl_convert_filter *filter)
{
int s=-1, n;
int s = -1;
if (c >= 0x100) {
/* look it up from the cp1252 table */
s = -1;
n = 31;
while (n >= 0) {
if (c == cp1252_ucs_table[n] && c != 0xfffe) {
/* Look it up from the CP1252 table */
for (int n = 31; n >= 0; n--) {
if (c == cp1252_ucs_table[n]) {
s = 0x80 + n;
break;
}
n--;
}
if (s <= 0 && (c & ~MBFL_WCSPLANE_MASK) == MBFL_WCSPLANE_8859_1)
{
s = c & MBFL_WCSPLANE_MASK;
}
}
else if (c >= 0 && c < 0x100) {
} else if (c <= 0x7F || c >= 0xA0) {
s = c;
}
if (s >= 0) {
CK((*filter->output_function)(s, filter->data));
} else {
@ -108,15 +98,15 @@ int mbfl_filt_conv_wchar_cp1252(int c, mbfl_convert_filter *filter)
return c;
}
/*
* cp1252 => wchar
*/
int mbfl_filt_conv_cp1252_wchar(int c, mbfl_convert_filter *filter)
{
int s;
if (c >= 0x80 && c < 0xa0) {
if (c >= 0x80 && c < 0xA0) {
s = cp1252_ucs_table[c - 0x80];
if (!s) {
s = c | MBFL_WCSGROUP_THROUGH;
}
} else {
s = c;
}
@ -126,17 +116,10 @@ int mbfl_filt_conv_cp1252_wchar(int c, mbfl_convert_filter *filter)
return c;
}
/* We only distinguish the MS extensions to ISO-8859-1.
* Actually, this is pretty much a NO-OP, since the identification
* system doesn't allow us to discriminate between a positive match,
* a possible match and a definite non-match.
* The problem here is that cp1252 looks like SJIS for certain chars.
* */
static int mbfl_filt_ident_cp1252(int c, mbfl_identify_filter *filter)
{
if (c >= 0x80 && c < 0xa0)
filter->flag = 0;
else
filter->flag = 1; /* not it */
if (c >= 0x80 && c < 0xA0 && !cp1252_ucs_table[c - 0x80]) {
filter->flag = 1;
}
return c;
}

8
ext/mbstring/libmbfl/filters/unicode_table_cp1252.h
View file

@ -32,9 +32,9 @@
* as it only covers this range, while the rest cover 0xa0 onwards */
static const unsigned short cp1252_ucs_table[] = {
0x20ac,0xfffd,0x201a,0x0192,0x201e,0x2026,0x2020,0x2021,
0x02c6,0x2030,0x0160,0x2039,0x0152,0xfffd,0x017d,0xfffd,
0xfffd,0x2018,0x2019,0x201c,0x201d,0x2022,0x2013,0x2014,
0x02dc,0x2122,0x0161,0x203a,0x0153,0xfffd,0x017e,0x0178
0x20ac,0x0000,0x201a,0x0192,0x201e,0x2026,0x2020,0x2021,
0x02c6,0x2030,0x0160,0x2039,0x0152,0x0000,0x017d,0x0000,
0x0000,0x2018,0x2019,0x201c,0x201d,0x2022,0x2013,0x2014,
0x02dc,0x2122,0x0161,0x203a,0x0153,0x0000,0x017e,0x0178
};
#endif /* UNICODE_TABLE_CP1252_H */