diff options
Diffstat (limited to 'Modules/cjkcodecs/_codecs_jp.c')
-rw-r--r-- | Modules/cjkcodecs/_codecs_jp.c | 325 |
1 files changed, 159 insertions, 166 deletions
diff --git a/Modules/cjkcodecs/_codecs_jp.c b/Modules/cjkcodecs/_codecs_jp.c index a500696e931..7ab318bcc11 100644 --- a/Modules/cjkcodecs/_codecs_jp.c +++ b/Modules/cjkcodecs/_codecs_jp.c @@ -19,38 +19,39 @@ ENCODER(cp932) { - while (inleft > 0) { - Py_UNICODE c = IN1; + while (*inpos < inlen) { + Py_UCS4 c = INCHAR1; DBCHAR code; unsigned char c1, c2; if (c <= 0x80) { - WRITE1((unsigned char)c) - NEXT(1, 1) + WRITEBYTE1((unsigned char)c) + NEXT(1, 1); continue; } else if (c >= 0xff61 && c <= 0xff9f) { - WRITE1(c - 0xfec0) - NEXT(1, 1) + WRITEBYTE1(c - 0xfec0) + NEXT(1, 1); continue; } else if (c >= 0xf8f0 && c <= 0xf8f3) { /* Windows compatibility */ REQUIRE_OUTBUF(1) if (c == 0xf8f0) - OUT1(0xa0) + OUTBYTE1(0xa0) else - OUT1(c - 0xfef1 + 0xfd) - NEXT(1, 1) + OUTBYTE1(c - 0xfef1 + 0xfd) + NEXT(1, 1); continue; } - UCS4INVALID(c) + if (c > 0xFFFF) + return 1; REQUIRE_OUTBUF(2) TRYMAP_ENC(cp932ext, code, c) { - OUT1(code >> 8) - OUT2(code & 0xff) + OUTBYTE1(code >> 8) + OUTBYTE2(code & 0xff) } else TRYMAP_ENC(jisxcommon, code, c) { if (code & 0x8000) /* MSB set: JIS X 0212 */ @@ -61,20 +62,20 @@ ENCODER(cp932) c2 = code & 0xff; c2 = (((c1 - 0x21) & 1) ? 0x5e : 0) + (c2 - 0x21); c1 = (c1 - 0x21) >> 1; - OUT1(c1 < 0x1f ? c1 + 0x81 : c1 + 0xc1) - OUT2(c2 < 0x3f ? c2 + 0x40 : c2 + 0x41) + OUTBYTE1(c1 < 0x1f ? c1 + 0x81 : c1 + 0xc1) + OUTBYTE2(c2 < 0x3f ? c2 + 0x40 : c2 + 0x41) } else if (c >= 0xe000 && c < 0xe758) { /* User-defined area */ - c1 = (Py_UNICODE)(c - 0xe000) / 188; - c2 = (Py_UNICODE)(c - 0xe000) % 188; - OUT1(c1 + 0xf0) - OUT2(c2 < 0x3f ? c2 + 0x40 : c2 + 0x41) + c1 = (Py_UCS4)(c - 0xe000) / 188; + c2 = (Py_UCS4)(c - 0xe000) % 188; + OUTBYTE1(c1 + 0xf0) + OUTBYTE2(c2 < 0x3f ? c2 + 0x40 : c2 + 0x41) } else return 1; - NEXT(1, 2) + NEXT(1, 2); } return 0; @@ -83,33 +84,32 @@ ENCODER(cp932) DECODER(cp932) { while (inleft > 0) { - unsigned char c = IN1, c2; + unsigned char c = INBYTE1, c2; - REQUIRE_OUTBUF(1) if (c <= 0x80) { - OUT1(c) - NEXT(1, 1) + OUTCHAR(c); + NEXT_IN(1); continue; } else if (c >= 0xa0 && c <= 0xdf) { if (c == 0xa0) - OUT1(0xf8f0) /* half-width katakana */ + OUTCHAR(0xf8f0); /* half-width katakana */ else - OUT1(0xfec0 + c) - NEXT(1, 1) + OUTCHAR(0xfec0 + c); + NEXT_IN(1); continue; } else if (c >= 0xfd/* && c <= 0xff*/) { /* Windows compatibility */ - OUT1(0xf8f1 - 0xfd + c) - NEXT(1, 1) + OUTCHAR(0xf8f1 - 0xfd + c); + NEXT_IN(1); continue; } REQUIRE_INBUF(2) - c2 = IN2; + c2 = INBYTE2; - TRYMAP_DEC(cp932ext, **outbuf, c, c2); + TRYMAP_DEC(cp932ext, writer, c, c2); else if ((c >= 0x81 && c <= 0x9f) || (c >= 0xe0 && c <= 0xea)){ if (c2 < 0x40 || (c2 > 0x7e && c2 < 0x80) || c2 > 0xfc) return 1; @@ -119,21 +119,21 @@ DECODER(cp932) c = (2 * c + (c2 < 0x5e ? 0 : 1) + 0x21); c2 = (c2 < 0x5e ? c2 : c2 - 0x5e) + 0x21; - TRYMAP_DEC(jisx0208, **outbuf, c, c2); + TRYMAP_DEC(jisx0208, writer, c, c2); else return 1; } else if (c >= 0xf0 && c <= 0xf9) { if ((c2 >= 0x40 && c2 <= 0x7e) || (c2 >= 0x80 && c2 <= 0xfc)) - OUT1(0xe000 + 188 * (c - 0xf0) + - (c2 < 0x80 ? c2 - 0x40 : c2 - 0x41)) + OUTCHAR(0xe000 + 188 * (c - 0xf0) + + (c2 < 0x80 ? c2 - 0x40 : c2 - 0x41)); else return 1; } else return 1; - NEXT(2, 1) + NEXT_IN(2); } return 0; @@ -146,25 +146,24 @@ DECODER(cp932) ENCODER(euc_jis_2004) { - while (inleft > 0) { - ucs4_t c = IN1; + while (*inpos < inlen) { + Py_UCS4 c = INCHAR1; DBCHAR code; Py_ssize_t insize; if (c < 0x80) { - WRITE1(c) - NEXT(1, 1) + WRITEBYTE1(c) + NEXT(1, 1); continue; } - DECODE_SURROGATE(c) - insize = GET_INSIZE(c); + insize = 1; if (c <= 0xFFFF) { EMULATE_JISX0213_2000_ENCODE_BMP(code, c) else TRYMAP_ENC(jisx0213_bmp, code, c) { if (code == MULTIC) { - if (inleft < 2) { + if (inlen - *inpos < 2) { if (flags & MBENC_FLUSH) { code = find_pairencmap( (ucs2_t)c, 0, @@ -177,8 +176,9 @@ ENCODER(euc_jis_2004) return MBERR_TOOFEW; } else { + Py_UCS4 c2 = INCHAR2; code = find_pairencmap( - (ucs2_t)c, (*inbuf)[1], + (ucs2_t)c, c2, jisx0213_pair_encmap, JISX0213_ENCPAIRS); if (code == DBCINV) { @@ -196,8 +196,8 @@ ENCODER(euc_jis_2004) else TRYMAP_ENC(jisxcommon, code, c); else if (c >= 0xff61 && c <= 0xff9f) { /* JIS X 0201 half-width katakana */ - WRITE2(0x8e, c - 0xfec0) - NEXT(1, 2) + WRITEBYTE2(0x8e, c - 0xfec0) + NEXT(1, 2); continue; } else if (c == 0xff3c) @@ -219,12 +219,12 @@ ENCODER(euc_jis_2004) if (code & 0x8000) { /* Codeset 2 */ - WRITE3(0x8f, code >> 8, (code & 0xFF) | 0x80) - NEXT(insize, 3) + WRITEBYTE3(0x8f, code >> 8, (code & 0xFF) | 0x80) + NEXT(insize, 3); } else { /* Codeset 1 */ - WRITE2((code >> 8) | 0x80, (code & 0xFF) | 0x80) - NEXT(insize, 2) + WRITEBYTE2((code >> 8) | 0x80, (code & 0xFF) | 0x80) + NEXT(insize, 2); } } @@ -234,14 +234,12 @@ ENCODER(euc_jis_2004) DECODER(euc_jis_2004) { while (inleft > 0) { - unsigned char c = IN1; - ucs4_t code; - - REQUIRE_OUTBUF(1) + unsigned char c = INBYTE1; + Py_UCS4 code; if (c < 0x80) { - OUT1(c) - NEXT(1, 1) + OUTCHAR(c); + NEXT_IN(1); continue; } @@ -250,10 +248,10 @@ DECODER(euc_jis_2004) unsigned char c2; REQUIRE_INBUF(2) - c2 = IN2; + c2 = INBYTE2; if (c2 >= 0xa1 && c2 <= 0xdf) { - OUT1(0xfec0 + c2) - NEXT(2, 1) + OUTCHAR(0xfec0 + c2); + NEXT_IN(2); } else return 1; @@ -262,46 +260,46 @@ DECODER(euc_jis_2004) unsigned char c2, c3; REQUIRE_INBUF(3) - c2 = IN2 ^ 0x80; - c3 = IN3 ^ 0x80; + c2 = INBYTE2 ^ 0x80; + c3 = INBYTE3 ^ 0x80; /* JIS X 0213 Plane 2 or JIS X 0212 (see NOTES) */ - EMULATE_JISX0213_2000_DECODE_PLANE2(**outbuf, c2, c3) - else TRYMAP_DEC(jisx0213_2_bmp, **outbuf, c2, c3) ; - else TRYMAP_DEC(jisx0213_2_emp, code, c2, c3) { - WRITEUCS4(EMPBASE | code) - NEXT_IN(3) + EMULATE_JISX0213_2000_DECODE_PLANE2(writer, c2, c3) + else TRYMAP_DEC(jisx0213_2_bmp, writer, c2, c3) ; + else TRYMAP_DEC_CHAR(jisx0213_2_emp, code, c2, c3) { + OUTCHAR(EMPBASE | code); + NEXT_IN(3); continue; } - else TRYMAP_DEC(jisx0212, **outbuf, c2, c3) ; + else TRYMAP_DEC(jisx0212, writer, c2, c3) ; else return 1; - NEXT(3, 1) + NEXT_IN(3); } else { unsigned char c2; REQUIRE_INBUF(2) c ^= 0x80; - c2 = IN2 ^ 0x80; + c2 = INBYTE2 ^ 0x80; /* JIS X 0213 Plane 1 */ - EMULATE_JISX0213_2000_DECODE_PLANE1(**outbuf, c, c2) - else if (c == 0x21 && c2 == 0x40) **outbuf = 0xff3c; - else if (c == 0x22 && c2 == 0x32) **outbuf = 0xff5e; - else TRYMAP_DEC(jisx0208, **outbuf, c, c2); - else TRYMAP_DEC(jisx0213_1_bmp, **outbuf, c, c2); - else TRYMAP_DEC(jisx0213_1_emp, code, c, c2) { - WRITEUCS4(EMPBASE | code) - NEXT_IN(2) + EMULATE_JISX0213_2000_DECODE_PLANE1(writer, c, c2) + else if (c == 0x21 && c2 == 0x40) OUTCHAR(0xff3c); + else if (c == 0x22 && c2 == 0x32) OUTCHAR(0xff5e); + else TRYMAP_DEC(jisx0208, writer, c, c2); + else TRYMAP_DEC(jisx0213_1_bmp, writer, c, c2); + else TRYMAP_DEC_CHAR(jisx0213_1_emp, code, c, c2) { + OUTCHAR(EMPBASE | code); + NEXT_IN(2); continue; } - else TRYMAP_DEC(jisx0213_pair, code, c, c2) { - WRITE2(code >> 16, code & 0xffff) - NEXT(2, 2) + else TRYMAP_DEC_CHAR(jisx0213_pair, code, c, c2) { + OUTCHAR2(code >> 16, code & 0xffff); + NEXT_IN(2); continue; } else return 1; - NEXT(2, 1) + NEXT_IN(2); } } @@ -315,35 +313,36 @@ DECODER(euc_jis_2004) ENCODER(euc_jp) { - while (inleft > 0) { - Py_UNICODE c = IN1; + while (*inpos < inlen) { + Py_UCS4 c = INCHAR1; DBCHAR code; if (c < 0x80) { - WRITE1((unsigned char)c) - NEXT(1, 1) + WRITEBYTE1((unsigned char)c) + NEXT(1, 1); continue; } - UCS4INVALID(c) + if (c > 0xFFFF) + return 1; TRYMAP_ENC(jisxcommon, code, c); else if (c >= 0xff61 && c <= 0xff9f) { /* JIS X 0201 half-width katakana */ - WRITE2(0x8e, c - 0xfec0) - NEXT(1, 2) + WRITEBYTE2(0x8e, c - 0xfec0) + NEXT(1, 2); continue; } #ifndef STRICT_BUILD else if (c == 0xff3c) /* FULL-WIDTH REVERSE SOLIDUS */ code = 0x2140; else if (c == 0xa5) { /* YEN SIGN */ - WRITE1(0x5c); - NEXT(1, 1) + WRITEBYTE1(0x5c); + NEXT(1, 1); continue; } else if (c == 0x203e) { /* OVERLINE */ - WRITE1(0x7e); - NEXT(1, 1) + WRITEBYTE1(0x7e); + NEXT(1, 1); continue; } #endif @@ -352,12 +351,12 @@ ENCODER(euc_jp) if (code & 0x8000) { /* JIS X 0212 */ - WRITE3(0x8f, code >> 8, (code & 0xFF) | 0x80) - NEXT(1, 3) + WRITEBYTE3(0x8f, code >> 8, (code & 0xFF) | 0x80) + NEXT(1, 3); } else { /* JIS X 0208 */ - WRITE2((code >> 8) | 0x80, (code & 0xFF) | 0x80) - NEXT(1, 2) + WRITEBYTE2((code >> 8) | 0x80, (code & 0xFF) | 0x80) + NEXT(1, 2); } } @@ -367,13 +366,11 @@ ENCODER(euc_jp) DECODER(euc_jp) { while (inleft > 0) { - unsigned char c = IN1; - - REQUIRE_OUTBUF(1) + unsigned char c = INBYTE1; if (c < 0x80) { - OUT1(c) - NEXT(1, 1) + OUTCHAR(c); + NEXT_IN(1); continue; } @@ -382,10 +379,10 @@ DECODER(euc_jp) unsigned char c2; REQUIRE_INBUF(2) - c2 = IN2; + c2 = INBYTE2; if (c2 >= 0xa1 && c2 <= 0xdf) { - OUT1(0xfec0 + c2) - NEXT(2, 1) + OUTCHAR(0xfec0 + c2); + NEXT_IN(2); } else return 1; @@ -394,11 +391,11 @@ DECODER(euc_jp) unsigned char c2, c3; REQUIRE_INBUF(3) - c2 = IN2; - c3 = IN3; + c2 = INBYTE2; + c3 = INBYTE3; /* JIS X 0212 */ - TRYMAP_DEC(jisx0212, **outbuf, c2 ^ 0x80, c3 ^ 0x80) { - NEXT(3, 1) + TRYMAP_DEC(jisx0212, writer, c2 ^ 0x80, c3 ^ 0x80) { + NEXT_IN(3); } else return 1; @@ -407,18 +404,18 @@ DECODER(euc_jp) unsigned char c2; REQUIRE_INBUF(2) - c2 = IN2; + c2 = INBYTE2; /* JIS X 0208 */ #ifndef STRICT_BUILD if (c == 0xa1 && c2 == 0xc0) /* FULL-WIDTH REVERSE SOLIDUS */ - **outbuf = 0xff3c; + OUTCHAR(0xff3c); else #endif - TRYMAP_DEC(jisx0208, **outbuf, + TRYMAP_DEC(jisx0208, writer, c ^ 0x80, c2 ^ 0x80) ; else return 1; - NEXT(2, 1) + NEXT_IN(2); } } @@ -432,8 +429,8 @@ DECODER(euc_jp) ENCODER(shift_jis) { - while (inleft > 0) { - Py_UNICODE c = IN1; + while (*inpos < inlen) { + Py_UCS4 c = INCHAR1; DBCHAR code; unsigned char c1, c2; @@ -445,14 +442,16 @@ ENCODER(shift_jis) else if (c == 0x203e) code = 0x7e; /* OVERLINE */ #endif else JISX0201_K_ENCODE(c, code) - else UCS4INVALID(c) - else code = NOCHAR; + else if (c > 0xFFFF) + return 1; + else + code = NOCHAR; if (code < 0x80 || (code >= 0xa1 && code <= 0xdf)) { REQUIRE_OUTBUF(1) - OUT1((unsigned char)code) - NEXT(1, 1) + OUTBYTE1((unsigned char)code) + NEXT(1, 1); continue; } @@ -475,9 +474,9 @@ ENCODER(shift_jis) c2 = code & 0xff; c2 = (((c1 - 0x21) & 1) ? 0x5e : 0) + (c2 - 0x21); c1 = (c1 - 0x21) >> 1; - OUT1(c1 < 0x1f ? c1 + 0x81 : c1 + 0xc1) - OUT2(c2 < 0x3f ? c2 + 0x40 : c2 + 0x41) - NEXT(1, 2) + OUTBYTE1(c1 < 0x1f ? c1 + 0x81 : c1 + 0xc1) + OUTBYTE2(c2 < 0x3f ? c2 + 0x40 : c2 + 0x41) + NEXT(1, 2); } return 0; @@ -486,21 +485,19 @@ ENCODER(shift_jis) DECODER(shift_jis) { while (inleft > 0) { - unsigned char c = IN1; - - REQUIRE_OUTBUF(1) + unsigned char c = INBYTE1; #ifdef STRICT_BUILD - JISX0201_R_DECODE(c, **outbuf) + JISX0201_R_DECODE(c, writer) #else - if (c < 0x80) **outbuf = c; + if (c < 0x80) OUTCHAR(c); #endif - else JISX0201_K_DECODE(c, **outbuf) + else JISX0201_K_DECODE(c, writer) else if ((c >= 0x81 && c <= 0x9f) || (c >= 0xe0 && c <= 0xea)){ unsigned char c1, c2; REQUIRE_INBUF(2) - c2 = IN2; + c2 = INBYTE2; if (c2 < 0x40 || (c2 > 0x7e && c2 < 0x80) || c2 > 0xfc) return 1; @@ -512,13 +509,13 @@ DECODER(shift_jis) #ifndef STRICT_BUILD if (c1 == 0x21 && c2 == 0x40) { /* FULL-WIDTH REVERSE SOLIDUS */ - OUT1(0xff3c) - NEXT(2, 1) + OUTCHAR(0xff3c); + NEXT_IN(2); continue; } #endif - TRYMAP_DEC(jisx0208, **outbuf, c1, c2) { - NEXT(2, 1) + TRYMAP_DEC(jisx0208, writer, c1, c2) { + NEXT_IN(2); continue; } else @@ -527,7 +524,7 @@ DECODER(shift_jis) else return 1; - NEXT(1, 1) /* JIS X 0201 */ + NEXT_IN(1); /* JIS X 0201 */ } return 0; @@ -540,30 +537,29 @@ DECODER(shift_jis) ENCODER(shift_jis_2004) { - while (inleft > 0) { - ucs4_t c = IN1; + while (*inpos < inlen) { + Py_UCS4 c = INCHAR1; DBCHAR code = NOCHAR; int c1, c2; Py_ssize_t insize; JISX0201_ENCODE(c, code) - else DECODE_SURROGATE(c) if (code < 0x80 || (code >= 0xa1 && code <= 0xdf)) { - WRITE1((unsigned char)code) - NEXT(1, 1) + WRITEBYTE1((unsigned char)code) + NEXT(1, 1); continue; } REQUIRE_OUTBUF(2) - insize = GET_INSIZE(c); + insize = 1; if (code == NOCHAR) { if (c <= 0xffff) { EMULATE_JISX0213_2000_ENCODE_BMP(code, c) else TRYMAP_ENC(jisx0213_bmp, code, c) { if (code == MULTIC) { - if (inleft < 2) { + if (inlen - *inpos < 2) { if (flags & MBENC_FLUSH) { code = find_pairencmap ((ucs2_t)c, 0, @@ -576,8 +572,9 @@ ENCODER(shift_jis_2004) return MBERR_TOOFEW; } else { + Py_UCS4 ch2 = INCHAR2; code = find_pairencmap( - (ucs2_t)c, IN2, + (ucs2_t)c, ch2, jisx0213_pair_encmap, JISX0213_ENCPAIRS); if (code == DBCINV) { @@ -622,10 +619,10 @@ ENCODER(shift_jis_2004) if (c1 & 1) c2 += 0x5e; c1 >>= 1; - OUT1(c1 + (c1 < 0x1f ? 0x81 : 0xc1)) - OUT2(c2 + (c2 < 0x3f ? 0x40 : 0x41)) + OUTBYTE1(c1 + (c1 < 0x1f ? 0x81 : 0xc1)) + OUTBYTE2(c2 + (c2 < 0x3f ? 0x40 : 0x41)) - NEXT(insize, 2) + NEXT(insize, 2); } return 0; @@ -634,16 +631,15 @@ ENCODER(shift_jis_2004) DECODER(shift_jis_2004) { while (inleft > 0) { - unsigned char c = IN1; + unsigned char c = INBYTE1; - REQUIRE_OUTBUF(1) - JISX0201_DECODE(c, **outbuf) + JISX0201_DECODE(c, writer) else if ((c >= 0x81 && c <= 0x9f) || (c >= 0xe0 && c <= 0xfc)){ unsigned char c1, c2; - ucs4_t code; + Py_UCS4 code; REQUIRE_INBUF(2) - c2 = IN2; + c2 = INBYTE2; if (c2 < 0x40 || (c2 > 0x7e && c2 < 0x80) || c2 > 0xfc) return 1; @@ -654,50 +650,47 @@ DECODER(shift_jis_2004) if (c1 < 0x5e) { /* Plane 1 */ c1 += 0x21; - EMULATE_JISX0213_2000_DECODE_PLANE1(**outbuf, + EMULATE_JISX0213_2000_DECODE_PLANE1(writer, c1, c2) - else TRYMAP_DEC(jisx0208, **outbuf, c1, c2) { - NEXT_OUT(1) + else TRYMAP_DEC(jisx0208, writer, c1, c2) { } - else TRYMAP_DEC(jisx0213_1_bmp, **outbuf, + else TRYMAP_DEC(jisx0213_1_bmp, writer, c1, c2) { - NEXT_OUT(1) } - else TRYMAP_DEC(jisx0213_1_emp, code, c1, c2) { - WRITEUCS4(EMPBASE | code) + else TRYMAP_DEC_CHAR(jisx0213_1_emp, code, c1, c2) { + OUTCHAR(EMPBASE | code); } - else TRYMAP_DEC(jisx0213_pair, code, c1, c2) { - WRITE2(code >> 16, code & 0xffff) - NEXT_OUT(2) + else TRYMAP_DEC_CHAR(jisx0213_pair, code, c1, c2) { + OUTCHAR2(code >> 16, code & 0xffff); } else return 1; - NEXT_IN(2) + NEXT_IN(2); } else { /* Plane 2 */ if (c1 >= 0x67) c1 += 0x07; else if (c1 >= 0x63 || c1 == 0x5f) c1 -= 0x37; else c1 -= 0x3d; - EMULATE_JISX0213_2000_DECODE_PLANE2(**outbuf, + EMULATE_JISX0213_2000_DECODE_PLANE2(writer, c1, c2) - else TRYMAP_DEC(jisx0213_2_bmp, **outbuf, - c1, c2) ; - else TRYMAP_DEC(jisx0213_2_emp, code, c1, c2) { - WRITEUCS4(EMPBASE | code) - NEXT_IN(2) + else TRYMAP_DEC(jisx0213_2_bmp, writer, + c1, c2) { + } else TRYMAP_DEC_CHAR(jisx0213_2_emp, code, c1, c2) { + OUTCHAR(EMPBASE | code); + NEXT_IN(2); continue; } else return 1; - NEXT(2, 1) + NEXT_IN(2); } continue; } else return 1; - NEXT(1, 1) /* JIS X 0201 */ + NEXT_IN(1); /* JIS X 0201 */ } return 0; |