18 #define ENABLE_ECONV_NEWLINE_OPTION 1
32 #ifdef ENABLE_ECONV_NEWLINE_OPTION
45 static unsigned char *
47 const unsigned char *str,
size_t len,
48 unsigned char *caller_dst_buf,
size_t caller_dst_bufsize,
80 char ary[
sizeof(double) >
sizeof(
void*) ?
sizeof(double) :
sizeof(
void*)];
84 #define TRANSCODING_READBUF(tc) \
85 ((tc)->transcoder->max_input <= (int)sizeof((tc)->readbuf.ary) ? \
88 #define TRANSCODING_WRITEBUF(tc) \
89 ((tc)->transcoder->max_output <= (int)sizeof((tc)->writebuf.ary) ? \
90 (tc)->writebuf.ary : \
92 #define TRANSCODING_WRITEBUF_SIZE(tc) \
93 ((tc)->transcoder->max_output <= (int)sizeof((tc)->writebuf.ary) ? \
94 sizeof((tc)->writebuf.ary) : \
95 (size_t)(tc)->transcoder->max_output)
96 #define TRANSCODING_STATE_EMBED_MAX ((int)sizeof(union rb_transcoding_state_t))
97 #define TRANSCODING_STATE(tc) \
98 ((tc)->transcoder->state_size <= (int)sizeof((tc)->state) ? \
154 #define DECORATOR_P(sname, dname) (*(sname) == '\0')
178 entry->
sname = sname;
179 entry->
dname = dname;
241 #define encoding_equal(enc1, enc2) (STRCASECMP((enc1), (enc2)) == 0)
258 const char *dname = (
const char *)key;
278 void (*
callback)(
const char *sname,
const char *dname,
int depth,
void *
arg),
335 const char *enc = dname;
343 enc = (
const char *)val;
351 callback((
const char *)val, enc, --depth, arg);
352 enc = (
const char *)val;
368 const char *
const lib = entry->
lib;
369 const size_t len =
strlen(lib);
377 memcpy(path, transcoder_lib_prefix,
sizeof(transcoder_lib_prefix) - 1);
378 memcpy(path +
sizeof(transcoder_lib_prefix) - 1, lib, len);
397 *repl_encname_ptr =
"UTF-8";
398 return "\xEF\xBF\xBD";
402 *repl_encname_ptr =
"US-ASCII";
411 static const unsigned char *
413 const unsigned char *in_start,
414 const unsigned char *inchar_start,
415 const unsigned char *in_p,
416 size_t *char_len_ptr)
418 const unsigned char *ptr;
419 if (inchar_start - in_start < tc->recognized_len) {
421 inchar_start,
unsigned char, in_p - inchar_start);
433 const unsigned char *in_stop,
unsigned char *out_stop,
439 ssize_t readagain_len = 0;
441 const unsigned char *inchar_start;
442 const unsigned char *in_p;
444 unsigned char *out_p;
446 in_p = inchar_start = *in_pos;
450 #define SUSPEND(ret, num) \
452 tc->resume_position = (num); \
453 if (0 < in_p - inchar_start) \
454 MEMMOVE(TRANSCODING_READBUF(tc)+tc->recognized_len, \
455 inchar_start, unsigned char, in_p - inchar_start); \
458 tc->recognized_len += in_p - inchar_start; \
459 if (readagain_len) { \
460 tc->recognized_len -= readagain_len; \
461 tc->readagain_len = readagain_len; \
464 resume_label ## num:; \
466 #define SUSPEND_OBUF(num) \
468 while (out_stop - out_p < 1) { SUSPEND(econv_destination_buffer_full, num); } \
471 #define SUSPEND_AFTER_OUTPUT(num) \
472 if ((opt & ECONV_AFTER_OUTPUT) && *out_pos != out_p) { \
473 SUSPEND(econv_after_output, num); \
476 #define next_table (tc->next_table)
477 #define next_info (tc->next_info)
478 #define next_byte (tc->next_byte)
479 #define writebuf_len (tc->writebuf_len)
480 #define writebuf_off (tc->writebuf_off)
484 case 1:
goto resume_label1;
485 case 2:
goto resume_label2;
486 case 3:
goto resume_label3;
487 case 4:
goto resume_label4;
488 case 5:
goto resume_label5;
489 case 6:
goto resume_label6;
490 case 7:
goto resume_label7;
491 case 8:
goto resume_label8;
492 case 9:
goto resume_label9;
493 case 10:
goto resume_label10;
494 case 11:
goto resume_label11;
495 case 12:
goto resume_label12;
496 case 13:
goto resume_label13;
497 case 14:
goto resume_label14;
498 case 15:
goto resume_label15;
499 case 16:
goto resume_label16;
500 case 17:
goto resume_label17;
501 case 18:
goto resume_label18;
502 case 19:
goto resume_label19;
503 case 20:
goto resume_label20;
504 case 21:
goto resume_label21;
505 case 22:
goto resume_label22;
506 case 23:
goto resume_label23;
507 case 24:
goto resume_label24;
508 case 25:
goto resume_label25;
509 case 26:
goto resume_label26;
510 case 27:
goto resume_label27;
511 case 28:
goto resume_label28;
512 case 29:
goto resume_label29;
513 case 30:
goto resume_label30;
514 case 31:
goto resume_label31;
515 case 32:
goto resume_label32;
516 case 33:
goto resume_label33;
517 case 34:
goto resume_label34;
527 if (in_stop <= in_p) {
534 #define BYTE_ADDR(index) (tr->byte_array + (index))
535 #define WORD_ADDR(index) (tr->word_array + INFO2WORDINDEX(index))
536 #define BL_BASE BYTE_ADDR(BYTE_LOOKUP_BASE(WORD_ADDR(next_table)))
537 #define BL_INFO WORD_ADDR(BYTE_LOOKUP_INFO(WORD_ADDR(next_table)))
538 #define BL_MIN_BYTE (BL_BASE[0])
539 #define BL_MAX_BYTE (BL_BASE[1])
540 #define BL_OFFSET(byte) (BL_BASE[2+(byte)-BL_MIN_BYTE])
541 #define BL_ACTION(byte) (BL_INFO[BL_OFFSET((byte))])
554 const unsigned char *
p = inchar_start;
567 case 0x00:
case 0x04:
case 0x08:
case 0x0C:
568 case 0x10:
case 0x14:
case 0x18:
case 0x1C:
570 while (in_p >= in_stop) {
616 const unsigned char *char_start;
640 const unsigned char *char_start;
646 char_start, (
size_t)char_len,
647 out_p, out_stop - out_p);
652 char_start, (
size_t)char_len,
664 const unsigned char *char_start;
671 out_p, out_stop - out_p);
705 discard_len = ((invalid_len - 1) / unitlen) * unitlen;
706 readagain_len = invalid_len - discard_len;
734 out_p, out_stop - out_p);
758 const unsigned char *in_stop,
unsigned char *out_stop,
764 const unsigned char *readagain_pos = readagain_buf;
765 const unsigned char *readagain_stop = readagain_buf + tc->
readagain_len;
774 readagain_pos,
unsigned char, readagain_stop - readagain_pos);
790 if (TRANSCODING_STATE_EMBED_MAX < tr->state_size)
811 const unsigned char **input_ptr,
const unsigned char *input_stop,
812 unsigned char **output_ptr,
unsigned char *output_stop,
816 input_ptr, output_ptr,
817 input_stop, output_stop,
828 if (TRANSCODING_STATE_EMBED_MAX < tr->state_size)
843 if (TRANSCODING_STATE_EMBED_MAX < tr->state_size) {
938 for (i = 0; i < n; i++) {
947 for (i = 0; i < n; i++) {
998 if (*sname ==
'\0' && *dname ==
'\0') {
1008 if (num_trans < 0) {
1019 ec->
flags = ecflags;
1026 #define MAX_ECFLAGS_DECORATORS 32
1049 if (ecflags & ECONV_XML_TEXT_DECORATOR)
1050 decorators_ret[num_decorators++] =
"xml_text_escape";
1051 if (ecflags & ECONV_XML_ATTR_CONTENT_DECORATOR)
1052 decorators_ret[num_decorators++] =
"xml_attr_content_escape";
1054 decorators_ret[num_decorators++] =
"xml_attr_quote";
1057 decorators_ret[num_decorators++] =
"crlf_newline";
1059 decorators_ret[num_decorators++] =
"cr_newline";
1061 decorators_ret[num_decorators++] =
"universal_newline";
1063 return num_decorators;
1075 if (num_decorators == -1)
1082 for (i = 0; i < num_decorators; i++)
1088 ec->
flags |= ecflags & ~ECONV_ERROR_HANDLER_MASK;
1095 const unsigned char **input_ptr,
const unsigned char *input_stop,
1096 unsigned char **output_ptr,
unsigned char *output_stop,
1103 const unsigned char **ipp, *is, *iold;
1104 unsigned char **opp, *os, *oold;
1110 for (i = start; i < ec->
num_trans; i++) {
1144 flags &= ~ECONV_AFTER_OUTPUT;
1147 f &= ~ECONV_AFTER_OUTPUT;
1151 if (iold != *ipp || oold != *opp)
1176 const unsigned char **input_ptr,
const unsigned char *input_stop,
1177 unsigned char **output_ptr,
unsigned char *output_stop,
1179 int *result_position_ptr)
1182 int needreport_index;
1185 unsigned char empty_buf;
1186 unsigned char *empty_ptr = &empty_buf;
1189 input_ptr = (
const unsigned char **)&empty_ptr;
1190 input_stop = empty_ptr;
1194 output_ptr = &empty_ptr;
1195 output_stop = empty_ptr;
1201 needreport_index = -1;
1202 for (i = ec->
num_trans-1; 0 <= i; i--) {
1210 needreport_index =
i;
1211 goto found_needreport;
1218 rb_bug(
"unexpected transcode last result");
1230 result_position_ptr);
1242 needreport_index =
trans_sweep(ec, input_ptr, input_stop, output_ptr, output_stop, flags, sweep_start);
1243 sweep_start = needreport_index + 1;
1244 }
while (needreport_index != -1 && needreport_index != ec->
num_trans-1);
1246 for (i = ec->
num_trans-1; 0 <= i; i--) {
1255 if (result_position_ptr)
1256 *result_position_ptr =
i;
1260 if (result_position_ptr)
1261 *result_position_ptr = -1;
1267 const unsigned char **input_ptr,
const unsigned char *input_stop,
1268 unsigned char **output_ptr,
unsigned char *output_stop,
1272 int result_position;
1280 if (output_stop - *output_ptr < ec->in_data_end - ec->
in_data_start) {
1281 len = output_stop - *output_ptr;
1283 *output_ptr = output_stop;
1297 if (output_stop - *output_ptr < input_stop - *input_ptr) {
1298 len = output_stop - *output_ptr;
1301 len = input_stop - *input_ptr;
1304 *(*output_ptr)++ = *(*input_ptr)++;
1308 memcpy(*output_ptr, *input_ptr, len);
1311 if (*input_ptr != input_stop)
1323 if (data_start != data_end) {
1325 if (output_stop - *output_ptr < data_end - data_start) {
1326 len = output_stop - *output_ptr;
1327 memcpy(*output_ptr, data_start, len);
1328 *output_ptr = output_stop;
1333 len = data_end - data_start;
1334 memcpy(*output_ptr, data_start, len);
1353 *input_ptr != input_stop) {
1354 input_stop = *input_ptr;
1355 res =
rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
1359 else if ((flags & ECONV_AFTER_OUTPUT) ||
1361 res =
rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
1366 res =
rb_trans_conv(ec, input_ptr, input_stop, output_ptr, output_stop, flags, &result_position);
1393 unsigned char utfbuf[1024];
1394 const unsigned char *utf;
1396 int utf_allocated = 0;
1397 char charef_buf[16];
1398 const unsigned char *
p;
1407 utfbuf,
sizeof(utfbuf),
1415 if (utf_len % 4 != 0)
1419 while (4 <= utf_len) {
1425 snprintf(charef_buf,
sizeof(charef_buf),
"&#x%X;", u);
1447 const unsigned char **input_ptr,
const unsigned char *input_stop,
1448 unsigned char **output_ptr,
unsigned char *output_stop,
1453 unsigned char empty_buf;
1454 unsigned char *empty_ptr = &empty_buf;
1459 input_ptr = (
const unsigned char **)&empty_ptr;
1460 input_stop = empty_ptr;
1464 output_ptr = &empty_ptr;
1465 output_stop = empty_ptr;
1469 ret =
rb_econv_convert0(ec, input_ptr, input_stop, output_ptr, output_stop, flags);
1518 static unsigned char *
1520 const unsigned char *str,
size_t len,
1521 unsigned char *caller_dst_buf,
size_t caller_dst_bufsize,
1522 size_t *dst_len_ptr)
1524 unsigned char *dst_str;
1531 const unsigned char *sp;
1535 dst_bufsize = caller_dst_bufsize;
1545 dst_str = caller_dst_buf;
1547 dst_str =
xmalloc(dst_bufsize);
1550 dp = dst_str+dst_len;
1552 dst_len = dp - dst_str;
1558 if (dst_str == caller_dst_buf) {
1561 memcpy(tmp, dst_str, dst_bufsize/2);
1565 dst_str =
xrealloc(dst_str, dst_bufsize);
1567 dp = dst_str+dst_len;
1569 dst_len = dp - dst_str;
1575 *dst_len_ptr = dst_len;
1579 if (dst_str != caller_dst_buf)
1588 const unsigned char *str,
size_t len,
const char *str_encoding)
1591 unsigned char insert_buf[4096];
1592 const unsigned char *insert_str =
NULL;
1595 int last_trans_index;
1598 unsigned char **buf_start_p;
1599 unsigned char **data_start_p;
1600 unsigned char **data_end_p;
1601 unsigned char **buf_end_p;
1616 str, len, insert_buf,
sizeof(insert_buf), &insert_len);
1617 if (insert_str ==
NULL)
1632 tc = ec->
elems[last_trans_index].
tc;
1634 if (need < insert_len)
1636 if (last_trans_index == 0) {
1656 tc = ec->
elems[last_trans_index].
tc;
1659 if (*buf_start_p ==
NULL) {
1662 *data_start_p =
buf;
1664 *buf_end_p = buf+need;
1666 else if ((
size_t)(*buf_end_p - *data_end_p) < need) {
1667 MEMMOVE(*buf_start_p, *data_start_p,
unsigned char, *data_end_p - *data_start_p);
1668 *data_end_p = *buf_start_p + (*data_end_p - *data_start_p);
1669 *data_start_p = *buf_start_p;
1670 if ((
size_t)(*buf_end_p - *data_end_p) < need) {
1672 size_t s = (*data_end_p - *buf_start_p) + need;
1676 *data_start_p =
buf;
1677 *data_end_p = buf + (*data_end_p - *buf_start_p);
1679 *buf_end_p = buf + s;
1683 memcpy(*data_end_p, insert_str, insert_len);
1684 *data_end_p += insert_len;
1691 if (insert_str != str && insert_str != insert_buf)
1692 xfree((
void*)insert_str);
1696 if (insert_str != str && insert_str != insert_buf)
1697 xfree((
void*)insert_str);
1746 #if SIZEOF_SIZE_T > SIZEOF_INT
1809 return data.ascii_compat_name;
1815 unsigned const char *ss, *sp, *se;
1816 unsigned char *ds, *
dp, *de;
1835 unsigned long new_capa = (
unsigned long)dlen + len + max_output;
1841 ss = sp = (
const unsigned char *)
RSTRING_PTR(src) + off;
1936 const char *dname = 0;
1940 dname =
"universal_newline";
1943 dname =
"crlf_newline";
1946 dname =
"cr_newline";
1955 for (i=0; i < num_trans; i++) {
1972 int has_description = 0;
1977 if (*sname !=
'\0' || *dname !=
'\0') {
1980 else if (*dname ==
'\0')
1984 has_description = 1;
1991 const char *pre =
"";
1992 if (has_description)
2018 has_description = 1;
2020 if (!has_description) {
2057 else if (readagain_len) {
2058 bytes2 =
rb_str_new(err+error_len, readagain_len);
2094 const char *start, *end;
2118 mesg =
rb_sprintf(
"%s to %s in conversion from %s",
2142 unsigned char *(*resize_destination)(
VALUE,
size_t,
size_t),
2144 unsigned char **out_start_ptr,
2145 unsigned char **out_pos,
2146 unsigned char **out_stop_ptr)
2148 size_t len = (*out_pos - *out_start_ptr);
2149 size_t new_len = (len + max_output) * 2;
2150 *out_start_ptr = resize_destination(destination, len, new_len);
2151 *out_pos = *out_start_ptr + len;
2152 *out_stop_ptr = *out_start_ptr + new_len;
2160 const unsigned char *replacement;
2161 const char *repl_enc;
2162 const char *ins_enc;
2177 replacement = (
unsigned char *)
"?";
2191 const unsigned char *str,
size_t len,
const char *encname)
2193 unsigned char *str2;
2195 const char *encname2;
2201 MEMCPY(str2, str,
unsigned char, len);
2237 #define hash_fallback rb_hash_aref
2259 const unsigned char *in_stop,
unsigned char *out_stop,
2261 unsigned char *(*resize_destination)(
VALUE,
size_t,
size_t),
2262 const char *src_encoding,
2263 const char *dst_encoding,
2270 unsigned char *out_start = *out_pos;
2306 rep = (*fallback_func)(fallback, rep);
2311 if ((
int)ret == -1) {
2327 more_output_buffer(destination, resize_destination, max_output, &out_start, out_pos, &out_stop);
2337 transcode_loop(
const unsigned char **in_pos,
unsigned char **out_pos,
2338 const unsigned char *in_stop,
unsigned char *out_stop,
2340 unsigned char *(*resize_destination)(
VALUE,
size_t,
size_t),
2341 const char *src_encoding,
2342 const char *dst_encoding,
2349 unsigned char *out_start = *out_pos;
2350 const unsigned char *ptr;
2364 unsigned char input_byte;
2365 const unsigned char *
p = &input_byte;
2368 if (ptr < in_stop) {
2379 if (&input_byte != p)
2380 ptr += p - &input_byte;
2391 more_output_buffer(destination, resize_destination, max_output, &out_start, out_pos, &out_stop);
2412 static unsigned char *
2465 #ifdef ENABLE_ECONV_NEWLINE_OPTION
2492 int setflags = 0, newlineflag = 0;
2497 newlineflag |= !
NIL_P(v);
2502 newlineflag |= !
NIL_P(v);
2507 newlineflag |= !
NIL_P(v);
2511 ecflags |= setflags;
2524 if (
NIL_P(opthash)) {
2556 if (!
NIL_P(newhash))
2575 if (
NIL_P(opthash)) {
2580 rb_bug(
"rb_econv_open_opts called with invalid opthash");
2584 ec =
rb_econv_open(source_encoding, destination_encoding, ecflags);
2588 if (!
NIL_P(replacement)) {
2634 const char *sname, *dname;
2635 int sencidx, dencidx;
2637 dencidx =
enc_arg(arg1, &dname, &denc);
2645 sencidx =
enc_arg(arg2, &sname, &senc);
2660 volatile VALUE arg1, arg2;
2662 unsigned char *
buf, *
bp, *sp;
2663 const unsigned char *fromp;
2665 const char *sname, *dname;
2673 if (!ecflags)
return -1;
2681 arg2 = argc<=1 ?
Qnil : argv[1];
2688 if (senc && senc == denc) {
2689 return NIL_P(arg2) ? -1 : dencidx;
2697 return NIL_P(arg2) ? -1 : dencidx;
2714 if (fromp != sp+slen) {
2787 if (encidx < 0)
return str;
2788 if (newstr == str) {
2872 int encidx =
str_transcode0(argc, argv, &newstr, ecflags, ecopts);
2880 if (newstr == str) {
2962 const char *arg_name, *result_name;
2965 enc_arg(&arg, &arg_name, &arg_enc);
2969 if (result_name ==
NULL)
2979 volatile VALUE *snamev_p,
volatile VALUE *dnamev_p,
2980 const char **sname_p,
const char **dname_p,
2985 VALUE opt, flags_v, ecopts;
2987 const char *sname, *dname;
2991 argc =
rb_scan_args(argc, argv,
"21:", snamev_p, dnamev_p, &flags_v, &opt);
2993 if (!
NIL_P(flags_v)) {
3000 else if (!
NIL_P(opt)) {
3033 *ecflags_p = ecflags;
3046 if (num_decorators == -1)
3062 rb_ary_store(convpath, len + num_decorators - 1, pair);
3066 rb_ary_store(convpath, len + num_decorators - 1, pair);
3070 for (i = 0; i < num_decorators; i++)
3082 if (*ary_p ==
Qnil) {
3123 volatile VALUE snamev, dnamev;
3124 const char *sname, *dname;
3130 econv_args(argc, argv, &snamev, &dnamev, &sname, &dname, &senc, &denc, &ecflags, &ecopts);
3135 if (
NIL_P(convpath))
3155 return RTEST(convpath);
3181 const char **sname_p,
const char **dname_p,
3189 const char *sname, *dname;
3195 volatile VALUE snamev, dnamev;
3202 enc_arg(&snamev, &sname, &senc);
3204 enc_arg(&dnamev, &dname, &denc);
3222 if (ret == -1 || arg.
ret == -1)
3357 volatile VALUE snamev, dnamev;
3358 const char *sname, *dname;
3374 econv_args(argc, argv, &snamev, &dnamev, &sname, &dname, &senc, &denc, &ecflags, &ecopts);
3415 return rb_sprintf(
"#<%s: uninitialized>", cname);
3665 const unsigned char *ip, *is;
3666 unsigned char *op, *os;
3667 long output_byteoffset, output_bytesize;
3668 unsigned long output_byteend;
3671 argc =
rb_scan_args(argc, argv,
"23:", &input, &output, &output_byteoffset_v, &output_bytesize_v, &flags_v, &opt);
3673 if (
NIL_P(output_byteoffset_v))
3674 output_byteoffset = 0;
3676 output_byteoffset =
NUM2LONG(output_byteoffset_v);
3678 if (
NIL_P(output_bytesize_v))
3679 output_bytesize = 0;
3681 output_bytesize =
NUM2LONG(output_bytesize_v);
3683 if (!
NIL_P(flags_v)) {
3689 else if (!
NIL_P(opt)) {
3708 if (
NIL_P(output_bytesize_v)) {
3716 if (
NIL_P(output_byteoffset_v))
3719 if (output_byteoffset < 0)
3725 if (output_bytesize < 0)
3728 output_byteend = (
unsigned long)output_byteoffset +
3729 (
unsigned long)output_bytesize;
3731 if (output_byteend < (
unsigned long)output_byteoffset ||
3746 op = (
unsigned char *)
RSTRING_PTR(output) + output_byteoffset;
3747 os = op + output_bytesize;
3755 if (
LONG_MAX / 2 < output_bytesize)
3757 output_bytesize *= 2;
3758 output_byteoffset_v =
Qnil;
3836 rb_bug(
"unexpected result of econv_primitive_convert");
3880 rb_bug(
"unexpected result of econv_primitive_convert");
4022 const char *insert_enc;
4079 if (putbackable < n)
4402 #ifdef ENABLE_ECONV_NEWLINE_OPTION
const char * ascii_incompat_name
#define RB_TYPE_P(obj, type)
int rb_econv_prepare_opts(VALUE opthash, VALUE *opts)
VALUE(* func_si)(void *, const unsigned char *, size_t)
search_path_queue_t * queue
int rb_enc_get_index(VALUE obj)
void rb_econv_check_error(rb_econv_t *ec)
static VALUE econv_destination_encoding(VALUE self)
static VALUE sym_undefined_conversion
VALUE rb_eConverterNotFoundError
VALUE rb_ary_new4(long n, const VALUE *elts)
int(* state_fini_func)(void *)
VALUE rb_ary_entry(VALUE ary, long offset)
unsigned char * in_buf_end
const unsigned char * error_bytes_start
void rb_bug(const char *fmt,...)
rb_econv_result_t last_result
VALUE rb_econv_make_exception(rb_econv_t *ec)
VALUE rb_require_safe(VALUE, int)
const char * dst_encoding
static VALUE sym_invalid_byte_sequence
size_t strlen(const char *)
struct search_path_queue_tag search_path_queue_t
#define DECORATOR_P(sname, dname)
const char * rb_obj_classname(VALUE)
rb_econv_result_t rb_econv_convert(rb_econv_t *ec, const unsigned char **input_ptr, const unsigned char *input_stop, unsigned char **output_ptr, unsigned char *output_stop, int flags)
VALUE rb_econv_open_exc(const char *sname, const char *dname, int ecflags)
#define ECONV_AFTER_OUTPUT
static int max(int a, int b)
int st_lookup(st_table *, st_data_t, st_data_t *)
void st_add_direct(st_table *, st_data_t, st_data_t)
VALUE rb_str_tmp_new(long)
void rb_define_singleton_method(VALUE obj, const char *name, VALUE(*func)(ANYARGS), int argc)
Defines a singleton method for obj.
static void transcode_loop(const unsigned char **in_pos, unsigned char **out_pos, const unsigned char *in_stop, unsigned char *out_stop, VALUE destination, unsigned char *(*resize_destination)(VALUE, size_t, size_t), const char *src_encoding, const char *dst_encoding, int ecflags, VALUE ecopts)
VALUE rb_eInvalidByteSequenceError
static void econv_args(int argc, VALUE *argv, volatile VALUE *snamev_p, volatile VALUE *dnamev_p, const char **sname_p, const char **dname_p, rb_encoding **senc_p, rb_encoding **denc_p, int *ecflags_p, VALUE *ecopts_p)
int(* state_init_func)(void *)
void rb_econv_binmode(rb_econv_t *ec)
static void rb_transcoding_close(rb_transcoding *tc)
rb_encoding * source_encoding
unsigned char * out_data_start
static int decorate_convpath(VALUE convpath, int ecflags)
static int enc_arg(volatile VALUE *arg, const char **name_p, rb_encoding **enc_p)
static VALUE sym_crlf_newline
#define MAX_ECFLAGS_DECORATORS
static size_t rb_transcoding_memsize(rb_transcoding *tc)
#define rb_check_frozen(obj)
int rb_econv_prepare_options(VALUE opthash, VALUE *opts, int ecflags)
unsigned char * in_data_start
int rb_econv_decorate_at_last(rb_econv_t *ec, const char *decorator_name)
static int str_transcode_enc_args(VALUE str, volatile VALUE *arg1, volatile VALUE *arg2, const char **sname_p, rb_encoding **senc_p, const char **dname_p, rb_encoding **denc_p)
rb_encoding * rb_to_encoding(VALUE enc)
VALUE rb_str_new_cstr(const char *)
rb_encoding * destination_encoding
struct rb_transcoding * tc
#define SUSPEND(ret, num)
VALUE rb_enc_from_encoding(rb_encoding *encoding)
static VALUE sym_cr_newline
void rb_define_alloc_func(VALUE, rb_alloc_func_t)
static int str_transcode(int argc, VALUE *argv, VALUE *self)
#define ECONV_NEWLINE_DECORATOR_MASK
VALUE rb_econv_substr_append(rb_econv_t *ec, VALUE src, long off, long len, VALUE dst, int flags)
VALUE rb_ary_push(VALUE ary, VALUE item)
st_table * st_init_strcasetable(void)
static VALUE econv_last_error(VALUE self)
VALUE rb_hash_freeze(VALUE)
struct rb_transcoding * error_tc
VALUE rb_enc_str_new(const char *, long, rb_encoding *)
static rb_econv_t * rb_econv_alloc(int n_hint)
static rb_econv_t * rb_econv_open_by_transcoder_entries(int n, transcoder_entry_t **entries)
VALUE rb_define_class_under(VALUE outer, const char *name, VALUE super)
Defines a class under the namespace of outer.
#define MEMMOVE(p1, p2, type, n)
void rb_raise(VALUE exc, const char *fmt,...)
VALUE rb_enc_associate(VALUE obj, rb_encoding *enc)
unsigned int conv_tree_start
VALUE rb_obj_class(VALUE)
static void rb_econv_init_by_convpath_i(const char *sname, const char *dname, int depth, void *arg)
VALUE rb_str_encode(VALUE str, VALUE to, int ecflags, VALUE ecopts)
rb_econv_t * rb_econv_open_opts(const char *source_encoding, const char *destination_encoding, int ecflags, VALUE opthash)
const rb_transcoder * transcoder
static int output_replacement_character(rb_econv_t *ec)
static rb_econv_result_t rb_trans_conv(rb_econv_t *ec, const unsigned char **input_ptr, const unsigned char *input_stop, unsigned char **output_ptr, unsigned char *output_stop, int flags, int *result_position_ptr)
void callback(ffi_cif *cif, void *resp, void **args, void *ctx)
union rb_transcoding::@141 readbuf
static rb_econv_result_t transcode_restartable(const unsigned char **in_pos, unsigned char **out_pos, const unsigned char *in_stop, unsigned char *out_stop, rb_transcoding *tc, const int opt)
static VALUE econv_finish(VALUE self)
rb_encoding * rb_utf8_encoding(void)
static VALUE econv_description(const char *sname, const char *dname, int ecflags, VALUE mesg)
static transcoder_entry_t * make_transcoder_entry(const char *sname, const char *dname)
static const rb_transcoder * load_transcoder_entry(transcoder_entry_t *entry)
#define ENC_CODERANGE_BROKEN
static int transcode_search_path(const char *sname, const char *dname, void(*callback)(const char *sname, const char *dname, int depth, void *arg), void *arg)
VALUE rb_str_new_frozen(VALUE)
unsigned char * in_buf_start
static rb_econv_t * rb_econv_open0(const char *sname, const char *dname, int ecflags)
static void econv_free(void *ptr)
static VALUE sym_source_buffer_empty
void rb_econv_putback(rb_econv_t *ec, unsigned char *p, int n)
ssize_t(* func_so)(void *, const unsigned char *, size_t, unsigned char *, size_t)
static VALUE econv_convert(VALUE self, VALUE source_string)
static VALUE sym_partial_input
static const char transcoder_lib_prefix[]
static rb_econv_t * rb_econv_init_by_convpath(VALUE self, VALUE convpath, const char **sname_p, const char **dname_p, rb_encoding **senc_p, rb_encoding **denc_p)
void rb_exc_raise(VALUE mesg)
#define ECONV_ERROR_HANDLER_MASK
static unsigned char * output
static const char * get_replacement_character(const char *encname, size_t *len_ret, const char **repl_encname_ptr)
static VALUE str_encode_associate(VALUE str, int encidx)
static VALUE ecerr_incomplete_input(VALUE self)
int rb_econv_has_convpath_p(const char *from_encoding, const char *to_encoding)
union rb_transcoding::@142 writebuf
static unsigned char * str_transcoding_resize(VALUE destination, size_t len, size_t new_len)
double dummy_for_alignment
int rb_to_encoding_index(VALUE enc)
static VALUE econv_primitive_errinfo(VALUE self)
ssize_t(* finish_func)(void *, unsigned char *, size_t)
unsigned int output_index
#define TRANSCODING_READBUF(tc)
static size_t econv_memsize(const void *ptr)
#define TypedData_Get_Struct(obj, type, data_type, sval)
void Init_transcode(void)
unsigned char * in_data_end
size_t rb_str_capacity(VALUE)
static VALUE str_encode_bang(int argc, VALUE *argv, VALUE str)
static VALUE str_encode(int argc, VALUE *argv, VALUE str)
const char * destination_encoding
static int rb_econv_decorate_at(rb_econv_t *ec, const char *decorator_name, int n)
#define SUSPEND_AFTER_OUTPUT(num)
int rb_typeddata_is_kind_of(VALUE obj, const rb_data_type_t *data_type)
void rb_econv_close(rb_econv_t *ec)
#define StringValueCStr(v)
static VALUE econv_get_replacement(VALUE self)
void rb_define_const(VALUE, const char *, VALUE)
VALUE rb_str_cat2(VALUE, const char *)
VALUE rb_method_call(int, VALUE *, VALUE)
static void more_output_buffer(VALUE destination, unsigned char *(*resize_destination)(VALUE, size_t, size_t), int max_output, unsigned char **out_start_ptr, unsigned char **out_pos, unsigned char **out_stop_ptr)
void rb_str_shared_replace(VALUE, VALUE)
void rb_ary_store(VALUE ary, long idx, VALUE val)
static VALUE econv_s_search_convpath(int argc, VALUE *argv, VALUE klass)
VALUE rb_econv_str_append(rb_econv_t *ec, VALUE src, VALUE dst, int flags)
static st_table * transcoder_table
#define MBCLEN_CHARFOUND_P(ret)
int rb_econv_insert_output(rb_econv_t *ec, const unsigned char *str, size_t len, const char *str_encoding)
static VALUE make_econv_exception(rb_econv_t *ec)
VALUE rb_cEncodingConverter
static const rb_data_type_t econv_data_type
ssize_t(* func_sio)(void *, const unsigned char *, size_t, VALUE, unsigned char *, size_t)
static VALUE econv_set_replacement(VALUE self, VALUE arg)
#define TypedData_Wrap_Struct(klass, data_type, sval)
VALUE rb_check_hash_type(VALUE)
#define TRANSCODING_STATE(tc)
static VALUE sym_fallback
char ary[sizeof(double) > sizeof(void *)?sizeof(double):sizeof(void *)]
VALUE rb_enc_associate_index(VALUE obj, int idx)
static VALUE method_fallback(VALUE fallback, VALUE c)
rb_transcoder_asciicompat_type_t asciicompat_type
#define ALLOCA_N(type, n)
void rb_declare_transcoder(const char *enc1, const char *enc2, const char *lib)
#define rb_enc_mbc_to_codepoint(p, e, enc)
static VALUE econv_equal(VALUE self, VALUE other)
#define ENC_CODERANGE_SET(obj, cr)
int rb_econv_decorate_at_first(rb_econv_t *ec, const char *decorator_name)
#define ECONV_XML_ATTR_CONTENT_DECORATOR
#define SUSPEND_OBUF(num)
static int str_transcode0(int argc, VALUE *argv, VALUE *self, int ecflags, VALUE ecopts)
void rb_register_transcoder(const rb_transcoder *tr)
unsigned char * out_buf_start
static int transcode_search_path_i(st_data_t key, st_data_t val, st_data_t arg)
static VALUE econv_putback(int argc, VALUE *argv, VALUE self)
VALUE rb_hash_aset(VALUE, VALUE, VALUE)
VALUE rb_str_resize(VALUE, long)
static void search_convpath_i(const char *sname, const char *dname, int depth, void *arg)
int st_foreach(st_table *, int(*)(ANYARGS), st_data_t)
static rb_econv_t * check_econv(VALUE self)
static VALUE econv_s_allocate(VALUE klass)
search_path_queue_t ** queue_last_ptr
VALUE rb_sprintf(const char *format,...)
#define MBCLEN_CHARFOUND_LEN(ret)
int rb_enc_precise_mbclen(const char *p, const char *e, rb_encoding *enc)
static VALUE econv_insert_output(VALUE self, VALUE string)
struct rb_econv_t rb_econv_t
static VALUE ecerr_destination_encoding(VALUE self)
int rb_econv_putbackable(rb_econv_t *ec)
unsigned char * out_buf_end
long rb_str_coderange_scan_restartable(const char *, const char *, rb_encoding *, int *)
static int decorator_names(int ecflags, const char **decorators_ret)
int rb_econv_set_replacement(rb_econv_t *ec, const unsigned char *str, size_t len, const char *encname)
struct rb_transcoding * last_tc
#define STR1_BYTEINDEX(w)
static VALUE aref_fallback(VALUE fallback, VALUE c)
static VALUE make_encobj(const char *name)
int rb_scan_args(int argc, const VALUE *argv, const char *fmt,...)
unsigned char buf[MIME_BUF_SIZE]
VALUE rb_assoc_new(VALUE car, VALUE cdr)
const char * source_encoding
int rb_define_dummy_encoding(const char *name)
VALUE rb_econv_substr_convert(rb_econv_t *ec, VALUE src, long byteoff, long bytesize, int flags)
static VALUE econv_init(int argc, VALUE *argv, VALUE self)
VALUE rb_str_buf_new(long)
static VALUE sym_universal_newline
union rb_transcoding::rb_transcoding_state_t state
const char * src_encoding
VALUE rb_obj_encoding(VALUE obj)
VALUE rb_attr_get(VALUE, ID)
static void trans_open_i(const char *sname, const char *dname, int depth, void *arg)
static VALUE sym_universal
int memcmp(const void *s1, const void *s2, size_t len)
static VALUE ecerr_error_char(VALUE self)
const char * ascii_compat_name
static rb_encoding * make_encoding(const char *name)
VALUE rb_obj_is_method(VALUE)
RUBY_EXTERN VALUE rb_cString
static VALUE econv_source_encoding(VALUE self)
static VALUE proc_fallback(VALUE fallback, VALUE c)
static VALUE sym_finished
#define RSTRING_EMBED_LEN_MAX
VALUE rb_obj_is_proc(VALUE)
#define MEMCPY(p1, p2, type, n)
static int make_replacement(rb_econv_t *ec)
#define ENC_CODERANGE_VALID
#define ECONV_XML_ATTR_QUOTE_DECORATOR
static rb_econv_result_t transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos, const unsigned char *in_stop, unsigned char *out_stop, rb_transcoding *tc, const int opt)
VALUE rb_ivar_set(VALUE, ID, VALUE)
void rb_str_modify(VALUE)
#define ENC_CODERANGE_7BIT
rb_encoding * rb_enc_get(VALUE obj)
#define ECONV_INVALID_REPLACE
static VALUE sym_after_output
static VALUE econv_inspect(VALUE self)
static rb_transcoding * rb_transcoding_open_by_transcoder(const rb_transcoder *tr, int flags)
void * rb_check_typeddata(VALUE obj, const rb_data_type_t *data_type)
VALUE rb_exc_new3(VALUE etype, VALUE str)
unsigned char * out_data_end
static rb_econv_result_t rb_econv_convert0(rb_econv_t *ec, const unsigned char **input_ptr, const unsigned char *input_stop, unsigned char **output_ptr, unsigned char *output_stop, int flags)
VALUE rb_proc_call(VALUE, VALUE)
static int asciicompat_encoding_i(st_data_t key, st_data_t val, st_data_t arg)
VALUE rb_funcall3(VALUE, ID, int, const VALUE *)
Calls a method.
size_t rb_econv_memsize(rb_econv_t *ec)
rb_econv_t * rb_econv_open(const char *sname, const char *dname, int ecflags)
#define TRANSCODING_WRITEBUF(tc)
static const unsigned char * transcode_char_start(rb_transcoding *tc, const unsigned char *in_start, const unsigned char *inchar_start, const unsigned char *in_p, size_t *char_len_ptr)
VALUE rb_check_array_type(VALUE ary)
void rb_error_arity(int argc, int min, int max)
static VALUE ecerr_error_bytes(VALUE self)
static rb_econv_result_t rb_transcoding_convert(rb_transcoding *tc, const unsigned char **input_ptr, const unsigned char *input_stop, unsigned char **output_ptr, unsigned char *output_stop, int flags)
VALUE rb_str_catf(VALUE str, const char *format,...)
#define ECONV_PARTIAL_INPUT
#define RARRAY_LENINT(ary)
static void declare_transcoder(const char *sname, const char *dname, const char *lib)
static int rb_econv_add_converter(rb_econv_t *ec, const char *sname, const char *dname, int n)
static int rb_econv_add_transcoder_at(rb_econv_t *ec, const rb_transcoder *tr, int i)
static VALUE econv_s_asciicompat_encoding(VALUE klass, VALUE arg)
int rb_respond_to(VALUE, ID)
const char * destination_encoding_name
static VALUE econv_convpath(VALUE self)
static int trans_sweep(rb_econv_t *ec, const unsigned char **input_ptr, const unsigned char *input_stop, unsigned char **output_ptr, unsigned char *output_stop, int flags, int start)
VALUE rb_enc_default_internal(void)
VALUE rb_ary_new2(long capa)
const char * rb_econv_asciicompat_encoding(const char *ascii_incompat_name)
#define ECONV_UNDEF_HEX_CHARREF
VALUE rb_str_new(const char *, long)
const char * rb_econv_encoding_to_insert_output(rb_econv_t *ec)
static VALUE ecerr_source_encoding(VALUE self)
static int output_hex_charref(rb_econv_t *ec)
VALUE rb_str_drop_bytes(VALUE, long)
VALUE rb_econv_str_convert(rb_econv_t *ec, VALUE src, int flags)
static VALUE ecerr_readagain_bytes(VALUE self)
#define rb_enc_asciicompat(enc)
VALUE rb_eUndefinedConversionError
const char * rb_id2name(ID id)
#define rb_check_arity(argc, min, max)
const char * replacement_enc
const char * source_encoding_name
int replacement_allocated
VALUE rb_hash_aref(VALUE, VALUE)
struct search_path_queue_tag * next
int rb_enc_find_index(const char *name)
static VALUE encoded_dup(VALUE newstr, VALUE str, int encidx)
static int econv_opts(VALUE opt, int ecflags)
static VALUE sym_destination_buffer_full
static unsigned char * allocate_converted_string(const char *sname, const char *dname, const unsigned char *str, size_t len, unsigned char *caller_dst_buf, size_t caller_dst_bufsize, size_t *dst_len_ptr)
#define ECONV_CRLF_NEWLINE_DECORATOR
const rb_transcoder * transcoder
static transcoder_entry_t * get_transcoder_entry(const char *sname, const char *dname)
RUBY_EXTERN VALUE rb_cData
struct rb_transcoding rb_transcoding
#define ECONV_UNDEF_REPLACE
#define REALLOC_N(var, type, n)
struct rb_econv_t::@143 last_error
int rb_enc_str_coderange(VALUE)
void rb_define_method(VALUE klass, const char *name, VALUE(*func)(ANYARGS), int argc)
const unsigned char * replacement_str
#define STR1_LENGTH(byte_addr)
VALUE(* func_ii)(void *, VALUE)
#define encoding_equal(enc1, enc2)
#define TRANSCODING_WRITEBUF_SIZE(tc)
static rb_encoding * make_dummy_encoding(const char *name)
#define ECONV_XML_TEXT_DECORATOR
rb_encoding * rb_enc_find(const char *name)
transcoder_entry_t ** entries
void st_free_table(st_table *)
static VALUE econv_result_to_symbol(rb_econv_result_t res)
#define ECONV_UNIVERSAL_NEWLINE_DECORATOR
#define ECONV_CR_NEWLINE_DECORATOR
static VALUE ecerr_source_encoding_name(VALUE self)
rb_encoding * rb_enc_from_index(int index)
static VALUE econv_primitive_convert(int argc, VALUE *argv, VALUE self)
ssize_t(* func_io)(void *, VALUE, const unsigned char *, size_t)
#define ECONV_INVALID_MASK
static VALUE ecerr_destination_encoding_name(VALUE self)
static VALUE sym_incomplete_input
void rb_str_set_len(VALUE, long)