mirrors/quickjs
1
0
Fork 0
mirror of https://github.com/bellard/quickjs.git synced 2025-05-29 01:49:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

regexp: added v flag support - fixed corner cases of case insensitive matching

Browse Source
This commit is contained in:
Fabrice Bellard 2025-05-16 17:43:03 +02:00
parent a8b2d7c2b2
commit d7cdfdc8d7
13 changed files with 2004 additions and 205 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

756
libregexp.c
View File

@ -71,6 +71,7 @@ typedef struct {
const uint8_t *buf_start;
int re_flags;
BOOL is_unicode;
BOOL unicode_sets; /* if set, is_unicode is also set */
BOOL ignore_case;
BOOL dotall;
int capture_count;
@ -102,11 +103,11 @@ static const REOpCode reopcode_info[REOP_COUNT] = {
};
#define RE_HEADER_FLAGS 0
#define RE_HEADER_CAPTURE_COUNT 1
#define RE_HEADER_STACK_SIZE 2
#define RE_HEADER_BYTECODE_LEN 3
#define RE_HEADER_CAPTURE_COUNT 2
#define RE_HEADER_STACK_SIZE 3
#define RE_HEADER_BYTECODE_LEN 4
#define RE_HEADER_LEN 7
#define RE_HEADER_LEN 8
static inline int is_digit(int c) {
return c >= '0' && c <= '9';
@ -122,6 +123,264 @@ static int dbuf_insert(DynBuf *s, int pos, int len)
return 0;
}
typedef struct REString {
struct REString *next;
uint32_t hash;
uint32_t len;
uint32_t buf[];
} REString;
typedef struct {
/* the string list is the union of 'char_range' and of the strings
in hash_table[]. The strings in hash_table[] have a length !=
1. */
CharRange cr;
uint32_t n_strings;
uint32_t hash_size;
int hash_bits;
REString **hash_table;
} REStringList;
static uint32_t re_string_hash(int len, const uint32_t *buf)
{
int i;
uint32_t h;
h = 1;
for(i = 0; i < len; i++)
h = h * 263 + buf[i];
return h * 0x61C88647;
}
static void re_string_list_init(REParseState *s1, REStringList *s)
{
cr_init(&s->cr, s1->opaque, lre_realloc);
s->n_strings = 0;
s->hash_size = 0;
s->hash_bits = 0;
s->hash_table = NULL;
}
static void re_string_list_free(REStringList *s)
{
REString *p, *p_next;
int i;
for(i = 0; i < s->hash_size; i++) {
for(p = s->hash_table[i]; p != NULL; p = p_next) {
p_next = p->next;
lre_realloc(s->cr.mem_opaque, p, 0);
}
}
lre_realloc(s->cr.mem_opaque, s->hash_table, 0);
cr_free(&s->cr);
}
static void lre_print_char(int c, BOOL is_range)
{
if (c == '\'' || c == '\\' ||
(is_range && (c == '-' || c == ']'))) {
printf("\\%c", c);
} else if (c >= ' ' && c <= 126) {
printf("%c", c);
} else {
printf("\\u{%04x}", c);
}
}
static __maybe_unused void re_string_list_dump(const char *str, const REStringList *s)
{
REString *p;
const CharRange *cr;
int i, j, k;
printf("%s:\n", str);
printf(" ranges: [");
cr = &s->cr;
for(i = 0; i < cr->len; i += 2) {
lre_print_char(cr->points[i], TRUE);
if (cr->points[i] != cr->points[i + 1] - 1) {
printf("-");
lre_print_char(cr->points[i + 1] - 1, TRUE);
}
}
printf("]\n");
j = 0;
for(i = 0; i < s->hash_size; i++) {
for(p = s->hash_table[i]; p != NULL; p = p->next) {
printf(" %d/%d: '", j, s->n_strings);
for(k = 0; k < p->len; k++) {
lre_print_char(p->buf[k], FALSE);
}
printf("'\n");
j++;
}
}
}
static int re_string_find2(REStringList *s, int len, const uint32_t *buf,
uint32_t h0, BOOL add_flag)
{
uint32_t h = 0; /* avoid warning */
REString *p;
if (s->n_strings != 0) {
h = h0 >> (32 - s->hash_bits);
for(p = s->hash_table[h]; p != NULL; p = p->next) {
if (p->hash == h0 && p->len == len &&
!memcmp(p->buf, buf, len * sizeof(buf[0]))) {
return 1;
}
}
}
/* not found */
if (!add_flag)
return 0;
/* increase the size of the hash table if needed */
if (unlikely((s->n_strings + 1) > s->hash_size)) {
REString **new_hash_table, *p_next;
int new_hash_bits, i;
uint32_t new_hash_size;
new_hash_bits = max_int(s->hash_bits + 1, 4);
new_hash_size = 1 << new_hash_bits;
new_hash_table = lre_realloc(s->cr.mem_opaque, NULL,
sizeof(new_hash_table[0]) * new_hash_size);
if (!new_hash_table)
return -1;
memset(new_hash_table, 0, sizeof(new_hash_table[0]) * new_hash_size);
for(i = 0; i < s->hash_size; i++) {
for(p = s->hash_table[i]; p != NULL; p = p_next) {
p_next = p->next;
h = p->hash >> (32 - new_hash_bits);
p->next = new_hash_table[h];
new_hash_table[h] = p;
}
}
lre_realloc(s->cr.mem_opaque, s->hash_table, 0);
s->hash_bits = new_hash_bits;
s->hash_size = new_hash_size;
s->hash_table = new_hash_table;
h = h0 >> (32 - s->hash_bits);
}
p = lre_realloc(s->cr.mem_opaque, NULL, sizeof(REString) + len * sizeof(buf[0]));
if (!p)
return -1;
p->next = s->hash_table[h];
s->hash_table[h] = p;
s->n_strings++;
p->hash = h0;
p->len = len;
memcpy(p->buf, buf, sizeof(buf[0]) * len);
return 1;
}
static int re_string_find(REStringList *s, int len, const uint32_t *buf,
BOOL add_flag)
{
uint32_t h0;
h0 = re_string_hash(len, buf);
return re_string_find2(s, len, buf, h0, add_flag);
}
/* return -1 if memory error, 0 if OK */
static int re_string_add(REStringList *s, int len, const uint32_t *buf)
{
if (len == 1) {
return cr_union_interval(&s->cr, buf[0], buf[0]);
}
if (re_string_find(s, len, buf, TRUE) < 0)
return -1;
return 0;
}
/* a = a op b */
static int re_string_list_op(REStringList *a, REStringList *b, int op)
{
int i, ret;
REString *p, **pp;
if (cr_op1(&a->cr, b->cr.points, b->cr.len, op))
return -1;
switch(op) {
case CR_OP_UNION:
if (b->n_strings != 0) {
for(i = 0; i < b->hash_size; i++) {
for(p = b->hash_table[i]; p != NULL; p = p->next) {
if (re_string_find2(a, p->len, p->buf, p->hash, TRUE) < 0)
return -1;
}
}
}
break;
case CR_OP_INTER:
case CR_OP_SUB:
for(i = 0; i < a->hash_size; i++) {
pp = &a->hash_table[i];
for(;;) {
p = *pp;
if (p == NULL)
break;
ret = re_string_find2(b, p->len, p->buf, p->hash, FALSE);
if (op == CR_OP_SUB)
ret = !ret;
if (!ret) {
/* remove it */
*pp = p->next;
a->n_strings--;
lre_realloc(a->cr.mem_opaque, p, 0);
} else {
/* keep it */
pp = &p->next;
}
}
}
break;
default:
abort();
}
return 0;
}
static int re_string_list_canonicalize(REParseState *s1,
REStringList *s, BOOL is_unicode)
{
if (cr_regexp_canonicalize(&s->cr, is_unicode))
return -1;
if (s->n_strings != 0) {
REStringList a_s, *a = &a_s;
int i, j;
REString *p;
/* XXX: simplify */
re_string_list_init(s1, a);
a->n_strings = s->n_strings;
a->hash_size = s->hash_size;
a->hash_bits = s->hash_bits;
a->hash_table = s->hash_table;
s->n_strings = 0;
s->hash_size = 0;
s->hash_bits = 0;
s->hash_table = NULL;
for(i = 0; i < a->hash_size; i++) {
for(p = a->hash_table[i]; p != NULL; p = p->next) {
for(j = 0; j < p->len; j++) {
p->buf[j] = lre_canonicalize(p->buf[j], is_unicode);
}
if (re_string_add(s, p->len, p->buf)) {
re_string_list_free(a);
return -1;
}
}
}
re_string_list_free(a);
}
return 0;
}
static const uint16_t char_range_d[] = {
1,
0x0030, 0x0039 + 1,
@ -170,7 +429,7 @@ static const uint16_t * const char_range_table[] = {
char_range_w,
};
static int cr_init_char_range(REParseState *s, CharRange *cr, uint32_t c)
static int cr_init_char_range(REParseState *s, REStringList *cr, uint32_t c)
{
BOOL invert;
const uint16_t *c_pt;
@ -179,18 +438,18 @@ static int cr_init_char_range(REParseState *s, CharRange *cr, uint32_t c)
invert = c & 1;
c_pt = char_range_table[c >> 1];
len = *c_pt++;
cr_init(cr, s->opaque, lre_realloc);
re_string_list_init(s, cr);
for(i = 0; i < len * 2; i++) {
if (cr_add_point(cr, c_pt[i]))
if (cr_add_point(&cr->cr, c_pt[i]))
goto fail;
}
if (invert) {
if (cr_invert(cr))
if (cr_invert(&cr->cr))
goto fail;
}
return 0;
fail:
cr_free(cr);
re_string_list_free(cr);
return -1;
}
@ -533,8 +792,16 @@ static BOOL is_unicode_char(int c)
(c == '_'));
}
static int parse_unicode_property(REParseState *s, CharRange *cr,
const uint8_t **pp, BOOL is_inv)
/* XXX: memory error test */
static void seq_prop_cb(void *opaque, const uint32_t *seq, int seq_len)
{
REStringList *sl = opaque;
re_string_add(sl, seq_len, seq);
}
static int parse_unicode_property(REParseState *s, REStringList *cr,
const uint8_t **pp, BOOL is_inv,
BOOL allow_sequence_prop)
{
const uint8_t *p;
char name[64], value[64];
@ -574,51 +841,76 @@ static int parse_unicode_property(REParseState *s, CharRange *cr,
} else if (!strcmp(name, "Script_Extensions") || !strcmp(name, "scx")) {
script_ext = TRUE;
do_script:
cr_init(cr, s->opaque, lre_realloc);
ret = unicode_script(cr, value, script_ext);
re_string_list_init(s, cr);
ret = unicode_script(&cr->cr, value, script_ext);
if (ret) {
cr_free(cr);
re_string_list_free(cr);
if (ret == -2)
return re_parse_error(s, "unknown unicode script");
else
goto out_of_memory;
}
} else if (!strcmp(name, "General_Category") || !strcmp(name, "gc")) {
cr_init(cr, s->opaque, lre_realloc);
ret = unicode_general_category(cr, value);
re_string_list_init(s, cr);
ret = unicode_general_category(&cr->cr, value);
if (ret) {
cr_free(cr);
re_string_list_free(cr);
if (ret == -2)
return re_parse_error(s, "unknown unicode general category");
else
goto out_of_memory;
}
} else if (value[0] == '\0') {
cr_init(cr, s->opaque, lre_realloc);
ret = unicode_general_category(cr, name);
re_string_list_init(s, cr);
ret = unicode_general_category(&cr->cr, name);
if (ret == -1) {
cr_free(cr);
re_string_list_free(cr);
goto out_of_memory;
}
if (ret < 0) {
ret = unicode_prop(cr, name);
if (ret) {
cr_free(cr);
if (ret == -2)
goto unknown_property_name;
else
ret = unicode_prop(&cr->cr, name);
if (ret == -1) {
re_string_list_free(cr);
goto out_of_memory;
}
}
if (ret < 0 && !is_inv && allow_sequence_prop) {
CharRange cr_tmp;
cr_init(&cr_tmp, s->opaque, lre_realloc);
ret = unicode_sequence_prop(name, seq_prop_cb, cr, &cr_tmp);
cr_free(&cr_tmp);
if (ret == -1) {
re_string_list_free(cr);
goto out_of_memory;
}
}
if (ret < 0)
goto unknown_property_name;
} else {
unknown_property_name:
return re_parse_error(s, "unknown unicode property name");
}
/* the ordering of case folding and inversion differs with
unicode_sets. 'unicode_sets' ordering is more consistent */
/* XXX: the spec seems incorrect, we do it as the other engines
seem to do it. */
if (s->ignore_case && s->unicode_sets) {
if (re_string_list_canonicalize(s, cr, s->is_unicode)) {
re_string_list_free(cr);
goto out_of_memory;
}
}
if (is_inv) {
if (cr_invert(cr)) {
cr_free(cr);
return -1;
if (cr_invert(&cr->cr)) {
re_string_list_free(cr);
goto out_of_memory;
}
}
if (s->ignore_case && !s->unicode_sets) {
if (re_string_list_canonicalize(s, cr, s->is_unicode)) {
re_string_list_free(cr);
goto out_of_memory;
}
}
*pp = p;
@ -628,10 +920,61 @@ static int parse_unicode_property(REParseState *s, CharRange *cr,
}
#endif /* CONFIG_ALL_UNICODE */
static int get_class_atom(REParseState *s, REStringList *cr,
const uint8_t **pp, BOOL inclass);
static int parse_class_string_disjunction(REParseState *s, REStringList *cr,
const uint8_t **pp)
{
const uint8_t *p;
DynBuf str;
int c;
p = *pp;
if (*p != '{')
return re_parse_error(s, "expecting '{' after \\q");
dbuf_init2(&str, s->opaque, lre_realloc);
re_string_list_init(s, cr);
p++;
for(;;) {
str.size = 0;
while (*p != '}' && *p != '|') {
c = get_class_atom(s, NULL, &p, FALSE);
if (c < 0)
goto fail;
if (dbuf_put_u32(&str, c)) {
re_parse_out_of_memory(s);
goto fail;
}
}
if (re_string_add(cr, str.size / 4, (uint32_t *)str.buf)) {
re_parse_out_of_memory(s);
goto fail;
}
if (*p == '}')
break;
p++;
}
if (s->ignore_case) {
if (re_string_list_canonicalize(s, cr, TRUE))
goto fail;
}
p++; /* skip the '}' */
dbuf_free(&str);
*pp = p;
return 0;
fail:
dbuf_free(&str);
re_string_list_free(cr);
return -1;
}
/* return -1 if error otherwise the character or a class range
(CLASS_RANGE_BASE). In case of class range, 'cr' is
(CLASS_RANGE_BASE) if cr != NULL. In case of class range, 'cr' is
initialized. Otherwise, it is ignored. */
static int get_class_atom(REParseState *s, CharRange *cr,
static int get_class_atom(REParseState *s, REStringList *cr,
const uint8_t **pp, BOOL inclass)
{
const uint8_t *p;
@ -666,6 +1009,8 @@ static int get_class_atom(REParseState *s, CharRange *cr,
case 'W':
c = CHAR_RANGE_W;
class_range:
if (!cr)
goto default_escape;
if (cr_init_char_range(s, cr, c))
return -1;
c = CLASS_RANGE_BASE;
@ -690,27 +1035,50 @@ static int get_class_atom(REParseState *s, CharRange *cr,
if (!inclass && s->is_unicode)
goto invalid_escape;
break;
case '^':
case '$':
case '\\':
case '.':
case '*':
case '+':
case '?':
case '(':
case ')':
case '[':
case ']':
case '{':
case '}':
case '|':
case '/':
/* always valid to escape these characters */
break;
#ifdef CONFIG_ALL_UNICODE
case 'p':
case 'P':
if (s->is_unicode) {
if (parse_unicode_property(s, cr, &p, (c == 'P')))
if (s->is_unicode && cr) {
if (parse_unicode_property(s, cr, &p, (c == 'P'), s->unicode_sets))
return -1;
c = CLASS_RANGE_BASE;
break;
}
/* fall thru */
goto default_escape;
#endif
case 'q':
if (s->unicode_sets && cr && inclass) {
if (parse_class_string_disjunction(s, cr, &p))
return -1;
c = CLASS_RANGE_BASE;
break;
}
goto default_escape;
default:
default_escape:
p--;
ret = lre_parse_escape(&p, s->is_unicode * 2);
if (ret >= 0) {
c = ret;
} else {
if (ret == -2 && *p != '\0' && strchr("^$\\.*+?()[]{}|/", *p)) {
/* always valid to escape these characters */
goto normal_char;
} else if (s->is_unicode) {
if (s->is_unicode) {
invalid_escape:
return re_parse_error(s, "invalid escape sequence in regular expression");
} else {
@ -727,6 +1095,48 @@ static int get_class_atom(REParseState *s, CharRange *cr,
return re_parse_error(s, "unexpected end");
}
/* fall thru */
goto normal_char;
case '&':
case '!':
case '#':
case '$':
case '%':
case '*':
case '+':
case ',':
case '.':
case ':':
case ';':
case '<':
case '=':
case '>':
case '?':
case '@':
case '^':
case '`':
case '~':
if (s->unicode_sets && p[1] == c) {
/* forbidden double characters */
return re_parse_error(s, "invalid class set operation in regular expression");
}
goto normal_char;
case '(':
case ')':
case '[':
case ']':
case '{':
case '}':
case '/':
case '-':
case '|':
if (s->unicode_sets) {
/* invalid characters in unicode sets */
return re_parse_error(s, "invalid character in class in regular expression");
}
goto normal_char;
default:
normal_char:
/* normal char */
@ -754,8 +1164,6 @@ static int re_emit_range(REParseState *s, const CharRange *cr)
if (len >= 65535)
return re_parse_error(s, "too many ranges");
if (len == 0) {
/* not sure it can really happen. Emit a match that is always
false */
re_emit_op_u32(s, REOP_char32, -1);
} else {
high = cr->points[cr->len - 1];
@ -783,15 +1191,135 @@ static int re_emit_range(REParseState *s, const CharRange *cr)
return 0;
}
static int re_parse_char_class(REParseState *s, const uint8_t **pp)
static int re_string_cmp_len(const void *a, const void *b, void *arg)
{
REString *p1 = *(REString **)a;
REString *p2 = *(REString **)b;
return (p1->len < p2->len) - (p1->len > p2->len);
}
static int re_emit_string_list(REParseState *s, const REStringList *sl)
{
REString **tab, *p;
int i, j, c, split_pos, last_match_pos, n;
BOOL has_empty_string, is_last;
// re_string_list_dump("sl", sl);
if (sl->n_strings == 0) {
/* simple case: only characters */
if (re_emit_range(s, &sl->cr))
return -1;
} else {
/* at least one string list is present : match the longest ones first */
/* XXX: add a new op_switch opcode to compile as a trie */
tab = lre_realloc(s->opaque, NULL, sizeof(tab[0]) * sl->n_strings);
if (!tab) {
re_parse_out_of_memory(s);
return -1;
}
has_empty_string = FALSE;
n = 0;
for(i = 0; i < sl->hash_size; i++) {
for(p = sl->hash_table[i]; p != NULL; p = p->next) {
if (p->len == 0) {
has_empty_string = TRUE;
} else {
tab[n++] = p;
}
}
}
assert(n <= sl->n_strings);
rqsort(tab, n, sizeof(tab[0]), re_string_cmp_len, NULL);
last_match_pos = -1;
for(i = 0; i < n; i++) {
p = tab[i];
is_last = !has_empty_string && sl->cr.len == 0 && i == (n - 1);
if (!is_last)
split_pos = re_emit_op_u32(s, REOP_split_next_first, 0);
else
split_pos = 0;
for(j = 0; j < p->len; j++) {
c = p->buf[j];
if (c <= 0xffff)
re_emit_op_u16(s, REOP_char, c);
else
re_emit_op_u32(s, REOP_char32, c);
}
if (!is_last) {
last_match_pos = re_emit_op_u32(s, REOP_goto, last_match_pos);
put_u32(s->byte_code.buf + split_pos, s->byte_code.size - (split_pos + 4));
}
}
if (sl->cr.len != 0) {
/* char range */
is_last = !has_empty_string;
if (!is_last)
split_pos = re_emit_op_u32(s, REOP_split_next_first, 0);
else
split_pos = 0; /* not used */
if (re_emit_range(s, &sl->cr)) {
lre_realloc(s->opaque, tab, 0);
return -1;
}
if (!is_last)
put_u32(s->byte_code.buf + split_pos, s->byte_code.size - (split_pos + 4));
}
/* patch the 'goto match' */
while (last_match_pos != -1) {
int next_pos = get_u32(s->byte_code.buf + last_match_pos);
put_u32(s->byte_code.buf + last_match_pos, s->byte_code.size - (last_match_pos + 4));
last_match_pos = next_pos;
}
lre_realloc(s->opaque, tab, 0);
}
return 0;
}
static int re_parse_nested_class(REParseState *s, REStringList *cr, const uint8_t **pp);
static int re_parse_class_set_operand(REParseState *s, REStringList *cr, const uint8_t **pp)
{
int c1;
const uint8_t *p = *pp;
if (*p == '[') {
if (re_parse_nested_class(s, cr, pp))
return -1;
} else {
c1 = get_class_atom(s, cr, pp, TRUE);
if (c1 < 0)
return -1;
if (c1 < CLASS_RANGE_BASE) {
/* create a range with a single character */
re_string_list_init(s, cr);
if (s->ignore_case)
c1 = lre_canonicalize(c1, s->is_unicode);
if (cr_union_interval(&cr->cr, c1, c1)) {
re_string_list_free(cr);
return -1;
}
}
}
return 0;
}
static int re_parse_nested_class(REParseState *s, REStringList *cr, const uint8_t **pp)
{
const uint8_t *p;
uint32_t c1, c2;
CharRange cr_s, *cr = &cr_s;
CharRange cr1_s, *cr1 = &cr1_s;
BOOL invert;
int ret;
REStringList cr1_s, *cr1 = &cr1_s;
BOOL invert, is_first;
cr_init(cr, s->opaque, lre_realloc);
if (lre_check_stack_overflow(s->opaque, 0))
return re_parse_error(s, "stack overflow");
re_string_list_init(s, cr);
p = *pp;
p++; /* skip '[' */
@ -801,17 +1329,26 @@ static int re_parse_char_class(REParseState *s, const uint8_t **pp)
invert = TRUE;
}
/* handle unions */
is_first = TRUE;
for(;;) {
if (*p == ']')
break;
if (*p == '[' && s->unicode_sets) {
if (re_parse_nested_class(s, cr1, &p))
goto fail;
goto class_union;
} else {
c1 = get_class_atom(s, cr1, &p, TRUE);
if ((int)c1 < 0)
goto fail;
if (*p == '-' && p[1] != ']') {
const uint8_t *p0 = p + 1;
if (p[1] == '-' && s->unicode_sets && is_first)
goto class_atom; /* first character class followed by '--' */
if (c1 >= CLASS_RANGE_BASE) {
if (s->is_unicode) {
cr_free(cr1);
re_string_list_free(cr1);
goto invalid_class_range;
}
/* Annex B: match '-' character */
@ -821,7 +1358,7 @@ static int re_parse_char_class(REParseState *s, const uint8_t **pp)
if ((int)c2 < 0)
goto fail;
if (c2 >= CLASS_RANGE_BASE) {
cr_free(cr1);
re_string_list_free(cr1);
if (s->is_unicode) {
goto invalid_class_range;
}
@ -834,40 +1371,112 @@ static int re_parse_char_class(REParseState *s, const uint8_t **pp)
re_parse_error(s, "invalid class range");
goto fail;
}
if (cr_union_interval(cr, c1, c2))
if (s->ignore_case) {
CharRange cr2_s, *cr2 = &cr2_s;
cr_init(cr2, s->opaque, lre_realloc);
if (cr_add_interval(cr2, c1, c2 + 1) ||
cr_regexp_canonicalize(cr2, s->is_unicode) ||
cr_op1(&cr->cr, cr2->points, cr2->len, CR_OP_UNION)) {
cr_free(cr2);
goto memory_error;
}
cr_free(cr2);
} else {
if (cr_union_interval(&cr->cr, c1, c2))
goto memory_error;
}
is_first = FALSE; /* union operation */
} else {
class_atom:
if (c1 >= CLASS_RANGE_BASE) {
int ret;
ret = cr_union1(cr, cr1->points, cr1->len);
cr_free(cr1);
class_union:
ret = re_string_list_op(cr, cr1, CR_OP_UNION);
re_string_list_free(cr1);
if (ret)
goto memory_error;
} else {
if (cr_union_interval(cr, c1, c1))
if (s->ignore_case)
c1 = lre_canonicalize(c1, s->is_unicode);
if (cr_union_interval(&cr->cr, c1, c1))
goto memory_error;
}
}
}
if (s->ignore_case) {
if (cr_regexp_canonicalize(cr, s->is_unicode))
goto memory_error;
if (s->unicode_sets && is_first) {
if (*p == '&' && p[1] == '&' && p[2] != '&') {
/* handle '&&' */
for(;;) {
if (*p == ']') {
break;
} else if (*p == '&' && p[1] == '&' && p[2] != '&') {
p += 2;
} else {
goto invalid_operation;
}
if (invert) {
if (cr_invert(cr))
goto memory_error;
}
if (re_emit_range(s, cr))
if (re_parse_class_set_operand(s, cr1, &p))
goto fail;
cr_free(cr);
ret = re_string_list_op(cr, cr1, CR_OP_INTER);
re_string_list_free(cr1);
if (ret)
goto memory_error;
}
} else if (*p == '-' && p[1] == '-') {
/* handle '--' */
for(;;) {
if (*p == ']') {
break;
} else if (*p == '-' && p[1] == '-') {
p += 2;
} else {
invalid_operation:
re_parse_error(s, "invalid operation in regular expression");
goto fail;
}
if (re_parse_class_set_operand(s, cr1, &p))
goto fail;
ret = re_string_list_op(cr, cr1, CR_OP_SUB);
re_string_list_free(cr1);
if (ret)
goto memory_error;
}
}
}
is_first = FALSE;
}
p++; /* skip ']' */
*pp = p;
if (invert) {
/* XXX: add may_contain_string syntax check to be fully
compliant. The test here accepts more input than the
spec. */
if (cr->n_strings != 0) {
re_parse_error(s, "negated character class with strings in regular expression debugger eval code");
goto fail;
}
if (cr_invert(&cr->cr))
goto memory_error;
}
return 0;
memory_error:
re_parse_out_of_memory(s);
fail:
cr_free(cr);
re_string_list_free(cr);
return -1;
}
static int re_parse_char_class(REParseState *s, const uint8_t **pp)
{
REStringList cr_s, *cr = &cr_s;
if (re_parse_nested_class(s, cr, pp))
return -1;
if (re_emit_string_list(s, cr))
goto fail;
re_string_list_free(cr);
return 0;
fail:
re_string_list_free(cr);
return -1;
}
@ -1121,7 +1730,7 @@ static int re_parse_term(REParseState *s, BOOL is_backward_dir)
const uint8_t *p;
int c, last_atom_start, quant_min, quant_max, last_capture_count;
BOOL greedy, add_zero_advance_check, is_neg, is_backward_lookahead;
CharRange cr_s, *cr = &cr_s;
REStringList cr_s, *cr = &cr_s;
last_atom_start = -1;
last_capture_count = 0;
@ -1385,9 +1994,8 @@ static int re_parse_term(REParseState *s, BOOL is_backward_dir)
re_emit_op(s, REOP_prev);
if (c >= CLASS_RANGE_BASE) {
int ret;
/* Note: canonicalization is not needed */
ret = re_emit_range(s, cr);
cr_free(cr);
ret = re_emit_string_list(s, cr);
re_string_list_free(cr);
if (ret)
return -1;
} else {
@ -1737,10 +2345,11 @@ uint8_t *lre_compile(int *plen, char *error_msg, int error_msg_size,
s->buf_end = s->buf_ptr + buf_len;
s->buf_start = s->buf_ptr;
s->re_flags = re_flags;
s->is_unicode = ((re_flags & LRE_FLAG_UNICODE) != 0);
s->is_unicode = ((re_flags & (LRE_FLAG_UNICODE | LRE_FLAG_UNICODE_SETS)) != 0);
is_sticky = ((re_flags & LRE_FLAG_STICKY) != 0);
s->ignore_case = ((re_flags & LRE_FLAG_IGNORECASE) != 0);
s->dotall = ((re_flags & LRE_FLAG_DOTALL) != 0);
s->unicode_sets = ((re_flags & LRE_FLAG_UNICODE_SETS) != 0);
s->capture_count = 1;
s->total_capture_count = -1;
s->has_named_captures = -1;
@ -1748,7 +2357,7 @@ uint8_t *lre_compile(int *plen, char *error_msg, int error_msg_size,
dbuf_init2(&s->byte_code, opaque, lre_realloc);
dbuf_init2(&s->group_names, opaque, lre_realloc);
dbuf_putc(&s->byte_code, re_flags); /* first element is the flags */
dbuf_put_u16(&s->byte_code, re_flags); /* first element is the flags */
dbuf_putc(&s->byte_code, 0); /* second element is the number of captures */
dbuf_putc(&s->byte_code, 0); /* stack size */
dbuf_put_u32(&s->byte_code, 0); /* bytecode length */
@ -1801,7 +2410,8 @@ uint8_t *lre_compile(int *plen, char *error_msg, int error_msg_size,
/* add the named groups if needed */
if (s->group_names.size > (s->capture_count - 1)) {
dbuf_put(&s->byte_code, s->group_names.buf, s->group_names.size);
s->byte_code.buf[RE_HEADER_FLAGS] |= LRE_FLAG_NAMED_GROUPS;
put_u16(s->byte_code.buf + RE_HEADER_FLAGS,
lre_get_flags(s->byte_code.buf) | LRE_FLAG_NAMED_GROUPS);
}
dbuf_free(&s->group_names);
@ -2221,6 +2831,8 @@ static intptr_t lre_exec_backtrack(REExecContext *s, uint8_t **capture,
v1 = FALSE;
} else {
PEEK_PREV_CHAR(c, cptr, s->cbuf, cbuf_type);
if (s->ignore_case)
c = lre_canonicalize(c, s->is_unicode);
v1 = is_word_char(c);
}
/* current char */
@ -2228,6 +2840,8 @@ static intptr_t lre_exec_backtrack(REExecContext *s, uint8_t **capture,
v2 = FALSE;
} else {
PEEK_CHAR(c, cptr, cbuf_end, cbuf_type);
if (s->ignore_case)
c = lre_canonicalize(c, s->is_unicode);
v2 = is_word_char(c);
}
if (v1 ^ v2 ^ (REOP_not_word_boundary - opcode))
@ -2424,7 +3038,7 @@ int lre_exec(uint8_t **capture,
re_flags = lre_get_flags(bc_buf);
s->multi_line = (re_flags & LRE_FLAG_MULTILINE) != 0;
s->ignore_case = (re_flags & LRE_FLAG_IGNORECASE) != 0;
s->is_unicode = (re_flags & LRE_FLAG_UNICODE) != 0;
s->is_unicode = (re_flags & (LRE_FLAG_UNICODE | LRE_FLAG_UNICODE_SETS)) != 0;
s->capture_count = bc_buf[RE_HEADER_CAPTURE_COUNT];
s->stack_size_max = bc_buf[RE_HEADER_STACK_SIZE];
s->cbuf = cbuf;
@ -2459,7 +3073,7 @@ int lre_get_capture_count(const uint8_t *bc_buf)
int lre_get_flags(const uint8_t *bc_buf)
{
return bc_buf[RE_HEADER_FLAGS];
return get_u16(bc_buf + RE_HEADER_FLAGS);
}
/* Return NULL if no group names. Otherwise, return a pointer to

1
libregexp.h
View File

@ -35,6 +35,7 @@
#define LRE_FLAG_STICKY (1 << 5)
#define LRE_FLAG_INDICES (1 << 6) /* Unused by libregexp, just recorded. */
#define LRE_FLAG_NAMED_GROUPS (1 << 7) /* named groups are present in the regexp */
#define LRE_FLAG_UNICODE_SETS (1 << 8)
#define LRE_RET_MEMORY_ERROR (-1)
#define LRE_RET_TIMEOUT (-2)

420
libunicode-table.h
View File

@ -4054,6 +4054,89 @@ static const uint8_t unicode_prop_Changes_When_NFKC_Casefolded1_table[450] = {
0x4f, 0xff,
};
static const uint8_t unicode_prop_Basic_Emoji1_table[143] = {
0x60, 0x23, 0x19, 0x81, 0x40, 0xcc, 0x1a, 0x01,
0x80, 0x42, 0x08, 0x81, 0x94, 0x81, 0xb1, 0x8b,
0xaa, 0x80, 0x92, 0x80, 0x8c, 0x07, 0x81, 0x90,
0x0c, 0x0f, 0x04, 0x80, 0x94, 0x06, 0x08, 0x03,
0x01, 0x06, 0x03, 0x81, 0x9b, 0x80, 0xa2, 0x00,
0x03, 0x10, 0x80, 0xbc, 0x82, 0x97, 0x80, 0x8d,
0x80, 0x43, 0x5a, 0x81, 0xb2, 0x03, 0x80, 0x61,
0xc4, 0xad, 0x80, 0x40, 0xc9, 0x80, 0x40, 0xbd,
0x01, 0x89, 0xe5, 0x80, 0x97, 0x80, 0x93, 0x01,
0x20, 0x82, 0x94, 0x81, 0x40, 0xad, 0xa0, 0x8b,
0x88, 0x80, 0xc5, 0x80, 0x95, 0x8b, 0xaa, 0x1c,
0x8b, 0x90, 0x10, 0x82, 0xc6, 0x00, 0x80, 0x40,
0xba, 0x81, 0xbe, 0x8c, 0x18, 0x97, 0x91, 0x80,
0x99, 0x81, 0x8c, 0x80, 0xd5, 0xd4, 0xaf, 0xc5,
0x28, 0x12, 0x0a, 0x1b, 0x8a, 0x0e, 0x88, 0x40,
0xe2, 0x8b, 0x18, 0x41, 0x1a, 0xae, 0x80, 0x89,
0x80, 0x40, 0xb8, 0xef, 0x8c, 0x82, 0x89, 0x84,
0xb7, 0x86, 0x8e, 0x81, 0x8a, 0x85, 0x88,
};
static const uint8_t unicode_prop_Basic_Emoji2_table[183] = {
0x40, 0xa8, 0x03, 0x80, 0x5f, 0x8c, 0x80, 0x8b,
0x80, 0x40, 0xd7, 0x80, 0x95, 0x80, 0xd9, 0x85,
0x8e, 0x81, 0x41, 0x7c, 0x80, 0x40, 0xa5, 0x80,
0x9c, 0x10, 0x0c, 0x82, 0x40, 0xc6, 0x80, 0x40,
0xe6, 0x81, 0x89, 0x80, 0x88, 0x80, 0xb9, 0x0a,
0x84, 0x88, 0x01, 0x05, 0x03, 0x01, 0x00, 0x09,
0x02, 0x02, 0x0f, 0x14, 0x00, 0x80, 0x9b, 0x09,
0x00, 0x08, 0x80, 0x91, 0x01, 0x80, 0x92, 0x00,
0x18, 0x00, 0x0a, 0x05, 0x07, 0x81, 0x95, 0x05,
0x00, 0x00, 0x80, 0x94, 0x05, 0x09, 0x01, 0x17,
0x04, 0x09, 0x08, 0x01, 0x00, 0x00, 0x05, 0x02,
0x80, 0x90, 0x81, 0x8e, 0x01, 0x80, 0x9a, 0x81,
0xbb, 0x80, 0x41, 0x91, 0x81, 0x41, 0xce, 0x82,
0x45, 0x27, 0x80, 0x8b, 0x80, 0x42, 0x58, 0x00,
0x80, 0x61, 0xbe, 0xd5, 0x81, 0x8b, 0x81, 0x40,
0x81, 0x80, 0xb3, 0x80, 0x40, 0xe8, 0x01, 0x88,
0x88, 0x80, 0xc5, 0x80, 0x97, 0x08, 0x11, 0x81,
0xaa, 0x1c, 0x8b, 0x92, 0x00, 0x00, 0x80, 0xc6,
0x00, 0x80, 0x40, 0xba, 0x80, 0xca, 0x81, 0xa3,
0x09, 0x86, 0x8c, 0x01, 0x19, 0x80, 0x93, 0x01,
0x07, 0x81, 0x88, 0x04, 0x82, 0x8b, 0x17, 0x11,
0x00, 0x03, 0x05, 0x02, 0x05, 0x80, 0x40, 0xcf,
0x00, 0x82, 0x8f, 0x2a, 0x05, 0x01, 0x80,
};
static const uint8_t unicode_prop_RGI_Emoji_Modifier_Sequence_table[73] = {
0x60, 0x26, 0x1c, 0x80, 0x40, 0xda, 0x80, 0x8f,
0x83, 0x61, 0xcc, 0x76, 0x80, 0xbb, 0x11, 0x01,
0x82, 0xf4, 0x09, 0x8a, 0x94, 0x18, 0x18, 0x88,
0x10, 0x1a, 0x02, 0x30, 0x00, 0x97, 0x80, 0x40,
0xc8, 0x0b, 0x80, 0x94, 0x03, 0x81, 0x40, 0xad,
0x12, 0x84, 0xd2, 0x80, 0x8f, 0x82, 0x88, 0x80,
0x8a, 0x80, 0x42, 0x3e, 0x01, 0x07, 0x3d, 0x80,
0x88, 0x89, 0x11, 0xb7, 0x80, 0xbc, 0x08, 0x08,
0x80, 0x90, 0x10, 0x8c, 0x40, 0xe4, 0x82, 0xa9,
0x88,
};
static const uint8_t unicode_prop_RGI_Emoji_Flag_Sequence_table[128] = {
0x0c, 0x00, 0x09, 0x00, 0x04, 0x01, 0x02, 0x06,
0x03, 0x03, 0x01, 0x02, 0x01, 0x03, 0x07, 0x0d,
0x18, 0x00, 0x09, 0x00, 0x00, 0x89, 0x08, 0x00,
0x00, 0x81, 0x88, 0x83, 0x8c, 0x10, 0x00, 0x01,
0x07, 0x08, 0x29, 0x10, 0x28, 0x00, 0x80, 0x8a,
0x00, 0x0a, 0x00, 0x0e, 0x15, 0x18, 0x83, 0x89,
0x06, 0x00, 0x81, 0x8d, 0x00, 0x12, 0x08, 0x00,
0x03, 0x00, 0x24, 0x00, 0x05, 0x21, 0x00, 0x00,
0x29, 0x90, 0x00, 0x02, 0x00, 0x08, 0x09, 0x00,
0x08, 0x18, 0x8b, 0x80, 0x8c, 0x02, 0x19, 0x1a,
0x11, 0x00, 0x00, 0x80, 0x9c, 0x80, 0x88, 0x02,
0x00, 0x00, 0x02, 0x20, 0x88, 0x0a, 0x00, 0x03,
0x01, 0x02, 0x05, 0x08, 0x00, 0x01, 0x09, 0x20,
0x21, 0x18, 0x22, 0x00, 0x00, 0x00, 0x00, 0x18,
0x28, 0x89, 0x80, 0x8b, 0x80, 0x90, 0x80, 0x92,
0x80, 0x8d, 0x05, 0x80, 0x8a, 0x80, 0x88, 0x80,
};
static const uint8_t unicode_prop_Emoji_Keycap_Sequence_table[4] = {
0xa2, 0x05, 0x04, 0x89,
};
static const uint8_t unicode_prop_ASCII_Hex_Digit_table[5] = {
0xaf, 0x89, 0x35, 0x99, 0x85,
};
@ -4493,6 +4576,11 @@ typedef enum {
UNICODE_PROP_Changes_When_Titlecased1,
UNICODE_PROP_Changes_When_Casefolded1,
UNICODE_PROP_Changes_When_NFKC_Casefolded1,
UNICODE_PROP_Basic_Emoji1,
UNICODE_PROP_Basic_Emoji2,
UNICODE_PROP_RGI_Emoji_Modifier_Sequence,
UNICODE_PROP_RGI_Emoji_Flag_Sequence,
UNICODE_PROP_Emoji_Keycap_Sequence,
UNICODE_PROP_ASCII_Hex_Digit,
UNICODE_PROP_Bidi_Control,
UNICODE_PROP_Dash,
@ -4633,6 +4721,11 @@ static const uint8_t * const unicode_prop_table[] = {
unicode_prop_Changes_When_Titlecased1_table,
unicode_prop_Changes_When_Casefolded1_table,
unicode_prop_Changes_When_NFKC_Casefolded1_table,
unicode_prop_Basic_Emoji1_table,
unicode_prop_Basic_Emoji2_table,
unicode_prop_RGI_Emoji_Modifier_Sequence_table,
unicode_prop_RGI_Emoji_Flag_Sequence_table,
unicode_prop_Emoji_Keycap_Sequence_table,
unicode_prop_ASCII_Hex_Digit_table,
unicode_prop_Bidi_Control_table,
unicode_prop_Dash_table,
@ -4688,6 +4781,11 @@ static const uint16_t unicode_prop_len_table[] = {
countof(unicode_prop_Changes_When_Titlecased1_table),
countof(unicode_prop_Changes_When_Casefolded1_table),
countof(unicode_prop_Changes_When_NFKC_Casefolded1_table),
countof(unicode_prop_Basic_Emoji1_table),
countof(unicode_prop_Basic_Emoji2_table),
countof(unicode_prop_RGI_Emoji_Modifier_Sequence_table),
countof(unicode_prop_RGI_Emoji_Flag_Sequence_table),
countof(unicode_prop_Emoji_Keycap_Sequence_table),
countof(unicode_prop_ASCII_Hex_Digit_table),
countof(unicode_prop_Bidi_Control_table),
countof(unicode_prop_Dash_table),
@ -4726,5 +4824,325 @@ static const uint16_t unicode_prop_len_table[] = {
countof(unicode_prop_Case_Ignorable_table),
};
typedef enum {
UNICODE_SEQUENCE_PROP_Basic_Emoji,
UNICODE_SEQUENCE_PROP_Emoji_Keycap_Sequence,
UNICODE_SEQUENCE_PROP_RGI_Emoji_Modifier_Sequence,
UNICODE_SEQUENCE_PROP_RGI_Emoji_Flag_Sequence,
UNICODE_SEQUENCE_PROP_RGI_Emoji_Tag_Sequence,
UNICODE_SEQUENCE_PROP_RGI_Emoji_ZWJ_Sequence,
UNICODE_SEQUENCE_PROP_RGI_Emoji,
UNICODE_SEQUENCE_PROP_COUNT,
} UnicodeSequencePropertyEnum;
static const char unicode_sequence_prop_name_table[] =
"Basic_Emoji" "\0"
"Emoji_Keycap_Sequence" "\0"
"RGI_Emoji_Modifier_Sequence" "\0"
"RGI_Emoji_Flag_Sequence" "\0"
"RGI_Emoji_Tag_Sequence" "\0"
"RGI_Emoji_ZWJ_Sequence" "\0"
"RGI_Emoji" "\0"
;
static const uint8_t unicode_rgi_emoji_tag_sequence[18] = {
0x67, 0x62, 0x65, 0x6e, 0x67, 0x00, 0x67, 0x62,
0x73, 0x63, 0x74, 0x00, 0x67, 0x62, 0x77, 0x6c,
0x73, 0x00,
};
static const uint8_t unicode_rgi_emoji_zwj_sequence[2320] = {
0x02, 0xb8, 0x19, 0x40, 0x86, 0x02, 0xd1, 0x39,
0xb0, 0x19, 0x02, 0x26, 0x39, 0x42, 0x86, 0x02,
0xb4, 0x36, 0x42, 0x86, 0x03, 0x68, 0x54, 0x64,
0x87, 0x68, 0x54, 0x02, 0xdc, 0x39, 0x42, 0x86,
0x02, 0xd1, 0x39, 0x73, 0x13, 0x02, 0x39, 0x39,
0x40, 0x86, 0x02, 0x69, 0x34, 0xbd, 0x19, 0x03,
0xb6, 0x36, 0x40, 0x86, 0xa1, 0x87, 0x03, 0x68,
0x74, 0x1d, 0x19, 0x68, 0x74, 0x03, 0x68, 0x34,
0xbd, 0x19, 0xa1, 0x87, 0x02, 0xf1, 0x7a, 0xf2,
0x7a, 0x02, 0xca, 0x33, 0x42, 0x86, 0x02, 0x69,
0x34, 0xb0, 0x19, 0x04, 0x68, 0x14, 0x68, 0x14,
0x67, 0x14, 0x66, 0x14, 0x02, 0xf9, 0x26, 0x42,
0x86, 0x03, 0x69, 0x74, 0x1d, 0x19, 0x69, 0x74,
0x03, 0xd1, 0x19, 0xbc, 0x19, 0xa1, 0x87, 0x02,
0x3c, 0x19, 0x40, 0x86, 0x02, 0x68, 0x34, 0xeb,
0x13, 0x02, 0xc3, 0x33, 0xa1, 0x87, 0x02, 0x70,
0x34, 0x40, 0x86, 0x02, 0xd4, 0x39, 0x42, 0x86,
0x02, 0xcf, 0x39, 0x42, 0x86, 0x02, 0x47, 0x36,
0x40, 0x86, 0x02, 0x39, 0x39, 0x42, 0x86, 0x04,
0xd1, 0x79, 0x64, 0x87, 0x8b, 0x14, 0xd1, 0x79,
0x02, 0xd1, 0x39, 0x95, 0x86, 0x02, 0x68, 0x34,
0x93, 0x13, 0x02, 0x69, 0x34, 0xed, 0x13, 0x02,
0xda, 0x39, 0x40, 0x86, 0x03, 0x69, 0x34, 0xaf,
0x19, 0xa1, 0x87, 0x02, 0xd1, 0x39, 0x93, 0x13,
0x03, 0xce, 0x39, 0x42, 0x86, 0xa1, 0x87, 0x03,
0xd1, 0x79, 0x64, 0x87, 0xd1, 0x79, 0x03, 0xc3,
0x33, 0x42, 0x86, 0xa1, 0x87, 0x03, 0x69, 0x74,
0x1d, 0x19, 0x68, 0x74, 0x02, 0x69, 0x34, 0x92,
0x16, 0x02, 0xd1, 0x39, 0x96, 0x86, 0x04, 0x69,
0x14, 0x64, 0x87, 0x8b, 0x14, 0x68, 0x14, 0x02,
0x68, 0x34, 0x7c, 0x13, 0x02, 0x47, 0x36, 0x42,
0x86, 0x02, 0x86, 0x34, 0x42, 0x86, 0x02, 0xd1,
0x39, 0x7c, 0x13, 0x02, 0x69, 0x14, 0xa4, 0x13,
0x02, 0xda, 0x39, 0x42, 0x86, 0x02, 0x37, 0x39,
0x40, 0x86, 0x02, 0xd1, 0x39, 0x08, 0x87, 0x04,
0x68, 0x54, 0x64, 0x87, 0x8b, 0x14, 0x68, 0x54,
0x02, 0x4d, 0x36, 0x40, 0x86, 0x02, 0x68, 0x34,
0x2c, 0x15, 0x02, 0x69, 0x34, 0xaf, 0x19, 0x02,
0x6e, 0x34, 0x40, 0x86, 0x02, 0xcd, 0x39, 0x42,
0x86, 0x02, 0xd1, 0x39, 0x2c, 0x15, 0x02, 0x6f,
0x14, 0x40, 0x86, 0x03, 0xd1, 0x39, 0xbc, 0x19,
0xa1, 0x87, 0x02, 0x68, 0x34, 0xa8, 0x13, 0x02,
0x69, 0x34, 0x73, 0x13, 0x04, 0x69, 0x54, 0x64,
0x87, 0x8b, 0x14, 0x68, 0x54, 0x02, 0x71, 0x34,
0x42, 0x86, 0x02, 0xd1, 0x39, 0xa8, 0x13, 0x02,
0x45, 0x36, 0x40, 0x86, 0x03, 0x69, 0x54, 0x64,
0x87, 0x68, 0x54, 0x03, 0x69, 0x54, 0x64, 0x87,
0x69, 0x54, 0x03, 0xce, 0x39, 0x40, 0x86, 0xa1,
0x87, 0x02, 0xd8, 0x39, 0x40, 0x86, 0x03, 0xc3,
0x33, 0x40, 0x86, 0xa1, 0x87, 0x02, 0x4d, 0x36,
0x42, 0x86, 0x02, 0xd1, 0x19, 0x92, 0x16, 0x02,
0xd1, 0x39, 0xeb, 0x13, 0x02, 0x68, 0x34, 0xbc,
0x14, 0x02, 0xd1, 0x39, 0xbc, 0x14, 0x02, 0x3d,
0x39, 0x40, 0x86, 0x02, 0xb8, 0x39, 0x42, 0x86,
0x02, 0xa3, 0x36, 0x40, 0x86, 0x02, 0x75, 0x35,
0x40, 0x86, 0x02, 0xd8, 0x39, 0x42, 0x86, 0x02,
0x69, 0x34, 0x93, 0x13, 0x02, 0x35, 0x39, 0x40,
0x86, 0x02, 0x4b, 0x36, 0x40, 0x86, 0x02, 0x3d,
0x39, 0x42, 0x86, 0x02, 0x38, 0x39, 0x42, 0x86,
0x02, 0xa3, 0x36, 0x42, 0x86, 0x03, 0x69, 0x14,
0x67, 0x14, 0x67, 0x14, 0x02, 0xb6, 0x36, 0x40,
0x86, 0x02, 0x69, 0x34, 0x7c, 0x13, 0x02, 0x75,
0x35, 0x42, 0x86, 0x02, 0xcc, 0x93, 0x40, 0x86,
0x02, 0xcc, 0x33, 0x40, 0x86, 0x03, 0xd1, 0x39,
0xbd, 0x19, 0xa1, 0x87, 0x02, 0x82, 0x34, 0x40,
0x86, 0x02, 0x87, 0x34, 0x40, 0x86, 0x02, 0x69,
0x14, 0x3e, 0x13, 0x02, 0xd6, 0x39, 0x40, 0x86,
0x02, 0x68, 0x14, 0xbd, 0x19, 0x02, 0x46, 0x36,
0x42, 0x86, 0x02, 0x4b, 0x36, 0x42, 0x86, 0x02,
0x69, 0x34, 0x2c, 0x15, 0x03, 0xb6, 0x36, 0x42,
0x86, 0xa1, 0x87, 0x02, 0xc4, 0x33, 0x40, 0x86,
0x02, 0x26, 0x19, 0x40, 0x86, 0x02, 0x69, 0x14,
0xb0, 0x19, 0x02, 0xde, 0x19, 0x42, 0x86, 0x02,
0x69, 0x34, 0xa8, 0x13, 0x02, 0xcc, 0x33, 0x42,
0x86, 0x02, 0x82, 0x34, 0x42, 0x86, 0x02, 0xd1,
0x19, 0x93, 0x13, 0x02, 0x81, 0x14, 0x42, 0x86,
0x02, 0x69, 0x34, 0x95, 0x86, 0x02, 0x68, 0x34,
0xbb, 0x14, 0x02, 0xd1, 0x39, 0xbb, 0x14, 0x02,
0x69, 0x34, 0xeb, 0x13, 0x02, 0xd1, 0x39, 0x84,
0x13, 0x02, 0x69, 0x34, 0xbc, 0x14, 0x04, 0x69,
0x54, 0x64, 0x87, 0x8b, 0x14, 0x69, 0x54, 0x02,
0x26, 0x39, 0x40, 0x86, 0x02, 0xb4, 0x36, 0x40,
0x86, 0x02, 0x47, 0x16, 0x42, 0x86, 0x02, 0xdc,
0x39, 0x40, 0x86, 0x02, 0xca, 0x33, 0x40, 0x86,
0x02, 0xf9, 0x26, 0x40, 0x86, 0x02, 0x69, 0x34,
0x08, 0x87, 0x03, 0x69, 0x14, 0x69, 0x14, 0x66,
0x14, 0x03, 0xd1, 0x59, 0x1d, 0x19, 0xd1, 0x59,
0x02, 0xd4, 0x39, 0x40, 0x86, 0x02, 0xcf, 0x39,
0x40, 0x86, 0x02, 0x68, 0x34, 0xa4, 0x13, 0x02,
0xd1, 0x39, 0xa4, 0x13, 0x02, 0xd1, 0x19, 0xa8,
0x13, 0x02, 0xd7, 0x39, 0x42, 0x86, 0x03, 0x69,
0x34, 0xbc, 0x19, 0xa1, 0x87, 0x02, 0x68, 0x14,
0xb0, 0x19, 0x02, 0x68, 0x14, 0x73, 0x13, 0x04,
0x69, 0x14, 0x69, 0x14, 0x66, 0x14, 0x66, 0x14,
0x03, 0x68, 0x34, 0xaf, 0x19, 0xa1, 0x87, 0x02,
0x68, 0x34, 0x80, 0x16, 0x02, 0x73, 0x34, 0x42,
0x86, 0x02, 0xd1, 0x39, 0x80, 0x16, 0x02, 0x68,
0x34, 0xb0, 0x19, 0x02, 0x86, 0x34, 0x40, 0x86,
0x02, 0x38, 0x19, 0x42, 0x86, 0x02, 0x69, 0x34,
0xbb, 0x14, 0x02, 0xb5, 0x36, 0x42, 0x86, 0x02,
0xcd, 0x39, 0x40, 0x86, 0x02, 0x68, 0x34, 0x95,
0x86, 0x02, 0x68, 0x34, 0x27, 0x15, 0x03, 0x68,
0x14, 0x68, 0x14, 0x66, 0x14, 0x02, 0x71, 0x34,
0x40, 0x86, 0x02, 0xd1, 0x39, 0x27, 0x15, 0x02,
0x2e, 0x16, 0xa8, 0x14, 0x02, 0xc3, 0x33, 0x42,
0x86, 0x02, 0x69, 0x14, 0x66, 0x14, 0x02, 0x68,
0x34, 0x96, 0x86, 0x02, 0x69, 0x34, 0xa4, 0x13,
0x03, 0x69, 0x14, 0x64, 0x87, 0x68, 0x14, 0x02,
0xb8, 0x39, 0x40, 0x86, 0x02, 0x68, 0x34, 0x3e,
0x13, 0x03, 0xd1, 0x19, 0xaf, 0x19, 0xa1, 0x87,
0x02, 0xd1, 0x39, 0x3e, 0x13, 0x02, 0x68, 0x34,
0xbd, 0x19, 0x02, 0xd1, 0x19, 0xbb, 0x14, 0x02,
0xd1, 0x19, 0x95, 0x86, 0x02, 0xdb, 0x39, 0x42,
0x86, 0x02, 0x38, 0x39, 0x40, 0x86, 0x02, 0x69,
0x34, 0x80, 0x16, 0x02, 0x69, 0x14, 0xeb, 0x13,
0x04, 0x68, 0x14, 0x69, 0x14, 0x67, 0x14, 0x67,
0x14, 0x02, 0x77, 0x34, 0x42, 0x86, 0x02, 0x46,
0x36, 0x40, 0x86, 0x02, 0x68, 0x34, 0x92, 0x16,
0x02, 0x4e, 0x36, 0x42, 0x86, 0x03, 0x69, 0x14,
0xbd, 0x19, 0xa1, 0x87, 0x02, 0xde, 0x19, 0x40,
0x86, 0x02, 0x69, 0x34, 0x27, 0x15, 0x03, 0xc3,
0x13, 0x40, 0x86, 0xa1, 0x87, 0x02, 0x81, 0x14,
0x40, 0x86, 0x03, 0xd1, 0x39, 0xaf, 0x19, 0xa1,
0x87, 0x02, 0x68, 0x34, 0xbc, 0x19, 0x02, 0xd1,
0x19, 0x80, 0x16, 0x02, 0xd9, 0x39, 0x42, 0x86,
0x02, 0xd1, 0x39, 0xbc, 0x19, 0x02, 0xdc, 0x19,
0x42, 0x86, 0x02, 0x68, 0x34, 0x73, 0x13, 0x02,
0x69, 0x34, 0x3e, 0x13, 0x02, 0x47, 0x16, 0x40,
0x86, 0x02, 0xd1, 0x39, 0xbd, 0x19, 0x02, 0x3e,
0x39, 0x42, 0x86, 0x02, 0x69, 0x14, 0x95, 0x86,
0x02, 0x68, 0x14, 0x96, 0x86, 0x03, 0x69, 0x34,
0xbd, 0x19, 0xa1, 0x87, 0x02, 0xd7, 0x39, 0x40,
0x86, 0x02, 0x45, 0x16, 0x42, 0x86, 0x02, 0x68,
0x34, 0xed, 0x13, 0x03, 0x68, 0x34, 0xbc, 0x19,
0xa1, 0x87, 0x02, 0xd1, 0x39, 0xed, 0x13, 0x02,
0xd1, 0x39, 0x92, 0x16, 0x02, 0x73, 0x34, 0x40,
0x86, 0x02, 0x38, 0x19, 0x40, 0x86, 0x02, 0xb5,
0x36, 0x40, 0x86, 0x02, 0x68, 0x34, 0xaf, 0x19,
0x02, 0xd1, 0x39, 0xaf, 0x19, 0x02, 0x69, 0x34,
0xbc, 0x19, 0x02, 0xb6, 0x16, 0x42, 0x86, 0x02,
0x26, 0x14, 0x25, 0x15, 0x02, 0xc3, 0x33, 0x40,
0x86, 0x02, 0xdd, 0x39, 0x42, 0x86, 0x02, 0xcb,
0x93, 0x42, 0x86, 0x02, 0xcb, 0x33, 0x42, 0x86,
0x02, 0x81, 0x34, 0x42, 0x86, 0x02, 0xce, 0x39,
0xa1, 0x87, 0x02, 0xdb, 0x39, 0x40, 0x86, 0x02,
0x68, 0x34, 0x08, 0x87, 0x02, 0xd1, 0x19, 0xb0,
0x19, 0x02, 0x77, 0x34, 0x40, 0x86, 0x02, 0x4e,
0x36, 0x40, 0x86, 0x02, 0xce, 0x39, 0x42, 0x86,
0x02, 0x4e, 0x16, 0x42, 0x86, 0x02, 0xd9, 0x39,
0x40, 0x86, 0x02, 0xdc, 0x19, 0x40, 0x86, 0x02,
0x3e, 0x39, 0x40, 0x86, 0x02, 0xb9, 0x39, 0x42,
0x86, 0x02, 0xda, 0x19, 0x42, 0x86, 0x02, 0x42,
0x16, 0x94, 0x81, 0x02, 0x45, 0x16, 0x40, 0x86,
0x02, 0x69, 0x14, 0xbd, 0x19, 0x02, 0x70, 0x34,
0x42, 0x86, 0x02, 0xce, 0x19, 0xa1, 0x87, 0x02,
0xc3, 0x13, 0x42, 0x86, 0x02, 0x68, 0x14, 0x08,
0x87, 0x02, 0xd1, 0x19, 0x7c, 0x13, 0x02, 0x68,
0x14, 0x92, 0x16, 0x02, 0xb6, 0x16, 0x40, 0x86,
0x02, 0x37, 0x39, 0x42, 0x86, 0x03, 0xce, 0x19,
0x42, 0x86, 0xa1, 0x87, 0x03, 0x68, 0x14, 0x67,
0x14, 0x67, 0x14, 0x02, 0xdd, 0x39, 0x40, 0x86,
0x02, 0xcf, 0x19, 0x42, 0x86, 0x02, 0xd1, 0x19,
0x2c, 0x15, 0x02, 0x4b, 0x13, 0xe9, 0x17, 0x02,
0x68, 0x14, 0x67, 0x14, 0x02, 0xcb, 0x93, 0x40,
0x86, 0x02, 0x6e, 0x34, 0x42, 0x86, 0x02, 0xcb,
0x33, 0x40, 0x86, 0x02, 0x81, 0x34, 0x40, 0x86,
0x02, 0xb6, 0x36, 0xa1, 0x87, 0x02, 0x45, 0x36,
0x42, 0x86, 0x02, 0xb4, 0x16, 0x42, 0x86, 0x02,
0x69, 0x14, 0x73, 0x13, 0x04, 0x69, 0x14, 0x69,
0x14, 0x67, 0x14, 0x66, 0x14, 0x02, 0x35, 0x39,
0x42, 0x86, 0x02, 0x68, 0x14, 0x93, 0x13, 0x02,
0xb6, 0x36, 0x42, 0x86, 0x03, 0x68, 0x14, 0x69,
0x14, 0x66, 0x14, 0x02, 0xce, 0x39, 0x40, 0x86,
0x02, 0x4e, 0x16, 0x40, 0x86, 0x02, 0x87, 0x34,
0x42, 0x86, 0x02, 0x86, 0x14, 0x42, 0x86, 0x02,
0xd6, 0x39, 0x42, 0x86, 0x02, 0xc4, 0x33, 0x42,
0x86, 0x02, 0x69, 0x34, 0x96, 0x86, 0x02, 0xb9,
0x39, 0x40, 0x86, 0x02, 0x68, 0x14, 0xa8, 0x13,
0x02, 0xd1, 0x19, 0x84, 0x13, 0x02, 0xda, 0x19,
0x40, 0x86, 0x02, 0xd8, 0x19, 0x42, 0x86, 0x02,
0xc3, 0x13, 0x40, 0x86, 0x02, 0xb9, 0x19, 0x42,
0x86, 0x02, 0x3d, 0x19, 0x42, 0x86, 0x02, 0xcf,
0x19, 0x40, 0x86, 0x04, 0x68, 0x14, 0x68, 0x14,
0x67, 0x14, 0x67, 0x14, 0x03, 0xd1, 0x19, 0xd1,
0x19, 0xd2, 0x19, 0x02, 0x68, 0x14, 0xbb, 0x14,
0x02, 0x3b, 0x14, 0x44, 0x87, 0x02, 0xd1, 0x19,
0x27, 0x15, 0x02, 0xb4, 0x16, 0x40, 0x86, 0x02,
0xcd, 0x19, 0x42, 0x86, 0x02, 0xd3, 0x86, 0xa5,
0x14, 0x02, 0x70, 0x14, 0x42, 0x86, 0x03, 0xb6,
0x16, 0x42, 0x86, 0xa1, 0x87, 0x04, 0x69, 0x14,
0x64, 0x87, 0x8b, 0x14, 0x69, 0x14, 0x02, 0x36,
0x16, 0x2b, 0x93, 0x02, 0x68, 0x14, 0x80, 0x16,
0x02, 0x86, 0x14, 0x40, 0x86, 0x02, 0x08, 0x14,
0x1b, 0x0b, 0x02, 0xd1, 0x19, 0xbc, 0x19, 0x02,
0xca, 0x13, 0x42, 0x86, 0x02, 0x41, 0x94, 0xe8,
0x95, 0x02, 0xd8, 0x19, 0x40, 0x86, 0x02, 0xb9,
0x19, 0x40, 0x86, 0x02, 0xd1, 0x19, 0xed, 0x13,
0x02, 0xf9, 0x86, 0x42, 0x86, 0x03, 0xd1, 0x19,
0xbd, 0x19, 0xa1, 0x87, 0x02, 0x3d, 0x19, 0x40,
0x86, 0x02, 0xd6, 0x19, 0x42, 0x86, 0x03, 0x69,
0x14, 0x66, 0x14, 0x66, 0x14, 0x02, 0xd1, 0x19,
0xaf, 0x19, 0x03, 0x69, 0x14, 0x69, 0x14, 0x67,
0x14, 0x02, 0xcd, 0x19, 0x40, 0x86, 0x02, 0x70,
0x14, 0x40, 0x86, 0x03, 0x68, 0x14, 0xbc, 0x19,
0xa1, 0x87, 0x02, 0x6e, 0x14, 0x42, 0x86, 0x02,
0x69, 0x14, 0x92, 0x16, 0x03, 0x68, 0x14, 0x68,
0x14, 0x67, 0x14, 0x02, 0x69, 0x14, 0x67, 0x14,
0x02, 0x75, 0x95, 0x42, 0x86, 0x03, 0x69, 0x14,
0x64, 0x87, 0x69, 0x14, 0x02, 0xd1, 0x19, 0xbc,
0x14, 0x02, 0xdf, 0x19, 0x42, 0x86, 0x02, 0xca,
0x13, 0x40, 0x86, 0x02, 0x82, 0x14, 0x42, 0x86,
0x02, 0x69, 0x14, 0x93, 0x13, 0x02, 0x68, 0x14,
0x7c, 0x13, 0x02, 0xf9, 0x86, 0x40, 0x86, 0x02,
0xd6, 0x19, 0x40, 0x86, 0x02, 0x68, 0x14, 0x2c,
0x15, 0x02, 0x69, 0x14, 0xa8, 0x13, 0x02, 0xd4,
0x19, 0x42, 0x86, 0x04, 0x68, 0x14, 0x69, 0x14,
0x66, 0x14, 0x66, 0x14, 0x02, 0x77, 0x14, 0x42,
0x86, 0x02, 0x39, 0x19, 0x42, 0x86, 0x02, 0xd1,
0x19, 0xa4, 0x13, 0x02, 0x6e, 0x14, 0x40, 0x86,
0x03, 0xd1, 0x19, 0xd2, 0x19, 0xd2, 0x19, 0x02,
0x69, 0x14, 0xbb, 0x14, 0x02, 0xd1, 0x19, 0x96,
0x86, 0x02, 0x75, 0x95, 0x40, 0x86, 0x04, 0x68,
0x14, 0x64, 0x87, 0x8b, 0x14, 0x68, 0x14, 0x02,
0xd1, 0x19, 0x3e, 0x13, 0x02, 0xdf, 0x19, 0x40,
0x86, 0x02, 0x82, 0x14, 0x40, 0x86, 0x02, 0x44,
0x13, 0xeb, 0x17, 0x02, 0xdd, 0x19, 0x42, 0x86,
0x02, 0x69, 0x14, 0x80, 0x16, 0x03, 0x68, 0x14,
0xaf, 0x19, 0xa1, 0x87, 0x02, 0xa3, 0x16, 0x42,
0x86, 0x02, 0x69, 0x14, 0x96, 0x86, 0x02, 0x46,
0x16, 0x42, 0x86, 0x02, 0xb6, 0x16, 0xa1, 0x87,
0x02, 0x68, 0x14, 0x27, 0x15, 0x02, 0x26, 0x14,
0x1b, 0x0b, 0x02, 0xd4, 0x19, 0x40, 0x86, 0x02,
0x77, 0x14, 0x40, 0x86, 0x02, 0x39, 0x19, 0x40,
0x86, 0x02, 0x37, 0x19, 0x42, 0x86, 0x03, 0x69,
0x14, 0x67, 0x14, 0x66, 0x14, 0x03, 0xc3, 0x13,
0x42, 0x86, 0xa1, 0x87, 0x02, 0x68, 0x14, 0xbc,
0x19, 0x02, 0xd1, 0x19, 0xeb, 0x13, 0x04, 0x69,
0x14, 0x69, 0x14, 0x67, 0x14, 0x67, 0x14, 0x02,
0xd1, 0x19, 0x08, 0x87, 0x02, 0x68, 0x14, 0xed,
0x13, 0x03, 0x69, 0x14, 0xbc, 0x19, 0xa1, 0x87,
0x02, 0xdd, 0x19, 0x40, 0x86, 0x02, 0xc3, 0x13,
0xa1, 0x87, 0x03, 0x68, 0x14, 0x66, 0x14, 0x66,
0x14, 0x03, 0x68, 0x14, 0x69, 0x14, 0x67, 0x14,
0x02, 0xa3, 0x16, 0x40, 0x86, 0x02, 0xdb, 0x19,
0x42, 0x86, 0x02, 0x68, 0x14, 0xaf, 0x19, 0x02,
0x46, 0x16, 0x40, 0x86, 0x02, 0x35, 0x16, 0xab,
0x14, 0x02, 0x68, 0x14, 0x95, 0x86, 0x02, 0x42,
0x16, 0x95, 0x81, 0x02, 0xc4, 0x13, 0x42, 0x86,
0x02, 0x15, 0x14, 0xba, 0x19, 0x02, 0x69, 0x14,
0x08, 0x87, 0x03, 0xd1, 0x19, 0x1d, 0x19, 0xd1,
0x19, 0x02, 0x69, 0x14, 0x7c, 0x13, 0x02, 0x37,
0x19, 0x40, 0x86, 0x02, 0x73, 0x14, 0x42, 0x86,
0x02, 0x69, 0x14, 0x2c, 0x15, 0x02, 0xb5, 0x16,
0x42, 0x86, 0x02, 0x35, 0x19, 0x42, 0x86, 0x04,
0x68, 0x14, 0x69, 0x14, 0x67, 0x14, 0x66, 0x14,
0x02, 0x64, 0x87, 0x25, 0x15, 0x02, 0x64, 0x87,
0x79, 0x1a, 0x02, 0x68, 0x14, 0xbc, 0x14, 0x03,
0xce, 0x19, 0x40, 0x86, 0xa1, 0x87, 0x02, 0x87,
0x14, 0x42, 0x86, 0x02, 0x4d, 0x16, 0x42, 0x86,
0x04, 0x68, 0x14, 0x68, 0x14, 0x66, 0x14, 0x66,
0x14, 0x02, 0xdb, 0x19, 0x40, 0x86, 0x02, 0xd9,
0x19, 0x42, 0x86, 0x02, 0xc4, 0x13, 0x40, 0x86,
0x02, 0xd1, 0x19, 0xbd, 0x19, 0x02, 0x68, 0x14,
0xa4, 0x13, 0x02, 0x3e, 0x19, 0x42, 0x86, 0x02,
0xf3, 0x93, 0xa7, 0x86, 0x03, 0x69, 0x14, 0xaf,
0x19, 0xa1, 0x87, 0x02, 0xf3, 0x93, 0x08, 0x13,
0x02, 0xd1, 0x19, 0xd2, 0x19, 0x02, 0x73, 0x14,
0x40, 0x86, 0x02, 0xb5, 0x16, 0x40, 0x86, 0x02,
0x35, 0x19, 0x40, 0x86, 0x02, 0x69, 0x14, 0x27,
0x15, 0x02, 0xce, 0x19, 0x42, 0x86, 0x02, 0x71,
0x14, 0x42, 0x86, 0x02, 0xd1, 0x19, 0x73, 0x13,
0x02, 0x68, 0x14, 0x3e, 0x13, 0x02, 0xf4, 0x13,
0x20, 0x86, 0x02, 0x87, 0x14, 0x40, 0x86, 0x03,
0xb6, 0x16, 0x40, 0x86, 0xa1, 0x87, 0x02, 0x4d,
0x16, 0x40, 0x86, 0x02, 0x69, 0x14, 0xbc, 0x19,
0x02, 0x4b, 0x16, 0x42, 0x86, 0x02, 0xd9, 0x19,
0x40, 0x86, 0x02, 0x3e, 0x19, 0x40, 0x86, 0x02,
0x69, 0x14, 0xed, 0x13, 0x02, 0xd7, 0x19, 0x42,
0x86, 0x02, 0xb8, 0x19, 0x42, 0x86, 0x03, 0x68,
0x14, 0x67, 0x14, 0x66, 0x14, 0x02, 0x3c, 0x19,
0x42, 0x86, 0x02, 0x68, 0x14, 0x66, 0x14, 0x03,
0x68, 0x14, 0x64, 0x87, 0x68, 0x14, 0x02, 0x69,
0x14, 0xaf, 0x19, 0x02, 0xce, 0x19, 0x40, 0x86,
0x02, 0x71, 0x14, 0x40, 0x86, 0x02, 0x68, 0x14,
0xeb, 0x13, 0x03, 0x68, 0x14, 0xbd, 0x19, 0xa1,
0x87, 0x02, 0x6f, 0x14, 0x42, 0x86, 0x04, 0xd1,
0x19, 0xd1, 0x19, 0xd2, 0x19, 0xd2, 0x19, 0x02,
0x69, 0x14, 0xbc, 0x14, 0x02, 0xcc, 0x93, 0x42,
0x86, 0x02, 0x4b, 0x16, 0x40, 0x86, 0x02, 0x26,
0x19, 0x42, 0x86, 0x02, 0xd7, 0x19, 0x40, 0x86,
};
#endif /* CONFIG_ALL_UNICODE */
/* 64 tables / 33442 bytes, 5 index / 351 bytes */
/* 71 tables / 36311 bytes, 5 index / 351 bytes */

215
libunicode.c
View File

@ -499,6 +499,9 @@ int cr_op(CharRange *cr, const uint32_t *a_pt, int a_len,
case CR_OP_XOR:
is_in = (a_idx & 1) ^ (b_idx & 1);
break;
case CR_OP_SUB:
is_in = (a_idx & 1) & ((b_idx & 1) ^ 1);
break;
default:
abort();
}
@ -511,14 +514,14 @@ int cr_op(CharRange *cr, const uint32_t *a_pt, int a_len,
return 0;
}
int cr_union1(CharRange *cr, const uint32_t *b_pt, int b_len)
int cr_op1(CharRange *cr, const uint32_t *b_pt, int b_len, int op)
{
CharRange a = *cr;
int ret;
cr->len = 0;
cr->size = 0;
cr->points = NULL;
ret = cr_op(cr, a.points, a.len, b_pt, b_len, CR_OP_UNION);
ret = cr_op(cr, a.points, a.len, b_pt, b_len, op);
cr_free(&a);
return ret;
}
@ -1554,6 +1557,7 @@ static int unicode_prop_ops(CharRange *cr, ...)
cr2 = &stack[stack_len - 1];
cr3 = &stack[stack_len++];
cr_init(cr3, cr->mem_opaque, cr->realloc_func);
/* CR_OP_XOR may be used here */
if (cr_op(cr3, cr1->points, cr1->len,
cr2->points, cr2->len, op - POP_UNION + CR_OP_UNION))
goto fail;
@ -1908,3 +1912,210 @@ BOOL lre_is_space_non_ascii(uint32_t c)
}
return FALSE;
}
#define SEQ_MAX_LEN 16
static int unicode_sequence_prop1(int seq_prop_idx, UnicodeSequencePropCB *cb, void *opaque,
CharRange *cr)
{
int i, c, j;
uint32_t seq[SEQ_MAX_LEN];
switch(seq_prop_idx) {
case UNICODE_SEQUENCE_PROP_Basic_Emoji:
if (unicode_prop1(cr, UNICODE_PROP_Basic_Emoji1) < 0)
return -1;
for(i = 0; i < cr->len; i += 2) {
for(c = cr->points[i]; c < cr->points[i + 1]; c++) {
seq[0] = c;
cb(opaque, seq, 1);
}
}
cr->len = 0;
if (unicode_prop1(cr, UNICODE_PROP_Basic_Emoji2) < 0)
return -1;
for(i = 0; i < cr->len; i += 2) {
for(c = cr->points[i]; c < cr->points[i + 1]; c++) {
seq[0] = c;
seq[1] = 0xfe0f;
cb(opaque, seq, 2);
}
}
break;
case UNICODE_SEQUENCE_PROP_RGI_Emoji_Modifier_Sequence:
if (unicode_prop1(cr, UNICODE_PROP_Emoji_Modifier_Base) < 0)
return -1;
for(i = 0; i < cr->len; i += 2) {
for(c = cr->points[i]; c < cr->points[i + 1]; c++) {
for(j = 0; j < 5; j++) {
seq[0] = c;
seq[1] = 0x1f3fb + j;
cb(opaque, seq, 2);
}
}
}
break;
case UNICODE_SEQUENCE_PROP_RGI_Emoji_Flag_Sequence:
if (unicode_prop1(cr, UNICODE_PROP_RGI_Emoji_Flag_Sequence) < 0)
return -1;
for(i = 0; i < cr->len; i += 2) {
for(c = cr->points[i]; c < cr->points[i + 1]; c++) {
int c0, c1;
c0 = c / 26;
c1 = c % 26;
seq[0] = 0x1F1E6 + c0;
seq[1] = 0x1F1E6 + c1;
cb(opaque, seq, 2);
}
}
break;
case UNICODE_SEQUENCE_PROP_RGI_Emoji_ZWJ_Sequence:
{
int len, code, pres, k, mod, mod_count, mod_pos[2], hc_pos, n_mod, n_hc, mod1;
int mod_idx, hc_idx, i0, i1;
const uint8_t *tab = unicode_rgi_emoji_zwj_sequence;
for(i = 0; i < countof(unicode_rgi_emoji_zwj_sequence);) {
len = tab[i++];
k = 0;
mod = 0;
mod_count = 0;
hc_pos = -1;
for(j = 0; j < len; j++) {
code = tab[i++];
code |= tab[i++] << 8;
pres = code >> 15;
mod1 = (code >> 13) & 3;
code &= 0x1fff;
if (code < 0x1000) {
c = code + 0x2000;
} else {
c = 0x1f000 + (code - 0x1000);
}
if (c == 0x1f9b0)
hc_pos = k;
seq[k++] = c;
if (mod1 != 0) {
assert(mod_count < 2);
mod = mod1;
mod_pos[mod_count++] = k;
seq[k++] = 0; /* will be filled later */
}
if (pres) {
seq[k++] = 0xfe0f;
}
if (j < len - 1) {
seq[k++] = 0x200d;
}
}
/* genrate all the variants */
switch(mod) {
case 1:
n_mod = 5;
break;
case 2:
n_mod = 25;
break;
case 3:
n_mod = 20;
break;
default:
n_mod = 1;
break;
}
if (hc_pos >= 0)
n_hc = 4;
else
n_hc = 1;
for(hc_idx = 0; hc_idx < n_hc; hc_idx++) {
for(mod_idx = 0; mod_idx < n_mod; mod_idx++) {
if (hc_pos >= 0)
seq[hc_pos] = 0x1f9b0 + hc_idx;
switch(mod) {
case 1:
seq[mod_pos[0]] = 0x1f3fb + mod_idx;
break;
case 2:
case 3:
i0 = mod_idx / 5;
i1 = mod_idx % 5;
/* avoid identical values */
if (mod == 3 && i0 >= i1)
i0++;
seq[mod_pos[0]] = 0x1f3fb + i0;
seq[mod_pos[1]] = 0x1f3fb + i1;
break;
default:
break;
}
#if 0
for(j = 0; j < k; j++)
printf(" %04x", seq[j]);
printf("\n");
#endif
cb(opaque, seq, k);
}
}
}
}
break;
case UNICODE_SEQUENCE_PROP_RGI_Emoji_Tag_Sequence:
{
for(i = 0; i < countof(unicode_rgi_emoji_tag_sequence);) {
j = 0;
seq[j++] = 0x1F3F4;
for(;;) {
c = unicode_rgi_emoji_tag_sequence[i++];
if (c == 0x00)
break;
seq[j++] = 0xe0000 + c;
}
seq[j++] = 0xe007f;
cb(opaque, seq, j);
}
}
break;
case UNICODE_SEQUENCE_PROP_Emoji_Keycap_Sequence:
if (unicode_prop1(cr, UNICODE_PROP_Emoji_Keycap_Sequence) < 0)
return -1;
for(i = 0; i < cr->len; i += 2) {
for(c = cr->points[i]; c < cr->points[i + 1]; c++) {
seq[0] = c;
seq[1] = 0xfe0f;
seq[2] = 0x20e3;
cb(opaque, seq, 3);
}
}
break;
case UNICODE_SEQUENCE_PROP_RGI_Emoji:
/* all prevous sequences */
for(i = UNICODE_SEQUENCE_PROP_Basic_Emoji; i <= UNICODE_SEQUENCE_PROP_RGI_Emoji_ZWJ_Sequence; i++) {
int ret;
ret = unicode_sequence_prop1(i, cb, opaque, cr);
if (ret < 0)
return ret;
cr->len = 0;
}
break;
default:
return -2;
}
return 0;
}
/* build a unicode sequence property */
/* return -2 if not found, -1 if other error. 'cr' is used as temporary memory. */
int unicode_sequence_prop(const char *prop_name, UnicodeSequencePropCB *cb, void *opaque,
CharRange *cr)
{
int seq_prop_idx;
seq_prop_idx = unicode_find_name(unicode_sequence_prop_name_table, prop_name);
if (seq_prop_idx < 0)
return -2;
return unicode_sequence_prop1(seq_prop_idx, cb, opaque, cr);
}

14
libunicode.h
View File

@ -45,6 +45,7 @@ typedef enum {
CR_OP_UNION,
CR_OP_INTER,
CR_OP_XOR,
CR_OP_SUB,
} CharRangeOpEnum;
void cr_init(CharRange *cr, void *mem_opaque, void *(*realloc_func)(void *opaque, void *ptr, size_t size));
@ -73,19 +74,18 @@ static inline int cr_add_interval(CharRange *cr, uint32_t c1, uint32_t c2)
return 0;
}
int cr_union1(CharRange *cr, const uint32_t *b_pt, int b_len);
int cr_op(CharRange *cr, const uint32_t *a_pt, int a_len,
const uint32_t *b_pt, int b_len, int op);
int cr_op1(CharRange *cr, const uint32_t *b_pt, int b_len, int op);
static inline int cr_union_interval(CharRange *cr, uint32_t c1, uint32_t c2)
{
uint32_t b_pt[2];
b_pt[0] = c1;
b_pt[1] = c2 + 1;
return cr_union1(cr, b_pt, 2);
return cr_op1(cr, b_pt, 2, CR_OP_UNION);
}
int cr_op(CharRange *cr, const uint32_t *a_pt, int a_len,
const uint32_t *b_pt, int b_len, int op);
int cr_invert(CharRange *cr);
int cr_regexp_canonicalize(CharRange *cr, int is_unicode);
@ -107,6 +107,10 @@ int unicode_script(CharRange *cr, const char *script_name, int is_ext);
int unicode_general_category(CharRange *cr, const char *gc_name);
int unicode_prop(CharRange *cr, const char *prop_name);
typedef void UnicodeSequencePropCB(void *opaque, const uint32_t *buf, int len);
int unicode_sequence_prop(const char *prop_name, UnicodeSequencePropCB *cb, void *opaque,
CharRange *cr);
int lre_case_conv(uint32_t *res, uint32_t c, int conv_type);
int lre_canonicalize(uint32_t c, int is_unicode);

6
quickjs-atom.h
View File

@ -177,6 +177,12 @@ DEF(minus_zero, "-0")
DEF(Infinity, "Infinity")
DEF(minus_Infinity, "-Infinity")
DEF(NaN, "NaN")
DEF(hasIndices, "hasIndices")
DEF(ignoreCase, "ignoreCase")
DEF(multiline, "multiline")
DEF(dotAll, "dotAll")
DEF(sticky, "sticky")
DEF(unicodeSets, "unicodeSets")
/* the following 3 atoms are only used with CONFIG_ATOMICS */
DEF(not_equal, "not-equal")
DEF(timed_out, "timed-out")

85
quickjs.c
View File

@ -44179,6 +44179,9 @@ static JSValue js_compile_regexp(JSContext *ctx, JSValueConst pattern,
case 'u':
mask = LRE_FLAG_UNICODE;
break;
case 'v':
mask = LRE_FLAG_UNICODE_SETS;
break;
case 'y':
mask = LRE_FLAG_STICKY;
break;
@ -44188,14 +44191,20 @@ static JSValue js_compile_regexp(JSContext *ctx, JSValueConst pattern,
if ((re_flags & mask) != 0) {
bad_flags:
JS_FreeCString(ctx, str);
return JS_ThrowSyntaxError(ctx, "invalid regular expression flags");
goto bad_flags1;
}
re_flags |= mask;
}
JS_FreeCString(ctx, str);
}
str = JS_ToCStringLen2(ctx, &len, pattern, !(re_flags & LRE_FLAG_UNICODE));
/* 'u' and 'v' cannot be both set */
if ((re_flags & LRE_FLAG_UNICODE_SETS) && (re_flags & LRE_FLAG_UNICODE)) {
bad_flags1:
return JS_ThrowSyntaxError(ctx, "invalid regular expression flags");
}
str = JS_ToCStringLen2(ctx, &len, pattern, !(re_flags & (LRE_FLAG_UNICODE | LRE_FLAG_UNICODE_SETS)));
if (!str)
return JS_EXCEPTION;
re_bytecode_buf = lre_compile(&re_bytecode_len, error_msg,
@ -44499,49 +44508,34 @@ static JSValue js_regexp_get_flag(JSContext *ctx, JSValueConst this_val, int mas
return JS_NewBool(ctx, flags & mask);
}
#define RE_FLAG_COUNT 8
static JSValue js_regexp_get_flags(JSContext *ctx, JSValueConst this_val)
{
char str[8], *p = str;
int res;
char str[RE_FLAG_COUNT], *p = str;
int res, i;
static const int flag_atom[RE_FLAG_COUNT] = {
JS_ATOM_hasIndices,
JS_ATOM_global,
JS_ATOM_ignoreCase,
JS_ATOM_multiline,
JS_ATOM_dotAll,
JS_ATOM_unicode,
JS_ATOM_unicodeSets,
JS_ATOM_sticky,
};
static const char flag_char[RE_FLAG_COUNT] = { 'd', 'g', 'i', 'm', 's', 'u', 'v', 'y' };
if (JS_VALUE_GET_TAG(this_val) != JS_TAG_OBJECT)
return JS_ThrowTypeErrorNotAnObject(ctx);
res = JS_ToBoolFree(ctx, JS_GetPropertyStr(ctx, this_val, "hasIndices"));
for(i = 0; i < RE_FLAG_COUNT; i++) {
res = JS_ToBoolFree(ctx, JS_GetProperty(ctx, this_val, flag_atom[i]));
if (res < 0)
goto exception;
if (res)
*p++ = 'd';
res = JS_ToBoolFree(ctx, JS_GetProperty(ctx, this_val, JS_ATOM_global));
if (res < 0)
goto exception;
if (res)
*p++ = 'g';
res = JS_ToBoolFree(ctx, JS_GetPropertyStr(ctx, this_val, "ignoreCase"));
if (res < 0)
goto exception;
if (res)
*p++ = 'i';
res = JS_ToBoolFree(ctx, JS_GetPropertyStr(ctx, this_val, "multiline"));
if (res < 0)
goto exception;
if (res)
*p++ = 'm';
res = JS_ToBoolFree(ctx, JS_GetPropertyStr(ctx, this_val, "dotAll"));
if (res < 0)
goto exception;
if (res)
*p++ = 's';
res = JS_ToBoolFree(ctx, JS_GetProperty(ctx, this_val, JS_ATOM_unicode));
if (res < 0)
goto exception;
if (res)
*p++ = 'u';
res = JS_ToBoolFree(ctx, JS_GetPropertyStr(ctx, this_val, "sticky"));
if (res < 0)
goto exception;
if (res)
*p++ = 'y';
*p++ = flag_char[i];
}
return JS_NewStringLen(ctx, str, p - str);
exception:
@ -45026,14 +45020,12 @@ static JSValue js_regexp_Symbol_match(JSContext *ctx, JSValueConst this_val,
goto exception;
p = JS_VALUE_GET_STRING(flags);
// TODO(bnoordhuis) query 'u' flag the same way?
global = (-1 != string_indexof_char(p, 'g', 0));
if (!global) {
A = JS_RegExpExec(ctx, rx, S);
} else {
fullUnicode = JS_ToBoolFree(ctx, JS_GetProperty(ctx, rx, JS_ATOM_unicode));
if (fullUnicode < 0)
goto exception;
fullUnicode = (string_indexof_char(p, 'u', 0) >= 0 ||
string_indexof_char(p, 'v', 0) >= 0);
if (JS_SetProperty(ctx, rx, JS_ATOM_lastIndex, JS_NewInt32(ctx, 0)) < 0)
goto exception;
@ -45217,7 +45209,8 @@ static JSValue js_regexp_Symbol_matchAll(JSContext *ctx, JSValueConst this_val,
it->iterated_string = S;
strp = JS_VALUE_GET_STRING(flags);
it->global = string_indexof_char(strp, 'g', 0) >= 0;
it->unicode = string_indexof_char(strp, 'u', 0) >= 0;
it->unicode = (string_indexof_char(strp, 'u', 0) >= 0 ||
string_indexof_char(strp, 'v', 0) >= 0);
it->done = FALSE;
JS_SetOpaque(iter, it);
@ -45364,13 +45357,11 @@ static JSValue js_regexp_Symbol_replace(JSContext *ctx, JSValueConst this_val,
goto exception;
p = JS_VALUE_GET_STRING(flags);
// TODO(bnoordhuis) query 'u' flag the same way?
fullUnicode = 0;
is_global = (-1 != string_indexof_char(p, 'g', 0));
if (is_global) {
fullUnicode = JS_ToBoolFree(ctx, JS_GetProperty(ctx, rx, JS_ATOM_unicode));
if (fullUnicode < 0)
goto exception;
fullUnicode = (string_indexof_char(p, 'u', 0) >= 0 ||
string_indexof_char(p, 'v', 0) >= 0);
if (JS_SetProperty(ctx, rx, JS_ATOM_lastIndex, JS_NewInt32(ctx, 0)) < 0)
goto exception;
}
@ -45596,7 +45587,8 @@ static JSValue js_regexp_Symbol_split(JSContext *ctx, JSValueConst this_val,
if (JS_IsException(flags))
goto exception;
strp = JS_VALUE_GET_STRING(flags);
unicodeMatching = string_indexof_char(strp, 'u', 0) >= 0;
unicodeMatching = (string_indexof_char(strp, 'u', 0) >= 0 ||
string_indexof_char(strp, 'v', 0) >= 0);
if (string_indexof_char(strp, 'y', 0) < 0) {
flags = JS_ConcatString3(ctx, "", flags, "y");
if (JS_IsException(flags))
@ -45707,6 +45699,7 @@ static const JSCFunctionListEntry js_regexp_proto_funcs[] = {
JS_CGETSET_MAGIC_DEF("multiline", js_regexp_get_flag, NULL, LRE_FLAG_MULTILINE ),
JS_CGETSET_MAGIC_DEF("dotAll", js_regexp_get_flag, NULL, LRE_FLAG_DOTALL ),
JS_CGETSET_MAGIC_DEF("unicode", js_regexp_get_flag, NULL, LRE_FLAG_UNICODE ),
JS_CGETSET_MAGIC_DEF("unicodeSets", js_regexp_get_flag, NULL, LRE_FLAG_UNICODE_SETS ),
JS_CGETSET_MAGIC_DEF("sticky", js_regexp_get_flag, NULL, LRE_FLAG_STICKY ),
JS_CGETSET_MAGIC_DEF("hasIndices", js_regexp_get_flag, NULL, LRE_FLAG_INDICES ),
JS_CFUNC_DEF("exec", 1, js_regexp_exec ),

30
test262.conf
View File

@ -180,7 +180,7 @@ regexp-match-indices
regexp-modifiers=skip
regexp-named-groups
regexp-unicode-property-escapes
regexp-v-flag=skip
regexp-v-flag
RegExp.escape
resizable-arraybuffer=skip
rest-parameters
@ -250,32 +250,6 @@ test262/test/built-ins/ThrowTypeError/unique-per-realm-function-proto.js
#test262/test/built-ins/RegExp/CharacterClassEscapes/
#test262/test/built-ins/RegExp/property-escapes/
# feature regexp-v-flag is missing in the tests
test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-digit-class-escape-negative-cases.js
test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-digit-class-escape-negative-cases.js
test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-digit-class-escape-positive-cases.js
test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-digit-class-escape-positive-cases.js
test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-non-digit-class-escape-negative-cases.js
test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-non-digit-class-escape-negative-cases.js
test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-non-digit-class-escape-positive-cases.js
test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-non-digit-class-escape-positive-cases.js
test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-non-whitespace-class-escape-negative-cases.js
test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-non-whitespace-class-escape-negative-cases.js
test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-non-whitespace-class-escape-positive-cases.js
test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-non-whitespace-class-escape-positive-cases.js
test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-non-word-class-escape-negative-cases.js
test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-non-word-class-escape-negative-cases.js
test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-non-word-class-escape-positive-cases.js
test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-non-word-class-escape-positive-cases.js
test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-whitespace-class-escape-negative-cases.js
test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-whitespace-class-escape-negative-cases.js
test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-whitespace-class-escape-positive-cases.js
test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-whitespace-class-escape-positive-cases.js
test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-word-class-escape-negative-cases.js
test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-word-class-escape-negative-cases.js
test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-word-class-escape-positive-cases.js
test262/test/built-ins/RegExp/CharacterClassEscapes/character-class-word-class-escape-positive-cases.js
# not yet in official specification
test262/test/built-ins/String/prototype/match/cstm-matcher-on-bigint-primitive.js
test262/test/built-ins/String/prototype/match/cstm-matcher-on-bigint-primitive.js
@ -341,8 +315,6 @@ test262/test/staging/sm/Set/symmetric-difference.js
test262/test/staging/sm/Set/union.js
test262/test/staging/sm/extensions/censor-strict-caller.js
test262/test/staging/sm/JSON/parse-with-source.js
test262/test/staging/sm/RegExp/flags.js
test262/test/staging/sm/RegExp/prototype.js
# not standard
test262/test/staging/sm/Function/builtin-no-construct.js

4
test262_errors.txt
View File

@ -16,13 +16,9 @@ test262/test/staging/sm/JSON/parse-number-syntax.js:39: Test262Error: parsing st
test262/test/staging/sm/JSON/parse-syntax-errors-02.js:51: Test262Error: parsing string <["Illegal backslash escape: \x15"]> threw a non-SyntaxError exception: Test262Error: string <["Illegal backslash escape: \x15"]> shouldn't have parsed as JSON Expected SameValue(«false», «true») to be true Expected SameValue(«true», «false») to be true
test262/test/staging/sm/Math/cbrt-approx.js:26: Error: got 1.39561242508609, expected a number near 1.3956124250860895 (relative error: 2)
test262/test/staging/sm/RegExp/constructor-ordering-2.js:15: Test262Error: Expected SameValue(«false», «true») to be true
test262/test/staging/sm/RegExp/match-trace.js:13: Test262Error: Expected SameValue(«"get:flags,get:unicode,set:lastIndex,get:exec,call:exec,get:result[0],get:exec,call:exec,get:result[0],get:exec,call:exec,"», «"get:flags,set:lastIndex,get:exec,call:exec,get:result[0],get:exec,call:exec,get:result[0],get:exec,call:exec,"») to be true
test262/test/staging/sm/RegExp/regress-613820-1.js:13: Test262Error: Expected SameValue(«"aaa"», «"aa"») to be true
test262/test/staging/sm/RegExp/regress-613820-2.js:13: Test262Error: Expected SameValue(«"f"», «undefined») to be true
test262/test/staging/sm/RegExp/regress-613820-3.js:13: Test262Error: Expected SameValue(«"aab"», «"aa"») to be true
test262/test/staging/sm/RegExp/replace-trace.js:13: Test262Error: Expected SameValue(«"get:flags,get:unicode,set:lastIndex,get:exec,call:exec,get:result[0],get:exec,call:exec,get:result[length],get:result[0],get:result[index],get:result[groups],"», «"get:flags,set:lastIndex,get:exec,call:exec,get:result[0],get:exec,call:exec,get:result[length],get:result[0],get:result[index],get:result[groups],"») to be true
test262/test/staging/sm/RegExp/unicode-ignoreCase-escape.js:22: Test262Error: Actual argument shouldn't be nullish.
test262/test/staging/sm/RegExp/unicode-ignoreCase-word-boundary.js:13: Test262Error: Expected SameValue(«false», «true») to be true
test262/test/staging/sm/String/match-defines-match-elements.js:52: Test262Error: Expected SameValue(«true», «false») to be true
test262/test/staging/sm/TypedArray/constructor-buffer-sequence.js:73: Error: Assertion failed: expected exception ExpectedError, got Error: Poisoned Value
test262/test/staging/sm/TypedArray/prototype-constructor-identity.js:17: Test262Error: Expected SameValue(«2», «6») to be true

28
tests/test_builtin.js
View File

@ -751,6 +751,34 @@ function test_regexp()
assert(a, ["123a23", "3"]);
a = /()*?a/.exec(",");
assert(a, null);
/* test \b escape */
assert(/[\q{a\b}]/.test("a\b"), true);
assert(/[\b]/.test("\b"), true);
/* test case insensitive matching (test262 hardly tests it) */
assert("aAbBcC#4".replace(/\p{Lower}/gu,"X"), "XAXBXC#4");
assert("aAbBcC#4".replace(/\p{Lower}/gui,"X"), "XXXXXX#4");
assert("aAbBcC#4".replace(/\p{Upper}/gui,"X"), "XXXXXX#4");
assert("aAbBcC#4".replace(/\P{Lower}/gui,"X"), "XXXXXXXX");
assert("aAbBcC#4".replace(/\P{Upper}/gui,"X"), "XXXXXXXX");
assert("aAbBcC".replace(/[^b]/gui, "X"), "XXbBXX");
assert("aAbBcC".replace(/[^A-B]/gui, "X"), "aAbBXX");
assert("aAbBcC#4".replace(/\p{Lower}/gvi,"X"), "XXXXXX#4");
assert("aAbBcC#4".replace(/\P{Lower}/gvi,"X"), "aAbBcCXX");
assert("aAbBcC#4".replace(/[^\P{Lower}]/gvi,"X"), "XXXXXX#4");
assert("aAbBcC#4".replace(/\P{Upper}/gvi,"X"), "aAbBcCXX");
assert("aAbBcC".replace(/[^b]/gvi, "X"), "XXbBXX");
assert("aAbBcC".replace(/[^A-B]/gvi, "X"), "aAbBXX");
assert("aAbBcC".replace(/[[a-c]&&B]/gvi, "X"), "aAXXcC");
assert("aAbBcC".replace(/[[a-c]--B]/gvi, "X"), "XXbBXX");
assert("abcAbC".replace(/[\q{AbC}]/gvi,"X"), "XX");
/* Note: SpiderMonkey and v8 may not be correct */
assert("abcAbC".replace(/[\q{BC|A}]/gvi,"X"), "XXXX");
assert("abcAbC".replace(/[\q{BC|A}--a]/gvi,"X"), "aXAX");
}
function test_symbol()

12
unicode_download.sh
View File

@ -1,8 +1,9 @@
#!/bin/sh
set -e
url="ftp://ftp.unicode.org/Public/16.0.0/ucd"
emoji_url="${url}/emoji/emoji-data.txt"
version="16.0.0"
emoji_version="16.0"
url="ftp://ftp.unicode.org/Public"
files="CaseFolding.txt DerivedNormalizationProps.txt PropList.txt \
SpecialCasing.txt CompositionExclusions.txt ScriptExtensions.txt \
@ -12,8 +13,11 @@ PropertyValueAliases.txt"
mkdir -p unicode
for f in $files; do
g="${url}/${f}"
g="${url}/${version}/ucd/${f}"
wget $g -O unicode/$f
done
wget $emoji_url -O unicode/emoji-data.txt
wget "${url}/${version}/ucd/emoji/emoji-data.txt" -O unicode/emoji-data.txt
wget "${url}/emoji/${emoji_version}/emoji-sequences.txt" -O unicode/emoji-sequences.txt
wget "${url}/emoji/${emoji_version}/emoji-zwj-sequences.txt" -O unicode/emoji-zwj-sequences.txt

541
unicode_gen.c
View File

@ -156,6 +156,153 @@ char *get_line(char *buf, int buf_size, FILE *f)
return buf;
}
typedef struct REString {
struct REString *next;
uint32_t hash;
uint32_t len;
uint32_t flags;
uint32_t buf[];
} REString;
typedef struct {
uint32_t n_strings;
uint32_t hash_size;
int hash_bits;
REString **hash_table;
} REStringList;
static uint32_t re_string_hash(int len, const uint32_t *buf)
{
int i;
uint32_t h;
h = 1;
for(i = 0; i < len; i++)
h = h * 263 + buf[i];
return h * 0x61C88647;
}
static void re_string_list_init(REStringList *s)
{
s->n_strings = 0;
s->hash_size = 0;
s->hash_bits = 0;
s->hash_table = NULL;
}
static __maybe_unused void re_string_list_free(REStringList *s)
{
REString *p, *p_next;
int i;
for(i = 0; i < s->hash_size; i++) {
for(p = s->hash_table[i]; p != NULL; p = p_next) {
p_next = p->next;
free(p);
}
}
free(s->hash_table);
}
static void lre_print_char(int c, BOOL is_range)
{
if (c == '\'' || c == '\\' ||
(is_range && (c == '-' || c == ']'))) {
printf("\\%c", c);
} else if (c >= ' ' && c <= 126) {
printf("%c", c);
} else {
printf("\\u{%04x}", c);
}
}
static __maybe_unused void re_string_list_dump(const char *str, const REStringList *s)
{
REString *p;
int i, j, k;
printf("%s:\n", str);
j = 0;
for(i = 0; i < s->hash_size; i++) {
for(p = s->hash_table[i]; p != NULL; p = p->next) {
printf(" %d/%d: '", j, s->n_strings);
for(k = 0; k < p->len; k++) {
lre_print_char(p->buf[k], FALSE);
}
printf("'\n");
j++;
}
}
}
static REString *re_string_find2(REStringList *s, int len, const uint32_t *buf,
uint32_t h0, BOOL add_flag)
{
uint32_t h = 0; /* avoid warning */
REString *p;
if (s->n_strings != 0) {
h = h0 >> (32 - s->hash_bits);
for(p = s->hash_table[h]; p != NULL; p = p->next) {
if (p->hash == h0 && p->len == len &&
!memcmp(p->buf, buf, len * sizeof(buf[0]))) {
return p;
}
}
}
/* not found */
if (!add_flag)
return NULL;
/* increase the size of the hash table if needed */
if (unlikely((s->n_strings + 1) > s->hash_size)) {
REString **new_hash_table, *p_next;
int new_hash_bits, i;
uint32_t new_hash_size;
new_hash_bits = max_int(s->hash_bits + 1, 4);
new_hash_size = 1 << new_hash_bits;
new_hash_table = malloc(sizeof(new_hash_table[0]) * new_hash_size);
if (!new_hash_table)
return NULL;
memset(new_hash_table, 0, sizeof(new_hash_table[0]) * new_hash_size);
for(i = 0; i < s->hash_size; i++) {
for(p = s->hash_table[i]; p != NULL; p = p_next) {
p_next = p->next;
h = p->hash >> (32 - new_hash_bits);
p->next = new_hash_table[h];
new_hash_table[h] = p;
}
}
free(s->hash_table);
s->hash_bits = new_hash_bits;
s->hash_size = new_hash_size;
s->hash_table = new_hash_table;
h = h0 >> (32 - s->hash_bits);
}
p = malloc(sizeof(REString) + len * sizeof(buf[0]));
if (!p)
return NULL;
p->next = s->hash_table[h];
s->hash_table[h] = p;
s->n_strings++;
p->hash = h0;
p->len = len;
p->flags = 0;
memcpy(p->buf, buf, sizeof(buf[0]) * len);
return p;
}
static REString *re_string_find(REStringList *s, int len, const uint32_t *buf,
BOOL add_flag)
{
uint32_t h0;
h0 = re_string_hash(len, buf);
return re_string_find2(s, len, buf, h0, add_flag);
}
static void re_string_add(REStringList *s, int len, const uint32_t *buf)
{
re_string_find(s, len, buf, TRUE);
}
#define UNICODE_GENERAL_CATEGORY
typedef enum {
@ -225,6 +372,23 @@ static const char *unicode_prop_short_name[] = {
#undef UNICODE_PROP_LIST
#define UNICODE_SEQUENCE_PROP_LIST
typedef enum {
#define DEF(id) SEQUENCE_PROP_ ## id,
#include "unicode_gen_def.h"
#undef DEF
SEQUENCE_PROP_COUNT,
} UnicodeSequencePropEnum1;
static const char *unicode_sequence_prop_name[] = {
#define DEF(id) #id,
#include "unicode_gen_def.h"
#undef DEF
};
#undef UNICODE_SEQUENCE_PROP_LIST
typedef struct {
/* case conv */
uint8_t u_len;
@ -247,7 +411,15 @@ typedef struct {
int *decomp_data;
} CCInfo;
typedef struct {
int count;
int size;
int *tab;
} UnicodeSequenceProperties;
CCInfo *unicode_db;
REStringList rgi_emoji_zwj_sequence;
DynBuf rgi_emoji_tag_sequence;
int find_name(const char **tab, int tab_len, const char *name)
{
@ -751,6 +923,147 @@ void parse_prop_list(const char *filename)
fclose(f);
}
#define SEQ_MAX_LEN 16
static BOOL is_emoji_modifier(uint32_t c)
{
return (c >= 0x1f3fb && c <= 0x1f3ff);
}
static void add_sequence_prop(int idx, int seq_len, int *seq)
{
int i;
assert(idx < SEQUENCE_PROP_COUNT);
switch(idx) {
case SEQUENCE_PROP_Basic_Emoji:
/* convert to 2 properties lists */
if (seq_len == 1) {
set_prop(seq[0], PROP_Basic_Emoji1, 1);
} else if (seq_len == 2 && seq[1] == 0xfe0f) {
set_prop(seq[0], PROP_Basic_Emoji2, 1);
} else {
abort();
}
break;
case SEQUENCE_PROP_RGI_Emoji_Modifier_Sequence:
assert(seq_len == 2);
assert(is_emoji_modifier(seq[1]));
assert(get_prop(seq[0], PROP_Emoji_Modifier_Base));
set_prop(seq[0], PROP_RGI_Emoji_Modifier_Sequence, 1);
break;
case SEQUENCE_PROP_RGI_Emoji_Flag_Sequence:
{
int code;
assert(seq_len == 2);
assert(seq[0] >= 0x1F1E6 && seq[0] <= 0x1F1FF);
assert(seq[1] >= 0x1F1E6 && seq[1] <= 0x1F1FF);
code = (seq[0] - 0x1F1E6) * 26 + (seq[1] - 0x1F1E6);
/* XXX: would be more compact with a simple bitmap -> 676 bits */
set_prop(code, PROP_RGI_Emoji_Flag_Sequence, 1);
}
break;
case SEQUENCE_PROP_RGI_Emoji_ZWJ_Sequence:
re_string_add(&rgi_emoji_zwj_sequence, seq_len, (uint32_t *)seq);
break;
case SEQUENCE_PROP_RGI_Emoji_Tag_Sequence:
{
assert(seq_len >= 3);
assert(seq[0] == 0x1F3F4);
assert(seq[seq_len - 1] == 0xE007F);
for(i = 1; i < seq_len - 1; i++) {
assert(seq[i] >= 0xe0001 && seq[i] <= 0xe007e);
dbuf_putc(&rgi_emoji_tag_sequence, seq[i] - 0xe0000);
}
dbuf_putc(&rgi_emoji_tag_sequence, 0);
}
break;
case SEQUENCE_PROP_Emoji_Keycap_Sequence:
assert(seq_len == 3);
assert(seq[1] == 0xfe0f);
assert(seq[2] == 0x20e3);
set_prop(seq[0], PROP_Emoji_Keycap_Sequence, 1);
break;
default:
assert(0);
}
}
void parse_sequence_prop_list(const char *filename)
{
FILE *f;
char line[4096], *p, buf[256], *q, *p_start;
uint32_t c0, c1, c;
int idx, seq_len;
int seq[SEQ_MAX_LEN];
f = fopen(filename, "rb");
if (!f) {
perror(filename);
exit(1);
}
for(;;) {
if (!get_line(line, sizeof(line), f))
break;
p = line;
while (isspace(*p))
p++;
if (*p == '#' || *p == '@' || *p == '\0')
continue;
p_start = p;
/* find the sequence property name */
p = strchr(p, ';');
if (!p)
continue;
p++;
p += strspn(p, " \t");
q = buf;
while (*p != '\0' && *p != ' ' && *p != '#' && *p != '\t' && *p != ';') {
if ((q - buf) < sizeof(buf) - 1)
*q++ = *p;
p++;
}
*q = '\0';
idx = find_name(unicode_sequence_prop_name,
countof(unicode_sequence_prop_name), buf);
if (idx < 0) {
fprintf(stderr, "Property not found: %s\n", buf);
exit(1);
}
p = p_start;
c0 = strtoul(p, (char **)&p, 16);
assert(c0 <= CHARCODE_MAX);
if (*p == '.' && p[1] == '.') {
p += 2;
c1 = strtoul(p, (char **)&p, 16);
assert(c1 <= CHARCODE_MAX);
for(c = c0; c <= c1; c++) {
seq[0] = c;
add_sequence_prop(idx, 1, seq);
}
} else {
seq_len = 0;
seq[seq_len++] = c0;
for(;;) {
while (isspace(*p))
p++;
if (*p == ';' || *p == '\0')
break;
c0 = strtoul(p, (char **)&p, 16);
assert(c0 <= CHARCODE_MAX);
assert(seq_len < countof(seq));
seq[seq_len++] = c0;
}
add_sequence_prop(idx, seq_len, seq);
}
}
fclose(f);
}
void parse_scripts(const char *filename)
{
FILE *f;
@ -1654,7 +1967,7 @@ void dump_name_table(FILE *f, const char *cname, const char **tab_name, int len,
maxw = 0;
for(i = 0; i < len; i++) {
w = strlen(tab_name[i]);
if (tab_short_name[i][0] != '\0') {
if (tab_short_name && tab_short_name[i][0] != '\0') {
w += 1 + strlen(tab_short_name[i]);
}
if (maxw < w)
@ -1666,7 +1979,7 @@ void dump_name_table(FILE *f, const char *cname, const char **tab_name, int len,
for(i = 0; i < len; i++) {
fprintf(f, " \"");
w = fprintf(f, "%s", tab_name[i]);
if (tab_short_name[i][0] != '\0') {
if (tab_short_name && tab_short_name[i][0] != '\0') {
w += fprintf(f, ",%s", tab_short_name[i]);
}
fprintf(f, "\"%*s\"\\0\"\n", 1 + maxw - w, "");
@ -1930,6 +2243,218 @@ void build_prop_list_table(FILE *f)
fprintf(f, "};\n\n");
}
static BOOL is_emoji_hair_color(uint32_t c)
{
return (c >= 0x1F9B0 && c <= 0x1F9B3);
}
#define EMOJI_MOD_NONE 0
#define EMOJI_MOD_TYPE1 1
#define EMOJI_MOD_TYPE2 2
#define EMOJI_MOD_TYPE2D 3
static BOOL mark_zwj_string(REStringList *sl, uint32_t *buf, int len, int mod_type, int *mod_pos,
int hc_pos, BOOL mark_flag)
{
REString *p;
int i, n_mod, i0, i1, hc_count, j;
#if 0
if (mark_flag)
printf("mod_type=%d\n", mod_type);
#endif
switch(mod_type) {
case EMOJI_MOD_NONE:
n_mod = 1;
break;
case EMOJI_MOD_TYPE1:
n_mod = 5;
break;
case EMOJI_MOD_TYPE2:
n_mod = 25;
break;
case EMOJI_MOD_TYPE2D:
n_mod = 20;
break;
default:
assert(0);
}
if (hc_pos >= 0)
hc_count = 4;
else
hc_count = 1;
/* check that all the related strings are present */
for(j = 0; j < hc_count; j++) {
for(i = 0; i < n_mod; i++) {
switch(mod_type) {
case EMOJI_MOD_NONE:
break;
case EMOJI_MOD_TYPE1:
buf[mod_pos[0]] = 0x1f3fb + i;
break;
case EMOJI_MOD_TYPE2:
case EMOJI_MOD_TYPE2D:
i0 = i / 5;
i1 = i % 5;
/* avoid identical values */
if (mod_type == EMOJI_MOD_TYPE2D && i0 >= i1)
i0++;
buf[mod_pos[0]] = 0x1f3fb + i0;
buf[mod_pos[1]] = 0x1f3fb + i1;
break;
default:
assert(0);
}
if (hc_pos >= 0)
buf[hc_pos] = 0x1F9B0 + j;
p = re_string_find(sl, len, buf, FALSE);
if (!p)
return FALSE;
if (mark_flag)
p->flags |= 1;
}
}
return TRUE;
}
static void zwj_encode_string(DynBuf *dbuf, const uint32_t *buf, int len, int mod_type, int *mod_pos,
int hc_pos)
{
int i, j;
int c, code;
uint32_t buf1[SEQ_MAX_LEN];
j = 0;
for(i = 0; i < len;) {
c = buf[i++];
if (c >= 0x2000 && c <= 0x2fff) {
code = c - 0x2000;
} else if (c >= 0x1f000 && c <= 0x1ffff) {
code = c - 0x1f000 + 0x1000;
} else {
assert(0);
}
if (i < len && is_emoji_modifier(buf[i])) {
/* modifier */
code |= (mod_type << 13);
i++;
}
if (i < len && buf[i] == 0xfe0f) {
/* presentation selector present */
code |= 0x8000;
i++;
}
if (i < len) {
/* zero width join */
assert(buf[i] == 0x200d);
i++;
}
buf1[j++] = code;
}
dbuf_putc(dbuf, j);
for(i = 0; i < j; i++) {
dbuf_putc(dbuf, buf1[i]);
dbuf_putc(dbuf, buf1[i] >> 8);
}
}
static void build_rgi_emoji_zwj_sequence(FILE *f, REStringList *sl)
{
int mod_pos[2], mod_count, hair_color_pos, j, h;
REString *p;
uint32_t buf[SEQ_MAX_LEN];
DynBuf dbuf;
#if 0
{
for(h = 0; h < sl->hash_size; h++) {
for(p = sl->hash_table[h]; p != NULL; p = p->next) {
for(j = 0; j < p->len; j++)
printf(" %04x", p->buf[j]);
printf("\n");
}
}
exit(0);
}
#endif
// printf("rgi_emoji_zwj_sequence: n=%d\n", sl->n_strings);
dbuf_init(&dbuf);
/* avoid duplicating strings with emoji modifiers or hair colors */
for(h = 0; h < sl->hash_size; h++) {
for(p = sl->hash_table[h]; p != NULL; p = p->next) {
if (p->flags) /* already examined */
continue;
mod_count = 0;
hair_color_pos = -1;
for(j = 0; j < p->len; j++) {
if (is_emoji_modifier(p->buf[j])) {
assert(mod_count < 2);
mod_pos[mod_count++] = j;
} else if (is_emoji_hair_color(p->buf[j])) {
hair_color_pos = j;
}
buf[j] = p->buf[j];
}
if (mod_count != 0 || hair_color_pos >= 0) {
int mod_type;
if (mod_count == 0)
mod_type = EMOJI_MOD_NONE;
else if (mod_count == 1)
mod_type = EMOJI_MOD_TYPE1;
else
mod_type = EMOJI_MOD_TYPE2;
if (mark_zwj_string(sl, buf, p->len, mod_type, mod_pos, hair_color_pos, FALSE)) {
mark_zwj_string(sl, buf, p->len, mod_type, mod_pos, hair_color_pos, TRUE);
} else if (mod_type == EMOJI_MOD_TYPE2) {
mod_type = EMOJI_MOD_TYPE2D;
if (mark_zwj_string(sl, buf, p->len, mod_type, mod_pos, hair_color_pos, FALSE)) {
mark_zwj_string(sl, buf, p->len, mod_type, mod_pos, hair_color_pos, TRUE);
} else {
dump_str("not_found", (int *)p->buf, p->len);
goto keep;
}
}
if (hair_color_pos >= 0)
buf[hair_color_pos] = 0x1f9b0;
/* encode the string */
zwj_encode_string(&dbuf, buf, p->len, mod_type, mod_pos, hair_color_pos);
} else {
keep:
zwj_encode_string(&dbuf, buf, p->len, EMOJI_MOD_NONE, NULL, -1);
}
}
}
/* Encode */
dump_byte_table(f, "unicode_rgi_emoji_zwj_sequence", dbuf.buf, dbuf.size);
dbuf_free(&dbuf);
}
void build_sequence_prop_list_table(FILE *f)
{
int i;
fprintf(f, "typedef enum {\n");
for(i = 0; i < SEQUENCE_PROP_COUNT; i++)
fprintf(f, " UNICODE_SEQUENCE_PROP_%s,\n", unicode_sequence_prop_name[i]);
fprintf(f, " UNICODE_SEQUENCE_PROP_COUNT,\n");
fprintf(f, "} UnicodeSequencePropertyEnum;\n\n");
dump_name_table(f, "unicode_sequence_prop_name_table",
unicode_sequence_prop_name, SEQUENCE_PROP_COUNT, NULL);
dump_byte_table(f, "unicode_rgi_emoji_tag_sequence", rgi_emoji_tag_sequence.buf, rgi_emoji_tag_sequence.size);
build_rgi_emoji_zwj_sequence(f, &rgi_emoji_zwj_sequence);
}
#ifdef USE_TEST
int check_conv(uint32_t *res, uint32_t c, int conv_type)
{
@ -3156,6 +3681,8 @@ int main(int argc, char *argv[])
outfilename = argv[arg++];
unicode_db = mallocz(sizeof(unicode_db[0]) * (CHARCODE_MAX + 1));
re_string_list_init(&rgi_emoji_zwj_sequence);
dbuf_init(&rgi_emoji_tag_sequence);
snprintf(filename, sizeof(filename), "%s/UnicodeData.txt", unicode_db_path);
@ -3190,6 +3717,14 @@ int main(int argc, char *argv[])
unicode_db_path);
parse_prop_list(filename);
snprintf(filename, sizeof(filename), "%s/emoji-sequences.txt",
unicode_db_path);
parse_sequence_prop_list(filename);
snprintf(filename, sizeof(filename), "%s/emoji-zwj-sequences.txt",
unicode_db_path);
parse_sequence_prop_list(filename);
// dump_unicode_data(unicode_db);
build_conv_table(unicode_db);
@ -3234,10 +3769,12 @@ int main(int argc, char *argv[])
build_script_table(fo);
build_script_ext_table(fo);
build_prop_list_table(fo);
build_sequence_prop_list_table(fo);
fprintf(fo, "#endif /* CONFIG_ALL_UNICODE */\n");
fprintf(fo, "/* %u tables / %u bytes, %u index / %u bytes */\n",
total_tables, total_table_bytes, total_index, total_index_bytes);
fclose(fo);
}
re_string_list_free(&rgi_emoji_zwj_sequence);
return 0;
}

15
unicode_gen_def.h
View File

@ -234,6 +234,11 @@ DEF(XID_Continue1, "")
DEF(Changes_When_Titlecased1, "")
DEF(Changes_When_Casefolded1, "")
DEF(Changes_When_NFKC_Casefolded1, "")
DEF(Basic_Emoji1, "")
DEF(Basic_Emoji2, "")
DEF(RGI_Emoji_Modifier_Sequence, "")
DEF(RGI_Emoji_Flag_Sequence, "")
DEF(Emoji_Keycap_Sequence, "")
/* Prop list exported to JS */
DEF(ASCII_Hex_Digit, "AHex")
@ -301,3 +306,13 @@ DEF(XID_Start, "XIDS")
DEF(Cased1, "")
#endif
#ifdef UNICODE_SEQUENCE_PROP_LIST
DEF(Basic_Emoji)
DEF(Emoji_Keycap_Sequence)
DEF(RGI_Emoji_Modifier_Sequence)
DEF(RGI_Emoji_Flag_Sequence)
DEF(RGI_Emoji_Tag_Sequence)
DEF(RGI_Emoji_ZWJ_Sequence)
DEF(RGI_Emoji)
#endif