mirrors/quickjs
1
0
Fork 0
mirror of https://github.com/bellard/quickjs.git synced 2024-11-24 06:38:12 +08:00
Code Issues Packages Projects Releases Wiki Activity
f3f2f42717
quickjs/unicode_gen.c
Charlie Gordon 7a2c6f42d4
Improve libunicode and libregexp headers (#288) 
- move all `lre_xxx` functions to libunicode
- use flags table `lre_ctype_bits` instead of bitmaps
- simplify `lre_is_space`, `lre_js_is_ident_first` and `lre_js_is_ident_next`
- simplify `simple_next_token`, handle UTF-8 correctly
- simplify `is_let`, remove dead code
2024-05-05 17:47:40 +02:00

3226 lines
87 KiB
C
Raw Blame History

/*
* Generation of Unicode tables
*
* Copyright (c) 2017-2018 Fabrice Bellard
* Copyright (c) 2017-2018 Charlie Gordon
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#include <stdlib.h>
#include <stdio.h>
#include <stdarg.h>
#include <inttypes.h>
#include <string.h>
#include <assert.h>
#include <ctype.h>
#include <time.h>
#include "cutils.h"
uint32_t total_tables;
uint32_t total_table_bytes;
uint32_t total_index;
uint32_t total_index_bytes;
/* define it to be able to test unicode.c */
//#define USE_TEST
/* profile tests */
//#define PROFILE
//#define DUMP_CASE_CONV_TABLE
//#define DUMP_TABLE_SIZE
//#define DUMP_CC_TABLE
//#define DUMP_DECOMP_TABLE
//#define DUMP_CASE_FOLDING_SPECIAL_CASES
/* Ideas:
- Generalize run length encoding + index for all tables
- remove redundant tables for ID_start, ID_continue, Case_Ignorable, Cased
Case conversion:
- use a single entry for consecutive U/LF runs
- allow EXT runs of length > 1
Decomposition:
- Greek lower case (+1f10/1f10) ?
- allow holes in B runs
- suppress more upper / lower case redundancy
*/
#ifdef USE_TEST
#include "libunicode.c"
#endif
#define CHARCODE_MAX 0x10ffff
#define CC_LEN_MAX 3
void *mallocz(size_t size)
{
void *ptr;
ptr = malloc(size);
memset(ptr, 0, size);
return ptr;
}
const char *get_field(const char *p, int n)
{
int i;
for(i = 0; i < n; i++) {
while (*p != ';' && *p != '\0')
p++;
if (*p == '\0')
return NULL;
p++;
}
return p;
}
const char *get_field_buf(char *buf, size_t buf_size, const char *p, int n)
{
char *q;
p = get_field(p, n);
q = buf;
while (*p != ';' && *p != '\0') {
if ((q - buf) < buf_size - 1)
*q++ = *p;
p++;
}
*q = '\0';
return buf;
}
void add_char(int **pbuf, int *psize, int *plen, int c)
{
int len, size, *buf;
buf = *pbuf;
size = *psize;
len = *plen;
if (len >= size) {
size = *psize;
size = max_int(len + 1, size * 3 / 2);
buf = realloc(buf, sizeof(buf[0]) * size);
*pbuf = buf;
*psize = size;
}
buf[len++] = c;
*plen = len;
}
int *get_field_str(int *plen, const char *str, int n)
{
const char *p;
int *buf, len, size;
p = get_field(str, n);
if (!p) {
*plen = 0;
return NULL;
}
len = 0;
size = 0;
buf = NULL;
for(;;) {
while (isspace(*p))
p++;
if (!isxdigit(*p))
break;
add_char(&buf, &size, &len, strtoul(p, (char **)&p, 16));
}
*plen = len;
return buf;
}
char *get_line(char *buf, int buf_size, FILE *f)
{
int len;
if (!fgets(buf, buf_size, f))
return NULL;
len = strlen(buf);
if (len > 0 && buf[len - 1] == '\n')
buf[len - 1] = '\0';
return buf;
}
#define UNICODE_GENERAL_CATEGORY
typedef enum {
#define DEF(id, str) GCAT_ ## id,
#include "unicode_gen_def.h"
#undef DEF
GCAT_COUNT,
} UnicodeGCEnum1;
static const char *unicode_gc_name[] = {
#define DEF(id, str) #id,
#include "unicode_gen_def.h"
#undef DEF
};
static const char *unicode_gc_short_name[] = {
#define DEF(id, str) str,
#include "unicode_gen_def.h"
#undef DEF
};
#undef UNICODE_GENERAL_CATEGORY
#define UNICODE_SCRIPT
typedef enum {
#define DEF(id, str) SCRIPT_ ## id,
#include "unicode_gen_def.h"
#undef DEF
SCRIPT_COUNT,
} UnicodeScriptEnum1;
static const char *unicode_script_name[] = {
#define DEF(id, str) #id,
#include "unicode_gen_def.h"
#undef DEF
};
const char *unicode_script_short_name[] = {
#define DEF(id, str) str,
#include "unicode_gen_def.h"
#undef DEF
};
#undef UNICODE_SCRIPT
#define UNICODE_PROP_LIST
typedef enum {
#define DEF(id, str) PROP_ ## id,
#include "unicode_gen_def.h"
#undef DEF
PROP_COUNT,
} UnicodePropEnum1;
static const char *unicode_prop_name[] = {
#define DEF(id, str) #id,
#include "unicode_gen_def.h"
#undef DEF
};
static const char *unicode_prop_short_name[] = {
#define DEF(id, str) str,
#include "unicode_gen_def.h"
#undef DEF
};
#undef UNICODE_PROP_LIST
typedef struct {
/* case conv */
uint8_t u_len;
uint8_t l_len;
uint8_t f_len;
int u_data[CC_LEN_MAX]; /* to upper case */
int l_data[CC_LEN_MAX]; /* to lower case */
int f_data[CC_LEN_MAX]; /* to case folding */
uint8_t combining_class;
uint8_t is_compat:1;
uint8_t is_excluded:1;
uint8_t general_category;
uint8_t script;
uint8_t script_ext_len;
uint8_t *script_ext;
uint32_t prop_bitmap_tab[3];
/* decomposition */
int decomp_len;
int *decomp_data;
} CCInfo;
CCInfo *unicode_db;
int find_name(const char **tab, int tab_len, const char *name)
{
int i, len, name_len;
const char *p, *r;
name_len = strlen(name);
for(i = 0; i < tab_len; i++) {
p = tab[i];
for(;;) {
r = strchr(p, ',');
if (!r)
len = strlen(p);
else
len = r - p;
if (len == name_len && memcmp(p, name, len) == 0)
return i;
if (!r)
break;
p = r + 1;
}
}
return -1;
}
static BOOL get_prop(uint32_t c, int prop_idx)
{
return (unicode_db[c].prop_bitmap_tab[prop_idx >> 5] >> (prop_idx & 0x1f)) & 1;
}
static void set_prop(uint32_t c, int prop_idx, int val)
{
uint32_t mask;
mask = 1U << (prop_idx & 0x1f);
if (val)
unicode_db[c].prop_bitmap_tab[prop_idx >> 5] |= mask;
else
unicode_db[c].prop_bitmap_tab[prop_idx >> 5] &= ~mask;
}
void parse_unicode_data(const char *filename)
{
FILE *f;
char line[1024];
char buf1[256];
const char *p;
int code, lc, uc, last_code;
CCInfo *ci, *tab = unicode_db;
f = fopen(filename, "rb");
if (!f) {
perror(filename);
exit(1);
}
last_code = 0;
for(;;) {
if (!get_line(line, sizeof(line), f))
break;
p = line;
while (isspace(*p))
p++;
if (*p == '#')
continue;
p = get_field(line, 0);
if (!p)
continue;
code = strtoul(p, NULL, 16);
lc = 0;
uc = 0;
p = get_field(line, 12);
if (p && *p != ';') {
uc = strtoul(p, NULL, 16);
}
p = get_field(line, 13);
if (p && *p != ';') {
lc = strtoul(p, NULL, 16);
}
ci = &tab[code];
if (uc > 0 || lc > 0) {
assert(code <= CHARCODE_MAX);
if (uc > 0) {
assert(ci->u_len == 0);
ci->u_len = 1;
ci->u_data[0] = uc;
}
if (lc > 0) {
assert(ci->l_len == 0);
ci->l_len = 1;
ci->l_data[0] = lc;
}
}
{
int i;
get_field_buf(buf1, sizeof(buf1), line, 2);
i = find_name(unicode_gc_name, countof(unicode_gc_name), buf1);
if (i < 0) {
fprintf(stderr, "General category '%s' not found\n",
buf1);
exit(1);
}
ci->general_category = i;
}
p = get_field(line, 3);
if (p && *p != ';' && *p != '\0') {
int cc;
cc = strtoul(p, NULL, 0);
if (cc != 0) {
assert(code <= CHARCODE_MAX);
ci->combining_class = cc;
// printf("%05x: %d\n", code, ci->combining_class);
}
}
p = get_field(line, 5);
if (p && *p != ';' && *p != '\0') {
int size;
assert(code <= CHARCODE_MAX);
ci->is_compat = 0;
if (*p == '<') {
while (*p != '\0' && *p != '>')
p++;
if (*p == '>')
p++;
ci->is_compat = 1;
}
size = 0;
for(;;) {
while (isspace(*p))
p++;
if (!isxdigit(*p))
break;
add_char(&ci->decomp_data, &size, &ci->decomp_len, strtoul(p, (char **)&p, 16));
}
#if 0
{
int i;
static int count, d_count;
printf("%05x: %c", code, ci->is_compat ? 'C': ' ');
for(i = 0; i < ci->decomp_len; i++)
printf(" %05x", ci->decomp_data[i]);
printf("\n");
count++;
d_count += ci->decomp_len;
// printf("%d %d\n", count, d_count);
}
#endif
}
p = get_field(line, 9);
if (p && *p == 'Y') {
set_prop(code, PROP_Bidi_Mirrored, 1);
}
/* handle ranges */
get_field_buf(buf1, sizeof(buf1), line, 1);
if (strstr(buf1, " Last>")) {
int i;
// printf("range: 0x%x-%0x\n", last_code, code);
assert(ci->decomp_len == 0);
assert(ci->script_ext_len == 0);
for(i = last_code + 1; i < code; i++) {
unicode_db[i] = *ci;
}
}
last_code = code;
}
fclose(f);
}
void parse_special_casing(CCInfo *tab, const char *filename)
{
FILE *f;
char line[1024];
const char *p;
int code;
CCInfo *ci;
f = fopen(filename, "rb");
if (!f) {
perror(filename);
exit(1);
}
for(;;) {
if (!get_line(line, sizeof(line), f))
break;
p = line;
while (isspace(*p))
p++;
if (*p == '#')
continue;
p = get_field(line, 0);
if (!p)
continue;
code = strtoul(p, NULL, 16);
assert(code <= CHARCODE_MAX);
ci = &tab[code];
p = get_field(line, 4);
if (p) {
/* locale dependent casing */
while (isspace(*p))
p++;
if (*p != '#' && *p != '\0')
continue;
}
p = get_field(line, 1);
if (p && *p != ';') {
ci->l_len = 0;
for(;;) {
while (isspace(*p))
p++;
if (*p == ';')
break;
assert(ci->l_len < CC_LEN_MAX);
ci->l_data[ci->l_len++] = strtoul(p, (char **)&p, 16);
}
if (ci->l_len == 1 && ci->l_data[0] == code)
ci->l_len = 0;
}
p = get_field(line, 3);
if (p && *p != ';') {
ci->u_len = 0;
for(;;) {
while (isspace(*p))
p++;
if (*p == ';')
break;
assert(ci->u_len < CC_LEN_MAX);
ci->u_data[ci->u_len++] = strtoul(p, (char **)&p, 16);
}
if (ci->u_len == 1 && ci->u_data[0] == code)
ci->u_len = 0;
}
}
fclose(f);
}
void parse_case_folding(CCInfo *tab, const char *filename)
{
FILE *f;
char line[1024];
const char *p;
int code, status;
CCInfo *ci;
f = fopen(filename, "rb");
if (!f) {
perror(filename);
exit(1);
}
for(;;) {
if (!get_line(line, sizeof(line), f))
break;
p = line;
while (isspace(*p))
p++;
if (*p == '#')
continue;
p = get_field(line, 0);
if (!p)
continue;
code = strtoul(p, NULL, 16);
assert(code <= CHARCODE_MAX);
ci = &tab[code];
p = get_field(line, 1);
if (!p)
continue;
/* locale dependent casing */
while (isspace(*p))
p++;
status = *p;
if (status != 'C' && status != 'S' && status != 'F')
continue;
p = get_field(line, 2);
assert(p != NULL);
if (status == 'S') {
/* we always select the simple case folding and assume it
* comes after the full case folding case */
assert(ci->f_len >= 2);
ci->f_len = 0;
} else {
assert(ci->f_len == 0);
}
for(;;) {
while (isspace(*p))
p++;
if (*p == ';')
break;
assert(ci->l_len < CC_LEN_MAX);
ci->f_data[ci->f_len++] = strtoul(p, (char **)&p, 16);
}
}
fclose(f);
}
void parse_composition_exclusions(const char *filename)
{
FILE *f;
char line[4096], *p;
uint32_t c0;
f = fopen(filename, "rb");
if (!f) {
perror(filename);
exit(1);
}
for(;;) {
if (!get_line(line, sizeof(line), f))
break;
p = line;
while (isspace(*p))
p++;
if (*p == '#' || *p == '@' || *p == '\0')
continue;
c0 = strtoul(p, (char **)&p, 16);
assert(c0 > 0 && c0 <= CHARCODE_MAX);
unicode_db[c0].is_excluded = TRUE;
}
fclose(f);
}
void parse_derived_core_properties(const char *filename)
{
FILE *f;
char line[4096], *p, buf[256], *q;
uint32_t c0, c1, c;
int i;
f = fopen(filename, "rb");
if (!f) {
perror(filename);
exit(1);
}
for(;;) {
if (!get_line(line, sizeof(line), f))
break;
p = line;
while (isspace(*p))
p++;
if (*p == '#' || *p == '@' || *p == '\0')
continue;
c0 = strtoul(p, (char **)&p, 16);
if (*p == '.' && p[1] == '.') {
p += 2;
c1 = strtoul(p, (char **)&p, 16);
} else {
c1 = c0;
}
assert(c1 <= CHARCODE_MAX);
p += strspn(p, " \t");
if (*p == ';') {
p++;
p += strspn(p, " \t");
q = buf;
while (*p != '\0' && *p != ' ' && *p != '#' && *p != '\t') {
if ((q - buf) < sizeof(buf) - 1)
*q++ = *p;
p++;
}
*q = '\0';
i = find_name(unicode_prop_name,
countof(unicode_prop_name), buf);
if (i < 0) {
if (!strcmp(buf, "Grapheme_Link"))
goto next;
fprintf(stderr, "Property not found: %s\n", buf);
exit(1);
}
for(c = c0; c <= c1; c++) {
set_prop(c, i, 1);
}
next: ;
}
}
fclose(f);
}
void parse_derived_norm_properties(const char *filename)
{
FILE *f;
char line[4096], *p, buf[256], *q;
uint32_t c0, c1, c;
f = fopen(filename, "rb");
if (!f) {
perror(filename);
exit(1);
}
for(;;) {
if (!get_line(line, sizeof(line), f))
break;
p = line;
while (isspace(*p))
p++;
if (*p == '#' || *p == '@' || *p == '\0')
continue;
c0 = strtoul(p, (char **)&p, 16);
if (*p == '.' && p[1] == '.') {
p += 2;
c1 = strtoul(p, (char **)&p, 16);
} else {
c1 = c0;
}
assert(c1 <= CHARCODE_MAX);
p += strspn(p, " \t");
if (*p == ';') {
p++;
p += strspn(p, " \t");
q = buf;
while (*p != '\0' && *p != ' ' && *p != '#' && *p != '\t') {
if ((q - buf) < sizeof(buf) - 1)
*q++ = *p;
p++;
}
*q = '\0';
if (!strcmp(buf, "Changes_When_NFKC_Casefolded")) {
for(c = c0; c <= c1; c++) {
set_prop(c, PROP_Changes_When_NFKC_Casefolded, 1);
}
}
}
}
fclose(f);
}
void parse_prop_list(const char *filename)
{
FILE *f;
char line[4096], *p, buf[256], *q;
uint32_t c0, c1, c;
int i;
f = fopen(filename, "rb");
if (!f) {
perror(filename);
exit(1);
}
for(;;) {
if (!get_line(line, sizeof(line), f))
break;
p = line;
while (isspace(*p))
p++;
if (*p == '#' || *p == '@' || *p == '\0')
continue;
c0 = strtoul(p, (char **)&p, 16);
if (*p == '.' && p[1] == '.') {
p += 2;
c1 = strtoul(p, (char **)&p, 16);
} else {
c1 = c0;
}
assert(c1 <= CHARCODE_MAX);
p += strspn(p, " \t");
if (*p == ';') {
p++;
p += strspn(p, " \t");
q = buf;
while (*p != '\0' && *p != ' ' && *p != '#' && *p != '\t') {
if ((q - buf) < sizeof(buf) - 1)
*q++ = *p;
p++;
}
*q = '\0';
i = find_name(unicode_prop_name,
countof(unicode_prop_name), buf);
if (i < 0) {
fprintf(stderr, "Property not found: %s\n", buf);
exit(1);
}
for(c = c0; c <= c1; c++) {
set_prop(c, i, 1);
}
}
}
fclose(f);
}
void parse_scripts(const char *filename)
{
FILE *f;
char line[4096], *p, buf[256], *q;
uint32_t c0, c1, c;
int i;
f = fopen(filename, "rb");
if (!f) {
perror(filename);
exit(1);
}
for(;;) {
if (!get_line(line, sizeof(line), f))
break;
p = line;
while (isspace(*p))
p++;
if (*p == '#' || *p == '@' || *p == '\0')
continue;
c0 = strtoul(p, (char **)&p, 16);
if (*p == '.' && p[1] == '.') {
p += 2;
c1 = strtoul(p, (char **)&p, 16);
} else {
c1 = c0;
}
assert(c1 <= CHARCODE_MAX);
p += strspn(p, " \t");
if (*p == ';') {
p++;
p += strspn(p, " \t");
q = buf;
while (*p != '\0' && *p != ' ' && *p != '#' && *p != '\t') {
if ((q - buf) < sizeof(buf) - 1)
*q++ = *p;
p++;
}
*q = '\0';
i = find_name(unicode_script_name,
countof(unicode_script_name), buf);
if (i < 0) {
fprintf(stderr, "Unknown script: '%s'\n", buf);
exit(1);
}
for(c = c0; c <= c1; c++)
unicode_db[c].script = i;
}
}
fclose(f);
}
void parse_script_extensions(const char *filename)
{
FILE *f;
char line[4096], *p, buf[256], *q;
uint32_t c0, c1, c;
int i;
uint8_t script_ext[255];
int script_ext_len;
f = fopen(filename, "rb");
if (!f) {
perror(filename);
exit(1);
}
for(;;) {
if (!get_line(line, sizeof(line), f))
break;
p = line;
while (isspace(*p))
p++;
if (*p == '#' || *p == '@' || *p == '\0')
continue;
c0 = strtoul(p, (char **)&p, 16);
if (*p == '.' && p[1] == '.') {
p += 2;
c1 = strtoul(p, (char **)&p, 16);
} else {
c1 = c0;
}
assert(c1 <= CHARCODE_MAX);
p += strspn(p, " \t");
script_ext_len = 0;
if (*p == ';') {
p++;
for(;;) {
p += strspn(p, " \t");
q = buf;
while (*p != '\0' && *p != ' ' && *p != '#' && *p != '\t') {
if ((q - buf) < sizeof(buf) - 1)
*q++ = *p;
p++;
}
*q = '\0';
if (buf[0] == '\0')
break;
i = find_name(unicode_script_short_name,
countof(unicode_script_short_name), buf);
if (i < 0) {
fprintf(stderr, "Script not found: %s\n", buf);
exit(1);
}
assert(script_ext_len < sizeof(script_ext));
script_ext[script_ext_len++] = i;
}
for(c = c0; c <= c1; c++) {
CCInfo *ci = &unicode_db[c];
ci->script_ext_len = script_ext_len;
ci->script_ext = malloc(sizeof(ci->script_ext[0]) * script_ext_len);
for(i = 0; i < script_ext_len; i++)
ci->script_ext[i] = script_ext[i];
}
}
}
fclose(f);
}
void dump_cc_info(CCInfo *ci, int i)
{
int j;
printf("%05x:", i);
if (ci->u_len != 0) {
printf(" U:");
for(j = 0; j < ci->u_len; j++)
printf(" %05x", ci->u_data[j]);
}
if (ci->l_len != 0) {
printf(" L:");
for(j = 0; j < ci->l_len; j++)
printf(" %05x", ci->l_data[j]);
}
if (ci->f_len != 0) {
printf(" F:");
for(j = 0; j < ci->f_len; j++)
printf(" %05x", ci->f_data[j]);
}
printf("\n");
}
void dump_unicode_data(CCInfo *tab)
{
int i;
CCInfo *ci;
for(i = 0; i <= CHARCODE_MAX; i++) {
ci = &tab[i];
if (ci->u_len != 0 || ci->l_len != 0 || ci->f_len != 0) {
dump_cc_info(ci, i);
}
}
}
BOOL is_complicated_case(const CCInfo *ci)
{
return (ci->u_len > 1 || ci->l_len > 1 ||
(ci->u_len > 0 && ci->l_len > 0) ||
(ci->f_len != ci->l_len) ||
(memcmp(ci->f_data, ci->l_data, ci->f_len * sizeof(ci->f_data[0])) != 0));
}
#ifndef USE_TEST
enum {
RUN_TYPE_U,
RUN_TYPE_L,
RUN_TYPE_UF,
RUN_TYPE_LF,
RUN_TYPE_UL,
RUN_TYPE_LSU,
RUN_TYPE_U2L_399_EXT2,
RUN_TYPE_UF_D20,
RUN_TYPE_UF_D1_EXT,
RUN_TYPE_U_EXT,
RUN_TYPE_LF_EXT,
RUN_TYPE_UF_EXT2,
RUN_TYPE_LF_EXT2,
RUN_TYPE_UF_EXT3,
};
#endif
const char *run_type_str[] = {
"U",
"L",
"UF",
"LF",
"UL",
"LSU",
"U2L_399_EXT2",
"UF_D20",
"UF_D1_EXT",
"U_EXT",
"LF_EXT",
"UF_EXT2",
"LF_EXT2",
"UF_EXT3",
};
typedef struct {
int code;
int len;
int type;
int data;
int ext_len;
int ext_data[3];
int data_index; /* 'data' coming from the table */
} TableEntry;
static int simple_to_lower(CCInfo *tab, int c)
{
if (tab[c].l_len != 1)
return c;
return tab[c].l_data[0];
}
/* code (17), len (7), type (4) */
void find_run_type(TableEntry *te, CCInfo *tab, int code)
{
int is_lower, len;
CCInfo *ci, *ci1, *ci2;
ci = &tab[code];
ci1 = &tab[code + 1];
ci2 = &tab[code + 2];
te->code = code;
if (ci->l_len == 1 && ci->l_data[0] == code + 2 &&
ci->f_len == 1 && ci->f_data[0] == ci->l_data[0] &&
ci->u_len == 0 &&
ci1->l_len == 1 && ci1->l_data[0] == code + 2 &&
ci1->f_len == 1 && ci1->f_data[0] == ci1->l_data[0] &&
ci1->u_len == 1 && ci1->u_data[0] == code &&
ci2->l_len == 0 &&
ci2->f_len == 0 &&
ci2->u_len == 1 && ci2->u_data[0] == code) {
te->len = 3;
te->data = 0;
te->type = RUN_TYPE_LSU;
return;
}
if (is_complicated_case(ci)) {
len = 1;
while (code + len <= CHARCODE_MAX) {
ci1 = &tab[code + len];
if (ci1->u_len != 1 ||
ci1->u_data[0] != ci->u_data[0] + len ||
ci1->l_len != 0 ||
ci1->f_len != 1 || ci1->f_data[0] != ci1->u_data[0])
break;
len++;
}
if (len > 1) {
te->len = len;
te->type = RUN_TYPE_UF;
te->data = ci->u_data[0];
return;
}
if (ci->l_len == 0 &&
ci->u_len == 2 && ci->u_data[1] == 0x399 &&
ci->f_len == 2 && ci->f_data[1] == 0x3B9 &&
ci->f_data[0] == simple_to_lower(tab, ci->u_data[0])) {
len = 1;
while (code + len <= CHARCODE_MAX) {
ci1 = &tab[code + len];
if (!(ci1->u_len == 2 &&
ci1->u_data[1] == ci->u_data[1] &&
ci1->u_data[0] == ci->u_data[0] + len &&
ci1->f_len == 2 &&
ci1->f_data[1] == ci->f_data[1] &&
ci1->f_data[0] == ci->f_data[0] + len &&
ci1->l_len == 0))
break;
len++;
}
te->len = len;
te->type = RUN_TYPE_UF_EXT2;
te->ext_data[0] = ci->u_data[0];
te->ext_data[1] = ci->u_data[1];
te->ext_len = 2;
return;
}
if (ci->u_len == 2 && ci->u_data[1] == 0x399 &&
ci->l_len == 1 &&
ci->f_len == 1 && ci->f_data[0] == ci->l_data[0]) {
len = 1;
while (code + len <= CHARCODE_MAX) {
ci1 = &tab[code + len];
if (!(ci1->u_len == 2 &&
ci1->u_data[1] == 0x399 &&
ci1->u_data[0] == ci->u_data[0] + len &&
ci1->l_len == 1 &&
ci1->l_data[0] == ci->l_data[0] + len &&
ci1->f_len == 1 && ci1->f_data[0] == ci1->l_data[0]))
break;
len++;
}
te->len = len;
te->type = RUN_TYPE_U2L_399_EXT2;
te->ext_data[0] = ci->u_data[0];
te->ext_data[1] = ci->l_data[0];
te->ext_len = 2;
return;
}
if (ci->l_len == 1 && ci->u_len == 0 && ci->f_len == 0) {
len = 1;
while (code + len <= CHARCODE_MAX) {
ci1 = &tab[code + len];
if (!(ci1->l_len == 1 &&
ci1->l_data[0] == ci->l_data[0] + len &&
ci1->u_len == 0 && ci1->f_len == 0))
break;
len++;
}
te->len = len;
te->type = RUN_TYPE_L;
te->data = ci->l_data[0];
return;
}
if (ci->l_len == 0 &&
ci->u_len == 1 &&
ci->u_data[0] < 0x1000 &&
ci->f_len == 1 && ci->f_data[0] == ci->u_data[0] + 0x20) {
te->len = 1;
te->type = RUN_TYPE_UF_D20;
te->data = ci->u_data[0];
} else if (ci->l_len == 0 &&
ci->u_len == 1 &&
ci->f_len == 1 && ci->f_data[0] == ci->u_data[0] + 1) {
te->len = 1;
te->type = RUN_TYPE_UF_D1_EXT;
te->ext_data[0] = ci->u_data[0];
te->ext_len = 1;
} else if (ci->l_len == 2 && ci->u_len == 0 && ci->f_len == 2 &&
ci->l_data[0] == ci->f_data[0] &&
ci->l_data[1] == ci->f_data[1]) {
te->len = 1;
te->type = RUN_TYPE_LF_EXT2;
te->ext_data[0] = ci->l_data[0];
te->ext_data[1] = ci->l_data[1];
te->ext_len = 2;
} else if (ci->u_len == 2 && ci->l_len == 0 && ci->f_len == 2 &&
ci->f_data[0] == simple_to_lower(tab, ci->u_data[0]) &&
ci->f_data[1] == simple_to_lower(tab, ci->u_data[1])) {
te->len = 1;
te->type = RUN_TYPE_UF_EXT2;
te->ext_data[0] = ci->u_data[0];
te->ext_data[1] = ci->u_data[1];
te->ext_len = 2;
} else if (ci->u_len == 3 && ci->l_len == 0 && ci->f_len == 3 &&
ci->f_data[0] == simple_to_lower(tab, ci->u_data[0]) &&
ci->f_data[1] == simple_to_lower(tab, ci->u_data[1]) &&
ci->f_data[2] == simple_to_lower(tab, ci->u_data[2])) {
te->len = 1;
te->type = RUN_TYPE_UF_EXT3;
te->ext_data[0] = ci->u_data[0];
te->ext_data[1] = ci->u_data[1];
te->ext_data[2] = ci->u_data[2];
te->ext_len = 3;
} else {
printf("unsupported encoding case:\n");
dump_cc_info(ci, code);
abort();
}
} else {
/* look for a run of identical conversions */
len = 0;
for(;;) {
if (code >= CHARCODE_MAX || len >= 126)
break;
ci = &tab[code + len];
ci1 = &tab[code + len + 1];
if (is_complicated_case(ci) || is_complicated_case(ci1)) {
break;
}
if (ci->l_len != 1 || ci->l_data[0] != code + len + 1)
break;
if (ci1->u_len != 1 || ci1->u_data[0] != code + len)
break;
len += 2;
}
if (len > 0) {
te->len = len;
te->type = RUN_TYPE_UL;
te->data = 0;
return;
}
ci = &tab[code];
is_lower = ci->l_len > 0;
len = 1;
while (code + len <= CHARCODE_MAX) {
ci1 = &tab[code + len];
if (is_complicated_case(ci1))
break;
if (is_lower) {
if (ci1->l_len != 1 ||
ci1->l_data[0] != ci->l_data[0] + len)
break;
} else {
if (ci1->u_len != 1 ||
ci1->u_data[0] != ci->u_data[0] + len)
break;
}
len++;
}
te->len = len;
if (is_lower) {
te->type = RUN_TYPE_LF;
te->data = ci->l_data[0];
} else {
te->type = RUN_TYPE_U;
te->data = ci->u_data[0];
}
}
}
TableEntry conv_table[1000];
int conv_table_len;
int ext_data[1000];
int ext_data_len;
void dump_case_conv_table1(void)
{
int i, j;
const TableEntry *te;
for(i = 0; i < conv_table_len; i++) {
te = &conv_table[i];
printf("%05x %02x %-10s %05x",
te->code, te->len, run_type_str[te->type], te->data);
for(j = 0; j < te->ext_len; j++) {
printf(" %05x", te->ext_data[j]);
}
printf("\n");
}
printf("table_len=%d ext_len=%d\n", conv_table_len, ext_data_len);
}
int find_data_index(const TableEntry *conv_table, int len, int data)
{
int i;
const TableEntry *te;
for(i = 0; i < len; i++) {
te = &conv_table[i];
if (te->code == data)
return i;
}
return -1;
}
int find_ext_data_index(int data)
{
int i;
for(i = 0; i < ext_data_len; i++) {
if (ext_data[i] == data)
return i;
}
assert(ext_data_len < countof(ext_data));
ext_data[ext_data_len++] = data;
return ext_data_len - 1;
}
void build_conv_table(CCInfo *tab)
{
int code, i, j;
CCInfo *ci;
TableEntry *te;
te = conv_table;
for(code = 0; code <= CHARCODE_MAX; code++) {
ci = &tab[code];
if (ci->u_len == 0 && ci->l_len == 0 && ci->f_len == 0)
continue;
assert(te - conv_table < countof(conv_table));
find_run_type(te, tab, code);
#if 0
if (te->type == RUN_TYPE_TODO) {
printf("TODO: ");
dump_cc_info(ci, code);
}
#endif
assert(te->len <= 127);
code += te->len - 1;
te++;
}
conv_table_len = te - conv_table;
/* find the data index */
for(i = 0; i < conv_table_len; i++) {
int data_index;
te = &conv_table[i];
switch(te->type) {
case RUN_TYPE_U:
case RUN_TYPE_L:
case RUN_TYPE_UF:
case RUN_TYPE_LF:
data_index = find_data_index(conv_table, conv_table_len, te->data);
if (data_index < 0) {
switch(te->type) {
case RUN_TYPE_U:
te->type = RUN_TYPE_U_EXT;
te->ext_len = 1;
te->ext_data[0] = te->data;
break;
case RUN_TYPE_LF:
te->type = RUN_TYPE_LF_EXT;
te->ext_len = 1;
te->ext_data[0] = te->data;
break;
default:
printf("%05x: index not found\n", te->code);
exit(1);
}
} else {
te->data_index = data_index;
}
break;
case RUN_TYPE_UF_D20:
te->data_index = te->data;
break;
}
}
/* find the data index for ext_data */
for(i = 0; i < conv_table_len; i++) {
te = &conv_table[i];
if (te->type == RUN_TYPE_UF_EXT3) {
int p, v;
v = 0;
for(j = 0; j < 3; j++) {
p = find_ext_data_index(te->ext_data[j]);
assert(p < 16);
v = (v << 4) | p;
}
te->data_index = v;
}
}
for(i = 0; i < conv_table_len; i++) {
te = &conv_table[i];
if (te->type == RUN_TYPE_LF_EXT2 ||
te->type == RUN_TYPE_UF_EXT2 ||
te->type == RUN_TYPE_U2L_399_EXT2) {
int p, v;
v = 0;
for(j = 0; j < 2; j++) {
p = find_ext_data_index(te->ext_data[j]);
assert(p < 64);
v = (v << 6) | p;
}
te->data_index = v;
}
}
for(i = 0; i < conv_table_len; i++) {
te = &conv_table[i];
if (te->type == RUN_TYPE_UF_D1_EXT ||
te->type == RUN_TYPE_U_EXT ||
te->type == RUN_TYPE_LF_EXT) {
te->data_index = find_ext_data_index(te->ext_data[0]);
}
}
#ifdef DUMP_CASE_CONV_TABLE
dump_case_conv_table1();
#endif
}
void dump_case_conv_table(FILE *f)
{
int i;
uint32_t v;
const TableEntry *te;
total_tables++;
total_table_bytes += conv_table_len * sizeof(uint32_t);
fprintf(f, "static const uint32_t case_conv_table1[%d] = {", conv_table_len);
for(i = 0; i < conv_table_len; i++) {
if (i % 4 == 0)
fprintf(f, "\n ");
te = &conv_table[i];
v = te->code << (32 - 17);
v |= te->len << (32 - 17 - 7);
v |= te->type << (32 - 17 - 7 - 4);
v |= te->data_index >> 8;
fprintf(f, " 0x%08x,", v);
}
fprintf(f, "\n};\n\n");
total_tables++;
total_table_bytes += conv_table_len;
fprintf(f, "static const uint8_t case_conv_table2[%d] = {", conv_table_len);
for(i = 0; i < conv_table_len; i++) {
if (i % 8 == 0)
fprintf(f, "\n ");
te = &conv_table[i];
fprintf(f, " 0x%02x,", te->data_index & 0xff);
}
fprintf(f, "\n};\n\n");
total_tables++;
total_table_bytes += ext_data_len * sizeof(uint16_t);
fprintf(f, "static const uint16_t case_conv_ext[%d] = {", ext_data_len);
for(i = 0; i < ext_data_len; i++) {
if (i % 8 == 0)
fprintf(f, "\n ");
fprintf(f, " 0x%04x,", ext_data[i]);
}
fprintf(f, "\n};\n\n");
}
static CCInfo *global_tab;
static int sp_cc_cmp(const void *p1, const void *p2)
{
CCInfo *c1 = &global_tab[*(const int *)p1];
CCInfo *c2 = &global_tab[*(const int *)p2];
if (c1->f_len < c2->f_len) {
return -1;
} else if (c2->f_len < c1->f_len) {
return 1;
} else {
return memcmp(c1->f_data, c2->f_data, sizeof(c1->f_data[0]) * c1->f_len);
}
}
/* dump the case special cases (multi character results which are
identical and need specific handling in lre_canonicalize() */
void dump_case_folding_special_cases(CCInfo *tab)
{
int i, len, j;
int *perm;
perm = malloc(sizeof(perm[0]) * (CHARCODE_MAX + 1));
for(i = 0; i <= CHARCODE_MAX; i++)
perm[i] = i;
global_tab = tab;
qsort(perm, CHARCODE_MAX + 1, sizeof(perm[0]), sp_cc_cmp);
for(i = 0; i <= CHARCODE_MAX;) {
if (tab[perm[i]].f_len <= 1) {
i++;
} else {
len = 1;
while ((i + len) <= CHARCODE_MAX && !sp_cc_cmp(&perm[i], &perm[i + len]))
len++;
if (len > 1) {
for(j = i; j < i + len; j++)
dump_cc_info(&tab[perm[j]], perm[j]);
}
i += len;
}
}
free(perm);
global_tab = NULL;
}
int tabcmp(const int *tab1, const int *tab2, int n)
{
int i;
for(i = 0; i < n; i++) {
if (tab1[i] != tab2[i])
return -1;
}
return 0;
}
void dump_str(const char *str, const int *buf, int len)
{
int i;
printf("%s=", str);
for(i = 0; i < len; i++)
printf(" %05x", buf[i]);
printf("\n");
}
void compute_internal_props(void)
{
int i;
BOOL has_ul;
for(i = 0; i <= CHARCODE_MAX; i++) {
CCInfo *ci = &unicode_db[i];
has_ul = (ci->u_len != 0 || ci->l_len != 0 || ci->f_len != 0);
if (has_ul) {
assert(get_prop(i, PROP_Cased));
} else {
set_prop(i, PROP_Cased1, get_prop(i, PROP_Cased));
}
set_prop(i, PROP_ID_Continue1,
get_prop(i, PROP_ID_Continue) & (get_prop(i, PROP_ID_Start) ^ 1));
set_prop(i, PROP_XID_Start1,
get_prop(i, PROP_ID_Start) ^ get_prop(i, PROP_XID_Start));
set_prop(i, PROP_XID_Continue1,
get_prop(i, PROP_ID_Continue) ^ get_prop(i, PROP_XID_Continue));
set_prop(i, PROP_Changes_When_Titlecased1,
get_prop(i, PROP_Changes_When_Titlecased) ^ (ci->u_len != 0));
set_prop(i, PROP_Changes_When_Casefolded1,
get_prop(i, PROP_Changes_When_Casefolded) ^ (ci->f_len != 0));
/* XXX: reduce table size (438 bytes) */
set_prop(i, PROP_Changes_When_NFKC_Casefolded1,
get_prop(i, PROP_Changes_When_NFKC_Casefolded) ^ (ci->f_len != 0));
#if 0
/* TEST */
#define M(x) (1U << GCAT_ ## x)
{
int b;
b = ((M(Mn) | M(Cf) | M(Lm) | M(Sk)) >>
unicode_db[i].general_category) & 1;
set_prop(i, PROP_Cased1,
get_prop(i, PROP_Case_Ignorable) ^ b);
}
#undef M
#endif
}
}
void dump_byte_table(FILE *f, const char *cname, const uint8_t *tab, int len)
{
int i;
total_tables++;
total_table_bytes += len;
fprintf(f, "static const uint8_t %s[%d] = {", cname, len);
for(i = 0; i < len; i++) {
if (i % 8 == 0)
fprintf(f, "\n ");
fprintf(f, " 0x%02x,", tab[i]);
}
fprintf(f, "\n};\n\n");
}
void dump_index_table(FILE *f, const char *cname, const uint8_t *tab, int len)
{
int i, code, offset;
total_index++;
total_index_bytes += len;
fprintf(f, "static const uint8_t %s[%d] = {\n", cname, len);
for(i = 0; i < len; i += 3) {
code = tab[i] + (tab[i+1] << 8) + ((tab[i+2] & 0x1f) << 16);
offset = ((i / 3) + 1) * 32 + (tab[i+2] >> 5);
fprintf(f, " 0x%02x, 0x%02x, 0x%02x,", tab[i], tab[i+1], tab[i+2]);
fprintf(f, " // %6.5X at %d%s\n", code, offset,
i == len - 3 ? " (upper bound)" : "");
}
fprintf(f, "};\n\n");
}
#define PROP_BLOCK_LEN 32
void build_prop_table(FILE *f, const char *name, int prop_index, BOOL add_index)
{
int i, j, n, v, offset, code;
DynBuf dbuf_s, *dbuf = &dbuf_s;
DynBuf dbuf1_s, *dbuf1 = &dbuf1_s;
DynBuf dbuf2_s, *dbuf2 = &dbuf2_s;
const uint32_t *buf;
int buf_len, block_end_pos, bit;
char cname[128];
dbuf_init(dbuf1);
for(i = 0; i <= CHARCODE_MAX;) {
v = get_prop(i, prop_index);
j = i + 1;
while (j <= CHARCODE_MAX && get_prop(j, prop_index) == v) {
j++;
}
n = j - i;
if (j == (CHARCODE_MAX + 1) && v == 0)
break; /* no need to encode last zero run */
//printf("%05x: %d %d\n", i, n, v);
dbuf_put_u32(dbuf1, n - 1);
i += n;
}
dbuf_init(dbuf);
dbuf_init(dbuf2);
buf = (uint32_t *)dbuf1->buf;
buf_len = dbuf1->size / sizeof(buf[0]);
/* the first value is assumed to be 0 */
assert(get_prop(0, prop_index) == 0);
block_end_pos = PROP_BLOCK_LEN;
i = 0;
code = 0;
bit = 0;
while (i < buf_len) {
if (add_index && dbuf->size >= block_end_pos && bit == 0) {
offset = (dbuf->size - block_end_pos);
/* XXX: offset could be larger in case of runs of small
lengths. Could add code to change the encoding to
prevent it at the expense of one byte loss */
assert(offset <= 7);
v = code | (offset << 21);
dbuf_putc(dbuf2, v);
dbuf_putc(dbuf2, v >> 8);
dbuf_putc(dbuf2, v >> 16);
block_end_pos += PROP_BLOCK_LEN;
}
/* Compressed byte encoding:
00..3F: 2 packed lengths: 3-bit + 3-bit
40..5F: 5-bits plus extra byte for length
60..7F: 5-bits plus 2 extra bytes for length
80..FF: 7-bit length
lengths must be incremented to get character count
Ranges alternate between false and true return value.
*/
v = buf[i];
code += v + 1;
bit ^= 1;
if (v < 8 && (i + 1) < buf_len && buf[i + 1] < 8) {
code += buf[i + 1] + 1;
bit ^= 1;
dbuf_putc(dbuf, (v << 3) | buf[i + 1]);
i += 2;
} else if (v < 128) {
dbuf_putc(dbuf, 0x80 + v);
i++;
} else if (v < (1 << 13)) {
dbuf_putc(dbuf, 0x40 + (v >> 8));
dbuf_putc(dbuf, v);
i++;
} else {
assert(v < (1 << 21));
dbuf_putc(dbuf, 0x60 + (v >> 16));
dbuf_putc(dbuf, v >> 8);
dbuf_putc(dbuf, v);
i++;
}
}
if (add_index) {
/* last index entry */
v = code;
dbuf_putc(dbuf2, v);
dbuf_putc(dbuf2, v >> 8);
dbuf_putc(dbuf2, v >> 16);
}
#ifdef DUMP_TABLE_SIZE
printf("prop %s: length=%d bytes\n", unicode_prop_name[prop_index],
(int)(dbuf->size + dbuf2->size));
#endif
snprintf(cname, sizeof(cname), "unicode_prop_%s_table", unicode_prop_name[prop_index]);
dump_byte_table(f, cname, dbuf->buf, dbuf->size);
if (add_index) {
snprintf(cname, sizeof(cname), "unicode_prop_%s_index", unicode_prop_name[prop_index]);
dump_index_table(f, cname, dbuf2->buf, dbuf2->size);
}
dbuf_free(dbuf);
dbuf_free(dbuf1);
dbuf_free(dbuf2);
}
void build_flags_tables(FILE *f)
{
build_prop_table(f, "Cased1", PROP_Cased1, TRUE);
build_prop_table(f, "Case_Ignorable", PROP_Case_Ignorable, TRUE);
build_prop_table(f, "ID_Start", PROP_ID_Start, TRUE);
build_prop_table(f, "ID_Continue1", PROP_ID_Continue1, TRUE);
}
void dump_name_table(FILE *f, const char *cname, const char **tab_name, int len,
const char **tab_short_name)
{
int i, w, maxw;
maxw = 0;
for(i = 0; i < len; i++) {
w = strlen(tab_name[i]);
if (tab_short_name[i][0] != '\0') {
w += 1 + strlen(tab_short_name[i]);
}
if (maxw < w)
maxw = w;
}
/* generate a sequence of strings terminated by an empty string */
fprintf(f, "static const char %s[] =\n", cname);
for(i = 0; i < len; i++) {
fprintf(f, " \"");
w = fprintf(f, "%s", tab_name[i]);
if (tab_short_name[i][0] != '\0') {
w += fprintf(f, ",%s", tab_short_name[i]);
}
fprintf(f, "\"%*s\"\\0\"\n", 1 + maxw - w, "");
}
fprintf(f, ";\n\n");
}
void build_general_category_table(FILE *f)
{
int i, v, j, n, n1;
DynBuf dbuf_s, *dbuf = &dbuf_s;
#ifdef DUMP_TABLE_SIZE
int cw_count, cw_len_count[4], cw_start;
#endif
fprintf(f, "typedef enum {\n");
for(i = 0; i < GCAT_COUNT; i++)
fprintf(f, " UNICODE_GC_%s,\n", unicode_gc_name[i]);
fprintf(f, " UNICODE_GC_COUNT,\n");
fprintf(f, "} UnicodeGCEnum;\n\n");
dump_name_table(f, "unicode_gc_name_table",
unicode_gc_name, GCAT_COUNT,
unicode_gc_short_name);
dbuf_init(dbuf);
#ifdef DUMP_TABLE_SIZE
cw_count = 0;
for(i = 0; i < 4; i++)
cw_len_count[i] = 0;
#endif
for(i = 0; i <= CHARCODE_MAX;) {
v = unicode_db[i].general_category;
j = i + 1;
while (j <= CHARCODE_MAX && unicode_db[j].general_category == v)
j++;
n = j - i;
/* compress Lu/Ll runs */
if (v == GCAT_Lu) {
n1 = 1;
while ((i + n1) <= CHARCODE_MAX && unicode_db[i + n1].general_category == (v + (n1 & 1))) {
n1++;
}
if (n1 > n) {
v = 31;
n = n1;
}
}
// printf("%05x %05x %d\n", i, n, v);
n--;
#ifdef DUMP_TABLE_SIZE
cw_count++;
cw_start = dbuf->size;
#endif
if (n < 7) {
dbuf_putc(dbuf, (n << 5) | v);
} else if (n < 7 + 128) {
n1 = n - 7;
assert(n1 < 128);
dbuf_putc(dbuf, (0xf << 5) | v);
dbuf_putc(dbuf, n1);
} else if (n < 7 + 128 + (1 << 14)) {
n1 = n - (7 + 128);
assert(n1 < (1 << 14));
dbuf_putc(dbuf, (0xf << 5) | v);
dbuf_putc(dbuf, (n1 >> 8) + 128);
dbuf_putc(dbuf, n1);
} else {
n1 = n - (7 + 128 + (1 << 14));
assert(n1 < (1 << 22));
dbuf_putc(dbuf, (0xf << 5) | v);
dbuf_putc(dbuf, (n1 >> 16) + 128 + 64);
dbuf_putc(dbuf, n1 >> 8);
dbuf_putc(dbuf, n1);
}
#ifdef DUMP_TABLE_SIZE
cw_len_count[dbuf->size - cw_start - 1]++;
#endif
i += n + 1;
}
#ifdef DUMP_TABLE_SIZE
printf("general category: %d entries [", cw_count);
for(i = 0; i < 4; i++)
printf(" %d", cw_len_count[i]);
printf(" ], length=%d bytes\n", (int)dbuf->size);
#endif
dump_byte_table(f, "unicode_gc_table", dbuf->buf, dbuf->size);
dbuf_free(dbuf);
}
void build_script_table(FILE *f)
{
int i, v, j, n, n1, type;
DynBuf dbuf_s, *dbuf = &dbuf_s;
#ifdef DUMP_TABLE_SIZE
int cw_count, cw_len_count[4], cw_start;
#endif
fprintf(f, "typedef enum {\n");
for(i = 0; i < SCRIPT_COUNT; i++)
fprintf(f, " UNICODE_SCRIPT_%s,\n", unicode_script_name[i]);
fprintf(f, " UNICODE_SCRIPT_COUNT,\n");
fprintf(f, "} UnicodeScriptEnum;\n\n");
i = 1;
dump_name_table(f, "unicode_script_name_table",
unicode_script_name + i, SCRIPT_COUNT - i,
unicode_script_short_name + i);
dbuf_init(dbuf);
#ifdef DUMP_TABLE_SIZE
cw_count = 0;
for(i = 0; i < 4; i++)
cw_len_count[i] = 0;
#endif
for(i = 0; i <= CHARCODE_MAX;) {
v = unicode_db[i].script;
j = i + 1;
while (j <= CHARCODE_MAX && unicode_db[j].script == v)
j++;
n = j - i;
if (v == 0 && j == (CHARCODE_MAX + 1))
break;
// printf("%05x %05x %d\n", i, n, v);
n--;
#ifdef DUMP_TABLE_SIZE
cw_count++;
cw_start = dbuf->size;
#endif
if (v == 0)
type = 0;
else
type = 1;
if (n < 96) {
dbuf_putc(dbuf, n | (type << 7));
} else if (n < 96 + (1 << 12)) {
n1 = n - 96;
assert(n1 < (1 << 12));
dbuf_putc(dbuf, ((n1 >> 8) + 96) | (type << 7));
dbuf_putc(dbuf, n1);
} else {
n1 = n - (96 + (1 << 12));
assert(n1 < (1 << 20));
dbuf_putc(dbuf, ((n1 >> 16) + 112) | (type << 7));
dbuf_putc(dbuf, n1 >> 8);
dbuf_putc(dbuf, n1);
}
if (type != 0)
dbuf_putc(dbuf, v);
#ifdef DUMP_TABLE_SIZE
cw_len_count[dbuf->size - cw_start - 1]++;
#endif
i += n + 1;
}
#ifdef DUMP_TABLE_SIZE
printf("script: %d entries [", cw_count);
for(i = 0; i < 4; i++)
printf(" %d", cw_len_count[i]);
printf(" ], length=%d bytes\n", (int)dbuf->size);
#endif
dump_byte_table(f, "unicode_script_table", dbuf->buf, dbuf->size);
dbuf_free(dbuf);
}
void build_script_ext_table(FILE *f)
{
int i, j, n, n1, script_ext_len;
DynBuf dbuf_s, *dbuf = &dbuf_s;
#if defined(DUMP_TABLE_SIZE)
int cw_count = 0;
#endif
dbuf_init(dbuf);
for(i = 0; i <= CHARCODE_MAX;) {
script_ext_len = unicode_db[i].script_ext_len;
j = i + 1;
while (j <= CHARCODE_MAX &&
unicode_db[j].script_ext_len == script_ext_len &&
!memcmp(unicode_db[j].script_ext, unicode_db[i].script_ext,
script_ext_len)) {
j++;
}
n = j - i;
#if defined(DUMP_TABLE_SIZE)
cw_count++;
#endif
n--;
if (n < 128) {
dbuf_putc(dbuf, n);
} else if (n < 128 + (1 << 14)) {
n1 = n - 128;
assert(n1 < (1 << 14));
dbuf_putc(dbuf, (n1 >> 8) + 128);
dbuf_putc(dbuf, n1);
} else {
n1 = n - (128 + (1 << 14));
assert(n1 < (1 << 22));
dbuf_putc(dbuf, (n1 >> 16) + 128 + 64);
dbuf_putc(dbuf, n1 >> 8);
dbuf_putc(dbuf, n1);
}
dbuf_putc(dbuf, script_ext_len);
for(j = 0; j < script_ext_len; j++)
dbuf_putc(dbuf, unicode_db[i].script_ext[j]);
i += n + 1;
}
#ifdef DUMP_TABLE_SIZE
printf("script_ext: %d entries", cw_count);
printf(", length=%d bytes\n", (int)dbuf->size);
#endif
dump_byte_table(f, "unicode_script_ext_table", dbuf->buf, dbuf->size);
dbuf_free(dbuf);
}
/* the following properties are synthetized so no table is necessary */
#define PROP_TABLE_COUNT PROP_ASCII
void build_prop_list_table(FILE *f)
{
int i;
for(i = 0; i < PROP_TABLE_COUNT; i++) {
if (i == PROP_ID_Start ||
i == PROP_Case_Ignorable ||
i == PROP_ID_Continue1) {
/* already generated */
} else {
build_prop_table(f, unicode_prop_name[i], i, FALSE);
}
}
fprintf(f, "typedef enum {\n");
for(i = 0; i < PROP_COUNT; i++)
fprintf(f, " UNICODE_PROP_%s,\n", unicode_prop_name[i]);
fprintf(f, " UNICODE_PROP_COUNT,\n");
fprintf(f, "} UnicodePropertyEnum;\n\n");
i = PROP_ASCII_Hex_Digit;
dump_name_table(f, "unicode_prop_name_table",
unicode_prop_name + i, PROP_XID_Start - i + 1,
unicode_prop_short_name + i);
fprintf(f, "static const uint8_t * const unicode_prop_table[] = {\n");
for(i = 0; i < PROP_TABLE_COUNT; i++) {
fprintf(f, " unicode_prop_%s_table,\n", unicode_prop_name[i]);
}
fprintf(f, "};\n\n");
fprintf(f, "static const uint16_t unicode_prop_len_table[] = {\n");
for(i = 0; i < PROP_TABLE_COUNT; i++) {
fprintf(f, " countof(unicode_prop_%s_table),\n", unicode_prop_name[i]);
}
fprintf(f, "};\n\n");
}
#ifdef USE_TEST
int check_conv(uint32_t *res, uint32_t c, int conv_type)
{
return lre_case_conv(res, c, conv_type);
}
void check_case_conv(void)
{
CCInfo *tab = unicode_db;
uint32_t res[3];
int l, error;
CCInfo ci_s, *ci1, *ci = &ci_s;
int code;
for(code = 0; code <= CHARCODE_MAX; code++) {
ci1 = &tab[code];
*ci = *ci1;
if (ci->l_len == 0) {
ci->l_len = 1;
ci->l_data[0] = code;
}
if (ci->u_len == 0) {
ci->u_len = 1;
ci->u_data[0] = code;
}
if (ci->f_len == 0) {
ci->f_len = 1;
ci->f_data[0] = code;
}
error = 0;
l = check_conv(res, code, 0);
if (l != ci->u_len || tabcmp((int *)res, ci->u_data, l)) {
printf("ERROR: L\n");
error++;
}
l = check_conv(res, code, 1);
if (l != ci->l_len || tabcmp((int *)res, ci->l_data, l)) {
printf("ERROR: U\n");
error++;
}
l = check_conv(res, code, 2);
if (l != ci->f_len || tabcmp((int *)res, ci->f_data, l)) {
printf("ERROR: F\n");
error++;
}
if (error) {
dump_cc_info(ci, code);
exit(1);
}
}
}
#ifdef PROFILE
static int64_t get_time_ns(void)
{
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (int64_t)ts.tv_sec * 1000000000 + ts.tv_nsec;
}
#endif
void check_flags(void)
{
int c;
BOOL flag_ref, flag;
for(c = 0; c <= CHARCODE_MAX; c++) {
flag_ref = get_prop(c, PROP_Cased);
flag = !!lre_is_cased(c);
if (flag != flag_ref) {
printf("ERROR: c=%05x cased=%d ref=%d\n",
c, flag, flag_ref);
exit(1);
}
flag_ref = get_prop(c, PROP_Case_Ignorable);
flag = !!lre_is_case_ignorable(c);
if (flag != flag_ref) {
printf("ERROR: c=%05x case_ignorable=%d ref=%d\n",
c, flag, flag_ref);
exit(1);
}
flag_ref = get_prop(c, PROP_ID_Start);
flag = !!lre_is_id_start(c);
if (flag != flag_ref) {
printf("ERROR: c=%05x id_start=%d ref=%d\n",
c, flag, flag_ref);
exit(1);
}
flag_ref = get_prop(c, PROP_ID_Continue);
flag = !!lre_is_id_continue(c);
if (flag != flag_ref) {
printf("ERROR: c=%05x id_cont=%d ref=%d\n",
c, flag, flag_ref);
exit(1);
}
}
#ifdef PROFILE
{
int64_t ti, count;
ti = get_time_ns();
count = 0;
for(c = 0x20; c <= 0xffff; c++) {
flag_ref = get_prop(c, PROP_ID_Start);
flag = !!lre_is_id_start(c);
assert(flag == flag_ref);
count++;
}
ti = get_time_ns() - ti;
printf("flags time=%0.1f ns/char\n",
(double)ti / count);
}
#endif
}
#endif
#define CC_BLOCK_LEN 32
void build_cc_table(FILE *f)
{
// Compress combining class table
// see: https://www.unicode.org/reports/tr44/#Canonical_Combining_Class_Values
int i, cc, n, type, n1, block_end_pos;
DynBuf dbuf_s, *dbuf = &dbuf_s;
DynBuf dbuf1_s, *dbuf1 = &dbuf1_s;
#if defined(DUMP_CC_TABLE) || defined(DUMP_TABLE_SIZE)
int cw_len_tab[3], cw_start, cc_table_len;
#endif
uint32_t v;
dbuf_init(dbuf);
dbuf_init(dbuf1);
#if defined(DUMP_CC_TABLE) || defined(DUMP_TABLE_SIZE)
cc_table_len = 0;
for(i = 0; i < countof(cw_len_tab); i++)
cw_len_tab[i] = 0;
#endif
block_end_pos = CC_BLOCK_LEN;
for(i = 0; i <= CHARCODE_MAX;) {
cc = unicode_db[i].combining_class;
assert(cc <= 255);
/* check increasing values */
n = 1;
while ((i + n) <= CHARCODE_MAX &&
unicode_db[i + n].combining_class == (cc + n))
n++;
if (n >= 2) {
type = 1;
} else {
type = 0;
n = 1;
while ((i + n) <= CHARCODE_MAX &&
unicode_db[i + n].combining_class == cc)
n++;
}
/* no need to encode the last run */
if (cc == 0 && (i + n - 1) == CHARCODE_MAX)
break;
#ifdef DUMP_CC_TABLE
printf("%05x %6d %d %d\n", i, n, type, cc);
#endif
if (type == 0) {
if (cc == 0)
type = 2;
else if (cc == 230)
type = 3;
}
n1 = n - 1;
/* add an entry to the index if necessary */
if (dbuf->size >= block_end_pos) {
v = i | ((dbuf->size - block_end_pos) << 21);
dbuf_putc(dbuf1, v);
dbuf_putc(dbuf1, v >> 8);
dbuf_putc(dbuf1, v >> 16);
block_end_pos += CC_BLOCK_LEN;
}
#if defined(DUMP_CC_TABLE) || defined(DUMP_TABLE_SIZE)
cw_start = dbuf->size;
#endif
/* Compressed run length encoding:
- 2 high order bits are combining class type
- 0:0, 1:230, 2:extra byte linear progression, 3:extra byte
- 00..2F: range length (add 1)
- 30..37: 3-bit range-length + 1 extra byte
- 38..3F: 3-bit range-length + 2 extra byte
*/
if (n1 < 48) {
dbuf_putc(dbuf, n1 | (type << 6));
} else if (n1 < 48 + (1 << 11)) {
n1 -= 48;
dbuf_putc(dbuf, ((n1 >> 8) + 48) | (type << 6));
dbuf_putc(dbuf, n1);
} else {
n1 -= 48 + (1 << 11);
assert(n1 < (1 << 20));
dbuf_putc(dbuf, ((n1 >> 16) + 56) | (type << 6));
dbuf_putc(dbuf, n1 >> 8);
dbuf_putc(dbuf, n1);
}
#if defined(DUMP_CC_TABLE) || defined(DUMP_TABLE_SIZE)
cw_len_tab[dbuf->size - cw_start - 1]++;
cc_table_len++;
#endif
if (type == 0 || type == 1)
dbuf_putc(dbuf, cc);
i += n;
}
/* last index entry */
v = i;
dbuf_putc(dbuf1, v);
dbuf_putc(dbuf1, v >> 8);
dbuf_putc(dbuf1, v >> 16);
dump_byte_table(f, "unicode_cc_table", dbuf->buf, dbuf->size);
dump_index_table(f, "unicode_cc_index", dbuf1->buf, dbuf1->size);
#if defined(DUMP_CC_TABLE) || defined(DUMP_TABLE_SIZE)
printf("CC table: size=%d (%d entries) [",
(int)(dbuf->size + dbuf1->size),
cc_table_len);
for(i = 0; i < countof(cw_len_tab); i++)
printf(" %d", cw_len_tab[i]);
printf(" ]\n");
#endif
dbuf_free(dbuf);
dbuf_free(dbuf1);
}
/* maximum length of decomposition: 18 chars (1), then 8 */
#ifndef USE_TEST
typedef enum {
DECOMP_TYPE_C1, /* 16 bit char */
DECOMP_TYPE_L1, /* 16 bit char table */
DECOMP_TYPE_L2,
DECOMP_TYPE_L3,
DECOMP_TYPE_L4,
DECOMP_TYPE_L5, /* XXX: not used */
DECOMP_TYPE_L6, /* XXX: could remove */
DECOMP_TYPE_L7, /* XXX: could remove */
DECOMP_TYPE_LL1, /* 18 bit char table */
DECOMP_TYPE_LL2,
DECOMP_TYPE_S1, /* 8 bit char table */
DECOMP_TYPE_S2,
DECOMP_TYPE_S3,
DECOMP_TYPE_S4,
DECOMP_TYPE_S5,
DECOMP_TYPE_I1, /* increment 16 bit char value */
DECOMP_TYPE_I2_0,
DECOMP_TYPE_I2_1,
DECOMP_TYPE_I3_1,
DECOMP_TYPE_I3_2,
DECOMP_TYPE_I4_1,
DECOMP_TYPE_I4_2,
DECOMP_TYPE_B1, /* 16 bit base + 8 bit offset */
DECOMP_TYPE_B2,
DECOMP_TYPE_B3,
DECOMP_TYPE_B4,
DECOMP_TYPE_B5,
DECOMP_TYPE_B6,
DECOMP_TYPE_B7,
DECOMP_TYPE_B8,
DECOMP_TYPE_B18,
DECOMP_TYPE_LS2,
DECOMP_TYPE_PAT3,
DECOMP_TYPE_S2_UL,
DECOMP_TYPE_LS2_UL,
} DecompTypeEnum;
#endif
const char *decomp_type_str[] = {
"C1",
"L1",
"L2",
"L3",
"L4",
"L5",
"L6",
"L7",
"LL1",
"LL2",
"S1",
"S2",
"S3",
"S4",
"S5",
"I1",
"I2_0",
"I2_1",
"I3_1",
"I3_2",
"I4_1",
"I4_2",
"B1",
"B2",
"B3",
"B4",
"B5",
"B6",
"B7",
"B8",
"B18",
"LS2",
"PAT3",
"S2_UL",
"LS2_UL",
};
const int decomp_incr_tab[4][4] = {
{ DECOMP_TYPE_I1, 0, -1 },
{ DECOMP_TYPE_I2_0, 0, 1, -1 },
{ DECOMP_TYPE_I3_1, 1, 2, -1 },
{ DECOMP_TYPE_I4_1, 1, 2, -1 },
};
/*
entry size:
type bits
code 18
len 7
compat 1
type 5
index 16
total 47
*/
typedef struct {
int code;
uint8_t len;
uint8_t type;
uint8_t c_len;
uint16_t c_min;
uint16_t data_index;
int cost; /* size in bytes from this entry to the end */
} DecompEntry;
int get_decomp_run_size(const DecompEntry *de)
{
int s;
s = 6;
if (de->type <= DECOMP_TYPE_C1) {
/* nothing more */
} else if (de->type <= DECOMP_TYPE_L7) {
s += de->len * de->c_len * 2;
} else if (de->type <= DECOMP_TYPE_LL2) {
/* 18 bits per char */
s += (de->len * de->c_len * 18 + 7) / 8;
} else if (de->type <= DECOMP_TYPE_S5) {
s += de->len * de->c_len;
} else if (de->type <= DECOMP_TYPE_I4_2) {
s += de->c_len * 2;
} else if (de->type <= DECOMP_TYPE_B18) {
s += 2 + de->len * de->c_len;
} else if (de->type <= DECOMP_TYPE_LS2) {
s += de->len * 3;
} else if (de->type <= DECOMP_TYPE_PAT3) {
s += 4 + de->len * 2;
} else if (de->type <= DECOMP_TYPE_S2_UL) {
s += de->len;
} else if (de->type <= DECOMP_TYPE_LS2_UL) {
s += (de->len / 2) * 3;
} else {
abort();
}
return s;
}
static const uint16_t unicode_short_table[2] = { 0x2044, 0x2215 };
/* return -1 if not found */
int get_short_code(int c)
{
int i;
if (c < 0x80) {
return c;
} else if (c >= 0x300 && c < 0x350) {
return c - 0x300 + 0x80;
} else {
for(i = 0; i < countof(unicode_short_table); i++) {
if (c == unicode_short_table[i])
return i + 0x80 + 0x50;
}
return -1;
}
}
static BOOL is_short(int code)
{
return get_short_code(code) >= 0;
}
static BOOL is_short_tab(const int *tab, int len)
{
int i;
for(i = 0; i < len; i++) {
if (!is_short(tab[i]))
return FALSE;
}
return TRUE;
}
static BOOL is_16bit(const int *tab, int len)
{
int i;
for(i = 0; i < len; i++) {
if (tab[i] > 0xffff)
return FALSE;
}
return TRUE;
}
static uint32_t to_lower_simple(uint32_t c)
{
/* Latin1 and Cyrillic */
if (c < 0x100 || (c >= 0x410 && c <= 0x42f))
c += 0x20;
else
c++;
return c;
}
/* select best encoding with dynamic programming */
void find_decomp_run(DecompEntry *tab_de, int i)
{
DecompEntry de_s, *de = &de_s;
CCInfo *ci, *ci1, *ci2;
int l, j, n, len_max;
ci = &unicode_db[i];
l = ci->decomp_len;
if (l == 0) {
tab_de[i].cost = tab_de[i + 1].cost;
return;
}
/* the offset for the compose table has only 6 bits, so we must
limit if it can be used by the compose table */
if (!ci->is_compat && !ci->is_excluded && l == 2)
len_max = 64;
else
len_max = 127;
tab_de[i].cost = 0x7fffffff;
if (!is_16bit(ci->decomp_data, l)) {
assert(l <= 2);
n = 1;
for(;;) {
de->code = i;
de->len = n;
de->type = DECOMP_TYPE_LL1 + l - 1;
de->c_len = l;
de->cost = get_decomp_run_size(de) + tab_de[i + n].cost;
if (de->cost < tab_de[i].cost) {
tab_de[i] = *de;
}
if (!((i + n) <= CHARCODE_MAX && n < len_max))
break;
ci1 = &unicode_db[i + n];
/* Note: we accept a hole */
if (!(ci1->decomp_len == 0 ||
(ci1->decomp_len == l &&
ci1->is_compat == ci->is_compat)))
break;
n++;
}
return;
}
if (l <= 7) {
n = 1;
for(;;) {
de->code = i;
de->len = n;
if (l == 1 && n == 1) {
de->type = DECOMP_TYPE_C1;
} else {
assert(l <= 8);
de->type = DECOMP_TYPE_L1 + l - 1;
}
de->c_len = l;
de->cost = get_decomp_run_size(de) + tab_de[i + n].cost;
if (de->cost < tab_de[i].cost) {
tab_de[i] = *de;
}
if (!((i + n) <= CHARCODE_MAX && n < len_max))
break;
ci1 = &unicode_db[i + n];
/* Note: we accept a hole */
if (!(ci1->decomp_len == 0 ||
(ci1->decomp_len == l &&
ci1->is_compat == ci->is_compat &&
is_16bit(ci1->decomp_data, l))))
break;
n++;
}
}
if (l <= 8 || l == 18) {
int c_min, c_max, c;
c_min = c_max = -1;
n = 1;
for(;;) {
ci1 = &unicode_db[i + n - 1];
for(j = 0; j < l; j++) {
c = ci1->decomp_data[j];
if (c == 0x20) {
/* we accept space for Arabic */
} else if (c_min == -1) {
c_min = c_max = c;
} else {
c_min = min_int(c_min, c);
c_max = max_int(c_max, c);
}
}
if ((c_max - c_min) > 254)
break;
de->code = i;
de->len = n;
if (l == 18)
de->type = DECOMP_TYPE_B18;
else
de->type = DECOMP_TYPE_B1 + l - 1;
de->c_len = l;
de->c_min = c_min;
de->cost = get_decomp_run_size(de) + tab_de[i + n].cost;
if (de->cost < tab_de[i].cost) {
tab_de[i] = *de;
}
if (!((i + n) <= CHARCODE_MAX && n < len_max))
break;
ci1 = &unicode_db[i + n];
if (!(ci1->decomp_len == l &&
ci1->is_compat == ci->is_compat))
break;
n++;
}
}
/* find an ascii run */
if (l <= 5 && is_short_tab(ci->decomp_data, l)) {
n = 1;
for(;;) {
de->code = i;
de->len = n;
de->type = DECOMP_TYPE_S1 + l - 1;
de->c_len = l;
de->cost = get_decomp_run_size(de) + tab_de[i + n].cost;
if (de->cost < tab_de[i].cost) {
tab_de[i] = *de;
}
if (!((i + n) <= CHARCODE_MAX && n < len_max))
break;
ci1 = &unicode_db[i + n];
/* Note: we accept a hole */
if (!(ci1->decomp_len == 0 ||
(ci1->decomp_len == l &&
ci1->is_compat == ci->is_compat &&
is_short_tab(ci1->decomp_data, l))))
break;
n++;
}
}
/* check if a single char is increasing */
if (l <= 4) {
int idx1, idx;
for(idx1 = 1; (idx = decomp_incr_tab[l - 1][idx1]) >= 0; idx1++) {
n = 1;
for(;;) {
de->code = i;
de->len = n;
de->type = decomp_incr_tab[l - 1][0] + idx1 - 1;
de->c_len = l;
de->cost = get_decomp_run_size(de) + tab_de[i + n].cost;
if (de->cost < tab_de[i].cost) {
tab_de[i] = *de;
}
if (!((i + n) <= CHARCODE_MAX && n < len_max))
break;
ci1 = &unicode_db[i + n];
if (!(ci1->decomp_len == l &&
ci1->is_compat == ci->is_compat))
goto next1;
for(j = 0; j < l; j++) {
if (j == idx) {
if (ci1->decomp_data[j] != ci->decomp_data[j] + n)
goto next1;
} else {
if (ci1->decomp_data[j] != ci->decomp_data[j])
goto next1;
}
}
n++;
}
next1: ;
}
}
if (l == 3) {
n = 1;
for(;;) {
de->code = i;
de->len = n;
de->type = DECOMP_TYPE_PAT3;
de->c_len = l;
de->cost = get_decomp_run_size(de) + tab_de[i + n].cost;
if (de->cost < tab_de[i].cost) {
tab_de[i] = *de;
}
if (!((i + n) <= CHARCODE_MAX && n < len_max))
break;
ci1 = &unicode_db[i + n];
if (!(ci1->decomp_len == l &&
ci1->is_compat == ci->is_compat &&
ci1->decomp_data[1] <= 0xffff &&
ci1->decomp_data[0] == ci->decomp_data[0] &&
ci1->decomp_data[l - 1] == ci->decomp_data[l - 1]))
break;
n++;
}
}
if (l == 2 && is_short(ci->decomp_data[1])) {
n = 1;
for(;;) {
de->code = i;
de->len = n;
de->type = DECOMP_TYPE_LS2;
de->c_len = l;
de->cost = get_decomp_run_size(de) + tab_de[i + n].cost;
if (de->cost < tab_de[i].cost) {
tab_de[i] = *de;
}
if (!((i + n) <= CHARCODE_MAX && n < len_max))
break;
ci1 = &unicode_db[i + n];
if (!(ci1->decomp_len == 0 ||
(ci1->decomp_len == l &&
ci1->is_compat == ci->is_compat &&
ci1->decomp_data[0] <= 0xffff &&
is_short(ci1->decomp_data[1]))))
break;
n++;
}
}
if (l == 2) {
BOOL is_16bit;
n = 0;
is_16bit = FALSE;
for(;;) {
if (!((i + n + 1) <= CHARCODE_MAX && n + 2 <= len_max))
break;
ci1 = &unicode_db[i + n];
if (!(ci1->decomp_len == l &&
ci1->is_compat == ci->is_compat &&
is_short(ci1->decomp_data[1])))
break;
if (!is_16bit && !is_short(ci1->decomp_data[0]))
is_16bit = TRUE;
ci2 = &unicode_db[i + n + 1];
if (!(ci2->decomp_len == l &&
ci2->is_compat == ci->is_compat &&
ci2->decomp_data[0] == to_lower_simple(ci1->decomp_data[0]) &&
ci2->decomp_data[1] == ci1->decomp_data[1]))
break;
n += 2;
de->code = i;
de->len = n;
de->type = DECOMP_TYPE_S2_UL + is_16bit;
de->c_len = l;
de->cost = get_decomp_run_size(de) + tab_de[i + n].cost;
if (de->cost < tab_de[i].cost) {
tab_de[i] = *de;
}
}
}
}
void put16(uint8_t *data_buf, int *pidx, uint16_t c)
{
int idx;
idx = *pidx;
data_buf[idx++] = c;
data_buf[idx++] = c >> 8;
*pidx = idx;
}
void add_decomp_data(uint8_t *data_buf, int *pidx, DecompEntry *de)
{
int i, j, idx, c;
CCInfo *ci;
idx = *pidx;
de->data_index = idx;
if (de->type <= DECOMP_TYPE_C1) {
ci = &unicode_db[de->code];
assert(ci->decomp_len == 1);
de->data_index = ci->decomp_data[0];
} else if (de->type <= DECOMP_TYPE_L7) {
for(i = 0; i < de->len; i++) {
ci = &unicode_db[de->code + i];
for(j = 0; j < de->c_len; j++) {
if (ci->decomp_len == 0)
c = 0;
else
c = ci->decomp_data[j];
put16(data_buf, &idx, c);
}
}
} else if (de->type <= DECOMP_TYPE_LL2) {
int n, p, k;
n = (de->len * de->c_len * 18 + 7) / 8;
p = de->len * de->c_len * 2;
memset(data_buf + idx, 0, n);
k = 0;
for(i = 0; i < de->len; i++) {
ci = &unicode_db[de->code + i];
for(j = 0; j < de->c_len; j++) {
if (ci->decomp_len == 0)
c = 0;
else
c = ci->decomp_data[j];
data_buf[idx + k * 2] = c;
data_buf[idx + k * 2 + 1] = c >> 8;
data_buf[idx + p + (k / 4)] |= (c >> 16) << ((k % 4) * 2);
k++;
}
}
idx += n;
} else if (de->type <= DECOMP_TYPE_S5) {
for(i = 0; i < de->len; i++) {
ci = &unicode_db[de->code + i];
for(j = 0; j < de->c_len; j++) {
if (ci->decomp_len == 0)
c = 0;
else
c = ci->decomp_data[j];
c = get_short_code(c);
assert(c >= 0);
data_buf[idx++] = c;
}
}
} else if (de->type <= DECOMP_TYPE_I4_2) {
ci = &unicode_db[de->code];
assert(ci->decomp_len == de->c_len);
for(j = 0; j < de->c_len; j++)
put16(data_buf, &idx, ci->decomp_data[j]);
} else if (de->type <= DECOMP_TYPE_B18) {
c = de->c_min;
data_buf[idx++] = c;
data_buf[idx++] = c >> 8;
for(i = 0; i < de->len; i++) {
ci = &unicode_db[de->code + i];
for(j = 0; j < de->c_len; j++) {
assert(ci->decomp_len == de->c_len);
c = ci->decomp_data[j];
if (c == 0x20) {
c = 0xff;
} else {
c -= de->c_min;
assert((uint32_t)c <= 254);
}
data_buf[idx++] = c;
}
}
} else if (de->type <= DECOMP_TYPE_LS2) {
assert(de->c_len == 2);
for(i = 0; i < de->len; i++) {
ci = &unicode_db[de->code + i];
if (ci->decomp_len == 0)
c = 0;
else
c = ci->decomp_data[0];
put16(data_buf, &idx, c);
if (ci->decomp_len == 0)
c = 0;
else
c = ci->decomp_data[1];
c = get_short_code(c);
assert(c >= 0);
data_buf[idx++] = c;
}
} else if (de->type <= DECOMP_TYPE_PAT3) {
ci = &unicode_db[de->code];
assert(ci->decomp_len == 3);
put16(data_buf, &idx, ci->decomp_data[0]);
put16(data_buf, &idx, ci->decomp_data[2]);
for(i = 0; i < de->len; i++) {
ci = &unicode_db[de->code + i];
assert(ci->decomp_len == 3);
put16(data_buf, &idx, ci->decomp_data[1]);
}
} else if (de->type <= DECOMP_TYPE_S2_UL) {
for(i = 0; i < de->len; i += 2) {
ci = &unicode_db[de->code + i];
c = ci->decomp_data[0];
c = get_short_code(c);
assert(c >= 0);
data_buf[idx++] = c;
c = ci->decomp_data[1];
c = get_short_code(c);
assert(c >= 0);
data_buf[idx++] = c;
}
} else if (de->type <= DECOMP_TYPE_LS2_UL) {
for(i = 0; i < de->len; i += 2) {
ci = &unicode_db[de->code + i];
c = ci->decomp_data[0];
put16(data_buf, &idx, c);
c = ci->decomp_data[1];
c = get_short_code(c);
assert(c >= 0);
data_buf[idx++] = c;
}
} else {
abort();
}
*pidx = idx;
}
#if 0
void dump_large_char(void)
{
int i, j;
for(i = 0; i <= CHARCODE_MAX; i++) {
CCInfo *ci = &unicode_db[i];
for(j = 0; j < ci->decomp_len; j++) {
if (ci->decomp_data[j] > 0xffff)
printf("%05x\n", ci->decomp_data[j]);
}
}
}
#endif
void build_compose_table(FILE *f, const DecompEntry *tab_de);
void build_decompose_table(FILE *f)
{
int i, array_len, code_max, data_len, count;
DecompEntry *tab_de, de_s, *de = &de_s;
uint8_t *data_buf;
code_max = CHARCODE_MAX;
tab_de = mallocz((code_max + 2) * sizeof(*tab_de));
for(i = code_max; i >= 0; i--) {
find_decomp_run(tab_de, i);
}
/* build the data buffer */
data_buf = malloc(100000);
data_len = 0;
array_len = 0;
for(i = 0; i <= code_max; i++) {
de = &tab_de[i];
if (de->len != 0) {
add_decomp_data(data_buf, &data_len, de);
i += de->len - 1;
array_len++;
}
}
#ifdef DUMP_DECOMP_TABLE
/* dump */
{
int size, size1;
printf("START LEN TYPE L C SIZE\n");
size = 0;
for(i = 0; i <= code_max; i++) {
de = &tab_de[i];
if (de->len != 0) {
size1 = get_decomp_run_size(de);
printf("%05x %3d %6s %2d %1d %4d\n", i, de->len,
decomp_type_str[de->type], de->c_len,
unicode_db[i].is_compat, size1);
i += de->len - 1;
size += size1;
}
}
printf("array_len=%d estimated size=%d bytes actual=%d bytes\n",
array_len, size, array_len * 6 + data_len);
}
#endif
total_tables++;
total_table_bytes += array_len * sizeof(uint32_t);
fprintf(f, "static const uint32_t unicode_decomp_table1[%d] = {", array_len);
count = 0;
for(i = 0; i <= code_max; i++) {
de = &tab_de[i];
if (de->len != 0) {
uint32_t v;
if (count++ % 4 == 0)
fprintf(f, "\n ");
v = (de->code << (32 - 18)) |
(de->len << (32 - 18 - 7)) |
(de->type << (32 - 18 - 7 - 6)) |
unicode_db[de->code].is_compat;
fprintf(f, " 0x%08x,", v);
i += de->len - 1;
}
}
fprintf(f, "\n};\n\n");
total_tables++;
total_table_bytes += array_len * sizeof(uint16_t);
fprintf(f, "static const uint16_t unicode_decomp_table2[%d] = {", array_len);
count = 0;
for(i = 0; i <= code_max; i++) {
de = &tab_de[i];
if (de->len != 0) {
if (count++ % 8 == 0)
fprintf(f, "\n ");
fprintf(f, " 0x%04x,", de->data_index);
i += de->len - 1;
}
}
fprintf(f, "\n};\n\n");
total_tables++;
total_table_bytes += data_len;
fprintf(f, "static const uint8_t unicode_decomp_data[%d] = {", data_len);
for(i = 0; i < data_len; i++) {
if (i % 8 == 0)
fprintf(f, "\n ");
fprintf(f, " 0x%02x,", data_buf[i]);
}
fprintf(f, "\n};\n\n");
build_compose_table(f, tab_de);
free(data_buf);
free(tab_de);
}
typedef struct {
uint32_t c[2];
uint32_t p;
} ComposeEntry;
#define COMPOSE_LEN_MAX 10000
static int ce_cmp(const void *p1, const void *p2)
{
const ComposeEntry *ce1 = p1;
const ComposeEntry *ce2 = p2;
int i;
for(i = 0; i < 2; i++) {
if (ce1->c[i] < ce2->c[i])
return -1;
else if (ce1->c[i] > ce2->c[i])
return 1;
}
return 0;
}
static int get_decomp_pos(const DecompEntry *tab_de, int c)
{
int i, v, k;
const DecompEntry *de;
k = 0;
for(i = 0; i <= CHARCODE_MAX; i++) {
de = &tab_de[i];
if (de->len != 0) {
if (c >= de->code && c < de->code + de->len) {
v = c - de->code;
assert(v < 64);
v |= k << 6;
assert(v < 65536);
return v;
}
i += de->len - 1;
k++;
}
}
return -1;
}
void build_compose_table(FILE *f, const DecompEntry *tab_de)
{
int i, v, tab_ce_len;
ComposeEntry *ce, *tab_ce;
tab_ce = malloc(sizeof(*tab_ce) * COMPOSE_LEN_MAX);
tab_ce_len = 0;
for(i = 0; i <= CHARCODE_MAX; i++) {
CCInfo *ci = &unicode_db[i];
if (ci->decomp_len == 2 && !ci->is_compat &&
!ci->is_excluded) {
assert(tab_ce_len < COMPOSE_LEN_MAX);
ce = &tab_ce[tab_ce_len++];
ce->c[0] = ci->decomp_data[0];
ce->c[1] = ci->decomp_data[1];
ce->p = i;
}
}
qsort(tab_ce, tab_ce_len, sizeof(*tab_ce), ce_cmp);
#if 0
{
printf("tab_ce_len=%d\n", tab_ce_len);
for(i = 0; i < tab_ce_len; i++) {
ce = &tab_ce[i];
printf("%05x %05x %05x\n", ce->c[0], ce->c[1], ce->p);
}
}
#endif
total_tables++;
total_table_bytes += tab_ce_len * sizeof(uint16_t);
fprintf(f, "static const uint16_t unicode_comp_table[%u] = {", tab_ce_len);
for(i = 0; i < tab_ce_len; i++) {
if (i % 8 == 0)
fprintf(f, "\n ");
v = get_decomp_pos(tab_de, tab_ce[i].p);
if (v < 0) {
printf("ERROR: entry for c=%04x not found\n",
tab_ce[i].p);
exit(1);
}
fprintf(f, " 0x%04x,", v);
}
fprintf(f, "\n};\n\n");
free(tab_ce);
}
#ifdef USE_TEST
void check_decompose_table(void)
{
int c;
CCInfo *ci;
int res[UNICODE_DECOMP_LEN_MAX], *ref;
int len, ref_len, is_compat;
for(is_compat = 0; is_compat <= 1; is_compat++) {
for(c = 0; c < CHARCODE_MAX; c++) {
ci = &unicode_db[c];
ref_len = ci->decomp_len;
ref = ci->decomp_data;
if (!is_compat && ci->is_compat) {
ref_len = 0;
}
len = unicode_decomp_char((uint32_t *)res, c, is_compat);
if (len != ref_len ||
tabcmp(res, ref, ref_len) != 0) {
printf("ERROR c=%05x compat=%d\n", c, is_compat);
dump_str("res", res, len);
dump_str("ref", ref, ref_len);
exit(1);
}
}
}
}
void check_compose_table(void)
{
int i, p;
/* XXX: we don't test all the cases */
for(i = 0; i <= CHARCODE_MAX; i++) {
CCInfo *ci = &unicode_db[i];
if (ci->decomp_len == 2 && !ci->is_compat &&
!ci->is_excluded) {
p = unicode_compose_pair(ci->decomp_data[0], ci->decomp_data[1]);
if (p != i) {
printf("ERROR compose: c=%05x %05x -> %05x ref=%05x\n",
ci->decomp_data[0], ci->decomp_data[1], p, i);
exit(1);
}
}
}
}
#endif
#ifdef USE_TEST
void check_str(const char *msg, int num, const int *in_buf, int in_len,
const int *buf1, int len1,
const int *buf2, int len2)
{
if (len1 != len2 || tabcmp(buf1, buf2, len1) != 0) {
printf("%d: ERROR %s:\n", num, msg);
dump_str(" in", in_buf, in_len);
dump_str("res", buf1, len1);
dump_str("ref", buf2, len2);
exit(1);
}
}
void check_cc_table(void)
{
int cc, cc_ref, c;
for(c = 0; c <= CHARCODE_MAX; c++) {
cc_ref = unicode_db[c].combining_class;
cc = unicode_get_cc(c);
if (cc != cc_ref) {
printf("ERROR: c=%04x cc=%d cc_ref=%d\n",
c, cc, cc_ref);
exit(1);
}
}
#ifdef PROFILE
{
int64_t ti, count;
ti = get_time_ns();
count = 0;
/* only do it on meaningful chars */
for(c = 0x20; c <= 0xffff; c++) {
cc_ref = unicode_db[c].combining_class;
cc = unicode_get_cc(c);
count++;
}
ti = get_time_ns() - ti;
printf("cc time=%0.1f ns/char\n",
(double)ti / count);
}
#endif
}
void normalization_test(const char *filename)
{
FILE *f;
char line[4096], *p;
int *in_str, *nfc_str, *nfd_str, *nfkc_str, *nfkd_str;
int in_len, nfc_len, nfd_len, nfkc_len, nfkd_len;
int *buf, buf_len, pos;
f = fopen(filename, "rb");
if (!f) {
perror(filename);
exit(1);
}
pos = 0;
for(;;) {
if (!get_line(line, sizeof(line), f))
break;
pos++;
p = line;
while (isspace(*p))
p++;
if (*p == '#' || *p == '@')
continue;
in_str = get_field_str(&in_len, p, 0);
nfc_str = get_field_str(&nfc_len, p, 1);
nfd_str = get_field_str(&nfd_len, p, 2);
nfkc_str = get_field_str(&nfkc_len, p, 3);
nfkd_str = get_field_str(&nfkd_len, p, 4);
// dump_str("in", in_str, in_len);
buf_len = unicode_normalize((uint32_t **)&buf, (uint32_t *)in_str, in_len, UNICODE_NFD, NULL, NULL);
check_str("nfd", pos, in_str, in_len, buf, buf_len, nfd_str, nfd_len);
free(buf);
buf_len = unicode_normalize((uint32_t **)&buf, (uint32_t *)in_str, in_len, UNICODE_NFKD, NULL, NULL);
check_str("nfkd", pos, in_str, in_len, buf, buf_len, nfkd_str, nfkd_len);
free(buf);
buf_len = unicode_normalize((uint32_t **)&buf, (uint32_t *)in_str, in_len, UNICODE_NFC, NULL, NULL);
check_str("nfc", pos, in_str, in_len, buf, buf_len, nfc_str, nfc_len);
free(buf);
buf_len = unicode_normalize((uint32_t **)&buf, (uint32_t *)in_str, in_len, UNICODE_NFKC, NULL, NULL);
check_str("nfkc", pos, in_str, in_len, buf, buf_len, nfkc_str, nfkc_len);
free(buf);
free(in_str);
free(nfc_str);
free(nfd_str);
free(nfkc_str);
free(nfkd_str);
}
fclose(f);
}
#endif
int main(int argc, char *argv[])
{
const char *unicode_db_path, *outfilename;
char filename[1024];
int arg = 1;
if (arg >= argc || (!strcmp(argv[arg], "-h") || !strcmp(argv[arg], "--help"))) {
printf("usage: %s PATH [OUTPUT]\n"
" PATH path to the Unicode database directory\n"
" OUTPUT name of the output file. If omitted, a self test is performed\n"
" using the files from the Unicode library\n"
, argv[0]);
return 1;
}
unicode_db_path = argv[arg++];
outfilename = NULL;
if (arg < argc)
outfilename = argv[arg++];
unicode_db = mallocz(sizeof(unicode_db[0]) * (CHARCODE_MAX + 1));
snprintf(filename, sizeof(filename), "%s/UnicodeData.txt", unicode_db_path);
parse_unicode_data(filename);
snprintf(filename, sizeof(filename), "%s/SpecialCasing.txt", unicode_db_path);
parse_special_casing(unicode_db, filename);
snprintf(filename, sizeof(filename), "%s/CaseFolding.txt", unicode_db_path);
parse_case_folding(unicode_db, filename);
snprintf(filename, sizeof(filename), "%s/CompositionExclusions.txt", unicode_db_path);
parse_composition_exclusions(filename);
snprintf(filename, sizeof(filename), "%s/DerivedCoreProperties.txt", unicode_db_path);
parse_derived_core_properties(filename);
snprintf(filename, sizeof(filename), "%s/DerivedNormalizationProps.txt", unicode_db_path);
parse_derived_norm_properties(filename);
snprintf(filename, sizeof(filename), "%s/PropList.txt", unicode_db_path);
parse_prop_list(filename);
snprintf(filename, sizeof(filename), "%s/Scripts.txt", unicode_db_path);
parse_scripts(filename);
snprintf(filename, sizeof(filename), "%s/ScriptExtensions.txt",
unicode_db_path);
parse_script_extensions(filename);
snprintf(filename, sizeof(filename), "%s/emoji-data.txt",
unicode_db_path);
parse_prop_list(filename);
// dump_unicode_data(unicode_db);
build_conv_table(unicode_db);
#ifdef DUMP_CASE_FOLDING_SPECIAL_CASES
dump_case_folding_special_cases(unicode_db);
#endif
if (!outfilename) {
#ifdef USE_TEST
check_case_conv();
check_flags();
check_decompose_table();
check_compose_table();
check_cc_table();
snprintf(filename, sizeof(filename), "%s/NormalizationTest.txt", unicode_db_path);
normalization_test(filename);
#else
fprintf(stderr, "Tests are not compiled\n");
exit(1);
#endif
} else
{
FILE *fo = fopen(outfilename, "wb");
if (!fo) {
perror(outfilename);
exit(1);
}
fprintf(fo,
"/* Compressed unicode tables */\n"
"/* Automatically generated file - do not edit */\n"
"\n"
"#include <stdint.h>\n"
"\n");
dump_case_conv_table(fo);
compute_internal_props();
build_flags_tables(fo);
fprintf(fo, "#ifdef CONFIG_ALL_UNICODE\n\n");
build_cc_table(fo);
build_decompose_table(fo);
build_general_category_table(fo);
build_script_table(fo);
build_script_ext_table(fo);
build_prop_list_table(fo);
fprintf(fo, "#endif /* CONFIG_ALL_UNICODE */\n");
fprintf(fo, "/* %u tables / %u bytes, %u index / %u bytes */\n",
total_tables, total_table_bytes, total_index, total_index_bytes);
fclose(fo);
}
return 0;
}
Reference in New Issue View Git Blame Copy Permalink