feat: use categories from unicode full emoji list (#446)

pull/449/head
Ika 2019-09-04 16:19:06 +08:00 committed by GitHub
parent 0a542ea784
commit 9f3bcd27f6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 4571 additions and 3392 deletions

View File

@ -1,15 +1,12 @@
language: node_js
node_js:
- "8"
- "12"
script:
- yarn lint
- yarn test --ci
after_success:
- if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then bash ./scripts/deploy.sh; fi
cache:
yarn: true
directories:

2116
README.md

File diff suppressed because it is too large Load Diff

View File

@ -17,14 +17,12 @@
},
"dependencies": {
"cheerio": "^0.22.0",
"dedent": "^0.7.0",
"request": "^2.88.0"
},
"devDependencies": {
"@types/cheerio": "0.22.13",
"@types/dedent": "0.7.0",
"@types/jest": "24.0.18",
"@types/node": "8.10.53",
"@types/node": "12.7.4",
"@types/request": "2.48.2",
"jest": "24.9.0",
"jest-playback": "2.0.2",
@ -34,6 +32,6 @@
"typescript": "3.6.2"
},
"engines": {
"node": ">= 8"
"node": ">= 12"
}
}

View File

@ -1,3 +0,0 @@
{
"extends": "ikatyang:library"
}

File diff suppressed because one or more lines are too long

View File

@ -1,24 +0,0 @@
{
"scope": "http://www.emoji-cheat-sheet.com:80",
"method": "GET",
"path": "/",
"body": "",
"status": 301,
"response": "<!DOCTYPE HTML PUBLIC \"-//IETF//DTD HTML 2.0//EN\">\n<html><head>\n<title>301 Moved Permanently</title>\n</head><body>\n<h1>Moved Permanently</h1>\n<p>The document has moved <a href=\"http://www.webpagefx.com/tools/emoji-cheat-sheet/\">here</a>.</p>\n</body></html>\n",
"rawHeaders": [
"Server",
"nginx",
"Date",
"Sat, 15 Jul 2017 15:29:18 GMT",
"Content-Type",
"text/html; charset=iso-8859-1",
"Content-Length",
"257",
"Connection",
"close",
"Location",
"http://www.webpagefx.com/tools/emoji-cheat-sheet/",
"ngpass_ngall",
"1"
]
}

File diff suppressed because one or more lines are too long

View File

@ -1,28 +0,0 @@
{
"scope": "http://www.webpagefx.com:80",
"method": "GET",
"path": "/tools/emoji-cheat-sheet/",
"body": "",
"status": 301,
"response": "<html>\r\n<head><title>301 Moved Permanently</title></head>\r\n<body bgcolor=\"white\">\r\n<center><h1>301 Moved Permanently</h1></center>\r\n<hr><center>CloudFront</center>\r\n</body>\r\n</html>\r\n",
"rawHeaders": [
"Server",
"CloudFront",
"Date",
"Sat, 15 Jul 2017 15:29:19 GMT",
"Content-Type",
"text/html",
"Content-Length",
"183",
"Connection",
"close",
"Location",
"https://www.webpagefx.com/tools/emoji-cheat-sheet/",
"X-Cache",
"Redirect from cloudfront",
"Via",
"1.1 76bce8bb4fbd102fc0b3aa2e41094b79.cloudfront.net (CloudFront)",
"X-Amz-Cf-Id",
"qpAvYqmevPQivnDy1hZxDykgTI_776Uql3rvV7I9cCJjoo-P9j28sw=="
]
}

View File

@ -1,9 +0,0 @@
yarn generate
git config --global user.name ikatyang-bot
git config --global user.email ikatyang+bot@gmail.com
git config --global push.default simple
git add --all
git commit -m "docs(readme): update emoji-cheat-sheet"
git push -q "https://$GH_TOKEN@github.com/$TRAVIS_REPO_SLUG.git" HEAD:master

208
scripts/fetch.js 100644
View File

@ -0,0 +1,208 @@
const $ = require("cheerio");
const request = require("request");
/**
* @typedef {string} EmojiLiteral
* @returns {Promise<{ [githubEmojiId: string]: EmojiLiteral | [string] }>}
*/
async function getGithubEmojiIdMap() {
return Object.fromEntries(
Object.entries(
/** @type {{ [id: string]: string }} */ (await fetchJson(
"https://api.github.com/emojis",
{
headers: {
"User-Agent": "https://github.com/ikatyang/emoji-cheat-sheet"
}
}
))
).map(([id, url]) => [
id,
url.includes("/unicode/")
? getLast(url.split("/"))
.split(".png")[0]
.split("-")
.map(codePointText =>
String.fromCodePoint(Number.parseInt(codePointText, 16))
)
.join("")
: [getLast(url.split("/")).split(".png")[0]] // github's custom emoji
])
);
}
async function getUnicodeEmojiCategoryIterator() {
return getUnicodeEmojiCategoryIteratorFromHtmlText(
await fetch("https://unicode.org/emoji/charts/full-emoji-list.html")
);
}
/**
* @param {string} htmlText
*/
function* getUnicodeEmojiCategoryIteratorFromHtmlText(htmlText) {
const $html = $.load(htmlText).root();
const $trs = $html
.find("table")
.children()
.toArray();
for (const $tr of $trs) {
if ($tr.firstChild.tagName === "th") {
if ($tr.firstChild.attribs.class === "bighead") {
const value = $tr.firstChild.firstChild.firstChild.nodeValue;
yield { type: "category", value };
} else if ($tr.firstChild.attribs.class === "mediumhead") {
const value = $tr.firstChild.firstChild.firstChild.nodeValue;
yield { type: "subcategory", value };
} else {
// skip column titles
}
} else if ($tr.firstChild.tagName === "td") {
if ($tr.children[4].attribs.class === "chars") {
yield { type: "emoji", value: $tr.children[4].firstChild.nodeValue };
} else {
throw new Error(`Unexpected situation.`);
}
} else {
throw new Error(
`Unexpected tagName ${JSON.stringify($tr.firstChild.tagName)}`
);
}
}
}
async function getCategorizeGithubEmojiIds() {
const githubEmojiIdMap = await getGithubEmojiIdMap();
/** @type {{ [emojiLiteral: string]: string[] }} */
const emojiLiteralToGithubEmojiIdsMap = {};
/** @type {{ [githubSpecificEmojiUri: string]: string[] }} */
const githubSpecificEmojiUriToGithubEmojiIdsMap = {};
for (const [emojiId, emojiLiteral] of Object.entries(githubEmojiIdMap)) {
if (Array.isArray(emojiLiteral)) {
const [uri] = emojiLiteral;
if (!githubSpecificEmojiUriToGithubEmojiIdsMap[uri]) {
githubSpecificEmojiUriToGithubEmojiIdsMap[uri] = [];
}
githubSpecificEmojiUriToGithubEmojiIdsMap[uri].push(emojiId);
delete githubEmojiIdMap[emojiId];
continue;
}
if (!emojiLiteralToGithubEmojiIdsMap[emojiLiteral]) {
emojiLiteralToGithubEmojiIdsMap[emojiLiteral] = [];
}
emojiLiteralToGithubEmojiIdsMap[emojiLiteral].push(emojiId);
}
/** @type {{ [category: string]: { [subcategory: string]: Array<string[]> } }} */
const categorizedEmojiIds = {};
const categoryStack = [];
for (const { type, value } of await getUnicodeEmojiCategoryIterator()) {
switch (type) {
case "category": {
while (categoryStack.length) categoryStack.pop();
const title = toTitleCase(value);
categoryStack.push(title);
categorizedEmojiIds[title] = {};
break;
}
case "subcategory": {
if (categoryStack.length > 1) categoryStack.pop();
const title = toTitleCase(value);
categoryStack.push(title);
categorizedEmojiIds[categoryStack[0]][title] = [];
break;
}
case "emoji": {
const key = value.replace(/[\ufe00-\ufe0f\u200d]/g, "");
if (key in emojiLiteralToGithubEmojiIdsMap) {
const githubEmojiIds = emojiLiteralToGithubEmojiIdsMap[key];
const [category, subcategory] = categoryStack;
categorizedEmojiIds[category][subcategory].push(githubEmojiIds);
for (const githubEmojiId of githubEmojiIds) {
delete githubEmojiIdMap[githubEmojiId];
}
}
break;
}
default:
throw new Error(`Unexpected type ${JSON.stringify(type)}`);
}
}
if (Object.keys(githubEmojiIdMap).length) {
throw new Error(`Uncategorized emoji(s) found.`);
}
for (const category of Object.keys(categorizedEmojiIds)) {
const subCategorizedEmojiIds = categorizedEmojiIds[category];
const subcategories = Object.keys(subCategorizedEmojiIds);
for (const subcategory of subcategories) {
if (subCategorizedEmojiIds[subcategory].length === 0) {
delete subCategorizedEmojiIds[subcategory];
}
}
if (Object.keys(subCategorizedEmojiIds).length === 0) {
delete categorizedEmojiIds[category];
}
}
if (Object.keys(githubSpecificEmojiUriToGithubEmojiIdsMap).length) {
categorizedEmojiIds["GitHub Custom Emoji"] = {
"": Object.entries(githubSpecificEmojiUriToGithubEmojiIdsMap).map(
([, v]) => v
)
};
}
return categorizedEmojiIds;
}
/**
* @param {string} str
*/
function toTitleCase(str) {
return str
.replace(/-/g, " ")
.replace(/\s+/g, " ")
.replace(/[a-zA-Z]+/g, word => word[0].toUpperCase() + word.slice(1));
}
/**
* @template T
* @param {Array<T>} array
*/
function getLast(array) {
return array[array.length - 1];
}
/**
* @param {string} url
* @param {Partial<request.Options>} options
* @returns {Promise<any>}
*/
async function fetchJson(url, options = {}) {
return JSON.parse(await fetch(url, options));
}
/**
* @param {string} url
* @param {Partial<request.Options>} options
* @returns {Promise<string>}
*/
async function fetch(url, options = {}) {
return new Promise((resolve, reject) => {
request.get(
/** @type {request.Options} */ ({ url, ...options }),
(error, response, html) => {
if (!error && response.statusCode === 200) {
resolve(html);
} else {
reject(
error
? error
: `Unexpected response status code: ${response.statusCode}`
);
}
}
);
});
}
module.exports = {
getCategorizeGithubEmojiIds
};

View File

@ -1,153 +1,12 @@
const $ = require("cheerio");
const dedent = require("dedent");
const request = require("request");
const packageJson = require("../package.json");
const { getCategorizeGithubEmojiIds } = require("./fetch");
const { generateCheatSheet } = require("./markdown");
const apiUrl = "https://api.github.com/emojis";
const sheetUrl = "http://www.emoji-cheat-sheet.com";
const columns = 2;
/**
* @typedef {string} EmojiId
* @typedef {{ [category: string]: EmojiId[] }} EmojiData
*/
const tocName = "Table of Contents";
const topName = "top";
const topHref = "#table-of-contents";
async function generateCheatSheet() {
return buildTable(await getData());
}
/**
* @returns {Promise<EmojiData>}
*/
async function getData() {
const apiHtml = await fetchHtml(apiUrl);
const sheetHtml = await fetchHtml(sheetUrl);
const apiJson = /** @type {Record<EmojiId, string>} */ (JSON.parse(apiHtml));
const emojiIds = Object.keys(apiJson);
const emojiData = /** @type {EmojiData} */ ({});
const $html = $.load(sheetHtml).root();
$html.find("h2").each((_, $category) => {
const localEmojiIds = /** @type {string[]} */ ([]);
const category = $($category).text();
$html
.find(`#emoji-${category.toLowerCase()} li .name`)
.each((_, $emoji) => {
const emoji = $($emoji).text();
const index = emojiIds.indexOf(emoji);
if (index !== -1) {
localEmojiIds.push(...emojiIds.splice(index, 1));
}
});
emojiData[category] = localEmojiIds;
});
if (emojiIds.length !== 0) {
emojiData["Uncategorized"] = emojiIds;
}
return emojiData;
}
/**
* @param {EmojiData} emojiData
* @returns {string}
*/
function buildTable(emojiData) {
const travisRepoUrl = `https://travis-ci.org/${packageJson.repository}`;
const travisBadgeUrl = `${travisRepoUrl}.svg?branch=master`;
const categories = Object.keys(emojiData);
return dedent(`
# ${packageJson.name}
[![build](${travisBadgeUrl})](${travisRepoUrl})
This cheat sheet is automatically generated from ${[
["GitHub Emoji API", apiUrl],
["Emoji Cheat Sheet", sheetUrl]
]
.map(([siteName, siteUrl]) => `[${siteName}](${siteUrl})`)
.join(" and ")}.
## ${tocName}
${categories
.map(category => `- [${category}](#${category.toLowerCase()})`)
.join("\n")}
${categories
.map(category => {
const emojis = emojiData[category];
return dedent(`
### ${category}
${buildTableHead()}
${buildTableContent(emojis)}
`);
})
.join("\n".repeat(2))}
`);
}
/**
* @param {string[]} emojis
*/
function buildTableContent(emojis) {
let tableContent = "";
for (let i = 0; i < emojis.length; i += columns) {
const rowEmojis = emojis.slice(i, i + columns);
while (rowEmojis.length < columns) {
rowEmojis.push("");
}
tableContent += `| [${topName}](${topHref}) |${rowEmojis
.map(x => (x.length !== 0 ? ` :${x}: | \`:${x}:\` ` : " | "))
.join("|")}|\n`;
}
return tableContent;
}
function buildTableHead() {
return dedent(`
| |${" ico | emoji |".repeat(columns)}
| - |${" --- | ----- |".repeat(columns)}
`);
}
/**
* @param {string} url
* @returns {Promise<string>}
*/
async function fetchHtml(url) {
return new Promise((resolve, reject) => {
const options = /** @type {request.Options} */ ({ url });
if (url === apiUrl) {
options.headers = {
"User-Agent": "https://github.com/ikatyang/emoji-cheat-sheet"
};
}
request.get(options, (error, response, html) => {
if (!error && response.statusCode === 200) {
resolve(html);
} else {
reject(
error
? error
: `Unexpected response status code: ${response.statusCode}`
);
}
});
});
async function generate() {
return generateCheatSheet(await getCategorizeGithubEmojiIds());
}
if (require.main === /** @type {unknown} */ (module)) {
generateCheatSheet().then(cheatSheet => console.log(cheatSheet));
generate().then(cheatSheet => console.log(cheatSheet));
} else {
module.exports = generateCheatSheet;
module.exports = generate;
}

File diff suppressed because it is too large Load Diff

134
scripts/markdown.js 100644
View File

@ -0,0 +1,134 @@
const { name: repoName, repository } = require("../package.json");
const resource1 = "[GitHub Emoji API](https://api.github.com/emojis)";
const resoruce2 =
"[Unicode Full Emoji List](https://unicode.org/emoji/charts/full-emoji-list.html)";
const columns = 2;
const tocName = "Table of Contents";
/**
* @typedef {Array<string[]>} GithubEmojiIds
*/
/**
* @param {{ [category: string]: { [subcategory: string]: GithubEmojiIds } }} categorizedGithubEmojiIds
*/
function generateCheatSheet(categorizedGithubEmojiIds) {
const lineTexts = [];
lineTexts.push(`# ${repoName}`);
lineTexts.push("");
lineTexts.push(
`[![build](https://travis-ci.org/${repository}.svg?branch=master)](https://travis-ci.org/${repository})`
);
lineTexts.push("");
lineTexts.push(
`This cheat sheet is automatically generated from ${resource1} and ${resoruce2}.`
);
lineTexts.push("");
const categories = Object.keys(categorizedGithubEmojiIds);
lineTexts.push(`## ${tocName}`);
lineTexts.push("");
lineTexts.push(...generateToc(categories));
lineTexts.push("");
for (const category of categories) {
lineTexts.push(`### ${category}`);
lineTexts.push("");
const subcategorizeGithubEmojiIds = categorizedGithubEmojiIds[category];
const subcategories = Object.keys(subcategorizeGithubEmojiIds);
if (subcategories.length > 1) {
lineTexts.push(...generateToc(subcategories));
lineTexts.push("");
}
for (const subcategory of subcategories) {
if (subcategory) {
lineTexts.push(`#### ${subcategory}`);
lineTexts.push("");
}
lineTexts.push(
...generateTable(
subcategorizeGithubEmojiIds[subcategory],
`[top](#${getHeaderId(category)})`,
`[top](#${getHeaderId(tocName)})`
)
);
lineTexts.push("");
}
}
return lineTexts.join("\n");
}
/**
* @param {string[]} headers
*/
function generateToc(headers) {
return headers.map(header => `- [${header}](#${getHeaderId(header)})`);
}
/**
* @param {string} header
*/
function getHeaderId(header) {
return header
.toLowerCase()
.replace(/ /g, "-")
.replace(/[^a-z0-9-]/g, "");
}
/**
* @param {GithubEmojiIds} githubEmojiIds
* @param {string} leftText
* @param {string} rightText
*/
function generateTable(githubEmojiIds, leftText, rightText) {
const lineTexts = [];
let header = "";
let delimieter = "";
header += "| ";
delimieter += "| - ";
for (let i = 0; i < columns && i < githubEmojiIds.length; i++) {
header += `| ico | shortcode `;
delimieter += "| :-: | - ";
}
header += "| |";
delimieter += "| - |";
lineTexts.push(header, delimieter);
for (let i = 0; i < githubEmojiIds.length; i += columns) {
let lineText = `| ${leftText} `;
for (let j = 0; j < columns; j++) {
if (i + j < githubEmojiIds.length) {
const emojiIds = githubEmojiIds[i + j];
const emojiId = emojiIds[0];
lineText += `| :${emojiId}: | \`:${emojiId}:\` `;
for (let k = 1; k < emojiIds.length; k++) {
lineText += `<br /> \`:${emojiIds[k]}:\` `;
}
} else if (githubEmojiIds.length > columns) {
lineText += "| | ";
}
}
lineText += `| ${rightText} |`;
lineTexts.push(lineText);
}
return lineTexts;
}
module.exports = {
generateCheatSheet
};

View File

@ -6,7 +6,7 @@
"noEmit": true,
"resolveJsonModule": true,
"strict": true,
"target": "es2018"
"target": "esnext"
},
"include": ["scripts/**/*.js"]
}

View File

@ -332,11 +332,6 @@
dependencies:
"@types/node" "*"
"@types/dedent@0.7.0":
version "0.7.0"
resolved "https://registry.yarnpkg.com/@types/dedent/-/dedent-0.7.0.tgz#155f339ca404e6dd90b9ce46a3f78fd69ca9b050"
integrity sha512-EGlKlgMhnLt/cM4DbUSafFdrkeJoC9Mvnj0PUCU7tFmTjMjNRT957kXCx0wYm3JuEq4o4ZsS5vG+NlkM2DMd2A==
"@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0":
version "2.0.1"
resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.1.tgz#42995b446db9a48a11a07ec083499a860e9138ff"
@ -374,10 +369,10 @@
resolved "https://registry.yarnpkg.com/@types/node/-/node-12.7.3.tgz#27b3f40addaf2f580459fdb405222685542f907a"
integrity sha512-3SiLAIBkDWDg6vFo0+5YJyHPWU9uwu40Qe+v+0MH8wRKYBimHvvAOyk3EzMrD/TrIlLYfXrqDqrg913PynrMJQ==
"@types/node@8.10.53":
version "8.10.53"
resolved "https://registry.yarnpkg.com/@types/node/-/node-8.10.53.tgz#5fa08eef810b08b2c03073e360b54f7bad899db1"
integrity sha512-aOmXdv1a1/vYUn1OT1CED8ftbkmmYbKhKGSyMDeJiidLvKRKvZUQOdXwG/wcNY7T1Qb0XTlVdiYjIq00U7pLrQ==
"@types/node@12.7.4":
version "12.7.4"
resolved "https://registry.yarnpkg.com/@types/node/-/node-12.7.4.tgz#64db61e0359eb5a8d99b55e05c729f130a678b04"
integrity sha512-W0+n1Y+gK/8G2P/piTkBBN38Qc5Q1ZSO6B5H3QmPCUewaiXOo2GCAWZ4ElZCcNhjJuBSUSLGFUJnmlCn5+nxOQ==
"@types/request@2.48.2":
version "2.48.2"
@ -980,11 +975,6 @@ decode-uri-component@^0.2.0:
resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.0.tgz#eb3913333458775cb84cd1a1fae062106bb87545"
integrity sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=
dedent@^0.7.0:
version "0.7.0"
resolved "https://registry.yarnpkg.com/dedent/-/dedent-0.7.0.tgz#2495ddbaf6eb874abb0e1be9df22d2e5a544326c"
integrity sha1-JJXduvbrh0q7Dhvp3yLS5aVEMmw=
deep-eql@^3.0.1:
version "3.0.1"
resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-3.0.1.tgz#dfc9404400ad1c8fe023e7da1df1c147c4b444df"