feat: use categories from unicode full emoji list (#446)
Ika
4 years ago
committed by
GitHub
parent
0a542ea784
commit
9f3bcd27f6
@ -1,3 +0,0 @@ |
||||
{ |
||||
"extends": "ikatyang:library" |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
@ -1,24 +0,0 @@ |
||||
{ |
||||
"scope": "http://www.emoji-cheat-sheet.com:80", |
||||
"method": "GET", |
||||
"path": "/", |
||||
"body": "", |
||||
"status": 301, |
||||
"response": "<!DOCTYPE HTML PUBLIC \"-//IETF//DTD HTML 2.0//EN\">\n<html><head>\n<title>301 Moved Permanently</title>\n</head><body>\n<h1>Moved Permanently</h1>\n<p>The document has moved <a href=\"http://www.webpagefx.com/tools/emoji-cheat-sheet/\">here</a>.</p>\n</body></html>\n", |
||||
"rawHeaders": [ |
||||
"Server", |
||||
"nginx", |
||||
"Date", |
||||
"Sat, 15 Jul 2017 15:29:18 GMT", |
||||
"Content-Type", |
||||
"text/html; charset=iso-8859-1", |
||||
"Content-Length", |
||||
"257", |
||||
"Connection", |
||||
"close", |
||||
"Location", |
||||
"http://www.webpagefx.com/tools/emoji-cheat-sheet/", |
||||
"ngpass_ngall", |
||||
"1" |
||||
] |
||||
} |
||||
File diff suppressed because one or more lines are too long
@ -1,28 +0,0 @@ |
||||
{ |
||||
"scope": "http://www.webpagefx.com:80", |
||||
"method": "GET", |
||||
"path": "/tools/emoji-cheat-sheet/", |
||||
"body": "", |
||||
"status": 301, |
||||
"response": "<html>\r\n<head><title>301 Moved Permanently</title></head>\r\n<body bgcolor=\"white\">\r\n<center><h1>301 Moved Permanently</h1></center>\r\n<hr><center>CloudFront</center>\r\n</body>\r\n</html>\r\n", |
||||
"rawHeaders": [ |
||||
"Server", |
||||
"CloudFront", |
||||
"Date", |
||||
"Sat, 15 Jul 2017 15:29:19 GMT", |
||||
"Content-Type", |
||||
"text/html", |
||||
"Content-Length", |
||||
"183", |
||||
"Connection", |
||||
"close", |
||||
"Location", |
||||
"https://www.webpagefx.com/tools/emoji-cheat-sheet/", |
||||
"X-Cache", |
||||
"Redirect from cloudfront", |
||||
"Via", |
||||
"1.1 76bce8bb4fbd102fc0b3aa2e41094b79.cloudfront.net (CloudFront)", |
||||
"X-Amz-Cf-Id", |
||||
"qpAvYqmevPQivnDy1hZxDykgTI_776Uql3rvV7I9cCJjoo-P9j28sw==" |
||||
] |
||||
} |
||||
@ -1,9 +0,0 @@ |
||||
yarn generate |
||||
|
||||
git config --global user.name ikatyang-bot |
||||
git config --global user.email ikatyang+bot@gmail.com |
||||
git config --global push.default simple |
||||
|
||||
git add --all |
||||
git commit -m "docs(readme): update emoji-cheat-sheet" |
||||
git push -q "https://$GH_TOKEN@github.com/$TRAVIS_REPO_SLUG.git" HEAD:master |
||||
@ -0,0 +1,208 @@ |
||||
const $ = require("cheerio"); |
||||
const request = require("request"); |
||||
|
||||
/** |
||||
* @typedef {string} EmojiLiteral |
||||
* @returns {Promise<{ [githubEmojiId: string]: EmojiLiteral | [string] }>} |
||||
*/ |
||||
async function getGithubEmojiIdMap() { |
||||
return Object.fromEntries( |
||||
Object.entries( |
||||
/** @type {{ [id: string]: string }} */ (await fetchJson( |
||||
"https://api.github.com/emojis", |
||||
{ |
||||
headers: { |
||||
"User-Agent": "https://github.com/ikatyang/emoji-cheat-sheet" |
||||
} |
||||
} |
||||
)) |
||||
).map(([id, url]) => [ |
||||
id, |
||||
url.includes("/unicode/") |
||||
? getLast(url.split("/")) |
||||
.split(".png")[0] |
||||
.split("-") |
||||
.map(codePointText => |
||||
String.fromCodePoint(Number.parseInt(codePointText, 16)) |
||||
) |
||||
.join("") |
||||
: [getLast(url.split("/")).split(".png")[0]] // github's custom emoji
|
||||
]) |
||||
); |
||||
} |
||||
|
||||
async function getUnicodeEmojiCategoryIterator() { |
||||
return getUnicodeEmojiCategoryIteratorFromHtmlText( |
||||
await fetch("https://unicode.org/emoji/charts/full-emoji-list.html") |
||||
); |
||||
} |
||||
|
||||
/** |
||||
* @param {string} htmlText |
||||
*/ |
||||
function* getUnicodeEmojiCategoryIteratorFromHtmlText(htmlText) { |
||||
const $html = $.load(htmlText).root(); |
||||
const $trs = $html |
||||
.find("table") |
||||
.children() |
||||
.toArray(); |
||||
for (const $tr of $trs) { |
||||
if ($tr.firstChild.tagName === "th") { |
||||
if ($tr.firstChild.attribs.class === "bighead") { |
||||
const value = $tr.firstChild.firstChild.firstChild.nodeValue; |
||||
yield { type: "category", value }; |
||||
} else if ($tr.firstChild.attribs.class === "mediumhead") { |
||||
const value = $tr.firstChild.firstChild.firstChild.nodeValue; |
||||
yield { type: "subcategory", value }; |
||||
} else { |
||||
// skip column titles
|
||||
} |
||||
} else if ($tr.firstChild.tagName === "td") { |
||||
if ($tr.children[4].attribs.class === "chars") { |
||||
yield { type: "emoji", value: $tr.children[4].firstChild.nodeValue }; |
||||
} else { |
||||
throw new Error(`Unexpected situation.`); |
||||
} |
||||
} else { |
||||
throw new Error( |
||||
`Unexpected tagName ${JSON.stringify($tr.firstChild.tagName)}` |
||||
); |
||||
} |
||||
} |
||||
} |
||||
|
||||
async function getCategorizeGithubEmojiIds() { |
||||
const githubEmojiIdMap = await getGithubEmojiIdMap(); |
||||
/** @type {{ [emojiLiteral: string]: string[] }} */ |
||||
const emojiLiteralToGithubEmojiIdsMap = {}; |
||||
/** @type {{ [githubSpecificEmojiUri: string]: string[] }} */ |
||||
const githubSpecificEmojiUriToGithubEmojiIdsMap = {}; |
||||
for (const [emojiId, emojiLiteral] of Object.entries(githubEmojiIdMap)) { |
||||
if (Array.isArray(emojiLiteral)) { |
||||
const [uri] = emojiLiteral; |
||||
if (!githubSpecificEmojiUriToGithubEmojiIdsMap[uri]) { |
||||
githubSpecificEmojiUriToGithubEmojiIdsMap[uri] = []; |
||||
} |
||||
githubSpecificEmojiUriToGithubEmojiIdsMap[uri].push(emojiId); |
||||
delete githubEmojiIdMap[emojiId]; |
||||
continue; |
||||
} |
||||
if (!emojiLiteralToGithubEmojiIdsMap[emojiLiteral]) { |
||||
emojiLiteralToGithubEmojiIdsMap[emojiLiteral] = []; |
||||
} |
||||
emojiLiteralToGithubEmojiIdsMap[emojiLiteral].push(emojiId); |
||||
} |
||||
/** @type {{ [category: string]: { [subcategory: string]: Array<string[]> } }} */ |
||||
const categorizedEmojiIds = {}; |
||||
const categoryStack = []; |
||||
for (const { type, value } of await getUnicodeEmojiCategoryIterator()) { |
||||
switch (type) { |
||||
case "category": { |
||||
while (categoryStack.length) categoryStack.pop(); |
||||
const title = toTitleCase(value); |
||||
categoryStack.push(title); |
||||
categorizedEmojiIds[title] = {}; |
||||
break; |
||||
} |
||||
case "subcategory": { |
||||
if (categoryStack.length > 1) categoryStack.pop(); |
||||
const title = toTitleCase(value); |
||||
categoryStack.push(title); |
||||
categorizedEmojiIds[categoryStack[0]][title] = []; |
||||
break; |
||||
} |
||||
case "emoji": { |
||||
const key = value.replace(/[\ufe00-\ufe0f\u200d]/g, ""); |
||||
if (key in emojiLiteralToGithubEmojiIdsMap) { |
||||
const githubEmojiIds = emojiLiteralToGithubEmojiIdsMap[key]; |
||||
const [category, subcategory] = categoryStack; |
||||
categorizedEmojiIds[category][subcategory].push(githubEmojiIds); |
||||
for (const githubEmojiId of githubEmojiIds) { |
||||
delete githubEmojiIdMap[githubEmojiId]; |
||||
} |
||||
} |
||||
break; |
||||
} |
||||
default: |
||||
throw new Error(`Unexpected type ${JSON.stringify(type)}`); |
||||
} |
||||
} |
||||
if (Object.keys(githubEmojiIdMap).length) { |
||||
throw new Error(`Uncategorized emoji(s) found.`); |
||||
} |
||||
for (const category of Object.keys(categorizedEmojiIds)) { |
||||
const subCategorizedEmojiIds = categorizedEmojiIds[category]; |
||||
const subcategories = Object.keys(subCategorizedEmojiIds); |
||||
for (const subcategory of subcategories) { |
||||
if (subCategorizedEmojiIds[subcategory].length === 0) { |
||||
delete subCategorizedEmojiIds[subcategory]; |
||||
} |
||||
} |
||||
if (Object.keys(subCategorizedEmojiIds).length === 0) { |
||||
delete categorizedEmojiIds[category]; |
||||
} |
||||
} |
||||
if (Object.keys(githubSpecificEmojiUriToGithubEmojiIdsMap).length) { |
||||
categorizedEmojiIds["GitHub Custom Emoji"] = { |
||||
"": Object.entries(githubSpecificEmojiUriToGithubEmojiIdsMap).map( |
||||
([, v]) => v |
||||
) |
||||
}; |
||||
} |
||||
return categorizedEmojiIds; |
||||
} |
||||
|
||||
/** |
||||
* @param {string} str |
||||
*/ |
||||
function toTitleCase(str) { |
||||
return str |
||||
.replace(/-/g, " ") |
||||
.replace(/\s+/g, " ") |
||||
.replace(/[a-zA-Z]+/g, word => word[0].toUpperCase() + word.slice(1)); |
||||
} |
||||
|
||||
/** |
||||
* @template T |
||||
* @param {Array<T>} array |
||||
*/ |
||||
function getLast(array) { |
||||
return array[array.length - 1]; |
||||
} |
||||
|
||||
/** |
||||
* @param {string} url |
||||
* @param {Partial<request.Options>} options |
||||
* @returns {Promise<any>} |
||||
*/ |
||||
async function fetchJson(url, options = {}) { |
||||
return JSON.parse(await fetch(url, options)); |
||||
} |
||||
|
||||
/** |
||||
* @param {string} url |
||||
* @param {Partial<request.Options>} options |
||||
* @returns {Promise<string>} |
||||
*/ |
||||
async function fetch(url, options = {}) { |
||||
return new Promise((resolve, reject) => { |
||||
request.get( |
||||
/** @type {request.Options} */ ({ url, ...options }), |
||||
(error, response, html) => { |
||||
if (!error && response.statusCode === 200) { |
||||
resolve(html); |
||||
} else { |
||||
reject( |
||||
error |
||||
? error |
||||
: `Unexpected response status code: ${response.statusCode}` |
||||
); |
||||
} |
||||
} |
||||
); |
||||
}); |
||||
} |
||||
|
||||
module.exports = { |
||||
getCategorizeGithubEmojiIds |
||||
}; |
||||
@ -1,153 +1,12 @@ |
||||
const $ = require("cheerio"); |
||||
const dedent = require("dedent"); |
||||
const request = require("request"); |
||||
const packageJson = require("../package.json"); |
||||
const { getCategorizeGithubEmojiIds } = require("./fetch"); |
||||
const { generateCheatSheet } = require("./markdown"); |
||||
|
||||
const apiUrl = "https://api.github.com/emojis"; |
||||
const sheetUrl = "http://www.emoji-cheat-sheet.com"; |
||||
|
||||
const columns = 2; |
||||
|
||||
/** |
||||
* @typedef {string} EmojiId |
||||
* @typedef {{ [category: string]: EmojiId[] }} EmojiData |
||||
*/ |
||||
|
||||
const tocName = "Table of Contents"; |
||||
const topName = "top"; |
||||
const topHref = "#table-of-contents"; |
||||
|
||||
async function generateCheatSheet() { |
||||
return buildTable(await getData()); |
||||
} |
||||
|
||||
/** |
||||
* @returns {Promise<EmojiData>} |
||||
*/ |
||||
async function getData() { |
||||
const apiHtml = await fetchHtml(apiUrl); |
||||
const sheetHtml = await fetchHtml(sheetUrl); |
||||
|
||||
const apiJson = /** @type {Record<EmojiId, string>} */ (JSON.parse(apiHtml)); |
||||
|
||||
const emojiIds = Object.keys(apiJson); |
||||
const emojiData = /** @type {EmojiData} */ ({}); |
||||
|
||||
const $html = $.load(sheetHtml).root(); |
||||
$html.find("h2").each((_, $category) => { |
||||
const localEmojiIds = /** @type {string[]} */ ([]); |
||||
const category = $($category).text(); |
||||
$html |
||||
.find(`#emoji-${category.toLowerCase()} li .name`) |
||||
.each((_, $emoji) => { |
||||
const emoji = $($emoji).text(); |
||||
const index = emojiIds.indexOf(emoji); |
||||
if (index !== -1) { |
||||
localEmojiIds.push(...emojiIds.splice(index, 1)); |
||||
} |
||||
}); |
||||
emojiData[category] = localEmojiIds; |
||||
}); |
||||
|
||||
if (emojiIds.length !== 0) { |
||||
emojiData["Uncategorized"] = emojiIds; |
||||
} |
||||
|
||||
return emojiData; |
||||
} |
||||
|
||||
/** |
||||
* @param {EmojiData} emojiData |
||||
* @returns {string} |
||||
*/ |
||||
function buildTable(emojiData) { |
||||
const travisRepoUrl = `https://travis-ci.org/${packageJson.repository}`; |
||||
const travisBadgeUrl = `${travisRepoUrl}.svg?branch=master`; |
||||
const categories = Object.keys(emojiData); |
||||
return dedent(` |
||||
# ${packageJson.name} |
||||
|
||||
[](${travisRepoUrl}) |
||||
|
||||
This cheat sheet is automatically generated from ${[ |
||||
["GitHub Emoji API", apiUrl], |
||||
["Emoji Cheat Sheet", sheetUrl] |
||||
] |
||||
.map(([siteName, siteUrl]) => `[${siteName}](${siteUrl})`) |
||||
.join(" and ")}. |
||||
|
||||
## ${tocName} |
||||
|
||||
${categories |
||||
.map(category => `- [${category}](#${category.toLowerCase()})`) |
||||
.join("\n")} |
||||
|
||||
${categories |
||||
.map(category => { |
||||
const emojis = emojiData[category]; |
||||
return dedent(` |
||||
### ${category} |
||||
|
||||
${buildTableHead()} |
||||
${buildTableContent(emojis)} |
||||
`);
|
||||
}) |
||||
.join("\n".repeat(2))} |
||||
`);
|
||||
} |
||||
|
||||
/** |
||||
* @param {string[]} emojis |
||||
*/ |
||||
function buildTableContent(emojis) { |
||||
let tableContent = ""; |
||||
for (let i = 0; i < emojis.length; i += columns) { |
||||
const rowEmojis = emojis.slice(i, i + columns); |
||||
while (rowEmojis.length < columns) { |
||||
rowEmojis.push(""); |
||||
} |
||||
tableContent += `| [${topName}](${topHref}) |${rowEmojis |
||||
.map(x => (x.length !== 0 ? ` :${x}: | \`:${x}:\` ` : " | ")) |
||||
.join("|")}|\n`;
|
||||
} |
||||
return tableContent; |
||||
} |
||||
|
||||
function buildTableHead() { |
||||
return dedent(` |
||||
| |${" ico | emoji |".repeat(columns)} |
||||
| - |${" --- | ----- |".repeat(columns)} |
||||
`);
|
||||
} |
||||
|
||||
/** |
||||
* @param {string} url |
||||
* @returns {Promise<string>} |
||||
*/ |
||||
async function fetchHtml(url) { |
||||
return new Promise((resolve, reject) => { |
||||
const options = /** @type {request.Options} */ ({ url }); |
||||
if (url === apiUrl) { |
||||
options.headers = { |
||||
"User-Agent": "https://github.com/ikatyang/emoji-cheat-sheet" |
||||
}; |
||||
} |
||||
request.get(options, (error, response, html) => { |
||||
if (!error && response.statusCode === 200) { |
||||
resolve(html); |
||||
} else { |
||||
reject( |
||||
error |
||||
? error |
||||
: `Unexpected response status code: ${response.statusCode}` |
||||
); |
||||
} |
||||
}); |
||||
}); |
||||
async function generate() { |
||||
return generateCheatSheet(await getCategorizeGithubEmojiIds()); |
||||
} |
||||
|
||||
if (require.main === /** @type {unknown} */ (module)) { |
||||
generateCheatSheet().then(cheatSheet => console.log(cheatSheet)); |
||||
generate().then(cheatSheet => console.log(cheatSheet)); |
||||
} else { |
||||
module.exports = generateCheatSheet; |
||||
module.exports = generate; |
||||
} |
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,134 @@ |
||||
const { name: repoName, repository } = require("../package.json"); |
||||
|
||||
const resource1 = "[GitHub Emoji API](https://api.github.com/emojis)"; |
||||
const resoruce2 = |
||||
"[Unicode Full Emoji List](https://unicode.org/emoji/charts/full-emoji-list.html)"; |
||||
|
||||
const columns = 2; |
||||
|
||||
const tocName = "Table of Contents"; |
||||
|
||||
/** |
||||
* @typedef {Array<string[]>} GithubEmojiIds |
||||
*/ |
||||
|
||||
/** |
||||
* @param {{ [category: string]: { [subcategory: string]: GithubEmojiIds } }} categorizedGithubEmojiIds |
||||
*/ |
||||
function generateCheatSheet(categorizedGithubEmojiIds) { |
||||
const lineTexts = []; |
||||
|
||||
lineTexts.push(`# ${repoName}`); |
||||
lineTexts.push(""); |
||||
|
||||
lineTexts.push( |
||||
`[](https://travis-ci.org/${repository})` |
||||
); |
||||
lineTexts.push(""); |
||||
|
||||
lineTexts.push( |
||||
`This cheat sheet is automatically generated from ${resource1} and ${resoruce2}.` |
||||
); |
||||
lineTexts.push(""); |
||||
|
||||
const categories = Object.keys(categorizedGithubEmojiIds); |
||||
|
||||
lineTexts.push(`## ${tocName}`); |
||||
lineTexts.push(""); |
||||
lineTexts.push(...generateToc(categories)); |
||||
lineTexts.push(""); |
||||
|
||||
for (const category of categories) { |
||||
lineTexts.push(`### ${category}`); |
||||
lineTexts.push(""); |
||||
|
||||
const subcategorizeGithubEmojiIds = categorizedGithubEmojiIds[category]; |
||||
const subcategories = Object.keys(subcategorizeGithubEmojiIds); |
||||
if (subcategories.length > 1) { |
||||
lineTexts.push(...generateToc(subcategories)); |
||||
lineTexts.push(""); |
||||
} |
||||
|
||||
for (const subcategory of subcategories) { |
||||
if (subcategory) { |
||||
lineTexts.push(`#### ${subcategory}`); |
||||
lineTexts.push(""); |
||||
} |
||||
|
||||
lineTexts.push( |
||||
...generateTable( |
||||
subcategorizeGithubEmojiIds[subcategory], |
||||
`[top](#${getHeaderId(category)})`, |
||||
`[top](#${getHeaderId(tocName)})` |
||||
) |
||||
); |
||||
lineTexts.push(""); |
||||
} |
||||
} |
||||
|
||||
return lineTexts.join("\n"); |
||||
} |
||||
|
||||
/** |
||||
* @param {string[]} headers |
||||
*/ |
||||
function generateToc(headers) { |
||||
return headers.map(header => `- [${header}](#${getHeaderId(header)})`); |
||||
} |
||||
|
||||
/** |
||||
* @param {string} header |
||||
*/ |
||||
function getHeaderId(header) { |
||||
return header |
||||
.toLowerCase() |
||||
.replace(/ /g, "-") |
||||
.replace(/[^a-z0-9-]/g, ""); |
||||
} |
||||
|
||||
/** |
||||
* @param {GithubEmojiIds} githubEmojiIds |
||||
* @param {string} leftText |
||||
* @param {string} rightText |
||||
*/ |
||||
function generateTable(githubEmojiIds, leftText, rightText) { |
||||
const lineTexts = []; |
||||
|
||||
let header = ""; |
||||
let delimieter = ""; |
||||
|
||||
header += "| "; |
||||
delimieter += "| - "; |
||||
for (let i = 0; i < columns && i < githubEmojiIds.length; i++) { |
||||
header += `| ico | shortcode `; |
||||
delimieter += "| :-: | - "; |
||||
} |
||||
header += "| |"; |
||||
delimieter += "| - |"; |
||||
|
||||
lineTexts.push(header, delimieter); |
||||
|
||||
for (let i = 0; i < githubEmojiIds.length; i += columns) { |
||||
let lineText = `| ${leftText} `; |
||||
for (let j = 0; j < columns; j++) { |
||||
if (i + j < githubEmojiIds.length) { |
||||
const emojiIds = githubEmojiIds[i + j]; |
||||
const emojiId = emojiIds[0]; |
||||
lineText += `| :${emojiId}: | \`:${emojiId}:\` `; |
||||
for (let k = 1; k < emojiIds.length; k++) { |
||||
lineText += `<br /> \`:${emojiIds[k]}:\` `; |
||||
} |
||||
} else if (githubEmojiIds.length > columns) { |
||||
lineText += "| | "; |
||||
} |
||||
} |
||||
lineText += `| ${rightText} |`; |
||||
lineTexts.push(lineText); |
||||
} |
||||
|
||||
return lineTexts; |
||||
} |
||||
|
||||
module.exports = { |
||||
generateCheatSheet |
||||
}; |
||||
Loading…
Reference in new issue