diff --git a/app/javascript/mastodon/features/emoji/database.ts b/app/javascript/mastodon/features/emoji/database.ts index 9e03b53d3c..fe4010a861 100644 --- a/app/javascript/mastodon/features/emoji/database.ts +++ b/app/javascript/mastodon/features/emoji/database.ts @@ -5,11 +5,7 @@ import { openDB } from 'idb'; import { EMOJI_DB_SHORTCODE_TEST } from './constants'; import { toSupportedLocale, toSupportedLocaleOrCustom } from './locale'; -import type { - CustomEmojiData, - UnicodeEmojiData, - LocaleOrCustom, -} from './types'; +import type { CustomEmojiData, UnicodeEmojiData, EtagTypes } from './types'; import { emojiLogger } from './utils'; interface EmojiDB extends LocaleTables, DBSchema { @@ -32,7 +28,7 @@ interface EmojiDB extends LocaleTables, DBSchema { }; }; etags: { - key: LocaleOrCustom; + key: EtagTypes; value: string; }; } @@ -197,10 +193,9 @@ export async function putLegacyShortcodes(shortcodes: ShortcodesDataset) { await trx.done; } -export async function putLatestEtag(etag: string, localeString: string) { - const locale = toSupportedLocaleOrCustom(localeString); +export async function putLatestEtag(etag: string, name: EtagTypes) { const db = await loadDB(); - await db.put('etags', etag, locale); + await db.put('etags', etag, name); } export async function clearEtag(localeString: string) { diff --git a/app/javascript/mastodon/features/emoji/index.ts b/app/javascript/mastodon/features/emoji/index.ts index b0b0fb49b2..b134d88441 100644 --- a/app/javascript/mastodon/features/emoji/index.ts +++ b/app/javascript/mastodon/features/emoji/index.ts @@ -1,13 +1,9 @@ -import type { Locale } from 'emojibase'; - import { initialState } from '@/mastodon/initial_state'; -import type { EMOJI_DB_NAME_SHORTCODES, EMOJI_TYPE_CUSTOM } from './constants'; +import type { EMOJI_DB_NAME_SHORTCODES } from './constants'; import { toSupportedLocale } from './locale'; import type { LocaleOrCustom } from './types'; import { emojiLogger } from './utils'; -// eslint-disable-next-line import/default -- Importing via worker loader. -import EmojiWorker from './worker?worker&inline'; const userLocale = toSupportedLocale(initialState?.meta.locale ?? 'en'); @@ -18,13 +14,14 @@ const log = emojiLogger('index'); // This is too short, but better to fallback quickly than wait. const WORKER_TIMEOUT = 1_000; -export function initializeEmoji() { +export async function initializeEmoji() { log('initializing emojis'); // Create a temp worker, and assign it to the module-level worker once we know it's ready. let tempWorker: Worker | null = null; if (!worker && 'Worker' in window) { try { + const { default: EmojiWorker } = await import('./worker?worker&inline'); tempWorker = new EmojiWorker(); } catch (err) { console.warn('Error creating web worker:', err); @@ -64,7 +61,7 @@ async function fallbackLoad() { await loadCustomEmoji(); const { importLegacyShortcodes } = await import('./loader'); const shortcodes = await importLegacyShortcodes(); - if (shortcodes.length) { + if (shortcodes?.length) { log('loaded %d legacy shortcodes', shortcodes.length); } await loadEmojiLocale(userLocale); @@ -72,14 +69,11 @@ async function fallbackLoad() { async function loadEmojiLocale(localeString: string) { const locale = toSupportedLocale(localeString); - const { importEmojiData, localeToEmojiPath, localeToShortcodesPath } = - await import('./loader'); + const { importEmojiData } = await import('./loader'); if (worker) { - const path = await localeToEmojiPath(locale); - const shortcodesPath = await localeToShortcodesPath(locale); - log('asking worker to load locale %s from %s', locale, path); - messageWorker(locale, path, shortcodesPath); + log('asking worker to load locale %s', locale); + messageWorker(locale); } else { const emojis = await importEmojiData(locale); if (emojis) { @@ -100,17 +94,11 @@ export async function loadCustomEmoji() { } } -function messageWorker( - locale: typeof EMOJI_TYPE_CUSTOM | typeof EMOJI_DB_NAME_SHORTCODES, -): void; -function messageWorker(locale: Locale, path: string, shortcodes?: string): void; function messageWorker( locale: LocaleOrCustom | typeof EMOJI_DB_NAME_SHORTCODES, - path?: string, - shortcodes?: string, ) { if (!worker) { return; } - worker.postMessage({ locale, path, shortcodes }); + worker.postMessage({ locale }); } diff --git a/app/javascript/mastodon/features/emoji/loader.ts b/app/javascript/mastodon/features/emoji/loader.ts index 0dfa22b99d..c6b64fe29c 100644 --- a/app/javascript/mastodon/features/emoji/loader.ts +++ b/app/javascript/mastodon/features/emoji/loader.ts @@ -13,46 +13,35 @@ import { putLatestEtag, putLegacyShortcodes, } from './database'; -import { toSupportedLocale, toSupportedLocaleOrCustom } from './locale'; +import { toSupportedLocale, toValidEtagName } from './locale'; import type { CustomEmojiData } from './types'; +import { emojiLogger } from './utils'; -export async function importEmojiData( - localeString: string, - path?: string, - shortcodes: boolean | string = true, -) { +const log = emojiLogger('loader'); + +export async function importEmojiData(localeString: string, shortcodes = true) { const locale = toSupportedLocale(localeString); - // Validate the provided path. - if (path && !/^[/a-z]*\/packs\/assets\/compact-\w+\.json$/.test(path)) { - throw new Error('Invalid path for emoji data'); - } else { - // Otherwise get the path if not provided. - path ??= await localeToEmojiPath(locale); - } + log( + 'importing emoji data for locale %s%s', + locale, + shortcodes ? ' and shortcodes' : '', + ); - const emojis = await fetchAndCheckEtag(locale, path); + const emojis = await fetchAndCheckEtag({ + etagString: locale, + path: localeToEmojiPath(locale), + }); if (!emojis) { return; } const shortcodesData: ShortcodesDataset[] = []; if (shortcodes) { - if ( - typeof shortcodes === 'string' && - !/^[/a-z]*\/packs\/assets\/shortcodes\/cldr\.json$/.test(shortcodes) - ) { - throw new Error('Invalid path for shortcodes data'); - } - const shortcodesPath = - typeof shortcodes === 'string' - ? shortcodes - : await localeToShortcodesPath(locale); - const shortcodesResponse = await fetchAndCheckEtag( - locale, - shortcodesPath, - false, - ); + const shortcodesResponse = await fetchAndCheckEtag({ + etagString: `${locale}-shortcodes`, + path: localeToShortcodesPath(locale), + }); if (shortcodesResponse) { shortcodesData.push(shortcodesResponse); } else { @@ -69,10 +58,10 @@ export async function importEmojiData( } export async function importCustomEmojiData() { - const emojis = await fetchAndCheckEtag( - 'custom', - '/api/v1/custom_emojis', - ); + const emojis = await fetchAndCheckEtag({ + etagString: 'custom', + path: '/api/v1/custom_emojis', + }); if (!emojis) { return; } @@ -81,76 +70,76 @@ export async function importCustomEmojiData() { } export async function importLegacyShortcodes() { - const { default: shortcodesPath } = - await import('emojibase-data/en/shortcodes/iamcal.json?url'); - const response = await fetch(shortcodesPath); - if (!response.ok) { - throw new Error( - `Failed to fetch legacy shortcodes data: ${response.statusText}`, - ); + const globPaths = import.meta.glob( + // We use import.meta.glob to eagerly load the URL, as the regular import() doesn't work inside the Web Worker. + '../../../../../node_modules/emojibase-data/en/shortcodes/iamcal.json', + { eager: true, import: 'default', query: '?url' }, + ); + const path = Object.values(globPaths)[0]; + if (!path) { + throw new Error('IAMCAL shortcodes path not found'); + } + const shortcodesData = await fetchAndCheckEtag({ + checkEtag: true, + etagString: 'shortcodes', + path, + }); + if (!shortcodesData) { + return; } - const shortcodesData = (await response.json()) as ShortcodesDataset; await putLegacyShortcodes(shortcodesData); return Object.keys(shortcodesData); } -const emojiModules = new Map( - Object.entries( - import.meta.glob( - '../../../../../node_modules/emojibase-data/**/compact.json', - { - query: '?url', - import: 'default', - }, - ), - ).map(([key, loader]) => { - const match = /emojibase-data\/([^/]+)\/compact\.json$/.exec(key); - return [match?.at(1) ?? key, loader]; - }), -); - -export function localeToEmojiPath(locale: Locale) { - const path = emojiModules.get(locale); +function localeToEmojiPath(locale: Locale) { + const key = `../../../../../node_modules/emojibase-data/${locale}/compact.json`; + const emojiModules = import.meta.glob( + '../../../../../node_modules/emojibase-data/**/compact.json', + { + query: '?url', + import: 'default', + eager: true, + }, + ); + const path = emojiModules[key]; if (!path) { throw new Error(`Unsupported locale: ${locale}`); } - return path(); + return path; } -const shortcodesModules = new Map( - Object.entries( - import.meta.glob( - '../../../../../node_modules/emojibase-data/**/shortcodes/cldr.json', - { - query: '?url', - import: 'default', - }, - ), - ).map(([key, loader]) => { - const match = /emojibase-data\/([^/]+)\/shortcodes\/cldr\.json$/.exec(key); - return [match?.at(1) ?? key, loader]; - }), -); - -export function localeToShortcodesPath(locale: Locale) { - const path = shortcodesModules.get(locale); +function localeToShortcodesPath(locale: Locale) { + const key = `../../../../../node_modules/emojibase-data/${locale}/shortcodes/cldr.json`; + const shortcodesModules = import.meta.glob( + '../../../../../node_modules/emojibase-data/**/shortcodes/cldr.json', + { + query: '?url', + import: 'default', + eager: true, + }, + ); + const path = shortcodesModules[key]; if (!path) { throw new Error(`Unsupported locale for shortcodes: ${locale}`); } - return path(); + return path; } -export async function fetchAndCheckEtag( - localeString: string, - path: string, - checkEtag = true, -): Promise { - const locale = toSupportedLocaleOrCustom(localeString); +async function fetchAndCheckEtag({ + etagString, + path, + checkEtag = false, +}: { + etagString: string; + path: string; + checkEtag?: boolean; +}): Promise { + const etagName = toValidEtagName(etagString); // Use location.origin as this script may be loaded from a CDN domain. const url = new URL(path, location.origin); - const oldEtag = checkEtag ? await loadLatestEtag(locale) : null; + const oldEtag = checkEtag ? await loadLatestEtag(etagName) : null; const response = await fetch(url, { headers: { 'Content-Type': 'application/json', @@ -163,7 +152,7 @@ export async function fetchAndCheckEtag( } if (!response.ok) { throw new Error( - `Failed to fetch emoji data for ${locale}: ${response.statusText}`, + `Failed to fetch emoji data for ${etagName}: ${response.statusText}`, ); } @@ -172,7 +161,8 @@ export async function fetchAndCheckEtag( // Store the ETag for future requests const etag = response.headers.get('ETag'); if (etag && checkEtag) { - await putLatestEtag(etag, localeString); + log(`storing new etag for ${etagName}: ${etag}`); + await putLatestEtag(etag, etagName); } return data; diff --git a/app/javascript/mastodon/features/emoji/locale.ts b/app/javascript/mastodon/features/emoji/locale.ts index 8ff23f5161..f39b56d47c 100644 --- a/app/javascript/mastodon/features/emoji/locale.ts +++ b/app/javascript/mastodon/features/emoji/locale.ts @@ -1,7 +1,8 @@ import type { Locale } from 'emojibase'; import { SUPPORTED_LOCALES } from 'emojibase'; -import type { LocaleOrCustom } from './types'; +import { EMOJI_DB_NAME_SHORTCODES, EMOJI_TYPE_CUSTOM } from './constants'; +import type { EtagTypes, LocaleOrCustom, LocaleWithShortcodes } from './types'; export function toSupportedLocale(localeBase: string): Locale { const locale = localeBase.toLowerCase(); @@ -12,12 +13,35 @@ export function toSupportedLocale(localeBase: string): Locale { } export function toSupportedLocaleOrCustom(locale: string): LocaleOrCustom { - if (locale.toLowerCase() === 'custom') { - return 'custom'; + if (locale.toLowerCase() === EMOJI_TYPE_CUSTOM) { + return EMOJI_TYPE_CUSTOM; } return toSupportedLocale(locale); } -function isSupportedLocale(locale: string): locale is Locale { - return SUPPORTED_LOCALES.includes(locale.toLowerCase() as Locale); +export function toValidEtagName(input: string): EtagTypes { + const lower = input.toLowerCase(); + if (lower === EMOJI_TYPE_CUSTOM || lower === EMOJI_DB_NAME_SHORTCODES) { + return lower; + } + + if (isLocaleWithShortcodes(lower)) { + return lower; + } + + return toSupportedLocale(lower); +} + +function isSupportedLocale(locale: string): locale is Locale { + return SUPPORTED_LOCALES.includes(locale as Locale); +} + +function isLocaleWithShortcodes(input: string): input is LocaleWithShortcodes { + const [baseLocale, shortcodes] = input.split('-'); + return ( + !!baseLocale && + !!shortcodes && + isSupportedLocale(baseLocale) && + shortcodes === EMOJI_DB_NAME_SHORTCODES + ); } diff --git a/app/javascript/mastodon/features/emoji/render.ts b/app/javascript/mastodon/features/emoji/render.ts index 38bc7fd7e5..8fe311014a 100644 --- a/app/javascript/mastodon/features/emoji/render.ts +++ b/app/javascript/mastodon/features/emoji/render.ts @@ -4,12 +4,6 @@ import { EMOJI_TYPE_UNICODE, EMOJI_TYPE_CUSTOM, } from './constants'; -import { - loadEmojiByHexcode, - loadLegacyShortcodesByShortcode, - LocaleNotLoadedError, -} from './database'; -import { importEmojiData } from './loader'; import { emojiToUnicodeHex } from './normalize'; import type { EmojiLoadedState, @@ -121,6 +115,12 @@ export async function loadEmojiDataToState( return null; } + const { + loadLegacyShortcodesByShortcode, + loadEmojiByHexcode, + LocaleNotLoadedError, + } = await import('./database'); + // First, try to load the data from IndexedDB. try { const legacyCode = await loadLegacyShortcodesByShortcode(state.code); @@ -155,6 +155,7 @@ export async function loadEmojiDataToState( state.code, locale, ); + const { importEmojiData } = await import('./loader'); await importEmojiData(locale); // Use this from the loader file as it can be awaited. return loadEmojiDataToState(state, locale, true); } diff --git a/app/javascript/mastodon/features/emoji/types.ts b/app/javascript/mastodon/features/emoji/types.ts index 541fc428e6..03002dda64 100644 --- a/app/javascript/mastodon/features/emoji/types.ts +++ b/app/javascript/mastodon/features/emoji/types.ts @@ -7,6 +7,7 @@ import type { CustomEmoji } from '@/mastodon/models/custom_emoji'; import type { RequiredExcept } from '@/mastodon/utils/types'; import type { + EMOJI_DB_NAME_SHORTCODES, EMOJI_MODE_NATIVE, EMOJI_MODE_NATIVE_WITH_FLAGS, EMOJI_MODE_TWEMOJI, @@ -20,6 +21,11 @@ export type EmojiMode = | typeof EMOJI_MODE_TWEMOJI; export type LocaleOrCustom = Locale | typeof EMOJI_TYPE_CUSTOM; +export type LocaleWithShortcodes = `${Locale}-shortcodes`; +export type EtagTypes = + | LocaleOrCustom + | typeof EMOJI_DB_NAME_SHORTCODES + | LocaleWithShortcodes; export interface EmojiAppState { locales: Locale[]; diff --git a/app/javascript/mastodon/features/emoji/worker.ts b/app/javascript/mastodon/features/emoji/worker.ts index 2243678276..5602577dbe 100644 --- a/app/javascript/mastodon/features/emoji/worker.ts +++ b/app/javascript/mastodon/features/emoji/worker.ts @@ -8,24 +8,23 @@ import { addEventListener('message', handleMessage); self.postMessage('ready'); // After the worker is ready, notify the main thread -function handleMessage(event: MessageEvent<{ locale: string; path?: string }>) { +function handleMessage(event: MessageEvent<{ locale: string }>) { const { - data: { locale, path }, + data: { locale }, } = event; - void loadData(locale, path); + void loadData(locale); } -async function loadData(locale: string, path?: string) { +async function loadData(locale: string) { let importCount: number | undefined; if (locale === EMOJI_TYPE_CUSTOM) { importCount = (await importCustomEmojiData())?.length; } else if (locale === EMOJI_DB_NAME_SHORTCODES) { - importCount = (await importLegacyShortcodes()).length; - } else if (path) { - importCount = (await importEmojiData(locale, path))?.length; + importCount = (await importLegacyShortcodes())?.length; } else { - throw new Error('Path is required for loading locale emoji data'); + importCount = (await importEmojiData(locale))?.length; } + if (importCount) { self.postMessage(`loaded ${importCount} emojis into ${locale}`); } diff --git a/app/javascript/mastodon/main.tsx b/app/javascript/mastodon/main.tsx index 5d1d0aa513..965645f9e6 100644 --- a/app/javascript/mastodon/main.tsx +++ b/app/javascript/mastodon/main.tsx @@ -30,7 +30,7 @@ function main() { } const { initializeEmoji } = await import('./features/emoji/index'); - initializeEmoji(); + await initializeEmoji(); const root = createRoot(mountNode); root.render();