diff --git a/Gemfile.lock b/Gemfile.lock index 094c9b1d6e..4c232743bf 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -362,7 +362,7 @@ GEM rack (>= 2.2, < 4) rdf (~> 3.3) rexml (~> 3.2) - json-ld-preloaded (3.3.1) + json-ld-preloaded (3.3.2) json-ld (~> 3.3) rdf (~> 3.3) json-schema (5.2.1) @@ -627,7 +627,7 @@ GEM prism (1.4.0) prometheus_exporter (2.2.0) webrick - propshaft (1.2.0) + propshaft (1.2.1) actionpack (>= 7.0.0) activesupport (>= 7.0.0) rack @@ -759,7 +759,7 @@ GEM rspec-expectations (~> 3.13) rspec-mocks (~> 3.13) rspec-support (~> 3.13) - rspec-sidekiq (5.1.0) + rspec-sidekiq (5.2.0) rspec-core (~> 3.0) rspec-expectations (~> 3.0) rspec-mocks (~> 3.0) diff --git a/app/controllers/admin/accounts_controller.rb b/app/controllers/admin/accounts_controller.rb index 10391aa3e2..e140693014 100644 --- a/app/controllers/admin/accounts_controller.rb +++ b/app/controllers/admin/accounts_controller.rb @@ -16,11 +16,14 @@ module Admin def batch authorize :account, :index? - @form = Form::AccountBatch.new(form_account_batch_params) - @form.current_account = current_account - @form.action = action_from_button - @form.select_all_matching = params[:select_all_matching] - @form.query = filtered_accounts + @form = Form::AccountBatch.new( + form_account_batch_params.merge( + action: action_from_button, + current_account:, + query: filtered_accounts, + select_all_matching: params[:select_all_matching] + ) + ) @form.save rescue ActionController::ParameterMissing flash[:alert] = I18n.t('admin.accounts.no_account_selected') diff --git a/app/controllers/admin/tags_controller.rb b/app/controllers/admin/tags_controller.rb index a7bfd64794..f2c28328f8 100644 --- a/app/controllers/admin/tags_controller.rb +++ b/app/controllers/admin/tags_controller.rb @@ -5,6 +5,7 @@ module Admin before_action :set_tag, except: [:index] PER_PAGE = 20 + PERIOD_DAYS = 6.days def index authorize :tag, :index? @@ -15,7 +16,7 @@ module Admin def show authorize @tag, :show? - @time_period = (6.days.ago.to_date...Time.now.utc.to_date) + @time_period = report_range end def update @@ -24,7 +25,7 @@ module Admin if @tag.update(tag_params.merge(reviewed_at: Time.now.utc)) redirect_to admin_tag_path(@tag.id), notice: I18n.t('admin.tags.updated_msg') else - @time_period = (6.days.ago.to_date...Time.now.utc.to_date) + @time_period = report_range render :show end @@ -36,6 +37,10 @@ module Admin @tag = Tag.find(params[:id]) end + def report_range + (PERIOD_DAYS.ago.to_date...Time.now.utc.to_date) + end + def tag_params params .expect(tag: [:name, :display_name, :trendable, :usable, :listable]) diff --git a/app/controllers/api/v1/statuses_controller.rb b/app/controllers/api/v1/statuses_controller.rb index 4037078fb3..a674883e17 100644 --- a/app/controllers/api/v1/statuses_controller.rb +++ b/app/controllers/api/v1/statuses_controller.rb @@ -2,6 +2,7 @@ class Api::V1::StatusesController < Api::BaseController include Authorization + include AsyncRefreshesConcern before_action -> { authorize_if_got_token! :read, :'read:statuses' }, except: [:create, :update, :destroy] before_action -> { doorkeeper_authorize! :write, :'write:statuses' }, only: [:create, :update, :destroy] @@ -57,9 +58,17 @@ class Api::V1::StatusesController < Api::BaseController @context = Context.new(ancestors: loaded_ancestors, descendants: loaded_descendants) statuses = [@status] + @context.ancestors + @context.descendants - render json: @context, serializer: REST::ContextSerializer, relationships: StatusRelationshipsPresenter.new(statuses, current_user&.account_id) + refresh_key = "context:#{@status.id}:refresh" + async_refresh = AsyncRefresh.new(refresh_key) - ActivityPub::FetchAllRepliesWorker.perform_async(@status.id) if !current_account.nil? && @status.should_fetch_replies? + if async_refresh.running? + add_async_refresh_header(async_refresh) + elsif !current_account.nil? && @status.should_fetch_replies? + add_async_refresh_header(AsyncRefresh.create(refresh_key)) + ActivityPub::FetchAllRepliesWorker.perform_async(@status.id) + end + + render json: @context, serializer: REST::ContextSerializer, relationships: StatusRelationshipsPresenter.new(statuses, current_user&.account_id) end def create diff --git a/app/controllers/concerns/auth/captcha_concern.rb b/app/controllers/concerns/auth/captcha_concern.rb index c01da21249..a6232db943 100644 --- a/app/controllers/concerns/auth/captcha_concern.rb +++ b/app/controllers/concerns/auth/captcha_concern.rb @@ -5,6 +5,18 @@ module Auth::CaptchaConcern include Hcaptcha::Adapters::ViewMethods + CAPTCHA_DIRECTIVES = %w( + connect_src + frame_src + script_src + style_src + ).freeze + + CAPTCHA_SOURCES = %w( + https://*.hcaptcha.com + https://hcaptcha.com + ).freeze + included do helper_method :render_captcha end @@ -42,20 +54,9 @@ module Auth::CaptchaConcern end def extend_csp_for_captcha! - policy = request.content_security_policy&.clone + return unless captcha_required? && request.content_security_policy.present? - return unless captcha_required? && policy.present? - - %w(script_src frame_src style_src connect_src).each do |directive| - values = policy.send(directive) - - values << 'https://hcaptcha.com' unless values.include?('https://hcaptcha.com') || values.include?('https:') - values << 'https://*.hcaptcha.com' unless values.include?('https://*.hcaptcha.com') || values.include?('https:') - - policy.send(directive, *values) - end - - request.content_security_policy = policy + request.content_security_policy = captcha_adjusted_policy end def render_captcha @@ -63,4 +64,24 @@ module Auth::CaptchaConcern hcaptcha_tags end + + private + + def captcha_adjusted_policy + request.content_security_policy.clone.tap do |policy| + populate_captcha_policy(policy) + end + end + + def populate_captcha_policy(policy) + CAPTCHA_DIRECTIVES.each do |directive| + values = policy.send(directive) + + CAPTCHA_SOURCES.each do |source| + values << source unless values.include?(source) || values.include?('https:') + end + + policy.send(directive, *values) + end + end end diff --git a/app/javascript/flavours/glitch/actions/statuses_typed.ts b/app/javascript/flavours/glitch/actions/statuses_typed.ts index 840dba5bc1..b79d98df07 100644 --- a/app/javascript/flavours/glitch/actions/statuses_typed.ts +++ b/app/javascript/flavours/glitch/actions/statuses_typed.ts @@ -1,3 +1,5 @@ +import { createAction } from '@reduxjs/toolkit'; + import { apiGetContext } from 'flavours/glitch/api/statuses'; import { createDataLoadingThunk } from 'flavours/glitch/store/typed_functions'; @@ -6,13 +8,18 @@ import { importFetchedStatuses } from './importer'; export const fetchContext = createDataLoadingThunk( 'status/context', ({ statusId }: { statusId: string }) => apiGetContext(statusId), - (context, { dispatch }) => { + ({ context, refresh }, { dispatch }) => { const statuses = context.ancestors.concat(context.descendants); dispatch(importFetchedStatuses(statuses)); return { context, + refresh, }; }, ); + +export const completeContextRefresh = createAction<{ statusId: string }>( + 'status/context/complete', +); diff --git a/app/javascript/flavours/glitch/api.ts b/app/javascript/flavours/glitch/api.ts index 912948f7d3..ca6dec0974 100644 --- a/app/javascript/flavours/glitch/api.ts +++ b/app/javascript/flavours/glitch/api.ts @@ -15,6 +15,50 @@ export const getLinks = (response: AxiosResponse) => { return LinkHeader.parse(value); }; +export interface AsyncRefreshHeader { + id: string; + retry: number; +} + +const isAsyncRefreshHeader = (obj: object): obj is AsyncRefreshHeader => + 'id' in obj && 'retry' in obj; + +export const getAsyncRefreshHeader = ( + response: AxiosResponse, +): AsyncRefreshHeader | null => { + const value = response.headers['mastodon-async-refresh'] as + | string + | undefined; + + if (!value) { + return null; + } + + const asyncRefreshHeader: Record = {}; + + value.split(/,\s*/).forEach((pair) => { + const [key, val] = pair.split('=', 2); + + let typedValue: string | number; + + if (key && ['id', 'retry'].includes(key) && val) { + if (val.startsWith('"')) { + typedValue = val.slice(1, -1); + } else { + typedValue = parseInt(val); + } + + asyncRefreshHeader[key] = typedValue; + } + }); + + if (isAsyncRefreshHeader(asyncRefreshHeader)) { + return asyncRefreshHeader; + } + + return null; +}; + const csrfHeader: RawAxiosRequestHeaders = {}; const setCSRFHeader = () => { @@ -62,7 +106,7 @@ export default function api(withAuthorization = true) { }); } -type ApiUrl = `v${1 | 2}/${string}`; +type ApiUrl = `v${1 | '1_alpha' | 2}/${string}`; type RequestParamsOrData = Record; export async function apiRequest( diff --git a/app/javascript/flavours/glitch/api/async_refreshes.ts b/app/javascript/flavours/glitch/api/async_refreshes.ts new file mode 100644 index 0000000000..8d0b3dba93 --- /dev/null +++ b/app/javascript/flavours/glitch/api/async_refreshes.ts @@ -0,0 +1,5 @@ +import { apiRequestGet } from 'flavours/glitch/api'; +import type { ApiAsyncRefreshJSON } from 'flavours/glitch/api_types/async_refreshes'; + +export const apiGetAsyncRefresh = (id: string) => + apiRequestGet(`v1_alpha/async_refreshes/${id}`); diff --git a/app/javascript/flavours/glitch/api/statuses.ts b/app/javascript/flavours/glitch/api/statuses.ts index 3b6053a858..7e2a9ee8e0 100644 --- a/app/javascript/flavours/glitch/api/statuses.ts +++ b/app/javascript/flavours/glitch/api/statuses.ts @@ -1,5 +1,14 @@ -import { apiRequestGet } from 'flavours/glitch/api'; +import api, { getAsyncRefreshHeader } from 'flavours/glitch/api'; import type { ApiContextJSON } from 'flavours/glitch/api_types/statuses'; -export const apiGetContext = (statusId: string) => - apiRequestGet(`v1/statuses/${statusId}/context`); +export const apiGetContext = async (statusId: string) => { + const response = await api().request({ + method: 'GET', + url: `/api/v1/statuses/${statusId}/context`, + }); + + return { + context: response.data, + refresh: getAsyncRefreshHeader(response), + }; +}; diff --git a/app/javascript/flavours/glitch/api_types/async_refreshes.ts b/app/javascript/flavours/glitch/api_types/async_refreshes.ts new file mode 100644 index 0000000000..2d2fed2412 --- /dev/null +++ b/app/javascript/flavours/glitch/api_types/async_refreshes.ts @@ -0,0 +1,7 @@ +export interface ApiAsyncRefreshJSON { + async_refresh: { + id: string; + status: 'running' | 'finished'; + result_count: number; + }; +} diff --git a/app/javascript/flavours/glitch/components/account_bio.tsx b/app/javascript/flavours/glitch/components/account_bio.tsx index fff7e70c62..d2bb9d6ebf 100644 --- a/app/javascript/flavours/glitch/components/account_bio.tsx +++ b/app/javascript/flavours/glitch/components/account_bio.tsx @@ -2,27 +2,44 @@ import { useCallback } from 'react'; import { useLinks } from 'flavours/glitch/hooks/useLinks'; +import { EmojiHTML } from '../features/emoji/emoji_html'; +import { isFeatureEnabled } from '../initial_state'; +import { useAppSelector } from '../store'; + interface AccountBioProps { - note: string; className: string; - dropdownAccountId?: string; + accountId: string; + showDropdown?: boolean; } export const AccountBio: React.FC = ({ - note, className, - dropdownAccountId, + accountId, + showDropdown = false, }) => { - const handleClick = useLinks(!!dropdownAccountId); + const handleClick = useLinks(showDropdown); const handleNodeChange = useCallback( (node: HTMLDivElement | null) => { - if (!dropdownAccountId || !node || node.childNodes.length === 0) { + if (!showDropdown || !node || node.childNodes.length === 0) { return; } - addDropdownToHashtags(node, dropdownAccountId); + addDropdownToHashtags(node, accountId); }, - [dropdownAccountId], + [showDropdown, accountId], ); + const note = useAppSelector((state) => { + const account = state.accounts.get(accountId); + if (!account) { + return ''; + } + return isFeatureEnabled('modern_emojis') + ? account.note + : account.note_emojified; + }); + const extraEmojis = useAppSelector((state) => { + const account = state.accounts.get(accountId); + return account?.emojis; + }); if (note.length === 0) { return null; @@ -31,10 +48,11 @@ export const AccountBio: React.FC = ({ return (
+ > + +
); }; diff --git a/app/javascript/flavours/glitch/components/hover_card_account.tsx b/app/javascript/flavours/glitch/components/hover_card_account.tsx index 6938506faf..58ce247c58 100644 --- a/app/javascript/flavours/glitch/components/hover_card_account.tsx +++ b/app/javascript/flavours/glitch/components/hover_card_account.tsx @@ -106,7 +106,7 @@ export const HoverCardAccount = forwardRef< <>
diff --git a/app/javascript/flavours/glitch/components/status_content.jsx b/app/javascript/flavours/glitch/components/status_content.jsx index 78e895b548..52a470e0de 100644 --- a/app/javascript/flavours/glitch/components/status_content.jsx +++ b/app/javascript/flavours/glitch/components/status_content.jsx @@ -13,8 +13,9 @@ import ChevronRightIcon from '@/material-icons/400-24px/chevron_right.svg?react' import { Icon } from 'flavours/glitch/components/icon'; import { Poll } from 'flavours/glitch/components/poll'; import { identityContextPropShape, withIdentity } from 'flavours/glitch/identity_context'; -import { autoPlayGif, languages as preloadedLanguages } from 'flavours/glitch/initial_state'; +import { autoPlayGif, isFeatureEnabled, languages as preloadedLanguages } from 'flavours/glitch/initial_state'; import { decode as decodeIDNA } from 'flavours/glitch/utils/idna'; +import { EmojiHTML } from '../features/emoji/emoji_html'; const MAX_HEIGHT = 706; // 22px * 32 (+ 2px padding at the top) @@ -79,6 +80,9 @@ const isLinkMisleading = (link) => { * @returns {string} */ export function getStatusContent(status) { + if (isFeatureEnabled('modern_emojis')) { + return status.getIn(['translation', 'content']) || status.get('content'); + } return status.getIn(['translation', 'contentHtml']) || status.get('contentHtml'); } @@ -325,7 +329,7 @@ class StatusContent extends PureComponent { const targetLanguages = this.props.languages?.get(status.get('language') || 'und'); const renderTranslate = this.props.onTranslate && this.props.identity.signedIn && ['public', 'unlisted'].includes(status.get('visibility')) && status.get('search_index').trim().length > 0 && targetLanguages?.includes(contentLocale); - const content = { __html: statusContent ?? getStatusContent(status) }; + const content = statusContent ?? getStatusContent(status); const language = status.getIn(['translation', 'language']) || status.get('language'); const classNames = classnames('status__content', { 'status__content--with-action': this.props.onClick && this.props.history, @@ -350,7 +354,12 @@ class StatusContent extends PureComponent { return ( <>
-
+ {poll} {translateButton} @@ -362,7 +371,12 @@ class StatusContent extends PureComponent { } else { return (
-
+ {poll} {translateButton} diff --git a/app/javascript/flavours/glitch/components/status_list.jsx b/app/javascript/flavours/glitch/components/status_list.jsx index 3323c6faae..fab7d57831 100644 --- a/app/javascript/flavours/glitch/components/status_list.jsx +++ b/app/javascript/flavours/glitch/components/status_list.jsx @@ -41,6 +41,12 @@ export default class StatusList extends ImmutablePureComponent { trackScroll: true, }; + componentDidMount() { + this.columnHeaderHeight = parseFloat( + getComputedStyle(this.node.node).getPropertyValue('--column-header-height') + ) || 0; + } + getFeaturedStatusCount = () => { return this.props.featuredStatusIds ? this.props.featuredStatusIds.size : 0; }; @@ -54,35 +60,68 @@ export default class StatusList extends ImmutablePureComponent { }; handleMoveUp = (id, featured) => { - const elementIndex = this.getCurrentStatusIndex(id, featured) - 1; - this._selectChild(elementIndex, true); + const index = this.getCurrentStatusIndex(id, featured); + this._selectChild(id, index, -1); }; handleMoveDown = (id, featured) => { - const elementIndex = this.getCurrentStatusIndex(id, featured) + 1; - this._selectChild(elementIndex, false); + const index = this.getCurrentStatusIndex(id, featured); + this._selectChild(id, index, 1); }; + _selectChild = (id, index, direction) => { + const listContainer = this.node.node; + let listItem = listContainer.querySelector( + // :nth-child uses 1-based indexing + `.item-list > :nth-child(${index + 1 + direction})` + ); + + if (!listItem) { + return; + } + + // If selected container element is empty, we skip it + if (listItem.matches(':empty')) { + this._selectChild(id, index + direction, direction); + return; + } + + // Check if the list item is a post + let targetElement = listItem.querySelector('.focusable'); + + // Otherwise, check if the item contains follow suggestions or + // is a 'load more' button. + if ( + !targetElement && ( + listItem.querySelector('.inline-follow-suggestions') || + listItem.matches('.load-more') + ) + ) { + targetElement = listItem; + } + + if (targetElement) { + const elementRect = targetElement.getBoundingClientRect(); + + const isFullyVisible = + elementRect.top >= this.columnHeaderHeight && + elementRect.bottom <= window.innerHeight; + + if (!isFullyVisible) { + targetElement.scrollIntoView({ + block: direction === 1 ? 'start' : 'center', + }); + } + + targetElement.focus(); + } + } + handleLoadOlder = debounce(() => { const { statusIds, lastId, onLoadMore } = this.props; onLoadMore(lastId || (statusIds.size > 0 ? statusIds.last() : undefined)); }, 300, { leading: true }); - _selectChild (index, align_top) { - const container = this.node.node; - // TODO: This breaks at the inline-follow-suggestions container - const element = container.querySelector(`article:nth-of-type(${index + 1}) .focusable`); - - if (element) { - if (align_top && container.scrollTop > element.offsetTop) { - element.scrollIntoView(true); - } else if (!align_top && container.scrollTop + container.clientHeight < element.offsetTop + element.offsetHeight) { - element.scrollIntoView(false); - } - element.focus(); - } - } - setRef = c => { this.node = c; }; diff --git a/app/javascript/flavours/glitch/features/account_timeline/components/account_header.tsx b/app/javascript/flavours/glitch/features/account_timeline/components/account_header.tsx index 1d49947553..1788effd7c 100644 --- a/app/javascript/flavours/glitch/features/account_timeline/components/account_header.tsx +++ b/app/javascript/flavours/glitch/features/account_timeline/components/account_header.tsx @@ -902,8 +902,7 @@ export const AccountHeader: React.FC<{ )} diff --git a/app/javascript/flavours/glitch/features/emoji/constants.ts b/app/javascript/flavours/glitch/features/emoji/constants.ts index d38f17f216..09022371b2 100644 --- a/app/javascript/flavours/glitch/features/emoji/constants.ts +++ b/app/javascript/flavours/glitch/features/emoji/constants.ts @@ -15,6 +15,16 @@ export const SKIN_TONE_CODES = [ 0x1f3ff, // Dark skin tone ] as const; +// Emoji rendering modes. A mode is what we are using to render emojis, a style is what the user has selected. +export const EMOJI_MODE_NATIVE = 'native'; +export const EMOJI_MODE_NATIVE_WITH_FLAGS = 'native-flags'; +export const EMOJI_MODE_TWEMOJI = 'twemoji'; + +export const EMOJI_TYPE_UNICODE = 'unicode'; +export const EMOJI_TYPE_CUSTOM = 'custom'; + +export const EMOJI_STATE_MISSING = 'missing'; + export const EMOJIS_WITH_DARK_BORDER = [ '🎱', // 1F3B1 '🐜', // 1F41C diff --git a/app/javascript/flavours/glitch/features/emoji/database.ts b/app/javascript/flavours/glitch/features/emoji/database.ts index 8d7e9c7a5e..0b8ddd34fb 100644 --- a/app/javascript/flavours/glitch/features/emoji/database.ts +++ b/app/javascript/flavours/glitch/features/emoji/database.ts @@ -1,17 +1,19 @@ import { SUPPORTED_LOCALES } from 'emojibase'; -import type { FlatCompactEmoji, Locale } from 'emojibase'; -import type { DBSchema } from 'idb'; +import type { Locale } from 'emojibase'; +import type { DBSchema, IDBPDatabase } from 'idb'; import { openDB } from 'idb'; -import type { ApiCustomEmojiJSON } from '@/flavours/glitch/api_types/custom_emoji'; - -import type { LocaleOrCustom } from './locale'; import { toSupportedLocale, toSupportedLocaleOrCustom } from './locale'; +import type { + CustomEmojiData, + UnicodeEmojiData, + LocaleOrCustom, +} from './types'; interface EmojiDB extends LocaleTables, DBSchema { custom: { key: string; - value: ApiCustomEmojiJSON; + value: CustomEmojiData; indexes: { category: string; }; @@ -24,7 +26,7 @@ interface EmojiDB extends LocaleTables, DBSchema { interface LocaleTable { key: string; - value: FlatCompactEmoji; + value: UnicodeEmojiData; indexes: { group: number; label: string; @@ -36,63 +38,114 @@ type LocaleTables = Record; const SCHEMA_VERSION = 1; -const db = await openDB('mastodon-emoji', SCHEMA_VERSION, { - upgrade(database) { - const customTable = database.createObjectStore('custom', { - keyPath: 'shortcode', - autoIncrement: false, - }); - customTable.createIndex('category', 'category'); +let db: IDBPDatabase | null = null; - database.createObjectStore('etags'); - - for (const locale of SUPPORTED_LOCALES) { - const localeTable = database.createObjectStore(locale, { - keyPath: 'hexcode', +async function loadDB() { + if (db) { + return db; + } + db = await openDB('mastodon-emoji', SCHEMA_VERSION, { + upgrade(database) { + const customTable = database.createObjectStore('custom', { + keyPath: 'shortcode', autoIncrement: false, }); - localeTable.createIndex('group', 'group'); - localeTable.createIndex('label', 'label'); - localeTable.createIndex('order', 'order'); - localeTable.createIndex('tags', 'tags', { multiEntry: true }); - } - }, -}); + customTable.createIndex('category', 'category'); -export async function putEmojiData(emojis: FlatCompactEmoji[], locale: Locale) { + database.createObjectStore('etags'); + + for (const locale of SUPPORTED_LOCALES) { + const localeTable = database.createObjectStore(locale, { + keyPath: 'hexcode', + autoIncrement: false, + }); + localeTable.createIndex('group', 'group'); + localeTable.createIndex('label', 'label'); + localeTable.createIndex('order', 'order'); + localeTable.createIndex('tags', 'tags', { multiEntry: true }); + } + }, + }); + return db; +} + +export async function putEmojiData(emojis: UnicodeEmojiData[], locale: Locale) { + const db = await loadDB(); const trx = db.transaction(locale, 'readwrite'); await Promise.all(emojis.map((emoji) => trx.store.put(emoji))); await trx.done; } -export async function putCustomEmojiData(emojis: ApiCustomEmojiJSON[]) { +export async function putCustomEmojiData(emojis: CustomEmojiData[]) { + const db = await loadDB(); const trx = db.transaction('custom', 'readwrite'); await Promise.all(emojis.map((emoji) => trx.store.put(emoji))); await trx.done; } -export function putLatestEtag(etag: string, localeString: string) { +export async function putLatestEtag(etag: string, localeString: string) { const locale = toSupportedLocaleOrCustom(localeString); + const db = await loadDB(); return db.put('etags', etag, locale); } -export function searchEmojiByHexcode(hexcode: string, localeString: string) { +export async function searchEmojiByHexcode( + hexcode: string, + localeString: string, +) { const locale = toSupportedLocale(localeString); + const db = await loadDB(); return db.get(locale, hexcode); } -export function searchEmojiByTag(tag: string, localeString: string) { +export async function searchEmojisByHexcodes( + hexcodes: string[], + localeString: string, +) { + const locale = toSupportedLocale(localeString); + const db = await loadDB(); + return db.getAll( + locale, + IDBKeyRange.bound(hexcodes[0], hexcodes[hexcodes.length - 1]), + ); +} + +export async function searchEmojiByTag(tag: string, localeString: string) { const locale = toSupportedLocale(localeString); const range = IDBKeyRange.only(tag.toLowerCase()); + const db = await loadDB(); return db.getAllFromIndex(locale, 'tags', range); } -export function searchCustomEmojiByShortcode(shortcode: string) { +export async function searchCustomEmojiByShortcode(shortcode: string) { + const db = await loadDB(); return db.get('custom', shortcode); } +export async function searchCustomEmojisByShortcodes(shortcodes: string[]) { + const db = await loadDB(); + return db.getAll( + 'custom', + IDBKeyRange.bound(shortcodes[0], shortcodes[shortcodes.length - 1]), + ); +} + +export async function findMissingLocales(localeStrings: string[]) { + const locales = new Set(localeStrings.map(toSupportedLocale)); + const missingLocales: Locale[] = []; + const db = await loadDB(); + for (const locale of locales) { + const rowCount = await db.count(locale); + if (!rowCount) { + missingLocales.push(locale); + } + } + return missingLocales; +} + export async function loadLatestEtag(localeString: string) { const locale = toSupportedLocaleOrCustom(localeString); + const db = await loadDB(); const rowCount = await db.count(locale); if (!rowCount) { return null; // No data for this locale, return null even if there is an etag. diff --git a/app/javascript/flavours/glitch/features/emoji/emoji_html.tsx b/app/javascript/flavours/glitch/features/emoji/emoji_html.tsx new file mode 100644 index 0000000000..82c245190e --- /dev/null +++ b/app/javascript/flavours/glitch/features/emoji/emoji_html.tsx @@ -0,0 +1,81 @@ +import type { HTMLAttributes } from 'react'; +import { useEffect, useMemo, useState } from 'react'; + +import type { List as ImmutableList } from 'immutable'; +import { isList } from 'immutable'; + +import type { ApiCustomEmojiJSON } from '@/flavours/glitch/api_types/custom_emoji'; +import { isFeatureEnabled } from '@/flavours/glitch/initial_state'; +import type { CustomEmoji } from '@/flavours/glitch/models/custom_emoji'; + +import { useEmojiAppState } from './hooks'; +import { emojifyElement } from './render'; +import type { ExtraCustomEmojiMap } from './types'; + +type EmojiHTMLProps = Omit< + HTMLAttributes, + 'dangerouslySetInnerHTML' +> & { + htmlString: string; + extraEmojis?: ExtraCustomEmojiMap | ImmutableList; +}; + +export const EmojiHTML: React.FC = ({ + htmlString, + extraEmojis, + ...props +}) => { + if (isFeatureEnabled('modern_emojis')) { + return ( + + ); + } + return
; +}; + +const ModernEmojiHTML: React.FC = ({ + extraEmojis: rawEmojis, + htmlString: text, + ...props +}) => { + const appState = useEmojiAppState(); + const [innerHTML, setInnerHTML] = useState(''); + + const extraEmojis: ExtraCustomEmojiMap = useMemo(() => { + if (!rawEmojis) { + return {}; + } + if (isList(rawEmojis)) { + return ( + rawEmojis.toJS() as ApiCustomEmojiJSON[] + ).reduce( + (acc, emoji) => ({ ...acc, [emoji.shortcode]: emoji }), + {}, + ); + } + return rawEmojis; + }, [rawEmojis]); + + useEffect(() => { + if (!text) { + return; + } + const cb = async () => { + const div = document.createElement('div'); + div.innerHTML = text; + const ele = await emojifyElement(div, appState, extraEmojis); + setInnerHTML(ele.innerHTML); + }; + void cb(); + }, [text, appState, extraEmojis]); + + if (!innerHTML) { + return null; + } + + return
; +}; diff --git a/app/javascript/flavours/glitch/features/emoji/emoji_text.tsx b/app/javascript/flavours/glitch/features/emoji/emoji_text.tsx new file mode 100644 index 0000000000..253371391a --- /dev/null +++ b/app/javascript/flavours/glitch/features/emoji/emoji_text.tsx @@ -0,0 +1,45 @@ +import { useEffect, useState } from 'react'; + +import { useEmojiAppState } from './hooks'; +import { emojifyText } from './render'; + +interface EmojiTextProps { + text: string; +} + +export const EmojiText: React.FC = ({ text }) => { + const appState = useEmojiAppState(); + const [rendered, setRendered] = useState<(string | HTMLImageElement)[]>([]); + + useEffect(() => { + const cb = async () => { + const rendered = await emojifyText(text, appState); + setRendered(rendered ?? []); + }; + void cb(); + }, [text, appState]); + + if (rendered.length === 0) { + return null; + } + + return ( + <> + {rendered.map((fragment, index) => { + if (typeof fragment === 'string') { + return {fragment}; + } + return ( + {fragment.alt} + ); + })} + + ); +}; diff --git a/app/javascript/flavours/glitch/features/emoji/hooks.ts b/app/javascript/flavours/glitch/features/emoji/hooks.ts new file mode 100644 index 0000000000..e4870346a3 --- /dev/null +++ b/app/javascript/flavours/glitch/features/emoji/hooks.ts @@ -0,0 +1,16 @@ +import { useAppSelector } from '@/flavours/glitch/store'; + +import { toSupportedLocale } from './locale'; +import { determineEmojiMode } from './mode'; +import type { EmojiAppState } from './types'; + +export function useEmojiAppState(): EmojiAppState { + const locale = useAppSelector((state) => + toSupportedLocale(state.meta.get('locale') as string), + ); + const mode = useAppSelector((state) => + determineEmojiMode(state.meta.get('emoji_style') as string), + ); + + return { currentLocale: locale, locales: [locale], mode }; +} diff --git a/app/javascript/flavours/glitch/features/emoji/index.ts b/app/javascript/flavours/glitch/features/emoji/index.ts index 6521d1dd35..27e88f74b3 100644 --- a/app/javascript/flavours/glitch/features/emoji/index.ts +++ b/app/javascript/flavours/glitch/features/emoji/index.ts @@ -2,27 +2,44 @@ import initialState from '@/flavours/glitch/initial_state'; import { toSupportedLocale } from './locale'; -const serverLocale = toSupportedLocale(initialState?.meta.locale ?? 'en'); +const userLocale = toSupportedLocale(initialState?.meta.locale ?? 'en'); -const worker = - 'Worker' in window - ? new Worker(new URL('./worker', import.meta.url), { - type: 'module', - }) - : null; +let worker: Worker | null = null; export async function initializeEmoji() { + if (!worker && 'Worker' in window) { + try { + worker = new Worker(new URL('./worker', import.meta.url), { + type: 'module', + credentials: 'omit', + }); + } catch (err) { + console.warn('Error creating web worker:', err); + } + } + if (worker) { - worker.addEventListener('message', (event: MessageEvent) => { + // Assign worker to const to make TS happy inside the event listener. + const thisWorker = worker; + thisWorker.addEventListener('message', (event: MessageEvent) => { const { data: message } = event; if (message === 'ready') { - worker.postMessage(serverLocale); - worker.postMessage('custom'); + thisWorker.postMessage('custom'); + void loadEmojiLocale(userLocale); + // Load English locale as well, because people are still used to + // using it from before we supported other locales. + if (userLocale !== 'en') { + void loadEmojiLocale('en'); + } } }); } else { - const { importCustomEmojiData, importEmojiData } = await import('./loader'); - await Promise.all([importCustomEmojiData(), importEmojiData(serverLocale)]); + const { importCustomEmojiData } = await import('./loader'); + await importCustomEmojiData(); + await loadEmojiLocale(userLocale); + if (userLocale !== 'en') { + await loadEmojiLocale('en'); + } } } diff --git a/app/javascript/flavours/glitch/features/emoji/loader.ts b/app/javascript/flavours/glitch/features/emoji/loader.ts index 6ae1f5873b..ff77f721f6 100644 --- a/app/javascript/flavours/glitch/features/emoji/loader.ts +++ b/app/javascript/flavours/glitch/features/emoji/loader.ts @@ -11,7 +11,7 @@ import { putLatestEtag, } from './database'; import { toSupportedLocale, toSupportedLocaleOrCustom } from './locale'; -import type { LocaleOrCustom } from './locale'; +import type { LocaleOrCustom } from './types'; export async function importEmojiData(localeString: string) { const locale = toSupportedLocale(localeString); diff --git a/app/javascript/flavours/glitch/features/emoji/locale.ts b/app/javascript/flavours/glitch/features/emoji/locale.ts index 561c94afb0..8ff23f5161 100644 --- a/app/javascript/flavours/glitch/features/emoji/locale.ts +++ b/app/javascript/flavours/glitch/features/emoji/locale.ts @@ -1,7 +1,7 @@ import type { Locale } from 'emojibase'; import { SUPPORTED_LOCALES } from 'emojibase'; -export type LocaleOrCustom = Locale | 'custom'; +import type { LocaleOrCustom } from './types'; export function toSupportedLocale(localeBase: string): Locale { const locale = localeBase.toLowerCase(); diff --git a/app/javascript/flavours/glitch/features/emoji/mode.ts b/app/javascript/flavours/glitch/features/emoji/mode.ts new file mode 100644 index 0000000000..994881bca1 --- /dev/null +++ b/app/javascript/flavours/glitch/features/emoji/mode.ts @@ -0,0 +1,119 @@ +// Credit to Nolan Lawson for the original implementation. +// See: https://github.com/nolanlawson/emoji-picker-element/blob/master/src/picker/utils/testColorEmojiSupported.js + +import { isDevelopment } from '@/flavours/glitch/utils/environment'; + +import { + EMOJI_MODE_NATIVE, + EMOJI_MODE_NATIVE_WITH_FLAGS, + EMOJI_MODE_TWEMOJI, +} from './constants'; +import type { EmojiMode } from './types'; + +type Feature = Uint8ClampedArray; + +// See: https://github.com/nolanlawson/emoji-picker-element/blob/master/src/picker/constants.js +const FONT_FAMILY = + '"Twemoji Mozilla","Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol",' + + '"Noto Color Emoji","EmojiOne Color","Android Emoji",sans-serif'; + +function getTextFeature(text: string, color: string) { + const canvas = document.createElement('canvas'); + canvas.width = canvas.height = 1; + + const ctx = canvas.getContext('2d', { + // Improves the performance of `getImageData()` + // https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/getContextAttributes#willreadfrequently + willReadFrequently: true, + }); + if (!ctx) { + throw new Error('Canvas context not available'); + } + ctx.textBaseline = 'top'; + ctx.font = `100px ${FONT_FAMILY}`; + ctx.fillStyle = color; + ctx.scale(0.01, 0.01); + ctx.fillText(text, 0, 0); + + return ctx.getImageData(0, 0, 1, 1).data satisfies Feature; +} + +function compareFeatures(feature1: Feature, feature2: Feature) { + const feature1Str = [...feature1].join(','); + const feature2Str = [...feature2].join(','); + // This is RGBA, so for 0,0,0, we are checking that the first RGB is not all zeroes. + // Most of the time when unsupported this is 0,0,0,0, but on Chrome on Mac it is + // 0,0,0,61 - there is a transparency here. + return feature1Str === feature2Str && !feature1Str.startsWith('0,0,0,'); +} + +function testEmojiSupport(text: string) { + // Render white and black and then compare them to each other and ensure they're the same + // color, and neither one is black. This shows that the emoji was rendered in color. + const feature1 = getTextFeature(text, '#000'); + const feature2 = getTextFeature(text, '#fff'); + return compareFeatures(feature1, feature2); +} + +const EMOJI_VERSION_TEST_EMOJI = '🫨'; // shaking head, from v15 +const EMOJI_FLAG_TEST_EMOJI = 'πŸ‡¨πŸ‡­'; + +export function determineEmojiMode(style: string): EmojiMode { + if (style === EMOJI_MODE_NATIVE) { + // If flags are not supported, we replace them with Twemoji. + if (shouldReplaceFlags()) { + return EMOJI_MODE_NATIVE_WITH_FLAGS; + } + return EMOJI_MODE_NATIVE; + } + if (style === EMOJI_MODE_TWEMOJI) { + return EMOJI_MODE_TWEMOJI; + } + + // Auto style so determine based on browser capabilities. + if (shouldUseTwemoji()) { + return EMOJI_MODE_TWEMOJI; + } else if (shouldReplaceFlags()) { + return EMOJI_MODE_NATIVE_WITH_FLAGS; + } + return EMOJI_MODE_NATIVE; +} + +export function shouldUseTwemoji(): boolean { + if (typeof window === 'undefined') { + return false; + } + try { + // Test a known color emoji to see if 15.1 is supported. + return !testEmojiSupport(EMOJI_VERSION_TEST_EMOJI); + } catch (err: unknown) { + // If an error occurs, fall back to Twemoji to be safe. + if (isDevelopment()) { + console.warn( + 'Emoji rendering test failed, defaulting to Twemoji. Error:', + err, + ); + } + return true; + } +} + +// Based on https://github.com/talkjs/country-flag-emoji-polyfill/blob/master/src/index.ts#L19 +export function shouldReplaceFlags(): boolean { + if (typeof window === 'undefined') { + return false; + } + try { + // Test a known flag emoji to see if it is rendered in color. + return !testEmojiSupport(EMOJI_FLAG_TEST_EMOJI); + } catch (err: unknown) { + // If an error occurs, assume flags should be replaced. + if (isDevelopment()) { + console.warn( + 'Flag emoji rendering test failed, defaulting to replacement. Error:', + err, + ); + } + return true; + } +} diff --git a/app/javascript/flavours/glitch/features/emoji/normalize.test.ts b/app/javascript/flavours/glitch/features/emoji/normalize.test.ts index ee9cd89487..f0ea140590 100644 --- a/app/javascript/flavours/glitch/features/emoji/normalize.test.ts +++ b/app/javascript/flavours/glitch/features/emoji/normalize.test.ts @@ -22,9 +22,9 @@ const emojiSVGFiles = await readdir( ); const svgFileNames = emojiSVGFiles .filter((file) => file.isFile() && file.name.endsWith('.svg')) - .map((file) => basename(file.name, '.svg').toUpperCase()); + .map((file) => basename(file.name, '.svg')); const svgFileNamesWithoutBorder = svgFileNames.filter( - (fileName) => !fileName.endsWith('_BORDER'), + (fileName) => !fileName.endsWith('_border'), ); const unicodeEmojis = flattenEmojiData(unicodeRawEmojis); @@ -60,13 +60,13 @@ describe('unicodeToTwemojiHex', () => { describe('twemojiHasBorder', () => { test.concurrent.for( svgFileNames - .filter((file) => file.endsWith('_BORDER')) + .filter((file) => file.endsWith('_border')) .map((file) => { - const hexCode = file.replace('_BORDER', ''); + const hexCode = file.replace('_border', ''); return [ hexCode, - CODES_WITH_LIGHT_BORDER.includes(hexCode), - CODES_WITH_DARK_BORDER.includes(hexCode), + CODES_WITH_LIGHT_BORDER.includes(hexCode.toUpperCase()), + CODES_WITH_DARK_BORDER.includes(hexCode.toUpperCase()), ] as const; }), )('twemojiHasBorder for %s', ([hexCode, isLight, isDark], { expect }) => { diff --git a/app/javascript/flavours/glitch/features/emoji/normalize.ts b/app/javascript/flavours/glitch/features/emoji/normalize.ts index 94dc33a6ea..6a64c3b8bf 100644 --- a/app/javascript/flavours/glitch/features/emoji/normalize.ts +++ b/app/javascript/flavours/glitch/features/emoji/normalize.ts @@ -7,6 +7,7 @@ import { EMOJIS_WITH_DARK_BORDER, EMOJIS_WITH_LIGHT_BORDER, } from './constants'; +import type { TwemojiBorderInfo } from './types'; // Misc codes that have special handling const SKIER_CODE = 0x26f7; @@ -51,13 +52,7 @@ export function unicodeToTwemojiHex(unicodeHex: string): string { normalizedCodes.push(code); } - return hexNumbersToString(normalizedCodes, 0); -} - -interface TwemojiBorderInfo { - hexCode: string; - hasLightBorder: boolean; - hasDarkBorder: boolean; + return hexNumbersToString(normalizedCodes, 0).toLowerCase(); } export const CODES_WITH_DARK_BORDER = @@ -77,7 +72,7 @@ export function twemojiHasBorder(twemojiHex: string): TwemojiBorderInfo { hasDarkBorder = true; } return { - hexCode: normalizedHex, + hexCode: twemojiHex, hasLightBorder, hasDarkBorder, }; diff --git a/app/javascript/flavours/glitch/features/emoji/render.test.ts b/app/javascript/flavours/glitch/features/emoji/render.test.ts new file mode 100644 index 0000000000..23f85c36b3 --- /dev/null +++ b/app/javascript/flavours/glitch/features/emoji/render.test.ts @@ -0,0 +1,163 @@ +import { + EMOJI_MODE_NATIVE, + EMOJI_MODE_NATIVE_WITH_FLAGS, + EMOJI_MODE_TWEMOJI, +} from './constants'; +import { emojifyElement, tokenizeText } from './render'; +import type { CustomEmojiData, UnicodeEmojiData } from './types'; + +vitest.mock('./database', () => ({ + searchCustomEmojisByShortcodes: vitest.fn( + () => + [ + { + shortcode: 'custom', + static_url: 'emoji/static', + url: 'emoji/custom', + category: 'test', + visible_in_picker: true, + }, + ] satisfies CustomEmojiData[], + ), + searchEmojisByHexcodes: vitest.fn( + () => + [ + { + hexcode: '1F60A', + group: 0, + label: 'smiling face with smiling eyes', + order: 0, + tags: ['smile', 'happy'], + unicode: '😊', + }, + { + hexcode: '1F1EA-1F1FA', + group: 0, + label: 'flag-eu', + order: 0, + tags: ['flag', 'european union'], + unicode: 'πŸ‡ͺπŸ‡Ί', + }, + ] satisfies UnicodeEmojiData[], + ), + findMissingLocales: vitest.fn(() => []), +})); + +describe('emojifyElement', () => { + const testElement = document.createElement('div'); + testElement.innerHTML = '

Hello 😊πŸ‡ͺπŸ‡Ί!

:custom:

'; + + const expectedSmileImage = + '😊'; + const expectedFlagImage = + 'πŸ‡ͺπŸ‡Ί'; + const expectedCustomEmojiImage = + ':custom:'; + + function cloneTestElement() { + return testElement.cloneNode(true) as HTMLElement; + } + + test('emojifies custom emoji in native mode', async () => { + const emojifiedElement = await emojifyElement(cloneTestElement(), { + locales: ['en'], + mode: EMOJI_MODE_NATIVE, + currentLocale: 'en', + }); + expect(emojifiedElement.innerHTML).toBe( + `

Hello 😊πŸ‡ͺπŸ‡Ί!

${expectedCustomEmojiImage}

`, + ); + }); + + test('emojifies flag emoji in native-with-flags mode', async () => { + const emojifiedElement = await emojifyElement(cloneTestElement(), { + locales: ['en'], + mode: EMOJI_MODE_NATIVE_WITH_FLAGS, + currentLocale: 'en', + }); + expect(emojifiedElement.innerHTML).toBe( + `

Hello 😊${expectedFlagImage}!

${expectedCustomEmojiImage}

`, + ); + }); + + test('emojifies everything in twemoji mode', async () => { + const emojifiedElement = await emojifyElement(cloneTestElement(), { + locales: ['en'], + mode: EMOJI_MODE_TWEMOJI, + currentLocale: 'en', + }); + expect(emojifiedElement.innerHTML).toBe( + `

Hello ${expectedSmileImage}${expectedFlagImage}!

${expectedCustomEmojiImage}

`, + ); + }); +}); + +describe('tokenizeText', () => { + test('returns empty array for string with only whitespace', () => { + expect(tokenizeText(' \n')).toEqual([]); + }); + + test('returns an array of text to be a single token', () => { + expect(tokenizeText('Hello')).toEqual(['Hello']); + }); + + test('returns tokens for text with emoji', () => { + expect(tokenizeText('Hello 😊 πŸ‡ΏπŸ‡Ό!!')).toEqual([ + 'Hello ', + { + type: 'unicode', + code: '😊', + }, + ' ', + { + type: 'unicode', + code: 'πŸ‡ΏπŸ‡Ό', + }, + '!!', + ]); + }); + + test('returns tokens for text with custom emoji', () => { + expect(tokenizeText('Hello :smile:!!')).toEqual([ + 'Hello ', + { + type: 'custom', + code: 'smile', + }, + '!!', + ]); + }); + + test('handles custom emoji with underscores and numbers', () => { + expect(tokenizeText('Hello :smile_123:!!')).toEqual([ + 'Hello ', + { + type: 'custom', + code: 'smile_123', + }, + '!!', + ]); + }); + + test('returns tokens for text with mixed emoji', () => { + expect(tokenizeText('Hello 😊 :smile:!!')).toEqual([ + 'Hello ', + { + type: 'unicode', + code: '😊', + }, + ' ', + { + type: 'custom', + code: 'smile', + }, + '!!', + ]); + }); + + test('does not capture custom emoji with invalid characters', () => { + expect(tokenizeText('Hello :smile-123:!!')).toEqual([ + 'Hello :smile-123:!!', + ]); + }); +}); diff --git a/app/javascript/flavours/glitch/features/emoji/render.ts b/app/javascript/flavours/glitch/features/emoji/render.ts new file mode 100644 index 0000000000..424a997b07 --- /dev/null +++ b/app/javascript/flavours/glitch/features/emoji/render.ts @@ -0,0 +1,331 @@ +import type { Locale } from 'emojibase'; +import EMOJI_REGEX from 'emojibase-regex/emoji-loose'; + +import { autoPlayGif } from '@/flavours/glitch/initial_state'; +import { assetHost } from '@/flavours/glitch/utils/config'; + +import { + EMOJI_MODE_NATIVE, + EMOJI_MODE_NATIVE_WITH_FLAGS, + EMOJI_TYPE_UNICODE, + EMOJI_TYPE_CUSTOM, + EMOJI_STATE_MISSING, +} from './constants'; +import { + findMissingLocales, + searchCustomEmojisByShortcodes, + searchEmojisByHexcodes, +} from './database'; +import { loadEmojiLocale } from './index'; +import { + emojiToUnicodeHex, + twemojiHasBorder, + unicodeToTwemojiHex, +} from './normalize'; +import type { + CustomEmojiToken, + EmojiAppState, + EmojiLoadedState, + EmojiMode, + EmojiState, + EmojiStateMap, + EmojiToken, + ExtraCustomEmojiMap, + LocaleOrCustom, + UnicodeEmojiToken, +} from './types'; +import { stringHasUnicodeFlags } from './utils'; + +const localeCacheMap = new Map([ + [EMOJI_TYPE_CUSTOM, new Map()], +]); + +// Emojifies an element. This modifies the element in place, replacing text nodes with emojified versions. +export async function emojifyElement( + element: Element, + appState: EmojiAppState, + extraEmojis: ExtraCustomEmojiMap = {}, +): Promise { + const queue: (HTMLElement | Text)[] = [element]; + while (queue.length > 0) { + const current = queue.shift(); + if ( + !current || + current instanceof HTMLScriptElement || + current instanceof HTMLStyleElement + ) { + continue; + } + + if ( + current.textContent && + (current instanceof Text || !current.hasChildNodes()) + ) { + const renderedContent = await emojifyText( + current.textContent, + appState, + extraEmojis, + ); + if (renderedContent) { + if (!(current instanceof Text)) { + current.textContent = null; // Clear the text content if it's not a Text node. + } + current.replaceWith(renderedToHTMLFragment(renderedContent)); + } + continue; + } + + for (const child of current.childNodes) { + if (child instanceof HTMLElement || child instanceof Text) { + queue.push(child); + } + } + } + return element; +} + +export async function emojifyText( + text: string, + appState: EmojiAppState, + extraEmojis: ExtraCustomEmojiMap = {}, +) { + // Exit if no text to convert. + if (!text.trim()) { + return null; + } + + const tokens = tokenizeText(text); + + // If only one token and it's a string, exit early. + if (tokens.length === 1 && typeof tokens[0] === 'string') { + return null; + } + + // Get all emoji from the state map, loading any missing ones. + await ensureLocalesAreLoaded(appState.locales); + await loadMissingEmojiIntoCache(tokens, appState.locales); + + const renderedFragments: (string | HTMLImageElement)[] = []; + for (const token of tokens) { + if (typeof token !== 'string' && shouldRenderImage(token, appState.mode)) { + let state: EmojiState | undefined; + if (token.type === EMOJI_TYPE_CUSTOM) { + const extraEmojiData = extraEmojis[token.code]; + if (extraEmojiData) { + state = { type: EMOJI_TYPE_CUSTOM, data: extraEmojiData }; + } else { + state = emojiForLocale(token.code, EMOJI_TYPE_CUSTOM); + } + } else { + state = emojiForLocale( + emojiToUnicodeHex(token.code), + appState.currentLocale, + ); + } + + // If the state is valid, create an image element. Otherwise, just append as text. + if (state && typeof state !== 'string') { + const image = stateToImage(state); + renderedFragments.push(image); + continue; + } + } + const text = typeof token === 'string' ? token : token.code; + renderedFragments.push(text); + } + + return renderedFragments; +} + +// Private functions + +async function ensureLocalesAreLoaded(locales: Locale[]) { + const missingLocales = await findMissingLocales(locales); + for (const locale of missingLocales) { + await loadEmojiLocale(locale); + } +} + +const CUSTOM_EMOJI_REGEX = /:([a-z0-9_]+):/i; +const TOKENIZE_REGEX = new RegExp( + `(${EMOJI_REGEX.source}|${CUSTOM_EMOJI_REGEX.source})`, + 'g', +); + +type TokenizedText = (string | EmojiToken)[]; + +export function tokenizeText(text: string): TokenizedText { + if (!text.trim()) { + return []; + } + + const tokens = []; + let lastIndex = 0; + for (const match of text.matchAll(TOKENIZE_REGEX)) { + if (match.index > lastIndex) { + tokens.push(text.slice(lastIndex, match.index)); + } + + const code = match[0]; + + if (code.startsWith(':') && code.endsWith(':')) { + // Custom emoji + tokens.push({ + type: EMOJI_TYPE_CUSTOM, + code: code.slice(1, -1), // Remove the colons + } satisfies CustomEmojiToken); + } else { + // Unicode emoji + tokens.push({ + type: EMOJI_TYPE_UNICODE, + code: code, + } satisfies UnicodeEmojiToken); + } + lastIndex = match.index + code.length; + } + if (lastIndex < text.length) { + tokens.push(text.slice(lastIndex)); + } + return tokens; +} + +function cacheForLocale(locale: LocaleOrCustom): EmojiStateMap { + return localeCacheMap.get(locale) ?? (new Map() as EmojiStateMap); +} + +function emojiForLocale( + code: string, + locale: LocaleOrCustom, +): EmojiState | undefined { + const cache = cacheForLocale(locale); + return cache.get(code); +} + +async function loadMissingEmojiIntoCache( + tokens: TokenizedText, + locales: Locale[], +) { + const missingUnicodeEmoji = new Set(); + const missingCustomEmoji = new Set(); + + // Iterate over tokens and check if they are in the cache already. + for (const token of tokens) { + if (typeof token === 'string') { + continue; // Skip plain strings. + } + + // If this is a custom emoji, check it separately. + if (token.type === EMOJI_TYPE_CUSTOM) { + const code = token.code; + const emojiState = emojiForLocale(code, EMOJI_TYPE_CUSTOM); + if (!emojiState) { + missingCustomEmoji.add(code); + } + // Otherwise this is a unicode emoji, so check it against all locales. + } else { + const code = emojiToUnicodeHex(token.code); + if (missingUnicodeEmoji.has(code)) { + continue; // Already marked as missing. + } + for (const locale of locales) { + const emojiState = emojiForLocale(code, locale); + if (!emojiState) { + // If it's missing in one locale, we consider it missing for all. + missingUnicodeEmoji.add(code); + } + } + } + } + + if (missingUnicodeEmoji.size > 0) { + const missingEmojis = Array.from(missingUnicodeEmoji).toSorted(); + for (const locale of locales) { + const emojis = await searchEmojisByHexcodes(missingEmojis, locale); + const cache = cacheForLocale(locale); + for (const emoji of emojis) { + cache.set(emoji.hexcode, { type: EMOJI_TYPE_UNICODE, data: emoji }); + } + const notFoundEmojis = missingEmojis.filter((code) => + emojis.every((emoji) => emoji.hexcode !== code), + ); + for (const code of notFoundEmojis) { + cache.set(code, EMOJI_STATE_MISSING); // Mark as missing if not found, as it's probably not a valid emoji. + } + localeCacheMap.set(locale, cache); + } + } + + if (missingCustomEmoji.size > 0) { + const missingEmojis = Array.from(missingCustomEmoji).toSorted(); + const emojis = await searchCustomEmojisByShortcodes(missingEmojis); + const cache = cacheForLocale(EMOJI_TYPE_CUSTOM); + for (const emoji of emojis) { + cache.set(emoji.shortcode, { type: EMOJI_TYPE_CUSTOM, data: emoji }); + } + const notFoundEmojis = missingEmojis.filter((code) => + emojis.every((emoji) => emoji.shortcode !== code), + ); + for (const code of notFoundEmojis) { + cache.set(code, EMOJI_STATE_MISSING); // Mark as missing if not found, as it's probably not a valid emoji. + } + localeCacheMap.set(EMOJI_TYPE_CUSTOM, cache); + } +} + +function shouldRenderImage(token: EmojiToken, mode: EmojiMode): boolean { + if (token.type === EMOJI_TYPE_UNICODE) { + // If the mode is native or native with flags for non-flag emoji + // we can just append the text node directly. + if ( + mode === EMOJI_MODE_NATIVE || + (mode === EMOJI_MODE_NATIVE_WITH_FLAGS && + !stringHasUnicodeFlags(token.code)) + ) { + return false; + } + } + + return true; +} + +function stateToImage(state: EmojiLoadedState) { + const image = document.createElement('img'); + image.draggable = false; + image.classList.add('emojione'); + + if (state.type === EMOJI_TYPE_UNICODE) { + const emojiInfo = twemojiHasBorder(unicodeToTwemojiHex(state.data.hexcode)); + if (emojiInfo.hasLightBorder) { + image.dataset.lightCode = `${emojiInfo.hexCode}_BORDER`; + } else if (emojiInfo.hasDarkBorder) { + image.dataset.darkCode = `${emojiInfo.hexCode}_BORDER`; + } + + image.alt = state.data.unicode; + image.title = state.data.label; + image.src = `${assetHost}/emoji/${emojiInfo.hexCode}.svg`; + } else { + // Custom emoji + const shortCode = `:${state.data.shortcode}:`; + image.classList.add('custom-emoji'); + image.alt = shortCode; + image.title = shortCode; + image.src = autoPlayGif ? state.data.url : state.data.static_url; + image.dataset.original = state.data.url; + image.dataset.static = state.data.static_url; + } + + return image; +} + +function renderedToHTMLFragment(renderedArray: (string | HTMLImageElement)[]) { + const fragment = document.createDocumentFragment(); + for (const fragmentItem of renderedArray) { + if (typeof fragmentItem === 'string') { + fragment.appendChild(document.createTextNode(fragmentItem)); + } else if (fragmentItem instanceof HTMLImageElement) { + fragment.appendChild(fragmentItem); + } + } + return fragment; +} diff --git a/app/javascript/flavours/glitch/features/emoji/types.ts b/app/javascript/flavours/glitch/features/emoji/types.ts new file mode 100644 index 0000000000..4e21775ac9 --- /dev/null +++ b/app/javascript/flavours/glitch/features/emoji/types.ts @@ -0,0 +1,64 @@ +import type { FlatCompactEmoji, Locale } from 'emojibase'; + +import type { ApiCustomEmojiJSON } from '@/flavours/glitch/api_types/custom_emoji'; + +import type { + EMOJI_MODE_NATIVE, + EMOJI_MODE_NATIVE_WITH_FLAGS, + EMOJI_MODE_TWEMOJI, + EMOJI_STATE_MISSING, + EMOJI_TYPE_CUSTOM, + EMOJI_TYPE_UNICODE, +} from './constants'; + +export type EmojiMode = + | typeof EMOJI_MODE_NATIVE + | typeof EMOJI_MODE_NATIVE_WITH_FLAGS + | typeof EMOJI_MODE_TWEMOJI; + +export type LocaleOrCustom = Locale | typeof EMOJI_TYPE_CUSTOM; + +export interface EmojiAppState { + locales: Locale[]; + currentLocale: Locale; + mode: EmojiMode; +} + +export interface UnicodeEmojiToken { + type: typeof EMOJI_TYPE_UNICODE; + code: string; +} +export interface CustomEmojiToken { + type: typeof EMOJI_TYPE_CUSTOM; + code: string; +} +export type EmojiToken = UnicodeEmojiToken | CustomEmojiToken; + +export type CustomEmojiData = ApiCustomEmojiJSON; +export type UnicodeEmojiData = FlatCompactEmoji; +export type AnyEmojiData = CustomEmojiData | UnicodeEmojiData; + +export type EmojiStateMissing = typeof EMOJI_STATE_MISSING; +export interface EmojiStateUnicode { + type: typeof EMOJI_TYPE_UNICODE; + data: UnicodeEmojiData; +} +export interface EmojiStateCustom { + type: typeof EMOJI_TYPE_CUSTOM; + data: CustomEmojiData; +} +export type EmojiState = + | EmojiStateMissing + | EmojiStateUnicode + | EmojiStateCustom; +export type EmojiLoadedState = EmojiStateUnicode | EmojiStateCustom; + +export type EmojiStateMap = Map; + +export type ExtraCustomEmojiMap = Record; + +export interface TwemojiBorderInfo { + hexCode: string; + hasLightBorder: boolean; + hasDarkBorder: boolean; +} diff --git a/app/javascript/flavours/glitch/features/emoji/utils.test.ts b/app/javascript/flavours/glitch/features/emoji/utils.test.ts new file mode 100644 index 0000000000..75cac8c5b4 --- /dev/null +++ b/app/javascript/flavours/glitch/features/emoji/utils.test.ts @@ -0,0 +1,47 @@ +import { stringHasUnicodeEmoji, stringHasUnicodeFlags } from './utils'; + +describe('stringHasEmoji', () => { + test.concurrent.for([ + ['only text', false], + ['text with emoji πŸ˜€', true], + ['multiple emojis πŸ˜€πŸ˜ƒπŸ˜„', true], + ['emoji with skin tone πŸ‘πŸ½', true], + ['emoji with ZWJ πŸ‘©β€β€οΈβ€πŸ‘¨', true], + ['emoji with variation selector ✊️', true], + ['emoji with keycap 1️⃣', true], + ['emoji with flags πŸ‡ΊπŸ‡Έ', true], + ['emoji with regional indicators πŸ‡¦πŸ‡Ί', true], + ['emoji with gender πŸ‘©β€βš•οΈ', true], + ['emoji with family πŸ‘¨β€πŸ‘©β€πŸ‘§β€πŸ‘¦', true], + ['emoji with zero width joiner πŸ‘©β€πŸ”¬', true], + ['emoji with non-BMP codepoint πŸ§‘β€πŸš€', true], + ['emoji with combining marks πŸ‘¨β€πŸ‘©β€πŸ‘§β€πŸ‘¦', true], + ['emoji with enclosing keycap #️⃣', true], + ['emoji with no visible glyph \u200D', false], + ] as const)( + 'stringHasEmoji has emojis in "%s": %o', + ([text, expected], { expect }) => { + expect(stringHasUnicodeEmoji(text)).toBe(expected); + }, + ); +}); + +describe('stringHasFlags', () => { + test.concurrent.for([ + ['EU πŸ‡ͺπŸ‡Ί', true], + ['Germany πŸ‡©πŸ‡ͺ', true], + ['Canada πŸ‡¨πŸ‡¦', true], + ['SΓ£o TomΓ© & PrΓ­ncipe πŸ‡ΈπŸ‡Ή', true], + ['Scotland 🏴󠁧󠁒󠁳󠁣󠁴󠁿', true], + ['black flag 🏴', false], + ['arrr πŸ΄β€β˜ οΈ', false], + ['rainbow flag πŸ³οΈβ€πŸŒˆ', false], + ['non-flag πŸ”₯', false], + ['only text', false], + ] as const)( + 'stringHasFlags has flag in "%s": %o', + ([text, expected], { expect }) => { + expect(stringHasUnicodeFlags(text)).toBe(expected); + }, + ); +}); diff --git a/app/javascript/flavours/glitch/features/emoji/utils.ts b/app/javascript/flavours/glitch/features/emoji/utils.ts new file mode 100644 index 0000000000..d00accea8c --- /dev/null +++ b/app/javascript/flavours/glitch/features/emoji/utils.ts @@ -0,0 +1,13 @@ +import EMOJI_REGEX from 'emojibase-regex/emoji-loose'; + +export function stringHasUnicodeEmoji(text: string): boolean { + return EMOJI_REGEX.test(text); +} + +// From https://github.com/talkjs/country-flag-emoji-polyfill/blob/master/src/index.ts#L49-L50 +const EMOJIS_FLAGS_REGEX = + /[\u{1F1E6}-\u{1F1FF}|\u{E0062}-\u{E0063}|\u{E0065}|\u{E0067}|\u{E006C}|\u{E006E}|\u{E0073}-\u{E0074}|\u{E0077}|\u{E007F}]+/u; + +export function stringHasUnicodeFlags(text: string): boolean { + return EMOJIS_FLAGS_REGEX.test(text); +} diff --git a/app/javascript/flavours/glitch/features/status/components/refresh_controller.tsx b/app/javascript/flavours/glitch/features/status/components/refresh_controller.tsx new file mode 100644 index 0000000000..524c5932af --- /dev/null +++ b/app/javascript/flavours/glitch/features/status/components/refresh_controller.tsx @@ -0,0 +1,111 @@ +import { useEffect, useState, useCallback } from 'react'; + +import { useIntl, defineMessages, FormattedMessage } from 'react-intl'; + +import classNames from 'classnames'; + +import { + fetchContext, + completeContextRefresh, +} from 'flavours/glitch/actions/statuses'; +import type { AsyncRefreshHeader } from 'flavours/glitch/api'; +import { apiGetAsyncRefresh } from 'flavours/glitch/api/async_refreshes'; +import { LoadingIndicator } from 'flavours/glitch/components/loading_indicator'; +import { useAppSelector, useAppDispatch } from 'flavours/glitch/store'; + +const messages = defineMessages({ + loading: { + id: 'status.context.loading', + defaultMessage: 'Checking for more replies', + }, +}); + +export const RefreshController: React.FC<{ + statusId: string; + withBorder?: boolean; +}> = ({ statusId, withBorder }) => { + const refresh = useAppSelector( + (state) => state.contexts.refreshing[statusId], + ); + const dispatch = useAppDispatch(); + const intl = useIntl(); + const [ready, setReady] = useState(false); + const [loading, setLoading] = useState(false); + + useEffect(() => { + let timeoutId: ReturnType; + + const scheduleRefresh = (refresh: AsyncRefreshHeader) => { + timeoutId = setTimeout(() => { + void apiGetAsyncRefresh(refresh.id).then((result) => { + if (result.async_refresh.status === 'finished') { + dispatch(completeContextRefresh({ statusId })); + + if (result.async_refresh.result_count > 0) { + setReady(true); + } + } else { + scheduleRefresh(refresh); + } + + return ''; + }); + }, refresh.retry * 1000); + }; + + if (refresh) { + scheduleRefresh(refresh); + } + + return () => { + clearTimeout(timeoutId); + }; + }, [dispatch, setReady, statusId, refresh]); + + const handleClick = useCallback(() => { + setLoading(true); + setReady(false); + + dispatch(fetchContext({ statusId })) + .then(() => { + setLoading(false); + return ''; + }) + .catch(() => { + setLoading(false); + }); + }, [dispatch, setReady, statusId]); + + if (ready && !loading) { + return ( + + ); + } + + if (!refresh && !loading) { + return null; + } + + return ( +
+ +
+ ); +}; diff --git a/app/javascript/flavours/glitch/features/status/index.jsx b/app/javascript/flavours/glitch/features/status/index.jsx index da04de0cc7..2a5d378503 100644 --- a/app/javascript/flavours/glitch/features/status/index.jsx +++ b/app/javascript/flavours/glitch/features/status/index.jsx @@ -62,7 +62,7 @@ import { attachFullscreenListener, detachFullscreenListener, isFullscreen } from import ActionBar from './components/action_bar'; import { DetailedStatus } from './components/detailed_status'; - +import { RefreshController } from './components/refresh_controller'; const messages = defineMessages({ revealAll: { id: 'status.show_more_all', defaultMessage: 'Show more for all' }, @@ -572,7 +572,7 @@ class Status extends ImmutablePureComponent { render () { let ancestors, descendants, remoteHint; - const { isLoading, status, settings, ancestorsIds, descendantsIds, intl, domain, multiColumn, pictureInPicture } = this.props; + const { isLoading, status, settings, ancestorsIds, descendantsIds, refresh, intl, domain, multiColumn, pictureInPicture } = this.props; const { fullscreen } = this.state; if (isLoading) { @@ -604,11 +604,9 @@ class Status extends ImmutablePureComponent { if (!isLocal) { remoteHint = ( - } - label={{status.getIn(['account', 'acct']).split('@')[1]} }} />} + ); } diff --git a/app/javascript/flavours/glitch/initial_state.js b/app/javascript/flavours/glitch/initial_state.js index 7114a1e28e..480773c2e1 100644 --- a/app/javascript/flavours/glitch/initial_state.js +++ b/app/javascript/flavours/glitch/initial_state.js @@ -48,6 +48,7 @@ * @property {string} sso_redirect * @property {string} status_page_url * @property {boolean} terms_of_service_enabled + * @property {string?} emoji_style * @property {boolean} system_emoji_font * @property {string} default_content_type */ @@ -114,6 +115,7 @@ export const disableHoverCards = getMeta('disable_hover_cards'); export const disabledAccountId = getMeta('disabled_account_id'); export const displayMedia = getMeta('display_media'); export const domain = getMeta('domain'); +export const emojiStyle = getMeta('emoji_style') || 'auto'; export const expandSpoilers = getMeta('expand_spoilers'); export const forceSingleColumn = !getMeta('advanced_layout'); export const limitedFederationMode = getMeta('limited_federation_mode'); diff --git a/app/javascript/flavours/glitch/main.tsx b/app/javascript/flavours/glitch/main.tsx index 3178f7aa53..37610f60ec 100644 --- a/app/javascript/flavours/glitch/main.tsx +++ b/app/javascript/flavours/glitch/main.tsx @@ -4,7 +4,11 @@ import { Globals } from '@react-spring/web'; import { setupBrowserNotifications } from 'flavours/glitch/actions/notifications'; import Mastodon from 'flavours/glitch/containers/mastodon'; -import { me, reduceMotion } from 'flavours/glitch/initial_state'; +import { + isFeatureEnabled, + me, + reduceMotion, +} from 'flavours/glitch/initial_state'; import * as perf from 'flavours/glitch/performance'; import ready from 'flavours/glitch/ready'; import { store } from 'flavours/glitch/store'; @@ -29,6 +33,13 @@ function main() { }); } + if (isFeatureEnabled('modern_emojis')) { + const { initializeEmoji } = await import( + '@/flavours/glitch/features/emoji' + ); + await initializeEmoji(); + } + const root = createRoot(mountNode); root.render(); store.dispatch(setupBrowserNotifications()); diff --git a/app/javascript/flavours/glitch/reducers/contexts.ts b/app/javascript/flavours/glitch/reducers/contexts.ts index 9c849a967b..19714963ba 100644 --- a/app/javascript/flavours/glitch/reducers/contexts.ts +++ b/app/javascript/flavours/glitch/reducers/contexts.ts @@ -4,6 +4,7 @@ import type { Draft, UnknownAction } from '@reduxjs/toolkit'; import type { List as ImmutableList } from 'immutable'; import { timelineDelete } from 'flavours/glitch/actions/timelines_typed'; +import type { AsyncRefreshHeader } from 'flavours/glitch/api'; import type { ApiRelationshipJSON } from 'flavours/glitch/api_types/relationships'; import type { ApiStatusJSON, @@ -12,7 +13,7 @@ import type { import type { Status } from 'flavours/glitch/models/status'; import { blockAccountSuccess, muteAccountSuccess } from '../actions/accounts'; -import { fetchContext } from '../actions/statuses'; +import { fetchContext, completeContextRefresh } from '../actions/statuses'; import { TIMELINE_UPDATE } from '../actions/timelines'; import { compareId } from '../compare_id'; @@ -25,11 +26,13 @@ interface TimelineUpdateAction extends UnknownAction { interface State { inReplyTos: Record; replies: Record; + refreshing: Record; } const initialState: State = { inReplyTos: {}, replies: {}, + refreshing: {}, }; const normalizeContext = ( @@ -127,6 +130,13 @@ export const contextsReducer = createReducer(initialState, (builder) => { builder .addCase(fetchContext.fulfilled, (state, action) => { normalizeContext(state, action.meta.arg.statusId, action.payload.context); + + if (action.payload.refresh) { + state.refreshing[action.meta.arg.statusId] = action.payload.refresh; + } + }) + .addCase(completeContextRefresh, (state, action) => { + delete state.refreshing[action.payload.statusId]; }) .addCase(blockAccountSuccess, (state, action) => { filterContexts( diff --git a/app/javascript/flavours/glitch/styles/components.scss b/app/javascript/flavours/glitch/styles/components.scss index 9fdf4dd209..ffd1c9ac35 100644 --- a/app/javascript/flavours/glitch/styles/components.scss +++ b/app/javascript/flavours/glitch/styles/components.scss @@ -2933,6 +2933,8 @@ a.account__display-name { } &__main { + --column-header-height: 62px; + box-sizing: border-box; width: 100%; flex: 0 1 auto; @@ -9118,6 +9120,10 @@ noscript { .conversation { position: relative; + // When scrolling these elements into view, take into account + // the column header height + scroll-margin-top: var(--column-header-height, 0); + &.unread { &::before { content: ''; diff --git a/app/javascript/mastodon/actions/statuses_typed.ts b/app/javascript/mastodon/actions/statuses_typed.ts index b98abbe122..cc9c389cda 100644 --- a/app/javascript/mastodon/actions/statuses_typed.ts +++ b/app/javascript/mastodon/actions/statuses_typed.ts @@ -1,3 +1,5 @@ +import { createAction } from '@reduxjs/toolkit'; + import { apiGetContext } from 'mastodon/api/statuses'; import { createDataLoadingThunk } from 'mastodon/store/typed_functions'; @@ -6,13 +8,18 @@ import { importFetchedStatuses } from './importer'; export const fetchContext = createDataLoadingThunk( 'status/context', ({ statusId }: { statusId: string }) => apiGetContext(statusId), - (context, { dispatch }) => { + ({ context, refresh }, { dispatch }) => { const statuses = context.ancestors.concat(context.descendants); dispatch(importFetchedStatuses(statuses)); return { context, + refresh, }; }, ); + +export const completeContextRefresh = createAction<{ statusId: string }>( + 'status/context/complete', +); diff --git a/app/javascript/mastodon/api.ts b/app/javascript/mastodon/api.ts index dc9c20b508..1820e00a53 100644 --- a/app/javascript/mastodon/api.ts +++ b/app/javascript/mastodon/api.ts @@ -20,6 +20,50 @@ export const getLinks = (response: AxiosResponse) => { return LinkHeader.parse(value); }; +export interface AsyncRefreshHeader { + id: string; + retry: number; +} + +const isAsyncRefreshHeader = (obj: object): obj is AsyncRefreshHeader => + 'id' in obj && 'retry' in obj; + +export const getAsyncRefreshHeader = ( + response: AxiosResponse, +): AsyncRefreshHeader | null => { + const value = response.headers['mastodon-async-refresh'] as + | string + | undefined; + + if (!value) { + return null; + } + + const asyncRefreshHeader: Record = {}; + + value.split(/,\s*/).forEach((pair) => { + const [key, val] = pair.split('=', 2); + + let typedValue: string | number; + + if (key && ['id', 'retry'].includes(key) && val) { + if (val.startsWith('"')) { + typedValue = val.slice(1, -1); + } else { + typedValue = parseInt(val); + } + + asyncRefreshHeader[key] = typedValue; + } + }); + + if (isAsyncRefreshHeader(asyncRefreshHeader)) { + return asyncRefreshHeader; + } + + return null; +}; + const csrfHeader: RawAxiosRequestHeaders = {}; const setCSRFHeader = () => { @@ -83,7 +127,7 @@ export default function api(withAuthorization = true) { return instance; } -type ApiUrl = `v${1 | 2}/${string}`; +type ApiUrl = `v${1 | '1_alpha' | 2}/${string}`; type RequestParamsOrData = Record; export async function apiRequest( diff --git a/app/javascript/mastodon/api/async_refreshes.ts b/app/javascript/mastodon/api/async_refreshes.ts new file mode 100644 index 0000000000..953300a4a8 --- /dev/null +++ b/app/javascript/mastodon/api/async_refreshes.ts @@ -0,0 +1,5 @@ +import { apiRequestGet } from 'mastodon/api'; +import type { ApiAsyncRefreshJSON } from 'mastodon/api_types/async_refreshes'; + +export const apiGetAsyncRefresh = (id: string) => + apiRequestGet(`v1_alpha/async_refreshes/${id}`); diff --git a/app/javascript/mastodon/api/statuses.ts b/app/javascript/mastodon/api/statuses.ts index 921a7bfe63..48eff2a692 100644 --- a/app/javascript/mastodon/api/statuses.ts +++ b/app/javascript/mastodon/api/statuses.ts @@ -1,5 +1,14 @@ -import { apiRequestGet } from 'mastodon/api'; +import api, { getAsyncRefreshHeader } from 'mastodon/api'; import type { ApiContextJSON } from 'mastodon/api_types/statuses'; -export const apiGetContext = (statusId: string) => - apiRequestGet(`v1/statuses/${statusId}/context`); +export const apiGetContext = async (statusId: string) => { + const response = await api().request({ + method: 'GET', + url: `/api/v1/statuses/${statusId}/context`, + }); + + return { + context: response.data, + refresh: getAsyncRefreshHeader(response), + }; +}; diff --git a/app/javascript/mastodon/api_types/async_refreshes.ts b/app/javascript/mastodon/api_types/async_refreshes.ts new file mode 100644 index 0000000000..2d2fed2412 --- /dev/null +++ b/app/javascript/mastodon/api_types/async_refreshes.ts @@ -0,0 +1,7 @@ +export interface ApiAsyncRefreshJSON { + async_refresh: { + id: string; + status: 'running' | 'finished'; + result_count: number; + }; +} diff --git a/app/javascript/mastodon/components/account_bio.tsx b/app/javascript/mastodon/components/account_bio.tsx index e0127f2092..cdac41b8a7 100644 --- a/app/javascript/mastodon/components/account_bio.tsx +++ b/app/javascript/mastodon/components/account_bio.tsx @@ -2,27 +2,44 @@ import { useCallback } from 'react'; import { useLinks } from 'mastodon/hooks/useLinks'; +import { EmojiHTML } from '../features/emoji/emoji_html'; +import { isFeatureEnabled } from '../initial_state'; +import { useAppSelector } from '../store'; + interface AccountBioProps { - note: string; className: string; - dropdownAccountId?: string; + accountId: string; + showDropdown?: boolean; } export const AccountBio: React.FC = ({ - note, className, - dropdownAccountId, + accountId, + showDropdown = false, }) => { - const handleClick = useLinks(!!dropdownAccountId); + const handleClick = useLinks(showDropdown); const handleNodeChange = useCallback( (node: HTMLDivElement | null) => { - if (!dropdownAccountId || !node || node.childNodes.length === 0) { + if (!showDropdown || !node || node.childNodes.length === 0) { return; } - addDropdownToHashtags(node, dropdownAccountId); + addDropdownToHashtags(node, accountId); }, - [dropdownAccountId], + [showDropdown, accountId], ); + const note = useAppSelector((state) => { + const account = state.accounts.get(accountId); + if (!account) { + return ''; + } + return isFeatureEnabled('modern_emojis') + ? account.note + : account.note_emojified; + }); + const extraEmojis = useAppSelector((state) => { + const account = state.accounts.get(accountId); + return account?.emojis; + }); if (note.length === 0) { return null; @@ -31,10 +48,11 @@ export const AccountBio: React.FC = ({ return (
+ > + +
); }; diff --git a/app/javascript/mastodon/components/hover_card_account.tsx b/app/javascript/mastodon/components/hover_card_account.tsx index a6bdda2168..a5a5e4c957 100644 --- a/app/javascript/mastodon/components/hover_card_account.tsx +++ b/app/javascript/mastodon/components/hover_card_account.tsx @@ -102,7 +102,7 @@ export const HoverCardAccount = forwardRef< <>
diff --git a/app/javascript/mastodon/components/status_content.jsx b/app/javascript/mastodon/components/status_content.jsx index 0628e0791b..02f06ec96a 100644 --- a/app/javascript/mastodon/components/status_content.jsx +++ b/app/javascript/mastodon/components/status_content.jsx @@ -13,7 +13,8 @@ import ChevronRightIcon from '@/material-icons/400-24px/chevron_right.svg?react' import { Icon } from 'mastodon/components/icon'; import { Poll } from 'mastodon/components/poll'; import { identityContextPropShape, withIdentity } from 'mastodon/identity_context'; -import { autoPlayGif, languages as preloadedLanguages } from 'mastodon/initial_state'; +import { autoPlayGif, isFeatureEnabled, languages as preloadedLanguages } from 'mastodon/initial_state'; +import { EmojiHTML } from '../features/emoji/emoji_html'; const MAX_HEIGHT = 706; // 22px * 32 (+ 2px padding at the top) @@ -23,6 +24,9 @@ const MAX_HEIGHT = 706; // 22px * 32 (+ 2px padding at the top) * @returns {string} */ export function getStatusContent(status) { + if (isFeatureEnabled('modern_emojis')) { + return status.getIn(['translation', 'content']) || status.get('content'); + } return status.getIn(['translation', 'contentHtml']) || status.get('contentHtml'); } @@ -228,7 +232,7 @@ class StatusContent extends PureComponent { const targetLanguages = this.props.languages?.get(status.get('language') || 'und'); const renderTranslate = this.props.onTranslate && this.props.identity.signedIn && ['public', 'unlisted'].includes(status.get('visibility')) && status.get('search_index').trim().length > 0 && targetLanguages?.includes(contentLocale); - const content = { __html: statusContent ?? getStatusContent(status) }; + const content = statusContent ?? getStatusContent(status); const language = status.getIn(['translation', 'language']) || status.get('language'); const classNames = classnames('status__content', { 'status__content--with-action': this.props.onClick && this.props.history, @@ -253,7 +257,12 @@ class StatusContent extends PureComponent { return ( <>
-
+ {poll} {translateButton} @@ -265,7 +274,12 @@ class StatusContent extends PureComponent { } else { return (
-
+ {poll} {translateButton} diff --git a/app/javascript/mastodon/components/status_list.jsx b/app/javascript/mastodon/components/status_list.jsx index cca449b0ca..70b7968fba 100644 --- a/app/javascript/mastodon/components/status_list.jsx +++ b/app/javascript/mastodon/components/status_list.jsx @@ -40,6 +40,12 @@ export default class StatusList extends ImmutablePureComponent { trackScroll: true, }; + componentDidMount() { + this.columnHeaderHeight = parseFloat( + getComputedStyle(this.node.node).getPropertyValue('--column-header-height') + ) || 0; + } + getFeaturedStatusCount = () => { return this.props.featuredStatusIds ? this.props.featuredStatusIds.size : 0; }; @@ -53,35 +59,68 @@ export default class StatusList extends ImmutablePureComponent { }; handleMoveUp = (id, featured) => { - const elementIndex = this.getCurrentStatusIndex(id, featured) - 1; - this._selectChild(elementIndex, true); + const index = this.getCurrentStatusIndex(id, featured); + this._selectChild(id, index, -1); }; handleMoveDown = (id, featured) => { - const elementIndex = this.getCurrentStatusIndex(id, featured) + 1; - this._selectChild(elementIndex, false); + const index = this.getCurrentStatusIndex(id, featured); + this._selectChild(id, index, 1); }; + _selectChild = (id, index, direction) => { + const listContainer = this.node.node; + let listItem = listContainer.querySelector( + // :nth-child uses 1-based indexing + `.item-list > :nth-child(${index + 1 + direction})` + ); + + if (!listItem) { + return; + } + + // If selected container element is empty, we skip it + if (listItem.matches(':empty')) { + this._selectChild(id, index + direction, direction); + return; + } + + // Check if the list item is a post + let targetElement = listItem.querySelector('.focusable'); + + // Otherwise, check if the item contains follow suggestions or + // is a 'load more' button. + if ( + !targetElement && ( + listItem.querySelector('.inline-follow-suggestions') || + listItem.matches('.load-more') + ) + ) { + targetElement = listItem; + } + + if (targetElement) { + const elementRect = targetElement.getBoundingClientRect(); + + const isFullyVisible = + elementRect.top >= this.columnHeaderHeight && + elementRect.bottom <= window.innerHeight; + + if (!isFullyVisible) { + targetElement.scrollIntoView({ + block: direction === 1 ? 'start' : 'center', + }); + } + + targetElement.focus(); + } + } + handleLoadOlder = debounce(() => { const { statusIds, lastId, onLoadMore } = this.props; onLoadMore(lastId || (statusIds.size > 0 ? statusIds.last() : undefined)); }, 300, { leading: true }); - _selectChild (index, align_top) { - const container = this.node.node; - // TODO: This breaks at the inline-follow-suggestions container - const element = container.querySelector(`article:nth-of-type(${index + 1}) .focusable`); - - if (element) { - if (align_top && container.scrollTop > element.offsetTop) { - element.scrollIntoView(true); - } else if (!align_top && container.scrollTop + container.clientHeight < element.offsetTop + element.offsetHeight) { - element.scrollIntoView(false); - } - element.focus(); - } - } - setRef = c => { this.node = c; }; diff --git a/app/javascript/mastodon/features/account_timeline/components/account_header.tsx b/app/javascript/mastodon/features/account_timeline/components/account_header.tsx index b9f83bebaa..0bae039503 100644 --- a/app/javascript/mastodon/features/account_timeline/components/account_header.tsx +++ b/app/javascript/mastodon/features/account_timeline/components/account_header.tsx @@ -898,8 +898,7 @@ export const AccountHeader: React.FC<{ )} diff --git a/app/javascript/mastodon/features/emoji/constants.ts b/app/javascript/mastodon/features/emoji/constants.ts index d38f17f216..09022371b2 100644 --- a/app/javascript/mastodon/features/emoji/constants.ts +++ b/app/javascript/mastodon/features/emoji/constants.ts @@ -15,6 +15,16 @@ export const SKIN_TONE_CODES = [ 0x1f3ff, // Dark skin tone ] as const; +// Emoji rendering modes. A mode is what we are using to render emojis, a style is what the user has selected. +export const EMOJI_MODE_NATIVE = 'native'; +export const EMOJI_MODE_NATIVE_WITH_FLAGS = 'native-flags'; +export const EMOJI_MODE_TWEMOJI = 'twemoji'; + +export const EMOJI_TYPE_UNICODE = 'unicode'; +export const EMOJI_TYPE_CUSTOM = 'custom'; + +export const EMOJI_STATE_MISSING = 'missing'; + export const EMOJIS_WITH_DARK_BORDER = [ '🎱', // 1F3B1 '🐜', // 1F41C diff --git a/app/javascript/mastodon/features/emoji/database.ts b/app/javascript/mastodon/features/emoji/database.ts index 618f010850..0b8ddd34fb 100644 --- a/app/javascript/mastodon/features/emoji/database.ts +++ b/app/javascript/mastodon/features/emoji/database.ts @@ -1,17 +1,19 @@ import { SUPPORTED_LOCALES } from 'emojibase'; -import type { FlatCompactEmoji, Locale } from 'emojibase'; -import type { DBSchema } from 'idb'; +import type { Locale } from 'emojibase'; +import type { DBSchema, IDBPDatabase } from 'idb'; import { openDB } from 'idb'; -import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji'; - -import type { LocaleOrCustom } from './locale'; import { toSupportedLocale, toSupportedLocaleOrCustom } from './locale'; +import type { + CustomEmojiData, + UnicodeEmojiData, + LocaleOrCustom, +} from './types'; interface EmojiDB extends LocaleTables, DBSchema { custom: { key: string; - value: ApiCustomEmojiJSON; + value: CustomEmojiData; indexes: { category: string; }; @@ -24,7 +26,7 @@ interface EmojiDB extends LocaleTables, DBSchema { interface LocaleTable { key: string; - value: FlatCompactEmoji; + value: UnicodeEmojiData; indexes: { group: number; label: string; @@ -36,63 +38,114 @@ type LocaleTables = Record; const SCHEMA_VERSION = 1; -const db = await openDB('mastodon-emoji', SCHEMA_VERSION, { - upgrade(database) { - const customTable = database.createObjectStore('custom', { - keyPath: 'shortcode', - autoIncrement: false, - }); - customTable.createIndex('category', 'category'); +let db: IDBPDatabase | null = null; - database.createObjectStore('etags'); - - for (const locale of SUPPORTED_LOCALES) { - const localeTable = database.createObjectStore(locale, { - keyPath: 'hexcode', +async function loadDB() { + if (db) { + return db; + } + db = await openDB('mastodon-emoji', SCHEMA_VERSION, { + upgrade(database) { + const customTable = database.createObjectStore('custom', { + keyPath: 'shortcode', autoIncrement: false, }); - localeTable.createIndex('group', 'group'); - localeTable.createIndex('label', 'label'); - localeTable.createIndex('order', 'order'); - localeTable.createIndex('tags', 'tags', { multiEntry: true }); - } - }, -}); + customTable.createIndex('category', 'category'); -export async function putEmojiData(emojis: FlatCompactEmoji[], locale: Locale) { + database.createObjectStore('etags'); + + for (const locale of SUPPORTED_LOCALES) { + const localeTable = database.createObjectStore(locale, { + keyPath: 'hexcode', + autoIncrement: false, + }); + localeTable.createIndex('group', 'group'); + localeTable.createIndex('label', 'label'); + localeTable.createIndex('order', 'order'); + localeTable.createIndex('tags', 'tags', { multiEntry: true }); + } + }, + }); + return db; +} + +export async function putEmojiData(emojis: UnicodeEmojiData[], locale: Locale) { + const db = await loadDB(); const trx = db.transaction(locale, 'readwrite'); await Promise.all(emojis.map((emoji) => trx.store.put(emoji))); await trx.done; } -export async function putCustomEmojiData(emojis: ApiCustomEmojiJSON[]) { +export async function putCustomEmojiData(emojis: CustomEmojiData[]) { + const db = await loadDB(); const trx = db.transaction('custom', 'readwrite'); await Promise.all(emojis.map((emoji) => trx.store.put(emoji))); await trx.done; } -export function putLatestEtag(etag: string, localeString: string) { +export async function putLatestEtag(etag: string, localeString: string) { const locale = toSupportedLocaleOrCustom(localeString); + const db = await loadDB(); return db.put('etags', etag, locale); } -export function searchEmojiByHexcode(hexcode: string, localeString: string) { +export async function searchEmojiByHexcode( + hexcode: string, + localeString: string, +) { const locale = toSupportedLocale(localeString); + const db = await loadDB(); return db.get(locale, hexcode); } -export function searchEmojiByTag(tag: string, localeString: string) { +export async function searchEmojisByHexcodes( + hexcodes: string[], + localeString: string, +) { + const locale = toSupportedLocale(localeString); + const db = await loadDB(); + return db.getAll( + locale, + IDBKeyRange.bound(hexcodes[0], hexcodes[hexcodes.length - 1]), + ); +} + +export async function searchEmojiByTag(tag: string, localeString: string) { const locale = toSupportedLocale(localeString); const range = IDBKeyRange.only(tag.toLowerCase()); + const db = await loadDB(); return db.getAllFromIndex(locale, 'tags', range); } -export function searchCustomEmojiByShortcode(shortcode: string) { +export async function searchCustomEmojiByShortcode(shortcode: string) { + const db = await loadDB(); return db.get('custom', shortcode); } +export async function searchCustomEmojisByShortcodes(shortcodes: string[]) { + const db = await loadDB(); + return db.getAll( + 'custom', + IDBKeyRange.bound(shortcodes[0], shortcodes[shortcodes.length - 1]), + ); +} + +export async function findMissingLocales(localeStrings: string[]) { + const locales = new Set(localeStrings.map(toSupportedLocale)); + const missingLocales: Locale[] = []; + const db = await loadDB(); + for (const locale of locales) { + const rowCount = await db.count(locale); + if (!rowCount) { + missingLocales.push(locale); + } + } + return missingLocales; +} + export async function loadLatestEtag(localeString: string) { const locale = toSupportedLocaleOrCustom(localeString); + const db = await loadDB(); const rowCount = await db.count(locale); if (!rowCount) { return null; // No data for this locale, return null even if there is an etag. diff --git a/app/javascript/mastodon/features/emoji/emoji_html.tsx b/app/javascript/mastodon/features/emoji/emoji_html.tsx new file mode 100644 index 0000000000..27af2dda27 --- /dev/null +++ b/app/javascript/mastodon/features/emoji/emoji_html.tsx @@ -0,0 +1,81 @@ +import type { HTMLAttributes } from 'react'; +import { useEffect, useMemo, useState } from 'react'; + +import type { List as ImmutableList } from 'immutable'; +import { isList } from 'immutable'; + +import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji'; +import { isFeatureEnabled } from '@/mastodon/initial_state'; +import type { CustomEmoji } from '@/mastodon/models/custom_emoji'; + +import { useEmojiAppState } from './hooks'; +import { emojifyElement } from './render'; +import type { ExtraCustomEmojiMap } from './types'; + +type EmojiHTMLProps = Omit< + HTMLAttributes, + 'dangerouslySetInnerHTML' +> & { + htmlString: string; + extraEmojis?: ExtraCustomEmojiMap | ImmutableList; +}; + +export const EmojiHTML: React.FC = ({ + htmlString, + extraEmojis, + ...props +}) => { + if (isFeatureEnabled('modern_emojis')) { + return ( + + ); + } + return
; +}; + +const ModernEmojiHTML: React.FC = ({ + extraEmojis: rawEmojis, + htmlString: text, + ...props +}) => { + const appState = useEmojiAppState(); + const [innerHTML, setInnerHTML] = useState(''); + + const extraEmojis: ExtraCustomEmojiMap = useMemo(() => { + if (!rawEmojis) { + return {}; + } + if (isList(rawEmojis)) { + return ( + rawEmojis.toJS() as ApiCustomEmojiJSON[] + ).reduce( + (acc, emoji) => ({ ...acc, [emoji.shortcode]: emoji }), + {}, + ); + } + return rawEmojis; + }, [rawEmojis]); + + useEffect(() => { + if (!text) { + return; + } + const cb = async () => { + const div = document.createElement('div'); + div.innerHTML = text; + const ele = await emojifyElement(div, appState, extraEmojis); + setInnerHTML(ele.innerHTML); + }; + void cb(); + }, [text, appState, extraEmojis]); + + if (!innerHTML) { + return null; + } + + return
; +}; diff --git a/app/javascript/mastodon/features/emoji/emoji_text.tsx b/app/javascript/mastodon/features/emoji/emoji_text.tsx new file mode 100644 index 0000000000..253371391a --- /dev/null +++ b/app/javascript/mastodon/features/emoji/emoji_text.tsx @@ -0,0 +1,45 @@ +import { useEffect, useState } from 'react'; + +import { useEmojiAppState } from './hooks'; +import { emojifyText } from './render'; + +interface EmojiTextProps { + text: string; +} + +export const EmojiText: React.FC = ({ text }) => { + const appState = useEmojiAppState(); + const [rendered, setRendered] = useState<(string | HTMLImageElement)[]>([]); + + useEffect(() => { + const cb = async () => { + const rendered = await emojifyText(text, appState); + setRendered(rendered ?? []); + }; + void cb(); + }, [text, appState]); + + if (rendered.length === 0) { + return null; + } + + return ( + <> + {rendered.map((fragment, index) => { + if (typeof fragment === 'string') { + return {fragment}; + } + return ( + {fragment.alt} + ); + })} + + ); +}; diff --git a/app/javascript/mastodon/features/emoji/hooks.ts b/app/javascript/mastodon/features/emoji/hooks.ts new file mode 100644 index 0000000000..fd38129a19 --- /dev/null +++ b/app/javascript/mastodon/features/emoji/hooks.ts @@ -0,0 +1,16 @@ +import { useAppSelector } from '@/mastodon/store'; + +import { toSupportedLocale } from './locale'; +import { determineEmojiMode } from './mode'; +import type { EmojiAppState } from './types'; + +export function useEmojiAppState(): EmojiAppState { + const locale = useAppSelector((state) => + toSupportedLocale(state.meta.get('locale') as string), + ); + const mode = useAppSelector((state) => + determineEmojiMode(state.meta.get('emoji_style') as string), + ); + + return { currentLocale: locale, locales: [locale], mode }; +} diff --git a/app/javascript/mastodon/features/emoji/index.ts b/app/javascript/mastodon/features/emoji/index.ts index 6975465b55..4f23dc5395 100644 --- a/app/javascript/mastodon/features/emoji/index.ts +++ b/app/javascript/mastodon/features/emoji/index.ts @@ -2,27 +2,44 @@ import initialState from '@/mastodon/initial_state'; import { toSupportedLocale } from './locale'; -const serverLocale = toSupportedLocale(initialState?.meta.locale ?? 'en'); +const userLocale = toSupportedLocale(initialState?.meta.locale ?? 'en'); -const worker = - 'Worker' in window - ? new Worker(new URL('./worker', import.meta.url), { - type: 'module', - }) - : null; +let worker: Worker | null = null; export async function initializeEmoji() { + if (!worker && 'Worker' in window) { + try { + worker = new Worker(new URL('./worker', import.meta.url), { + type: 'module', + credentials: 'omit', + }); + } catch (err) { + console.warn('Error creating web worker:', err); + } + } + if (worker) { - worker.addEventListener('message', (event: MessageEvent) => { + // Assign worker to const to make TS happy inside the event listener. + const thisWorker = worker; + thisWorker.addEventListener('message', (event: MessageEvent) => { const { data: message } = event; if (message === 'ready') { - worker.postMessage(serverLocale); - worker.postMessage('custom'); + thisWorker.postMessage('custom'); + void loadEmojiLocale(userLocale); + // Load English locale as well, because people are still used to + // using it from before we supported other locales. + if (userLocale !== 'en') { + void loadEmojiLocale('en'); + } } }); } else { - const { importCustomEmojiData, importEmojiData } = await import('./loader'); - await Promise.all([importCustomEmojiData(), importEmojiData(serverLocale)]); + const { importCustomEmojiData } = await import('./loader'); + await importCustomEmojiData(); + await loadEmojiLocale(userLocale); + if (userLocale !== 'en') { + await loadEmojiLocale('en'); + } } } diff --git a/app/javascript/mastodon/features/emoji/loader.ts b/app/javascript/mastodon/features/emoji/loader.ts index f9c6971351..482d9e5c35 100644 --- a/app/javascript/mastodon/features/emoji/loader.ts +++ b/app/javascript/mastodon/features/emoji/loader.ts @@ -11,7 +11,7 @@ import { putLatestEtag, } from './database'; import { toSupportedLocale, toSupportedLocaleOrCustom } from './locale'; -import type { LocaleOrCustom } from './locale'; +import type { LocaleOrCustom } from './types'; export async function importEmojiData(localeString: string) { const locale = toSupportedLocale(localeString); diff --git a/app/javascript/mastodon/features/emoji/locale.ts b/app/javascript/mastodon/features/emoji/locale.ts index 561c94afb0..8ff23f5161 100644 --- a/app/javascript/mastodon/features/emoji/locale.ts +++ b/app/javascript/mastodon/features/emoji/locale.ts @@ -1,7 +1,7 @@ import type { Locale } from 'emojibase'; import { SUPPORTED_LOCALES } from 'emojibase'; -export type LocaleOrCustom = Locale | 'custom'; +import type { LocaleOrCustom } from './types'; export function toSupportedLocale(localeBase: string): Locale { const locale = localeBase.toLowerCase(); diff --git a/app/javascript/mastodon/features/emoji/mode.ts b/app/javascript/mastodon/features/emoji/mode.ts new file mode 100644 index 0000000000..0f581d8b50 --- /dev/null +++ b/app/javascript/mastodon/features/emoji/mode.ts @@ -0,0 +1,119 @@ +// Credit to Nolan Lawson for the original implementation. +// See: https://github.com/nolanlawson/emoji-picker-element/blob/master/src/picker/utils/testColorEmojiSupported.js + +import { isDevelopment } from '@/mastodon/utils/environment'; + +import { + EMOJI_MODE_NATIVE, + EMOJI_MODE_NATIVE_WITH_FLAGS, + EMOJI_MODE_TWEMOJI, +} from './constants'; +import type { EmojiMode } from './types'; + +type Feature = Uint8ClampedArray; + +// See: https://github.com/nolanlawson/emoji-picker-element/blob/master/src/picker/constants.js +const FONT_FAMILY = + '"Twemoji Mozilla","Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol",' + + '"Noto Color Emoji","EmojiOne Color","Android Emoji",sans-serif'; + +function getTextFeature(text: string, color: string) { + const canvas = document.createElement('canvas'); + canvas.width = canvas.height = 1; + + const ctx = canvas.getContext('2d', { + // Improves the performance of `getImageData()` + // https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/getContextAttributes#willreadfrequently + willReadFrequently: true, + }); + if (!ctx) { + throw new Error('Canvas context not available'); + } + ctx.textBaseline = 'top'; + ctx.font = `100px ${FONT_FAMILY}`; + ctx.fillStyle = color; + ctx.scale(0.01, 0.01); + ctx.fillText(text, 0, 0); + + return ctx.getImageData(0, 0, 1, 1).data satisfies Feature; +} + +function compareFeatures(feature1: Feature, feature2: Feature) { + const feature1Str = [...feature1].join(','); + const feature2Str = [...feature2].join(','); + // This is RGBA, so for 0,0,0, we are checking that the first RGB is not all zeroes. + // Most of the time when unsupported this is 0,0,0,0, but on Chrome on Mac it is + // 0,0,0,61 - there is a transparency here. + return feature1Str === feature2Str && !feature1Str.startsWith('0,0,0,'); +} + +function testEmojiSupport(text: string) { + // Render white and black and then compare them to each other and ensure they're the same + // color, and neither one is black. This shows that the emoji was rendered in color. + const feature1 = getTextFeature(text, '#000'); + const feature2 = getTextFeature(text, '#fff'); + return compareFeatures(feature1, feature2); +} + +const EMOJI_VERSION_TEST_EMOJI = '🫨'; // shaking head, from v15 +const EMOJI_FLAG_TEST_EMOJI = 'πŸ‡¨πŸ‡­'; + +export function determineEmojiMode(style: string): EmojiMode { + if (style === EMOJI_MODE_NATIVE) { + // If flags are not supported, we replace them with Twemoji. + if (shouldReplaceFlags()) { + return EMOJI_MODE_NATIVE_WITH_FLAGS; + } + return EMOJI_MODE_NATIVE; + } + if (style === EMOJI_MODE_TWEMOJI) { + return EMOJI_MODE_TWEMOJI; + } + + // Auto style so determine based on browser capabilities. + if (shouldUseTwemoji()) { + return EMOJI_MODE_TWEMOJI; + } else if (shouldReplaceFlags()) { + return EMOJI_MODE_NATIVE_WITH_FLAGS; + } + return EMOJI_MODE_NATIVE; +} + +export function shouldUseTwemoji(): boolean { + if (typeof window === 'undefined') { + return false; + } + try { + // Test a known color emoji to see if 15.1 is supported. + return !testEmojiSupport(EMOJI_VERSION_TEST_EMOJI); + } catch (err: unknown) { + // If an error occurs, fall back to Twemoji to be safe. + if (isDevelopment()) { + console.warn( + 'Emoji rendering test failed, defaulting to Twemoji. Error:', + err, + ); + } + return true; + } +} + +// Based on https://github.com/talkjs/country-flag-emoji-polyfill/blob/master/src/index.ts#L19 +export function shouldReplaceFlags(): boolean { + if (typeof window === 'undefined') { + return false; + } + try { + // Test a known flag emoji to see if it is rendered in color. + return !testEmojiSupport(EMOJI_FLAG_TEST_EMOJI); + } catch (err: unknown) { + // If an error occurs, assume flags should be replaced. + if (isDevelopment()) { + console.warn( + 'Flag emoji rendering test failed, defaulting to replacement. Error:', + err, + ); + } + return true; + } +} diff --git a/app/javascript/mastodon/features/emoji/normalize.test.ts b/app/javascript/mastodon/features/emoji/normalize.test.ts index ee9cd89487..f0ea140590 100644 --- a/app/javascript/mastodon/features/emoji/normalize.test.ts +++ b/app/javascript/mastodon/features/emoji/normalize.test.ts @@ -22,9 +22,9 @@ const emojiSVGFiles = await readdir( ); const svgFileNames = emojiSVGFiles .filter((file) => file.isFile() && file.name.endsWith('.svg')) - .map((file) => basename(file.name, '.svg').toUpperCase()); + .map((file) => basename(file.name, '.svg')); const svgFileNamesWithoutBorder = svgFileNames.filter( - (fileName) => !fileName.endsWith('_BORDER'), + (fileName) => !fileName.endsWith('_border'), ); const unicodeEmojis = flattenEmojiData(unicodeRawEmojis); @@ -60,13 +60,13 @@ describe('unicodeToTwemojiHex', () => { describe('twemojiHasBorder', () => { test.concurrent.for( svgFileNames - .filter((file) => file.endsWith('_BORDER')) + .filter((file) => file.endsWith('_border')) .map((file) => { - const hexCode = file.replace('_BORDER', ''); + const hexCode = file.replace('_border', ''); return [ hexCode, - CODES_WITH_LIGHT_BORDER.includes(hexCode), - CODES_WITH_DARK_BORDER.includes(hexCode), + CODES_WITH_LIGHT_BORDER.includes(hexCode.toUpperCase()), + CODES_WITH_DARK_BORDER.includes(hexCode.toUpperCase()), ] as const; }), )('twemojiHasBorder for %s', ([hexCode, isLight, isDark], { expect }) => { diff --git a/app/javascript/mastodon/features/emoji/normalize.ts b/app/javascript/mastodon/features/emoji/normalize.ts index 94dc33a6ea..6a64c3b8bf 100644 --- a/app/javascript/mastodon/features/emoji/normalize.ts +++ b/app/javascript/mastodon/features/emoji/normalize.ts @@ -7,6 +7,7 @@ import { EMOJIS_WITH_DARK_BORDER, EMOJIS_WITH_LIGHT_BORDER, } from './constants'; +import type { TwemojiBorderInfo } from './types'; // Misc codes that have special handling const SKIER_CODE = 0x26f7; @@ -51,13 +52,7 @@ export function unicodeToTwemojiHex(unicodeHex: string): string { normalizedCodes.push(code); } - return hexNumbersToString(normalizedCodes, 0); -} - -interface TwemojiBorderInfo { - hexCode: string; - hasLightBorder: boolean; - hasDarkBorder: boolean; + return hexNumbersToString(normalizedCodes, 0).toLowerCase(); } export const CODES_WITH_DARK_BORDER = @@ -77,7 +72,7 @@ export function twemojiHasBorder(twemojiHex: string): TwemojiBorderInfo { hasDarkBorder = true; } return { - hexCode: normalizedHex, + hexCode: twemojiHex, hasLightBorder, hasDarkBorder, }; diff --git a/app/javascript/mastodon/features/emoji/render.test.ts b/app/javascript/mastodon/features/emoji/render.test.ts new file mode 100644 index 0000000000..23f85c36b3 --- /dev/null +++ b/app/javascript/mastodon/features/emoji/render.test.ts @@ -0,0 +1,163 @@ +import { + EMOJI_MODE_NATIVE, + EMOJI_MODE_NATIVE_WITH_FLAGS, + EMOJI_MODE_TWEMOJI, +} from './constants'; +import { emojifyElement, tokenizeText } from './render'; +import type { CustomEmojiData, UnicodeEmojiData } from './types'; + +vitest.mock('./database', () => ({ + searchCustomEmojisByShortcodes: vitest.fn( + () => + [ + { + shortcode: 'custom', + static_url: 'emoji/static', + url: 'emoji/custom', + category: 'test', + visible_in_picker: true, + }, + ] satisfies CustomEmojiData[], + ), + searchEmojisByHexcodes: vitest.fn( + () => + [ + { + hexcode: '1F60A', + group: 0, + label: 'smiling face with smiling eyes', + order: 0, + tags: ['smile', 'happy'], + unicode: '😊', + }, + { + hexcode: '1F1EA-1F1FA', + group: 0, + label: 'flag-eu', + order: 0, + tags: ['flag', 'european union'], + unicode: 'πŸ‡ͺπŸ‡Ί', + }, + ] satisfies UnicodeEmojiData[], + ), + findMissingLocales: vitest.fn(() => []), +})); + +describe('emojifyElement', () => { + const testElement = document.createElement('div'); + testElement.innerHTML = '

Hello 😊πŸ‡ͺπŸ‡Ί!

:custom:

'; + + const expectedSmileImage = + '😊'; + const expectedFlagImage = + 'πŸ‡ͺπŸ‡Ί'; + const expectedCustomEmojiImage = + ':custom:'; + + function cloneTestElement() { + return testElement.cloneNode(true) as HTMLElement; + } + + test('emojifies custom emoji in native mode', async () => { + const emojifiedElement = await emojifyElement(cloneTestElement(), { + locales: ['en'], + mode: EMOJI_MODE_NATIVE, + currentLocale: 'en', + }); + expect(emojifiedElement.innerHTML).toBe( + `

Hello 😊πŸ‡ͺπŸ‡Ί!

${expectedCustomEmojiImage}

`, + ); + }); + + test('emojifies flag emoji in native-with-flags mode', async () => { + const emojifiedElement = await emojifyElement(cloneTestElement(), { + locales: ['en'], + mode: EMOJI_MODE_NATIVE_WITH_FLAGS, + currentLocale: 'en', + }); + expect(emojifiedElement.innerHTML).toBe( + `

Hello 😊${expectedFlagImage}!

${expectedCustomEmojiImage}

`, + ); + }); + + test('emojifies everything in twemoji mode', async () => { + const emojifiedElement = await emojifyElement(cloneTestElement(), { + locales: ['en'], + mode: EMOJI_MODE_TWEMOJI, + currentLocale: 'en', + }); + expect(emojifiedElement.innerHTML).toBe( + `

Hello ${expectedSmileImage}${expectedFlagImage}!

${expectedCustomEmojiImage}

`, + ); + }); +}); + +describe('tokenizeText', () => { + test('returns empty array for string with only whitespace', () => { + expect(tokenizeText(' \n')).toEqual([]); + }); + + test('returns an array of text to be a single token', () => { + expect(tokenizeText('Hello')).toEqual(['Hello']); + }); + + test('returns tokens for text with emoji', () => { + expect(tokenizeText('Hello 😊 πŸ‡ΏπŸ‡Ό!!')).toEqual([ + 'Hello ', + { + type: 'unicode', + code: '😊', + }, + ' ', + { + type: 'unicode', + code: 'πŸ‡ΏπŸ‡Ό', + }, + '!!', + ]); + }); + + test('returns tokens for text with custom emoji', () => { + expect(tokenizeText('Hello :smile:!!')).toEqual([ + 'Hello ', + { + type: 'custom', + code: 'smile', + }, + '!!', + ]); + }); + + test('handles custom emoji with underscores and numbers', () => { + expect(tokenizeText('Hello :smile_123:!!')).toEqual([ + 'Hello ', + { + type: 'custom', + code: 'smile_123', + }, + '!!', + ]); + }); + + test('returns tokens for text with mixed emoji', () => { + expect(tokenizeText('Hello 😊 :smile:!!')).toEqual([ + 'Hello ', + { + type: 'unicode', + code: '😊', + }, + ' ', + { + type: 'custom', + code: 'smile', + }, + '!!', + ]); + }); + + test('does not capture custom emoji with invalid characters', () => { + expect(tokenizeText('Hello :smile-123:!!')).toEqual([ + 'Hello :smile-123:!!', + ]); + }); +}); diff --git a/app/javascript/mastodon/features/emoji/render.ts b/app/javascript/mastodon/features/emoji/render.ts new file mode 100644 index 0000000000..6ef9492147 --- /dev/null +++ b/app/javascript/mastodon/features/emoji/render.ts @@ -0,0 +1,331 @@ +import type { Locale } from 'emojibase'; +import EMOJI_REGEX from 'emojibase-regex/emoji-loose'; + +import { autoPlayGif } from '@/mastodon/initial_state'; +import { assetHost } from '@/mastodon/utils/config'; + +import { + EMOJI_MODE_NATIVE, + EMOJI_MODE_NATIVE_WITH_FLAGS, + EMOJI_TYPE_UNICODE, + EMOJI_TYPE_CUSTOM, + EMOJI_STATE_MISSING, +} from './constants'; +import { + findMissingLocales, + searchCustomEmojisByShortcodes, + searchEmojisByHexcodes, +} from './database'; +import { loadEmojiLocale } from './index'; +import { + emojiToUnicodeHex, + twemojiHasBorder, + unicodeToTwemojiHex, +} from './normalize'; +import type { + CustomEmojiToken, + EmojiAppState, + EmojiLoadedState, + EmojiMode, + EmojiState, + EmojiStateMap, + EmojiToken, + ExtraCustomEmojiMap, + LocaleOrCustom, + UnicodeEmojiToken, +} from './types'; +import { stringHasUnicodeFlags } from './utils'; + +const localeCacheMap = new Map([ + [EMOJI_TYPE_CUSTOM, new Map()], +]); + +// Emojifies an element. This modifies the element in place, replacing text nodes with emojified versions. +export async function emojifyElement( + element: Element, + appState: EmojiAppState, + extraEmojis: ExtraCustomEmojiMap = {}, +): Promise { + const queue: (HTMLElement | Text)[] = [element]; + while (queue.length > 0) { + const current = queue.shift(); + if ( + !current || + current instanceof HTMLScriptElement || + current instanceof HTMLStyleElement + ) { + continue; + } + + if ( + current.textContent && + (current instanceof Text || !current.hasChildNodes()) + ) { + const renderedContent = await emojifyText( + current.textContent, + appState, + extraEmojis, + ); + if (renderedContent) { + if (!(current instanceof Text)) { + current.textContent = null; // Clear the text content if it's not a Text node. + } + current.replaceWith(renderedToHTMLFragment(renderedContent)); + } + continue; + } + + for (const child of current.childNodes) { + if (child instanceof HTMLElement || child instanceof Text) { + queue.push(child); + } + } + } + return element; +} + +export async function emojifyText( + text: string, + appState: EmojiAppState, + extraEmojis: ExtraCustomEmojiMap = {}, +) { + // Exit if no text to convert. + if (!text.trim()) { + return null; + } + + const tokens = tokenizeText(text); + + // If only one token and it's a string, exit early. + if (tokens.length === 1 && typeof tokens[0] === 'string') { + return null; + } + + // Get all emoji from the state map, loading any missing ones. + await ensureLocalesAreLoaded(appState.locales); + await loadMissingEmojiIntoCache(tokens, appState.locales); + + const renderedFragments: (string | HTMLImageElement)[] = []; + for (const token of tokens) { + if (typeof token !== 'string' && shouldRenderImage(token, appState.mode)) { + let state: EmojiState | undefined; + if (token.type === EMOJI_TYPE_CUSTOM) { + const extraEmojiData = extraEmojis[token.code]; + if (extraEmojiData) { + state = { type: EMOJI_TYPE_CUSTOM, data: extraEmojiData }; + } else { + state = emojiForLocale(token.code, EMOJI_TYPE_CUSTOM); + } + } else { + state = emojiForLocale( + emojiToUnicodeHex(token.code), + appState.currentLocale, + ); + } + + // If the state is valid, create an image element. Otherwise, just append as text. + if (state && typeof state !== 'string') { + const image = stateToImage(state); + renderedFragments.push(image); + continue; + } + } + const text = typeof token === 'string' ? token : token.code; + renderedFragments.push(text); + } + + return renderedFragments; +} + +// Private functions + +async function ensureLocalesAreLoaded(locales: Locale[]) { + const missingLocales = await findMissingLocales(locales); + for (const locale of missingLocales) { + await loadEmojiLocale(locale); + } +} + +const CUSTOM_EMOJI_REGEX = /:([a-z0-9_]+):/i; +const TOKENIZE_REGEX = new RegExp( + `(${EMOJI_REGEX.source}|${CUSTOM_EMOJI_REGEX.source})`, + 'g', +); + +type TokenizedText = (string | EmojiToken)[]; + +export function tokenizeText(text: string): TokenizedText { + if (!text.trim()) { + return []; + } + + const tokens = []; + let lastIndex = 0; + for (const match of text.matchAll(TOKENIZE_REGEX)) { + if (match.index > lastIndex) { + tokens.push(text.slice(lastIndex, match.index)); + } + + const code = match[0]; + + if (code.startsWith(':') && code.endsWith(':')) { + // Custom emoji + tokens.push({ + type: EMOJI_TYPE_CUSTOM, + code: code.slice(1, -1), // Remove the colons + } satisfies CustomEmojiToken); + } else { + // Unicode emoji + tokens.push({ + type: EMOJI_TYPE_UNICODE, + code: code, + } satisfies UnicodeEmojiToken); + } + lastIndex = match.index + code.length; + } + if (lastIndex < text.length) { + tokens.push(text.slice(lastIndex)); + } + return tokens; +} + +function cacheForLocale(locale: LocaleOrCustom): EmojiStateMap { + return localeCacheMap.get(locale) ?? (new Map() as EmojiStateMap); +} + +function emojiForLocale( + code: string, + locale: LocaleOrCustom, +): EmojiState | undefined { + const cache = cacheForLocale(locale); + return cache.get(code); +} + +async function loadMissingEmojiIntoCache( + tokens: TokenizedText, + locales: Locale[], +) { + const missingUnicodeEmoji = new Set(); + const missingCustomEmoji = new Set(); + + // Iterate over tokens and check if they are in the cache already. + for (const token of tokens) { + if (typeof token === 'string') { + continue; // Skip plain strings. + } + + // If this is a custom emoji, check it separately. + if (token.type === EMOJI_TYPE_CUSTOM) { + const code = token.code; + const emojiState = emojiForLocale(code, EMOJI_TYPE_CUSTOM); + if (!emojiState) { + missingCustomEmoji.add(code); + } + // Otherwise this is a unicode emoji, so check it against all locales. + } else { + const code = emojiToUnicodeHex(token.code); + if (missingUnicodeEmoji.has(code)) { + continue; // Already marked as missing. + } + for (const locale of locales) { + const emojiState = emojiForLocale(code, locale); + if (!emojiState) { + // If it's missing in one locale, we consider it missing for all. + missingUnicodeEmoji.add(code); + } + } + } + } + + if (missingUnicodeEmoji.size > 0) { + const missingEmojis = Array.from(missingUnicodeEmoji).toSorted(); + for (const locale of locales) { + const emojis = await searchEmojisByHexcodes(missingEmojis, locale); + const cache = cacheForLocale(locale); + for (const emoji of emojis) { + cache.set(emoji.hexcode, { type: EMOJI_TYPE_UNICODE, data: emoji }); + } + const notFoundEmojis = missingEmojis.filter((code) => + emojis.every((emoji) => emoji.hexcode !== code), + ); + for (const code of notFoundEmojis) { + cache.set(code, EMOJI_STATE_MISSING); // Mark as missing if not found, as it's probably not a valid emoji. + } + localeCacheMap.set(locale, cache); + } + } + + if (missingCustomEmoji.size > 0) { + const missingEmojis = Array.from(missingCustomEmoji).toSorted(); + const emojis = await searchCustomEmojisByShortcodes(missingEmojis); + const cache = cacheForLocale(EMOJI_TYPE_CUSTOM); + for (const emoji of emojis) { + cache.set(emoji.shortcode, { type: EMOJI_TYPE_CUSTOM, data: emoji }); + } + const notFoundEmojis = missingEmojis.filter((code) => + emojis.every((emoji) => emoji.shortcode !== code), + ); + for (const code of notFoundEmojis) { + cache.set(code, EMOJI_STATE_MISSING); // Mark as missing if not found, as it's probably not a valid emoji. + } + localeCacheMap.set(EMOJI_TYPE_CUSTOM, cache); + } +} + +function shouldRenderImage(token: EmojiToken, mode: EmojiMode): boolean { + if (token.type === EMOJI_TYPE_UNICODE) { + // If the mode is native or native with flags for non-flag emoji + // we can just append the text node directly. + if ( + mode === EMOJI_MODE_NATIVE || + (mode === EMOJI_MODE_NATIVE_WITH_FLAGS && + !stringHasUnicodeFlags(token.code)) + ) { + return false; + } + } + + return true; +} + +function stateToImage(state: EmojiLoadedState) { + const image = document.createElement('img'); + image.draggable = false; + image.classList.add('emojione'); + + if (state.type === EMOJI_TYPE_UNICODE) { + const emojiInfo = twemojiHasBorder(unicodeToTwemojiHex(state.data.hexcode)); + if (emojiInfo.hasLightBorder) { + image.dataset.lightCode = `${emojiInfo.hexCode}_BORDER`; + } else if (emojiInfo.hasDarkBorder) { + image.dataset.darkCode = `${emojiInfo.hexCode}_BORDER`; + } + + image.alt = state.data.unicode; + image.title = state.data.label; + image.src = `${assetHost}/emoji/${emojiInfo.hexCode}.svg`; + } else { + // Custom emoji + const shortCode = `:${state.data.shortcode}:`; + image.classList.add('custom-emoji'); + image.alt = shortCode; + image.title = shortCode; + image.src = autoPlayGif ? state.data.url : state.data.static_url; + image.dataset.original = state.data.url; + image.dataset.static = state.data.static_url; + } + + return image; +} + +function renderedToHTMLFragment(renderedArray: (string | HTMLImageElement)[]) { + const fragment = document.createDocumentFragment(); + for (const fragmentItem of renderedArray) { + if (typeof fragmentItem === 'string') { + fragment.appendChild(document.createTextNode(fragmentItem)); + } else if (fragmentItem instanceof HTMLImageElement) { + fragment.appendChild(fragmentItem); + } + } + return fragment; +} diff --git a/app/javascript/mastodon/features/emoji/types.ts b/app/javascript/mastodon/features/emoji/types.ts new file mode 100644 index 0000000000..f5932ed97f --- /dev/null +++ b/app/javascript/mastodon/features/emoji/types.ts @@ -0,0 +1,64 @@ +import type { FlatCompactEmoji, Locale } from 'emojibase'; + +import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji'; + +import type { + EMOJI_MODE_NATIVE, + EMOJI_MODE_NATIVE_WITH_FLAGS, + EMOJI_MODE_TWEMOJI, + EMOJI_STATE_MISSING, + EMOJI_TYPE_CUSTOM, + EMOJI_TYPE_UNICODE, +} from './constants'; + +export type EmojiMode = + | typeof EMOJI_MODE_NATIVE + | typeof EMOJI_MODE_NATIVE_WITH_FLAGS + | typeof EMOJI_MODE_TWEMOJI; + +export type LocaleOrCustom = Locale | typeof EMOJI_TYPE_CUSTOM; + +export interface EmojiAppState { + locales: Locale[]; + currentLocale: Locale; + mode: EmojiMode; +} + +export interface UnicodeEmojiToken { + type: typeof EMOJI_TYPE_UNICODE; + code: string; +} +export interface CustomEmojiToken { + type: typeof EMOJI_TYPE_CUSTOM; + code: string; +} +export type EmojiToken = UnicodeEmojiToken | CustomEmojiToken; + +export type CustomEmojiData = ApiCustomEmojiJSON; +export type UnicodeEmojiData = FlatCompactEmoji; +export type AnyEmojiData = CustomEmojiData | UnicodeEmojiData; + +export type EmojiStateMissing = typeof EMOJI_STATE_MISSING; +export interface EmojiStateUnicode { + type: typeof EMOJI_TYPE_UNICODE; + data: UnicodeEmojiData; +} +export interface EmojiStateCustom { + type: typeof EMOJI_TYPE_CUSTOM; + data: CustomEmojiData; +} +export type EmojiState = + | EmojiStateMissing + | EmojiStateUnicode + | EmojiStateCustom; +export type EmojiLoadedState = EmojiStateUnicode | EmojiStateCustom; + +export type EmojiStateMap = Map; + +export type ExtraCustomEmojiMap = Record; + +export interface TwemojiBorderInfo { + hexCode: string; + hasLightBorder: boolean; + hasDarkBorder: boolean; +} diff --git a/app/javascript/mastodon/features/emoji/utils.test.ts b/app/javascript/mastodon/features/emoji/utils.test.ts new file mode 100644 index 0000000000..75cac8c5b4 --- /dev/null +++ b/app/javascript/mastodon/features/emoji/utils.test.ts @@ -0,0 +1,47 @@ +import { stringHasUnicodeEmoji, stringHasUnicodeFlags } from './utils'; + +describe('stringHasEmoji', () => { + test.concurrent.for([ + ['only text', false], + ['text with emoji πŸ˜€', true], + ['multiple emojis πŸ˜€πŸ˜ƒπŸ˜„', true], + ['emoji with skin tone πŸ‘πŸ½', true], + ['emoji with ZWJ πŸ‘©β€β€οΈβ€πŸ‘¨', true], + ['emoji with variation selector ✊️', true], + ['emoji with keycap 1️⃣', true], + ['emoji with flags πŸ‡ΊπŸ‡Έ', true], + ['emoji with regional indicators πŸ‡¦πŸ‡Ί', true], + ['emoji with gender πŸ‘©β€βš•οΈ', true], + ['emoji with family πŸ‘¨β€πŸ‘©β€πŸ‘§β€πŸ‘¦', true], + ['emoji with zero width joiner πŸ‘©β€πŸ”¬', true], + ['emoji with non-BMP codepoint πŸ§‘β€πŸš€', true], + ['emoji with combining marks πŸ‘¨β€πŸ‘©β€πŸ‘§β€πŸ‘¦', true], + ['emoji with enclosing keycap #️⃣', true], + ['emoji with no visible glyph \u200D', false], + ] as const)( + 'stringHasEmoji has emojis in "%s": %o', + ([text, expected], { expect }) => { + expect(stringHasUnicodeEmoji(text)).toBe(expected); + }, + ); +}); + +describe('stringHasFlags', () => { + test.concurrent.for([ + ['EU πŸ‡ͺπŸ‡Ί', true], + ['Germany πŸ‡©πŸ‡ͺ', true], + ['Canada πŸ‡¨πŸ‡¦', true], + ['SΓ£o TomΓ© & PrΓ­ncipe πŸ‡ΈπŸ‡Ή', true], + ['Scotland 🏴󠁧󠁒󠁳󠁣󠁴󠁿', true], + ['black flag 🏴', false], + ['arrr πŸ΄β€β˜ οΈ', false], + ['rainbow flag πŸ³οΈβ€πŸŒˆ', false], + ['non-flag πŸ”₯', false], + ['only text', false], + ] as const)( + 'stringHasFlags has flag in "%s": %o', + ([text, expected], { expect }) => { + expect(stringHasUnicodeFlags(text)).toBe(expected); + }, + ); +}); diff --git a/app/javascript/mastodon/features/emoji/utils.ts b/app/javascript/mastodon/features/emoji/utils.ts new file mode 100644 index 0000000000..d00accea8c --- /dev/null +++ b/app/javascript/mastodon/features/emoji/utils.ts @@ -0,0 +1,13 @@ +import EMOJI_REGEX from 'emojibase-regex/emoji-loose'; + +export function stringHasUnicodeEmoji(text: string): boolean { + return EMOJI_REGEX.test(text); +} + +// From https://github.com/talkjs/country-flag-emoji-polyfill/blob/master/src/index.ts#L49-L50 +const EMOJIS_FLAGS_REGEX = + /[\u{1F1E6}-\u{1F1FF}|\u{E0062}-\u{E0063}|\u{E0065}|\u{E0067}|\u{E006C}|\u{E006E}|\u{E0073}-\u{E0074}|\u{E0077}|\u{E007F}]+/u; + +export function stringHasUnicodeFlags(text: string): boolean { + return EMOJIS_FLAGS_REGEX.test(text); +} diff --git a/app/javascript/mastodon/features/status/components/refresh_controller.tsx b/app/javascript/mastodon/features/status/components/refresh_controller.tsx new file mode 100644 index 0000000000..04046302b6 --- /dev/null +++ b/app/javascript/mastodon/features/status/components/refresh_controller.tsx @@ -0,0 +1,111 @@ +import { useEffect, useState, useCallback } from 'react'; + +import { useIntl, defineMessages, FormattedMessage } from 'react-intl'; + +import classNames from 'classnames'; + +import { + fetchContext, + completeContextRefresh, +} from 'mastodon/actions/statuses'; +import type { AsyncRefreshHeader } from 'mastodon/api'; +import { apiGetAsyncRefresh } from 'mastodon/api/async_refreshes'; +import { LoadingIndicator } from 'mastodon/components/loading_indicator'; +import { useAppSelector, useAppDispatch } from 'mastodon/store'; + +const messages = defineMessages({ + loading: { + id: 'status.context.loading', + defaultMessage: 'Checking for more replies', + }, +}); + +export const RefreshController: React.FC<{ + statusId: string; + withBorder?: boolean; +}> = ({ statusId, withBorder }) => { + const refresh = useAppSelector( + (state) => state.contexts.refreshing[statusId], + ); + const dispatch = useAppDispatch(); + const intl = useIntl(); + const [ready, setReady] = useState(false); + const [loading, setLoading] = useState(false); + + useEffect(() => { + let timeoutId: ReturnType; + + const scheduleRefresh = (refresh: AsyncRefreshHeader) => { + timeoutId = setTimeout(() => { + void apiGetAsyncRefresh(refresh.id).then((result) => { + if (result.async_refresh.status === 'finished') { + dispatch(completeContextRefresh({ statusId })); + + if (result.async_refresh.result_count > 0) { + setReady(true); + } + } else { + scheduleRefresh(refresh); + } + + return ''; + }); + }, refresh.retry * 1000); + }; + + if (refresh) { + scheduleRefresh(refresh); + } + + return () => { + clearTimeout(timeoutId); + }; + }, [dispatch, setReady, statusId, refresh]); + + const handleClick = useCallback(() => { + setLoading(true); + setReady(false); + + dispatch(fetchContext({ statusId })) + .then(() => { + setLoading(false); + return ''; + }) + .catch(() => { + setLoading(false); + }); + }, [dispatch, setReady, statusId]); + + if (ready && !loading) { + return ( + + ); + } + + if (!refresh && !loading) { + return null; + } + + return ( +
+ +
+ ); +}; diff --git a/app/javascript/mastodon/features/status/index.jsx b/app/javascript/mastodon/features/status/index.jsx index 64cd0c4f82..77d23f55f6 100644 --- a/app/javascript/mastodon/features/status/index.jsx +++ b/app/javascript/mastodon/features/status/index.jsx @@ -68,7 +68,7 @@ import { attachFullscreenListener, detachFullscreenListener, isFullscreen } from import ActionBar from './components/action_bar'; import { DetailedStatus } from './components/detailed_status'; - +import { RefreshController } from './components/refresh_controller'; const messages = defineMessages({ revealAll: { id: 'status.show_more_all', defaultMessage: 'Show more for all' }, @@ -548,7 +548,7 @@ class Status extends ImmutablePureComponent { render () { let ancestors, descendants, remoteHint; - const { isLoading, status, ancestorsIds, descendantsIds, intl, domain, multiColumn, pictureInPicture } = this.props; + const { isLoading, status, ancestorsIds, descendantsIds, refresh, intl, domain, multiColumn, pictureInPicture } = this.props; const { fullscreen } = this.state; if (isLoading) { @@ -578,11 +578,9 @@ class Status extends ImmutablePureComponent { if (!isLocal) { remoteHint = ( - } - label={{status.getIn(['account', 'acct']).split('@')[1]} }} />} + ); } diff --git a/app/javascript/mastodon/initial_state.js b/app/javascript/mastodon/initial_state.js index 590c4c8d2b..7763d9cb79 100644 --- a/app/javascript/mastodon/initial_state.js +++ b/app/javascript/mastodon/initial_state.js @@ -45,6 +45,7 @@ * @property {string} sso_redirect * @property {string} status_page_url * @property {boolean} terms_of_service_enabled + * @property {string?} emoji_style */ /** @@ -95,6 +96,7 @@ export const disableHoverCards = getMeta('disable_hover_cards'); export const disabledAccountId = getMeta('disabled_account_id'); export const displayMedia = getMeta('display_media'); export const domain = getMeta('domain'); +export const emojiStyle = getMeta('emoji_style') || 'auto'; export const expandSpoilers = getMeta('expand_spoilers'); export const forceSingleColumn = !getMeta('advanced_layout'); export const limitedFederationMode = getMeta('limited_federation_mode'); diff --git a/app/javascript/mastodon/locales/en.json b/app/javascript/mastodon/locales/en.json index 59d39a1536..13b7aa4212 100644 --- a/app/javascript/mastodon/locales/en.json +++ b/app/javascript/mastodon/locales/en.json @@ -424,8 +424,6 @@ "hints.profiles.see_more_followers": "See more followers on {domain}", "hints.profiles.see_more_follows": "See more follows on {domain}", "hints.profiles.see_more_posts": "See more posts on {domain}", - "hints.threads.replies_may_be_missing": "Replies from other servers may be missing.", - "hints.threads.see_more": "See more replies on {domain}", "home.column_settings.show_quotes": "Show quotes", "home.column_settings.show_reblogs": "Show boosts", "home.column_settings.show_replies": "Show replies", @@ -847,6 +845,8 @@ "status.bookmark": "Bookmark", "status.cancel_reblog_private": "Unboost", "status.cannot_reblog": "This post cannot be boosted", + "status.context.load_new_replies": "New replies available", + "status.context.loading": "Checking for more replies", "status.continued_thread": "Continued thread", "status.copy": "Copy link to post", "status.delete": "Delete", diff --git a/app/javascript/mastodon/main.tsx b/app/javascript/mastodon/main.tsx index 70e6391bee..e840429c41 100644 --- a/app/javascript/mastodon/main.tsx +++ b/app/javascript/mastodon/main.tsx @@ -4,7 +4,7 @@ import { Globals } from '@react-spring/web'; import { setupBrowserNotifications } from 'mastodon/actions/notifications'; import Mastodon from 'mastodon/containers/mastodon'; -import { me, reduceMotion } from 'mastodon/initial_state'; +import { isFeatureEnabled, me, reduceMotion } from 'mastodon/initial_state'; import * as perf from 'mastodon/performance'; import ready from 'mastodon/ready'; import { store } from 'mastodon/store'; @@ -29,6 +29,11 @@ function main() { }); } + if (isFeatureEnabled('modern_emojis')) { + const { initializeEmoji } = await import('@/mastodon/features/emoji'); + await initializeEmoji(); + } + const root = createRoot(mountNode); root.render(); store.dispatch(setupBrowserNotifications()); diff --git a/app/javascript/mastodon/reducers/contexts.ts b/app/javascript/mastodon/reducers/contexts.ts index 7ecc6e3b29..cf378b4c04 100644 --- a/app/javascript/mastodon/reducers/contexts.ts +++ b/app/javascript/mastodon/reducers/contexts.ts @@ -4,6 +4,7 @@ import type { Draft, UnknownAction } from '@reduxjs/toolkit'; import type { List as ImmutableList } from 'immutable'; import { timelineDelete } from 'mastodon/actions/timelines_typed'; +import type { AsyncRefreshHeader } from 'mastodon/api'; import type { ApiRelationshipJSON } from 'mastodon/api_types/relationships'; import type { ApiStatusJSON, @@ -12,7 +13,7 @@ import type { import type { Status } from 'mastodon/models/status'; import { blockAccountSuccess, muteAccountSuccess } from '../actions/accounts'; -import { fetchContext } from '../actions/statuses'; +import { fetchContext, completeContextRefresh } from '../actions/statuses'; import { TIMELINE_UPDATE } from '../actions/timelines'; import { compareId } from '../compare_id'; @@ -25,11 +26,13 @@ interface TimelineUpdateAction extends UnknownAction { interface State { inReplyTos: Record; replies: Record; + refreshing: Record; } const initialState: State = { inReplyTos: {}, replies: {}, + refreshing: {}, }; const normalizeContext = ( @@ -127,6 +130,13 @@ export const contextsReducer = createReducer(initialState, (builder) => { builder .addCase(fetchContext.fulfilled, (state, action) => { normalizeContext(state, action.meta.arg.statusId, action.payload.context); + + if (action.payload.refresh) { + state.refreshing[action.meta.arg.statusId] = action.payload.refresh; + } + }) + .addCase(completeContextRefresh, (state, action) => { + delete state.refreshing[action.payload.statusId]; }) .addCase(blockAccountSuccess, (state, action) => { filterContexts( diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss index 40b073f68b..0fd97fb712 100644 --- a/app/javascript/styles/mastodon/components.scss +++ b/app/javascript/styles/mastodon/components.scss @@ -2868,6 +2868,8 @@ a.account__display-name { } &__main { + --column-header-height: 62px; + box-sizing: border-box; width: 100%; flex: 0 1 auto; @@ -8815,6 +8817,10 @@ noscript { .conversation { position: relative; + // When scrolling these elements into view, take into account + // the column header height + scroll-margin-top: var(--column-header-height, 0); + &.unread { &::before { content: ''; diff --git a/app/models/concerns/status/fetch_replies_concern.rb b/app/models/concerns/status/fetch_replies_concern.rb index fd9929aba4..cc117cb5ac 100644 --- a/app/models/concerns/status/fetch_replies_concern.rb +++ b/app/models/concerns/status/fetch_replies_concern.rb @@ -3,9 +3,6 @@ module Status::FetchRepliesConcern extend ActiveSupport::Concern - # enable/disable fetching all replies - FETCH_REPLIES_ENABLED = ENV['FETCH_REPLIES_ENABLED'] == 'true' - # debounce fetching all replies to minimize DoS FETCH_REPLIES_COOLDOWN_MINUTES = (ENV['FETCH_REPLIES_COOLDOWN_MINUTES'] || 15).to_i.minutes FETCH_REPLIES_INITIAL_WAIT_MINUTES = (ENV['FETCH_REPLIES_INITIAL_WAIT_MINUTES'] || 5).to_i.minutes @@ -36,7 +33,7 @@ module Status::FetchRepliesConcern def should_fetch_replies? # we aren't brand new, and we haven't fetched replies since the debounce window - FETCH_REPLIES_ENABLED && !local? && created_at <= FETCH_REPLIES_INITIAL_WAIT_MINUTES.ago && ( + !local? && created_at <= FETCH_REPLIES_INITIAL_WAIT_MINUTES.ago && ( fetched_replies_at.nil? || fetched_replies_at <= FETCH_REPLIES_COOLDOWN_MINUTES.ago ) end diff --git a/app/models/form/account_batch.rb b/app/models/form/account_batch.rb index 98e3be1a0c..f3109ad62a 100644 --- a/app/models/form/account_batch.rb +++ b/app/models/form/account_batch.rb @@ -1,13 +1,11 @@ # frozen_string_literal: true -class Form::AccountBatch - include ActiveModel::Model - include Authorization - include AccountableConcern +class Form::AccountBatch < Form::BaseBatch include Payloadable - attr_accessor :account_ids, :action, :current_account, - :select_all_matching, :query + attr_accessor :account_ids, + :query, + :select_all_matching def save case action diff --git a/app/models/form/base_batch.rb b/app/models/form/base_batch.rb new file mode 100644 index 0000000000..d3af923784 --- /dev/null +++ b/app/models/form/base_batch.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class Form::BaseBatch + include ActiveModel::Model + include Authorization + include AccountableConcern + + attr_accessor :action, + :current_account + + def save + raise 'Override in subclass' + end +end diff --git a/app/models/form/custom_emoji_batch.rb b/app/models/form/custom_emoji_batch.rb index c63996e069..b8cfb36399 100644 --- a/app/models/form/custom_emoji_batch.rb +++ b/app/models/form/custom_emoji_batch.rb @@ -1,12 +1,10 @@ # frozen_string_literal: true -class Form::CustomEmojiBatch - include ActiveModel::Model - include Authorization - include AccountableConcern - - attr_accessor :custom_emoji_ids, :action, :current_account, - :category_id, :category_name, :visible_in_picker +class Form::CustomEmojiBatch < Form::BaseBatch + attr_accessor :category_id, + :category_name, + :visible_in_picker, + :custom_emoji_ids def save case action diff --git a/app/models/form/domain_block_batch.rb b/app/models/form/domain_block_batch.rb index 39012df517..af792fd41f 100644 --- a/app/models/form/domain_block_batch.rb +++ b/app/models/form/domain_block_batch.rb @@ -1,11 +1,7 @@ # frozen_string_literal: true -class Form::DomainBlockBatch - include ActiveModel::Model - include Authorization - include AccountableConcern - - attr_accessor :domain_blocks_attributes, :action, :current_account +class Form::DomainBlockBatch < Form::BaseBatch + attr_accessor :domain_blocks_attributes def save case action diff --git a/app/models/form/email_domain_block_batch.rb b/app/models/form/email_domain_block_batch.rb index df120182bc..6292f2b1e1 100644 --- a/app/models/form/email_domain_block_batch.rb +++ b/app/models/form/email_domain_block_batch.rb @@ -1,11 +1,7 @@ # frozen_string_literal: true -class Form::EmailDomainBlockBatch - include ActiveModel::Model - include Authorization - include AccountableConcern - - attr_accessor :email_domain_block_ids, :action, :current_account +class Form::EmailDomainBlockBatch < Form::BaseBatch + attr_accessor :email_domain_block_ids def save case action diff --git a/app/models/form/ip_block_batch.rb b/app/models/form/ip_block_batch.rb index bdfeb91c8a..b6a189750e 100644 --- a/app/models/form/ip_block_batch.rb +++ b/app/models/form/ip_block_batch.rb @@ -1,11 +1,7 @@ # frozen_string_literal: true -class Form::IpBlockBatch - include ActiveModel::Model - include Authorization - include AccountableConcern - - attr_accessor :ip_block_ids, :action, :current_account +class Form::IpBlockBatch < Form::BaseBatch + attr_accessor :ip_block_ids def save case action diff --git a/app/models/worker_batch.rb b/app/models/worker_batch.rb new file mode 100644 index 0000000000..f741071ba9 --- /dev/null +++ b/app/models/worker_batch.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +class WorkerBatch + include Redisable + + TTL = 3600 + + def initialize(id = nil) + @id = id || SecureRandom.hex(12) + end + + attr_reader :id + + # Connect the batch with an async refresh. When the number of processed jobs + # passes the given threshold, the async refresh will be marked as finished. + # @param [String] async_refresh_key + # @param [Float] threshold + def connect(async_refresh_key, threshold: 1.0) + redis.hset(key, { 'async_refresh_key' => async_refresh_key, 'threshold' => threshold }) + end + + # Add jobs to the batch. Usually when the batch is created. + # @param [Array] jids + def add_jobs(jids) + if jids.blank? + async_refresh_key = redis.hget(key, 'async_refresh_key') + + if async_refresh_key.present? + async_refresh = AsyncRefresh.new(async_refresh_key) + async_refresh.finish! + end + + return + end + + redis.multi do |pipeline| + pipeline.sadd(key('jobs'), jids) + pipeline.expire(key('jobs'), TTL) + pipeline.hincrby(key, 'pending', jids.size) + pipeline.expire(key, TTL) + end + end + + # Remove a job from the batch, such as when it's been processed or it has failed. + # @param [String] jid + def remove_job(jid) + _, pending, processed, async_refresh_key, threshold = redis.multi do |pipeline| + pipeline.srem(key('jobs'), jid) + pipeline.hincrby(key, 'pending', -1) + pipeline.hincrby(key, 'processed', 1) + pipeline.hget(key, 'async_refresh_key') + pipeline.hget(key, 'threshold') + end + + if async_refresh_key.present? + async_refresh = AsyncRefresh.new(async_refresh_key) + async_refresh.increment_result_count(by: 1) + async_refresh.finish! if pending.zero? || processed >= threshold.to_f * (processed + pending) + end + end + + # Get pending jobs. + # @returns [Array] + def jobs + redis.smembers(key('jobs')) + end + + # Inspect the batch. + # @returns [Hash] + def info + redis.hgetall(key) + end + + private + + def key(suffix = nil) + "worker_batch:#{@id}#{":#{suffix}" if suffix}" + end +end diff --git a/app/serializers/initial_state_serializer.rb b/app/serializers/initial_state_serializer.rb index 1d0e1134f5..09384db683 100644 --- a/app/serializers/initial_state_serializer.rb +++ b/app/serializers/initial_state_serializer.rb @@ -47,6 +47,7 @@ class InitialStateSerializer < ActiveModel::Serializer store[:default_content_type] = object_account_user.setting_default_content_type store[:system_emoji_font] = object_account_user.setting_system_emoji_font store[:show_trends] = Setting.trends && object_account_user.setting_trends + store[:emoji_style] = object_account_user.settings['web.emoji_style'] if Mastodon::Feature.modern_emojis_enabled? else store[:auto_play_gif] = Setting.auto_play_gif store[:display_media] = Setting.display_media diff --git a/app/services/activitypub/fetch_all_replies_service.rb b/app/services/activitypub/fetch_all_replies_service.rb index 765e5c8ae8..e9c1712ed6 100644 --- a/app/services/activitypub/fetch_all_replies_service.rb +++ b/app/services/activitypub/fetch_all_replies_service.rb @@ -6,7 +6,7 @@ class ActivityPub::FetchAllRepliesService < ActivityPub::FetchRepliesService # Limit of replies to fetch per status MAX_REPLIES = (ENV['FETCH_REPLIES_MAX_SINGLE'] || 500).to_i - def call(status_uri, collection_or_uri, max_pages: 1, request_id: nil) + def call(status_uri, collection_or_uri, max_pages: 1, async_refresh_key: nil, request_id: nil) @status_uri = status_uri super diff --git a/app/services/activitypub/fetch_replies_service.rb b/app/services/activitypub/fetch_replies_service.rb index 6a6d9e391a..25eb275ca5 100644 --- a/app/services/activitypub/fetch_replies_service.rb +++ b/app/services/activitypub/fetch_replies_service.rb @@ -6,7 +6,7 @@ class ActivityPub::FetchRepliesService < BaseService # Limit of fetched replies MAX_REPLIES = 5 - def call(reference_uri, collection_or_uri, max_pages: 1, allow_synchronous_requests: true, request_id: nil) + def call(reference_uri, collection_or_uri, max_pages: 1, allow_synchronous_requests: true, async_refresh_key: nil, request_id: nil) @reference_uri = reference_uri @allow_synchronous_requests = allow_synchronous_requests @@ -14,7 +14,10 @@ class ActivityPub::FetchRepliesService < BaseService return if @items.nil? @items = filter_replies(@items) - FetchReplyWorker.push_bulk(@items) { |reply_uri| [reply_uri, { 'request_id' => request_id }] } + + batch = WorkerBatch.new + batch.connect(async_refresh_key) if async_refresh_key.present? + batch.add_jobs(FetchReplyWorker.push_bulk(@items) { |reply_uri| [reply_uri, { 'request_id' => request_id, 'batch_id' => batch.id }] }) [@items, n_pages] end diff --git a/app/workers/activitypub/fetch_all_replies_worker.rb b/app/workers/activitypub/fetch_all_replies_worker.rb index 40b251cf14..ab9eebc4ec 100644 --- a/app/workers/activitypub/fetch_all_replies_worker.rb +++ b/app/workers/activitypub/fetch_all_replies_worker.rb @@ -55,7 +55,7 @@ class ActivityPub::FetchAllRepliesWorker replies_collection_or_uri = get_replies_uri(status) return if replies_collection_or_uri.nil? - ActivityPub::FetchAllRepliesService.new.call(value_or_id(status), replies_collection_or_uri, max_pages: max_pages, **options.deep_symbolize_keys) + ActivityPub::FetchAllRepliesService.new.call(value_or_id(status), replies_collection_or_uri, max_pages: max_pages, async_refresh_key: "context:#{@root_status.id}:refresh", **options.deep_symbolize_keys) end # Get the URI of the replies collection of a status diff --git a/app/workers/fetch_reply_worker.rb b/app/workers/fetch_reply_worker.rb index ecb232bbbb..da3b9a8c13 100644 --- a/app/workers/fetch_reply_worker.rb +++ b/app/workers/fetch_reply_worker.rb @@ -7,6 +7,9 @@ class FetchReplyWorker sidekiq_options queue: 'pull', retry: 3 def perform(child_url, options = {}) + batch = WorkerBatch.new(options.delete('batch_id')) if options['batch_id'] FetchRemoteStatusService.new.call(child_url, **options.symbolize_keys) + ensure + batch&.remove_job(jid) end end diff --git a/package.json b/package.json index 9089397403..2e4f2a3ba9 100644 --- a/package.json +++ b/package.json @@ -69,6 +69,7 @@ "emoji-mart": "npm:emoji-mart-lazyload@latest", "emojibase": "^16.0.0", "emojibase-data": "^16.0.3", + "emojibase-regex": "^16.0.0", "escape-html": "^1.0.3", "fast-glob": "^3.3.3", "favico.js": "^0.3.10", diff --git a/spec/models/worker_batch_spec.rb b/spec/models/worker_batch_spec.rb new file mode 100644 index 0000000000..b58dc48618 --- /dev/null +++ b/spec/models/worker_batch_spec.rb @@ -0,0 +1,104 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe WorkerBatch do + subject { described_class.new } + + let(:async_refresh_key) { 'test_refresh' } + let(:async_refresh) { nil } + + describe '#id' do + it 'returns a string' do + expect(subject.id).to be_a String + end + end + + describe '#connect' do + before do + subject.connect(async_refresh_key, threshold: 0.75) + end + + it 'persists the async refresh key' do + expect(subject.info['async_refresh_key']).to eq async_refresh_key + end + + it 'persists the threshold' do + expect(subject.info['threshold']).to eq '0.75' + end + end + + describe '#add_jobs' do + before do + subject.connect(async_refresh_key, threshold: 0.5) if async_refresh.present? + subject.add_jobs([]) + end + + context 'when called with empty array' do + it 'does not persist the number of pending jobs' do + expect(subject.info).to be_empty + end + + it 'does not persist the job IDs' do + expect(subject.jobs).to eq [] + end + + context 'when async refresh is connected' do + let(:async_refresh) { AsyncRefresh.new(async_refresh_key) } + + it 'immediately marks the async refresh as finished' do + expect(async_refresh.reload.finished?).to be true + end + end + end + + context 'when called with an array of job IDs' do + before do + subject.add_jobs(%w(foo bar)) + end + + it 'persists the number of pending jobs' do + expect(subject.info['pending']).to eq '2' + end + + it 'persists the job IDs' do + expect(subject.jobs).to eq %w(foo bar) + end + end + end + + describe '#remove_job' do + before do + subject.connect(async_refresh_key, threshold: 0.5) if async_refresh.present? + subject.add_jobs(%w(foo bar baz)) + subject.remove_job('foo') + end + + it 'removes the job from pending jobs' do + expect(subject.jobs).to eq %w(bar baz) + end + + it 'decrements the number of pending jobs' do + expect(subject.info['pending']).to eq '2' + end + + context 'when async refresh is connected' do + let(:async_refresh) { AsyncRefresh.new(async_refresh_key) } + + it 'increments async refresh progress' do + expect(async_refresh.reload.result_count).to eq 1 + end + + it 'marks the async refresh as finished when the threshold is reached' do + subject.remove_job('bar') + expect(async_refresh.reload.finished?).to be true + end + end + end + + describe '#info' do + it 'returns a hash' do + expect(subject.info).to be_a Hash + end + end +end diff --git a/spec/workers/activitypub/fetch_all_replies_worker_spec.rb b/spec/workers/activitypub/fetch_all_replies_worker_spec.rb index 9a8bdac030..9795c4619a 100644 --- a/spec/workers/activitypub/fetch_all_replies_worker_spec.rb +++ b/spec/workers/activitypub/fetch_all_replies_worker_spec.rb @@ -123,7 +123,6 @@ RSpec.describe ActivityPub::FetchAllRepliesWorker do end before do - stub_const('Status::FetchRepliesConcern::FETCH_REPLIES_ENABLED', true) all_items.each do |item| next if [top_note_uri, reply_note_uri].include? item diff --git a/yarn.lock b/yarn.lock index ba5f4c1789..a2495c7644 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2672,6 +2672,7 @@ __metadata: emoji-mart: "npm:emoji-mart-lazyload@latest" emojibase: "npm:^16.0.0" emojibase-data: "npm:^16.0.3" + emojibase-regex: "npm:^16.0.0" escape-html: "npm:^1.0.3" eslint: "npm:^9.23.0" eslint-import-resolver-typescript: "npm:^4.2.5" @@ -3295,23 +3296,7 @@ __metadata: languageName: node linkType: hard -"@rollup/pluginutils@npm:^5.0.1, @rollup/pluginutils@npm:^5.0.2, @rollup/pluginutils@npm:^5.1.0": - version: 5.1.4 - resolution: "@rollup/pluginutils@npm:5.1.4" - dependencies: - "@types/estree": "npm:^1.0.0" - estree-walker: "npm:^2.0.2" - picomatch: "npm:^4.0.2" - peerDependencies: - rollup: ^1.20.0||^2.0.0||^3.0.0||^4.0.0 - peerDependenciesMeta: - rollup: - optional: true - checksum: 10c0/6d58fbc6f1024eb4b087bc9bf59a1d655a8056a60c0b4021d3beaeec3f0743503f52467fd89d2cf0e7eccf2831feb40a05ad541a17637ea21ba10b21c2004deb - languageName: node - linkType: hard - -"@rollup/pluginutils@npm:^5.1.3": +"@rollup/pluginutils@npm:^5.0.1, @rollup/pluginutils@npm:^5.0.2, @rollup/pluginutils@npm:^5.1.0, @rollup/pluginutils@npm:^5.1.3": version: 5.2.0 resolution: "@rollup/pluginutils@npm:5.2.0" dependencies: @@ -5397,13 +5382,13 @@ __metadata: linkType: hard "axios@npm:^1.4.0": - version: 1.10.0 - resolution: "axios@npm:1.10.0" + version: 1.11.0 + resolution: "axios@npm:1.11.0" dependencies: follow-redirects: "npm:^1.15.6" - form-data: "npm:^4.0.0" + form-data: "npm:^4.0.4" proxy-from-env: "npm:^1.1.0" - checksum: 10c0/2239cb269cc789eac22f5d1aabd58e1a83f8f364c92c2caa97b6f5cbb4ab2903d2e557d9dc670b5813e9bcdebfb149e783fb8ab3e45098635cd2f559b06bd5d8 + checksum: 10c0/5de273d33d43058610e4d252f0963cc4f10714da0bfe872e8ef2cbc23c2c999acc300fd357b6bce0fc84a2ca9bd45740fa6bb28199ce2c1266c8b1a393f2b36e languageName: node linkType: hard @@ -6616,6 +6601,13 @@ __metadata: languageName: node linkType: hard +"emojibase-regex@npm:^16.0.0": + version: 16.0.0 + resolution: "emojibase-regex@npm:16.0.0" + checksum: 10c0/8ee5ff798e51caa581434b1cb2f9737e50195093c4efa1739df21a50a5496f80517924787d865e8cf7d6a0b4c90dbedc04bdc506dcbcc582e14cdf0bb47af0f0 + languageName: node + linkType: hard + "emojibase@npm:^16.0.0": version: 16.0.0 resolution: "emojibase@npm:16.0.0" @@ -7609,14 +7601,16 @@ __metadata: languageName: node linkType: hard -"form-data@npm:^4.0.0": - version: 4.0.1 - resolution: "form-data@npm:4.0.1" +"form-data@npm:^4.0.4": + version: 4.0.4 + resolution: "form-data@npm:4.0.4" dependencies: asynckit: "npm:^0.4.0" combined-stream: "npm:^1.0.8" + es-set-tostringtag: "npm:^2.1.0" + hasown: "npm:^2.0.2" mime-types: "npm:^2.1.12" - checksum: 10c0/bb102d570be8592c23f4ea72d7df9daa50c7792eb0cf1c5d7e506c1706e7426a4e4ae48a35b109e91c85f1c0ec63774a21ae252b66f4eb981cb8efef7d0463c8 + checksum: 10c0/373525a9a034b9d57073e55eab79e501a714ffac02e7a9b01be1c820780652b16e4101819785e1e18f8d98f0aee866cc654d660a435c378e16a72f2e7cac9695 languageName: node linkType: hard