Merge commit '7d3ef27a8dc8fb281bae959b245d3db63aa82260' into glitch-soc/merge-upstream

Gemfile.lock

@@ -362,7 +362,7 @@ GEM
       rack (>= 2.2, < 4)
       rdf (~> 3.3)
       rexml (~> 3.2)
-    json-ld-preloaded (3.3.1)
+    json-ld-preloaded (3.3.2)
       json-ld (~> 3.3)
       rdf (~> 3.3)
     json-schema (5.2.1)
@@ -627,7 +627,7 @@ GEM
     prism (1.4.0)
     prometheus_exporter (2.2.0)
       webrick
-    propshaft (1.2.0)
+    propshaft (1.2.1)
       actionpack (>= 7.0.0)
       activesupport (>= 7.0.0)
       rack
@@ -759,7 +759,7 @@ GEM
       rspec-expectations (~> 3.13)
       rspec-mocks (~> 3.13)
       rspec-support (~> 3.13)
-    rspec-sidekiq (5.1.0)
+    rspec-sidekiq (5.2.0)
       rspec-core (~> 3.0)
       rspec-expectations (~> 3.0)
       rspec-mocks (~> 3.0)

app/controllers/admin/accounts_controller.rb

@@ -16,11 +16,14 @@ module Admin
     def batch
       authorize :account, :index?
 
-      @form = Form::AccountBatch.new(form_account_batch_params)
+      @form = Form::AccountBatch.new(
-      @form.current_account = current_account
+        form_account_batch_params.merge(
-      @form.action = action_from_button
+          action: action_from_button,
-      @form.select_all_matching = params[:select_all_matching]
+          current_account:,
-      @form.query = filtered_accounts
+          query: filtered_accounts,
+          select_all_matching: params[:select_all_matching]
+        )
+      )
       @form.save
     rescue ActionController::ParameterMissing
       flash[:alert] = I18n.t('admin.accounts.no_account_selected')

app/controllers/admin/tags_controller.rb

@@ -5,6 +5,7 @@ module Admin
     before_action :set_tag, except: [:index]
 
     PER_PAGE = 20
+    PERIOD_DAYS = 6.days
 
     def index
       authorize :tag, :index?
@@ -15,7 +16,7 @@ module Admin
     def show
       authorize @tag, :show?
 
-      @time_period = (6.days.ago.to_date...Time.now.utc.to_date)
+      @time_period = report_range
     end
 
     def update
@@ -24,7 +25,7 @@ module Admin
       if @tag.update(tag_params.merge(reviewed_at: Time.now.utc))
         redirect_to admin_tag_path(@tag.id), notice: I18n.t('admin.tags.updated_msg')
       else
-        @time_period = (6.days.ago.to_date...Time.now.utc.to_date)
+        @time_period = report_range
 
         render :show
       end
@@ -36,6 +37,10 @@ module Admin
       @tag = Tag.find(params[:id])
     end
 
+    def report_range
+      (PERIOD_DAYS.ago.to_date...Time.now.utc.to_date)
+    end
+
     def tag_params
       params
         .expect(tag: [:name, :display_name, :trendable, :usable, :listable])

app/controllers/api/v1/statuses_controller.rb

@@ -2,6 +2,7 @@
 
 class Api::V1::StatusesController < Api::BaseController
   include Authorization
+  include AsyncRefreshesConcern
 
   before_action -> { authorize_if_got_token! :read, :'read:statuses' }, except: [:create, :update, :destroy]
   before_action -> { doorkeeper_authorize! :write, :'write:statuses' }, only:   [:create, :update, :destroy]
@@ -57,9 +58,17 @@ class Api::V1::StatusesController < Api::BaseController
     @context = Context.new(ancestors: loaded_ancestors, descendants: loaded_descendants)
     statuses = [@status] + @context.ancestors + @context.descendants
 
-    render json: @context, serializer: REST::ContextSerializer, relationships: StatusRelationshipsPresenter.new(statuses, current_user&.account_id)
+    refresh_key = "context:#{@status.id}:refresh"
+    async_refresh = AsyncRefresh.new(refresh_key)
 
-    ActivityPub::FetchAllRepliesWorker.perform_async(@status.id) if !current_account.nil? && @status.should_fetch_replies?
+    if async_refresh.running?
+      add_async_refresh_header(async_refresh)
+    elsif !current_account.nil? && @status.should_fetch_replies?
+      add_async_refresh_header(AsyncRefresh.create(refresh_key))
+      ActivityPub::FetchAllRepliesWorker.perform_async(@status.id)
+    end
+
+    render json: @context, serializer: REST::ContextSerializer, relationships: StatusRelationshipsPresenter.new(statuses, current_user&.account_id)
   end
 
   def create

app/controllers/concerns/auth/captcha_concern.rb

@@ -5,6 +5,18 @@ module Auth::CaptchaConcern
 
   include Hcaptcha::Adapters::ViewMethods
 
+  CAPTCHA_DIRECTIVES = %w(
+    connect_src
+    frame_src
+    script_src
+    style_src
+  ).freeze
+
+  CAPTCHA_SOURCES = %w(
+    https://*.hcaptcha.com
+    https://hcaptcha.com
+  ).freeze
+
   included do
     helper_method :render_captcha
   end
@@ -42,20 +54,9 @@ module Auth::CaptchaConcern
   end
 
   def extend_csp_for_captcha!
-    policy = request.content_security_policy&.clone
+    return unless captcha_required? && request.content_security_policy.present?
 
-    return unless captcha_required? && policy.present?
+    request.content_security_policy = captcha_adjusted_policy
-
-    %w(script_src frame_src style_src connect_src).each do |directive|
-      values = policy.send(directive)
-
-      values << 'https://hcaptcha.com' unless values.include?('https://hcaptcha.com') || values.include?('https:')
-      values << 'https://*.hcaptcha.com' unless values.include?('https://*.hcaptcha.com') || values.include?('https:')
-
-      policy.send(directive, *values)
-    end
-
-    request.content_security_policy = policy
   end
 
   def render_captcha
@@ -63,4 +64,24 @@ module Auth::CaptchaConcern
 
     hcaptcha_tags
   end
+
+  private
+
+  def captcha_adjusted_policy
+    request.content_security_policy.clone.tap do |policy|
+      populate_captcha_policy(policy)
+    end
+  end
+
+  def populate_captcha_policy(policy)
+    CAPTCHA_DIRECTIVES.each do |directive|
+      values = policy.send(directive)
+
+      CAPTCHA_SOURCES.each do |source|
+        values << source unless values.include?(source) || values.include?('https:')
+      end
+
+      policy.send(directive, *values)
+    end
+  end
 end

app/javascript/mastodon/actions/statuses_typed.ts

@@ -1,3 +1,5 @@
+import { createAction } from '@reduxjs/toolkit';
+
 import { apiGetContext } from 'mastodon/api/statuses';
 import { createDataLoadingThunk } from 'mastodon/store/typed_functions';
 
@@ -6,13 +8,18 @@ import { importFetchedStatuses } from './importer';
 export const fetchContext = createDataLoadingThunk(
   'status/context',
   ({ statusId }: { statusId: string }) => apiGetContext(statusId),
-  (context, { dispatch }) => {
+  ({ context, refresh }, { dispatch }) => {
     const statuses = context.ancestors.concat(context.descendants);
 
     dispatch(importFetchedStatuses(statuses));
 
     return {
       context,
+      refresh,
     };
   },
 );
+
+export const completeContextRefresh = createAction<{ statusId: string }>(
+  'status/context/complete',
+);

app/javascript/mastodon/api.ts

@@ -20,6 +20,50 @@ export const getLinks = (response: AxiosResponse) => {
   return LinkHeader.parse(value);
 };
 
+export interface AsyncRefreshHeader {
+  id: string;
+  retry: number;
+}
+
+const isAsyncRefreshHeader = (obj: object): obj is AsyncRefreshHeader =>
+  'id' in obj && 'retry' in obj;
+
+export const getAsyncRefreshHeader = (
+  response: AxiosResponse,
+): AsyncRefreshHeader | null => {
+  const value = response.headers['mastodon-async-refresh'] as
+    | string
+    | undefined;
+
+  if (!value) {
+    return null;
+  }
+
+  const asyncRefreshHeader: Record<string, unknown> = {};
+
+  value.split(/,\s*/).forEach((pair) => {
+    const [key, val] = pair.split('=', 2);
+
+    let typedValue: string | number;
+
+    if (key && ['id', 'retry'].includes(key) && val) {
+      if (val.startsWith('"')) {
+        typedValue = val.slice(1, -1);
+      } else {
+        typedValue = parseInt(val);
+      }
+
+      asyncRefreshHeader[key] = typedValue;
+    }
+  });
+
+  if (isAsyncRefreshHeader(asyncRefreshHeader)) {
+    return asyncRefreshHeader;
+  }
+
+  return null;
+};
+
 const csrfHeader: RawAxiosRequestHeaders = {};
 
 const setCSRFHeader = () => {
@@ -83,7 +127,7 @@ export default function api(withAuthorization = true) {
   return instance;
 }
 
-type ApiUrl = `v${1 | 2}/${string}`;
+type ApiUrl = `v${1 | '1_alpha' | 2}/${string}`;
 type RequestParamsOrData = Record<string, unknown>;
 
 export async function apiRequest<ApiResponse = unknown>(

app/javascript/mastodon/api/async_refreshes.ts

@@ -0,0 +1,5 @@
+import { apiRequestGet } from 'mastodon/api';
+import type { ApiAsyncRefreshJSON } from 'mastodon/api_types/async_refreshes';
+
+export const apiGetAsyncRefresh = (id: string) =>
+  apiRequestGet<ApiAsyncRefreshJSON>(`v1_alpha/async_refreshes/${id}`);

app/javascript/mastodon/api/statuses.ts

@@ -1,5 +1,14 @@
-import { apiRequestGet } from 'mastodon/api';
+import api, { getAsyncRefreshHeader } from 'mastodon/api';
 import type { ApiContextJSON } from 'mastodon/api_types/statuses';
 
-export const apiGetContext = (statusId: string) =>
+export const apiGetContext = async (statusId: string) => {
-  apiRequestGet<ApiContextJSON>(`v1/statuses/${statusId}/context`);
+  const response = await api().request<ApiContextJSON>({
+    method: 'GET',
+    url: `/api/v1/statuses/${statusId}/context`,
+  });
+
+  return {
+    context: response.data,
+    refresh: getAsyncRefreshHeader(response),
+  };
+};

app/javascript/mastodon/api_types/async_refreshes.ts

@@ -0,0 +1,7 @@
+export interface ApiAsyncRefreshJSON {
+  async_refresh: {
+    id: string;
+    status: 'running' | 'finished';
+    result_count: number;
+  };
+}

app/javascript/mastodon/components/account_bio.tsx

@@ -2,27 +2,44 @@ import { useCallback } from 'react';
 
 import { useLinks } from 'mastodon/hooks/useLinks';
 
+import { EmojiHTML } from '../features/emoji/emoji_html';
+import { isFeatureEnabled } from '../initial_state';
+import { useAppSelector } from '../store';
+
 interface AccountBioProps {
-  note: string;
   className: string;
-  dropdownAccountId?: string;
+  accountId: string;
+  showDropdown?: boolean;
 }
 
 export const AccountBio: React.FC<AccountBioProps> = ({
-  note,
   className,
-  dropdownAccountId,
+  accountId,
+  showDropdown = false,
 }) => {
-  const handleClick = useLinks(!!dropdownAccountId);
+  const handleClick = useLinks(showDropdown);
   const handleNodeChange = useCallback(
     (node: HTMLDivElement | null) => {
-      if (!dropdownAccountId || !node || node.childNodes.length === 0) {
+      if (!showDropdown || !node || node.childNodes.length === 0) {
         return;
       }
-      addDropdownToHashtags(node, dropdownAccountId);
+      addDropdownToHashtags(node, accountId);
     },
-    [dropdownAccountId],
+    [showDropdown, accountId],
   );
+  const note = useAppSelector((state) => {
+    const account = state.accounts.get(accountId);
+    if (!account) {
+      return '';
+    }
+    return isFeatureEnabled('modern_emojis')
+      ? account.note
+      : account.note_emojified;
+  });
+  const extraEmojis = useAppSelector((state) => {
+    const account = state.accounts.get(accountId);
+    return account?.emojis;
+  });
 
   if (note.length === 0) {
     return null;
@@ -31,10 +48,11 @@ export const AccountBio: React.FC<AccountBioProps> = ({
   return (
     <div
       className={`${className} translate`}
-      dangerouslySetInnerHTML={{ __html: note }}
       onClickCapture={handleClick}
       ref={handleNodeChange}
-    />
+    >
+      <EmojiHTML htmlString={note} extraEmojis={extraEmojis} />
+    </div>
   );
 };
 

app/javascript/mastodon/components/hover_card_account.tsx

@@ -102,7 +102,7 @@ export const HoverCardAccount = forwardRef<
             <>
               <div className='hover-card__text-row'>
                 <AccountBio
-                  note={account.note_emojified}
+                  accountId={account.id}
                   className='hover-card__bio'
                 />
                 <AccountFields fields={account.fields} limit={2} />

app/javascript/mastodon/components/status_content.jsx

@@ -13,7 +13,8 @@ import ChevronRightIcon from '@/material-icons/400-24px/chevron_right.svg?react'
 import { Icon }  from 'mastodon/components/icon';
 import { Poll } from 'mastodon/components/poll';
 import { identityContextPropShape, withIdentity } from 'mastodon/identity_context';
-import { autoPlayGif, languages as preloadedLanguages } from 'mastodon/initial_state';
+import { autoPlayGif, isFeatureEnabled, languages as preloadedLanguages } from 'mastodon/initial_state';
+import { EmojiHTML } from '../features/emoji/emoji_html';
 
 const MAX_HEIGHT = 706; // 22px * 32 (+ 2px padding at the top)
 
@@ -23,6 +24,9 @@ const MAX_HEIGHT = 706; // 22px * 32 (+ 2px padding at the top)
  * @returns {string}
  */
 export function getStatusContent(status) {
+  if (isFeatureEnabled('modern_emojis')) {
+    return status.getIn(['translation', 'content']) || status.get('content');
+  }
   return status.getIn(['translation', 'contentHtml']) || status.get('contentHtml');
 }
 
@@ -228,7 +232,7 @@ class StatusContent extends PureComponent {
     const targetLanguages = this.props.languages?.get(status.get('language') || 'und');
     const renderTranslate = this.props.onTranslate && this.props.identity.signedIn && ['public', 'unlisted'].includes(status.get('visibility')) && status.get('search_index').trim().length > 0 && targetLanguages?.includes(contentLocale);
 
-    const content = { __html: statusContent ?? getStatusContent(status) };
+    const content = statusContent ?? getStatusContent(status);
     const language = status.getIn(['translation', 'language']) || status.get('language');
     const classNames = classnames('status__content', {
       'status__content--with-action': this.props.onClick && this.props.history,
@@ -253,7 +257,12 @@ class StatusContent extends PureComponent {
       return (
         <>
           <div className={classNames} ref={this.setRef} onMouseDown={this.handleMouseDown} onMouseUp={this.handleMouseUp} key='status-content' onMouseEnter={this.handleMouseEnter} onMouseLeave={this.handleMouseLeave}>
-            <div className='status__content__text status__content__text--visible translate' lang={language} dangerouslySetInnerHTML={content} />
+            <EmojiHTML
+              className='status__content__text status__content__text--visible translate'
+              lang={language}
+              htmlString={content}
+              extraEmojis={status.get('emojis')}
+            />
 
             {poll}
             {translateButton}
@@ -265,7 +274,12 @@ class StatusContent extends PureComponent {
     } else {
       return (
         <div className={classNames} ref={this.setRef} onMouseEnter={this.handleMouseEnter} onMouseLeave={this.handleMouseLeave}>
-          <div className='status__content__text status__content__text--visible translate' lang={language} dangerouslySetInnerHTML={content} />
+          <EmojiHTML
+            className='status__content__text status__content__text--visible translate'
+            lang={language}
+            htmlString={content}
+            extraEmojis={status.get('emojis')}
+          />
 
           {poll}
           {translateButton}

app/javascript/mastodon/components/status_list.jsx

@@ -40,6 +40,12 @@ export default class StatusList extends ImmutablePureComponent {
     trackScroll: true,
   };
 
+  componentDidMount() {
+    this.columnHeaderHeight = parseFloat(
+      getComputedStyle(this.node.node).getPropertyValue('--column-header-height')
+    ) || 0;
+  }
+
   getFeaturedStatusCount = () => {
     return this.props.featuredStatusIds ? this.props.featuredStatusIds.size : 0;
   };
@@ -53,35 +59,68 @@ export default class StatusList extends ImmutablePureComponent {
   };
 
   handleMoveUp = (id, featured) => {
-    const elementIndex = this.getCurrentStatusIndex(id, featured) - 1;
+    const index = this.getCurrentStatusIndex(id, featured);
-    this._selectChild(elementIndex, true);
+    this._selectChild(id, index, -1);
   };
   
   handleMoveDown = (id, featured) => {
-    const elementIndex = this.getCurrentStatusIndex(id, featured) + 1;
+    const index = this.getCurrentStatusIndex(id, featured);
-    this._selectChild(elementIndex, false);
+    this._selectChild(id, index, 1);
   };
 
+  _selectChild = (id, index, direction) => {
+    const listContainer = this.node.node;
+    let listItem = listContainer.querySelector(
+      // :nth-child uses 1-based indexing
+      `.item-list > :nth-child(${index + 1 + direction})`
+    );
+    
+    if (!listItem) {
+      return;
+    }
+
+    // If selected container element is empty, we skip it
+    if (listItem.matches(':empty')) {
+      this._selectChild(id, index + direction, direction);
+      return;
+    }
+
+    // Check if the list item is a post
+    let targetElement = listItem.querySelector('.focusable');
+
+    // Otherwise, check if the item contains follow suggestions or
+    // is a 'load more' button.
+    if (
+      !targetElement && (
+        listItem.querySelector('.inline-follow-suggestions') ||
+        listItem.matches('.load-more')
+      )
+    ) {
+      targetElement = listItem;
+    }
+
+    if (targetElement) {
+      const elementRect = targetElement.getBoundingClientRect();
+
+      const isFullyVisible =
+        elementRect.top >= this.columnHeaderHeight &&
+        elementRect.bottom <= window.innerHeight;
+
+      if (!isFullyVisible) {
+        targetElement.scrollIntoView({
+          block: direction === 1 ? 'start' : 'center',
+        });
+      }
+
+      targetElement.focus();
+    }
+  }
+
   handleLoadOlder = debounce(() => {
     const { statusIds, lastId, onLoadMore } = this.props;
     onLoadMore(lastId || (statusIds.size > 0 ? statusIds.last() : undefined));
   }, 300, { leading: true });
 
-  _selectChild (index, align_top) {
-    const container = this.node.node;
-    // TODO: This breaks at the inline-follow-suggestions container
-    const element = container.querySelector(`article:nth-of-type(${index + 1}) .focusable`);
-
-    if (element) {
-      if (align_top && container.scrollTop > element.offsetTop) {
-        element.scrollIntoView(true);
-      } else if (!align_top && container.scrollTop + container.clientHeight < element.offsetTop + element.offsetHeight) {
-        element.scrollIntoView(false);
-      }
-      element.focus();
-    }
-  }
-
   setRef = c => {
     this.node = c;
   };

app/javascript/mastodon/features/account_timeline/components/account_header.tsx

@@ -898,8 +898,7 @@ export const AccountHeader: React.FC<{
                 )}
 
                 <AccountBio
-                  note={account.note_emojified}
+                  accountId={accountId}
-                  dropdownAccountId={accountId}
                   className='account__header__content'
                 />
 

app/javascript/mastodon/features/emoji/constants.ts

@@ -15,6 +15,16 @@ export const SKIN_TONE_CODES = [
   0x1f3ff, // Dark skin tone
 ] as const;
 
+// Emoji rendering modes. A mode is what we are using to render emojis, a style is what the user has selected.
+export const EMOJI_MODE_NATIVE = 'native';
+export const EMOJI_MODE_NATIVE_WITH_FLAGS = 'native-flags';
+export const EMOJI_MODE_TWEMOJI = 'twemoji';
+
+export const EMOJI_TYPE_UNICODE = 'unicode';
+export const EMOJI_TYPE_CUSTOM = 'custom';
+
+export const EMOJI_STATE_MISSING = 'missing';
+
 export const EMOJIS_WITH_DARK_BORDER = [
   '🎱', // 1F3B1
   '🐜', // 1F41C

app/javascript/mastodon/features/emoji/database.ts

@@ -1,17 +1,19 @@
 import { SUPPORTED_LOCALES } from 'emojibase';
-import type { FlatCompactEmoji, Locale } from 'emojibase';
+import type { Locale } from 'emojibase';
-import type { DBSchema } from 'idb';
+import type { DBSchema, IDBPDatabase } from 'idb';
 import { openDB } from 'idb';
 
-import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji';
-
-import type { LocaleOrCustom } from './locale';
 import { toSupportedLocale, toSupportedLocaleOrCustom } from './locale';
+import type {
+  CustomEmojiData,
+  UnicodeEmojiData,
+  LocaleOrCustom,
+} from './types';
 
 interface EmojiDB extends LocaleTables, DBSchema {
   custom: {
     key: string;
-    value: ApiCustomEmojiJSON;
+    value: CustomEmojiData;
     indexes: {
       category: string;
     };
@@ -24,7 +26,7 @@ interface EmojiDB extends LocaleTables, DBSchema {
 
 interface LocaleTable {
   key: string;
-  value: FlatCompactEmoji;
+  value: UnicodeEmojiData;
   indexes: {
     group: number;
     label: string;
@@ -36,63 +38,114 @@ type LocaleTables = Record<Locale, LocaleTable>;
 
 const SCHEMA_VERSION = 1;
 
-const db = await openDB<EmojiDB>('mastodon-emoji', SCHEMA_VERSION, {
+let db: IDBPDatabase<EmojiDB> | null = null;
-  upgrade(database) {
-    const customTable = database.createObjectStore('custom', {
-      keyPath: 'shortcode',
-      autoIncrement: false,
-    });
-    customTable.createIndex('category', 'category');
 
-    database.createObjectStore('etags');
+async function loadDB() {
-
+  if (db) {
-    for (const locale of SUPPORTED_LOCALES) {
+    return db;
-      const localeTable = database.createObjectStore(locale, {
+  }
-        keyPath: 'hexcode',
+  db = await openDB<EmojiDB>('mastodon-emoji', SCHEMA_VERSION, {
+    upgrade(database) {
+      const customTable = database.createObjectStore('custom', {
+        keyPath: 'shortcode',
         autoIncrement: false,
       });
-      localeTable.createIndex('group', 'group');
+      customTable.createIndex('category', 'category');
-      localeTable.createIndex('label', 'label');
-      localeTable.createIndex('order', 'order');
-      localeTable.createIndex('tags', 'tags', { multiEntry: true });
-    }
-  },
-});
 
-export async function putEmojiData(emojis: FlatCompactEmoji[], locale: Locale) {
+      database.createObjectStore('etags');
+
+      for (const locale of SUPPORTED_LOCALES) {
+        const localeTable = database.createObjectStore(locale, {
+          keyPath: 'hexcode',
+          autoIncrement: false,
+        });
+        localeTable.createIndex('group', 'group');
+        localeTable.createIndex('label', 'label');
+        localeTable.createIndex('order', 'order');
+        localeTable.createIndex('tags', 'tags', { multiEntry: true });
+      }
+    },
+  });
+  return db;
+}
+
+export async function putEmojiData(emojis: UnicodeEmojiData[], locale: Locale) {
+  const db = await loadDB();
   const trx = db.transaction(locale, 'readwrite');
   await Promise.all(emojis.map((emoji) => trx.store.put(emoji)));
   await trx.done;
 }
 
-export async function putCustomEmojiData(emojis: ApiCustomEmojiJSON[]) {
+export async function putCustomEmojiData(emojis: CustomEmojiData[]) {
+  const db = await loadDB();
   const trx = db.transaction('custom', 'readwrite');
   await Promise.all(emojis.map((emoji) => trx.store.put(emoji)));
   await trx.done;
 }
 
-export function putLatestEtag(etag: string, localeString: string) {
+export async function putLatestEtag(etag: string, localeString: string) {
   const locale = toSupportedLocaleOrCustom(localeString);
+  const db = await loadDB();
   return db.put('etags', etag, locale);
 }
 
-export function searchEmojiByHexcode(hexcode: string, localeString: string) {
+export async function searchEmojiByHexcode(
+  hexcode: string,
+  localeString: string,
+) {
   const locale = toSupportedLocale(localeString);
+  const db = await loadDB();
   return db.get(locale, hexcode);
 }
 
-export function searchEmojiByTag(tag: string, localeString: string) {
+export async function searchEmojisByHexcodes(
+  hexcodes: string[],
+  localeString: string,
+) {
+  const locale = toSupportedLocale(localeString);
+  const db = await loadDB();
+  return db.getAll(
+    locale,
+    IDBKeyRange.bound(hexcodes[0], hexcodes[hexcodes.length - 1]),
+  );
+}
+
+export async function searchEmojiByTag(tag: string, localeString: string) {
   const locale = toSupportedLocale(localeString);
   const range = IDBKeyRange.only(tag.toLowerCase());
+  const db = await loadDB();
   return db.getAllFromIndex(locale, 'tags', range);
 }
 
-export function searchCustomEmojiByShortcode(shortcode: string) {
+export async function searchCustomEmojiByShortcode(shortcode: string) {
+  const db = await loadDB();
   return db.get('custom', shortcode);
 }
 
+export async function searchCustomEmojisByShortcodes(shortcodes: string[]) {
+  const db = await loadDB();
+  return db.getAll(
+    'custom',
+    IDBKeyRange.bound(shortcodes[0], shortcodes[shortcodes.length - 1]),
+  );
+}
+
+export async function findMissingLocales(localeStrings: string[]) {
+  const locales = new Set(localeStrings.map(toSupportedLocale));
+  const missingLocales: Locale[] = [];
+  const db = await loadDB();
+  for (const locale of locales) {
+    const rowCount = await db.count(locale);
+    if (!rowCount) {
+      missingLocales.push(locale);
+    }
+  }
+  return missingLocales;
+}
+
 export async function loadLatestEtag(localeString: string) {
   const locale = toSupportedLocaleOrCustom(localeString);
+  const db = await loadDB();
   const rowCount = await db.count(locale);
   if (!rowCount) {
     return null; // No data for this locale, return null even if there is an etag.

app/javascript/mastodon/features/emoji/emoji_html.tsx

@@ -0,0 +1,81 @@
+import type { HTMLAttributes } from 'react';
+import { useEffect, useMemo, useState } from 'react';
+
+import type { List as ImmutableList } from 'immutable';
+import { isList } from 'immutable';
+
+import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji';
+import { isFeatureEnabled } from '@/mastodon/initial_state';
+import type { CustomEmoji } from '@/mastodon/models/custom_emoji';
+
+import { useEmojiAppState } from './hooks';
+import { emojifyElement } from './render';
+import type { ExtraCustomEmojiMap } from './types';
+
+type EmojiHTMLProps = Omit<
+  HTMLAttributes<HTMLDivElement>,
+  'dangerouslySetInnerHTML'
+> & {
+  htmlString: string;
+  extraEmojis?: ExtraCustomEmojiMap | ImmutableList<CustomEmoji>;
+};
+
+export const EmojiHTML: React.FC<EmojiHTMLProps> = ({
+  htmlString,
+  extraEmojis,
+  ...props
+}) => {
+  if (isFeatureEnabled('modern_emojis')) {
+    return (
+      <ModernEmojiHTML
+        htmlString={htmlString}
+        extraEmojis={extraEmojis}
+        {...props}
+      />
+    );
+  }
+  return <div dangerouslySetInnerHTML={{ __html: htmlString }} {...props} />;
+};
+
+const ModernEmojiHTML: React.FC<EmojiHTMLProps> = ({
+  extraEmojis: rawEmojis,
+  htmlString: text,
+  ...props
+}) => {
+  const appState = useEmojiAppState();
+  const [innerHTML, setInnerHTML] = useState('');
+
+  const extraEmojis: ExtraCustomEmojiMap = useMemo(() => {
+    if (!rawEmojis) {
+      return {};
+    }
+    if (isList(rawEmojis)) {
+      return (
+        rawEmojis.toJS() as ApiCustomEmojiJSON[]
+      ).reduce<ExtraCustomEmojiMap>(
+        (acc, emoji) => ({ ...acc, [emoji.shortcode]: emoji }),
+        {},
+      );
+    }
+    return rawEmojis;
+  }, [rawEmojis]);
+
+  useEffect(() => {
+    if (!text) {
+      return;
+    }
+    const cb = async () => {
+      const div = document.createElement('div');
+      div.innerHTML = text;
+      const ele = await emojifyElement(div, appState, extraEmojis);
+      setInnerHTML(ele.innerHTML);
+    };
+    void cb();
+  }, [text, appState, extraEmojis]);
+
+  if (!innerHTML) {
+    return null;
+  }
+
+  return <div {...props} dangerouslySetInnerHTML={{ __html: innerHTML }} />;
+};

app/javascript/mastodon/features/emoji/emoji_text.tsx

@@ -0,0 +1,45 @@
+import { useEffect, useState } from 'react';
+
+import { useEmojiAppState } from './hooks';
+import { emojifyText } from './render';
+
+interface EmojiTextProps {
+  text: string;
+}
+
+export const EmojiText: React.FC<EmojiTextProps> = ({ text }) => {
+  const appState = useEmojiAppState();
+  const [rendered, setRendered] = useState<(string | HTMLImageElement)[]>([]);
+
+  useEffect(() => {
+    const cb = async () => {
+      const rendered = await emojifyText(text, appState);
+      setRendered(rendered ?? []);
+    };
+    void cb();
+  }, [text, appState]);
+
+  if (rendered.length === 0) {
+    return null;
+  }
+
+  return (
+    <>
+      {rendered.map((fragment, index) => {
+        if (typeof fragment === 'string') {
+          return <span key={index}>{fragment}</span>;
+        }
+        return (
+          <img
+            key={index}
+            draggable='false'
+            src={fragment.src}
+            alt={fragment.alt}
+            title={fragment.title}
+            className={fragment.className}
+          />
+        );
+      })}
+    </>
+  );
+};

app/javascript/mastodon/features/emoji/hooks.ts

@@ -0,0 +1,16 @@
+import { useAppSelector } from '@/mastodon/store';
+
+import { toSupportedLocale } from './locale';
+import { determineEmojiMode } from './mode';
+import type { EmojiAppState } from './types';
+
+export function useEmojiAppState(): EmojiAppState {
+  const locale = useAppSelector((state) =>
+    toSupportedLocale(state.meta.get('locale') as string),
+  );
+  const mode = useAppSelector((state) =>
+    determineEmojiMode(state.meta.get('emoji_style') as string),
+  );
+
+  return { currentLocale: locale, locales: [locale], mode };
+}

app/javascript/mastodon/features/emoji/index.ts

@@ -2,27 +2,44 @@ import initialState from '@/mastodon/initial_state';
 
 import { toSupportedLocale } from './locale';
 
-const serverLocale = toSupportedLocale(initialState?.meta.locale ?? 'en');
+const userLocale = toSupportedLocale(initialState?.meta.locale ?? 'en');
 
-const worker =
+let worker: Worker | null = null;
-  'Worker' in window
-    ? new Worker(new URL('./worker', import.meta.url), {
-        type: 'module',
-      })
-    : null;
 
 export async function initializeEmoji() {
+  if (!worker && 'Worker' in window) {
+    try {
+      worker = new Worker(new URL('./worker', import.meta.url), {
+        type: 'module',
+        credentials: 'omit',
+      });
+    } catch (err) {
+      console.warn('Error creating web worker:', err);
+    }
+  }
+
   if (worker) {
-    worker.addEventListener('message', (event: MessageEvent<string>) => {
+    // Assign worker to const to make TS happy inside the event listener.
+    const thisWorker = worker;
+    thisWorker.addEventListener('message', (event: MessageEvent<string>) => {
       const { data: message } = event;
       if (message === 'ready') {
-        worker.postMessage(serverLocale);
+        thisWorker.postMessage('custom');
-        worker.postMessage('custom');
+        void loadEmojiLocale(userLocale);
+        // Load English locale as well, because people are still used to
+        // using it from before we supported other locales.
+        if (userLocale !== 'en') {
+          void loadEmojiLocale('en');
+        }
       }
     });
   } else {
-    const { importCustomEmojiData, importEmojiData } = await import('./loader');
+    const { importCustomEmojiData } = await import('./loader');
-    await Promise.all([importCustomEmojiData(), importEmojiData(serverLocale)]);
+    await importCustomEmojiData();
+    await loadEmojiLocale(userLocale);
+    if (userLocale !== 'en') {
+      await loadEmojiLocale('en');
+    }
   }
 }
 

app/javascript/mastodon/features/emoji/loader.ts

@@ -11,7 +11,7 @@ import {
   putLatestEtag,
 } from './database';
 import { toSupportedLocale, toSupportedLocaleOrCustom } from './locale';
-import type { LocaleOrCustom } from './locale';
+import type { LocaleOrCustom } from './types';
 
 export async function importEmojiData(localeString: string) {
   const locale = toSupportedLocale(localeString);

app/javascript/mastodon/features/emoji/locale.ts

@@ -1,7 +1,7 @@
 import type { Locale } from 'emojibase';
 import { SUPPORTED_LOCALES } from 'emojibase';
 
-export type LocaleOrCustom = Locale | 'custom';
+import type { LocaleOrCustom } from './types';
 
 export function toSupportedLocale(localeBase: string): Locale {
   const locale = localeBase.toLowerCase();

app/javascript/mastodon/features/emoji/mode.ts

@@ -0,0 +1,119 @@
+// Credit to Nolan Lawson for the original implementation.
+// See: https://github.com/nolanlawson/emoji-picker-element/blob/master/src/picker/utils/testColorEmojiSupported.js
+
+import { isDevelopment } from '@/mastodon/utils/environment';
+
+import {
+  EMOJI_MODE_NATIVE,
+  EMOJI_MODE_NATIVE_WITH_FLAGS,
+  EMOJI_MODE_TWEMOJI,
+} from './constants';
+import type { EmojiMode } from './types';
+
+type Feature = Uint8ClampedArray;
+
+// See: https://github.com/nolanlawson/emoji-picker-element/blob/master/src/picker/constants.js
+const FONT_FAMILY =
+  '"Twemoji Mozilla","Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol",' +
+  '"Noto Color Emoji","EmojiOne Color","Android Emoji",sans-serif';
+
+function getTextFeature(text: string, color: string) {
+  const canvas = document.createElement('canvas');
+  canvas.width = canvas.height = 1;
+
+  const ctx = canvas.getContext('2d', {
+    // Improves the performance of `getImageData()`
+    // https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/getContextAttributes#willreadfrequently
+    willReadFrequently: true,
+  });
+  if (!ctx) {
+    throw new Error('Canvas context not available');
+  }
+  ctx.textBaseline = 'top';
+  ctx.font = `100px ${FONT_FAMILY}`;
+  ctx.fillStyle = color;
+  ctx.scale(0.01, 0.01);
+  ctx.fillText(text, 0, 0);
+
+  return ctx.getImageData(0, 0, 1, 1).data satisfies Feature;
+}
+
+function compareFeatures(feature1: Feature, feature2: Feature) {
+  const feature1Str = [...feature1].join(',');
+  const feature2Str = [...feature2].join(',');
+  // This is RGBA, so for 0,0,0, we are checking that the first RGB is not all zeroes.
+  // Most of the time when unsupported this is 0,0,0,0, but on Chrome on Mac it is
+  // 0,0,0,61 - there is a transparency here.
+  return feature1Str === feature2Str && !feature1Str.startsWith('0,0,0,');
+}
+
+function testEmojiSupport(text: string) {
+  // Render white and black and then compare them to each other and ensure they're the same
+  // color, and neither one is black. This shows that the emoji was rendered in color.
+  const feature1 = getTextFeature(text, '#000');
+  const feature2 = getTextFeature(text, '#fff');
+  return compareFeatures(feature1, feature2);
+}
+
+const EMOJI_VERSION_TEST_EMOJI = '🫨'; // shaking head, from v15
+const EMOJI_FLAG_TEST_EMOJI = '🇨🇭';
+
+export function determineEmojiMode(style: string): EmojiMode {
+  if (style === EMOJI_MODE_NATIVE) {
+    // If flags are not supported, we replace them with Twemoji.
+    if (shouldReplaceFlags()) {
+      return EMOJI_MODE_NATIVE_WITH_FLAGS;
+    }
+    return EMOJI_MODE_NATIVE;
+  }
+  if (style === EMOJI_MODE_TWEMOJI) {
+    return EMOJI_MODE_TWEMOJI;
+  }
+
+  // Auto style so determine based on browser capabilities.
+  if (shouldUseTwemoji()) {
+    return EMOJI_MODE_TWEMOJI;
+  } else if (shouldReplaceFlags()) {
+    return EMOJI_MODE_NATIVE_WITH_FLAGS;
+  }
+  return EMOJI_MODE_NATIVE;
+}
+
+export function shouldUseTwemoji(): boolean {
+  if (typeof window === 'undefined') {
+    return false;
+  }
+  try {
+    // Test a known color emoji to see if 15.1 is supported.
+    return !testEmojiSupport(EMOJI_VERSION_TEST_EMOJI);
+  } catch (err: unknown) {
+    // If an error occurs, fall back to Twemoji to be safe.
+    if (isDevelopment()) {
+      console.warn(
+        'Emoji rendering test failed, defaulting to Twemoji. Error:',
+        err,
+      );
+    }
+    return true;
+  }
+}
+
+// Based on https://github.com/talkjs/country-flag-emoji-polyfill/blob/master/src/index.ts#L19
+export function shouldReplaceFlags(): boolean {
+  if (typeof window === 'undefined') {
+    return false;
+  }
+  try {
+    // Test a known flag emoji to see if it is rendered in color.
+    return !testEmojiSupport(EMOJI_FLAG_TEST_EMOJI);
+  } catch (err: unknown) {
+    // If an error occurs, assume flags should be replaced.
+    if (isDevelopment()) {
+      console.warn(
+        'Flag emoji rendering test failed, defaulting to replacement. Error:',
+        err,
+      );
+    }
+    return true;
+  }
+}

app/javascript/mastodon/features/emoji/normalize.test.ts

@@ -22,9 +22,9 @@ const emojiSVGFiles = await readdir(
 );
 const svgFileNames = emojiSVGFiles
   .filter((file) => file.isFile() && file.name.endsWith('.svg'))
-  .map((file) => basename(file.name, '.svg').toUpperCase());
+  .map((file) => basename(file.name, '.svg'));
 const svgFileNamesWithoutBorder = svgFileNames.filter(
-  (fileName) => !fileName.endsWith('_BORDER'),
+  (fileName) => !fileName.endsWith('_border'),
 );
 
 const unicodeEmojis = flattenEmojiData(unicodeRawEmojis);
@@ -60,13 +60,13 @@ describe('unicodeToTwemojiHex', () => {
 describe('twemojiHasBorder', () => {
   test.concurrent.for(
     svgFileNames
-      .filter((file) => file.endsWith('_BORDER'))
+      .filter((file) => file.endsWith('_border'))
       .map((file) => {
-        const hexCode = file.replace('_BORDER', '');
+        const hexCode = file.replace('_border', '');
         return [
           hexCode,
-          CODES_WITH_LIGHT_BORDER.includes(hexCode),
+          CODES_WITH_LIGHT_BORDER.includes(hexCode.toUpperCase()),
-          CODES_WITH_DARK_BORDER.includes(hexCode),
+          CODES_WITH_DARK_BORDER.includes(hexCode.toUpperCase()),
         ] as const;
       }),
   )('twemojiHasBorder for %s', ([hexCode, isLight, isDark], { expect }) => {

app/javascript/mastodon/features/emoji/normalize.ts

@@ -7,6 +7,7 @@ import {
   EMOJIS_WITH_DARK_BORDER,
   EMOJIS_WITH_LIGHT_BORDER,
 } from './constants';
+import type { TwemojiBorderInfo } from './types';
 
 // Misc codes that have special handling
 const SKIER_CODE = 0x26f7;
@@ -51,13 +52,7 @@ export function unicodeToTwemojiHex(unicodeHex: string): string {
     normalizedCodes.push(code);
   }
 
-  return hexNumbersToString(normalizedCodes, 0);
+  return hexNumbersToString(normalizedCodes, 0).toLowerCase();
-}
-
-interface TwemojiBorderInfo {
-  hexCode: string;
-  hasLightBorder: boolean;
-  hasDarkBorder: boolean;
 }
 
 export const CODES_WITH_DARK_BORDER =
@@ -77,7 +72,7 @@ export function twemojiHasBorder(twemojiHex: string): TwemojiBorderInfo {
     hasDarkBorder = true;
   }
   return {
-    hexCode: normalizedHex,
+    hexCode: twemojiHex,
     hasLightBorder,
     hasDarkBorder,
   };

app/javascript/mastodon/features/emoji/render.test.ts

@@ -0,0 +1,163 @@
+import {
+  EMOJI_MODE_NATIVE,
+  EMOJI_MODE_NATIVE_WITH_FLAGS,
+  EMOJI_MODE_TWEMOJI,
+} from './constants';
+import { emojifyElement, tokenizeText } from './render';
+import type { CustomEmojiData, UnicodeEmojiData } from './types';
+
+vitest.mock('./database', () => ({
+  searchCustomEmojisByShortcodes: vitest.fn(
+    () =>
+      [
+        {
+          shortcode: 'custom',
+          static_url: 'emoji/static',
+          url: 'emoji/custom',
+          category: 'test',
+          visible_in_picker: true,
+        },
+      ] satisfies CustomEmojiData[],
+  ),
+  searchEmojisByHexcodes: vitest.fn(
+    () =>
+      [
+        {
+          hexcode: '1F60A',
+          group: 0,
+          label: 'smiling face with smiling eyes',
+          order: 0,
+          tags: ['smile', 'happy'],
+          unicode: '😊',
+        },
+        {
+          hexcode: '1F1EA-1F1FA',
+          group: 0,
+          label: 'flag-eu',
+          order: 0,
+          tags: ['flag', 'european union'],
+          unicode: '🇪🇺',
+        },
+      ] satisfies UnicodeEmojiData[],
+  ),
+  findMissingLocales: vitest.fn(() => []),
+}));
+
+describe('emojifyElement', () => {
+  const testElement = document.createElement('div');
+  testElement.innerHTML = '<p>Hello 😊🇪🇺!</p><p>:custom:</p>';
+
+  const expectedSmileImage =
+    '<img draggable="false" class="emojione" alt="😊" title="smiling face with smiling eyes" src="/emoji/1f60a.svg">';
+  const expectedFlagImage =
+    '<img draggable="false" class="emojione" alt="🇪🇺" title="flag-eu" src="/emoji/1f1ea-1f1fa.svg">';
+  const expectedCustomEmojiImage =
+    '<img draggable="false" class="emojione custom-emoji" alt=":custom:" title=":custom:" src="emoji/static" data-original="emoji/custom" data-static="emoji/static">';
+
+  function cloneTestElement() {
+    return testElement.cloneNode(true) as HTMLElement;
+  }
+
+  test('emojifies custom emoji in native mode', async () => {
+    const emojifiedElement = await emojifyElement(cloneTestElement(), {
+      locales: ['en'],
+      mode: EMOJI_MODE_NATIVE,
+      currentLocale: 'en',
+    });
+    expect(emojifiedElement.innerHTML).toBe(
+      `<p>Hello 😊🇪🇺!</p><p>${expectedCustomEmojiImage}</p>`,
+    );
+  });
+
+  test('emojifies flag emoji in native-with-flags mode', async () => {
+    const emojifiedElement = await emojifyElement(cloneTestElement(), {
+      locales: ['en'],
+      mode: EMOJI_MODE_NATIVE_WITH_FLAGS,
+      currentLocale: 'en',
+    });
+    expect(emojifiedElement.innerHTML).toBe(
+      `<p>Hello 😊${expectedFlagImage}!</p><p>${expectedCustomEmojiImage}</p>`,
+    );
+  });
+
+  test('emojifies everything in twemoji mode', async () => {
+    const emojifiedElement = await emojifyElement(cloneTestElement(), {
+      locales: ['en'],
+      mode: EMOJI_MODE_TWEMOJI,
+      currentLocale: 'en',
+    });
+    expect(emojifiedElement.innerHTML).toBe(
+      `<p>Hello ${expectedSmileImage}${expectedFlagImage}!</p><p>${expectedCustomEmojiImage}</p>`,
+    );
+  });
+});
+
+describe('tokenizeText', () => {
+  test('returns empty array for string with only whitespace', () => {
+    expect(tokenizeText('   \n')).toEqual([]);
+  });
+
+  test('returns an array of text to be a single token', () => {
+    expect(tokenizeText('Hello')).toEqual(['Hello']);
+  });
+
+  test('returns tokens for text with emoji', () => {
+    expect(tokenizeText('Hello 😊 🇿🇼!!')).toEqual([
+      'Hello ',
+      {
+        type: 'unicode',
+        code: '😊',
+      },
+      ' ',
+      {
+        type: 'unicode',
+        code: '🇿🇼',
+      },
+      '!!',
+    ]);
+  });
+
+  test('returns tokens for text with custom emoji', () => {
+    expect(tokenizeText('Hello :smile:!!')).toEqual([
+      'Hello ',
+      {
+        type: 'custom',
+        code: 'smile',
+      },
+      '!!',
+    ]);
+  });
+
+  test('handles custom emoji with underscores and numbers', () => {
+    expect(tokenizeText('Hello :smile_123:!!')).toEqual([
+      'Hello ',
+      {
+        type: 'custom',
+        code: 'smile_123',
+      },
+      '!!',
+    ]);
+  });
+
+  test('returns tokens for text with mixed emoji', () => {
+    expect(tokenizeText('Hello 😊 :smile:!!')).toEqual([
+      'Hello ',
+      {
+        type: 'unicode',
+        code: '😊',
+      },
+      ' ',
+      {
+        type: 'custom',
+        code: 'smile',
+      },
+      '!!',
+    ]);
+  });
+
+  test('does not capture custom emoji with invalid characters', () => {
+    expect(tokenizeText('Hello :smile-123:!!')).toEqual([
+      'Hello :smile-123:!!',
+    ]);
+  });
+});

app/javascript/mastodon/features/emoji/render.ts

@@ -0,0 +1,331 @@
+import type { Locale } from 'emojibase';
+import EMOJI_REGEX from 'emojibase-regex/emoji-loose';
+
+import { autoPlayGif } from '@/mastodon/initial_state';
+import { assetHost } from '@/mastodon/utils/config';
+
+import {
+  EMOJI_MODE_NATIVE,
+  EMOJI_MODE_NATIVE_WITH_FLAGS,
+  EMOJI_TYPE_UNICODE,
+  EMOJI_TYPE_CUSTOM,
+  EMOJI_STATE_MISSING,
+} from './constants';
+import {
+  findMissingLocales,
+  searchCustomEmojisByShortcodes,
+  searchEmojisByHexcodes,
+} from './database';
+import { loadEmojiLocale } from './index';
+import {
+  emojiToUnicodeHex,
+  twemojiHasBorder,
+  unicodeToTwemojiHex,
+} from './normalize';
+import type {
+  CustomEmojiToken,
+  EmojiAppState,
+  EmojiLoadedState,
+  EmojiMode,
+  EmojiState,
+  EmojiStateMap,
+  EmojiToken,
+  ExtraCustomEmojiMap,
+  LocaleOrCustom,
+  UnicodeEmojiToken,
+} from './types';
+import { stringHasUnicodeFlags } from './utils';
+
+const localeCacheMap = new Map<LocaleOrCustom, EmojiStateMap>([
+  [EMOJI_TYPE_CUSTOM, new Map()],
+]);
+
+// Emojifies an element. This modifies the element in place, replacing text nodes with emojified versions.
+export async function emojifyElement<Element extends HTMLElement>(
+  element: Element,
+  appState: EmojiAppState,
+  extraEmojis: ExtraCustomEmojiMap = {},
+): Promise<Element> {
+  const queue: (HTMLElement | Text)[] = [element];
+  while (queue.length > 0) {
+    const current = queue.shift();
+    if (
+      !current ||
+      current instanceof HTMLScriptElement ||
+      current instanceof HTMLStyleElement
+    ) {
+      continue;
+    }
+
+    if (
+      current.textContent &&
+      (current instanceof Text || !current.hasChildNodes())
+    ) {
+      const renderedContent = await emojifyText(
+        current.textContent,
+        appState,
+        extraEmojis,
+      );
+      if (renderedContent) {
+        if (!(current instanceof Text)) {
+          current.textContent = null; // Clear the text content if it's not a Text node.
+        }
+        current.replaceWith(renderedToHTMLFragment(renderedContent));
+      }
+      continue;
+    }
+
+    for (const child of current.childNodes) {
+      if (child instanceof HTMLElement || child instanceof Text) {
+        queue.push(child);
+      }
+    }
+  }
+  return element;
+}
+
+export async function emojifyText(
+  text: string,
+  appState: EmojiAppState,
+  extraEmojis: ExtraCustomEmojiMap = {},
+) {
+  // Exit if no text to convert.
+  if (!text.trim()) {
+    return null;
+  }
+
+  const tokens = tokenizeText(text);
+
+  // If only one token and it's a string, exit early.
+  if (tokens.length === 1 && typeof tokens[0] === 'string') {
+    return null;
+  }
+
+  // Get all emoji from the state map, loading any missing ones.
+  await ensureLocalesAreLoaded(appState.locales);
+  await loadMissingEmojiIntoCache(tokens, appState.locales);
+
+  const renderedFragments: (string | HTMLImageElement)[] = [];
+  for (const token of tokens) {
+    if (typeof token !== 'string' && shouldRenderImage(token, appState.mode)) {
+      let state: EmojiState | undefined;
+      if (token.type === EMOJI_TYPE_CUSTOM) {
+        const extraEmojiData = extraEmojis[token.code];
+        if (extraEmojiData) {
+          state = { type: EMOJI_TYPE_CUSTOM, data: extraEmojiData };
+        } else {
+          state = emojiForLocale(token.code, EMOJI_TYPE_CUSTOM);
+        }
+      } else {
+        state = emojiForLocale(
+          emojiToUnicodeHex(token.code),
+          appState.currentLocale,
+        );
+      }
+
+      // If the state is valid, create an image element. Otherwise, just append as text.
+      if (state && typeof state !== 'string') {
+        const image = stateToImage(state);
+        renderedFragments.push(image);
+        continue;
+      }
+    }
+    const text = typeof token === 'string' ? token : token.code;
+    renderedFragments.push(text);
+  }
+
+  return renderedFragments;
+}
+
+// Private functions
+
+async function ensureLocalesAreLoaded(locales: Locale[]) {
+  const missingLocales = await findMissingLocales(locales);
+  for (const locale of missingLocales) {
+    await loadEmojiLocale(locale);
+  }
+}
+
+const CUSTOM_EMOJI_REGEX = /:([a-z0-9_]+):/i;
+const TOKENIZE_REGEX = new RegExp(
+  `(${EMOJI_REGEX.source}|${CUSTOM_EMOJI_REGEX.source})`,
+  'g',
+);
+
+type TokenizedText = (string | EmojiToken)[];
+
+export function tokenizeText(text: string): TokenizedText {
+  if (!text.trim()) {
+    return [];
+  }
+
+  const tokens = [];
+  let lastIndex = 0;
+  for (const match of text.matchAll(TOKENIZE_REGEX)) {
+    if (match.index > lastIndex) {
+      tokens.push(text.slice(lastIndex, match.index));
+    }
+
+    const code = match[0];
+
+    if (code.startsWith(':') && code.endsWith(':')) {
+      // Custom emoji
+      tokens.push({
+        type: EMOJI_TYPE_CUSTOM,
+        code: code.slice(1, -1), // Remove the colons
+      } satisfies CustomEmojiToken);
+    } else {
+      // Unicode emoji
+      tokens.push({
+        type: EMOJI_TYPE_UNICODE,
+        code: code,
+      } satisfies UnicodeEmojiToken);
+    }
+    lastIndex = match.index + code.length;
+  }
+  if (lastIndex < text.length) {
+    tokens.push(text.slice(lastIndex));
+  }
+  return tokens;
+}
+
+function cacheForLocale(locale: LocaleOrCustom): EmojiStateMap {
+  return localeCacheMap.get(locale) ?? (new Map() as EmojiStateMap);
+}
+
+function emojiForLocale(
+  code: string,
+  locale: LocaleOrCustom,
+): EmojiState | undefined {
+  const cache = cacheForLocale(locale);
+  return cache.get(code);
+}
+
+async function loadMissingEmojiIntoCache(
+  tokens: TokenizedText,
+  locales: Locale[],
+) {
+  const missingUnicodeEmoji = new Set<string>();
+  const missingCustomEmoji = new Set<string>();
+
+  // Iterate over tokens and check if they are in the cache already.
+  for (const token of tokens) {
+    if (typeof token === 'string') {
+      continue; // Skip plain strings.
+    }
+
+    // If this is a custom emoji, check it separately.
+    if (token.type === EMOJI_TYPE_CUSTOM) {
+      const code = token.code;
+      const emojiState = emojiForLocale(code, EMOJI_TYPE_CUSTOM);
+      if (!emojiState) {
+        missingCustomEmoji.add(code);
+      }
+      // Otherwise this is a unicode emoji, so check it against all locales.
+    } else {
+      const code = emojiToUnicodeHex(token.code);
+      if (missingUnicodeEmoji.has(code)) {
+        continue; // Already marked as missing.
+      }
+      for (const locale of locales) {
+        const emojiState = emojiForLocale(code, locale);
+        if (!emojiState) {
+          // If it's missing in one locale, we consider it missing for all.
+          missingUnicodeEmoji.add(code);
+        }
+      }
+    }
+  }
+
+  if (missingUnicodeEmoji.size > 0) {
+    const missingEmojis = Array.from(missingUnicodeEmoji).toSorted();
+    for (const locale of locales) {
+      const emojis = await searchEmojisByHexcodes(missingEmojis, locale);
+      const cache = cacheForLocale(locale);
+      for (const emoji of emojis) {
+        cache.set(emoji.hexcode, { type: EMOJI_TYPE_UNICODE, data: emoji });
+      }
+      const notFoundEmojis = missingEmojis.filter((code) =>
+        emojis.every((emoji) => emoji.hexcode !== code),
+      );
+      for (const code of notFoundEmojis) {
+        cache.set(code, EMOJI_STATE_MISSING); // Mark as missing if not found, as it's probably not a valid emoji.
+      }
+      localeCacheMap.set(locale, cache);
+    }
+  }
+
+  if (missingCustomEmoji.size > 0) {
+    const missingEmojis = Array.from(missingCustomEmoji).toSorted();
+    const emojis = await searchCustomEmojisByShortcodes(missingEmojis);
+    const cache = cacheForLocale(EMOJI_TYPE_CUSTOM);
+    for (const emoji of emojis) {
+      cache.set(emoji.shortcode, { type: EMOJI_TYPE_CUSTOM, data: emoji });
+    }
+    const notFoundEmojis = missingEmojis.filter((code) =>
+      emojis.every((emoji) => emoji.shortcode !== code),
+    );
+    for (const code of notFoundEmojis) {
+      cache.set(code, EMOJI_STATE_MISSING); // Mark as missing if not found, as it's probably not a valid emoji.
+    }
+    localeCacheMap.set(EMOJI_TYPE_CUSTOM, cache);
+  }
+}
+
+function shouldRenderImage(token: EmojiToken, mode: EmojiMode): boolean {
+  if (token.type === EMOJI_TYPE_UNICODE) {
+    // If the mode is native or native with flags for non-flag emoji
+    // we can just append the text node directly.
+    if (
+      mode === EMOJI_MODE_NATIVE ||
+      (mode === EMOJI_MODE_NATIVE_WITH_FLAGS &&
+        !stringHasUnicodeFlags(token.code))
+    ) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+function stateToImage(state: EmojiLoadedState) {
+  const image = document.createElement('img');
+  image.draggable = false;
+  image.classList.add('emojione');
+
+  if (state.type === EMOJI_TYPE_UNICODE) {
+    const emojiInfo = twemojiHasBorder(unicodeToTwemojiHex(state.data.hexcode));
+    if (emojiInfo.hasLightBorder) {
+      image.dataset.lightCode = `${emojiInfo.hexCode}_BORDER`;
+    } else if (emojiInfo.hasDarkBorder) {
+      image.dataset.darkCode = `${emojiInfo.hexCode}_BORDER`;
+    }
+
+    image.alt = state.data.unicode;
+    image.title = state.data.label;
+    image.src = `${assetHost}/emoji/${emojiInfo.hexCode}.svg`;
+  } else {
+    // Custom emoji
+    const shortCode = `:${state.data.shortcode}:`;
+    image.classList.add('custom-emoji');
+    image.alt = shortCode;
+    image.title = shortCode;
+    image.src = autoPlayGif ? state.data.url : state.data.static_url;
+    image.dataset.original = state.data.url;
+    image.dataset.static = state.data.static_url;
+  }
+
+  return image;
+}
+
+function renderedToHTMLFragment(renderedArray: (string | HTMLImageElement)[]) {
+  const fragment = document.createDocumentFragment();
+  for (const fragmentItem of renderedArray) {
+    if (typeof fragmentItem === 'string') {
+      fragment.appendChild(document.createTextNode(fragmentItem));
+    } else if (fragmentItem instanceof HTMLImageElement) {
+      fragment.appendChild(fragmentItem);
+    }
+  }
+  return fragment;
+}

app/javascript/mastodon/features/emoji/types.ts

@@ -0,0 +1,64 @@
+import type { FlatCompactEmoji, Locale } from 'emojibase';
+
+import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji';
+
+import type {
+  EMOJI_MODE_NATIVE,
+  EMOJI_MODE_NATIVE_WITH_FLAGS,
+  EMOJI_MODE_TWEMOJI,
+  EMOJI_STATE_MISSING,
+  EMOJI_TYPE_CUSTOM,
+  EMOJI_TYPE_UNICODE,
+} from './constants';
+
+export type EmojiMode =
+  | typeof EMOJI_MODE_NATIVE
+  | typeof EMOJI_MODE_NATIVE_WITH_FLAGS
+  | typeof EMOJI_MODE_TWEMOJI;
+
+export type LocaleOrCustom = Locale | typeof EMOJI_TYPE_CUSTOM;
+
+export interface EmojiAppState {
+  locales: Locale[];
+  currentLocale: Locale;
+  mode: EmojiMode;
+}
+
+export interface UnicodeEmojiToken {
+  type: typeof EMOJI_TYPE_UNICODE;
+  code: string;
+}
+export interface CustomEmojiToken {
+  type: typeof EMOJI_TYPE_CUSTOM;
+  code: string;
+}
+export type EmojiToken = UnicodeEmojiToken | CustomEmojiToken;
+
+export type CustomEmojiData = ApiCustomEmojiJSON;
+export type UnicodeEmojiData = FlatCompactEmoji;
+export type AnyEmojiData = CustomEmojiData | UnicodeEmojiData;
+
+export type EmojiStateMissing = typeof EMOJI_STATE_MISSING;
+export interface EmojiStateUnicode {
+  type: typeof EMOJI_TYPE_UNICODE;
+  data: UnicodeEmojiData;
+}
+export interface EmojiStateCustom {
+  type: typeof EMOJI_TYPE_CUSTOM;
+  data: CustomEmojiData;
+}
+export type EmojiState =
+  | EmojiStateMissing
+  | EmojiStateUnicode
+  | EmojiStateCustom;
+export type EmojiLoadedState = EmojiStateUnicode | EmojiStateCustom;
+
+export type EmojiStateMap = Map<string, EmojiState>;
+
+export type ExtraCustomEmojiMap = Record<string, ApiCustomEmojiJSON>;
+
+export interface TwemojiBorderInfo {
+  hexCode: string;
+  hasLightBorder: boolean;
+  hasDarkBorder: boolean;
+}

app/javascript/mastodon/features/emoji/utils.test.ts

@@ -0,0 +1,47 @@
+import { stringHasUnicodeEmoji, stringHasUnicodeFlags } from './utils';
+
+describe('stringHasEmoji', () => {
+  test.concurrent.for([
+    ['only text', false],
+    ['text with emoji 😀', true],
+    ['multiple emojis 😀😃😄', true],
+    ['emoji with skin tone 👍🏽', true],
+    ['emoji with ZWJ 👩‍❤️‍👨', true],
+    ['emoji with variation selector ✊️', true],
+    ['emoji with keycap 1️⃣', true],
+    ['emoji with flags 🇺🇸', true],
+    ['emoji with regional indicators 🇦🇺', true],
+    ['emoji with gender 👩‍⚕️', true],
+    ['emoji with family 👨‍👩‍👧‍👦', true],
+    ['emoji with zero width joiner 👩‍🔬', true],
+    ['emoji with non-BMP codepoint 🧑‍🚀', true],
+    ['emoji with combining marks 👨‍👩‍👧‍👦', true],
+    ['emoji with enclosing keycap #️⃣', true],
+    ['emoji with no visible glyph \u200D', false],
+  ] as const)(
+    'stringHasEmoji has emojis in "%s": %o',
+    ([text, expected], { expect }) => {
+      expect(stringHasUnicodeEmoji(text)).toBe(expected);
+    },
+  );
+});
+
+describe('stringHasFlags', () => {
+  test.concurrent.for([
+    ['EU 🇪🇺', true],
+    ['Germany 🇩🇪', true],
+    ['Canada 🇨🇦', true],
+    ['São Tomé & Príncipe 🇸🇹', true],
+    ['Scotland 🏴󠁧󠁢󠁳󠁣󠁴󠁿', true],
+    ['black flag 🏴', false],
+    ['arrr 🏴‍☠️', false],
+    ['rainbow flag 🏳️‍🌈', false],
+    ['non-flag 🔥', false],
+    ['only text', false],
+  ] as const)(
+    'stringHasFlags has flag in "%s": %o',
+    ([text, expected], { expect }) => {
+      expect(stringHasUnicodeFlags(text)).toBe(expected);
+    },
+  );
+});

app/javascript/mastodon/features/emoji/utils.ts

@@ -0,0 +1,13 @@
+import EMOJI_REGEX from 'emojibase-regex/emoji-loose';
+
+export function stringHasUnicodeEmoji(text: string): boolean {
+  return EMOJI_REGEX.test(text);
+}
+
+// From https://github.com/talkjs/country-flag-emoji-polyfill/blob/master/src/index.ts#L49-L50
+const EMOJIS_FLAGS_REGEX =
+  /[\u{1F1E6}-\u{1F1FF}|\u{E0062}-\u{E0063}|\u{E0065}|\u{E0067}|\u{E006C}|\u{E006E}|\u{E0073}-\u{E0074}|\u{E0077}|\u{E007F}]+/u;
+
+export function stringHasUnicodeFlags(text: string): boolean {
+  return EMOJIS_FLAGS_REGEX.test(text);
+}

app/javascript/mastodon/features/status/components/refresh_controller.tsx

@@ -0,0 +1,111 @@
+import { useEffect, useState, useCallback } from 'react';
+
+import { useIntl, defineMessages, FormattedMessage } from 'react-intl';
+
+import classNames from 'classnames';
+
+import {
+  fetchContext,
+  completeContextRefresh,
+} from 'mastodon/actions/statuses';
+import type { AsyncRefreshHeader } from 'mastodon/api';
+import { apiGetAsyncRefresh } from 'mastodon/api/async_refreshes';
+import { LoadingIndicator } from 'mastodon/components/loading_indicator';
+import { useAppSelector, useAppDispatch } from 'mastodon/store';
+
+const messages = defineMessages({
+  loading: {
+    id: 'status.context.loading',
+    defaultMessage: 'Checking for more replies',
+  },
+});
+
+export const RefreshController: React.FC<{
+  statusId: string;
+  withBorder?: boolean;
+}> = ({ statusId, withBorder }) => {
+  const refresh = useAppSelector(
+    (state) => state.contexts.refreshing[statusId],
+  );
+  const dispatch = useAppDispatch();
+  const intl = useIntl();
+  const [ready, setReady] = useState(false);
+  const [loading, setLoading] = useState(false);
+
+  useEffect(() => {
+    let timeoutId: ReturnType<typeof setTimeout>;
+
+    const scheduleRefresh = (refresh: AsyncRefreshHeader) => {
+      timeoutId = setTimeout(() => {
+        void apiGetAsyncRefresh(refresh.id).then((result) => {
+          if (result.async_refresh.status === 'finished') {
+            dispatch(completeContextRefresh({ statusId }));
+
+            if (result.async_refresh.result_count > 0) {
+              setReady(true);
+            }
+          } else {
+            scheduleRefresh(refresh);
+          }
+
+          return '';
+        });
+      }, refresh.retry * 1000);
+    };
+
+    if (refresh) {
+      scheduleRefresh(refresh);
+    }
+
+    return () => {
+      clearTimeout(timeoutId);
+    };
+  }, [dispatch, setReady, statusId, refresh]);
+
+  const handleClick = useCallback(() => {
+    setLoading(true);
+    setReady(false);
+
+    dispatch(fetchContext({ statusId }))
+      .then(() => {
+        setLoading(false);
+        return '';
+      })
+      .catch(() => {
+        setLoading(false);
+      });
+  }, [dispatch, setReady, statusId]);
+
+  if (ready && !loading) {
+    return (
+      <button
+        className={classNames('load-more load-gap', {
+          'timeline-hint--with-descendants': withBorder,
+        })}
+        onClick={handleClick}
+      >
+        <FormattedMessage
+          id='status.context.load_new_replies'
+          defaultMessage='New replies available'
+        />
+      </button>
+    );
+  }
+
+  if (!refresh && !loading) {
+    return null;
+  }
+
+  return (
+    <div
+      className={classNames('load-more load-gap', {
+        'timeline-hint--with-descendants': withBorder,
+      })}
+      aria-busy
+      aria-live='polite'
+      aria-label={intl.formatMessage(messages.loading)}
+    >
+      <LoadingIndicator />
+    </div>
+  );
+};

app/javascript/mastodon/features/status/index.jsx

@@ -68,7 +68,7 @@ import { attachFullscreenListener, detachFullscreenListener, isFullscreen } from
 
 import ActionBar from './components/action_bar';
 import { DetailedStatus } from './components/detailed_status';
-
+import { RefreshController } from './components/refresh_controller';
 
 const messages = defineMessages({
   revealAll: { id: 'status.show_more_all', defaultMessage: 'Show more for all' },
@@ -548,7 +548,7 @@ class Status extends ImmutablePureComponent {
 
   render () {
     let ancestors, descendants, remoteHint;
-    const { isLoading, status, ancestorsIds, descendantsIds, intl, domain, multiColumn, pictureInPicture } = this.props;
+    const { isLoading, status, ancestorsIds, descendantsIds, refresh, intl, domain, multiColumn, pictureInPicture } = this.props;
     const { fullscreen } = this.state;
 
     if (isLoading) {
@@ -578,11 +578,9 @@ class Status extends ImmutablePureComponent {
 
     if (!isLocal) {
       remoteHint = (
-        <TimelineHint
+        <RefreshController
-          className={classNames(!!descendants && 'timeline-hint--with-descendants')}
+          statusId={status.get('id')}
-          url={status.get('url')}
+          withBorder={!!descendants}
-          message={<FormattedMessage id='hints.threads.replies_may_be_missing' defaultMessage='Replies from other servers may be missing.' />}
-          label={<FormattedMessage id='hints.threads.see_more' defaultMessage='See more replies on {domain}' values={{ domain: <strong>{status.getIn(['account', 'acct']).split('@')[1]}</strong> }} />}
         />
       );
     }

app/javascript/mastodon/initial_state.js

@@ -45,6 +45,7 @@
  * @property {string} sso_redirect
  * @property {string} status_page_url
  * @property {boolean} terms_of_service_enabled
+ * @property {string?} emoji_style
  */
 
 /**
@@ -95,6 +96,7 @@ export const disableHoverCards = getMeta('disable_hover_cards');
 export const disabledAccountId = getMeta('disabled_account_id');
 export const displayMedia = getMeta('display_media');
 export const domain = getMeta('domain');
+export const emojiStyle = getMeta('emoji_style') || 'auto';
 export const expandSpoilers = getMeta('expand_spoilers');
 export const forceSingleColumn = !getMeta('advanced_layout');
 export const limitedFederationMode = getMeta('limited_federation_mode');

app/javascript/mastodon/locales/en.json

@@ -424,8 +424,6 @@
   "hints.profiles.see_more_followers": "See more followers on {domain}",
   "hints.profiles.see_more_follows": "See more follows on {domain}",
   "hints.profiles.see_more_posts": "See more posts on {domain}",
-  "hints.threads.replies_may_be_missing": "Replies from other servers may be missing.",
-  "hints.threads.see_more": "See more replies on {domain}",
   "home.column_settings.show_quotes": "Show quotes",
   "home.column_settings.show_reblogs": "Show boosts",
   "home.column_settings.show_replies": "Show replies",
@@ -847,6 +845,8 @@
   "status.bookmark": "Bookmark",
   "status.cancel_reblog_private": "Unboost",
   "status.cannot_reblog": "This post cannot be boosted",
+  "status.context.load_new_replies": "New replies available",
+  "status.context.loading": "Checking for more replies",
   "status.continued_thread": "Continued thread",
   "status.copy": "Copy link to post",
   "status.delete": "Delete",

app/javascript/mastodon/main.tsx

@@ -4,7 +4,7 @@ import { Globals } from '@react-spring/web';
 
 import { setupBrowserNotifications } from 'mastodon/actions/notifications';
 import Mastodon from 'mastodon/containers/mastodon';
-import { me, reduceMotion } from 'mastodon/initial_state';
+import { isFeatureEnabled, me, reduceMotion } from 'mastodon/initial_state';
 import * as perf from 'mastodon/performance';
 import ready from 'mastodon/ready';
 import { store } from 'mastodon/store';
@@ -29,6 +29,11 @@ function main() {
       });
     }
 
+    if (isFeatureEnabled('modern_emojis')) {
+      const { initializeEmoji } = await import('@/mastodon/features/emoji');
+      await initializeEmoji();
+    }
+
     const root = createRoot(mountNode);
     root.render(<Mastodon {...props} />);
     store.dispatch(setupBrowserNotifications());

app/javascript/mastodon/reducers/contexts.ts

@@ -4,6 +4,7 @@ import type { Draft, UnknownAction } from '@reduxjs/toolkit';
 import type { List as ImmutableList } from 'immutable';
 
 import { timelineDelete } from 'mastodon/actions/timelines_typed';
+import type { AsyncRefreshHeader } from 'mastodon/api';
 import type { ApiRelationshipJSON } from 'mastodon/api_types/relationships';
 import type {
   ApiStatusJSON,
@@ -12,7 +13,7 @@ import type {
 import type { Status } from 'mastodon/models/status';
 
 import { blockAccountSuccess, muteAccountSuccess } from '../actions/accounts';
-import { fetchContext } from '../actions/statuses';
+import { fetchContext, completeContextRefresh } from '../actions/statuses';
 import { TIMELINE_UPDATE } from '../actions/timelines';
 import { compareId } from '../compare_id';
 
@@ -25,11 +26,13 @@ interface TimelineUpdateAction extends UnknownAction {
 interface State {
   inReplyTos: Record<string, string>;
   replies: Record<string, string[]>;
+  refreshing: Record<string, AsyncRefreshHeader>;
 }
 
 const initialState: State = {
   inReplyTos: {},
   replies: {},
+  refreshing: {},
 };
 
 const normalizeContext = (
@@ -127,6 +130,13 @@ export const contextsReducer = createReducer(initialState, (builder) => {
   builder
     .addCase(fetchContext.fulfilled, (state, action) => {
       normalizeContext(state, action.meta.arg.statusId, action.payload.context);
+
+      if (action.payload.refresh) {
+        state.refreshing[action.meta.arg.statusId] = action.payload.refresh;
+      }
+    })
+    .addCase(completeContextRefresh, (state, action) => {
+      delete state.refreshing[action.payload.statusId];
     })
     .addCase(blockAccountSuccess, (state, action) => {
       filterContexts(

app/javascript/styles/mastodon/components.scss

@@ -2868,6 +2868,8 @@ a.account__display-name {
     }
 
     &__main {
+      --column-header-height: 62px;
+
       box-sizing: border-box;
       width: 100%;
       flex: 0 1 auto;
@@ -8815,6 +8817,10 @@ noscript {
 .conversation {
   position: relative;
 
+  // When scrolling these elements into view, take into account
+  // the column header height
+  scroll-margin-top: var(--column-header-height, 0);
+
   &.unread {
     &::before {
       content: '';

app/models/concerns/status/fetch_replies_concern.rb

@@ -3,9 +3,6 @@
 module Status::FetchRepliesConcern
   extend ActiveSupport::Concern
 
-  # enable/disable fetching all replies
-  FETCH_REPLIES_ENABLED = ENV['FETCH_REPLIES_ENABLED'] == 'true'
-
   # debounce fetching all replies to minimize DoS
   FETCH_REPLIES_COOLDOWN_MINUTES = (ENV['FETCH_REPLIES_COOLDOWN_MINUTES'] || 15).to_i.minutes
   FETCH_REPLIES_INITIAL_WAIT_MINUTES = (ENV['FETCH_REPLIES_INITIAL_WAIT_MINUTES'] || 5).to_i.minutes
@@ -36,7 +33,7 @@ module Status::FetchRepliesConcern
 
   def should_fetch_replies?
     # we aren't brand new, and we haven't fetched replies since the debounce window
-    FETCH_REPLIES_ENABLED && !local? && created_at <= FETCH_REPLIES_INITIAL_WAIT_MINUTES.ago && (
+    !local? && created_at <= FETCH_REPLIES_INITIAL_WAIT_MINUTES.ago && (
       fetched_replies_at.nil? || fetched_replies_at <= FETCH_REPLIES_COOLDOWN_MINUTES.ago
     )
   end

app/models/form/account_batch.rb

@@ -1,13 +1,11 @@
 # frozen_string_literal: true
 
-class Form::AccountBatch
+class Form::AccountBatch < Form::BaseBatch
-  include ActiveModel::Model
-  include Authorization
-  include AccountableConcern
   include Payloadable
 
-  attr_accessor :account_ids, :action, :current_account,
+  attr_accessor :account_ids,
-                :select_all_matching, :query
+                :query,
+                :select_all_matching
 
   def save
     case action

app/models/form/base_batch.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class Form::BaseBatch
+  include ActiveModel::Model
+  include Authorization
+  include AccountableConcern
+
+  attr_accessor :action,
+                :current_account
+
+  def save
+    raise 'Override in subclass'
+  end
+end

app/models/form/custom_emoji_batch.rb

@@ -1,12 +1,10 @@
 # frozen_string_literal: true
 
-class Form::CustomEmojiBatch
+class Form::CustomEmojiBatch < Form::BaseBatch
-  include ActiveModel::Model
+  attr_accessor :category_id,
-  include Authorization
+                :category_name,
-  include AccountableConcern
+                :visible_in_picker,
-
+                :custom_emoji_ids
-  attr_accessor :custom_emoji_ids, :action, :current_account,
-                :category_id, :category_name, :visible_in_picker
 
   def save
     case action

app/models/form/domain_block_batch.rb

@@ -1,11 +1,7 @@
 # frozen_string_literal: true
 
-class Form::DomainBlockBatch
+class Form::DomainBlockBatch < Form::BaseBatch
-  include ActiveModel::Model
+  attr_accessor :domain_blocks_attributes
-  include Authorization
-  include AccountableConcern
-
-  attr_accessor :domain_blocks_attributes, :action, :current_account
 
   def save
     case action

app/models/form/email_domain_block_batch.rb

@@ -1,11 +1,7 @@
 # frozen_string_literal: true
 
-class Form::EmailDomainBlockBatch
+class Form::EmailDomainBlockBatch < Form::BaseBatch
-  include ActiveModel::Model
+  attr_accessor :email_domain_block_ids
-  include Authorization
-  include AccountableConcern
-
-  attr_accessor :email_domain_block_ids, :action, :current_account
 
   def save
     case action

app/models/form/ip_block_batch.rb

@@ -1,11 +1,7 @@
 # frozen_string_literal: true
 
-class Form::IpBlockBatch
+class Form::IpBlockBatch < Form::BaseBatch
-  include ActiveModel::Model
+  attr_accessor :ip_block_ids
-  include Authorization
-  include AccountableConcern
-
-  attr_accessor :ip_block_ids, :action, :current_account
 
   def save
     case action

app/models/worker_batch.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+class WorkerBatch
+  include Redisable
+
+  TTL = 3600
+
+  def initialize(id = nil)
+    @id = id || SecureRandom.hex(12)
+  end
+
+  attr_reader :id
+
+  # Connect the batch with an async refresh. When the number of processed jobs
+  # passes the given threshold, the async refresh will be marked as finished.
+  # @param [String] async_refresh_key
+  # @param [Float] threshold
+  def connect(async_refresh_key, threshold: 1.0)
+    redis.hset(key, { 'async_refresh_key' => async_refresh_key, 'threshold' => threshold })
+  end
+
+  # Add jobs to the batch. Usually when the batch is created.
+  # @param [Array<String>] jids
+  def add_jobs(jids)
+    if jids.blank?
+      async_refresh_key = redis.hget(key, 'async_refresh_key')
+
+      if async_refresh_key.present?
+        async_refresh = AsyncRefresh.new(async_refresh_key)
+        async_refresh.finish!
+      end
+
+      return
+    end
+
+    redis.multi do |pipeline|
+      pipeline.sadd(key('jobs'), jids)
+      pipeline.expire(key('jobs'), TTL)
+      pipeline.hincrby(key, 'pending', jids.size)
+      pipeline.expire(key, TTL)
+    end
+  end
+
+  # Remove a job from the batch, such as when it's been processed or it has failed.
+  # @param [String] jid
+  def remove_job(jid)
+    _, pending, processed, async_refresh_key, threshold = redis.multi do |pipeline|
+      pipeline.srem(key('jobs'), jid)
+      pipeline.hincrby(key, 'pending', -1)
+      pipeline.hincrby(key, 'processed', 1)
+      pipeline.hget(key, 'async_refresh_key')
+      pipeline.hget(key, 'threshold')
+    end
+
+    if async_refresh_key.present?
+      async_refresh = AsyncRefresh.new(async_refresh_key)
+      async_refresh.increment_result_count(by: 1)
+      async_refresh.finish! if pending.zero? || processed >= threshold.to_f * (processed + pending)
+    end
+  end
+
+  # Get pending jobs.
+  # @returns [Array<String>]
+  def jobs
+    redis.smembers(key('jobs'))
+  end
+
+  # Inspect the batch.
+  # @returns [Hash]
+  def info
+    redis.hgetall(key)
+  end
+
+  private
+
+  def key(suffix = nil)
+    "worker_batch:#{@id}#{":#{suffix}" if suffix}"
+  end
+end

app/serializers/initial_state_serializer.rb

@@ -47,6 +47,7 @@ class InitialStateSerializer < ActiveModel::Serializer
       store[:default_content_type] = object_account_user.setting_default_content_type
       store[:system_emoji_font] = object_account_user.setting_system_emoji_font
       store[:show_trends]       = Setting.trends && object_account_user.setting_trends
+      store[:emoji_style]       = object_account_user.settings['web.emoji_style'] if Mastodon::Feature.modern_emojis_enabled?
     else
       store[:auto_play_gif] = Setting.auto_play_gif
       store[:display_media] = Setting.display_media

app/services/activitypub/fetch_all_replies_service.rb

@@ -6,7 +6,7 @@ class ActivityPub::FetchAllRepliesService < ActivityPub::FetchRepliesService
   # Limit of replies to fetch per status
   MAX_REPLIES = (ENV['FETCH_REPLIES_MAX_SINGLE'] || 500).to_i
 
-  def call(status_uri, collection_or_uri, max_pages: 1, request_id: nil)
+  def call(status_uri, collection_or_uri, max_pages: 1, async_refresh_key: nil, request_id: nil)
     @status_uri = status_uri
 
     super

app/services/activitypub/fetch_replies_service.rb

@@ -6,7 +6,7 @@ class ActivityPub::FetchRepliesService < BaseService
   # Limit of fetched replies
   MAX_REPLIES = 5
 
-  def call(reference_uri, collection_or_uri, max_pages: 1, allow_synchronous_requests: true, request_id: nil)
+  def call(reference_uri, collection_or_uri, max_pages: 1, allow_synchronous_requests: true, async_refresh_key: nil, request_id: nil)
     @reference_uri = reference_uri
     @allow_synchronous_requests = allow_synchronous_requests
 
@@ -14,7 +14,10 @@ class ActivityPub::FetchRepliesService < BaseService
     return if @items.nil?
 
     @items = filter_replies(@items)
-    FetchReplyWorker.push_bulk(@items) { |reply_uri| [reply_uri, { 'request_id' => request_id }] }
+
+    batch = WorkerBatch.new
+    batch.connect(async_refresh_key) if async_refresh_key.present?
+    batch.add_jobs(FetchReplyWorker.push_bulk(@items) { |reply_uri| [reply_uri, { 'request_id' => request_id, 'batch_id' => batch.id }] })
 
     [@items, n_pages]
   end

app/workers/activitypub/fetch_all_replies_worker.rb

@@ -55,7 +55,7 @@ class ActivityPub::FetchAllRepliesWorker
     replies_collection_or_uri = get_replies_uri(status)
     return if replies_collection_or_uri.nil?
 
-    ActivityPub::FetchAllRepliesService.new.call(value_or_id(status), replies_collection_or_uri, max_pages: max_pages, **options.deep_symbolize_keys)
+    ActivityPub::FetchAllRepliesService.new.call(value_or_id(status), replies_collection_or_uri, max_pages: max_pages, async_refresh_key: "context:#{@root_status.id}:refresh", **options.deep_symbolize_keys)
   end
 
   # Get the URI of the replies collection of a status

app/workers/fetch_reply_worker.rb

@@ -7,6 +7,9 @@ class FetchReplyWorker
   sidekiq_options queue: 'pull', retry: 3
 
   def perform(child_url, options = {})
+    batch = WorkerBatch.new(options.delete('batch_id')) if options['batch_id']
     FetchRemoteStatusService.new.call(child_url, **options.symbolize_keys)
+  ensure
+    batch&.remove_job(jid)
   end
 end

package.json

@@ -69,6 +69,7 @@
     "emoji-mart": "npm:emoji-mart-lazyload@latest",
     "emojibase": "^16.0.0",
     "emojibase-data": "^16.0.3",
+    "emojibase-regex": "^16.0.0",
     "escape-html": "^1.0.3",
     "fast-glob": "^3.3.3",
     "favico.js": "^0.3.10",

spec/models/worker_batch_spec.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe WorkerBatch do
+  subject { described_class.new }
+
+  let(:async_refresh_key) { 'test_refresh' }
+  let(:async_refresh) { nil }
+
+  describe '#id' do
+    it 'returns a string' do
+      expect(subject.id).to be_a String
+    end
+  end
+
+  describe '#connect' do
+    before do
+      subject.connect(async_refresh_key, threshold: 0.75)
+    end
+
+    it 'persists the async refresh key' do
+      expect(subject.info['async_refresh_key']).to eq async_refresh_key
+    end
+
+    it 'persists the threshold' do
+      expect(subject.info['threshold']).to eq '0.75'
+    end
+  end
+
+  describe '#add_jobs' do
+    before do
+      subject.connect(async_refresh_key, threshold: 0.5) if async_refresh.present?
+      subject.add_jobs([])
+    end
+
+    context 'when called with empty array' do
+      it 'does not persist the number of pending jobs' do
+        expect(subject.info).to be_empty
+      end
+
+      it 'does not persist the job IDs' do
+        expect(subject.jobs).to eq []
+      end
+
+      context 'when async refresh is connected' do
+        let(:async_refresh) { AsyncRefresh.new(async_refresh_key) }
+
+        it 'immediately marks the async refresh as finished' do
+          expect(async_refresh.reload.finished?).to be true
+        end
+      end
+    end
+
+    context 'when called with an array of job IDs' do
+      before do
+        subject.add_jobs(%w(foo bar))
+      end
+
+      it 'persists the number of pending jobs' do
+        expect(subject.info['pending']).to eq '2'
+      end
+
+      it 'persists the job IDs' do
+        expect(subject.jobs).to eq %w(foo bar)
+      end
+    end
+  end
+
+  describe '#remove_job' do
+    before do
+      subject.connect(async_refresh_key, threshold: 0.5) if async_refresh.present?
+      subject.add_jobs(%w(foo bar baz))
+      subject.remove_job('foo')
+    end
+
+    it 'removes the job from pending jobs' do
+      expect(subject.jobs).to eq %w(bar baz)
+    end
+
+    it 'decrements the number of pending jobs' do
+      expect(subject.info['pending']).to eq '2'
+    end
+
+    context 'when async refresh is connected' do
+      let(:async_refresh) { AsyncRefresh.new(async_refresh_key) }
+
+      it 'increments async refresh progress' do
+        expect(async_refresh.reload.result_count).to eq 1
+      end
+
+      it 'marks the async refresh as finished when the threshold is reached' do
+        subject.remove_job('bar')
+        expect(async_refresh.reload.finished?).to be true
+      end
+    end
+  end
+
+  describe '#info' do
+    it 'returns a hash' do
+      expect(subject.info).to be_a Hash
+    end
+  end
+end

spec/workers/activitypub/fetch_all_replies_worker_spec.rb

@@ -123,7 +123,6 @@ RSpec.describe ActivityPub::FetchAllRepliesWorker do
   end
 
   before do
-    stub_const('Status::FetchRepliesConcern::FETCH_REPLIES_ENABLED', true)
     all_items.each do |item|
       next if [top_note_uri, reply_note_uri].include? item
 

yarn.lock

@@ -2672,6 +2672,7 @@ __metadata:
     emoji-mart: "npm:emoji-mart-lazyload@latest"
     emojibase: "npm:^16.0.0"
     emojibase-data: "npm:^16.0.3"
+    emojibase-regex: "npm:^16.0.0"
     escape-html: "npm:^1.0.3"
     eslint: "npm:^9.23.0"
     eslint-import-resolver-typescript: "npm:^4.2.5"
@@ -3295,23 +3296,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@rollup/pluginutils@npm:^5.0.1, @rollup/pluginutils@npm:^5.0.2, @rollup/pluginutils@npm:^5.1.0":
+"@rollup/pluginutils@npm:^5.0.1, @rollup/pluginutils@npm:^5.0.2, @rollup/pluginutils@npm:^5.1.0, @rollup/pluginutils@npm:^5.1.3":
-  version: 5.1.4
-  resolution: "@rollup/pluginutils@npm:5.1.4"
-  dependencies:
-    "@types/estree": "npm:^1.0.0"
-    estree-walker: "npm:^2.0.2"
-    picomatch: "npm:^4.0.2"
-  peerDependencies:
-    rollup: ^1.20.0||^2.0.0||^3.0.0||^4.0.0
-  peerDependenciesMeta:
-    rollup:
-      optional: true
-  checksum: 10c0/6d58fbc6f1024eb4b087bc9bf59a1d655a8056a60c0b4021d3beaeec3f0743503f52467fd89d2cf0e7eccf2831feb40a05ad541a17637ea21ba10b21c2004deb
-  languageName: node
-  linkType: hard
-
-"@rollup/pluginutils@npm:^5.1.3":
   version: 5.2.0
   resolution: "@rollup/pluginutils@npm:5.2.0"
   dependencies:
@@ -5397,13 +5382,13 @@ __metadata:
   linkType: hard
 
 "axios@npm:^1.4.0":
-  version: 1.10.0
+  version: 1.11.0
-  resolution: "axios@npm:1.10.0"
+  resolution: "axios@npm:1.11.0"
   dependencies:
     follow-redirects: "npm:^1.15.6"
-    form-data: "npm:^4.0.0"
+    form-data: "npm:^4.0.4"
     proxy-from-env: "npm:^1.1.0"
-  checksum: 10c0/2239cb269cc789eac22f5d1aabd58e1a83f8f364c92c2caa97b6f5cbb4ab2903d2e557d9dc670b5813e9bcdebfb149e783fb8ab3e45098635cd2f559b06bd5d8
+  checksum: 10c0/5de273d33d43058610e4d252f0963cc4f10714da0bfe872e8ef2cbc23c2c999acc300fd357b6bce0fc84a2ca9bd45740fa6bb28199ce2c1266c8b1a393f2b36e
   languageName: node
   linkType: hard
 
@@ -6616,6 +6601,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"emojibase-regex@npm:^16.0.0":
+  version: 16.0.0
+  resolution: "emojibase-regex@npm:16.0.0"
+  checksum: 10c0/8ee5ff798e51caa581434b1cb2f9737e50195093c4efa1739df21a50a5496f80517924787d865e8cf7d6a0b4c90dbedc04bdc506dcbcc582e14cdf0bb47af0f0
+  languageName: node
+  linkType: hard
+
 "emojibase@npm:^16.0.0":
   version: 16.0.0
   resolution: "emojibase@npm:16.0.0"
@@ -7609,14 +7601,16 @@ __metadata:
   languageName: node
   linkType: hard
 
-"form-data@npm:^4.0.0":
+"form-data@npm:^4.0.4":
-  version: 4.0.1
+  version: 4.0.4
-  resolution: "form-data@npm:4.0.1"
+  resolution: "form-data@npm:4.0.4"
   dependencies:
     asynckit: "npm:^0.4.0"
     combined-stream: "npm:^1.0.8"
+    es-set-tostringtag: "npm:^2.1.0"
+    hasown: "npm:^2.0.2"
     mime-types: "npm:^2.1.12"
-  checksum: 10c0/bb102d570be8592c23f4ea72d7df9daa50c7792eb0cf1c5d7e506c1706e7426a4e4ae48a35b109e91c85f1c0ec63774a21ae252b66f4eb981cb8efef7d0463c8
+  checksum: 10c0/373525a9a034b9d57073e55eab79e501a714ffac02e7a9b01be1c820780652b16e4101819785e1e18f8d98f0aee866cc654d660a435c378e16a72f2e7cac9695
   languageName: node
   linkType: hard