import { memo, useMemo } from "react"; import { ClickableWord } from "@/components/clickableWord"; const TOKEN_PATTERN = /[\p{L}\p{M}\p{N}]+(?:['’-][\p{L}\p{M}\p{N}]+)*|[^\p{L}\p{M}\p{N}\s]|\s+/gu; const WORD_PATTERN = /^[\p{L}\p{M}\p{N}]+(?:['’-][\p{L}\p{M}\p{N}]+)*$/u; interface TokenizedTextProps { text: string; onSelect?: (word: string) => void; } export const TokenizedText = memo(function TokenizedText({ text, onSelect }: TokenizedTextProps) { const tokens = useMemo(() => text.match(TOKEN_PATTERN) ?? [], [text]); return ( <> {tokens.map((token, i) => WORD_PATTERN.test(token) ? ( ) : ( {token} ) )} ); });