import { memo, useMemo } from "react";
import { ClickableWord } from "@/components/clickableWord";
const TOKEN_PATTERN = /[\p{L}\p{M}\p{N}]+(?:['’-][\p{L}\p{M}\p{N}]+)*|[^\p{L}\p{M}\p{N}\s]|\s+/gu;
const WORD_PATTERN = /^[\p{L}\p{M}\p{N}]+(?:['’-][\p{L}\p{M}\p{N}]+)*$/u;
interface TokenizedTextProps {
text: string;
onSelect?: (word: string) => void;
}
export const TokenizedText = memo(function TokenizedText({ text, onSelect }: TokenizedTextProps) {
const tokens = useMemo(() => text.match(TOKEN_PATTERN) ?? [], [text]);
return (
<>
{tokens.map((token, i) =>
WORD_PATTERN.test(token) ? (
) : (
{token}
)
)}
>
);
});