[^\p{L}\p{M}\p{N}]*)' . '(?(?:[A-Za-z]{1,3}\.){2,})(?[^\p{L}\p{M}\p{N}]*)$/u', $token, $m )) { return new Word(0, $m['lead'], $m['core'], $m['core'], $m['trail'], TokenType::DOTTED); } // Dot-words like "node.js" (exactly one dot), but avoid URLs/emails and domains. // Only handle a tiny allowlist of suffixes (js/ts/jsx/tsx). if (strpos($token, '://') === false && strpos($token, '@') === false) { if (preg_match( '/^(?[^\p{L}\p{M}\p{N}]*)' . '(?[A-Za-z]+)\.(?[A-Za-z]+)' . '(?[^\p{L}\p{M}\p{N}]*)$/u', $token, $m )) { $rightLower = strtolower($m['right']); $allow = ['js' => true, 'ts' => true, 'jsx' => true, 'tsx' => true]; if (isset($allow[$rightLower])) { $core = $m['left'] . '.' . $m['right']; return new Word(0, $m['lead'], $core, $core, $m['trail'], TokenType::DOTWORD); } } } // Compound tokens with &, /, + (e.g., r&d, input/output, c++). // Conservative match; formatting/casing rules are handled in Pass 1. if (preg_match( '/^(?[^\p{L}\p{M}\p{N}]*)' . '(?[\p{L}\p{M}\p{N}]+' . '(?:(?:[&\/+][\p{L}\p{M}\p{N}]+)+|(?:\+{2,}))' . ')(?[^\p{L}\p{M}\p{N}]*)$/u', $token, $m )) { return new Word(0, $m['lead'], $m['core'], $m['core'], $m['trail'], TokenType::COMPOUND); } // Normal words (letters/digits) with internal apostrophes/hyphens if (preg_match( '/^(?[^\p{L}\p{M}\p{N}]*)' . '(?[\p{L}\p{M}\p{N}][\p{L}\p{M}\p{N}\'’\-\x{2010}\x{2011}]*)' . '(?[^\p{L}\p{M}\p{N}]*)$/u', $token, $m )) { return new Word(0, $m['lead'], $m['core'], $m['core'], $m['trail'], TokenType::WORD); } // Opaque tokens (domains, URLs, emails, paths, etc.) that contain letters/digits // but are not normal word tokens. Keep them for word-position logic. if (preg_match('/[\p{L}\p{M}\p{N}]/u', $token) === 1) { if (preg_match( '/^(?[^\p{L}\p{M}\p{N}]*)' . '(?.*?)' . '(?[^\p{L}\p{M}\p{N}]*)$/u', $token, $m )) { if (preg_match('/[\p{L}\p{M}\p{N}]/u', $m['core']) === 1) { return new Word(0, $m['lead'], $m['core'], $m['core'], $m['trail'], TokenType::OPAQUE); } } } return null; } /** * @param array $parts * @return array */ public function buildWords(array $parts): array { $words = []; foreach ($parts as $i => $part) { if (preg_match('/^\s+$/u', $part) === 1) { continue; // whitespace token } $split = $this->splitToken($part); if ($split === null) { continue; // punctuation-only or unsupported token } $split->partIndex = $i; $words[] = $split; } return $words; } }