Performance optimizations

src/Tokens/GroupTokens.php

@@ -25,28 +25,43 @@ public function __invoke(array $tokens): array
         /** @var Token[] $groupedTokens */
         $groupedTokens = [];
 
-        while ($token = current($tokens)) {
-            $token = $token->cloneWithoutParent();
+        $count = count($tokens);
+        $removed = [];
+        for ($i = 0; $i < $count; $i++) {
+            if (isset($removed[$i])) {
+                continue;
+            }
+
+            $token = $tokens[$i]->cloneWithoutParent();
 
-            foreach ($tokens as $compareKey => $compareToken) {
-                if ($token->equals($compareToken)) {
+            // Since tokens are sorted by start, only check subsequent tokens
+            // that could overlap (start < token->end)
+            for ($j = $i + 1; $j < $count; $j++) {
+                if (isset($removed[$j])) {
                     continue;
                 }
 
-                if ($token->containsOrOverlaps($compareToken)) {
-                    if ($token->canContain($compareToken)) {
-                        $token->addChild($compareToken);
-                    }
+                $compareToken = $tokens[$j];
+
+                // Since tokens are sorted by start position,
+                // once compareToken->start >= token->end, no more overlaps possible
+                if ($compareToken->start >= $token->end) {
+                    break;
+                }
 
-                    unset($tokens[$compareKey]);
+                // At this point we know: token->start <= compareToken->start < token->end
+                // and they are not equal (different indices, and sorted order means
+                // same-start tokens differ in end). This means containsOrOverlaps is true.
+                if ($token->canContain($compareToken)) {
+                    $token->addChild($compareToken);
                 }
+
+                $removed[$j] = true;
             }
 
             if ($token->parent === null) {
                 $groupedTokens[] = $token;
             }
-
-            next($tokens);
         }
 
         return $groupedTokens;

src/Tokens/ParseTokens.php

@@ -15,6 +15,7 @@
     public function __invoke(string $content, Language $language): array
     {
         $tokens = [];
+        $seen = [];
 
         // Match tokens from patterns
         foreach ($language->getPatterns() as $key => $pattern) {
@@ -33,34 +34,30 @@ public function __invoke(string $content, Language $language): array
                 continue;
             }
 
+            $tokenType = $pattern->getTokenType();
+            $tokenTypeValue = $tokenType->getValue();
+
             foreach ($match as $item) {
                 $offset = $item[1];
                 $value = $item[0];
 
-                $token = new Token(
+                $hashKey = $offset . ':' . $tokenTypeValue . ':' . $value;
+
+                if (isset($seen[$hashKey])) {
+                    continue;
+                }
+
+                $seen[$hashKey] = true;
+
+                $tokens[] = new Token(
                     offset: $offset,
                     value: $value,
-                    type: $pattern->getTokenType(),
+                    type: $tokenType,
                     pattern: $pattern,
                 );
-
-                if (! $this->tokenAlreadyPresent($tokens, $token)) {
-                    $tokens[] = $token;
-                }
             }
         }
 
         return $tokens;
     }
-
-    private function tokenAlreadyPresent(array $tokens, Token $token): bool
-    {
-        foreach ($tokens as $tokenToCompare) {
-            if ($tokenToCompare->equals($token)) {
-                return true;
-            }
-        }
-
-        return false;
-    }
 }