Lexer - Solving ambiguity on function keywords

Signed-off-by: iifawzi <[email protected]>

Author: iifawzi

src/Lexer.php

@@ -87,6 +87,30 @@ class Lexer extends Core
         'parseUnknown',
     ];
 
+
+    /**
+     * A list of keywords that indicate that the function keyword
+     * is not used as a function
+     *
+     * @var string[]
+     */
+    public $KEYWORD_NAME_INDICATORS = [
+        'FROM',
+        'SET',
+        'WHERE',
+    ];
+
+    /**
+     * A list of operators that indicate that the function keyword
+     * is not used as a function
+     *
+     * @var string[]
+     */
+    public $OPERATOR_NAME_INDICATORS = [
+        ',',
+        '.',
+    ];
+
     /**
      * The string to be parsed.
      *
@@ -344,6 +368,7 @@ public function lex()
         $this->list = $list;
 
         $this->solveAmbiguityOnStarOperator();
+        $this->solveAmbiguityOnFunctionKeywords();
     }
 
     /**
@@ -358,10 +383,8 @@ public function lex()
      * - ")" (a closing parenthesis like in "COUNT(*)").
      * This methods will change the flag of the "*" tokens when any of those condition above is true. Otherwise, the
      * default flag (arithmetic) will be kept.
-     *
-     * @return void
      */
-    private function solveAmbiguityOnStarOperator()
+    private function solveAmbiguityOnStarOperator(): void
     {
         $iBak = $this->list->idx;
         while (($starToken = $this->list->getNextOfTypeAndValue(Token::TYPE_OPERATOR, '*')) !== null) {
@@ -385,6 +408,51 @@ private function solveAmbiguityOnStarOperator()
         $this->list->idx = $iBak;
     }
 
+    /**
+     * Resolves the ambiguity when dealing with the functions keywords.
+     *
+     * In SQL statements, the function keywords might be used as table names or columns names.
+     * To solve this ambiguity, the solution is to find the next token, excluding whitespaces and
+     * comments, right after the function keyword position. The function keyword is for sure used
+     * as column name or table name if the next token found is any of:
+     *
+     * - "FROM" (the FROM keyword like in "SELECT Country x, AverageSalary avg FROM...");
+     * - "WHERE" (the WHERE keyword like in "DELETE FROM emp x WHERE x.salary = 20");
+     * - "SET" (the SET keyword like in "UPDATE Country x, City y set x.Name=x.Name");
+     * - "," (a comma separator like 'x,' in "UPDATE Country x, City y set x.Name=x.Name");
+     * - "." (a dot separator like in "x.asset_id FROM (SELECT evt.asset_id FROM evt)".
+     * - "NULL" (when used as a table alias like in "avg.col FROM (SELECT ev.col FROM ev) avg").
+     *
+     * This method will change the flag of the function keyword tokens when any of those
+     * condition above is true. Otherwise, the
+     * default flag (function keyword) will be kept.
+     */
+    private function solveAmbiguityOnFunctionKeywords(): void
+    {
+        $iBak = $this->list->idx;
+        $keywordFunction = Token::TYPE_KEYWORD | Token::FLAG_KEYWORD_FUNCTION;
+        while (($keywordToken = $this->list->getNextOfTypeAndFlag(Token::TYPE_KEYWORD, $keywordFunction)) !== null) {
+            $next = $this->list->getNext();
+            if (
+                ($next->type !== Token::TYPE_KEYWORD
+                    || ! in_array($next->value, $this->KEYWORD_NAME_INDICATORS, true)
+                )
+                && ($next->type !== Token::TYPE_OPERATOR
+                    || ! in_array($next->value, $this->OPERATOR_NAME_INDICATORS, true)
+                )
+                && ($next->value !== null)
+            ) {
+                continue;
+            }
+
+            $keywordToken->type = Token::TYPE_NONE;
+            $keywordToken->flags = Token::TYPE_NONE;
+            $keywordToken->keyword = $keywordToken->value;
+        }
+
+        $this->list->idx = $iBak;
+    }
+
     /**
      * Creates a new error log.
      *

src/TokensList.php

@@ -168,6 +168,23 @@ public function getNextOfTypeAndValue($type, $value)
         return null;
     }
 
+    /**
+     * Gets the next token.
+     *
+     * @param int $type the type of the token
+     * @param int $flag the flag of the token
+     */
+    public function getNextOfTypeAndFlag(int $type, int $flag): ?Token
+    {
+        for (; $this->idx < $this->count; ++$this->idx) {
+            if (($this->tokens[$this->idx]->type === $type) && ($this->tokens[$this->idx]->flags === $flag)) {
+                return $this->tokens[$this->idx++];
+            }
+        }
+
+        return null;
+    }
+
     /**
      * Sets an value inside the container.
      *

tests/Components/ExpressionTest.php

@@ -22,6 +22,23 @@ public function testParse2(): void
         $this->assertEquals($component->expr, 'col');
     }
 
+    public function testParse3(): void
+    {
+        $component = Expression::parse(new Parser(), $this->getTokensList('col xx'));
+        $this->assertEquals($component->alias, 'xx');
+
+        $component = Expression::parse(new Parser(), $this->getTokensList('col y'));
+        $this->assertEquals($component->alias, 'y');
+
+        $component = Expression::parse(new Parser(), $this->getTokensList('avg.col FROM (SELECT ev.col FROM ev)'));
+        $this->assertEquals($component->table, 'avg');
+        $this->assertEquals($component->expr, 'avg.col');
+
+        $component = Expression::parse(new Parser(), $this->getTokensList('x.id FROM (SELECT a.id FROM a) x'));
+        $this->assertEquals($component->table, 'x');
+        $this->assertEquals($component->expr, 'x.id');
+    }
+
     /**
      * @dataProvider parseErrProvider
      */

tests/Lexer/TokensListTest.php

@@ -30,17 +30,22 @@ public function setUp(): void
             new Token(' ', Token::TYPE_WHITESPACE),
             new Token('*', Token::TYPE_OPERATOR),
             new Token(' ', Token::TYPE_WHITESPACE),
-            new Token('FROM', Token::TYPE_KEYWORD),
+            new Token('FROM', Token::TYPE_KEYWORD, Token::FLAG_KEYWORD_RESERVED),
             new Token(' ', Token::TYPE_WHITESPACE),
             new Token('`test`', Token::TYPE_SYMBOL),
             new Token(' ', Token::TYPE_WHITESPACE),
+            new Token('WHERE', Token::TYPE_KEYWORD, Token::FLAG_KEYWORD_RESERVED),
+            new Token(' ', Token::TYPE_WHITESPACE),
+            new Token('name', Token::TYPE_NONE),
+            new Token('=', Token::TYPE_OPERATOR),
+            new Token('fa', Token::TYPE_NONE),
         ];
     }
 
     public function testBuild(): void
     {
         $list = new TokensList($this->tokens);
-        $this->assertEquals('SELECT * FROM `test` ', TokensList::build($list));
+        $this->assertEquals('SELECT * FROM `test` WHERE name=fa', TokensList::build($list));
     }
 
     public function testAdd(): void
@@ -60,6 +65,10 @@ public function testGetNext(): void
         $this->assertEquals($this->tokens[2], $list->getNext());
         $this->assertEquals($this->tokens[4], $list->getNext());
         $this->assertEquals($this->tokens[6], $list->getNext());
+        $this->assertEquals($this->tokens[8], $list->getNext());
+        $this->assertEquals($this->tokens[10], $list->getNext());
+        $this->assertEquals($this->tokens[11], $list->getNext());
+        $this->assertEquals($this->tokens[12], $list->getNext());
         $this->assertNull($list->getNext());
     }
 
@@ -78,9 +87,24 @@ public function testGetNextOfType(): void
         $list = new TokensList($this->tokens);
         $this->assertEquals($this->tokens[0], $list->getNextOfType(Token::TYPE_KEYWORD));
         $this->assertEquals($this->tokens[4], $list->getNextOfType(Token::TYPE_KEYWORD));
+        $this->assertEquals($this->tokens[8], $list->getNextOfType(Token::TYPE_KEYWORD));
         $this->assertNull($list->getNextOfType(Token::TYPE_KEYWORD));
     }
 
+    public function testGetNextOfTypeAndFlag(): void
+    {
+        $list = new TokensList($this->tokens);
+        $this->assertEquals($this->tokens[4], $list->getNextOfTypeAndFlag(
+            Token::TYPE_KEYWORD,
+            Token::FLAG_KEYWORD_RESERVED
+        ));
+        $this->assertEquals($this->tokens[8], $list->getNextOfTypeAndFlag(
+            Token::TYPE_KEYWORD,
+            Token::FLAG_KEYWORD_RESERVED
+        ));
+        $this->assertNull($list->getNextOfTypeAndFlag(Token::TYPE_KEYWORD, Token::FLAG_KEYWORD_RESERVED));
+    }
+
     public function testGetNextOfTypeAndValue(): void
     {
         $list = new TokensList($this->tokens);
@@ -107,7 +131,7 @@ public function testArrayAccess(): void
 
         // offsetExists($offset)
         $this->assertArrayHasKey(2, $list);
-        $this->assertArrayNotHasKey(8, $list);
+        $this->assertArrayNotHasKey(13, $list);
 
         // offsetUnset($offset)
         unset($list[2]);

tests/data/parser/parseDelete13.in

@@ -0,0 +1 @@
+DELETE FROM emp x WHERE x.salary = 20