From 8246b9761dcf96b0151a251d5bf65c2815ce7831 Mon Sep 17 00:00:00 2001 From: Ondrej Hlavacek Date: Tue, 15 Aug 2017 15:38:36 -0700 Subject: [PATCH 01/18] fix: remove open block comments --- lib/SqlFormatter.php | 8 ++++++-- tests/SqlFormatterTest.php | 7 +++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/SqlFormatter.php b/lib/SqlFormatter.php index 49e1d2f..7d6785a 100644 --- a/lib/SqlFormatter.php +++ b/lib/SqlFormatter.php @@ -219,8 +219,12 @@ protected static function getNextToken($string, $previous = null) if ($string[0] === '-' || $string[0] === '#') { $last = strpos($string, "\n"); $type = self::TOKEN_TYPE_COMMENT; - } else { // Comment until closing comment tag - $last = strpos($string, "*/", 2) + 2; + } else { // Comment until closing comment tag or end of query + if (strpos($string, "*/", 2)!== false) { + $last = strpos($string, "*/", 2) + 2; + } else { + $last = strlen($string); + } $type = self::TOKEN_TYPE_BLOCK_COMMENT; } diff --git a/tests/SqlFormatterTest.php b/tests/SqlFormatterTest.php index a7951e8..52e4068 100644 --- a/tests/SqlFormatterTest.php +++ b/tests/SqlFormatterTest.php @@ -93,6 +93,13 @@ function testRemoveComments() { $this->assertEquals($expected, $actual); } + + function testRemoveOpenComments() { + $sql = "/*\nopen block comment"; + $actual = SqlFormatter::removeComments($sql); + + $this->assertEquals("", $actual); + } function testCacheStats() { $stats = SqlFormatter::getCacheStats(); From 51c1cc44dbabb8efe25ad2852c73854ab47cff13 Mon Sep 17 00:00:00 2001 From: Ondrej Hlavacek Date: Tue, 15 Aug 2017 16:08:51 -0700 Subject: [PATCH 02/18] fix: coding style --- lib/SqlFormatter.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/SqlFormatter.php b/lib/SqlFormatter.php index 7d6785a..12fb471 100644 --- a/lib/SqlFormatter.php +++ b/lib/SqlFormatter.php @@ -220,7 +220,7 @@ protected static function getNextToken($string, $previous = null) $last = strpos($string, "\n"); $type = self::TOKEN_TYPE_COMMENT; } else { // Comment until closing comment tag or end of query - if (strpos($string, "*/", 2)!== false) { + if (strpos($string, "*/", 2) !== false) { $last = strpos($string, "*/", 2) + 2; } else { $last = strlen($string); From 4b5076f63a446cd911a24c021b7de9dd294b9d14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Hlav=C3=A1=C4=8Dek?= Date: Tue, 12 Feb 2019 14:02:24 +0100 Subject: [PATCH 03/18] feat: turn on error reporting --- bootstrap.php | 4 ++++ phpunit.xml.dist | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 bootstrap.php diff --git a/bootstrap.php b/bootstrap.php new file mode 100644 index 0000000..5f42aeb --- /dev/null +++ b/bootstrap.php @@ -0,0 +1,4 @@ + - + ./tests From 80abd6fcd272da04ecfb67d3935296ccdcf30ab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Hlav=C3=A1=C4=8Dek?= Date: Tue, 12 Feb 2019 14:02:47 +0100 Subject: [PATCH 04/18] fix: trailing spaces --- tests/SqlFormatterTest.php | 68 +++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/tests/SqlFormatterTest.php b/tests/SqlFormatterTest.php index 52e4068..07df772 100644 --- a/tests/SqlFormatterTest.php +++ b/tests/SqlFormatterTest.php @@ -6,7 +6,7 @@ class SqlFormatterTest extends PHPUnit_Framework_TestCase { protected $sqlData; - + /** * @dataProvider formatHighlightData */ @@ -54,43 +54,43 @@ function testCliHighlight($sql, $html) { function testCompress($sql, $html) { $this->assertEquals(trim($html), trim(SqlFormatter::compress($sql))); } - + function testUsePre() { SqlFormatter::$use_pre = false; $actual = SqlFormatter::highlight("test"); $expected = 'test'; $this->assertEquals($actual,$expected); - + SqlFormatter::$use_pre = true; $actual = SqlFormatter::highlight("test"); $expected = '
test
'; $this->assertEquals($actual,$expected); } - + function testSplitQuery() { $expected = array( "SELECT 'test' FROM MyTable;", "SELECT Column2 FROM SomeOther Table WHERE (test = true);" ); - + $actual = SqlFormatter::splitQuery(implode(';',$expected)); - + $this->assertEquals($expected, $actual); } - + function testSplitQueryEmpty() { $sql = "SELECT 1;SELECT 2;\n-- This is a comment\n;SELECT 3"; $expected = array("SELECT 1;","SELECT 2;","SELECT 3"); $actual = SqlFormatter::splitQuery($sql); - + $this->assertEquals($expected, $actual); } - + function testRemoveComments() { $expected = SqlFormatter::format("SELECT\n * FROM\n MyTable",false); $sql = "/* this is a comment */SELECT#This is another comment\n * FROM-- One final comment\n MyTable"; $actual = SqlFormatter::removeComments($sql); - + $this->assertEquals($expected, $actual); } @@ -105,11 +105,11 @@ function testCacheStats() { $stats = SqlFormatter::getCacheStats(); $this->assertGreaterThan(1,$stats['hits']); } - + function formatHighlightData() { $formatHighlightData = explode("\n\n",file_get_contents(__DIR__."/format-highlight.html")); $sqlData = $this->sqlData(); - + $return = array(); foreach($formatHighlightData as $i=>$data) { $return[] = array( @@ -117,14 +117,14 @@ function formatHighlightData() { $data ); } - + return $return; } - + function highlightCliData() { $clidata = explode("\n\n",file_get_contents(__DIR__."/clihighlight.html")); $sqlData = $this->sqlData(); - + $return = array(); foreach($clidata as $i=>$data) { $return[] = array( @@ -132,14 +132,14 @@ function highlightCliData() { $data ); } - + return $return; } - + function formatData() { $formatData = explode("\n\n",file_get_contents(__DIR__."/format.html")); $sqlData = $this->sqlData(); - + $return = array(); foreach($formatData as $i=>$data) { $return[] = array( @@ -147,14 +147,14 @@ function formatData() { $data ); } - + return $return; } - + function compressData() { $compressData = explode("\n\n",file_get_contents(__DIR__."/compress.html")); $sqlData = $this->sqlData(); - + $return = array(); foreach($compressData as $i=>$data) { $return[] = array( @@ -162,14 +162,14 @@ function compressData() { $data ); } - + return $return; } - + function highlightData() { $highlightData = explode("\n\n",file_get_contents(__DIR__."/highlight.html")); $sqlData = $this->sqlData(); - + $return = array(); foreach($highlightData as $i=>$data) { $return[] = array( @@ -177,43 +177,43 @@ function highlightData() { $data ); } - + return $return; } - - - + + + function sqlData() { if(!$this->sqlData) { $this->sqlData = explode("\n\n",file_get_contents(__DIR__."/sql.sql")); } - + /** $formatHighlight = array(); $highlight = array(); $format = array(); $compress = array(); $clihighlight = array(); - + foreach($this->sqlData as $sql) { $formatHighlight[] = trim(SqlFormatter::format($sql)); $highlight[] = trim(SqlFormatter::highlight($sql)); $format[] = trim(SqlFormatter::format($sql, false)); $compress[] = trim(SqlFormatter::compress($sql)); - + SqlFormatter::$cli = true; $clihighlight[] = trim(SqlFormatter::format($sql)); SqlFormatter::$cli = false; } - + file_put_contents(__DIR__."/format-highlight.html", implode("\n\n",$formatHighlight)); file_put_contents(__DIR__."/highlight.html", implode("\n\n",$highlight)); file_put_contents(__DIR__."/format.html", implode("\n\n",$format)); file_put_contents(__DIR__."/compress.html", implode("\n\n",$compress)); file_put_contents(__DIR__."/clihighlight.html", implode("\n\n",$clihighlight)); /**/ - + return $this->sqlData; } - + } From 79405f4211c170f0a4f44b0322edbbf4e7c29591 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Hlav=C3=A1=C4=8Dek?= Date: Tue, 12 Feb 2019 14:02:59 +0100 Subject: [PATCH 05/18] feat: failing trailing open comment test --- tests/SqlFormatterTest.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/SqlFormatterTest.php b/tests/SqlFormatterTest.php index 07df772..645e573 100644 --- a/tests/SqlFormatterTest.php +++ b/tests/SqlFormatterTest.php @@ -100,7 +100,14 @@ function testRemoveOpenComments() { $this->assertEquals("", $actual); } - + + function testRemoveTrailingClosingComments() { + $sql = "SELECT 1 */"; + $actual = SqlFormatter::removeComments($sql); + + $this->assertEquals("SELECT \n 1", $actual); + } + function testCacheStats() { $stats = SqlFormatter::getCacheStats(); $this->assertGreaterThan(1,$stats['hits']); From 75608b1bf47777fc506d4ab4d4dd842b3d936a7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Hlav=C3=A1=C4=8Dek?= Date: Tue, 12 Feb 2019 14:06:49 +0100 Subject: [PATCH 06/18] fix: the test should not assume removing the trailing closing comment --- tests/SqlFormatterTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/SqlFormatterTest.php b/tests/SqlFormatterTest.php index 645e573..b5dd73f 100644 --- a/tests/SqlFormatterTest.php +++ b/tests/SqlFormatterTest.php @@ -101,11 +101,11 @@ function testRemoveOpenComments() { $this->assertEquals("", $actual); } - function testRemoveTrailingClosingComments() { + function testTrailingClosingComments() { $sql = "SELECT 1 */"; $actual = SqlFormatter::removeComments($sql); - $this->assertEquals("SELECT \n 1", $actual); + $this->assertEquals("SELECT \n 1 */", $actual); } function testCacheStats() { From 79a471b26ec94207981dec73a94a3aeb10b5bfa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Hlav=C3=A1=C4=8Dek?= Date: Tue, 12 Feb 2019 14:07:02 +0100 Subject: [PATCH 07/18] fix: missing string index --- lib/SqlFormatter.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/SqlFormatter.php b/lib/SqlFormatter.php index 12fb471..ac20c75 100644 --- a/lib/SqlFormatter.php +++ b/lib/SqlFormatter.php @@ -214,7 +214,7 @@ protected static function getNextToken($string, $previous = null) } // Comment - if ($string[0] === '#' || (isset($string[1])&&($string[0]==='-'&&$string[1]==='-') || ($string[0]==='/'&&$string[1]==='*'))) { + if ($string[0] === '#' || (isset($string[1])&&($string[0]==='-'&&$string[1]==='-') || ($string[0]==='/'&&isset($string[1])&&$string[1]==='*'))) { // Comment until end of line if ($string[0] === '-' || $string[0] === '#') { $last = strpos($string, "\n"); From 474d0648f7ccf3e328181553246d4eade13eeb5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Hlav=C3=A1=C4=8Dek?= Date: Tue, 12 Feb 2019 14:24:11 +0100 Subject: [PATCH 08/18] fix: remove buggy versions, add newer versions --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a5c6dbd..fb4d637 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,11 @@ language: php php: + - 7.3 + - 7.2 + - 7.1 + - 7.0 - 5.6 - 5.5 - 5.4 - - 5.3 - - hhvm script: phpunit --coverage-text From a101cec0d34b5734c4c1e398f5ef73f50775bb63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Hlav=C3=A1=C4=8Dek?= Date: Tue, 12 Feb 2019 14:35:31 +0100 Subject: [PATCH 09/18] fix: remove unsupported php versions --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index fb4d637..005b2a6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,5 @@ language: php php: - - 7.3 - - 7.2 - - 7.1 - - 7.0 - 5.6 - 5.5 - 5.4 From a2d6fe7f43490d061ab007e58ee1f97a21b857e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Hlav=C3=A1=C4=8Dek?= Date: Tue, 12 Feb 2019 23:25:39 +0100 Subject: [PATCH 10/18] fix: remove further unsupported php versions --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 005b2a6..4e85cf9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,5 @@ language: php php: - 5.6 - - 5.5 - - 5.4 script: phpunit --coverage-text From da5373dada8ff18f41f31306b3e1a25cb566b79c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Hlav=C3=A1=C4=8Dek?= Date: Tue, 12 Feb 2019 23:28:42 +0100 Subject: [PATCH 11/18] fix: use tabs --- tests/SqlFormatterTest.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/SqlFormatterTest.php b/tests/SqlFormatterTest.php index b5dd73f..8c2654c 100644 --- a/tests/SqlFormatterTest.php +++ b/tests/SqlFormatterTest.php @@ -101,12 +101,12 @@ function testRemoveOpenComments() { $this->assertEquals("", $actual); } - function testTrailingClosingComments() { - $sql = "SELECT 1 */"; - $actual = SqlFormatter::removeComments($sql); + function testTrailingClosingComments() { + $sql = "SELECT 1 */"; + $actual = SqlFormatter::removeComments($sql); - $this->assertEquals("SELECT \n 1 */", $actual); - } + $this->assertEquals("SELECT \n 1 */", $actual); + } function testCacheStats() { $stats = SqlFormatter::getCacheStats(); From 24fab8dc2258dbfac04acb0d681e85ce60cee47c Mon Sep 17 00:00:00 2001 From: Ondra Jodas Date: Thu, 3 Sep 2020 11:23:36 +0200 Subject: [PATCH 12/18] format sql in removeComments method is optional choice --- lib/SqlFormatter.php | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/SqlFormatter.php b/lib/SqlFormatter.php index ac20c75..0d9dee2 100644 --- a/lib/SqlFormatter.php +++ b/lib/SqlFormatter.php @@ -790,10 +790,11 @@ public static function splitQuery($string) * Remove all comments from a SQL string * * @param String $string The SQL string + * @param bool $format format sql? * * @return String The SQL string without comments */ - public static function removeComments($string) + public static function removeComments($string, $format = true) { $result = ''; @@ -807,7 +808,10 @@ public static function removeComments($string) $result .= $token[self::TOKEN_VALUE]; } - $result = self::format( $result,false); + + if ($format) { + $result = self::format( $result,false); + } return $result; } From 20913af1412e8599ba333048f5168469ca8e400f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Jure=C4=8Dko?= Date: Fri, 11 Dec 2020 09:59:39 +0100 Subject: [PATCH 13/18] Run tests from PHP 5.6, 7.1, 7.4 --- .travis.yml | 7 ++++++- README.md | 6 ++++++ bootstrap.php | 2 ++ composer.json | 4 ++-- tests/SqlFormatterTest.php | 1 - 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4e85cf9..6fd9610 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,10 @@ language: php php: - 5.6 + - 7.1 + - 7.4 -script: phpunit --coverage-text +script: + - if [[ ${TRAVIS_PHP_VERSION:0:3} != "5.6" ]]; then phpenv config-rm xdebug.ini; fi + - composer install + - ./vendor/bin/phpunit --coverage-text diff --git a/README.md b/README.md index a144323..3a5b7f7 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,12 @@ A lightweight php class for formatting sql statements. It can automatically indent and add line breaks in addition to syntax highlighting. +For local development run: +``` +docker run --rm -it -v $(pwd):/app --entrypoint "" prooph/composer:5.6 composer install +docker run --rm -it -v $(pwd):/app --entrypoint "" prooph/composer:5.6 ./vendor/bin/phpunit +``` + History ============ diff --git a/bootstrap.php b/bootstrap.php index 5f42aeb..f903d74 100644 --- a/bootstrap.php +++ b/bootstrap.php @@ -1,4 +1,6 @@ =5.2.4" + "php": ">=5.6" }, "require-dev": { - "phpunit/phpunit": "3.7.*" + "phpunit/phpunit": "^5.0" }, "authors": [ { diff --git a/tests/SqlFormatterTest.php b/tests/SqlFormatterTest.php index 8c2654c..bdfed5f 100644 --- a/tests/SqlFormatterTest.php +++ b/tests/SqlFormatterTest.php @@ -1,5 +1,4 @@ Date: Fri, 11 Dec 2020 10:33:12 +0100 Subject: [PATCH 14/18] Generalization of comment implementation --- .gitignore | 1 + lib/SqlFormatter.php | 58 +++++++++++++++++++++++--------------------- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index 48b8bf9..4c36e38 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ +.idea/ vendor/ diff --git a/lib/SqlFormatter.php b/lib/SqlFormatter.php index 0d9dee2..e673856 100644 --- a/lib/SqlFormatter.php +++ b/lib/SqlFormatter.php @@ -39,7 +39,7 @@ class SqlFormatter 'CHARSET', 'CHECK', 'CHECKSUM', 'COLLATE', 'COLLATION', 'COLUMN', 'COLUMNS', 'COMMENT', 'COMMIT', 'COMMITTED', 'COMPRESSED', 'CONCURRENT', 'CONSTRAINT', 'CONTAINS', 'CONVERT', 'CREATE', 'CROSS', 'CURRENT_TIMESTAMP', 'DATABASE', 'DATABASES', 'DAY', 'DAY_HOUR', 'DAY_MINUTE', 'DAY_SECOND', 'DEFAULT', 'DEFINER', 'DELAYED', 'DELETE', 'DESC', 'DESCRIBE', 'DETERMINISTIC', 'DISTINCT', 'DISTINCTROW', 'DIV', - 'DO', 'DUMPFILE', 'DUPLICATE', 'DYNAMIC', 'ELSE', 'ENCLOSED', 'END', 'ENGINE', 'ENGINE_TYPE', 'ENGINES', 'ESCAPE', 'ESCAPED', 'EVENTS', 'EXEC', + 'DO', 'DUMPFILE', 'DUPLICATE', 'DYNAMIC', 'ELSE', 'ENCLOSED', 'END', 'ENGINE', 'ENGINE_TYPE', 'ENGINES', 'ESCAPE', 'ESCAPED', 'EVENTS', 'EXEC', 'EXECUTE', 'EXISTS', 'EXPLAIN', 'EXTENDED', 'FAST', 'FIELDS', 'FILE', 'FIRST', 'FIXED', 'FLUSH', 'FOR', 'FORCE', 'FOREIGN', 'FULL', 'FULLTEXT', 'FUNCTION', 'GLOBAL', 'GRANT', 'GRANTS', 'GROUP_CONCAT', 'HEAP', 'HIGH_PRIORITY', 'HOSTS', 'HOUR', 'HOUR_MINUTE', 'HOUR_SECOND', 'IDENTIFIED', 'IF', 'IFNULL', 'IGNORE', 'IN', 'INDEX', 'INDEXES', 'INFILE', 'INSERT', 'INSERT_ID', 'INSERT_METHOD', 'INTERVAL', @@ -122,6 +122,13 @@ class SqlFormatter // If not defined, it will be determined automatically public static $cli; + // Comments + public static $commentTokens = [ + ['#'], + ['--'], + ['/*', '*/'] + ]; + // For CLI syntax highlighting public static $cli_quote = "\x1b[34;1m"; public static $cli_backtick_quote = "\x1b[35;1m"; @@ -214,38 +221,33 @@ protected static function getNextToken($string, $previous = null) } // Comment - if ($string[0] === '#' || (isset($string[1])&&($string[0]==='-'&&$string[1]==='-') || ($string[0]==='/'&&isset($string[1])&&$string[1]==='*'))) { - // Comment until end of line - if ($string[0] === '-' || $string[0] === '#') { - $last = strpos($string, "\n"); - $type = self::TOKEN_TYPE_COMMENT; - } else { // Comment until closing comment tag or end of query - if (strpos($string, "*/", 2) !== false) { - $last = strpos($string, "*/", 2) + 2; - } else { - $last = strlen($string); - } - $type = self::TOKEN_TYPE_BLOCK_COMMENT; - } - - if ($last === false) { - $last = strlen($string); + $found = false; + foreach (self::$commentTokens as $comment) { + $start = $comment[0]; + $end = isset($comment[1]) ? $comment[1] : "\n"; + $found = substr($string, 0, strlen($start)) === $start; + if ($found) { + break; } - + } + if ($found) { + /** @var string $start */ + /** @var string $end */ + $endPos = strpos($string, $end, strlen($start)); + $last = $endPos ? + ($endPos + ($end === "\n" ? 0 : strlen($end))) : strlen($string); return array( self::TOKEN_VALUE => substr($string, 0, $last), - self::TOKEN_TYPE => $type + self::TOKEN_TYPE => $end === "\n" ? self::TOKEN_TYPE_COMMENT : self::TOKEN_TYPE_BLOCK_COMMENT ); } // Quoted String if ($string[0]==='"' || $string[0]==='\'' || $string[0]==='`' || $string[0]==='[') { - $return = array( + return array( self::TOKEN_TYPE => (($string[0]==='`' || $string[0]==='[')? self::TOKEN_TYPE_BACKTICK_QUOTE : self::TOKEN_TYPE_QUOTE), self::TOKEN_VALUE => self::getQuotedString($string) ); - - return $return; } // User-defined Variable @@ -254,7 +256,7 @@ protected static function getNextToken($string, $previous = null) self::TOKEN_VALUE => null, self::TOKEN_TYPE => self::TOKEN_TYPE_VARIABLE ); - + // If the variable name is quoted if ($string[1]==='"' || $string[1]==='\'' || $string[1]==='`') { $ret[self::TOKEN_VALUE] = $string[0].self::getQuotedString(substr($string,1)); @@ -266,7 +268,7 @@ protected static function getNextToken($string, $previous = null) $ret[self::TOKEN_VALUE] = $matches[1]; } } - + if($ret[self::TOKEN_VALUE] !== null) return $ret; } @@ -336,7 +338,7 @@ protected static function getNextToken($string, $previous = null) protected static function getQuotedString($string) { $ret = null; - + // This checks for the following patterns: // 1. backtick quoted string using `` to escape // 2. square bracket quoted string (SQL Server) using ]] to escape @@ -345,7 +347,7 @@ protected static function getQuotedString($string) if ( preg_match('/^(((`[^`]*($|`))+)|((\[[^\]]*($|\]))(\][^\]]*($|\]))*)|(("[^"\\\\]*(?:\\\\.[^"\\\\]*)*("|$))+)|((\'[^\'\\\\]*(?:\\\\.[^\'\\\\]*)*(\'|$))+))/s', $string, $matches)) { $ret = $matches[1]; } - + return $ret; } @@ -699,14 +701,14 @@ public static function format($string, $highlight=true) if ($token[self::TOKEN_VALUE] === '(' || $token[self::TOKEN_VALUE] === '.') { $return = rtrim($return,' '); } - + // If this is the "-" of a negative number, it shouldn't have a space after it if($token[self::TOKEN_VALUE] === '-' && isset($tokens[$i+1]) && $tokens[$i+1][self::TOKEN_TYPE] === self::TOKEN_TYPE_NUMBER && isset($tokens[$i-1])) { $prev = $tokens[$i-1][self::TOKEN_TYPE]; if($prev !== self::TOKEN_TYPE_QUOTE && $prev !== self::TOKEN_TYPE_BACKTICK_QUOTE && $prev !== self::TOKEN_TYPE_WORD && $prev !== self::TOKEN_TYPE_NUMBER) { $return = rtrim($return,' '); } - } + } } // If there are unmatched parentheses From d5dad3c69b965391ae53ff3e65b20cc5931b7995 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Jure=C4=8Dko?= Date: Fri, 11 Dec 2020 10:41:40 +0100 Subject: [PATCH 15/18] Ignore composer.lock --- .gitignore | 1 + composer.lock | 422 -------------------------------------------------- 2 files changed, 1 insertion(+), 422 deletions(-) delete mode 100644 composer.lock diff --git a/.gitignore b/.gitignore index 4c36e38..072705c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .idea/ vendor/ +composer.lock diff --git a/composer.lock b/composer.lock deleted file mode 100644 index 379ab13..0000000 --- a/composer.lock +++ /dev/null @@ -1,422 +0,0 @@ -{ - "hash": "a709b40d4a35e7077aa40fbd0f78f6c6", - "packages": [ - - ], - "packages-dev": [ - { - "name": "phpunit/php-code-coverage", - "version": "1.2.x-dev", - "source": { - "type": "git", - "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "1.2.9" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/1.2.9", - "reference": "1.2.9", - "shasum": "" - }, - "require": { - "php": ">=5.3.3", - "phpunit/php-file-iterator": ">=1.3.0@stable", - "phpunit/php-text-template": ">=1.1.1@stable", - "phpunit/php-token-stream": ">=1.1.3@stable" - }, - "suggest": { - "ext-dom": "*", - "ext-xdebug": ">=2.0.5" - }, - "type": "library", - "autoload": { - "classmap": [ - "PHP/" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "include-path": [ - "" - ], - "license": [ - "BSD-3-Clause" - ], - "authors": [ - { - "name": "Sebastian Bergmann", - "email": "sb@sebastian-bergmann.de", - "role": "lead" - } - ], - "description": "Library that provides collection, processing, and rendering functionality for PHP code coverage information.", - "homepage": "https://github.com/sebastianbergmann/php-code-coverage", - "keywords": [ - "coverage", - "testing", - "xunit" - ], - "time": "2013-02-26 18:55:56" - }, - { - "name": "phpunit/php-file-iterator", - "version": "dev-master", - "source": { - "type": "git", - "url": "https://github.com/sebastianbergmann/php-file-iterator.git", - "reference": "2deb24c65ea78e126daa8d45b2089ddc29ec1d26" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/2deb24c65ea78e126daa8d45b2089ddc29ec1d26", - "reference": "2deb24c65ea78e126daa8d45b2089ddc29ec1d26", - "shasum": "" - }, - "require": { - "php": ">=5.3.3" - }, - "type": "library", - "autoload": { - "classmap": [ - "File/" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "include-path": [ - "" - ], - "license": [ - "BSD-3-Clause" - ], - "authors": [ - { - "name": "Sebastian Bergmann", - "email": "sb@sebastian-bergmann.de", - "role": "lead" - } - ], - "description": "FilterIterator implementation that filters files based on a list of suffixes.", - "homepage": "https://github.com/sebastianbergmann/php-file-iterator/", - "keywords": [ - "filesystem", - "iterator" - ], - "time": "2013-01-07 10:47:05" - }, - { - "name": "phpunit/php-text-template", - "version": "dev-master", - "source": { - "type": "git", - "url": "https://github.com/sebastianbergmann/php-text-template.git", - "reference": "1eeef106193d2f8c539728e566bb4793071a9e18" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-text-template/zipball/1eeef106193d2f8c539728e566bb4793071a9e18", - "reference": "1eeef106193d2f8c539728e566bb4793071a9e18", - "shasum": "" - }, - "require": { - "php": ">=5.3.3" - }, - "type": "library", - "autoload": { - "classmap": [ - "Text/" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "include-path": [ - "" - ], - "license": [ - "BSD-3-Clause" - ], - "authors": [ - { - "name": "Sebastian Bergmann", - "email": "sb@sebastian-bergmann.de", - "role": "lead" - } - ], - "description": "Simple template engine.", - "homepage": "https://github.com/sebastianbergmann/php-text-template/", - "keywords": [ - "template" - ], - "time": "2013-01-07 10:56:17" - }, - { - "name": "phpunit/php-timer", - "version": "1.0.x-dev", - "source": { - "type": "git", - "url": "https://github.com/sebastianbergmann/php-timer.git", - "reference": "ecf7920b27003a9412b07dad79dbb5ad1249e6c3" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-timer/zipball/ecf7920b27003a9412b07dad79dbb5ad1249e6c3", - "reference": "ecf7920b27003a9412b07dad79dbb5ad1249e6c3", - "shasum": "" - }, - "require": { - "php": ">=5.3.3" - }, - "type": "library", - "autoload": { - "classmap": [ - "PHP/" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "include-path": [ - "" - ], - "license": [ - "BSD-3-Clause" - ], - "authors": [ - { - "name": "Sebastian Bergmann", - "email": "sb@sebastian-bergmann.de", - "role": "lead" - } - ], - "description": "Utility class for timing", - "homepage": "https://github.com/sebastianbergmann/php-timer/", - "keywords": [ - "timer" - ], - "time": "2013-01-30 06:08:51" - }, - { - "name": "phpunit/php-token-stream", - "version": "dev-master", - "source": { - "type": "git", - "url": "https://github.com/sebastianbergmann/php-token-stream.git", - "reference": "c25dd88e1592e66dee2553c99ef244203d5a1b98" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-token-stream/zipball/c25dd88e1592e66dee2553c99ef244203d5a1b98", - "reference": "c25dd88e1592e66dee2553c99ef244203d5a1b98", - "shasum": "" - }, - "require": { - "ext-tokenizer": "*", - "php": ">=5.3.3" - }, - "type": "library", - "autoload": { - "classmap": [ - "PHP/" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "include-path": [ - "" - ], - "license": [ - "BSD-3-Clause" - ], - "authors": [ - { - "name": "Sebastian Bergmann", - "email": "sb@sebastian-bergmann.de", - "role": "lead" - } - ], - "description": "Wrapper around PHP's tokenizer extension.", - "homepage": "https://github.com/sebastianbergmann/php-token-stream/", - "keywords": [ - "tokenizer" - ], - "time": "2013-01-07 10:56:35" - }, - { - "name": "phpunit/phpunit", - "version": "3.7.x-dev", - "source": { - "type": "git", - "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "2c67e52445416bb7c14046b432acd7eb79e4e612" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/2c67e52445416bb7c14046b432acd7eb79e4e612", - "reference": "2c67e52445416bb7c14046b432acd7eb79e4e612", - "shasum": "" - }, - "require": { - "ext-dom": "*", - "ext-pcre": "*", - "ext-reflection": "*", - "ext-spl": "*", - "php": ">=5.3.3", - "phpunit/php-code-coverage": ">=1.2.1,<1.3.0", - "phpunit/php-file-iterator": ">=1.3.1", - "phpunit/php-text-template": ">=1.1.1", - "phpunit/php-timer": ">=1.0.2,<1.1.0", - "phpunit/phpunit-mock-objects": ">=1.2.0,<1.3.0", - "symfony/yaml": ">=2.2.0" - }, - "require-dev": { - "pear-pear/pear": "1.9.4" - }, - "suggest": { - "ext-json": "*", - "ext-simplexml": "*", - "ext-tokenizer": "*", - "phpunit/php-invoker": ">=1.1.0,<1.2.0" - }, - "bin": [ - "composer/bin/phpunit" - ], - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "3.7.x-dev" - } - }, - "autoload": { - "classmap": [ - "PHPUnit/" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "include-path": [ - "", - "../../symfony/yaml/" - ], - "license": [ - "BSD-3-Clause" - ], - "authors": [ - { - "name": "Sebastian Bergmann", - "email": "sebastian@phpunit.de", - "role": "lead" - } - ], - "description": "The PHP Unit Testing framework.", - "homepage": "http://www.phpunit.de/", - "keywords": [ - "phpunit", - "testing", - "xunit" - ], - "time": "2013-03-11 07:06:05" - }, - { - "name": "phpunit/phpunit-mock-objects", - "version": "1.2.x-dev", - "source": { - "type": "git", - "url": "https://github.com/sebastianbergmann/phpunit-mock-objects.git", - "reference": "d49b5683200b5db9b1c64cb06f52f50d147891c4" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit-mock-objects/zipball/d49b5683200b5db9b1c64cb06f52f50d147891c4", - "reference": "d49b5683200b5db9b1c64cb06f52f50d147891c4", - "shasum": "" - }, - "require": { - "php": ">=5.3.3", - "phpunit/php-text-template": ">=1.1.1@stable" - }, - "suggest": { - "ext-soap": "*" - }, - "type": "library", - "autoload": { - "classmap": [ - "PHPUnit/" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "include-path": [ - "" - ], - "license": [ - "BSD-3-Clause" - ], - "authors": [ - { - "name": "Sebastian Bergmann", - "email": "sb@sebastian-bergmann.de", - "role": "lead" - } - ], - "description": "Mock Object library for PHPUnit", - "homepage": "https://github.com/sebastianbergmann/phpunit-mock-objects/", - "keywords": [ - "mock", - "xunit" - ], - "time": "2013-02-05 07:46:41" - }, - { - "name": "symfony/yaml", - "version": "dev-master", - "target-dir": "Symfony/Component/Yaml", - "source": { - "type": "git", - "url": "https://github.com/symfony/Yaml.git", - "reference": "f198ac28048eeceae852419c076123aaee59cd1c" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/symfony/Yaml/zipball/f198ac28048eeceae852419c076123aaee59cd1c", - "reference": "f198ac28048eeceae852419c076123aaee59cd1c", - "shasum": "" - }, - "require": { - "php": ">=5.3.3" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "2.3-dev" - } - }, - "autoload": { - "psr-0": { - "Symfony\\Component\\Yaml\\": "" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Fabien Potencier", - "email": "fabien@symfony.com" - }, - { - "name": "Symfony Community", - "homepage": "http://symfony.com/contributors" - } - ], - "description": "Symfony Yaml Component", - "homepage": "http://symfony.com", - "time": "2013-01-31 21:39:01" - } - ], - "aliases": [ - - ], - "minimum-stability": "dev", - "stability-flags": [ - - ], - "platform": { - "php": ">=5.2.4" - }, - "platform-dev": [ - - ] -} From 0a1f73a85599f7b474a3516958d1e8a8e8af9ca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Jure=C4=8Dko?= Date: Fri, 11 Dec 2020 11:43:35 +0100 Subject: [PATCH 16/18] Add test for Synapse temp table syntax --- lib/SqlFormatter.php | 4 ++-- tests/SqlFormatterTest.php | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/lib/SqlFormatter.php b/lib/SqlFormatter.php index e673856..ccc018d 100644 --- a/lib/SqlFormatter.php +++ b/lib/SqlFormatter.php @@ -123,7 +123,7 @@ class SqlFormatter public static $cli; // Comments - public static $commentTokens = [ + public static $comment_tokens = [ ['#'], ['--'], ['/*', '*/'] @@ -222,7 +222,7 @@ protected static function getNextToken($string, $previous = null) // Comment $found = false; - foreach (self::$commentTokens as $comment) { + foreach (self::$comment_tokens as $comment) { $start = $comment[0]; $end = isset($comment[1]) ? $comment[1] : "\n"; $found = substr($string, 0, strlen($start)) === $start; diff --git a/tests/SqlFormatterTest.php b/tests/SqlFormatterTest.php index bdfed5f..48df48d 100644 --- a/tests/SqlFormatterTest.php +++ b/tests/SqlFormatterTest.php @@ -112,6 +112,25 @@ function testCacheStats() { $this->assertGreaterThan(1,$stats['hits']); } + public function testSynapseTempTables() { + // In Azure Synapse is # char reserved for temp tables, not for comments + SqlFormatter::$comment_tokens = [ + ['--'], + ]; + + $sql = 'SELECT * INTO #temp_table FROM SOURCE_TABLE;'; + $sqlWithComment = "-- This is comment\n" . $sql; + $expected = <<assertEquals($expected, SqlFormatter::removeComments($sql)); + $this->assertEquals($expected, SqlFormatter::removeComments($sqlWithComment)); + } + function formatHighlightData() { $formatHighlightData = explode("\n\n",file_get_contents(__DIR__."/format-highlight.html")); $sqlData = $this->sqlData(); From 94fee9b147c0cf7f3f57c640c01c6a88bed541ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Jure=C4=8Dko?= Date: Fri, 11 Dec 2020 14:02:00 +0100 Subject: [PATCH 17/18] Improve code readability --- lib/SqlFormatter.php | 14 +++++++++++--- tests/SqlFormatterTest.php | 2 +- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/lib/SqlFormatter.php b/lib/SqlFormatter.php index ccc018d..4d5774f 100644 --- a/lib/SqlFormatter.php +++ b/lib/SqlFormatter.php @@ -233,11 +233,19 @@ protected static function getNextToken($string, $previous = null) if ($found) { /** @var string $start */ /** @var string $end */ + // Get position of the comment's end $endPos = strpos($string, $end, strlen($start)); - $last = $endPos ? - ($endPos + ($end === "\n" ? 0 : strlen($end))) : strlen($string); + if (!$endPos) { + // If the end is not found at all, then we take the whole rest of the string. + $endPos = strlen($string); + } elseif ($end !== "\n" ) { + // If the end is NOT the new line, we must skip end characters + $endPos += strlen($end); + } + return array( - self::TOKEN_VALUE => substr($string, 0, $last), + self::TOKEN_VALUE => substr($string, 0, $endPos), + // Comments that do not end with a new line are block comments. self::TOKEN_TYPE => $end === "\n" ? self::TOKEN_TYPE_COMMENT : self::TOKEN_TYPE_BLOCK_COMMENT ); } diff --git a/tests/SqlFormatterTest.php b/tests/SqlFormatterTest.php index 48df48d..3697a94 100644 --- a/tests/SqlFormatterTest.php +++ b/tests/SqlFormatterTest.php @@ -117,7 +117,7 @@ public function testSynapseTempTables() { SqlFormatter::$comment_tokens = [ ['--'], ]; - + $sql = 'SELECT * INTO #temp_table FROM SOURCE_TABLE;'; $sqlWithComment = "-- This is comment\n" . $sql; $expected = << Date: Fri, 11 Dec 2020 14:45:36 +0100 Subject: [PATCH 18/18] Added method setBoundaries --- lib/SqlFormatter.php | 9 ++++++++- tests/SqlFormatterTest.php | 3 ++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/SqlFormatter.php b/lib/SqlFormatter.php index 4d5774f..1dab84d 100644 --- a/lib/SqlFormatter.php +++ b/lib/SqlFormatter.php @@ -102,7 +102,8 @@ class SqlFormatter ); // Punctuation that can be used as a boundary between other tokens - protected static $boundaries = array(',', ';',':', ')', '(', '.', '=', '<', '>', '+', '-', '*', '/', '!', '^', '%', '|', '&', '#'); + const DEFAULT_BOUNDARIES = array(',', ';',':', ')', '(', '.', '=', '<', '>', '+', '-', '*', '/', '!', '^', '%', '|', '&', '#'); + protected static $boundaries = self::DEFAULT_BOUNDARIES; // For HTML syntax highlighting // Styles applied to different token types @@ -178,6 +179,12 @@ public static function getCacheStats() ); } + public static function setBoundaries(array $boundaries) + { + self::$boundaries = $boundaries; + self::$regex_boundaries = '('.implode('|',array_map(array(__CLASS__, 'quote_regex'),self::$boundaries)).')'; + } + /** * Stuff that only needs to be done once. Builds regular expressions and sorts the reserved words. */ diff --git a/tests/SqlFormatterTest.php b/tests/SqlFormatterTest.php index 3697a94..b934885 100644 --- a/tests/SqlFormatterTest.php +++ b/tests/SqlFormatterTest.php @@ -114,6 +114,7 @@ function testCacheStats() { public function testSynapseTempTables() { // In Azure Synapse is # char reserved for temp tables, not for comments + SqlFormatter::setBoundaries(array_diff(SqlFormatter::DEFAULT_BOUNDARIES, ['#'])); SqlFormatter::$comment_tokens = [ ['--'], ]; @@ -122,7 +123,7 @@ public function testSynapseTempTables() { $sqlWithComment = "-- This is comment\n" . $sql; $expected = <<