Merge branch 'master' into 2.1.2

Author: Nick-S-2018

.github/workflows/ci.yml

@@ -14,7 +14,7 @@ jobs:
     strategy:
       matrix:
         os: ['ubuntu-latest']
-        php-version: ['7.1', '7.3', '7.4', '8.0']
+        php-version: ['7.1', '7.3', '7.4', '8.0', '8.1']
         phpunit-version: ['latest']
         transport: ['', 'PhpHttp']
 

docs/searchclass.md

@@ -219,6 +219,15 @@ You can enable weight calculation by setting the `track_scores` option to `true`
     $search->trackScores(null); // unset track_scores option
 ```
 
+### stripBadUtf8()
+You can enable removal of bad utf8 characters from the results by setting the `strip_bad_utf8` option to `true`.
+
+```php
+    $search->stripBadUtf8(true); // enable removal of bad utf8 characters
+    $search->stripBadUtf8(false); // disable removal of bad utf8 characters
+    $search->stripBadUtf8(null); // unset strip_bad_utf8 option
+```
+
 ### profile()
 
 If included, result set will provide query profiling.

src/Manticoresearch/Response.php

@@ -70,9 +70,12 @@ public function getResponse()
     {
         if (null === $this->response) {
             $this->response = json_decode($this->string, true);
-
             if (json_last_error() !== JSON_ERROR_NONE) {
-                throw new RuntimeException('fatal error while trying to decode JSON response');
+                if (json_last_error() === JSON_ERROR_UTF8 && $this->stripBadUtf8()) {
+                    $this->response = json_decode(preg_replace('/[\x00-\x1F\x80-\xFF]/', '', $this->string), true);
+                } else {
+                    throw new RuntimeException('fatal error while trying to decode JSON response');
+                }
             }
 
             if (empty($this->response)) {
@@ -81,6 +84,15 @@ public function getResponse()
         }
         return $this->response;
     }
+    
+    /**
+     * check if strip_bad_utf8 as been set to true
+     * @return boolean
+     */
+    private function stripBadUtf8()
+    {
+        return !empty($this->transportInfo['body']) && !empty($this->transportInfo['body']['strip_bad_utf8']);
+    }
 
     /*
      * Check whenever response has error

src/Manticoresearch/ResultSet.php

@@ -55,27 +55,27 @@ public function __construct($responseObj)
         }
     }
 
-    public function rewind()
+    public function rewind(): void
     {
         $this->position = 0;
     }
 
-    public function current()
+    public function current(): ResultHit
     {
         return new ResultHit($this->array[$this->position]);
     }
 
-    public function next()
+    public function next(): void
     {
         $this->position++;
     }
 
-    public function valid()
+    public function valid(): bool
     {
         return isset($this->array[$this->position]);
     }
 
-    public function key()
+    public function key(): int
     {
         return $this->position;
     }
@@ -103,7 +103,7 @@ public function getResponse()
         return $this->response;
     }
 
-    public function count()
+    public function count(): int
     {
         return count($this->array);
     }

src/Manticoresearch/Results/PercolateResultSet.php

@@ -4,10 +4,11 @@
 namespace Manticoresearch\Results;
 
 use Manticoresearch\ResultSet;
+use Manticoresearch\ResultHit;
 
 class PercolateResultSet extends ResultSet
 {
-    public function current()
+    public function current(): ResultHit
     {
         return new PercolateResultHit($this->array[$this->position]);
     }

src/Manticoresearch/Search.php

@@ -75,6 +75,17 @@ public function trackScores($trackScores): self
         return $this;
     }
 
+    public function stripBadUtf8($stripBadUtf8): self
+    {
+        if (is_null($stripBadUtf8)) {
+            unset($this->params['strip_bad_utf8']);
+        } else {
+            $this->params['strip_bad_utf8'] = (bool)$stripBadUtf8;
+        }
+        
+        return $this;
+    }
+    
     /**
      * @param string $queryString
      * @return $this

test/Manticoresearch/SearchTest.php

@@ -774,6 +774,18 @@ public function testTrackScores()
             $this->assertGreaterThan(1, $resultHit->getScore());
         }
     }
+    
+    public function testStripBadUtf8Compiles()
+    {
+        $body = self::$search->stripBadUtf8(true)->compile();
+        $this->assertTrue($body['strip_bad_utf8']);
+        
+        $body = self::$search->stripBadUtf8(false)->compile();
+        $this->assertFalse($body['strip_bad_utf8']);
+        
+        $body = self::$search->stripBadUtf8(null)->compile();
+        $this->assertArrayNotHasKey('strip_bad_utf8', $body);
+    }
 
     public function testResultHitGetData()
     {