Add ability to process prefetched content

Kdecherf · j0k3r · commit 897e44ce68d5 · 2022-02-02T13:20:03.000+01:00
fetchContent() now accepts an optional parameter, prefetchedContent, which
can contain the content of a page that was fetched before calling Graby.

If we take the example of Wallabag it gives the ability of sending the
content of a page (through a browser extension for example) without
making network calls to fetch the page.

Signed-off-by: Kevin Decherf &lt;kevin@kdecherf.com&gt;
diff --git a/README.md b/README.md
@@ -129,6 +129,22 @@ array(
 
 The `date` result is the same as displayed in the content. If `date` is not `null` in the result, we recommend you to parse it using [`date_parse`](http://php.net/date_parse) (this is what we are using to validate that the date is correct).
 
+### Retrieve content from a prefetched page
+
+If you want to extract content from a page you fetched outside of Graby, you can call `setContentAsPrefetched()` before calling `fetchContent()`, e.g.:
+
+``` php
+use Graby\Graby;
+
+$article = 'http://www.bbc.com/news/entertainment-arts-32547474';
+
+$input = '<html>[...]</html>';
+
+$graby = new Graby();
+$graby->setContentAsPrefetched($input);
+$result = $graby->fetchContent($article);
+```
+
 ### Cleanup content
 
 Since the 1.9.0 version, you can also send html content to be cleanup in the same way graby clean content retrieved from an url. The url is still needed to convert links to absolute, etc.
diff --git a/src/Graby.php b/src/Graby.php
@@ -42,6 +42,7 @@ class Graby
     private $punycode;
 
     private $imgNoReferrer = false;
+    private $prefetchedContent = null;
 
     /**
      * @param array       $config
@@ -161,6 +162,11 @@ public function getConfig($key)
         return $this->config[$key];
     }
 
+    public function setContentAsPrefetched(string $content): void
+    {
+        $this->prefetchedContent = $content;
+    }
+
     /**
      * Fetch content from the given url and return a readable content.
      *
@@ -272,6 +278,18 @@ public function cleanupHtml($contentBlock, $url)
         return trim($this->cleanupXss((string) $html));
     }
 
+    private function getResponseForPrefetchedContent(string $url): array
+    {
+        return [
+            'body' => $this->prefetchedContent,
+            'effective_url' => $url,
+            'headers' => [
+                'content-type' => 'text/html',
+            ],
+            'status' => 200,
+        ];
+    }
+
     /**
      * Do fetch content from an url.
      *
@@ -284,9 +302,13 @@ private function doFetchContent($url)
         $url = $this->validateUrl($url);
         $siteConfig = $this->configBuilder->buildFromUrl($url);
 
-        $this->logger->info('Fetching url: {url}', ['url' => $url]);
-
-        $response = $this->httpClient->fetch($url, false, $siteConfig->http_header);
+        if (null === $this->prefetchedContent) {
+            $this->logger->info('Fetching url: {url}', ['url' => $url]);
+            $response = $this->httpClient->fetch($url, false, $siteConfig->http_header);
+        } else {
+            $this->logger->info('Content provided as prefetched for url: {url}', ['url' => $url]);
+            $response = $this->getResponseForPrefetchedContent($url);
+        }
 
         $effectiveUrl = $response['effective_url'];
         $effectiveUrl = str_replace(' ', '%20', $effectiveUrl);
@@ -330,7 +352,7 @@ private function doFetchContent($url)
 
         // check site config for single page URL - fetch it if found
         $isSinglePage = false;
-        if ($this->config['singlepage'] && ($singlePageResponse = $this->getSinglePage($html, $effectiveUrl))) {
+        if ($this->config['singlepage'] && null === $this->prefetchedContent && ($singlePageResponse = $this->getSinglePage($html, $effectiveUrl))) {
             $isSinglePage = true;
             $effectiveUrl = $singlePageResponse['effective_url'];
 
@@ -370,7 +392,7 @@ private function doFetchContent($url)
 
         // Deal with multi-page articles
         $isMultiPage = (!$isSinglePage && $extractResult && null !== $this->extractor->getNextPageUrl());
-        if ($this->config['multipage'] && $isMultiPage) {
+        if ($this->config['multipage'] && null === $this->prefetchedContent && $isMultiPage) {
             $this->logger->info('Attempting to process multi-page article');
             // store first page to avoid parsing it again (previous url content is in `$contentBlock`)
             $multiPageUrls = [$effectiveUrl];
diff --git a/tests/GrabyTest.php b/tests/GrabyTest.php
@@ -1396,6 +1396,23 @@ public function testWithTooLongHtmlJitFail(): void
         $this->assertNotSame('No title found', $res['title']);
     }
 
+    public function testPrefetchedContent(): void
+    {
+        $httpMockClient = new HttpMockClient();
+        $graby = new Graby([
+            'debug' => true,
+        ], $httpMockClient);
+
+        $input = '<html><body><h1>This is my awesome article</h1><article><p>' . str_repeat('This is an awesome text with some links, here there are the awesome', 7) . '</p></article></body></html>';
+
+        $graby->setContentAsPrefetched($input);
+        $res = $graby->fetchContent('https://example.com/prefetched-content');
+
+        $this->assertSame('This is my awesome article', $res['title']);
+        $this->assertSame('https://example.com/prefetched-content', $res['url']);
+        $this->assertStringContainsString('here there are the awesome', $res['html']);
+    }
+
     /**
      * Return an instance of graby with a mocked Guzzle client returning data from a predefined file.
      */
