shot-scraper multi --har, --har-zip, --har-file options

* shot-scraper multi --har, --har-zip, --har-file
* Ability to run shot-craper multi --har and not take shots at all

Closes #166

Author: simonw

docs/har.md

@@ -1,3 +1,4 @@
+(har)=
 # Saving a web page to an HTTP Archive
 
 An HTTP Archive file captures the full details of a series of HTTP requests and responses as JSON.
@@ -51,6 +52,8 @@ Archive:  datasette-io.har.zip
    154803                     14 files
 ```
 
+You can record multiple pages to a single HTTP Archive using the {ref}`shot-scraper multi --har option<multi-har>`.
+
 ## `shot-scraper har --help`
 
 Full `--help` for this command:

docs/multi.md

@@ -109,6 +109,39 @@ You can include desired `height`, `width`, `quality`, `wait` and `wait_for` opti
   wait_for: document.querySelector('#bighead')
 ```
 
+(multi-har)=
+## Recording to an HTTP Archive
+
+Similar to the {ref}`shot-scraper har command<har>`, `shot-scraper multi` can optionally record HTTP Archive files of the requests made during a session.
+
+Add the `--har` flag to record all requests and responses to a `trace.har` JSON file, or `--har-zip` for a `trace.har.zip` file. Use `--har-file filename.har` to provide a path to a custom filename - this will be recorded as JSON or zip depending on the file extension.
+
+For example:
+
+```bash
+shot-scraper multi shots.yml --har
+```
+Will output something like this:
+```
+Screenshot of 'http://www.example.com/' written to 'example.com.png'
+Screenshot of 'https://www.w3.org/' written to 'w3c.org.png'
+Wrote to HAR file: trace.har
+```
+When writing to a HAR you can omit the `output:` key in a YAML file to skip taking a screenshot of that file. This `shots.yml` file for example:
+```yaml
+- url: https://example.com/
+- url: https://datasette.io/
+```
+When run like this:
+```bash
+shot-scraper multi shots.yml --har-zip
+```
+Will produce this output, recording a HAR without taking any screenshots:
+```
+Skipping screenshot of 'https://example.com/'
+Skipping screenshot of 'https://datasette.io/'
+Wrote to HAR file: trace.har.zip
+```
 ## Running a server for the duration of the session
 
 If you need to run a server for the duration of the `shot-scraper multi` session you can specify that using a `server:` block, like this:
@@ -213,6 +246,9 @@ Options:
   --auth-password TEXT            Password for HTTP Basic authentication
   --auth-username TEXT            Username for HTTP Basic authentication
   --leave-server                  Leave servers running when script finishes
+  --har                           Save all requests to trace.har file
+  --har-zip                       Save all requests to trace.har.zip file
+  --har-file FILE                 Path to HAR file to save all requests
   --help                          Show this message and exit.
 ```
 <!-- [[[end]]] -->

shot_scraper/cli.py

@@ -486,6 +486,21 @@ def _browser_context(
     is_flag=True,
     help="Leave servers running when script finishes",
 )
+@click.option(
+    "--har",
+    is_flag=True,
+    help="Save all requests to trace.har file",
+)
+@click.option(
+    "--har-zip",
+    is_flag=True,
+    help="Save all requests to trace.har.zip file",
+)
+@click.option(
+    "--har-file",
+    type=click.Path(file_okay=True, writable=True, dir_okay=False),
+    help="Path to HAR file to save all requests",
+)
 def multi(
     config,
     auth,
@@ -506,6 +521,9 @@ def multi(
     auth_username,
     auth_password,
     leave_server,
+    har,
+    har_zip,
+    har_file,
 ):
     """
     Take multiple screenshots, defined by a YAML file
@@ -524,8 +542,20 @@ def multi(
     For full YAML syntax documentation, see:
     https://shot-scraper.datasette.io/en/stable/multi.html
     """
+    if (har or har_zip) and not har_file:
+        har_file = filename_for_url(
+            "trace", ext="har.zip" if har_zip else "har", file_exists=os.path.exists
+        )
+
     scale_factor = normalize_scale_factor(retina, scale_factor)
     shots = yaml.safe_load(config)
+
+    # Special case: if we are recording a har_file output can be blank to skip a shot
+    if har_file:
+        for shot in shots:
+            if not shot.get("output"):
+                shot["skip_shot"] = True
+
     server_processes = []
     if shots is None:
         shots = []
@@ -543,6 +573,7 @@ def multi(
             reduced_motion=reduced_motion,
             auth_username=auth_username,
             auth_password=auth_password,
+            record_har_path=har_file or None,
         )
         try:
             for shot in shots:
@@ -599,11 +630,16 @@ def multi(
             browser_obj.close()
             if leave_server:
                 for process, details in server_processes:
-                    print("Leaving server PID:", process.pid, " details:", details)
+                    click.echo(
+                        f"Leaving server PID: {process.pid} details: {details}",
+                        err=True,
+                    )
             else:
                 if server_processes:
                     for process, _ in server_processes:
                         process.kill()
+            if har_file and not silent:
+                click.echo(f"Wrote to HAR file: {har_file}", err=True)
 
 
 @cli.command()
@@ -1341,12 +1377,16 @@ def on_response(response):
                 ", ".join(list(selectors) + list(selectors_all)), url, output
             )
     else:
-        # Whole page
-        if return_bytes:
-            return page.screenshot(**screenshot_args)
+        if shot.get("skip_shot"):
+            message = "Skipping screenshot of '{}'".format(url)
         else:
-            page.screenshot(**screenshot_args)
-            message = "Screenshot of '{}' written to '{}'".format(url, output)
+            # Whole page
+            if return_bytes:
+                return page.screenshot(**screenshot_args)
+            else:
+                page.screenshot(**screenshot_args)
+                message = "Screenshot of '{}' written to '{}'".format(url, output)
+
     if not silent:
         click.echo(message, err=True)
 

tests/test_shot_scraper.py

@@ -259,3 +259,53 @@ def test_har(http_server, args, expect_zip):
         # Verify entries is a non-empty list
         assert isinstance(har_content["log"]["entries"], list)
         assert len(har_content["log"]["entries"]) > 0
+
+
+@pytest.mark.parametrize(
+    "args,expect_zip,record_shots",
+    (
+        (["--har"], False, True),
+        (["--har-zip"], True, True),
+        (["--har-file", "output.har"], False, True),
+        (["--har-file", "output.har.zip"], True, True),
+        # And one where we don't record the shots:
+        (["--har"], False, False),
+    ),
+)
+def test_multi_har(http_server, args, expect_zip, record_shots):
+    runner = CliRunner()
+    (http_server.base_dir / "two.html").write_text("<h1>Two</h1>")
+    with runner.isolated_filesystem():
+        pathlib.Path("shots.yml").write_text(
+            f"- url: {http_server.base_url}/\n"
+            + (f"  output: index.png\n" if record_shots else "")
+            + f"- url: {http_server.base_url}/two.html\n"
+            + (f"  output: two.png\n" if record_shots else "")
+        )
+        # Should be no files
+        here = pathlib.Path(".")
+        files = [str(p) for p in here.glob("*.*")]
+        assert files == ["shots.yml"]
+        result = runner.invoke(cli, ["multi", "shots.yml"] + args)
+        assert result.exit_code == 0
+        if record_shots:
+            assert result.output.startswith("Screenshot of 'http://localhost")
+        else:
+            assert result.output.startswith("Skipping screenshot of 'http://localhost")
+        assert "Wrote to HAR file:" in result.output
+        assert (".har.zip" in result.output) == expect_zip
+        # HAR file should have been created
+        if expect_zip:
+            files = here.glob("*.har.zip")
+        else:
+            files = here.glob("*.har")
+        har_files = list(files)
+        # Should have created exactly one .har file
+        assert len(har_files) == 1
+        assert bool(zipfile.is_zipfile(har_files[0])) == expect_zip
+        shot_files = list(here.glob("*.png"))
+        num_shots = len(shot_files)
+        if record_shots:
+            assert num_shots == 2
+        else:
+            assert num_shots == 0