print stopped docker logs when we can't get the container

Author: obs-gh-mattcotter

integration/scripts/utils.py

@@ -14,6 +14,10 @@ def die(message: str) -> None:
     sys.exit(1)
 
 
+def print_remote_result(result: Result) -> None:
+    print(str(result))
+
+
 def mask_credentials(env_vars: Dict[str, Any]) -> Dict[str, Any]:
     masked_env_vars = env_vars.copy()
     # Only mask if vars exist
@@ -249,12 +253,7 @@ def get_docker_image(remote_host: Host) -> str:
     )
     images = [line.strip() for line in result.stdout.splitlines() if "SNAPSHOT" in line]
     if len(images) != 1:
-        die(
-            "❌ Error in finding observe-agent image, command output:\n{}\ncommand error:\n{}".format(
-                result.stdout or "<empty>",
-                result.stderr or "<empty>",
-            )
-        )
+        die("❌ Error in finding observe-agent image\n" + str(result))
 
     return images[0]
 
@@ -288,6 +287,22 @@ def get_docker_container(remote_host: Host, retries: int = 3) -> str:
             )
             time.sleep(1)
             return get_docker_container(remote_host, retries - 1)
+        elif retries == 0:
+            # No retries left, see if we can get logs from stopped containers to help debug.
+            result = remote_host.run_command('sudo docker ps --format "{{.ID}}"')
+            if result.stdout != "":
+                container_ids = result.stdout.splitlines()
+                for container_id in container_ids:
+                    print(
+                        "Logs for container {}:".format(container_id),
+                        file=sys.stderr,
+                    )
+                    result = remote_host.run_command(
+                        "sudo docker logs {}".format(container_id)
+                    )
+                    print_remote_result(result)
+            else:
+                print_remote_result(result)
         die(
             "❌ Error in finding observe-agent container; command output:\n{}\ncommand error:\n{}".format(
                 result.stdout or "<empty>",