Merge pull request #2113 from pupil-labs/develop

Pupil v3.2 Release Candidate 1

Author: papr

pupil_src/launchables/player.py

@@ -572,6 +572,18 @@ def set_window_size():
         g_pool.gui.append(g_pool.quickbar)
 
         # we always load these plugins
+        _pupil_producer_plugins = [
+            # In priority order (first is default)
+            ("Pupil_From_Recording", {}),
+            ("Offline_Pupil_Detection", {}),
+        ]
+        _pupil_producer_plugins = list(reversed(_pupil_producer_plugins))
+        _gaze_producer_plugins = [
+            # In priority order (first is default)
+            ("GazeFromRecording", {}),
+            ("GazeFromOfflineCalibration", {}),
+        ]
+        _gaze_producer_plugins = list(reversed(_gaze_producer_plugins))
         default_plugins = [
             ("Plugin_Manager", {}),
             ("Seek_Control", {}),
@@ -582,14 +594,25 @@ def set_window_size():
             ("System_Graphs", {}),
             ("System_Timelines", {}),
             ("World_Video_Exporter", {}),
-            ("Pupil_From_Recording", {}),
-            ("GazeFromRecording", {}),
+            *_pupil_producer_plugins,
+            *_gaze_producer_plugins,
             ("Audio_Playback", {}),
         ]
-
-        g_pool.plugins = Plugin_List(
-            g_pool, session_settings.get("loaded_plugins", default_plugins)
-        )
+        _plugins_to_load = session_settings.get("loaded_plugins", None)
+        if _plugins_to_load is None:
+            # If no plugins are available from a previous session,
+            # then use the default plugin list
+            _plugins_to_load = default_plugins
+        else:
+            # If there are plugins available from a previous session,
+            # then prepend plugins that are required, but might have not been available before
+            _plugins_to_load = [
+                *_pupil_producer_plugins,
+                *_gaze_producer_plugins,
+                *_plugins_to_load,
+            ]
+
+        g_pool.plugins = Plugin_List(g_pool, _plugins_to_load)
 
         # Manually add g_pool.capture to the plugin list
         g_pool.plugins._plugins.append(g_pool.capture)

pupil_src/shared_modules/accuracy_visualizer.py

@@ -36,9 +36,49 @@
 
 logger = logging.getLogger(__name__)
 
-Calculation_Result = namedtuple(
-    "Calculation_Result", ["result", "num_used", "num_total"]
-)
+
+class CalculationResult(T.NamedTuple):
+    result: float
+    num_used: int
+    num_total: int
+
+
+class CorrelatedAndCoordinateTransformedResult(T.NamedTuple):
+    """Holds result from correlating reference and gaze data and their respective
+    transformations into norm, image, and camera coordinate systems.
+    """
+
+    norm_space: np.ndarray  # shape: 2*n, 2
+    image_space: np.ndarray  # shape: 2*n, 2
+    camera_space: np.ndarray  # shape: 2*n, 3
+
+    @staticmethod
+    def empty() -> "CorrelatedAndCoordinateTransformedResult":
+        return CorrelatedAndCoordinateTransformedResult(
+            norm_space=np.ndarray([]),
+            image_space=np.ndarray([]),
+            camera_space=np.ndarray([]),
+        )
+
+
+class CorrelationError(ValueError):
+    pass
+
+
+class AccuracyPrecisionResult(T.NamedTuple):
+    accuracy: CalculationResult
+    precision: CalculationResult
+    error_lines: np.ndarray
+    correlation: CorrelatedAndCoordinateTransformedResult
+
+    @staticmethod
+    def failed() -> "AccuracyPrecisionResult":
+        return AccuracyPrecisionResult(
+            accuracy=CalculationResult(0.0, 0, 0),
+            precision=CalculationResult(0.0, 0, 0),
+            error_lines=np.array([]),
+            correlation=CorrelatedAndCoordinateTransformedResult.empty(),
+        )
 
 
 class ValidationInput:
@@ -105,10 +145,6 @@ def update(
 
     @staticmethod
     def __gazer_class_from_name(gazer_class_name: str) -> T.Optional[T.Any]:
-        if "HMD" in gazer_class_name:
-            logger.info("Accuracy visualization is disabled for HMD calibration")
-            return None
-
         gazers_by_name = gazer_classes_by_class_name(registered_gazer_classes())
 
         try:
@@ -337,7 +373,7 @@ def recalculate(self):
             succession_threshold=self.succession_threshold,
         )
 
-        accuracy = results[0].result
+        accuracy = results.accuracy.result
         if np.isnan(accuracy):
             self.accuracy = None
             logger.warning(
@@ -349,7 +385,7 @@ def recalculate(self):
                 "Angular accuracy: {}. Used {} of {} samples.".format(*results[0])
             )
 
-        precision = results[1].result
+        precision = results.precision.result
         if np.isnan(precision):
             self.precision = None
             logger.warning(
@@ -361,9 +397,8 @@ def recalculate(self):
                 "Angular precision: {}. Used {} of {} samples.".format(*results[1])
             )
 
-        self.error_lines = results[2]
-
-        ref_locations = [loc["norm_pos"] for loc in self.recent_input.ref_list]
+        self.error_lines = results.error_lines
+        ref_locations = results.correlation.norm_space[1::2, :]
         if len(ref_locations) >= 3:
             hull = ConvexHull(ref_locations)  # requires at least 3 points
             self.calibration_area = hull.points[hull.vertices, :]
@@ -378,36 +413,25 @@ def calc_acc_prec_errlines(
         intrinsics,
         outlier_threshold,
         succession_threshold=np.cos(np.deg2rad(0.5)),
-    ):
+    ) -> AccuracyPrecisionResult:
         gazer = gazer_class(g_pool, params=gazer_params)
 
         gaze_pos = gazer.map_pupil_to_gaze(pupil_list)
         ref_pos = ref_list
 
-        width, height = intrinsics.resolution
-
-        # reuse closest_matches_monocular to correlate one label to each prediction
-        # correlated['ref']: prediction, correlated['pupil']: label location
-        correlated = closest_matches_monocular(gaze_pos, ref_pos)
-        # [[pred.x, pred.y, label.x, label.y], ...], shape: n x 4
-        locations = np.array(
-            [(*e["ref"]["norm_pos"], *e["pupil"]["norm_pos"]) for e in correlated]
-        )
-        if locations.size == 0:
-            accuracy_result = Calculation_Result(0.0, 0, 0)
-            precision_result = Calculation_Result(0.0, 0, 0)
-            error_lines = np.array([])
-            return accuracy_result, precision_result, error_lines
-        error_lines = locations.copy()  # n x 4
-        locations[:, ::2] *= width
-        locations[:, 1::2] = (1.0 - locations[:, 1::2]) * height
-        locations.shape = -1, 2
+        try:
+            correlation_result = Accuracy_Visualizer.correlate_and_coordinate_transform(
+                gaze_pos, ref_pos, intrinsics
+            )
+            error_lines = correlation_result.norm_space.reshape(-1, 4)
+            undistorted_3d = correlation_result.camera_space
+        except CorrelationError:
+            return AccuracyPrecisionResult.failed()
 
         # Accuracy is calculated as the average angular
         # offset (distance) (in degrees of visual angle)
         # between fixations locations and the corresponding
         # locations of the fixation targets.
-        undistorted_3d = intrinsics.unprojectPoints(locations, normalize=True)
 
         # Cosine distance of A and B: (A @ B) / (||A|| * ||B||)
         # No need to calculate norms, since A and B are normalized in our case.
@@ -426,7 +450,7 @@ def calc_acc_prec_errlines(
             -1, 2
         )  # shape: num_used x 2
         accuracy = np.rad2deg(np.arccos(selected_samples.clip(-1.0, 1.0).mean()))
-        accuracy_result = Calculation_Result(accuracy, num_used, num_total)
+        accuracy_result = CalculationResult(accuracy, num_used, num_total)
 
         # lets calculate precision:  (RMS of distance of succesive samples.)
         # This is a little rough as we do not compensate headmovements in this test.
@@ -457,9 +481,89 @@ def calc_acc_prec_errlines(
         precision = np.sqrt(
             np.mean(np.rad2deg(np.arccos(succesive_distances.clip(-1.0, 1.0))) ** 2)
         )
-        precision_result = Calculation_Result(precision, num_used, num_total)
+        precision_result = CalculationResult(precision, num_used, num_total)
+
+        return AccuracyPrecisionResult(
+            accuracy_result, precision_result, error_lines, correlation_result
+        )
+
+    @staticmethod
+    def correlate_and_coordinate_transform(
+        gaze_pos, ref_pos, intrinsics
+    ) -> CorrelatedAndCoordinateTransformedResult:
+        # reuse closest_matches_monocular to correlate one label to each prediction
+        # correlated['ref']: prediction, correlated['pupil']: label location
+        # NOTE the switch of the ref and pupil keys! This effects mostly hmd data.
+        correlated = closest_matches_monocular(gaze_pos, ref_pos)
+        # [[pred.x, pred.y, label.x, label.y], ...], shape: n x 4
+        if not correlated:
+            raise CorrelationError("No correlation possible")
 
-        return accuracy_result, precision_result, error_lines
+        try:
+            return Accuracy_Visualizer._coordinate_transform_ref_in_norm_space(
+                correlated, intrinsics
+            )
+        except KeyError as err:
+            if "norm_pos" in err.args:
+                return Accuracy_Visualizer._coordinate_transform_ref_in_camera_space(
+                    correlated, intrinsics
+                )
+            else:
+                raise
+
+    @staticmethod
+    def _coordinate_transform_ref_in_norm_space(
+        correlated, intrinsics
+    ) -> CorrelatedAndCoordinateTransformedResult:
+        width, height = intrinsics.resolution
+        locations_norm = np.array(
+            [(*e["ref"]["norm_pos"], *e["pupil"]["norm_pos"]) for e in correlated]
+        )
+        locations_image = locations_norm.copy()  # n x 4
+        locations_image[:, ::2] *= width
+        locations_image[:, 1::2] = (1.0 - locations_image[:, 1::2]) * height
+        locations_image.shape = -1, 2
+        locations_norm.shape = -1, 2
+        locations_camera = intrinsics.unprojectPoints(locations_image, normalize=True)
+        return CorrelatedAndCoordinateTransformedResult(
+            locations_norm, locations_image, locations_camera
+        )
+
+    @staticmethod
+    def _coordinate_transform_ref_in_camera_space(
+        correlated, intrinsics
+    ) -> CorrelatedAndCoordinateTransformedResult:
+        width, height = intrinsics.resolution
+        locations_mixed = np.array(
+            # NOTE: This looks incorrect, but is actually correct. The switch comes from
+            # using closest_matches_monocular() above with switched arguments.
+            [(*e["ref"]["norm_pos"], *e["pupil"]["mm_pos"]) for e in correlated]
+        )  # n x 5
+        pupil_norm = locations_mixed[:, 0:2]  # n x 2
+        pupil_image = pupil_norm.copy()
+        pupil_image[:, 0] *= width
+        pupil_image[:, 1] = (1.0 - pupil_image[:, 1]) * height
+        pupil_camera = intrinsics.unprojectPoints(pupil_image, normalize=True)  # n x 3
+
+        ref_camera = locations_mixed[:, 2:5]  # n x 3
+        ref_camera /= np.linalg.norm(ref_camera, axis=1, keepdims=True)
+        ref_image = intrinsics.projectPoints(ref_camera)  # n x 2
+        ref_norm = ref_image.copy()
+        ref_norm[:, 0] /= width
+        ref_norm[:, 1] = 1.0 - (ref_norm[:, 1] / height)
+
+        locations_norm = np.hstack([pupil_norm, ref_norm])  # n x 4
+        locations_norm.shape = -1, 2
+
+        locations_image = np.hstack([pupil_image, ref_image])  # n x 4
+        locations_image.shape = -1, 2
+
+        locations_camera = np.hstack([pupil_camera, ref_camera])  # n x 6
+        locations_camera.shape = -1, 3
+
+        return CorrelatedAndCoordinateTransformedResult(
+            locations_norm, locations_image, locations_camera
+        )
 
     def gl_display(self):
         if self.vis_mapping_error and self.error_lines is not None:

pupil_src/shared_modules/audio_playback.py

@@ -123,17 +123,10 @@ def _setup_input_audio_part(self, part_idx):
         self.audio_paused = False
 
         self.audio.stream.seek(0)
-        first_frame = next(self.audio_frame_iterator)
-        self.audio_pts_rate = first_frame.samples
-        self.audio_start_pts = first_frame.pts
-
-        logger.debug(
-            "audio_pts_rate = {} start_pts = {}".format(
-                self.audio_pts_rate, self.audio_start_pts
-            )
-        )
-        self.check_ts_consistency(reference_frame=first_frame)
-        self.seek_to_audio_frame(0)
+        if self.should_check_ts_consistency:
+            first_frame = next(self.audio_frame_iterator)
+            self.check_ts_consistency(reference_frame=first_frame)
+            self.seek_to_audio_frame(0)
 
         logger.debug(
             "Audio file format {} chans {} rate {} framesize {}".format(
@@ -166,6 +159,12 @@ def _setup_output_audio(self):
 
         except ValueError:
             self.pa_stream = None
+        except OSError:
+            self.pa_stream = None
+            import traceback
+
+            logger.warning("Audio found, but playback failed (#2103)")
+            logger.debug(traceback.format_exc())
 
     def _setup_audio_vis(self):
         self.audio_timeline = None
@@ -254,13 +253,11 @@ def get_audio_frame_iterator(self):
                     yield frame
 
     def audio_idx_to_pts(self, idx):
-        return idx * self.audio_pts_rate
+        return self.audio.pts[idx]
 
     def seek_to_audio_frame(self, seek_pos):
         try:
-            self.audio.stream.seek(
-                self.audio_start_pts + self.audio_idx_to_pts(seek_pos)
-            )
+            self.audio.stream.seek(self.audio_idx_to_pts(seek_pos))
         except av.AVError:
             raise FileSeekError()
         else:
@@ -321,7 +318,7 @@ def update_audio_viz(self):
             self.audio_viz_data, finished = self.audio_viz_trans.get_data(
                 log_scale=self.log_scale
             )
-            if not finished:
+            if not finished and self.audio_timeline:
                 self.audio_timeline.refresh()
 
     def setup_pyaudio_output_if_necessary(self):