Add hostless detection science module for Rubin

Author: utthishtastro

fink_science/rubin/hostless_detection/__init__.py

@@ -0,0 +1,34 @@
+# Copyright 2024-2025 AstroLab Software
+# Author: Rupesh Durgesh
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of the paper: ELEPHANT: ExtragaLactic alErt Pipeline for Hostless AstroNomical Transients https://arxiv.org/abs/2404.18165"""
+
+import io
+
+from astropy.io import fits
+import numpy as np
+
+
+def read_cutout_stamp(fits_bytes: bytes) -> np.ndarray:
+    """
+    Reads Rubin cutout stamps
+
+    Parameters
+    ----------
+    fits_bytes
+       input byte string
+    """
+    fits_buffer = io.BytesIO(fits_bytes)
+    with fits.open(fits_buffer) as hdulist:
+        return hdulist[0].data

fink_science/rubin/hostless_detection/pipeline_utils.py

@@ -0,0 +1,94 @@
+# Copyright 2020-2025 AstroLab Software
+# Author: Rupesh Durgesh
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from line_profiler import profile
+import os
+
+import pandas as pd
+import pyspark.sql.functions as F
+from pyspark.sql.types import ArrayType, FloatType
+
+from fink_science.rubin.hostless_detection.run_pipeline import (
+    HostLessExtragalacticRubin,
+)
+from fink_science.ztf.hostless_detection.pipeline_utils import load_json
+from fink_science.tester import spark_unit_tests
+
+CONFIGS_BASE = load_json("fink_science/ztf/hostless_detection/config_base.json")
+CONFIGS = load_json("fink_science/ztf/hostless_detection/config.json")
+CONFIGS.update(CONFIGS_BASE)
+
+
+@F.pandas_udf(ArrayType(FloatType()))
+@profile
+def run_potential_hostless(
+    cutoutScience: pd.Series, cutoutTemplate: pd.Series
+) -> pd.Series:
+    """Runs potential hostless candidate detection for Rubin without any filtering
+
+    Parameters
+    ----------
+    cutoutScience: pd.Series
+        science stamp images
+    cutoutTemplate: pd.Series
+        template stamp images
+
+    Returns
+    -------
+    pd.Series
+        Scores (array of 2 floats) for being hostless
+
+    References
+    ----------
+    1. ELEPHANT: ExtragaLactic alErt Pipeline for Hostless AstroNomical
+     Transients
+        https://arxiv.org/abs/2404.18165
+
+    Examples
+    --------
+    >>> df = spark.read.format('parquet').load(rubin_alert_sample)
+    >>> df.count()
+    50
+
+    # Add a new column
+    >>> df = df.withColumn('kstest_static',
+    ...     run_potential_hostless(
+    ...         df["cutoutScience"],
+    ...         df["cutoutTemplate"]))
+    >>> df.filter(df.kstest_static[0] >= 0).count()
+    3
+    """
+    kstest_results = []
+    hostless_science_class = HostLessExtragalacticRubin(CONFIGS_BASE)
+
+    for index in range(cutoutScience.shape[0]):
+        science_stamp = cutoutScience[index]
+        template_stamp = cutoutTemplate[index]
+        kstest_science, kstest_template = (
+            hostless_science_class.process_candidate_fink_rubin(
+                science_stamp, template_stamp
+            )
+        )
+        kstest_results.append([kstest_science, kstest_template])
+    return pd.Series(kstest_results)
+
+
+if __name__ == "__main__":
+    globs = globals()
+    path = os.path.dirname(__file__)
+
+    rubin_alert_sample = "./fink_science/data/alerts/or4_lsst7.1"
+    globs["sample_file"] = rubin_alert_sample
+    spark_unit_tests(globs)

fink_science/rubin/hostless_detection/processor.py

@@ -0,0 +1,72 @@
+# Copyright 2024-2025 AstroLab Software
+# Author: Rupesh Durgesh
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Implementation of the paper: ELEPHANT: ExtragaLactic alErt Pipeline for Hostless AstroNomical Transients https://arxiv.org/abs/2404.18165"""
+
+from typing import Tuple
+
+from fink_science.ztf.hostless_detection.pipeline_utils import (
+    run_hostless_detection_with_clipped_data,
+    run_powerspectrum_analysis,
+)
+from fink_science.ztf.hostless_detection.run_pipeline import HostLessExtragalactic
+from fink_science.rubin.hostless_detection.pipeline_utils import read_cutout_stamp
+
+
+class HostLessExtragalacticRubin(HostLessExtragalactic):
+    """
+    Detects potential hostless candidates for extragalactic transients.
+
+    Inherits from the HostLessExtragalactic base class.
+
+    """
+
+    def process_candidate_fink_rubin(
+        self, science_stamp: bytes, template_stamp: bytes
+    ) -> Tuple[float, float]:
+        """
+        Processes each candidate
+
+        Parameters
+        ----------
+        science_stamp
+           science stamp data
+        template_stamp
+           template stamp data
+        """
+        science_stamp = read_cutout_stamp(science_stamp)
+
+        template_stamp = read_cutout_stamp(template_stamp)
+
+        if science_stamp.shape != template_stamp.shape:
+            return -99, -99
+
+        science_stamp_clipped, template_stamp_clipped = self._run_sigma_clipping(
+            science_stamp, template_stamp
+        )
+        is_hostless_candidate = run_hostless_detection_with_clipped_data(
+            science_stamp_clipped, template_stamp_clipped, self.configs
+        )
+        if is_hostless_candidate:
+            power_spectrum_results = run_powerspectrum_analysis(
+                science_stamp,
+                template_stamp,
+                science_stamp_clipped.mask.astype(int),
+                template_stamp_clipped.mask.astype(int),
+                self._image_shape,
+            )
+            return power_spectrum_results[
+                "kstest_SCIENCE_15_statistic"
+            ], power_spectrum_results["kstest_TEMPLATE_15_statistic"]
+        return -99, -99

fink_science/rubin/hostless_detection/run_pipeline.py

@@ -221,8 +221,12 @@ def run_powerspectrum_analysis(
         number of iterations for powerspectrum analysis shuffling
 
     """
-    science_data = create_noise_filled_mask(science_image, science_mask, image_size)
-    template_data = create_noise_filled_mask(template_image, template_mask, image_size)
+    science_data = create_noise_filled_mask(
+        science_image, science_mask, list(science_mask.shape)
+    )
+    template_data = create_noise_filled_mask(
+        template_image, template_mask, list(template_mask.shape)
+    )
     _, kstest_results_dict, _, _ = ps.detect_host_with_powerspectrum(
         science_data,
         template_data,