Fix merging events with intraday prices

If Yahoo returns intraday price data with dividend or stock-split event in future, then this broke the merge.
Fix is to discard out-of-range events.
Assumes that if user requesting intraday then they aren't interested in events.

Author: ValueRaider

tests/prices.py

@@ -114,6 +114,43 @@ def test_duplicatingWeekly(self):
         if not test_run:
             self.skipTest("Skipping test_duplicatingWeekly() because not possible to fail Monday/weekend")
 
+    def test_pricesEventsMerge(self):
+        # Test case: dividend occurs after last row in price data
+        tkr = 'INTC'
+        start_d = _dt.date(2022, 1, 1)
+        end_d = _dt.date(2023, 1, 1)
+        df = yf.Ticker(tkr, session=self.session).history(interval='1d', start=start_d, end=end_d)
+        div = 1.0
+        future_div_dt = df.index[-1] + _dt.timedelta(days=1)
+        if future_div_dt.weekday() in [5, 6]:
+            future_div_dt += _dt.timedelta(days=1) * (7 - future_div_dt.weekday())
+        divs = _pd.DataFrame(data={"Dividends":[div]}, index=[future_div_dt])
+        df2 = yf.utils.safe_merge_dfs(df.drop(['Dividends', 'Stock Splits'], axis=1), divs, '1d')
+        self.assertIn(future_div_dt, df2.index)
+        self.assertIn("Dividends", df2.columns)
+        self.assertEqual(df2['Dividends'].iloc[-1], div)
+
+    def test_pricesEventsMerge_bug(self):
+        # Reproduce exception when merging intraday prices with future dividend
+        tkr = 'S32.AX'
+        interval = '30m'
+        df_index = []
+        d = 13
+        for h in range(0, 16):
+            for m in [0, 30]:
+                df_index.append(_dt.datetime(2023, 9, d, h, m))
+        df_index.append(_dt.datetime(2023, 9, d, 16))
+        df = _pd.DataFrame(index=df_index)
+        df.index = _pd.to_datetime(df.index)
+        df['Close'] = 1.0
+
+        div = 1.0
+        future_div_dt = _dt.datetime(2023, 9, 14, 10)
+        divs = _pd.DataFrame(data={"Dividends":[div]}, index=[future_div_dt])
+
+        df2 = yf.utils.safe_merge_dfs(df, divs, interval)
+        # No exception = test pass
+
     def test_intraDayWithEvents(self):
         tkrs = ["BHP.AX", "IMP.JO", "BP.L", "PNL.L", "INTC"]
         test_run = False

yfinance/utils.py

@@ -650,8 +650,10 @@ def fix_Yahoo_returning_live_separate(quotes, interval, tz_exchange):
 
 
 def safe_merge_dfs(df_main, df_sub, interval):
-    if df_sub.shape[0] == 0:
+    if df_sub.empty:
         raise Exception("No data to merge")
+    if df_main.empty:
+        return df_main
 
     df_sub_backup = df_sub.copy()
     data_cols = [c for c in df_sub.columns if c not in df_main]
@@ -675,47 +677,54 @@ def safe_merge_dfs(df_main, df_sub, interval):
     else:
         indices = _np.searchsorted(_np.append(df_main.index, df_main.index[-1] + td), df_sub.index, side='right')
         indices -= 1  # Convert from [[i-1], [i]) to [[i], [i+1])
-        # Numpy.searchsorted does not handle out-of-range well, so handle manually:
-        for i in range(len(df_sub.index)):
-            dt = df_sub.index[i]
-            if dt < df_main.index[0] or dt >= df_main.index[-1] + td:
-                # Out-of-range
-                indices[i] = -1
+    # Numpy.searchsorted does not handle out-of-range well, so handle manually:
+    for i in range(len(df_sub.index)):
+        dt = df_sub.index[i]
+        if dt < df_main.index[0] or dt >= df_main.index[-1] + td:
+            # Out-of-range
+            indices[i] = -1
 
     f_outOfRange = indices == -1
-    if f_outOfRange.any() and not intraday:
-        empty_row_data = {c:[_np.nan] for c in const.price_colnames}|{'Volume':[0]}
-        if interval == '1d':
-            # For 1d, add all out-of-range event dates
-            for i in _np.where(f_outOfRange)[0]:
-                dt = df_sub.index[i]
-                get_yf_logger().debug(f"Adding out-of-range {data_col} @ {dt.date()} in new prices row of NaNs")
-                empty_row = _pd.DataFrame(data=empty_row_data, index=[dt])
-                df_main = _pd.concat([df_main, empty_row], sort=True)
+    if f_outOfRange.any():
+        if intraday:
+            # Discard out-of-range dividends in intraday data, assume user not interested
+            df_sub = df_sub[~f_outOfRange]
+            if df_sub.empty:
+                df_main['Dividends'] = 0.0
+                return df_main
         else:
-            # Else, only add out-of-range event dates if occurring in interval 
-            # immediately after last pricfe row
-            last_dt = df_main.index[-1]
-            next_interval_start_dt = last_dt + td
-            next_interval_end_dt = next_interval_start_dt + td
-            for i in _np.where(f_outOfRange)[0]:
-                dt = df_sub.index[i]
-                if next_interval_start_dt <= dt < next_interval_end_dt:
-                    new_dt = next_interval_start_dt
+            empty_row_data = {c:[_np.nan] for c in const.price_colnames}|{'Volume':[0]}
+            if interval == '1d':
+                # For 1d, add all out-of-range event dates
+                for i in _np.where(f_outOfRange)[0]:
+                    dt = df_sub.index[i]
                     get_yf_logger().debug(f"Adding out-of-range {data_col} @ {dt.date()} in new prices row of NaNs")
                     empty_row = _pd.DataFrame(data=empty_row_data, index=[dt])
                     df_main = _pd.concat([df_main, empty_row], sort=True)
-        df_main = df_main.sort_index()
-
-        # Re-calculate indices
-        indices = _np.searchsorted(_np.append(df_main.index, df_main.index[-1] + td), df_sub.index, side='right')
-        indices -= 1  # Convert from [[i-1], [i]) to [[i], [i+1])
-        # Numpy.searchsorted does not handle out-of-range well, so handle manually:
-        for i in range(len(df_sub.index)):
-            dt = df_sub.index[i]
-            if dt < df_main.index[0] or dt >= df_main.index[-1] + td:
-                # Out-of-range
-                indices[i] = -1
+            else:
+                # Else, only add out-of-range event dates if occurring in interval 
+                # immediately after last price row
+                last_dt = df_main.index[-1]
+                next_interval_start_dt = last_dt + td
+                next_interval_end_dt = next_interval_start_dt + td
+                for i in _np.where(f_outOfRange)[0]:
+                    dt = df_sub.index[i]
+                    if next_interval_start_dt <= dt < next_interval_end_dt:
+                        new_dt = next_interval_start_dt
+                        get_yf_logger().debug(f"Adding out-of-range {data_col} @ {dt.date()} in new prices row of NaNs")
+                        empty_row = _pd.DataFrame(data=empty_row_data, index=[dt])
+                        df_main = _pd.concat([df_main, empty_row], sort=True)
+            df_main = df_main.sort_index()
+
+            # Re-calculate indices
+            indices = _np.searchsorted(_np.append(df_main.index, df_main.index[-1] + td), df_sub.index, side='right')
+            indices -= 1  # Convert from [[i-1], [i]) to [[i], [i+1])
+            # Numpy.searchsorted does not handle out-of-range well, so handle manually:
+            for i in range(len(df_sub.index)):
+                dt = df_sub.index[i]
+                if dt < df_main.index[0] or dt >= df_main.index[-1] + td:
+                    # Out-of-range
+                    indices[i] = -1
 
     f_outOfRange = indices == -1
     if f_outOfRange.any():