IndexError: index 3 is out of bounds for dimension 0 with size 3

[Tagar]

Got this exception -

IndexError: index 3 is out of bounds for dimension 0 with size 3

on a following cell of the intro's notebook

from fastai.vision.all import *
path = untar_data(URLs.PETS)/'images'

def is_cat(x): return x[0].isupper()
dls = ImageDataLoaders.from_name_func(
    path, get_image_files(path), valid_pct=0.2, seed=42,
    label_func=is_cat, item_tfms=Resize(224))

learn = cnn_learner(dls, resnet34, metrics=error_rate)
learn.fine_tune(1)

Full exception:

---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<command-8165228> in <module>
     11 
     12 learn = cnn_learner(dls, resnet34, metrics=error_rate)
---> 13 learn.fine_tune(1)

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastcore/utils.py in _f(*args, **kwargs)
    471         init_args.update(log)
    472         setattr(inst, 'init_args', init_args)
--> 473         return inst if to_return else f(*args, **kwargs)
    474     return _f
    475 

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastai/callback/schedule.py in fine_tune(self, epochs, base_lr, freeze_epochs, lr_mult, pct_start, div, **kwargs)
    159     "Fine tune with `freeze` for `freeze_epochs` then with `unfreeze` from `epochs` using discriminative LR"
    160     self.freeze()
--> 161     self.fit_one_cycle(freeze_epochs, slice(base_lr), pct_start=0.99, **kwargs)
    162     base_lr /= 2
    163     self.unfreeze()

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastcore/utils.py in _f(*args, **kwargs)
    471         init_args.update(log)
    472         setattr(inst, 'init_args', init_args)
--> 473         return inst if to_return else f(*args, **kwargs)
    474     return _f
    475 

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastai/callback/schedule.py in fit_one_cycle(self, n_epoch, lr_max, div, div_final, pct_start, wd, moms, cbs, reset_opt)
    111     scheds = {'lr': combined_cos(pct_start, lr_max/div, lr_max, lr_max/div_final),
    112               'mom': combined_cos(pct_start, *(self.moms if moms is None else moms))}
--> 113     self.fit(n_epoch, cbs=ParamScheduler(scheds)+L(cbs), reset_opt=reset_opt, wd=wd)
    114 
    115 # Cell

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastcore/utils.py in _f(*args, **kwargs)
    471         init_args.update(log)
    472         setattr(inst, 'init_args', init_args)
--> 473         return inst if to_return else f(*args, **kwargs)
    474     return _f
    475 

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastai/learner.py in fit(self, n_epoch, lr, wd, cbs, reset_opt)
    205             self.opt.set_hypers(lr=self.lr if lr is None else lr)
    206             self.n_epoch,self.loss = n_epoch,tensor(0.)
--> 207             self._with_events(self._do_fit, 'fit', CancelFitException, self._end_cleanup)
    208 
    209     def _end_cleanup(self): self.dl,self.xb,self.yb,self.pred,self.loss = None,(None,),(None,),None,None

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastai/learner.py in _with_events(self, f, event_type, ex, final)
    153 
    154     def _with_events(self, f, event_type, ex, final=noop):
--> 155         try:       self(f'before_{event_type}')       ;f()
    156         except ex: self(f'after_cancel_{event_type}')
    157         finally:   self(f'after_{event_type}')        ;final()

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastai/learner.py in _do_fit(self)
    195         for epoch in range(self.n_epoch):
    196             self.epoch=epoch
--> 197             self._with_events(self._do_epoch, 'epoch', CancelEpochException)
    198 
    199     @log_args(but='cbs')

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastai/learner.py in _with_events(self, f, event_type, ex, final)
    153 
    154     def _with_events(self, f, event_type, ex, final=noop):
--> 155         try:       self(f'before_{event_type}')       ;f()
    156         except ex: self(f'after_cancel_{event_type}')
    157         finally:   self(f'after_{event_type}')        ;final()

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastai/learner.py in _do_epoch(self)
    189 
    190     def _do_epoch(self):
--> 191         self._do_epoch_train()
    192         self._do_epoch_validate()
    193 

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastai/learner.py in _do_epoch_train(self)
    181     def _do_epoch_train(self):
    182         self.dl = self.dls.train
--> 183         self._with_events(self.all_batches, 'train', CancelTrainException)
    184 
    185     def _do_epoch_validate(self, ds_idx=1, dl=None):

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastai/learner.py in _with_events(self, f, event_type, ex, final)
    153 
    154     def _with_events(self, f, event_type, ex, final=noop):
--> 155         try:       self(f'before_{event_type}')       ;f()
    156         except ex: self(f'after_cancel_{event_type}')
    157         finally:   self(f'after_{event_type}')        ;final()

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastai/learner.py in all_batches(self)
    159     def all_batches(self):
    160         self.n_iter = len(self.dl)
--> 161         for o in enumerate(self.dl): self.one_batch(*o)
    162 
    163     def _do_one_batch(self):

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastai/learner.py in one_batch(self, i, b)
    177         self.iter = i
    178         self._split(b)
--> 179         self._with_events(self._do_one_batch, 'batch', CancelBatchException)
    180 
    181     def _do_epoch_train(self):

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastai/learner.py in _with_events(self, f, event_type, ex, final)
    153 
    154     def _with_events(self, f, event_type, ex, final=noop):
--> 155         try:       self(f'before_{event_type}')       ;f()
    156         except ex: self(f'after_cancel_{event_type}')
    157         finally:   self(f'after_{event_type}')        ;final()

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastai/learner.py in __call__(self, event_name)
    131     def ordered_cbs(self, event): return [cb for cb in sort_by_run(self.cbs) if hasattr(cb, event)]
    132 
--> 133     def __call__(self, event_name): L(event_name).map(self._call_one)
    134 
    135     def _call_one(self, event_name):

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastcore/foundation.py in map(self, f, *args, **kwargs)
    394              else f.format if isinstance(f,str)
    395              else f.__getitem__)
--> 396         return self._new(map(g, self))
    397 
    398     def filter(self, f, negate=False, **kwargs):

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastcore/foundation.py in _new(self, items, *args, **kwargs)
    340     @property
    341     def _xtra(self): return None
--> 342     def _new(self, items, *args, **kwargs): return type(self)(items, *args, use_list=None, **kwargs)
    343     def __getitem__(self, idx): return self._get(idx) if is_indexer(idx) else L(self._get(idx), use_list=None)
    344     def copy(self): return self._new(self.items.copy())

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastcore/foundation.py in __call__(cls, x, *args, **kwargs)
     49             return x
     50 
---> 51         res = super().__call__(*((x,) + args), **kwargs)
     52         res._newchk = 0
     53         return res

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastcore/foundation.py in __init__(self, items, use_list, match, *rest)
    331         if items is None: items = []
    332         if (use_list is not None) or not _is_array(items):
--> 333             items = list(items) if use_list else _listify(items)
    334         if match is not None:
    335             if is_coll(match): match = len(match)

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastcore/foundation.py in _listify(o)
    244     if isinstance(o, list): return o
    245     if isinstance(o, str) or _is_array(o): return [o]
--> 246     if is_iter(o): return list(o)
    247     return [o]
    248 

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastcore/foundation.py in __call__(self, *args, **kwargs)
    307             if isinstance(v,_Arg): kwargs[k] = args.pop(v.i)
    308         fargs = [args[x.i] if isinstance(x, _Arg) else x for x in self.pargs] + args[self.maxi+1:]
--> 309         return self.fn(*fargs, **kwargs)
    310 
    311 # Cell

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastai/learner.py in _call_one(self, event_name)
    135     def _call_one(self, event_name):
    136         assert hasattr(event, event_name), event_name
--> 137         [cb(event_name) for cb in sort_by_run(self.cbs)]
    138 
    139     def _bn_bias_state(self, with_bias): return norm_bias_params(self.model, with_bias).map(self.opt.state)

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastai/learner.py in <listcomp>(.0)
    135     def _call_one(self, event_name):
    136         assert hasattr(event, event_name), event_name
--> 137         [cb(event_name) for cb in sort_by_run(self.cbs)]
    138 
    139     def _bn_bias_state(self, with_bias): return norm_bias_params(self.model, with_bias).map(self.opt.state)

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastai/callback/core.py in __call__(self, event_name)
     42                (self.run_valid and not getattr(self, 'training', False)))
     43         res = None
---> 44         if self.run and _run: res = getattr(self, event_name, noop)()
     45         if event_name=='after_fit': self.run=True #Reset self.run to True at each end of fit
     46         return res

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastai/callback/schedule.py in before_batch(self)
     84     def __init__(self, scheds): self.scheds = scheds
     85     def before_fit(self): self.hps = {p:[] for p in self.scheds.keys()}
---> 86     def before_batch(self): self._update_val(self.pct_train)
     87 
     88     def _update_val(self, pct):

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastai/callback/schedule.py in _update_val(self, pct)
     87 
     88     def _update_val(self, pct):
---> 89         for n,f in self.scheds.items(): self.opt.set_hyper(n, f(pct))
     90 
     91     def after_batch(self):

/local_disk0/.ephemeral_nfs/envs/pythonEnv-22e37778-b406-4688-8a34-66d9ba762e34/lib/python3.7/site-packages/fastai/callback/schedule.py in _inner(pos)
     67         if pos == 1.: return scheds[-1](1.)
     68         idx = (pos >= pcts).nonzero().max()
---> 69         actual_pos = (pos-pcts[idx]) / (pcts[idx+1]-pcts[idx])
     70         return scheds[idx](actual_pos.item())
     71     return _inner

IndexError: index 3 is out of bounds for dimension 0 with size 3

[Tagar]

also reported in #236

[Tagar]

Same errors were reported multiple times in FastAI forums, for example,
here, here, here, here

[Tagar]

cc @jph00 - Jeremy any chance you can suggest possible workarounds / fixes for this? Thanks!!

[Tagar]

List of some of the conda/pip packages -

...
fastai                    2.0.10                     py_0    fastai
fastbook                  0.0.11             pyh39e3cac_0    fastai
fastcore                  1.0.9                      py_0    fastai
fastprogress              1.0.0              pyh39e3cac_0    fastai
fastscript                1.0.0                         0    fastai
nbdev                     1.0.18                     py_0    fastai
...
pytorch                   1.6.0           py3.7_cuda10.1.243_cudnn7.6.3_0    pytorch
torchvision               0.7.0                py37_cu101    pytorch
...
tensorboard               2.3.0                    pypi_0    pypi
tensorboard-plugin-wit    1.7.0                    pypi_0    pypi
tensorflow                2.3.0                    pypi_0    pypi
tensorflow-base           2.2.0           mkl_py37hd506778_0  
tensorflow-estimator      2.3.0                    pypi_0    pypi
...

[Tagar]

Tried pre-v3 fastai (fastai==1.0.61), but it has its own issues with the version of the 01_intro we have (latest version)

[Tagar]

Error happens a minute while inside of running learn.fine_tune(1) of the intro notebook

[Tagar]

Issue happens inside of the combine_scheds(pos):
https://github.com/fastai/fastai/blob/a07f271ac6a03cd14ff7f8c031c38527e5b238ed/fastai/callback/schedule.py#L60-L71

And values inside of the combine_scheds when exception happens -

{'pos': 1.0000000000000018,
 'idx': tensor(2), 
 'pcts': tensor([0.0000, 0.9900, 1.0000]), 
'scheds': [<fastai.callback.schedule._Annealer object at 0x7fb103503690>, 
<fastai.callback.schedule._Annealer object at 0x7fb10e6a4e90>]}

IndexError: index 3 is out of bounds for dimension 0 with size 3

So the issue is that pcts[] has only 3 elements, and pcts[idx+1] is trying to reach 4th element (beyond last element).

Probable root cause: pos has to be an integer, but it's actually a floating point number
pos = 1.0000000000000018 above. So the following if check fails -

https://github.com/fastai/fastai/blob/a07f271ac6a03cd14ff7f8c031c38527e5b238ed/fastai/callback/schedule.py#L67

The fix seems to be easy - cast pos to int before the comparison.

[youcefjd]

how to fix the issue?