torchrun defaults for concurrent distributed training jobs (#2015)

Author: ebsmothers

torchtune/_cli/run.py

@@ -88,6 +88,12 @@ def _run_distributed(self, args: argparse.Namespace, is_builtin: bool):
         # Have to reset the argv so that the recipe can be run with the correct arguments
         args.training_script = args.recipe
         args.training_script_args = args.recipe_args
+
+        # If the user does not explicitly pass a rendezvous endpoint, run in standalone mode.
+        # This allows running multiple distributed training jobs simultaneously.
+        if not args.rdzv_endpoint:
+            args.standalone = True
+
         # torchtune built-in recipes are specified with an absolute posix path, but
         # custom recipes are specified as a relative module dot path and need to be
         # run with python -m