use deterministic

configs/recipes/smollm/sft/135m/quickstart_train.yaml

@@ -37,7 +37,10 @@ training:
   compile: False
 
   dataloader_num_workers: "auto"
-  dataloader_prefetch_factor: 32
+  dataloader_prefetch_factor:
+
+  seed: 192847
+  use_deterministic: True
 
   logging_steps: 5
   log_model_summary: False

docs/user_guides/train/configuration.md

@@ -145,6 +145,7 @@ training:
   output_dir: "output"                    # Directory for saving outputs
   run_name: null                          # Unique identifier for the run
   seed: 42                                # Random seed for reproducibility
+  use_deterministic: false                # Use deterministic CuDNN algorithms
 
   # Training duration
   num_train_epochs: 3                     # Number of training epochs

src/oumi/cli/train.py

@@ -57,7 +57,9 @@ def train(
 
     limit_per_process_memory()
     device_cleanup()
-    set_random_seeds(parsed_config.training.seed)
+    set_random_seeds(
+        parsed_config.training.seed, parsed_config.training.use_deterministic
+    )
 
     # Run training
     oumi_train(parsed_config)

src/oumi/core/configs/params/training_params.py

@@ -252,6 +252,12 @@ class TrainingParams(BaseParams):
     weight initialization, and any stochastic operations.
     """
 
+    use_deterministic: bool = False
+    """Whether to use deterministic algorithms for reproducibility.
+    If set to True, this will only allow those CuDNN algorithms
+    that are (believed to be) deterministic.
+    """
+
     run_name: Optional[str] = None
     """A unique identifier for the current training run.