diff --git a/rlkit/core/batch_rl_algorithm.py b/rlkit/core/batch_rl_algorithm.py
index 848c5eb80..7bc328d90 100644
--- a/rlkit/core/batch_rl_algorithm.py
+++ b/rlkit/core/batch_rl_algorithm.py
@@ -40,6 +40,7 @@ def __init__(
         self.num_train_loops_per_epoch = num_train_loops_per_epoch
         self.num_expl_steps_per_train_loop = num_expl_steps_per_train_loop
         self.min_num_steps_before_training = min_num_steps_before_training
+        gt.reset_root()
 
     def _train(self):
         if self.min_num_steps_before_training > 0: