diff --git a/tests/lib/TestCase.php b/tests/lib/TestCase.php
index aa091b98113..0c90b7fb917 100644
--- a/tests/lib/TestCase.php
+++ b/tests/lib/TestCase.php
@@ -96,7 +96,9 @@ abstract class TestCase extends \PHPUnit\Framework\TestCase {
 					return $oldService;
 				});
 			} else {
-				unset($container[$oldService]);
+				// The service was not registered before the test override.
+				// Remove the test registration so the container returns to its prior state.
+				unset($container[$name]);
 			}
 
 			unset($this->services[$name]);