diff --git a/src/poetry_plugin_shell/command.py b/src/poetry_plugin_shell/command.py
index 234ac85..fb05ae6 100644
--- a/src/poetry_plugin_shell/command.py
+++ b/src/poetry_plugin_shell/command.py
@@ -28,6 +28,9 @@ class ShellCommand(EnvCommand):
     def handle(self) -> int:
         from poetry_plugin_shell.shell import Shell
 
+        if os.environ.get("POETRY_ACTIVE") == "1" and not os.environ.get("VIRTUAL_ENV"):
+            os.environ.pop("POETRY_ACTIVE")
+
         # Check if it's already activated or doesn't exist and won't be created
         if self._is_venv_activated():
             self.line(
diff --git a/tests/test_shell_command.py b/tests/test_shell_command.py
index 8b05732..c24fa30 100644
--- a/tests/test_shell_command.py
+++ b/tests/test_shell_command.py
@@ -35,7 +35,9 @@ def test_shell(tester: CommandTester, mocker: MockerFixture) -> None:
 
 
 def test_shell_already_active(tester: CommandTester, mocker: MockerFixture) -> None:
+    assert isinstance(tester.command, ShellCommand)
     os.environ["POETRY_ACTIVE"] = "1"
+    os.environ["VIRTUAL_ENV"] = str(tester.command.env.path)
     shell_activate = mocker.patch("poetry_plugin_shell.shell.Shell.activate")
 
     tester.execute()