tests: re-use installs in upgrade tests

Tests that use the RedpandaInstaller are bandwidth-intensive and often
take several minutes to complete on account of the download of hundreds
of MBs worth of tarballs.

This commit mitigates this in local ducktape by having all test
containers share a single bind mount. The installer now uses a lock file
to prevent concurrent operations on the mount (e.g. when downloading
binaries, checking to see what binaries exist, etc).

With this commit, regardless of whether in local or clustered ducktape,
we also no longer get rid of downloaded binaries between test runs.
Instead, after a test completes, we just revert any changes to the
original binaries, and leave the rest be.

tests/docker/docker-compose.yml

@@ -42,5 +42,6 @@ services:
     - minio
     volumes:
     - '${BUILD_ROOT}:${BUILD_ROOT}'
+    - '${BUILD_ROOT}/ducktape/redpanda_installs:/opt/redpanda_installs'
     networks:
     - redpanda-test

tests/rptest/services/redpanda.py

@@ -447,7 +447,6 @@ def __init__(self,
                  environment: Optional[dict[str, str]] = None,
                  security: SecurityConfig = SecurityConfig(),
                  node_ready_timeout_s=None,
-                 enable_installer=False,
                  superuser: Optional[SaslCredentials] = None):
         super(RedpandaService, self).__init__(context, num_nodes=num_brokers)
         self._context = context
@@ -456,9 +455,7 @@ def __init__(self,
         self._enable_pp = enable_pp
         self._enable_sr = enable_sr
         self._security = security
-        self._installer: Optional[RedpandaInstaller] = None
-        if enable_installer:
-            self._installer = RedpandaInstaller(self)
+        self._installer: RedpandaInstaller = RedpandaInstaller(self)
 
         if superuser is None:
             superuser = self.SUPERUSER_CREDENTIALS
@@ -601,7 +598,7 @@ def start(self, nodes=None, clean_nodes=True, start_si=True):
                     # Expected usage is that we may install new binaries before
                     # starting the cluster, and installation-cleaning happened
                     # when we started the installer.
-                    self.clean_node(node, clean_installs=False)
+                    self.clean_node(node, preserve_current_install=True)
                 else:
                     self.logger.debug("%s: skip cleaning node" %
                                       self.who_am_i(node))
@@ -1134,7 +1131,7 @@ def decode_backtraces(self):
                 self.logger.exception("Failed to run seastar-addr2line")
 
     def rp_install_path(self):
-        if self._installer and self._installer._started:
+        if self._installer._started:
             # The installer sets up binaries to always use /opt/redpanda.
             return "/opt/redpanda"
         return self._context.globals.get("rp_install_path_root", None)
@@ -1207,7 +1204,10 @@ def clean(self, **kwargs):
         if self._s3client:
             self.delete_bucket_from_si()
 
-    def clean_node(self, node, preserve_logs=False, clean_installs=True):
+    def clean_node(self,
+                   node,
+                   preserve_logs=False,
+                   preserve_current_install=False):
         # These are allow_fail=True to allow for a race where kill_process finds
         # the PID, but then the process has died before it sends the SIGKILL.  This
         # should be safe against actual failures to of the process to stop, because
@@ -1235,9 +1235,11 @@ def clean_node(self, node, preserve_logs=False, clean_installs=True):
                 self.EXECUTABLE_SAVE_PATH):
             node.account.remove(self.EXECUTABLE_SAVE_PATH)
 
-        if clean_installs and self._installer is not None:
-            # Get rid of any installed packages.
-            self._installer.clean(node)
+        if not preserve_current_install or not self._installer._started:
+            # Reset the binaries to use the original binaries.
+            # NOTE: if the installer hasn't been started, there is no
+            # installation to preserve!
+            self._installer.reset_current_install([node])
 
     def remove_local_data(self, node):
         node.account.remove(f"{RedpandaService.PERSISTENT_ROOT}/data/*")
@@ -1728,7 +1730,7 @@ def save_executable(self):
         # Any node will do. Even in a mixed-version upgrade test, we should
         # still have the original binaries available.
         node = self.nodes[0]
-        if self._installer and self._installer._started:
+        if self._installer._started:
             head_root_path = self._installer.path_for_version(
                 RedpandaInstaller.HEAD)
             binary = f"{head_root_path}/libexec/redpanda"