Bug fixes for supporting several QA repos

alts/shared/models.py

@@ -238,6 +238,11 @@ class PulpLogsConfig(BaseLogsConfig):
     pulp_password: Optional[str] = None
 
 
+class ThirdPartyRepoSshHost(BaseModel):
+    host: str
+    user: Optional[str] = None
+
+
 class CeleryConfig(BaseModel):
     def __init__(self, **data):
         super().__init__(**data)
@@ -342,6 +347,12 @@ def __init__(self, **data):
     extra_alpine_repo_baseurl: str = ''
     epel_mirror_replacement: str = ''
     git_reference_directory: Optional[str] = None
+    # SSH client config seeded into test VMs so third-party test repos
+    # (gerrit, gitlab) can be cloned. Each entry maps to a `Host` block
+    # in `/root/.ssh/config` on the VM; missing `user` leaves the
+    # default (current SSH user). Configured per-deployment; empty by
+    # default so no `~/.ssh/config` is written unless explicitly set.
+    third_party_repo_ssh_hosts: List[ThirdPartyRepoSshHost] = []
     tests_base_dir: str = '/tests'
     package_proxy: str = ''
     disabled_al_repos: List[str] = []

alts/shared/utils/git_utils.py

@@ -43,7 +43,7 @@ def repo_reference_subpath(repo_url: str) -> str:
             host = scp_match.group('host')
             path = scp_match.group('path')
     if not host or not path:
-        # Unparseable URL — fall back to basename so behaviour stays
+        # Unparsable URL — fall back to basename so behaviour stays
         # predictable; collisions across hosts remain the caller's problem
         # in that degenerate case.
         basename = os.path.basename(repo_url) or 'repo'
@@ -87,19 +87,20 @@ def __clone_git_repo(
     reference_directory: Optional[str] = None,
     cmd_timeout: int = 300,
 ):
-    git_repo_path = Path(
-        work_dir,
-        Path(repo_url).name.replace('.git', ''),
-    )
+    subpath = repo_reference_subpath(repo_url)
+    if subpath.endswith('.git'):
+        subpath = subpath[:-4]
+    git_repo_path = Path(work_dir, subpath)
     if git_repo_path.exists():
         return git_repo_path
+    git_repo_path.parent.mkdir(parents=True, exist_ok=True)
     logger.debug('Cloning the git repo: %s', repo_url)
-    args = ['clone', repo_url, '--depth', '1']
+    args = ['clone', repo_url, str(git_repo_path), '--depth', '1']
     if reference_directory:
         args.extend(
             ['--reference-if-able', get_abspath(reference_directory)]
         )
-    exit_code, stdout, stderr = local['git'].with_cwd(work_dir).run(
+    exit_code, stdout, stderr = local['git'].run(
         args,
         retcode=None,
         timeout=cmd_timeout,

alts/worker/runners/base.py

@@ -747,6 +747,10 @@ def initial_provision(self, verbose=False):
             'pytest_is_needed': self.pytest_is_needed,
             'development_mode': CONFIG.development_mode,
             'package_proxy': CONFIG.package_proxy,
+            'third_party_repo_ssh_hosts': [
+                entry.model_dump(exclude_none=True)
+                for entry in CONFIG.third_party_repo_ssh_hosts
+            ],
         }
         dist_major_version = self.dist_version[0]
         if self.dist_name in CONFIG.rhel_flavors and dist_major_version in ('6', '7'):
@@ -1425,9 +1429,12 @@ def find_and_run_init_tests():
                 if not test_repo_path:
                     errors.append(f'Cannot clone test repository {repo_url}')
                     continue
+                remote_subpath = repo_reference_subpath(repo_url)
+                if remote_subpath.endswith('.git'):
+                    remote_subpath = remote_subpath[:-4]
                 remote_workdir = os.path.join(
                     CONFIG.tests_base_dir,
-                    test_repo_path.name,
+                    remote_subpath,
                     test_dir,
                 )
                 local_workdir = self._work_dir
@@ -1482,9 +1489,12 @@ def find_and_run_init_tests():
             if not test_repo_path:
                 errors.append(f'Cannot clone test repository {repo_url}')
                 continue
+            remote_subpath = repo_reference_subpath(repo_url)
+            if remote_subpath.endswith('.git'):
+                remote_subpath = remote_subpath[:-4]
             remote_workdir = os.path.join(
                 CONFIG.tests_base_dir,
-                test_repo_path.name,
+                remote_subpath,
                 test_dir,
             )
             local_workdir = self._work_dir
@@ -1883,15 +1893,16 @@ def clone_third_party_repo(
         if not git_repo_path:
             return
         if self._ssh_client:
-            repo_path = Path(
-                self._tests_dir,
-                Path(repo_url).name.replace('.git', ''),
-            )
+            subpath = repo_reference_subpath(repo_url)
+            if subpath.endswith('.git'):
+                subpath = subpath[:-4]
+            repo_path = Path(self._tests_dir, subpath)
             result = None
             for attempt in range(1, 6):
                 cmd = (f'if [ -e {repo_path} ]; then cd {repo_path} && '
                        f'git reset --hard origin/master && git checkout master && git pull; '
-                       f'else cd {self._tests_dir} && git clone {repo_url}; fi')
+                       f'else mkdir -p {repo_path.parent} && '
+                       f'git clone {repo_url} {repo_path}; fi')
                 result = self._ssh_client.sync_run_command(cmd)
                 if result.is_successful():
                     break
@@ -1903,10 +1914,6 @@ def clone_third_party_repo(
             if not result or (result and not result.is_successful()):
                 return
 
-            repo_path = Path(
-                self._tests_dir,
-                Path(repo_url).name.replace('.git', ''),
-            )
             command = f'git fetch origin && git checkout {git_ref}'
             if 'gerrit' in repo_url:
                 command = prepare_gerrit_command(git_ref)

alts/worker/runners/docker.py

@@ -28,6 +28,7 @@
 from alts.worker.executors.bats import BatsExecutor
 from alts.worker.executors.command import CommandExecutor
 from alts.worker.executors.shell import ShellExecutor
+from alts.shared.utils.git_utils import repo_reference_subpath
 from alts.worker.runners.base import (
     TESTS_SECTION_NAME,
     BaseRunner,
@@ -376,10 +377,15 @@ def clone_third_party_repo(
             return
         self._logger.info('Copying tests to container')
         self._logger.debug('Repo path: %s', test_repo_path)
-        self.exec_command('mkdir', '-p', CONFIG.tests_base_dir)
+        remote_subpath = repo_reference_subpath(repo_url)
+        if remote_subpath.endswith('.git'):
+            remote_subpath = remote_subpath[:-4]
+        remote_repo_path = f'{CONFIG.tests_base_dir}/{remote_subpath}'
+        remote_parent = os.path.dirname(remote_repo_path)
+        self.exec_command('mkdir', '-p', remote_parent)
         self.copy([
             str(test_repo_path),
-            f'{self.env_name}:{CONFIG.tests_base_dir}/{test_repo_path.name}',
+            f'{self.env_name}:{remote_repo_path}',
         ])
         return test_repo_path
 

resources/roles/preparation/defaults/main.yml

@@ -1,3 +1,8 @@
 ---
 procps_pkg: "procps"
 centos_repo_baseurl: "https://vault.centos.org"
+# Source of truth lives in the alts config
+# (`CelerConfig.third_party_repo_ssh_hosts`) and is passed in as an
+# extra-var by the runner. Empty list here is a no-op fallback so the
+# role still works if invoked standalone (e.g. ansible-lint).
+third_party_repo_ssh_hosts: []

resources/roles/preparation/tasks/main.yml

@@ -21,6 +21,47 @@
   tags:
     - initial_provision
 
+# SSH client config is only needed where the third-party repo is cloned
+# *inside* the environment (VMs cloned over SSH). Docker envs receive the
+# repo via `docker cp` from the worker, so they never clone and need no
+# SSH config.
+- name: Ensure root ~/.ssh directory exists
+  file:
+    path: /root/.ssh
+    state: directory
+    owner: root
+    group: root
+    mode: '0700'
+  when:
+    - connection_type != 'docker'
+    - third_party_repo_ssh_hosts | length > 0
+  tags:
+    - initial_provision
+
+- name: Configure SSH client for third-party repo hosts
+  copy:
+    dest: /root/.ssh/config
+    owner: root
+    group: root
+    mode: '0600'
+    content: |
+      {% for entry in third_party_repo_ssh_hosts %}
+      Host {{ entry.host }}
+      {% if entry.user is defined %}
+          User {{ entry.user }}
+      {% endif %}
+          StrictHostKeyChecking no
+          UserKnownHostsFile /dev/null
+      {% if not loop.last %}
+
+      {% endif %}
+      {% endfor %}
+  when:
+    - connection_type != 'docker'
+    - third_party_repo_ssh_hosts | length > 0
+  tags:
+    - initial_provision
+
 - name: Copy tests to test environment
   copy:
     src: "{{ integrity_tests_dir }}"

tests/runners/test_clone_paths_docker.py

@@ -0,0 +1,85 @@
+"""Container-side path-collision regression tests.
+
+DockerRunner copies the locally-cloned QA tree into the container.
+With the host-aware layout, two QA upstreams must be copied to two
+distinct paths inside the container, matching the layout the test
+runner expects when running tests by ``remote_workdir``.
+"""
+import logging
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import pytest
+
+from alts.worker import CONFIG
+from alts.worker.runners.docker import DockerRunner
+
+
+GERRIT_QA = 'ssh://gerrit.cloudlinux.com:29418/QA'
+GITLAB_QA = 'git@gitlab.corp.cloudlinux.com:clos/ci-tools/QA.git'
+
+
+def _make_runner(monkeypatch, env_name='env-x'):
+    runner = DockerRunner.__new__(DockerRunner)
+    runner._logger = logging.getLogger('test')
+    runner._env_name = env_name  # env_name is a read-only property
+    runner.exec_command = MagicMock(return_value=(0, '', ''))
+    runner.copy = MagicMock(return_value=(0, '', ''))
+
+    fake_local_clone = Path('/tmp/fake-local-clone')
+    monkeypatch.setattr(
+        'alts.worker.runners.base.BaseRunner.clone_third_party_repo',
+        lambda self, repo_url, git_ref: fake_local_clone,
+    )
+    return runner
+
+
+class TestDockerCopyDestinationIsHostAware:
+    def test_gerrit_qa_copy_target(self, monkeypatch):
+        runner = _make_runner(monkeypatch)
+        runner.clone_third_party_repo(GERRIT_QA, 'master')
+
+        expected_dest = (
+            f'env-x:{CONFIG.tests_base_dir}/gerrit.cloudlinux.com/QA'
+        )
+        runner.copy.assert_called_once()
+        copy_args = runner.copy.call_args.args[0]
+        assert copy_args[1] == expected_dest
+
+    def test_gitlab_qa_copy_target(self, monkeypatch):
+        runner = _make_runner(monkeypatch)
+        runner.clone_third_party_repo(GITLAB_QA, 'master')
+
+        expected_dest = (
+            f'env-x:{CONFIG.tests_base_dir}'
+            '/gitlab.corp.cloudlinux.com/clos/ci-tools/QA'
+        )
+        runner.copy.assert_called_once()
+        assert runner.copy.call_args.args[0][1] == expected_dest
+
+    def test_parent_directory_is_created_in_container(self, monkeypatch):
+        runner = _make_runner(monkeypatch)
+        runner.clone_third_party_repo(GITLAB_QA, 'master')
+
+        expected_parent = (
+            f'{CONFIG.tests_base_dir}'
+            '/gitlab.corp.cloudlinux.com/clos/ci-tools'
+        )
+        runner.exec_command.assert_called_once_with(
+            'mkdir', '-p', expected_parent,
+        )
+
+    def test_two_qa_upstreams_get_distinct_container_paths(self, monkeypatch):
+        runner = _make_runner(monkeypatch)
+        runner.clone_third_party_repo(GERRIT_QA, 'master')
+        gerrit_dest = runner.copy.call_args_list[0].args[0][1]
+        runner.copy.reset_mock()
+        runner.exec_command.reset_mock()
+        runner.clone_third_party_repo(GITLAB_QA, 'master')
+        gitlab_dest = runner.copy.call_args_list[0].args[0][1]
+
+        assert gerrit_dest != gitlab_dest
+        # Neither destination is the bare basename path.
+        bad = f'env-x:{CONFIG.tests_base_dir}/QA'
+        assert gerrit_dest != bad
+        assert gitlab_dest != bad