Merge pull request #2008 from batrick/fragment-merge-base-config

teuthology/suite/merge.py

@@ -5,6 +5,7 @@
 from types import MappingProxyType
 import yaml
 
+from teuthology.config import JobConfig
 from teuthology.suite.build_matrix import combine_path
 from teuthology.suite.util import strip_fragment_path
 from teuthology.misc import deep_merge
@@ -115,6 +116,7 @@ def config_merge(configs, suite_name=None, **kwargs):
     the entire job (config) from the list.
     """
     seed = kwargs.setdefault('seed', 1)
+    base_config = kwargs.setdefault('base_config', JobConfig())
     if not isinstance(seed, int):
         log.debug("no valid seed input: using 1")
         seed = 1
@@ -128,7 +130,7 @@ def config_merge(configs, suite_name=None, **kwargs):
         if suite_name is not None:
             desc = combine_path(suite_name, desc)
 
-        yaml_complete_obj = {}
+        yaml_complete_obj = copy.deepcopy(base_config.to_dict())
         deep_merge(yaml_complete_obj, dict(TEUTHOLOGY_TEMPLATE))
         for path in paths:
             if path not in yaml_cache:

teuthology/suite/run.py

@@ -9,7 +9,6 @@
 
 from humanfriendly import format_timespan
 
-from tempfile import NamedTemporaryFile
 from teuthology import repo_utils
 
 from teuthology.config import config, JobConfig
@@ -167,13 +166,16 @@ def choose_kernel(self):
         # Put together a stanza specifying the kernel hash
         if self.args.kernel_branch == 'distro':
             kernel_hash = 'distro'
+            kernel_branch = 'distro'
         # Skip the stanza if '-k none' is given
         elif self.args.kernel_branch is None or \
              self.args.kernel_branch.lower() == 'none':
             kernel_hash = None
+            kernel_branch = None
         else:
+            kernel_branch = self.args.kernel_branch
             kernel_hash = util.get_gitbuilder_hash(
-                'kernel', self.args.kernel_branch, 'default',
+                'kernel', kernel_branch, 'default',
                 self.args.machine_type, self.args.distro,
                 self.args.distro_version,
             )
@@ -189,7 +191,7 @@ def choose_kernel(self):
 
         if kernel_hash:
             log.info("kernel sha1: {hash}".format(hash=kernel_hash))
-            kernel_dict = dict(kernel=dict(kdb=kdb, sha1=kernel_hash))
+            kernel_dict = dict(kernel=dict(branch=kernel_branch, kdb=kdb, sha1=kernel_hash))
             if kernel_hash != 'distro':
                 kernel_dict['kernel']['flavor'] = 'default'
         else:
@@ -622,6 +624,9 @@ def schedule_suite(self):
         log.debug('Suite %s in %s' % (suite_name, suite_path))
         log.debug(f"subset = {self.args.subset}")
         log.debug(f"no_nested_subset = {self.args.no_nested_subset}")
+        if self.args.dry_run:
+            log.debug("Base job config:\n%s" % self.base_config)
+
         configs = build_matrix(suite_path,
                                subset=self.args.subset,
                                no_nested_subset=self.args.no_nested_subset,
@@ -633,20 +638,10 @@ def schedule_suite(self):
             filter_out=self.args.filter_out,
             filter_all=self.args.filter_all,
             filter_fragments=self.args.filter_fragments,
+            base_config=self.base_config,
             seed=self.args.seed,
             suite_name=suite_name))
 
-        if self.args.dry_run:
-            log.debug("Base job config:\n%s" % self.base_config)
-
-        # create, but do not write, the temp file here, so it can be
-        # added to the args in collect_jobs, but not filled until
-        # any backtracking is done
-        base_yaml_path = NamedTemporaryFile(
-            prefix='schedule_suite_', delete=False
-        ).name
-        self.base_yaml_paths.insert(0, base_yaml_path)
-
         # compute job limit in respect of --sleep-before-teardown
         job_limit = self.args.limit or 0
         sleep_before_teardown = int(self.args.sleep_before_teardown or 0)
@@ -711,9 +706,6 @@ def schedule_suite(self):
                     dry_run=self.args.dry_run,
                 )
 
-        with open(base_yaml_path, 'w+b') as base_yaml:
-            base_yaml.write(str(self.base_config).encode())
-
         if jobs_to_schedule:
             self.write_rerun_memo()
 
@@ -725,8 +717,6 @@ def schedule_suite(self):
 
         self.schedule_jobs(jobs_missing_packages, jobs_to_schedule, name)
 
-        os.remove(base_yaml_path)
-
         count = len(jobs_to_schedule)
         missing_count = len(jobs_missing_packages)
         total_count = count

teuthology/suite/test/test_run_.py

@@ -1,3 +1,4 @@
+import logging
 import os
 import pytest
 import requests
@@ -14,6 +15,7 @@
 from teuthology.suite import run
 from teuthology.util.time import TIMESTAMP_FMT
 
+log = logging.getLogger(__name__)
 
 class TestRun(object):
     klass = run.Run
@@ -270,14 +272,35 @@ def test_successful_schedule(
             [call('ceph_sha1', 'default', 'ubuntu', '14.04', 'machine_type')],
         )
         y = {
-          'teuthology': {
-            'fragments_dropped': [],
-            'meta': {},
-            'postmerge': []
-          },
           'field1': 'val1',
           'field2': 'val2'
         }
+        teuthology_keys = [
+          'branch',
+          'machine_type',
+          'name',
+          'os_type',
+          'os_version',
+          'overrides',
+          'priority',
+          'repo',
+          'seed',
+          'sha1',
+          'sleep_before_teardown',
+          'suite',
+          'suite_branch',
+          'suite_relpath',
+          'suite_repo',
+          'suite_sha1',
+          'tasks',
+          'teuthology_branch',
+          'teuthology_sha1',
+          'timestamp',
+          'user',
+          'teuthology',
+        ]
+        for t in teuthology_keys:
+            y[t] = ANY
         expected_job = dict(
             yaml=y,
             sha1='ceph_sha1',
@@ -287,16 +310,23 @@ def test_successful_schedule(
                 '--description',
                 os.path.join(self.args.suite, build_matrix_desc),
                 '--',
-                ANY, # base config
                 '-'
             ],
-            stdin=yaml.dump(y),
+            stdin=ANY,
             desc=os.path.join(self.args.suite, build_matrix_desc),
         )
 
         m_schedule_jobs.assert_has_calls(
             [call([], [expected_job], runobj.name)],
         )
+        args = m_schedule_jobs.call_args.args
+        log.debug("args =\n%s", args)
+        jobargs  = args[1][0]
+        stdin_yaml = yaml.safe_load(jobargs['stdin'])
+        for k in y:
+            assert y[k] == stdin_yaml[k]
+        for k in teuthology_keys:
+            assert k in stdin_yaml
         m_write_rerun_memo.assert_called_once_with()
 
     @patch('teuthology.suite.util.find_git_parents')

teuthology/task/kernel.py

@@ -33,7 +33,7 @@
 
 log = logging.getLogger(__name__)
 
-CONFIG_DEFAULT = {'branch': 'main'}
+CONFIG_DEFAULT = {'branch': 'distro', 'sha1': 'distro'}
 TIMEOUT_DEFAULT = 300
 
 VERSION_KEYS = ['branch', 'tag', 'sha1', 'deb', 'rpm', 'koji', 'koji_task']