From b2508bcba5e90a136a0cb3119e01519baeac8f79 Mon Sep 17 00:00:00 2001 From: Jo-Philipp Wich Date: Mon, 3 Feb 2020 16:50:06 +0100 Subject: [PATCH] phase1, phase2: implement force scheduler logic Signed-off-by: Jo-Philipp Wich --- phase1/master.cfg | 152 ++++++++++++++++++++++++++++++++++++++++++---- phase2/master.cfg | 90 ++++++++++++++++++++++++--- 2 files changed, 222 insertions(+), 20 deletions(-) diff --git a/phase1/master.cfg b/phase1/master.cfg index 24262ad..83d9e8b 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -205,17 +205,107 @@ c['change_source'].append(GitPoller( from buildbot.schedulers.basic import SingleBranchScheduler from buildbot.schedulers.forcesched import ForceScheduler +from buildbot.schedulers.forcesched import BaseParameter +from buildbot.schedulers.forcesched import ValidationError +from buildbot.plugins import schedulers +from buildbot.plugins import util from buildbot.changes import filter + +class TagChoiceParameter(BaseParameter): + spec_attributes = ["strict", "choices"] + type = "list" + strict = True + + def __init__(self, name, label=None, **kw): + super().__init__(name, label, **kw) + self._choice_list = [] + + @property + def choices(self): + taglist = [] + basever = re.search(r'-([0-9]+\.[0-9]+)$', repo_branch) + + if basever: + findtags = subprocess.Popen( + ['git', 'ls-remote', '--tags', repo_url], + stdout = subprocess.PIPE) + + while True: + line = findtags.stdout.readline() + + if not line: + break + + tagver = re.search(r'\brefs/tags/v([0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', line.decode().strip()) + + if tagver and tagver[1].find(basever[1]) == 0: + taglist.append(tagver[1]) + + taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z') + taglist.insert(0, '') + + self._choice_list = taglist + + return self._choice_list + + def parse_from_arg(self, s): + if self.strict and s not in self._choice_list: + raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list)) + return s + c['schedulers'] = [] c['schedulers'].append(SingleBranchScheduler( - name="all", - change_filter=filter.ChangeFilter(branch=repo_branch), - treeStableTimer=60, - builderNames=targets)) + name = "all", + change_filter = filter.ChangeFilter(branch=repo_branch), + treeStableTimer = 60, + builderNames = targets)) c['schedulers'].append(ForceScheduler( - name="force", - builderNames=targets)) + name = "force", + buttonName = "Force builds", + label = "Force build details", + builderNames = [ "00_force_build" ], + + codebases = [ + util.CodebaseParameter( + "", + label = "Repository", + branch = util.FixedParameter(name = "branch", default = ""), + revision = util.FixedParameter(name = "revision", default = ""), + repository = util.FixedParameter(name = "repository", default = ""), + project = util.FixedParameter(name = "project", default = "") + ) + ], + + reason = util.StringParameter( + name = "reason", + label = "Reason", + default = "Trigger build", + required = True, + size = 80 + ), + + properties = [ + util.NestedParameter( + name="options", + label="Build Options", + layout="vertical", + fields=[ + util.ChoiceStringParameter( + name = "target", + label = "Build target", + default = "all", + choices = [ "all" ] + targets + ), + TagChoiceParameter( + name = "tag", + label = "Build tag", + default = "" + ) + ] + ) + ] +)) ####### BUILDERS @@ -231,8 +321,11 @@ from buildbot.steps.transfer import FileUpload from buildbot.steps.transfer import FileDownload from buildbot.steps.transfer import StringDownload from buildbot.steps.master import MasterShellCommand +from buildbot.plugins import steps from buildbot.process.properties import Interpolate +from buildbot.process.properties import Property from buildbot.process import properties +from buildbot.config import BuilderConfig CleanTargetMap = [ @@ -283,7 +376,7 @@ def IsGitCleanRequested(step): def IsTaggingRequested(step): val = step.getProperty("tag") - if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val): + if val and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val): return True else: return False @@ -295,7 +388,7 @@ def IsNoMasterBuild(step): return repo_branch != "master" def GetBaseVersion(): - if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch): + if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", repo_branch): return repo_branch.split('-')[1] else: return "master" @@ -303,7 +396,7 @@ def GetBaseVersion(): @properties.renderer def GetVersionPrefix(props): basever = GetBaseVersion() - if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]): + if props.hasProperty("tag") and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]): return "%s/" % props["tag"] elif basever != "master": return "%s-SNAPSHOT/" % basever @@ -390,6 +483,29 @@ def NetLockUl(props): else: return [] +@util.renderer +def TagPropertyValue(props): + if props.hasProperty("options"): + options = props.getProperty("options") + if type(options) is dict: + return options.get("tag") + return None + +def IsTargetSelected(target): + def CheckTargetProperty(step): + try: + options = step.getProperty("options") + if type(options) is dict: + selected_target = options.get("target", "all") + if selected_target != "all" and selected_target != target: + return False + except KeyError: + pass + + return True + + return CheckTargetProperty + def UsignSec2Pub(seckey, comment="untrusted comment: secret key"): try: seckey = base64.b64decode(seckey) @@ -462,6 +578,13 @@ slaveNames = [ ] for slave in c['workers']: slaveNames.append(slave.workername) +force_factory = BuildFactory() + +c['builders'].append(BuilderConfig( + name = "00_force_build", + workernames = slaveNames, + factory = force_factory)) + for target in targets: ts = target.split('/') @@ -1167,10 +1290,17 @@ for target in targets: alwaysRun = True, )) - from buildbot.config import BuilderConfig - c['builders'].append(BuilderConfig(name=target, workernames=slaveNames, factory=factory, nextBuild=GetNextBuild)) + c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % target, builderNames=[ target ])) + force_factory.addStep(steps.Trigger( + name = "trigger_%s" % target, + description = "Triggering %s build" % target, + schedulerNames = [ "trigger_%s" % target ], + set_properties = { "reason": Property("reason"), "tag": TagPropertyValue }, + doStepIf = IsTargetSelected(target) + )) + ####### STATUS TARGETS diff --git a/phase2/master.cfg b/phase2/master.cfg index f5be23e..d3a7614 100644 --- a/phase2/master.cfg +++ b/phase2/master.cfg @@ -195,17 +195,59 @@ def branch_change_filter(change): from buildbot.schedulers.basic import SingleBranchScheduler from buildbot.schedulers.forcesched import ForceScheduler +from buildbot.plugins import schedulers +from buildbot.plugins import util +from buildbot.plugins import steps from buildbot.changes import filter + c['schedulers'] = [] c['schedulers'].append(SingleBranchScheduler( - name="all", - change_filter=filter.ChangeFilter(filter_fn=branch_change_filter), - treeStableTimer=60, - builderNames=archnames)) + name = "all", + change_filter = filter.ChangeFilter(filter_fn=branch_change_filter), + treeStableTimer = 60, + builderNames = archnames)) c['schedulers'].append(ForceScheduler( - name="force", - builderNames=archnames)) + name = "force", + buttonName = "Force builds", + label = "Force build details", + builderNames = [ "00_force_build" ], + + codebases = [ + util.CodebaseParameter( + "", + label = "Repository", + branch = util.FixedParameter(name = "branch", default = ""), + revision = util.FixedParameter(name = "revision", default = ""), + repository = util.FixedParameter(name = "repository", default = ""), + project = util.FixedParameter(name = "project", default = "") + ) + ], + + reason = util.StringParameter( + name = "reason", + label = "Reason", + default = "Trigger build", + required = True, + size = 80 + ), + + properties = [ + util.NestedParameter( + name="options", + label="Build Options", + layout="vertical", + fields=[ + util.ChoiceStringParameter( + name = "architecture", + label = "Build architecture", + default = "all", + choices = [ "all" ] + archnames + ) + ] + ) + ] +)) ####### BUILDERS @@ -221,10 +263,12 @@ from buildbot.steps.transfer import FileDownload from buildbot.steps.transfer import StringDownload from buildbot.steps.master import MasterShellCommand from buildbot.process.properties import WithProperties +from buildbot.process.properties import Property +from buildbot.config import BuilderConfig def GetDirectorySuffix(props): - verpat = re.compile('^([0-9]{2})\.([0-9]{2})(?:\.([0-9]+)(?:-rc([0-9]+))?|-(SNAPSHOT))$') + verpat = re.compile(r'^([0-9]{2})\.([0-9]{2})(?:\.([0-9]+)(?:-rc([0-9]+))?|-(SNAPSHOT))$') if props.hasProperty("release_version"): m = verpat.match(props["release_version"]) if m is not None: @@ -245,6 +289,21 @@ def GetCwd(props): else: return "/" +def IsArchitectureSelected(target): + def CheckArchitectureProperty(step): + try: + options = step.getProperty("options") + if type(options) is dict: + selected_arch = options.get("architecture", "all") + if selected_arch != "all" and selected_arch != target: + return False + except KeyError: + pass + + return True + + return CheckArchitectureProperty + def UsignSec2Pub(seckey, comment="untrusted comment: secret key"): try: seckey = base64.b64decode(seckey) @@ -267,6 +326,13 @@ slaveNames = [ ] for slave in c['workers']: slaveNames.append(slave.workername) +force_factory = BuildFactory() + +c['builders'].append(BuilderConfig( + name = "00_force_build", + workernames = slaveNames, + factory = force_factory)) + for arch in arches: ts = arch[1].split('/') @@ -604,10 +670,16 @@ for arch in arches: alwaysRun = True )) - from buildbot.config import BuilderConfig - c['builders'].append(BuilderConfig(name=arch[0], workernames=slaveNames, factory=factory)) + c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % arch[0], builderNames=[ arch[0] ])) + force_factory.addStep(steps.Trigger( + name = "trigger_%s" % arch[0], + description = "Triggering %s build" % arch[0], + schedulerNames = [ "trigger_%s" % arch[0] ], + set_properties = { "reason": Property("reason") }, + doStepIf = IsArchitectureSelected(arch[0]) + )) ####### STATUS arches -- 2.30.2