From aef8d4aa1e284e45400103fa3526e7a90d4b3690 Mon Sep 17 00:00:00 2001 From: Jo-Philipp Wich Date: Fri, 28 Jun 2019 13:05:58 +0200 Subject: [PATCH] treewide: untangle phase1 and phase2 masters - Move shared scripts into a common scripts/ directory - Move SSH Git clone key directly into the configuration - Update configuration examples Signed-off-by: Jo-Philipp Wich --- .gitignore | 2 ++ phase1/config.ini.example | 15 +++++++++++---- phase1/master.cfg | 23 +++++++++++++++-------- phase2/config.ini.example | 16 +++++++++++++--- phase2/master.cfg | 30 ++++++++++++++++++++++++------ {phase1 => scripts}/ccache.sh | 0 {phase1 => scripts}/dumpinfo.pl | 0 {phase1 => scripts}/expire.sh | 0 {phase1 => scripts}/findbin.pl | 0 {phase1 => scripts}/makebranch.sh | 0 {phase1 => scripts}/rsync.sh | 0 {phase1 => scripts}/sha2rsync.pl | 0 {phase1 => scripts}/signall.sh | 0 13 files changed, 65 insertions(+), 21 deletions(-) rename {phase1 => scripts}/ccache.sh (100%) rename {phase1 => scripts}/dumpinfo.pl (100%) rename {phase1 => scripts}/expire.sh (100%) rename {phase1 => scripts}/findbin.pl (100%) rename {phase1 => scripts}/makebranch.sh (100%) rename {phase1 => scripts}/rsync.sh (100%) rename {phase1 => scripts}/sha2rsync.pl (100%) rename {phase1 => scripts}/signall.sh (100%) diff --git a/.gitignore b/.gitignore index ee72b86..6eb6b27 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ * !.gitignore +!scripts +!scripts/* !phase[12] !phase[12]/* phase[12]/*/* diff --git a/phase1/config.ini.example b/phase1/config.ini.example index f1b677c..7032806 100644 --- a/phase1/config.ini.example +++ b/phase1/config.ini.example @@ -1,12 +1,19 @@ [general] -title = LEDE Project -title_url = http://lede-project.org/ -buildbot_url = http://phase1.builds.lede-project.org/ +title = OpenWrt Project +title_url = http://openwrt.org/ +buildbot_url = http://phase1.builds.openwrt.org/ homedir = . expire = 1209600 port = 9989 cc_version = le 4.9 git_ssh = true +git_ssh_key = -----BEGIN RSA PRIVATE KEY----- + MIIEpAIBAAKCAQEAuCJwo6OmrRDxcGfsMgBhq0vdzp2ZIdqnedFH8u6tVYLt9WDU + ... + mHzkh8Uv4OAWTjiLGycbXa0/31hu9PCeNzYmjjrp8tcGjsiJJFxydgS+wc0i2UPV + nSI+JbmAAF9vw6gj2i+Hqx7UloRd0tEv/leX354T5lO06LMiNhvN9g== + -----END RSA PRIVATE KEY----- + [status] bind = tcp:8010:interface=127.0.0.1 @@ -21,7 +28,7 @@ nickname = example-builder password = example [repo] -url = https://git.lede-project.org/source.git +url = https://git.openwrt.org/openwrt/openwrt.git branch = master [rsync] diff --git a/phase1/master.cfg b/phase1/master.cfg index edb4907..7521011 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -94,6 +94,7 @@ c['logHorizon'] = 20 ####### CHANGESOURCES home_dir = os.path.abspath(ini.get("general", "homedir")) +scripts_dir = os.path.abspath("../scripts") tree_expire = 0 other_builds = 0 cc_version = None @@ -102,6 +103,7 @@ cc_command = "gcc" cxx_command = "g++" git_ssh = False +git_ssh_key = None if ini.has_option("general", "expire"): tree_expire = ini.getint("general", "expire") @@ -117,6 +119,11 @@ if ini.has_option("general", "cc_version"): if ini.has_option("general", "git_ssh"): git_ssh = ini.getboolean("general", "git_ssh") +if ini.has_option("general", "git_ssh_key"): + git_ssh_key = ini.get("general", "git_ssh_key") +else: + git_ssh = False + repo_url = ini.get("repo", "url") repo_branch = "master" @@ -163,7 +170,7 @@ if not os.path.isdir(home_dir+'/source.git'): else: subprocess.call(["git", "pull"], cwd = home_dir+'/source.git') -findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'], +findtargets = subprocess.Popen([scripts_dir + '/dumpinfo.pl', 'targets'], stdout = subprocess.PIPE, cwd = home_dir+'/source.git') while True: @@ -453,7 +460,7 @@ for target in targets: if cc_version is not None: factory.addStep(FileDownload( name = "dlfindbinpl", - mastersrc = "findbin.pl", + mastersrc = scripts_dir + '/findbin.pl', slavedest = "../findbin.pl", mode = 0755)) @@ -486,7 +493,7 @@ for target in targets: factory.addStep(FileDownload( name = "dlexpiresh", doStepIf = IsExpireRequested, - mastersrc = "expire.sh", + mastersrc = scripts_dir + '/expire.sh', slavedest = "../expire.sh", mode = 0755)) @@ -628,9 +635,9 @@ for target in targets: # Git SSH if git_ssh: - factory.addStep(FileDownload( + factory.addStep(StringDownload( name = "dlgitclonekey", - mastersrc = "git-clone.key", + s = git_ssh_key, slavedest = "../git-clone.key", mode = 0600, )) @@ -923,7 +930,7 @@ for target in targets: factory.addStep(MasterShellCommand( name = "signfiles", description = "Signing files", - command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment], + command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment], env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile}, haltOnFailure = True )) @@ -990,7 +997,7 @@ for target in targets: # build list of files to upload factory.addStep(FileDownload( name = "dlsha2rsyncpl", - mastersrc = "sha2rsync.pl", + mastersrc = scripts_dir + '/sha2rsync.pl', slavedest = "../sha2rsync.pl", mode = 0755, )) @@ -1004,7 +1011,7 @@ for target in targets: factory.addStep(FileDownload( name = "dlrsync.sh", - mastersrc = "rsync.sh", + mastersrc = scripts_dir + '/rsync.sh', slavedest = "../rsync.sh", mode = 0755 )) diff --git a/phase2/config.ini.example b/phase2/config.ini.example index 2df80f8..991a835 100644 --- a/phase2/config.ini.example +++ b/phase2/config.ini.example @@ -1,17 +1,27 @@ [general] -title = LEDE Project -title_url = http://lede-project.org/ -buildbot_url = http://phase2.builds.lede-project.org/ +title = OpenWrt Project +title_url = http://openwrt.org/ +buildbot_url = http://phase2.builds.openwrt.org/ homedir = ../phase1 port = 9990 persistent = false git_ssh = true +git_ssh_key = -----BEGIN RSA PRIVATE KEY----- + MIIEpAIBAAKCAQEAuCJwo6OmrRDxcGfsMgBhq0vdzp2ZIdqnedFH8u6tVYLt9WDU + ... + mHzkh8Uv4OAWTjiLGycbXa0/31hu9PCeNzYmjjrp8tcGjsiJJFxydgS+wc0i2UPV + nSI+JbmAAF9vw6gj2i+Hqx7UloRd0tEv/leX354T5lO06LMiNhvN9g== + -----END RSA PRIVATE KEY----- [status] bind = tcp:8011:interface=127.0.0.1 user = example password = example +[repo] +url = https://git.openwrt.org/openwrt/openwrt.git +branch = master + [rsync] binary_url = user@example.org::upload-packages binary_password = example diff --git a/phase2/master.cfg b/phase2/master.cfg index 58f1b9f..bb0afaf 100644 --- a/phase2/master.cfg +++ b/phase2/master.cfg @@ -32,6 +32,7 @@ persistent = False other_builds = 0 tree_expire = 0 git_ssh = False +git_ssh_key = None if ini.has_option("general", "port"): slave_port = ini.getint("general", "port") @@ -48,6 +49,11 @@ if ini.has_option("general", "expire"): if ini.has_option("general", "git_ssh"): git_ssh = ini.getboolean("general", "git_ssh") +if ini.has_option("general", "git_ssh_key"): + git_ssh_key = ini.get("general", "git_ssh_key") +else: + git_ssh = False + c['slaves'] = [] max_builds = dict() @@ -76,6 +82,7 @@ c['logHorizon'] = 20 ####### CHANGESOURCES home_dir = os.path.abspath(ini.get("general", "homedir")) +scripts_dir = os.path.abspath("../scripts") rsync_bin_url = ini.get("rsync", "binary_url") rsync_bin_key = ini.get("rsync", "binary_password") @@ -117,12 +124,23 @@ if ini.has_option("gpg", "comment"): if ini.has_option("gpg", "passfile"): gpg_passfile = ini.get("gpg", "passfile") +repo_url = ini.get("repo", "url") +repo_branch = "master" + +if ini.has_option("repo", "branch"): + repo_branch = ini.get("repo", "branch") + # find arches arches = [ ] archnames = [ ] -findarches = subprocess.Popen([home_dir+'/dumpinfo.pl', 'architectures'], +if not os.path.isdir(home_dir+'/source.git'): + subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git']) +else: + subprocess.call(["git", "pull"], cwd = home_dir+'/source.git') + +findarches = subprocess.Popen([scripts_dir + '/dumpinfo.pl', 'architectures'], stdout = subprocess.PIPE, cwd = home_dir+'/source.git') while True: @@ -266,7 +284,7 @@ for arch in arches: # expire tree if needed elif tree_expire > 0: factory.addStep(FileDownload( - mastersrc = home_dir+"/expire.sh", + mastersrc = scripts_dir + '/expire.sh', slavedest = "../expire.sh", mode = 0755)) @@ -333,7 +351,7 @@ for arch in arches: command = ["sh", "-c", "rm -f .config && make defconfig"])) factory.addStep(FileDownload( - mastersrc = home_dir+'/ccache.sh', + mastersrc = scripts_dir + '/ccache.sh', slavedest = 'sdk/ccache.sh', mode = 0755)) @@ -345,9 +363,9 @@ for arch in arches: haltOnFailure = True)) if git_ssh: - factory.addStep(FileDownload( + factory.addStep(StringDownload( name = "dlgitclonekey", - mastersrc = home_dir+"/git-clone.key", + s = git_ssh_key, slavedest = "../git-clone.key", mode = 0600)) @@ -429,7 +447,7 @@ for arch in arches: factory.addStep(MasterShellCommand( name = "signfiles", description = "Signing files", - command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.tar.gz" %(home_dir, arch[0]), gpg_keyid, gpg_comment], + command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.tar.gz" %(home_dir, arch[0]), gpg_keyid, gpg_comment], env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile}, haltOnFailure = True )) diff --git a/phase1/ccache.sh b/scripts/ccache.sh similarity index 100% rename from phase1/ccache.sh rename to scripts/ccache.sh diff --git a/phase1/dumpinfo.pl b/scripts/dumpinfo.pl similarity index 100% rename from phase1/dumpinfo.pl rename to scripts/dumpinfo.pl diff --git a/phase1/expire.sh b/scripts/expire.sh similarity index 100% rename from phase1/expire.sh rename to scripts/expire.sh diff --git a/phase1/findbin.pl b/scripts/findbin.pl similarity index 100% rename from phase1/findbin.pl rename to scripts/findbin.pl diff --git a/phase1/makebranch.sh b/scripts/makebranch.sh similarity index 100% rename from phase1/makebranch.sh rename to scripts/makebranch.sh diff --git a/phase1/rsync.sh b/scripts/rsync.sh similarity index 100% rename from phase1/rsync.sh rename to scripts/rsync.sh diff --git a/phase1/sha2rsync.pl b/scripts/sha2rsync.pl similarity index 100% rename from phase1/sha2rsync.pl rename to scripts/sha2rsync.pl diff --git a/phase1/signall.sh b/scripts/signall.sh similarity index 100% rename from phase1/signall.sh rename to scripts/signall.sh -- 2.30.2