#!/usr/bin/python # # $Id$ # # Copyright (C) 2014 Dragon Research Labs ("DRL") # Portions copyright (C) 2013 Internet Systems Consortium ("ISC") # # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notices and this permission notice appear in all copies. # # THE SOFTWARE IS PROVIDED "AS IS" AND DRL AND ISC DISCLAIM ALL # WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL OR # ISC BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL # DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA # OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER # TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR # PERFORMANCE OF THIS SOFTWARE. """ Debian/Ubuntu package build tool, based on pbuilder-dist and reprepro. """ import os import sys import time import fcntl import errno import socket import logging import argparse import subprocess from textwrap import dedent rpki_packages = ("rpki-rp", "rpki-ca") rpki_source_package = "rpki" parser = argparse.ArgumentParser(description = __doc__, formatter_class = argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("--debug", action = "store_true", help = "enable debugging code") parser.add_argument("--update-build-after", type = int, default = 7 * 24 * 60 * 60, help = "interval (in seconds) after which we should update the pbuilder environment") parser.add_argument("--lockfile", default = os.path.expanduser("~/builder.lock"), help = "avoid collisions between multiple instances of this script") parser.add_argument("--keyring", default = os.path.expanduser("~/.gnupg/pubring.gpg"), help = "PGP keyring") parser.add_argument("--git-tree", default = os.path.expanduser("~/source/master/"), help = "git tree") parser.add_argument("--apt-tree", default = os.path.expanduser("~/repository/"), help = "reprepro repository") parser.add_argument("--url-host", default = "download.rpki.net", help = "hostname of public web server") parser.add_argument("--url-scheme", default = "http", help = "URL scheme of public web server") parser.add_argument("--url-path", default = "/APT", help = "path of apt repository on public web server") parser.add_argument("--backports", nargs = "+", default = ["python-django", "python-tornado"], help = "backports needed for this build") parser.add_argument("--releases", nargs = "+", default = ["debian/jessie", "ubuntu/xenial"], help = "releases for which to build") args = parser.parse_args() # Maybe logging should be configurable too. Later. logging.basicConfig(level = logging.INFO, timefmt = "%Y-%m-%dT%H:%M:%S", format = "%(asctime)s [%(process)d] %(levelname)s %(message)s") upload = socket.getfqdn() == "build-u.rpki.net" def run(*cmd, **kwargs): if args.debug: #logging.info("Running %r %r", cmd, kwargs) logging.info("Running %s", " ".join(cmd)) subprocess.check_call(cmd, **kwargs) # Getting this to work right also required adding: # # DEBBUILDOPTS="-b" # # to /etc/pbuilderrc; without this, reprepro (eventually, a year after # we set this up) started failing to incorporate some of the built # packages, because the regenerated source packages had different # checksums than the ones loaded initially. See: # # http://stackoverflow.com/questions/21563872/reprepro-complains-about-the-generated-pbuilder-debian-tar-gz-archive-md5 # # Putting stuff in ~/.pbuilderrc didn't work with pbuilder-dist when I # tried it last year, this may just be that sudo isn't configured to # pass HOME through, thus pbuilder is looking for ~root/.pbuilderrc. # Worth trying again at some point but not all that critical. logging.info("Starting") try: lock = os.open(args.lockfile, os.O_RDONLY | os.O_CREAT | os.O_NONBLOCK, 0666) fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) except (IOError, OSError), e: sys.exit(0 if e.errno == errno.EAGAIN else "Error {!r} opening lock {!r}".format(e, args.lockfile)) run("git", "fetch", "--all", "--prune", cwd = args.git_tree) run("git", "pull", cwd = args.git_tree) run("git", "verify-commit", "HEAD", cwd = args.git_tree) source_version = subprocess.check_output((sys.executable, os.path.join(args.git_tree, "buildtools/make-version.py"), "--build-tag", "--stdout"), cwd = args.git_tree).strip() assert source_version.startswith("buildbot-") source_version = source_version[len("buildbot-"):].replace("-", ".") logging.info("Source version is %s", source_version) if not args.debug: try: run("git", "diff-index", "--quiet", "HEAD", cwd = args.git_tree) except subprocess.CalledProcessError: sys.exit("Sources don't look pristine, not building ({!r})".format(source_version)) search_version = "_" + source_version + "~" dsc_dir = os.path.abspath(os.path.join(args.git_tree, "..")) if not os.path.isdir(args.apt_tree): logging.info("Creating %s", args.apt_tree) os.makedirs(args.apt_tree) fn = os.path.join(args.apt_tree, "apt-gpg-key.asc") if not os.path.exists(fn): logging.info("Creating %s", fn) run("gpg", "--export", "--armor", "--keyring", args.keyring, stdout = open(fn, "w")) class Release(object): architectures = dict(amd64 = "", i386 = "-i386") releases = [] packages = {} def __init__(self, distribution_release, backports): self.distribution, self.release = distribution_release.split("/") self.backports = backports self.apt_source = "{scheme}://{host}/{path}/{distribution} {release} main".format( scheme = args.url_scheme, host = args.url_host, path = args.url_path.strip("/"), distribution = self.distribution, release = self.release) self.env = dict(os.environ) if backports: self.env.update(OTHERMIRROR = "deb " + self.apt_source) self.releases.append(self) @classmethod def do_all_releases(cls): for release in cls.releases: release.setup_reprepro() for release in cls.releases: release.list_repository() for release in cls.releases: for release.arch, release.tag in cls.architectures.iteritems(): release.do_one_architecture() del release.arch, release.tag @staticmethod def repokey(release, architecture, package): return (release, architecture, package) def list_repository(self): cmd = ("reprepro", "list", self.release) logging.info("Running %s", " ".join(cmd)) listing = subprocess.check_output(cmd, cwd = self.tree) for line in listing.replace(":", " ").replace("|", " ").splitlines(): rel, comp, arch, pkg, ver = line.split() key = (rel, arch, pkg) assert key not in self.packages self.packages[key] = ver @property def deb_in_repository(self): ret = all(self.packages.get((self.release, self.arch, package)) == self.version for package in rpki_packages) #logging.info("Got %s looking for %r in %r", ret, self.version, self.packages) return ret @property def src_in_repository(self): ret = self.packages.get((self.release, "source", rpki_source_package)) == self.version #logging.info("Got %s looking for %r in %r", ret, self.version, self.packages) return ret @property def version(self): return source_version + "~" + self.release @property def dsc(self): return os.path.join(dsc_dir, "rpki_{}.dsc".format(self.version)) @property def tree(self): return os.path.join(args.apt_tree, self.distribution, "") @property def basefile(self): return os.path.expanduser("~/pbuilder/{0.release}{0.tag}-base.tgz".format(self)) @property def result(self): return os.path.expanduser("~/pbuilder/{0.release}{0.tag}_result".format(self)) @property def changes(self): return os.path.join(self.result, "rpki_{0.version}_{0.arch}.changes".format(self)) def do_one_architecture(self): logging.info("Running build for %s %s %s", self.distribution, self.release, self.arch) if not os.path.exists(self.dsc): logging.info("%s does not exist", self.dsc) for fn in os.listdir(dsc_dir): fn = os.path.join(dsc_dir, fn) if not os.path.isdir(fn) and search_version not in fn: logging.info("Removing %s", fn) os.unlink(fn) run("rm", "-rf", "debian", cwd = args.git_tree) logging.info("Building source package %s", self.version) run(sys.executable, "buildtools/build-debian-packages.py", "--version-suffix", self.release, cwd = args.git_tree) run("dpkg-buildpackage", "-S", "-us", "-uc", "-rfakeroot", cwd = args.git_tree) if not os.path.exists(self.basefile): logging.info("Creating build environment %s %s", self.release, self.arch) run("pbuilder-dist", self.release, self.arch, "create", env = self.env) elif time.time() > os.stat(self.basefile).st_mtime + args.update_build_after: logging.info("Updating build environment %s %s", self.release, self.arch) run("pbuilder-dist", self.release, self.arch, "update", env = self.env) if not self.deb_in_repository: for fn in os.listdir(self.result): fn = os.path.join(self.result, fn) logging.info("Removing %s", fn) os.unlink(fn) logging.info("Building binary packages %s %s %s", self.release, self.arch, self.version) run("pbuilder-dist", self.release, self.arch, "build", "--keyring", args.keyring, self.dsc, env = self.env) logging.info("Updating repository for %s %s %s", self.release, self.arch, self.version) run("reprepro", "--ignore=wrongdistribution", "include", self.release, self.changes, cwd = self.tree) if not self.src_in_repository: logging.info("Updating repository for %s source %s", self.release, self.version) run("reprepro", "--ignore=wrongdistribution", "includedsc", self.release, self.dsc, cwd = self.tree) def setup_reprepro(self): logging.info("Configuring reprepro for %s/%s", self.distribution, self.release) dn = os.path.join(self.tree, "conf") if not os.path.isdir(dn): logging.info("Creating %s", dn) os.makedirs(dn) fn = os.path.join(self.tree, "conf", "distributions") distributions = open(fn, "r").read() if os.path.exists(fn) else "" if "Codename: {0.release}\n".format(self) not in distributions: logging.info("%s %s", "Editing" if distributions else "Creating", fn) with open(fn, "w") as f: if distributions: f.write(distributions) f.write("\n") f.write(dedent("""\ Origin: rpki.net Label: rpki.net {self.distribution} repository Codename: {self.release} Architectures: {architectures} source Components: main Description: rpki.net {Distribution} APT Repository SignWith: yes DebOverride: override.{self.release} DscOverride: override.{self.release} """.format( self = self, Distribution = self.distribution.capitalize(), architectures = " ".join(self.architectures)))) fn = os.path.join(self.tree, "conf", "options") if not os.path.exists(fn): logging.info("Creating %s", fn) with open(fn, "w") as f: f.write(dedent("""\ verbose ask-passphrase basedir . """)) fn = os.path.join(self.tree, "conf", "override." + self.release) if not os.path.exists(fn): logging.info("Creating %s", fn) with open(fn, "w") as f: for pkg in self.backports: f.write(dedent("""\ {pkg:<30} Priority optional {pkg:<30} Section python """.format(pkg = pkg))) f.write(dedent("""\ rpki-ca Priority extra rpki-ca Section net rpki-rp Priority extra rpki-rp Section net """)) fn = os.path.join(args.apt_tree, "rpki.{}.list".format(self.release)) if not os.path.exists(fn): logging.info("Creating %s", fn) with open(fn, "w") as f: f.write(dedent("""\ deb {self.apt_source} deb-src {self.apt_source} """.format(self = self))) # Load whatever releases the user specified for r in args.releases: Release(r, args.backports) # Do all the real work. Release.do_all_releases() # Push any tags created above to the public git repository. if upload: run("git", "push", "--tags", cwd = args.git_tree) # Upload results, maybe. We do this in two stages, to minimize the window # during which the uploaded repository might be in an inconsistent state. def rsync(*flags): cmd = ["rsync", "--archive", "--itemize-changes", "--rsh", "ssh"] cmd.extend(flags) cmd.append(args.apt_tree) cmd.append("rsync://{host}/{path}/".format(host = args.url_host, path = args.url_path.strip("/"))) if upload: logging.info("Synching repository to %s with flags %s", cmd[-1], " ".join(flags)) run(*cmd) else: logging.info("Would have synched repository to %s with flags %s", cmd[-1], " ".join(flags)) rsync("--ignore-existing") rsync("--exclude", "HEADER.html", "--exclude", "HEADER.css", "--delete", "--delete-delay") logging.info("Done")