123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363 |
- #!/usr/bin/python
- #
- # $Id$
- #
- # Copyright (C) 2014 Dragon Research Labs ("DRL")
- # Portions copyright (C) 2013 Internet Systems Consortium ("ISC")
- #
- # Permission to use, copy, modify, and distribute this software for any
- # purpose with or without fee is hereby granted, provided that the above
- # copyright notices and this permission notice appear in all copies.
- #
- # THE SOFTWARE IS PROVIDED "AS IS" AND DRL AND ISC DISCLAIM ALL
- # WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
- # WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL OR
- # ISC BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
- # DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA
- # OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
- # TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
- # PERFORMANCE OF THIS SOFTWARE.
- """
- Debian/Ubuntu package build tool, based on pbuilder-dist and reprepro.
- """
- import os
- import sys
- import time
- import fcntl
- import errno
- import socket
- import logging
- import argparse
- import subprocess
- from textwrap import dedent
- rpki_packages = ("rpki-rp", "rpki-ca")
- rpki_source_package = "rpki"
- parser = argparse.ArgumentParser(description = __doc__,
- formatter_class = argparse.ArgumentDefaultsHelpFormatter)
- parser.add_argument("--debug", action = "store_true",
- help = "enable debugging code")
- parser.add_argument("--update-build-after", type = int, default = 7 * 24 * 60 * 60,
- help = "interval (in seconds) after which we should update the pbuilder environment")
- parser.add_argument("--lockfile", default = os.path.expanduser("~/builder.lock"),
- help = "avoid collisions between multiple instances of this script")
- parser.add_argument("--keyring", default = os.path.expanduser("~/.gnupg/pubring.gpg"),
- help = "PGP keyring")
- parser.add_argument("--git-tree", default = os.path.expanduser("~/source/master/"),
- help = "git tree")
- parser.add_argument("--apt-tree", default = os.path.expanduser("~/repository/"),
- help = "reprepro repository")
- parser.add_argument("--url-host", default = "download.rpki.net",
- help = "hostname of public web server")
- parser.add_argument("--url-scheme", default = "http",
- help = "URL scheme of public web server")
- parser.add_argument("--url-path", default = "/APT",
- help = "path of apt repository on public web server")
- parser.add_argument("--backports", nargs = "+", default = ["python-django", "python-tornado"],
- help = "backports needed for this build")
- parser.add_argument("--releases", nargs = "+", default = ["debian/jessie", "ubuntu/xenial"],
- help = "releases for which to build")
- args = parser.parse_args()
- # Maybe logging should be configurable too. Later.
- logging.basicConfig(level = logging.INFO, timefmt = "%Y-%m-%dT%H:%M:%S",
- format = "%(asctime)s [%(process)d] %(levelname)s %(message)s")
- upload = socket.getfqdn() == "build-u.rpki.net"
- def run(*cmd, **kwargs):
- if args.debug:
- #logging.info("Running %r %r", cmd, kwargs)
- logging.info("Running %s", " ".join(cmd))
- subprocess.check_call(cmd, **kwargs)
- # Getting this to work right also required adding:
- #
- # DEBBUILDOPTS="-b"
- #
- # to /etc/pbuilderrc; without this, reprepro (eventually, a year after
- # we set this up) started failing to incorporate some of the built
- # packages, because the regenerated source packages had different
- # checksums than the ones loaded initially. See:
- #
- # http://stackoverflow.com/questions/21563872/reprepro-complains-about-the-generated-pbuilder-debian-tar-gz-archive-md5
- #
- # Putting stuff in ~/.pbuilderrc didn't work with pbuilder-dist when I
- # tried it last year, this may just be that sudo isn't configured to
- # pass HOME through, thus pbuilder is looking for ~root/.pbuilderrc.
- # Worth trying again at some point but not all that critical.
- logging.info("Starting")
- try:
- lock = os.open(args.lockfile, os.O_RDONLY | os.O_CREAT | os.O_NONBLOCK, 0666)
- fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
- except (IOError, OSError), e:
- sys.exit(0 if e.errno == errno.EAGAIN else "Error {!r} opening lock {!r}".format(e, args.lockfile))
- run("git", "fetch", "--all", "--prune", cwd = args.git_tree)
- run("git", "pull", cwd = args.git_tree)
- source_version = subprocess.check_output((sys.executable, os.path.join(args.git_tree, "buildtools/make-version.py"),
- "--build-tag", "--stdout"), cwd = args.git_tree).strip()
- assert source_version.startswith("buildbot-")
- source_version = source_version[len("buildbot-"):].replace("-", ".")
- logging.info("Source version is %s", source_version)
- if not args.debug:
- try:
- run("git", "diff-index", "--quiet", "HEAD", cwd = args.git_tree)
- except subprocess.CalledProcessError:
- sys.exit("Sources don't look pristine, not building ({!r})".format(source_version))
- search_version = "_" + source_version + "~"
- dsc_dir = os.path.abspath(os.path.join(args.git_tree, ".."))
- if not os.path.isdir(args.apt_tree):
- logging.info("Creating %s", args.apt_tree)
- os.makedirs(args.apt_tree)
- fn = os.path.join(args.apt_tree, "apt-gpg-key.asc")
- if not os.path.exists(fn):
- logging.info("Creating %s", fn)
- run("gpg", "--export", "--armor", "--keyring", args.keyring, stdout = open(fn, "w"))
- class Release(object):
- architectures = dict(amd64 = "", i386 = "-i386")
- releases = []
- packages = {}
- def __init__(self, distribution_release, backports):
- self.distribution, self.release = distribution_release.split("/")
- self.backports = backports
- self.apt_source = "{scheme}://{host}/{path}/{distribution} {release} main".format(
- scheme = args.url_scheme,
- host = args.url_host,
- path = args.url_path.strip("/"),
- distribution = self.distribution,
- release = self.release)
- self.env = dict(os.environ)
- if backports:
- self.env.update(OTHERMIRROR = "deb " + self.apt_source)
- self.releases.append(self)
- @classmethod
- def do_all_releases(cls):
- for release in cls.releases:
- release.setup_reprepro()
- for release in cls.releases:
- release.list_repository()
- for release in cls.releases:
- for release.arch, release.tag in cls.architectures.iteritems():
- release.do_one_architecture()
- del release.arch, release.tag
- @staticmethod
- def repokey(release, architecture, package):
- return (release, architecture, package)
- def list_repository(self):
- cmd = ("reprepro", "list", self.release)
- logging.info("Running %s", " ".join(cmd))
- listing = subprocess.check_output(cmd, cwd = self.tree)
- for line in listing.replace(":", " ").replace("|", " ").splitlines():
- rel, comp, arch, pkg, ver = line.split()
- key = (rel, arch, pkg)
- assert key not in self.packages
- self.packages[key] = ver
- @property
- def deb_in_repository(self):
- ret = all(self.packages.get((self.release, self.arch, package)) == self.version
- for package in rpki_packages)
- #logging.info("Got %s looking for %r in %r", ret, self.version, self.packages)
- return ret
- @property
- def src_in_repository(self):
- ret = self.packages.get((self.release, "source", rpki_source_package)) == self.version
- #logging.info("Got %s looking for %r in %r", ret, self.version, self.packages)
- return ret
- @property
- def version(self):
- return source_version + "~" + self.release
- @property
- def dsc(self):
- return os.path.join(dsc_dir, "rpki_{}.dsc".format(self.version))
- @property
- def tree(self):
- return os.path.join(args.apt_tree, self.distribution, "")
- @property
- def basefile(self):
- return os.path.expanduser("~/pbuilder/{0.release}{0.tag}-base.tgz".format(self))
- @property
- def result(self):
- return os.path.expanduser("~/pbuilder/{0.release}{0.tag}_result".format(self))
- @property
- def changes(self):
- return os.path.join(self.result, "rpki_{0.version}_{0.arch}.changes".format(self))
- def do_one_architecture(self):
- logging.info("Running build for %s %s %s", self.distribution, self.release, self.arch)
- if not os.path.exists(self.dsc):
- logging.info("%s does not exist", self.dsc)
- for fn in os.listdir(dsc_dir):
- fn = os.path.join(dsc_dir, fn)
- if not os.path.isdir(fn) and search_version not in fn:
- logging.info("Removing %s", fn)
- os.unlink(fn)
- run("rm", "-rf", "debian", cwd = args.git_tree)
- logging.info("Building source package %s", self.version)
- run(sys.executable, "buildtools/build-debian-packages.py", "--version-suffix", self.release, cwd = args.git_tree)
- run("dpkg-buildpackage", "-S", "-us", "-uc", "-rfakeroot", cwd = args.git_tree)
- if not os.path.exists(self.basefile):
- logging.info("Creating build environment %s %s", self.release, self.arch)
- run("pbuilder-dist", self.release, self.arch, "create", env = self.env)
- elif time.time() > os.stat(self.basefile).st_mtime + args.update_build_after:
- logging.info("Updating build environment %s %s", self.release, self.arch)
- run("pbuilder-dist", self.release, self.arch, "update", env = self.env)
- if not self.deb_in_repository:
- for fn in os.listdir(self.result):
- fn = os.path.join(self.result, fn)
- logging.info("Removing %s", fn)
- os.unlink(fn)
- logging.info("Building binary packages %s %s %s", self.release, self.arch, self.version)
- run("pbuilder-dist", self.release, self.arch, "build", "--keyring", args.keyring, self.dsc, env = self.env)
- logging.info("Updating repository for %s %s %s", self.release, self.arch, self.version)
- run("reprepro", "--ignore=wrongdistribution", "include", self.release, self.changes, cwd = self.tree)
- if not self.src_in_repository:
- logging.info("Updating repository for %s source %s", self.release, self.version)
- run("reprepro", "--ignore=wrongdistribution", "includedsc", self.release, self.dsc, cwd = self.tree)
- def setup_reprepro(self):
- logging.info("Configuring reprepro for %s/%s", self.distribution, self.release)
- dn = os.path.join(self.tree, "conf")
- if not os.path.isdir(dn):
- logging.info("Creating %s", dn)
- os.makedirs(dn)
- fn = os.path.join(self.tree, "conf", "distributions")
- distributions = open(fn, "r").read() if os.path.exists(fn) else ""
- if "Codename: {0.release}\n".format(self) not in distributions:
- logging.info("%s %s", "Editing" if distributions else "Creating", fn)
- with open(fn, "w") as f:
- if distributions:
- f.write(distributions)
- f.write("\n")
- f.write(dedent("""\
- Origin: rpki.net
- Label: rpki.net {self.distribution} repository
- Codename: {self.release}
- Architectures: {architectures} source
- Components: main
- Description: rpki.net {Distribution} APT Repository
- SignWith: yes
- DebOverride: override.{self.release}
- DscOverride: override.{self.release}
- """.format(
- self = self,
- Distribution = self.distribution.capitalize(),
- architectures = " ".join(self.architectures))))
- fn = os.path.join(self.tree, "conf", "options")
- if not os.path.exists(fn):
- logging.info("Creating %s", fn)
- with open(fn, "w") as f:
- f.write(dedent("""\
- verbose
- ask-passphrase
- basedir .
- """))
- fn = os.path.join(self.tree, "conf", "override." + self.release)
- if not os.path.exists(fn):
- logging.info("Creating %s", fn)
- with open(fn, "w") as f:
- for pkg in self.backports:
- f.write(dedent("""\
- {pkg:<30} Priority optional
- {pkg:<30} Section python
- """.format(pkg = pkg)))
- f.write(dedent("""\
- rpki-ca Priority extra
- rpki-ca Section net
- rpki-rp Priority extra
- rpki-rp Section net
- """))
- fn = os.path.join(args.apt_tree, "rpki.{}.list".format(self.release))
- if not os.path.exists(fn):
- logging.info("Creating %s", fn)
- with open(fn, "w") as f:
- f.write(dedent("""\
- deb {self.apt_source}
- deb-src {self.apt_source}
- """.format(self = self)))
- # Load whatever releases the user specified
- for r in args.releases:
- Release(r, args.backports)
- # Do all the real work.
- Release.do_all_releases()
- # Push any tags created above to the public git repository.
- if upload:
- run("git", "push", "--tags", cwd = args.git_tree)
- # Upload results, maybe. We do this in two stages, to minimize the window
- # during which the uploaded repository might be in an inconsistent state.
- def rsync(*flags):
- cmd = ["rsync", "--archive", "--itemize-changes", "--rsh", "ssh"]
- cmd.extend(flags)
- cmd.append(args.apt_tree)
- cmd.append("rsync://{host}/{path}/".format(host = args.url_host,
- path = args.url_path.strip("/")))
- if upload:
- logging.info("Synching repository to %s with flags %s",
- cmd[-1], " ".join(flags))
- run(*cmd)
- else:
- logging.info("Would have synched repository to %s with flags %s",
- cmd[-1], " ".join(flags))
- rsync("--ignore-existing")
- rsync("--exclude", "HEADER.html",
- "--exclude", "HEADER.css",
- "--delete", "--delete-delay")
- logging.info("Done")
|