Add separate directory for rpki-pbuilder et al.

svn path=/apt-tools/; revision=6328

5 files changed, 427 insertions, 0 deletions

diff --git a/README b/README
new file mode 100644
index 0000000..9002d04
--- /dev/null
+++ b/README
@@ -0,0 +1,9 @@
+Tools and scripts related to automated building of Debian packages and
+maintenance of an APT repository containing the result.  At present,
+this is based on pbuilder and reprepro running on Ubuntu.
+
+This used to be part of the buildtools/ directory in the main
+repository, but the build automation isn't really tied to any
+particular branch, and now that it has to maintain packages from
+multiple branches it's less confusing to put it in its own little
+corner of the repository.
diff --git a/rpki-pbuilder.crontab b/rpki-pbuilder.crontab
new file mode 100644
index 0000000..25b596b
--- /dev/null
+++ b/rpki-pbuilder.crontab
@@ -0,0 +1 @@
+*/10 * * * *	. $HOME/builder.sh
diff --git a/rpki-pbuilder.logrotate b/rpki-pbuilder.logrotate
new file mode 100644
index 0000000..a7df6d4
--- /dev/null
+++ b/rpki-pbuilder.logrotate
@@ -0,0 +1,12 @@
+/home/sra/builder.log
+{
+	rotate 30
+	daily
+	missingok
+	notifempty
+	dateext
+	compress
+	compresscmd /usr/bin/xz
+	uncompresscmd /usr/bin/unxz
+	compressext .xz
+}
diff --git a/rpki-pbuilder.py b/rpki-pbuilder.py
new file mode 100644
index 0000000..3af0e11
--- /dev/null
+++ b/rpki-pbuilder.py
@@ -0,0 +1,351 @@
+#!/usr/bin/python
+#
+# $Id$
+#
+# Copyright (C) 2014  Dragon Research Labs ("DRL")
+# Portions copyright (C) 2013  Internet Systems Consortium ("ISC")
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notices and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND DRL AND ISC DISCLAIM ALL
+# WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS.  IN NO EVENT SHALL DRL OR
+# ISC BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
+# DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA
+# OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+# TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+# PERFORMANCE OF THIS SOFTWARE.
+
+"""
+Debian/Ubuntu package build tool, based on pbuilder-dist and reprepro.
+"""
+
+import os
+import sys
+import time
+import fcntl
+import errno
+import socket
+import logging
+import argparse
+import subprocess
+
+from textwrap import dedent
+
+rpki_packages = ("rpki-rp", "rpki-ca")
+rpki_source_package = "rpki"
+
+parser = argparse.ArgumentParser(description = __doc__,
+                                 formatter_class = argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument("--debug", action = "store_true",
+                    help = "enable debugging code")
+parser.add_argument("--update-build-after", type = int, default = 7 * 24 * 60 * 60,
+                    help = "interval (in seconds) after which we should update the pbuilder environment")
+parser.add_argument("--lockfile", default = os.path.expanduser("~/builder.lock"),
+                    help = "avoid collisions between multiple instances of this script")
+parser.add_argument("--keyring",  default = os.path.expanduser("~/.gnupg/pubring.gpg"),
+                    help = "PGP keyring")
+parser.add_argument("--svn-tree", default = os.path.expanduser("~/source/trunk/"),
+                    help = "subversion tree")
+parser.add_argument("--apt-tree", default = os.path.expanduser("~/repository/"),
+                    help = "reprepro repository")
+parser.add_argument("--apt-user", default = "aptbot",
+                    help = "username for uploading apt repository to public web server")
+parser.add_argument("--url-host", default = "download.rpki.net",
+                    help = "hostname of public web server")
+parser.add_argument("--url-path", default = "/APT",
+                    help = "path of apt repository on public web server")
+parser.add_argument("--backports", nargs = "+", default = ["python-django", "python-tornado"],
+                    help = "backports needed for this build")
+parser.add_argument("--releases", nargs = "+", default = ["ubuntu/trusty", "debian/wheezy", "ubuntu/precise"],
+                    help = "releases for which to build")
+args = parser.parse_args()
+
+# Maybe logging should be configurable too.  Later.
+
+logging.basicConfig(level = logging.INFO, timefmt = "%Y-%m-%dT%H:%M:%S",
+                    format = "%(asctime)s [%(process)d] %(levelname)s %(message)s")
+
+upload = socket.getfqdn() == "build-u.rpki.net"
+
+def run(*cmd, **kwargs):
+    if args.debug:
+        #logging.info("Running %r %r", cmd, kwargs)
+        logging.info("Running %s", " ".join(cmd))
+    subprocess.check_call(cmd, **kwargs)
+
+# Getting this to work right also required adding:
+#
+#   DEBBUILDOPTS="-b"
+#
+# to /etc/pbuilderrc; without this, reprepro (eventually, a year after
+# we set this up) started failing to incorporate some of the built
+# packages, because the regenerated source packages had different
+# checksums than the ones loaded initially.  See:
+#
+# http://stackoverflow.com/questions/21563872/reprepro-complains-about-the-generated-pbuilder-debian-tar-gz-archive-md5
+#
+# Putting stuff in ~/.pbuilderrc didn't work with pbuilder-dist when I
+# tried it last year, this may just be that sudo isn't configured to
+# pass HOME through, thus pbuilder is looking for ~root/.pbuilderrc.
+# Worth trying again at some point but not all that critical.
+
+logging.info("Starting")
+
+try:
+    lock = os.open(args.lockfile, os.O_RDONLY | os.O_CREAT | os.O_NONBLOCK, 0666)
+    fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+except (IOError, OSError), e:
+    sys.exit(0 if e.errno == errno.EAGAIN else "Error {!r} opening lock {!r}".format(e, args.lockfile))
+
+run("svn", "--quiet", "update", cwd = args.svn_tree)
+
+source_version = subprocess.check_output(("svnversion", "-c"), cwd = args.svn_tree).strip().split(":")[-1]
+
+logging.info("Source version is %s", source_version)
+
+if not source_version.isdigit() and not args.debug:
+    sys.exit("Sources don't look pristine, not building ({!r})".format(source_version))
+
+source_version = "0." + source_version
+search_version = "_" + source_version + "~"
+
+dsc_dir = os.path.abspath(os.path.join(args.svn_tree, ".."))
+
+if not os.path.isdir(args.apt_tree):
+    logging.info("Creating %s", args.apt_tree)
+    os.makedirs(args.apt_tree)
+
+fn = os.path.join(args.apt_tree, "apt-gpg-key.asc")
+if not os.path.exists(fn):
+    logging.info("Creating %s", fn)
+    run("gpg", "--export", "--armor", "--keyring", args.keyring, stdout = open(fn, "w"))
+
+class Release(object):
+
+    architectures = dict(amd64 = "", i386 = "-i386")
+
+    releases = []
+    packages = {}
+
+    def __init__(self, distribution_release, backports):
+        self.distribution, self.release = distribution_release.split("/")
+        self.backports = backports
+        self.apt_source = "http://{host}/{path}/{distribution} {release} main".format(
+            host = args.url_host,
+            path = args.url_path.strip("/"),
+            distribution = self.distribution,
+            release = self.release)
+        self.env = dict(os.environ)
+        if backports:
+            self.env.update(OTHERMIRROR = "deb " + self.apt_source)
+        self.releases.append(self)
+
+    @classmethod
+    def do_all_releases(cls):
+        for release in cls.releases:
+            release.setup_reprepro()
+        for release in cls.releases:
+            release.list_repository()
+        for release in cls.releases:
+            for release.arch, release.tag in cls.architectures.iteritems():
+                release.do_one_architecture()
+            del release.arch, release.tag
+
+    @staticmethod
+    def repokey(release, architecture, package):
+        return (release, architecture, package)
+
+    def list_repository(self):
+        cmd = ("reprepro", "list", self.release)
+        logging.info("Running %s", " ".join(cmd))
+        listing = subprocess.check_output(cmd, cwd = self.tree)
+        for line in listing.replace(":", " ").replace("|", " ").splitlines():
+            rel, comp, arch, pkg, ver = line.split()
+            key = (rel, arch, pkg)
+            assert key not in self.packages
+            self.packages[key] = ver
+
+    @property
+    def deb_in_repository(self):
+        ret = all(self.packages.get((self.release, self.arch, package)) == self.version
+                  for package in rpki_packages)
+        #logging.info("Got %s looking for %r in %r", ret, self.version, self.packages)
+        return ret
+
+    @property
+    def src_in_repository(self):
+        ret = self.packages.get((self.release, "source", rpki_source_package)) == self.version
+        #logging.info("Got %s looking for %r in %r", ret, self.version, self.packages)
+        return ret
+
+    @property
+    def version(self):
+        return source_version + "~" + self.release
+
+    @property
+    def dsc(self):
+        return os.path.join(dsc_dir, "rpki_{}.dsc".format(self.version))
+
+    @property
+    def tree(self):
+        return os.path.join(args.apt_tree, self.distribution, "")
+
+    @property
+    def basefile(self):
+        return os.path.expanduser("~/pbuilder/{0.release}{0.tag}-base.tgz".format(self))
+
+    @property
+    def result(self):
+        return os.path.expanduser("~/pbuilder/{0.release}{0.tag}_result".format(self))
+
+    @property
+    def changes(self):
+        return os.path.join(self.result, "rpki_{0.version}_{0.arch}.changes".format(self))
+
+    def do_one_architecture(self):
+        logging.info("Running build for %s %s %s", self.distribution, self.release, self.arch)
+
+        if not os.path.exists(self.dsc):
+
+            logging.info("%s does not exist", self.dsc)
+            for fn in os.listdir(dsc_dir):
+                fn = os.path.join(dsc_dir, fn)
+                if not os.path.isdir(fn) and search_version not in fn:
+                    logging.info("Removing %s", fn)
+                    os.unlink(fn)
+            run("rm", "-rf", "debian", cwd = args.svn_tree)
+
+            logging.info("Building source package %s", self.version)
+            run(sys.executable, "buildtools/make-version.py", cwd = args.svn_tree)
+            run(sys.executable, "buildtools/build-debian-packages.py", "--version-suffix", self.release, cwd = args.svn_tree)
+            run("dpkg-buildpackage", "-S", "-us", "-uc", "-rfakeroot", cwd = args.svn_tree)
+
+        if not os.path.exists(self.basefile):
+            logging.info("Creating build environment %s %s", self.release, self.arch)
+            run("pbuilder-dist", self.release, self.arch, "create", env = self.env)
+
+        elif time.time() > os.stat(self.basefile).st_mtime + args.update_build_after:
+            logging.info("Updating build environment %s %s", self.release, self.arch)
+            run("pbuilder-dist", self.release, self.arch, "update", env = self.env)
+
+        if not self.deb_in_repository:
+
+            for fn in os.listdir(self.result):
+                fn = os.path.join(self.result, fn)
+                logging.info("Removing %s", fn)
+                os.unlink(fn)
+
+            logging.info("Building binary packages %s %s %s", self.release, self.arch, self.version)
+            run("pbuilder-dist", self.release, self.arch, "build", "--keyring", args.keyring, self.dsc, env = self.env)
+
+            logging.info("Updating repository for %s %s %s", self.release, self.arch, self.version)
+            run("reprepro", "--ignore=wrongdistribution", "include", self.release, self.changes, cwd = self.tree)
+
+        if not self.src_in_repository:
+            logging.info("Updating repository for %s source %s", self.release, self.version)
+            run("reprepro", "--ignore=wrongdistribution", "includedsc", self.release, self.dsc, cwd = self.tree)
+
+    def setup_reprepro(self):
+
+        logging.info("Configuring reprepro for %s/%s", self.distribution, self.release)
+
+        dn = os.path.join(self.tree, "conf")
+        if not os.path.isdir(dn):
+            logging.info("Creating %s", dn)
+            os.makedirs(dn)
+
+        fn = os.path.join(self.tree, "conf", "distributions")
+        distributions = open(fn, "r").read() if os.path.exists(fn) else ""
+        if "Codename: {0.release}\n".format(self) not in distributions:
+            logging.info("%s %s", "Editing" if distributions else "Creating", fn)
+            with open(fn, "w") as f:
+                if distributions:
+                    f.write(distributions)
+                    f.write("\n")
+                f.write(dedent("""\
+                               Origin: rpki.net
+                               Label: rpki.net {self.distribution} repository
+                               Codename: {self.release}
+                               Architectures: {architectures} source
+                               Components: main
+                               Description: rpki.net {Distribution} APT Repository
+                               SignWith: yes
+                               DebOverride: override.{self.release}
+                               DscOverride: override.{self.release}
+                               """.format(
+                            self = self,
+                            Distribution  = self.distribution.capitalize(),
+                            architectures = " ".join(self.architectures))))
+
+        fn = os.path.join(self.tree, "conf", "options")
+        if not os.path.exists(fn):
+            logging.info("Creating %s", fn)
+            with open(fn, "w") as f:
+                f.write(dedent("""\
+                               verbose
+                               ask-passphrase
+                               basedir .
+                               """))
+
+        fn = os.path.join(self.tree, "conf", "override." + self.release)
+        if not os.path.exists(fn):
+            logging.info("Creating %s", fn)
+            with open(fn, "w") as f:
+                for pkg in self.backports:
+                    f.write(dedent("""\
+                                   {pkg:<30}   Priority        optional
+                                   {pkg:<30}   Section         python
+                                   """.format(pkg = pkg)))
+                f.write(dedent("""\
+                               rpki-ca                 Priority        extra
+                               rpki-ca                 Section         net
+                               rpki-rp                 Priority        extra
+                               rpki-rp                 Section         net
+                               """))
+
+        fn = os.path.join(args.apt_tree, "rpki.{}.list".format(self.release))
+        if not os.path.exists(fn):
+            logging.info("Creating %s", fn)
+            with open(fn, "w") as f:
+                f.write(dedent("""\
+                               deb {self.apt_source}
+                               deb-src {self.apt_source}
+                               """.format(self = self)))
+
+# Load whatever releases the user specified
+
+for r in args.releases:
+    Release(r, args.backports)
+
+# Do all the real work.
+
+Release.do_all_releases()
+
+# Upload results, maybe.  We do this in two stages, to minimize the window
+# during which the uploaded repository might be in an inconsistent state.
+
+def rsync(*flags):
+    cmd = ["rsync", "--archive", "--itemize-changes",
+           "--rsh", "ssh -l {}".format(args.apt_user)]
+    cmd.extend(flags)
+    cmd.append(args.apt_tree)
+    cmd.append("rsync://{host}/{path}/".format(host = args.url_host,
+                                               path = args.url_path.strip("/")))
+    if upload:
+        logging.info("Synching repository to %s with flags %s",
+                     cmd[-1], " ".join(flags))
+        run(*cmd)
+    else:
+        logging.info("Would have synched repository to %s with flags %",
+                     cmd[-1], " ".join(flags))
+
+rsync("--ignore-existing")
+
+rsync("--exclude", "HEADER.html",
+      "--exclude", "HEADER.css",
+      "--delete", "--delete-delay")
+
+logging.info("Done")
diff --git a/rpki-pbuilder.sh b/rpki-pbuilder.sh
new file mode 100755
index 0000000..dbf68d6
--- /dev/null
+++ b/rpki-pbuilder.sh
@@ -0,0 +1,54 @@
+#!/bin/sh -
+
+# New top-level script, now that we need to run rpki-pbuilder.py multiple times with different arguments.
+# May eventually want to consolidate all of this back into rpki-pbuilder.py, but not today.
+
+# At the moment, coverage with the new code is a bit scattershot.  It works with trusty, with backports
+# from xenial; it works for stretch, and it works for jessie with a lot of backports from stretch.
+# Backports for wheezy are probably more trouble than they're worth, and there seems little point
+# in putting effort into the non-LTS Ubuntu releases.
+#
+# In practice, the old code no longer builds for precise, which is an LTS that's still under support.
+# Don't yet know why.
+
+/usr/sbin/logrotate -s $HOME/logrotate.state $HOME/logrotate.conf
+
+exec >> $HOME/builder.log 2>&1
+
+set -x
+
+cd $HOME
+
+python rpki-pbuilder.py					\
+    --svn-tree $HOME/source.ng/tk705/			\
+    --apt-tree $HOME/repository.ng/			\
+    --url-path /APTng					\
+    --backports python-django python-tornado		\
+    --releases ubuntu/trusty
+
+python rpki-pbuilder.py					\
+    --svn-tree $HOME/source.ng/tk705/			\
+    --apt-tree $HOME/repository.ng/			\
+    --url-path /APTng					\
+    --releases ubuntu/xenial
+
+python rpki-pbuilder.py					\
+    --svn-tree $HOME/source.tos/trunk/			\
+    --apt-tree $HOME/repository.tos/			\
+    --url-path /APT					\
+    --backports python-django-south			\
+    --releases ubuntu/trusty
+
+python rpki-pbuilder.py					\
+    --svn-tree $HOME/source.tos/trunk/			\
+    --apt-tree $HOME/repository.tos/			\
+    --url-path /APT					\
+    --backports python-django python-django-south	\
+    --releases debian/wheezy
+
+python rpki-pbuilder.py					\
+    --svn-tree $HOME/source.tos/trunk/			\
+    --apt-tree $HOME/repository.tos/			\
+    --url-path /APT					\
+    --backports python-django python-django-south	\
+    --releases ubuntu/precise