Rewrite pbuilder script, add support for Ubutu Trusty Tahir (14.04).

Builds for Ubuntu Precise Pangolin (12.04) disabled, either for good
or until we figure out how to convince reprepo to accept the same
package compiled for two different releases (sigh).  See #712.

svn path=/trunk/; revision=5971

1 files changed, 245 insertions, 74 deletions

diff --git a/buildtools/rpki-pbuilder.py b/buildtools/rpki-pbuilder.py
index 1fc09354..59add547 100644
--- a/buildtools/rpki-pbuilder.py
+++ b/buildtools/rpki-pbuilder.py
@@ -16,38 +16,58 @@
 # OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 # PERFORMANCE OF THIS SOFTWARE.
 
+"""
+Debian/Ubuntu package build tool, based on pbuilder-dist and reprepro.
+"""
+
 import os
 import sys
 import time
 import fcntl
 import errno
 import socket
+import logging
+import argparse
 import subprocess
 
-debug  = False
+from textwrap import dedent
+
+#from apt_pkg import version_compare
+
+rpki_packages = ("rpki-rp", "rpki-ca")
+rpki_source_package = "rpki"
+
+parser = argparse.ArgumentParser(description = __doc__,
+                                 formatter_class = argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument("--debug", action = "store_true",
+                    help = "enable debugging code")
+parser.add_argument("--update-build-after", type = int, default = 7 * 24 * 60 * 60,
+                    help = "interval (in seconds) after which we should update the pbuilder environment")
+parser.add_argument("--lockfile", default = os.path.expanduser("~/builder.lock"),
+                    help = "avoid collisions between multiple instances of this script")
+parser.add_argument("--keyring",  default = os.path.expanduser("~/.gnupg/pubring.gpg"),
+                    help = "PGP keyring")
+parser.add_argument("--svn-tree", default = os.path.expanduser("~/source/trunk/"),
+                    help = "subversion tree")
+parser.add_argument("--apt-tree", default = os.path.expanduser("~/repository/"),
+                    help = "reprepro repository")
+parser.add_argument("--srv-path", default = "aptbot@download.rpki.net:/usr/local/www/data/download.rpki.net/APT/",
+                    help = "upload destination")
+parser.add_argument("--source-format", default = "http://download.rpki.net/APT/%(distribution)s %(release)s main",
+                    help = "source.list format string")
+args = parser.parse_args()
+
+# Maybe logging should be conigurable too.  Later.
+
+logging.basicConfig(level = logging.INFO)
+
 upload = socket.getfqdn() == "build-u.rpki.net"
 
-def run(*args, **kwargs):
-    if debug:
-        log("Running %r %r" % (args, kwargs))
-    subprocess.check_call(args, **kwargs)
-
-def log(msg):
-    # Maybe this should go to syslog instead, but this works for now.
-    sys.stdout.write(time.strftime("%Y-%m-%dT%H:%M:%SZ ", time.gmtime()))
-    sys.stdout.write(msg)
-    sys.stdout.write("\n")
-    sys.stdout.flush()
-
-lockfile = os.path.expanduser("~/builder.lock")
-svn_tree = os.path.expanduser("~/source/trunk/")
-apt_tree = os.path.expanduser("~/repository/")
-ubu_tree = os.path.join(apt_tree, "ubuntu/")
-deb_tree = os.path.join(apt_tree, "debian/")
-srv_path = "aptbot@download.rpki.net:/usr/local/www/data/download.rpki.net/APT/"
-ubu_env  = dict(os.environ,
-                OTHERMIRROR = "deb http://download.rpki.net/APT/ubuntu precise main")
-deb_env  = os.environ
+def run(*cmd, **kwargs):
+    if args.debug:
+        #logging.info("Running %r %r", cmd, kwargs)
+        logging.info("Running %s", " ".join(cmd))
+    subprocess.check_call(cmd, **kwargs)
 
 # Getting this to work right also required adding:
 #
@@ -60,78 +80,229 @@ deb_env  = os.environ
 #
 # http://stackoverflow.com/questions/21563872/reprepro-complains-about-the-generated-pbuilder-debian-tar-gz-archive-md5
 
-log("Starting")
+logging.info("Starting")
 
 try:
-    lock = os.open(lockfile, os.O_RDONLY | os.O_CREAT | os.O_NONBLOCK, 0666)
+    lock = os.open(args.lockfile, os.O_RDONLY | os.O_CREAT | os.O_NONBLOCK, 0666)
     fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
 except (IOError, OSError), e:
-    sys.exit(0 if e.errno == errno.EAGAIN else "Error %r opening lock %r" % lockfile)
-
-os.chdir(svn_tree)
+    sys.exit(0 if e.errno == errno.EAGAIN else "Error %r opening lock %r" % args.lockfile)
 
-run("svn", "--quiet", "update")
+run("svn", "--quiet", "update", cwd = args.svn_tree)
 
-version = subprocess.check_output(("svnversion", "-c")).strip().split(":")[-1]
+version = subprocess.check_output(("svnversion", "-c"), cwd = args.svn_tree).strip().split(":")[-1]
 
-if not version.isdigit() and not debug:
+if not version.isdigit() and not args.debug:
     sys.exit("Sources don't look pristine, not building (%r)" % version)
 
 version = "0." + version
 
-dsc = os.path.join(svn_tree, "..", "rpki_%s.dsc" % version)
+dsc_dir = os.path.abspath(os.path.join(args.svn_tree, ".."))
+dsc = os.path.join(dsc_dir, "rpki_%s.dsc" % version)
 
 if not os.path.exists(dsc):
-    log("Building source package %s" % version)
-    for fn in os.listdir(".."):
+    logging.info("Building source package %s", version)
+    for fn in os.listdir(dsc_dir):
         if fn != "trunk":
-            os.unlink(os.path.join("..", fn))
-    run("rm", "-rf", "debian")
-    run("python", "buildtools/make-version.py")
-    run("python", "buildtools/build-ubuntu-ports.py")
-    run("dpkg-buildpackage", "-S", "-us", "-uc", "-rfakeroot")
-
-for dist, tree, env in (("precise", ubu_tree, ubu_env),
-                        ("wheezy",  deb_tree, deb_env)):
-
-    for arch, tag in (("amd64", ""), ("i386",  "-i386")):
-
-        basedir = os.path.expanduser("~/pbuilder/%s%s-base.tgz" % (dist, tag))
-        result  = os.path.expanduser("~/pbuilder/%s%s_result" % (dist, tag))
-        changes = os.path.join(result, "rpki_%s_%s.changes" % (version, arch))
-
-        # Update the build environment if it's been more than a week since
-        # we last did that.  If this turns out to be error-prone, we might
-        # want to put it in a cron job of its own so it doesn't crash the
-        # normal cycle, but let's try it this way for a start.
-        #
-        if time.time() > os.stat(basedir).st_mtime + (7 * 24 * 60 * 60):
-            log("Updating build environment %s %s" % (dist, arch))
-            run("pbuilder-dist", dist, arch, "update",
-                env = env)
-
-        if not os.path.exists(changes):
-            # The need for --ignore=wrongdistribution may indicate
-            # something I'm doing wrong in hack-debian-changelog.py,
-            # revisit that later.  For now, just whack with a stick.
-            #
-            log("Building binary packages %s %s %s" % (dist, arch, version))
-            for fn in os.listdir(result):
-                os.unlink(os.path.join(result, fn))
-            run("pbuilder-dist", dist, arch, "build", dsc,
-                env = env)
-            run("reprepro", "--ignore=wrongdistribution", "include", dist, changes,
-                cwd = tree)
+            os.unlink(os.path.join(dsc_dir, fn))
+    run("rm", "-rf", "debian", cwd = args.svn_tree)
+    run("python", "buildtools/make-version.py", cwd = args.svn_tree)
+    run("python", "buildtools/build-ubuntu-ports.py", cwd = args.svn_tree)
+    run("dpkg-buildpackage", "-S", "-us", "-uc", "-rfakeroot", cwd = args.svn_tree)
+
+if not os.path.isdir(args.apt_tree):
+    logging.info("Creating %s", args.apt_tree)
+    os.makedirs(args.apt_tree)
+
+fn = os.path.join(args.apt_tree, "apt-gpg-key.asc")
+if not os.path.exists(fn):
+    logging.info("Creating %s", fn)
+    run("gpg", "--export", "--armor", "--keyring", args.keyring, stdout = open(fn, "w"))
+
+class Release(object):
+
+    architectures = dict(amd64 = "", i386 = "-i386")
+
+    releases = []
+    packages = {}
+
+    def __init__(self, release, distribution, *backports):
+        self.release = release
+        self.distribution = distribution
+        self.backports = backports
+        if backports:
+            self.env = dict(os.environ,
+                            OTHERMIRROR = "deb " + args.source_format  % dict(distribution = distribution, release = release))
+        else:
+            self.env = os.environ
+        self.releases.append(self)
+
+    @classmethod
+    def do_all_releases(cls):
+        for release in cls.releases:
+            release.setup_reprepro()
+        for release in cls.releases:
+            release.list_repository()
+        for release in cls.releases:
+            for release.arch, release.tag in cls.architectures.iteritems():
+                release.do_one_architecture()
+            del release.arch, release.tag
+
+    @staticmethod
+    def repokey(release, architecture, package):
+        return (release, architecture, package)
+
+    def list_repository(self):
+        cmd = ("reprepro", "list", self.release)
+        logging.info("Running %s", " ".join(cmd))
+        listing = subprocess.check_output(cmd, cwd = self.tree)
+        for line in listing.replace(":", " ").replace("|", " ").splitlines():
+            rel, comp, arch, pkg, ver = line.split()
+            key = (rel, arch, pkg)
+            assert key not in self.packages
+            self.packages[key] = ver
+
+    @property
+    def deb_in_repository(self):
+        return all(self.packages.get((self.release, self.arch, package)) == version
+                   for package in rpki_packages)
+
+    @property
+    def src_in_repository(self):
+        return self.packages.get((self.release, "source", rpki_source_package)) == version
+
+    @property
+    def tree(self):
+        return os.path.join(args.apt_tree, self.distribution, "")
+
+    @property
+    def basefile(self):
+        return os.path.expanduser("~/pbuilder/%s%s-base.tgz" % (self.release, self.tag))
+
+    @property
+    def result(self):
+        return os.path.expanduser("~/pbuilder/%s%s_result" % (self.release, self.tag))
+
+    @property
+    def changes(self):
+        return os.path.join(self.result, "rpki_%s_%s.changes" % (version, self.arch))
+
+    def do_one_architecture(self):
+        logging.info("Running build for %s %s %s", self.distribution, self.release, self.arch)
+
+        if not os.path.exists(self.basefile):
+            logging.info("Creating build environment %s %s", self.release, self.arch)
+            run("pbuilder-dist", self.release, self.arch, "create", env = self.env)
+
+        elif time.time() > os.stat(self.basefile).st_mtime + args.update_build_after:
+            logging.info("Updating build environment %s %s", self.release, self.arch)
+            run("pbuilder-dist", self.release, self.arch, "update", env = self.env)
+
+        if not os.path.exists(self.changes):
+            logging.info("Building binary packages %s %s %s", self.release, self.arch, version)
+            for fn in os.listdir(self.result):
+                os.unlink(os.path.join(self.result, fn))
+            run("pbuilder-dist", self.release, self.arch, "build", "--keyring", args.keyring, dsc, env = self.env)
+
+        if not self.deb_in_repository:
+            logging.info("Updating repository for %s %s %s", self.release, self.arch, version)
+            run("reprepro", "--ignore=wrongdistribution", "include", self.release, self.changes, cwd = self.tree)
+
+        if not self.src_in_repository:
+            logging.info("Updating repository for %s source %s", self.release, version)
+            run("reprepro", "--ignore=wrongdistribution", "includedsc", self.release, dsc, cwd = self.tree)
+
+    def setup_reprepro(self):
+
+        logging.info("Configuring reprepro for %s/%s", self.distribution, self.release)
+
+        dn = os.path.join(self.tree, "conf")
+        if not os.path.isdir(dn):
+            logging.info("Creating %s", dn)
+            os.makedirs(dn)
+
+        fn = os.path.join(self.tree, "conf", "distributions")
+        distributions = open(fn, "r").read() if os.path.exists(fn) else ""
+        if ("Codename: %s\n" % self.release) not in distributions:
+            logging.info("%s %s", "Editing" if distributions else "Creating", fn)
+            with open(fn, "w") as f:
+                if distributions:
+                    f.write(distributions)
+                    f.write("\n")
+                f.write(dedent("""\
+                        Origin: rpki.net
+                        Label: rpki.net %(distribution)s repository
+                        Codename: %(release)s
+                        Architectures: %(architectures)s source
+                        Components: main
+                        Description: rpki.net %(Distribution)s APT Repository
+                        SignWith: yes
+                        DebOverride: override.%(release)s
+                        DscOverride: override.%(release)s
+                        """ % dict(
+                            distribution = self.distribution,
+                            Distribution = self.distribution.capitalize(),
+                            architectures = " ".join(self.architectures),
+                            release = self.release)))
+
+        fn = os.path.join(self.tree, "conf", "options")
+        if not os.path.exists(fn):
+            logging.info("Creating %s", fn)
+            with open(fn, "w") as f:
+                f.write(dedent("""\
+                        verbose
+                        ask-passphrase
+                        basedir .
+                        """))
+
+        fn = os.path.join(self.tree, "conf", "override." + self.release)
+        if not os.path.exists(fn):
+            logging.info("Creating %s", fn)
+            with open(fn, "w") as f:
+                for pkg in backports:
+                    f.write(dedent("""\
+                        %-30s	Priority	optional
+                        %-30s	Section		python
+                        """ % (pkg, pkg)))
+                f.write(dedent("""\
+                        rpki-ca			Priority	extra
+                        rpki-ca			Section		net
+                        rpki-rp			Priority	extra
+                        rpki-rp			Section		net
+                        """))
+
+        fn = os.path.join(args.apt_tree, "rpki.%s.list" % self.release)
+        if not os.path.exists(fn):
+            logging.info("Creating %s", fn)
+            source = args.source_format % dict(distribution = self.distribution, release = self.release)
+            with open(fn, "w") as f:
+                f.write("deb %s\n" % source)
+                f.write("deb-src %s\n" % source)
+
+# At the moment, none of these distributions include South 1.1,
+# and only trusty includes Django 1.6.
+#
+# reprepro seems unable to cope with multipel packages with the same
+# name and version even when they really are different and are for
+# different releases.  Oh well, we didn't want to support precise
+# forever.
+
+Release("trusty", "ubuntu", "python-django-south")
+Release("wheezy", "debian", "python-django", "python-django-south")
+
+#Release("precise", "ubuntu", "python-django", "python-django-south")
+
+Release.do_all_releases()
 
 if upload:
-    log("Synching repository to server")
+    logging.info("Synching repository to server")
     run("rsync", "-ai4",
         "--ignore-existing",
-        apt_tree, srv_path)
+        args.apt_tree, args.srv_path)
     run("rsync", "-ai4",
         "--exclude", "HEADER.html",
         "--exclude", "HEADER.css",
         "--delete", "--delete-delay",
-        apt_tree, srv_path)
+        args.apt_tree, args.srv_path)
 
-log("Done")
+logging.info("Done")