|
|
@@ -0,0 +1,351 @@
|
|
|
+#!/usr/bin/python
|
|
|
+#
|
|
|
+# $Id$
|
|
|
+#
|
|
|
+# Copyright (C) 2014 Dragon Research Labs ("DRL")
|
|
|
+# Portions copyright (C) 2013 Internet Systems Consortium ("ISC")
|
|
|
+#
|
|
|
+# Permission to use, copy, modify, and distribute this software for any
|
|
|
+# purpose with or without fee is hereby granted, provided that the above
|
|
|
+# copyright notices and this permission notice appear in all copies.
|
|
|
+#
|
|
|
+# THE SOFTWARE IS PROVIDED "AS IS" AND DRL AND ISC DISCLAIM ALL
|
|
|
+# WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
|
|
|
+# WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL OR
|
|
|
+# ISC BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
|
|
|
+# DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA
|
|
|
+# OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
|
|
|
+# TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
|
|
+# PERFORMANCE OF THIS SOFTWARE.
|
|
|
+
|
|
|
+"""
|
|
|
+Debian/Ubuntu package build tool, based on pbuilder-dist and reprepro.
|
|
|
+"""
|
|
|
+
|
|
|
+import os
|
|
|
+import sys
|
|
|
+import time
|
|
|
+import fcntl
|
|
|
+import errno
|
|
|
+import socket
|
|
|
+import logging
|
|
|
+import argparse
|
|
|
+import subprocess
|
|
|
+
|
|
|
+from textwrap import dedent
|
|
|
+
|
|
|
+rpki_packages = ("rpki-rp", "rpki-ca")
|
|
|
+rpki_source_package = "rpki"
|
|
|
+
|
|
|
+parser = argparse.ArgumentParser(description = __doc__,
|
|
|
+ formatter_class = argparse.ArgumentDefaultsHelpFormatter)
|
|
|
+parser.add_argument("--debug", action = "store_true",
|
|
|
+ help = "enable debugging code")
|
|
|
+parser.add_argument("--update-build-after", type = int, default = 7 * 24 * 60 * 60,
|
|
|
+ help = "interval (in seconds) after which we should update the pbuilder environment")
|
|
|
+parser.add_argument("--lockfile", default = os.path.expanduser("~/builder.lock"),
|
|
|
+ help = "avoid collisions between multiple instances of this script")
|
|
|
+parser.add_argument("--keyring", default = os.path.expanduser("~/.gnupg/pubring.gpg"),
|
|
|
+ help = "PGP keyring")
|
|
|
+parser.add_argument("--svn-tree", default = os.path.expanduser("~/source/trunk/"),
|
|
|
+ help = "subversion tree")
|
|
|
+parser.add_argument("--apt-tree", default = os.path.expanduser("~/repository/"),
|
|
|
+ help = "reprepro repository")
|
|
|
+parser.add_argument("--apt-user", default = "aptbot",
|
|
|
+ help = "username for uploading apt repository to public web server")
|
|
|
+parser.add_argument("--url-host", default = "download.rpki.net",
|
|
|
+ help = "hostname of public web server")
|
|
|
+parser.add_argument("--url-path", default = "/APT",
|
|
|
+ help = "path of apt repository on public web server")
|
|
|
+parser.add_argument("--backports", nargs = "+", default = ["python-django", "python-tornado"],
|
|
|
+ help = "backports needed for this build")
|
|
|
+parser.add_argument("--releases", nargs = "+", default = ["ubuntu/trusty", "debian/wheezy", "ubuntu/precise"],
|
|
|
+ help = "releases for which to build")
|
|
|
+args = parser.parse_args()
|
|
|
+
|
|
|
+# Maybe logging should be configurable too. Later.
|
|
|
+
|
|
|
+logging.basicConfig(level = logging.INFO, timefmt = "%Y-%m-%dT%H:%M:%S",
|
|
|
+ format = "%(asctime)s [%(process)d] %(levelname)s %(message)s")
|
|
|
+
|
|
|
+upload = socket.getfqdn() == "build-u.rpki.net"
|
|
|
+
|
|
|
+def run(*cmd, **kwargs):
|
|
|
+ if args.debug:
|
|
|
+ #logging.info("Running %r %r", cmd, kwargs)
|
|
|
+ logging.info("Running %s", " ".join(cmd))
|
|
|
+ subprocess.check_call(cmd, **kwargs)
|
|
|
+
|
|
|
+# Getting this to work right also required adding:
|
|
|
+#
|
|
|
+# DEBBUILDOPTS="-b"
|
|
|
+#
|
|
|
+# to /etc/pbuilderrc; without this, reprepro (eventually, a year after
|
|
|
+# we set this up) started failing to incorporate some of the built
|
|
|
+# packages, because the regenerated source packages had different
|
|
|
+# checksums than the ones loaded initially. See:
|
|
|
+#
|
|
|
+# http://stackoverflow.com/questions/21563872/reprepro-complains-about-the-generated-pbuilder-debian-tar-gz-archive-md5
|
|
|
+#
|
|
|
+# Putting stuff in ~/.pbuilderrc didn't work with pbuilder-dist when I
|
|
|
+# tried it last year, this may just be that sudo isn't configured to
|
|
|
+# pass HOME through, thus pbuilder is looking for ~root/.pbuilderrc.
|
|
|
+# Worth trying again at some point but not all that critical.
|
|
|
+
|
|
|
+logging.info("Starting")
|
|
|
+
|
|
|
+try:
|
|
|
+ lock = os.open(args.lockfile, os.O_RDONLY | os.O_CREAT | os.O_NONBLOCK, 0666)
|
|
|
+ fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
|
+except (IOError, OSError), e:
|
|
|
+ sys.exit(0 if e.errno == errno.EAGAIN else "Error {!r} opening lock {!r}".format(e, args.lockfile))
|
|
|
+
|
|
|
+run("svn", "--quiet", "update", cwd = args.svn_tree)
|
|
|
+
|
|
|
+source_version = subprocess.check_output(("svnversion", "-c"), cwd = args.svn_tree).strip().split(":")[-1]
|
|
|
+
|
|
|
+logging.info("Source version is %s", source_version)
|
|
|
+
|
|
|
+if not source_version.isdigit() and not args.debug:
|
|
|
+ sys.exit("Sources don't look pristine, not building ({!r})".format(source_version))
|
|
|
+
|
|
|
+source_version = "0." + source_version
|
|
|
+search_version = "_" + source_version + "~"
|
|
|
+
|
|
|
+dsc_dir = os.path.abspath(os.path.join(args.svn_tree, ".."))
|
|
|
+
|
|
|
+if not os.path.isdir(args.apt_tree):
|
|
|
+ logging.info("Creating %s", args.apt_tree)
|
|
|
+ os.makedirs(args.apt_tree)
|
|
|
+
|
|
|
+fn = os.path.join(args.apt_tree, "apt-gpg-key.asc")
|
|
|
+if not os.path.exists(fn):
|
|
|
+ logging.info("Creating %s", fn)
|
|
|
+ run("gpg", "--export", "--armor", "--keyring", args.keyring, stdout = open(fn, "w"))
|
|
|
+
|
|
|
+class Release(object):
|
|
|
+
|
|
|
+ architectures = dict(amd64 = "", i386 = "-i386")
|
|
|
+
|
|
|
+ releases = []
|
|
|
+ packages = {}
|
|
|
+
|
|
|
+ def __init__(self, distribution_release, backports):
|
|
|
+ self.distribution, self.release = distribution_release.split("/")
|
|
|
+ self.backports = backports
|
|
|
+ self.apt_source = "http://{host}/{path}/{distribution} {release} main".format(
|
|
|
+ host = args.url_host,
|
|
|
+ path = args.url_path.strip("/"),
|
|
|
+ distribution = self.distribution,
|
|
|
+ release = self.release)
|
|
|
+ self.env = dict(os.environ)
|
|
|
+ if backports:
|
|
|
+ self.env.update(OTHERMIRROR = "deb " + self.apt_source)
|
|
|
+ self.releases.append(self)
|
|
|
+
|
|
|
+ @classmethod
|
|
|
+ def do_all_releases(cls):
|
|
|
+ for release in cls.releases:
|
|
|
+ release.setup_reprepro()
|
|
|
+ for release in cls.releases:
|
|
|
+ release.list_repository()
|
|
|
+ for release in cls.releases:
|
|
|
+ for release.arch, release.tag in cls.architectures.iteritems():
|
|
|
+ release.do_one_architecture()
|
|
|
+ del release.arch, release.tag
|
|
|
+
|
|
|
+ @staticmethod
|
|
|
+ def repokey(release, architecture, package):
|
|
|
+ return (release, architecture, package)
|
|
|
+
|
|
|
+ def list_repository(self):
|
|
|
+ cmd = ("reprepro", "list", self.release)
|
|
|
+ logging.info("Running %s", " ".join(cmd))
|
|
|
+ listing = subprocess.check_output(cmd, cwd = self.tree)
|
|
|
+ for line in listing.replace(":", " ").replace("|", " ").splitlines():
|
|
|
+ rel, comp, arch, pkg, ver = line.split()
|
|
|
+ key = (rel, arch, pkg)
|
|
|
+ assert key not in self.packages
|
|
|
+ self.packages[key] = ver
|
|
|
+
|
|
|
+ @property
|
|
|
+ def deb_in_repository(self):
|
|
|
+ ret = all(self.packages.get((self.release, self.arch, package)) == self.version
|
|
|
+ for package in rpki_packages)
|
|
|
+ #logging.info("Got %s looking for %r in %r", ret, self.version, self.packages)
|
|
|
+ return ret
|
|
|
+
|
|
|
+ @property
|
|
|
+ def src_in_repository(self):
|
|
|
+ ret = self.packages.get((self.release, "source", rpki_source_package)) == self.version
|
|
|
+ #logging.info("Got %s looking for %r in %r", ret, self.version, self.packages)
|
|
|
+ return ret
|
|
|
+
|
|
|
+ @property
|
|
|
+ def version(self):
|
|
|
+ return source_version + "~" + self.release
|
|
|
+
|
|
|
+ @property
|
|
|
+ def dsc(self):
|
|
|
+ return os.path.join(dsc_dir, "rpki_{}.dsc".format(self.version))
|
|
|
+
|
|
|
+ @property
|
|
|
+ def tree(self):
|
|
|
+ return os.path.join(args.apt_tree, self.distribution, "")
|
|
|
+
|
|
|
+ @property
|
|
|
+ def basefile(self):
|
|
|
+ return os.path.expanduser("~/pbuilder/{0.release}{0.tag}-base.tgz".format(self))
|
|
|
+
|
|
|
+ @property
|
|
|
+ def result(self):
|
|
|
+ return os.path.expanduser("~/pbuilder/{0.release}{0.tag}_result".format(self))
|
|
|
+
|
|
|
+ @property
|
|
|
+ def changes(self):
|
|
|
+ return os.path.join(self.result, "rpki_{0.version}_{0.arch}.changes".format(self))
|
|
|
+
|
|
|
+ def do_one_architecture(self):
|
|
|
+ logging.info("Running build for %s %s %s", self.distribution, self.release, self.arch)
|
|
|
+
|
|
|
+ if not os.path.exists(self.dsc):
|
|
|
+
|
|
|
+ logging.info("%s does not exist", self.dsc)
|
|
|
+ for fn in os.listdir(dsc_dir):
|
|
|
+ fn = os.path.join(dsc_dir, fn)
|
|
|
+ if not os.path.isdir(fn) and search_version not in fn:
|
|
|
+ logging.info("Removing %s", fn)
|
|
|
+ os.unlink(fn)
|
|
|
+ run("rm", "-rf", "debian", cwd = args.svn_tree)
|
|
|
+
|
|
|
+ logging.info("Building source package %s", self.version)
|
|
|
+ run(sys.executable, "buildtools/make-version.py", cwd = args.svn_tree)
|
|
|
+ run(sys.executable, "buildtools/build-debian-packages.py", "--version-suffix", self.release, cwd = args.svn_tree)
|
|
|
+ run("dpkg-buildpackage", "-S", "-us", "-uc", "-rfakeroot", cwd = args.svn_tree)
|
|
|
+
|
|
|
+ if not os.path.exists(self.basefile):
|
|
|
+ logging.info("Creating build environment %s %s", self.release, self.arch)
|
|
|
+ run("pbuilder-dist", self.release, self.arch, "create", env = self.env)
|
|
|
+
|
|
|
+ elif time.time() > os.stat(self.basefile).st_mtime + args.update_build_after:
|
|
|
+ logging.info("Updating build environment %s %s", self.release, self.arch)
|
|
|
+ run("pbuilder-dist", self.release, self.arch, "update", env = self.env)
|
|
|
+
|
|
|
+ if not self.deb_in_repository:
|
|
|
+
|
|
|
+ for fn in os.listdir(self.result):
|
|
|
+ fn = os.path.join(self.result, fn)
|
|
|
+ logging.info("Removing %s", fn)
|
|
|
+ os.unlink(fn)
|
|
|
+
|
|
|
+ logging.info("Building binary packages %s %s %s", self.release, self.arch, self.version)
|
|
|
+ run("pbuilder-dist", self.release, self.arch, "build", "--keyring", args.keyring, self.dsc, env = self.env)
|
|
|
+
|
|
|
+ logging.info("Updating repository for %s %s %s", self.release, self.arch, self.version)
|
|
|
+ run("reprepro", "--ignore=wrongdistribution", "include", self.release, self.changes, cwd = self.tree)
|
|
|
+
|
|
|
+ if not self.src_in_repository:
|
|
|
+ logging.info("Updating repository for %s source %s", self.release, self.version)
|
|
|
+ run("reprepro", "--ignore=wrongdistribution", "includedsc", self.release, self.dsc, cwd = self.tree)
|
|
|
+
|
|
|
+ def setup_reprepro(self):
|
|
|
+
|
|
|
+ logging.info("Configuring reprepro for %s/%s", self.distribution, self.release)
|
|
|
+
|
|
|
+ dn = os.path.join(self.tree, "conf")
|
|
|
+ if not os.path.isdir(dn):
|
|
|
+ logging.info("Creating %s", dn)
|
|
|
+ os.makedirs(dn)
|
|
|
+
|
|
|
+ fn = os.path.join(self.tree, "conf", "distributions")
|
|
|
+ distributions = open(fn, "r").read() if os.path.exists(fn) else ""
|
|
|
+ if "Codename: {0.release}\n".format(self) not in distributions:
|
|
|
+ logging.info("%s %s", "Editing" if distributions else "Creating", fn)
|
|
|
+ with open(fn, "w") as f:
|
|
|
+ if distributions:
|
|
|
+ f.write(distributions)
|
|
|
+ f.write("\n")
|
|
|
+ f.write(dedent("""\
|
|
|
+ Origin: rpki.net
|
|
|
+ Label: rpki.net {self.distribution} repository
|
|
|
+ Codename: {self.release}
|
|
|
+ Architectures: {architectures} source
|
|
|
+ Components: main
|
|
|
+ Description: rpki.net {Distribution} APT Repository
|
|
|
+ SignWith: yes
|
|
|
+ DebOverride: override.{self.release}
|
|
|
+ DscOverride: override.{self.release}
|
|
|
+ """.format(
|
|
|
+ self = self,
|
|
|
+ Distribution = self.distribution.capitalize(),
|
|
|
+ architectures = " ".join(self.architectures))))
|
|
|
+
|
|
|
+ fn = os.path.join(self.tree, "conf", "options")
|
|
|
+ if not os.path.exists(fn):
|
|
|
+ logging.info("Creating %s", fn)
|
|
|
+ with open(fn, "w") as f:
|
|
|
+ f.write(dedent("""\
|
|
|
+ verbose
|
|
|
+ ask-passphrase
|
|
|
+ basedir .
|
|
|
+ """))
|
|
|
+
|
|
|
+ fn = os.path.join(self.tree, "conf", "override." + self.release)
|
|
|
+ if not os.path.exists(fn):
|
|
|
+ logging.info("Creating %s", fn)
|
|
|
+ with open(fn, "w") as f:
|
|
|
+ for pkg in self.backports:
|
|
|
+ f.write(dedent("""\
|
|
|
+ {pkg:<30} Priority optional
|
|
|
+ {pkg:<30} Section python
|
|
|
+ """.format(pkg = pkg)))
|
|
|
+ f.write(dedent("""\
|
|
|
+ rpki-ca Priority extra
|
|
|
+ rpki-ca Section net
|
|
|
+ rpki-rp Priority extra
|
|
|
+ rpki-rp Section net
|
|
|
+ """))
|
|
|
+
|
|
|
+ fn = os.path.join(args.apt_tree, "rpki.{}.list".format(self.release))
|
|
|
+ if not os.path.exists(fn):
|
|
|
+ logging.info("Creating %s", fn)
|
|
|
+ with open(fn, "w") as f:
|
|
|
+ f.write(dedent("""\
|
|
|
+ deb {self.apt_source}
|
|
|
+ deb-src {self.apt_source}
|
|
|
+ """.format(self = self)))
|
|
|
+
|
|
|
+# Load whatever releases the user specified
|
|
|
+
|
|
|
+for r in args.releases:
|
|
|
+ Release(r, args.backports)
|
|
|
+
|
|
|
+# Do all the real work.
|
|
|
+
|
|
|
+Release.do_all_releases()
|
|
|
+
|
|
|
+# Upload results, maybe. We do this in two stages, to minimize the window
|
|
|
+# during which the uploaded repository might be in an inconsistent state.
|
|
|
+
|
|
|
+def rsync(*flags):
|
|
|
+ cmd = ["rsync", "--archive", "--itemize-changes",
|
|
|
+ "--rsh", "ssh -l {}".format(args.apt_user)]
|
|
|
+ cmd.extend(flags)
|
|
|
+ cmd.append(args.apt_tree)
|
|
|
+ cmd.append("rsync://{host}/{path}/".format(host = args.url_host,
|
|
|
+ path = args.url_path.strip("/")))
|
|
|
+ if upload:
|
|
|
+ logging.info("Synching repository to %s with flags %s",
|
|
|
+ cmd[-1], " ".join(flags))
|
|
|
+ run(*cmd)
|
|
|
+ else:
|
|
|
+ logging.info("Would have synched repository to %s with flags %",
|
|
|
+ cmd[-1], " ".join(flags))
|
|
|
+
|
|
|
+rsync("--ignore-existing")
|
|
|
+
|
|
|
+rsync("--exclude", "HEADER.html",
|
|
|
+ "--exclude", "HEADER.css",
|
|
|
+ "--delete", "--delete-delay")
|
|
|
+
|
|
|
+logging.info("Done")
|
