#!/usr/bin/env python import debian.changelog import debian.deb822 import subprocess import textwrap import argparse import tempfile import tarfile import hashlib import shutil import sys import os # Python decorator voodoo to simplify argparse subparser setup. def arg(*a, **k): return a, k def cmd(*args): def wrapper(func): def setup(subp): for a, k in args: subp.add_argument(*a, **k) subp.set_defaults(func = func) func._setup_parser = setup return func return wrapper # Context manager for temporary directories. class tempdir(object): def __enter__(self): self.dn = tempfile.mkdtemp() return self.dn def __exit__(self, *oops): shutil.rmtree(self.dn) # Docker process, mostly a context manager around subprocess.Popen. # We could use the native Python Docker interface, but the packaged # Debian version of that has a wildly different API than the version # on GitHub. class Docker(subprocess.Popen): class DockerError(Exception): "Docker returned failure." def __init__(self, *args, **kwargs): super(Docker, self).__init__(("docker",) + args, **kwargs) def __enter__(self): return self def __exit__(self, *oops): if any(oops): return if self.stdin: self.stdin.close() status = self.wait() if status: raise self.DockerError() # Filter which acts like fakeroot for tarfile.TarFile.add() def fakeroot_filter(info): info.uname = info.gname = "root" info.uid = info.gid = 0 return info # Commands @cmd(arg("--dist", default = "jessie", help = "distribution for base docker image"), arg("--tag", default = "baiji:jessie", help = "tag to use for constructed base docker image"), ) def create(args): """ Construct a base Docker image. This is mostly just the output of debootstrap, with a bit of extra setup to include git, build-essentials, and fakeroot. """ with tempdir() as dn: subprocess.check_call(("fakeroot", "/usr/sbin/debootstrap", "--foreign", "--variant=buildd", args.dist, dn)) with Docker("import", "-", args.tag, stdin = subprocess.PIPE) as docker: with tarfile.open(mode = "w|", fileobj = docker.stdin) as tar: tar.add(dn, ".", filter = fakeroot_filter) with Docker("build", "-t", args.tag, "-", stdin = subprocess.PIPE) as docker: docker.communicate(textwrap.dedent('''\ FROM {args.tag} RUN sed -i '/mount -t proc /d; /mount -t sysfs /d' /debootstrap/functions && \\ /debootstrap/debootstrap --second-stage RUN apt-get update && \\ apt-get install -y --no-install-recommends build-essential fakeroot git apt-utils RUN useradd -U -m -d /build baiji WORKDIR /build '''.format(args = args))) @cmd(arg("--tag", default = "baiji:jessie", help = "tag of base docker image to update"), ) def update(args): """ Update a base Docker image. """ with Docker("build", "-t", args.tag, "-", stdin = subprocess.PIPE) as docker: docker.communicate(textwrap.dedent('''\ FROM {args.tag} RUN apt-get update && \\ apt-get upgrade -y --with-new-pkgs --no-install-recommends && \\ apt-get autoremove && \\ apt-get clean '''.format(args = args))) @cmd(arg("--tag", default = "baiji:jessie", help = "tag of base docker image to use"), arg("--dsc", type = argparse.FileType("r"), help = ".dsc file to build"), arg("--local-package", default = [], nargs = "+", help = "local packages to make available to build"), arg("--force-image", action = "store_true", help = "don't rebuild Docker image"), arg("--dont-clean", action = "store_true", help = "don't clean up old Docker images"), arg("--just-image", action = "store_true", help = "don't build, just generate Docker image"), ) def build(args): """ Build a binary package given a source package. If no source package supplied, try to build one from the current directory, like debuild. """ if args.dsc is None: try: subprocess.check_call(("dpkg-buildpackage", "-S", "-us", "-uc", "-rfakeroot")) except Exception as e: sys.exit("Couldn't build source package: {!s}".format(e)) try: with open("debian/changelog") as f: changelog = debian.changelog.Changelog(f) args.dsc = open("../{}_{}{}.dsc".format( changelog.package, changelog.upstream_version, "" if changelog.debian_revision is None else "-" + changelog.debian_revision)) except Exception as e: sys.exit("Couldn't find .dsc file: {!s}".format(e)) dsc = debian.deb822.Dsc(args.dsc) dummy = debian.deb822.Deb822() dummy_name = "baiji-depends-" + dsc["Source"] dummy_fn = "{}_{}_all.deb".format(dummy_name, dsc["Version"]) dummy["Depends"] = ", ".join(dsc[i] for i in ("Build-Depends", "Build-Depends-Indep", "Build-Depends-Arch") if i in dsc) dummy["Package"] = dummy_name for tag in ("Version", "Maintainer", "Homepage"): dummy[tag] = dsc[tag] build_image_name = "baiji/build/{}/{}".format(dsc["Source"], dsc["Version"]) build_image_hash = hashlib.sha1(dummy["Depends"]).hexdigest() build_image = "{}:{}".format(build_image_name, build_image_hash) with Docker("image", "ls", build_image_name, "--format", "{{.Tag}}", stdout = subprocess.PIPE) as docker: build_image_hashes = set(docker.stdout.read().split()) build_image_exists = build_image_hash in build_image_hashes make_build_image = args.force_image or not build_image_exists if args.dont_clean: build_image_hashes = { build_image_hash } if build_image_exists else set() if not make_build_image: build_image_hashes.discard(build_image_hash) for h in build_image_hashes: with Docker("rmi", "{}:{}".format(build_image_name, h)): pass if make_build_image: with tempdir() as dn: equivs = subprocess.Popen(("equivs-build", "/dev/stdin"), stdin = subprocess.PIPE, stdout = subprocess.PIPE, cwd = dn) equivs.communicate(str(dummy)) if equivs.wait(): sys.exit("Couldn't generate dummy dependency package") with open(os.path.join(dn, "Dockerfile"), "w") as f: f.write(textwrap.dedent('''\ FROM {args.tag} COPY build.sh /baiji/ COPY micro-apt /micro-apt/ RUN cd /micro-apt && \\ apt-ftparchive packages . > Packages RUN cd /etc/apt/sources.list.d && \\ echo 'deb [trusted=yes] file:///micro-apt ./' > micro-apt.list RUN apt-get update && \\ apt-get install -y --no-install-recommends {dummy_name} && \\ apt-get clean USER baiji '''.format(args = args, dummy_name = dummy_name))) with open(os.path.join(dn, "build.sh"), "w") as f: f.write(textwrap.dedent('''\ #!/bin/bash - set -eo pipefail arch=`dpkg-architecture -qDEB_BUILD_ARCH` dpkg-source -x /source/{source}_{version}.dsc cd {source}-{version} dpkg-buildpackage -b -uc -us 2>&1 | tee ../{source}_{version}_$arch.build cd .. rm -rf {source}-{version} '''.format(source = dsc["Source"], version = dsc["Version"]))) with Docker("build", "-t", build_image, "-", stdin = subprocess.PIPE) as docker: with tarfile.open(mode = "w|", fileobj = docker.stdin) as tar: for fn in ("Dockerfile", "build.sh"): tar.add(os.path.join(dn, fn), fn, filter = fakeroot_filter) for pkg in [os.path.join(dn, dummy_fn)] + args.local_package: tar.add(pkg, os.path.join("micro-apt", os.path.basename(pkg)), filter = fakeroot_filter) if not args.just_image: container_name = "baiji-build-{}".format(dsc["Source"]) dn = os.path.dirname(args.dsc.name) with Docker("run", "-i", "--name", container_name, "--network", "none", "-v", "{}:/source:ro".format(os.path.abspath(dn)), build_image, "/bin/bash", "-x", "/baiji/build.sh"): pass with Docker("cp", "{}:/build/.".format(container_name), "-", stdout = subprocess.PIPE) as docker: with tarfile.open(mode = "r|*", fileobj = docker.stdout) as tar: for member in tar: fn = os.path.basename(member.name) if any(fn.endswith(fn2) for fn2 in (".deb", ".changes")): with open(os.path.join(dn, fn), "w") as f: f.write(tar.extractfile(member).read()) with Docker("rm", container_name): pass # Parse arguments and dispatch to one of the commands above. def main(): HF = type("HF", (argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter), {}) parser = argparse.ArgumentParser(formatter_class = HF, description = __doc__) subparsers = parser.add_subparsers(title = "Commands", metavar = "") for name in sorted(globals()): func = globals()[name] try: setup_parser = func._setup_parser except: continue setup_parser(subparsers.add_parser(name.replace("_", "-"), formatter_class = HF, description = func.__doc__, help = (func.__doc__ or "").lstrip().partition("\n")[0])) args = parser.parse_args() args.func(args) if __name__ == "__main__": main()