baiji/baiji

#!/usr/bin/env python

# Copyright (c) 2018-2019, Grunchweather Associates
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
# AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
# PERFORMANCE OF THIS SOFTWARE.

import debian.changelog
import debian.deb822
import subprocess
import textwrap
import argparse
import tempfile
import tarfile
import hashlib
import shutil
import sys
import os

# Python decorator voodoo to simplify argparse subparser setup.

def arg(*a, **k):
    return a, k

def cmd(*args):
    def wrapper(func):
        def setup(subp):
            for a, k in args:
                subp.add_argument(*a, **k)
            subp.set_defaults(func = func)
        func._setup_parser = setup
        return func
    return wrapper


# Context manager for temporary directories.

class tempdir(object):

    def __enter__(self):
        self.dn = tempfile.mkdtemp()
        return self.dn

    def __exit__(self, *oops):
        shutil.rmtree(self.dn)

# Docker process, mostly a context manager around subprocess.Popen.
# We could use the native Python Docker interface, but the packaged
# Debian version of that has a wildly different API than the version
# on GitHub.

class Docker(subprocess.Popen):

    class DockerError(Exception):
        "Docker returned failure."

    def __init__(self, *args, **kwargs):
        super(Docker, self).__init__(("docker",) + args, **kwargs)

    def __enter__(self):
        return self

    def __exit__(self, *oops):
        if any(oops):
            return
        if self.stdin:
            self.stdin.close()
        status = self.wait()
        if status:
            raise self.DockerError()

# Filter which acts like fakeroot for tarfile.TarFile.add() 

def fakeroot_filter(info):
    info.uname = info.gname = "root"
    info.uid   = info.gid   = 0
    return info


# Commands

@cmd(arg("--dist", default = "jessie",
         help = "distribution for base docker image"),
     arg("--tag",  default = "baiji:jessie",
         help = "tag to use for constructed base docker image"),
)
def create(args):
    """
    Construct a base Docker image.

    This is mostly just the output of debootstrap, with a bit of extra
    setup to include git, build-essentials, and fakeroot.
    """

    with tempdir() as dn:
        subprocess.check_call(("fakeroot", "/usr/sbin/debootstrap",
                               "--foreign", "--variant=buildd", args.dist, dn))
        with Docker("import", "-", args.tag, stdin = subprocess.PIPE) as docker:
            with tarfile.open(mode = "w|", fileobj = docker.stdin) as tar:
                tar.add(dn, ".", filter = fakeroot_filter)

    with Docker("build", "-t", args.tag, "-", stdin = subprocess.PIPE) as docker:
        docker.communicate(textwrap.dedent('''\
            FROM {args.tag}
            RUN sed -i '/mount -t proc /d; /mount -t sysfs /d' /debootstrap/functions && \\
                /debootstrap/debootstrap --second-stage
            RUN apt-get update && \\
                apt-get install -y --no-install-recommends build-essential fakeroot git apt-utils
            RUN useradd -U -m -d /build baiji
            WORKDIR /build
        '''.format(args = args)))


@cmd(arg("--tag",  default = "baiji:jessie",
         help = "tag of base docker image to update"),
)
def update(args):
    """
    Update a base Docker image.
    """

    with Docker("build", "-t", args.tag, "-", stdin = subprocess.PIPE) as docker:
       docker.communicate(textwrap.dedent('''\
           FROM {args.tag}
           RUN apt-get update && \\
               apt-get upgrade -y --with-new-pkgs --no-install-recommends && \\
               apt-get autoremove && \\
               apt-get clean
       '''.format(args = args)))


@cmd(arg("--tag", default = "baiji:jessie",
         help = "tag of base docker image to use"),
     arg("--dsc",  type = argparse.FileType("r"),
         help = ".dsc file to build"),
     arg("--local-package", default = [], nargs = "+",
         help = "local packages to make available to build"),
     arg("--force-image", action = "store_true",
         help = "don't rebuild Docker image"),
     arg("--dont-clean", action = "store_true",
         help = "don't clean up old Docker images"),
     arg("--just-image", action = "store_true",
         help = "don't build, just generate Docker image"),
)
def build(args):
    """
    Build a binary package given a source package.

    If no source package supplied, try to build one from the current
    directory, like debuild.
    """

    if args.dsc is None:
        try:
            subprocess.check_call(("dpkg-buildpackage", "-S", "-us", "-uc", "-rfakeroot"))
        except Exception as e:
            sys.exit("Couldn't build source package: {!s}".format(e))
        try:
            with open("debian/changelog") as f:
                changelog = debian.changelog.Changelog(f)
            args.dsc = open("../{}_{}{}.dsc".format(
                changelog.package, changelog.upstream_version,
                "" if changelog.debian_revision is None else "-" + changelog.debian_revision))
        except Exception as e:
            sys.exit("Couldn't find .dsc file: {!s}".format(e))

    dsc = debian.deb822.Dsc(args.dsc)

    dummy = debian.deb822.Deb822()
    dummy_name = "baiji-depends-" + dsc["Source"]
    dummy_fn = "{}_{}_all.deb".format(dummy_name, dsc["Version"])
    dummy["Depends"] = ", ".join(dsc[i]
                                 for i in ("Build-Depends",
                                           "Build-Depends-Indep",
                                           "Build-Depends-Arch")
                                 if i in dsc)

    dummy["Package"] = dummy_name
    for tag in ("Version", "Maintainer", "Homepage"):
        dummy[tag] = dsc[tag]

    build_image_hash = hashlib.sha1(dummy["Depends"]).hexdigest()
    build_image_name = "baiji/build/{}".format(dsc["Source"])
    build_image_vers = "{}-{}".format(dsc["Version"], build_image_hash)
    build_image      = "{}:{}".format(build_image_name, build_image_vers)

    with Docker("image", "ls", build_image_name, "--format", "{{.Tag}}",
                stdout = subprocess.PIPE) as docker:
        build_image_versions = set(docker.stdout.read().split())

    build_image_exists = build_image_vers in build_image_versions

    make_build_image = args.force_image or not build_image_exists

    if args.dont_clean:
        build_image_versions = { build_image_vers } if build_image_exists else set()

    if not make_build_image:
        build_image_versions.discard(build_image_vers)

    for v in build_image_versions:
        with Docker("rmi", "{}:{}".format(build_image_name, v)):
            pass

    if make_build_image:

        with tempdir() as dn:
            equivs = subprocess.Popen(("equivs-build", "/dev/stdin"),
                                      stdin = subprocess.PIPE, stdout = subprocess.PIPE, cwd = dn)
            equivs.communicate(str(dummy))
            if equivs.wait():
                sys.exit("Couldn't generate dummy dependency package")

            with open(os.path.join(dn, "Dockerfile"), "w") as f:
                f.write(textwrap.dedent('''\
                    FROM {args.tag}
                    COPY build.sh  /baiji/
                    COPY micro-apt /micro-apt/
                    RUN  cd /micro-apt && \\
                         apt-ftparchive packages . > Packages
                    RUN  cd /etc/apt/sources.list.d && \\
                         echo 'deb [trusted=yes] file:///micro-apt ./' > micro-apt.list
                    RUN  apt-get update && \\
                         apt-get install -y --no-install-recommends {dummy_name} && \\
                         apt-get clean
                    USER baiji
                    '''.format(args = args, dummy_name = dummy_name)))

            with open(os.path.join(dn, "build.sh"), "w") as f:
                f.write(textwrap.dedent('''\
                    #!/bin/bash -
                    set -eo pipefail
                    arch=`dpkg-architecture -qDEB_BUILD_ARCH`
                    dpkg-source -x /source/{source}_{version}.dsc {source}-{version}
                    cd {source}-{version}
                    dpkg-buildpackage -b -uc -us 2>&1 | tee ../{source}_{version}_$arch.build
                    cd ..
                    rm -rf {source}-{version}
                    '''.format(source = dsc["Source"], version = dsc["Version"])))

            with Docker("build", "-t", build_image, "-", stdin = subprocess.PIPE) as docker:
                with tarfile.open(mode = "w|", fileobj = docker.stdin) as tar:
                    for fn in ("Dockerfile", "build.sh"):
                        tar.add(os.path.join(dn, fn), fn, filter = fakeroot_filter)
                    for pkg in [os.path.join(dn, dummy_fn)] + args.local_package:
                        tar.add(pkg, os.path.join("micro-apt", os.path.basename(pkg)),
                                filter = fakeroot_filter)

    if not args.just_image:

        container_name = "baiji-build-{}".format(dsc["Source"])
        dn = os.path.dirname(args.dsc.name)

        with Docker("run", "-i", "--name", container_name, "--network", "none",
                    "-v", "{}:/source:ro".format(os.path.abspath(dn)),
                    build_image, "/bin/bash", "-x", "/baiji/build.sh"):
            pass

        with Docker("cp", "{}:/build/.".format(container_name), "-",
                    stdout = subprocess.PIPE) as docker:
            with tarfile.open(mode = "r|*", fileobj = docker.stdout) as tar:
                for member in tar:
                    fn = os.path.basename(member.name)
                    if any(fn.endswith(fn2) for fn2 in (".deb", ".changes")):
                        with open(os.path.join(dn, fn), "w") as f:
                            f.write(tar.extractfile(member).read())

        with Docker("rm", container_name):
            pass


# Parse arguments and dispatch to one of the commands above.

def main():
    HF = type("HF", (argparse.ArgumentDefaultsHelpFormatter,
                     argparse.RawDescriptionHelpFormatter), {})
    parser = argparse.ArgumentParser(formatter_class = HF, description = __doc__)
    subparsers = parser.add_subparsers(title = "Commands", metavar = "")
    for name in sorted(globals()):
        func = globals()[name]
        try:
            setup_parser = func._setup_parser
        except:
            continue
        setup_parser(subparsers.add_parser(name.replace("_", "-"),
                                           formatter_class = HF,
                                           description = func.__doc__,
                                           help = (func.__doc__ or "").lstrip().partition("\n")[0]))
    args = parser.parse_args()
    args.func(args)

if __name__ == "__main__":
    main()