sra/rpki.net - Dragon Research Labs RPKI Toolkit

diff options
Diffstat (limited to 'openssl/trunk/doc/README')
0 files changed, 0 insertions, 0 deletions
'>77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336
#!/usr/bin/python
#
# $Id$
#
# Copyright (C) 2014  Dragon Research Labs ("DRL")
# Portions copyright (C) 2013  Internet Systems Consortium ("ISC")
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notices and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND DRL AND ISC DISCLAIM ALL
# WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS.  IN NO EVENT SHALL DRL OR
# ISC BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
# DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA
# OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
# TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
# PERFORMANCE OF THIS SOFTWARE.

"""
Debian/Ubuntu package build tool, based on pbuilder-dist and reprepro.
"""

import os
import sys
import time
import fcntl
import errno
import socket
import logging
import argparse
import subprocess

from textwrap import dedent

rpki_packages = ("rpki-rp", "rpki-ca")
rpki_source_package = "rpki"

parser = argparse.ArgumentParser(description = __doc__,
                                 formatter_class = argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--debug", action = "store_true",
                    help = "enable debugging code")
parser.add_argument("--update-build-after", type = int, default = 7 * 24 * 60 * 60,
                    help = "interval (in seconds) after which we should update the pbuilder environment")
parser.add_argument("--lockfile", default = os.path.expanduser("~/builder.lock"),
                    help = "avoid collisions between multiple instances of this script")
parser.add_argument("--keyring",  default = os.path.expanduser("~/.gnupg/pubring.gpg"),
                    help = "PGP keyring")
parser.add_argument("--svn-tree", default = os.path.expanduser("~/source/trunk/"),
                    help = "subversion tree")
parser.add_argument("--apt-tree", default = os.path.expanduser("~/repository/"),
                    help = "reprepro repository")
parser.add_argument("--apt-user", default = "aptbot",
                    help = "username for uploading apt repository to public web server")
parser.add_argument("--url-host", default = "download.rpki.net",
                    help = "hostname of public web server")
parser.add_argument("--url-path", default = "/APT",
                    help = "path of apt repository on public web server")
parser.add_argument("--backports", nargs = "+", default = ["python-django", "python-tornado"],
                    help = "backports needed for this build")
parser.add_argument("--releases", nargs = "+", default = ["ubuntu/trusty", "debian/wheezy", "ubuntu/precise"],
                    help = "releases for which to build")
args = parser.parse_args()

# Maybe logging should be configurable too.  Later.

logging.basicConfig(level = logging.INFO, timefmt = "%Y-%m-%dT%H:%M:%S",
                    format = "%(asctime)s [%(process)d] %(levelname)s %(message)s")

upload = socket.getfqdn() == "build-u.rpki.net"

def run(*cmd, **kwargs):
    if args.debug:
        #logging.info("Running %r %r", cmd, kwargs)
        logging.info("Running %s", " ".join(cmd))
    subprocess.check_call(cmd, **kwargs)

# Getting this to work right also required adding:
#
#   DEBBUILDOPTS="-b"
#
# to /etc/pbuilderrc; without this, reprepro (eventually, a year after
# we set this up) started failing to incorporate some of the built
# packages, because the regenerated source packages had different
# checksums than the ones loaded initially.  See:
#
# http://stackoverflow.com/questions/21563872/reprepro-complains-about-the-generated-pbuilder-debian-tar-gz-archive-md5
#
# Putting stuff in ~/.pbuilderrc didn't work with pbuilder-dist when I
# tried it last year, this may just be that sudo isn't configured to
# pass HOME through, thus pbuilder is looking for ~root/.pbuilderrc.
# Worth trying again at some point but not all that critical.

logging.info("Starting")

try:
    lock = os.open(args.lockfile, os.O_RDONLY | os.O_CREAT | os.O_NONBLOCK, 0666)
    fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except (IOError, OSError), e:
    sys.exit(0 if e.errno == errno.EAGAIN else "Error {!r} opening lock {!r}".format(e, args.lockfile))

run("svn", "--quiet", "update", cwd = args.svn_tree)

source_version = subprocess.check_output(("svnversion", "-c"), cwd = args.svn_tree).strip().split(":")[-1]

if not source_version.isdigit() and not args.debug:
    sys.exit("Sources don't look pristine, not building ({!r})".format(source_version))

source_version = "0." + source_version
search_version = "_" + source_version + "~"

dsc_dir = os.path.abspath(os.path.join(args.svn_tree, ".."))

if not os.path.isdir(args.apt_tree):
    logging.info("Creating %s", args.apt_tree)
    os.makedirs(args.apt_tree)

fn = os.path.join(args.apt_tree, "apt-gpg-key.asc")
if not os.path.exists(fn):
    logging.info("Creating %s", fn)
    run("gpg", "--export", "--armor", "--keyring", args.keyring, stdout = open(fn, "w"))

class Release(object):

    architectures = dict(amd64 = "", i386 = "-i386")

    releases = []
    packages = {}

    def __init__(self, distribution_release, backports):
        self.distribution, self.release = distribution_release.split("/")
        self.backports = backports
        self.apt_source = "http://{host}/{path}/{distribution} {release} main".format(
            host = args.url_host,
            path = args.url_path.strip("/"),
            distribution = self.distribution,
            release = self.release)
        self.env = dict(os.environ)
        if backports:
            self.env.update(OTHERMIRROR = "deb " + self.apt_source)
        self.releases.append(self)

    @classmethod
    def do_all_releases(cls):
        for release in cls.releases:
            release.setup_reprepro()
        for release in cls.releases:
            release.list_repository()
        for release in cls.releases:
            for release.arch, release.tag in cls.architectures.iteritems():
                release.do_one_architecture()
            del release.arch, release.tag

    @staticmethod
    def repokey(release, architecture, package):
        return (release, architecture, package)

    def list_repository(self):
        cmd = ("reprepro", "list", self.release)
        logging.info("Running %s", " ".join(cmd))
        listing = subprocess.check_output(cmd, cwd = self.tree)
        for line in listing.replace(":", " ").replace("|", " ").splitlines():
            rel, comp, arch, pkg, ver = line.split()
            key = (rel, arch, pkg)
            assert key not in self.packages
            self.packages[key] = ver

    @property
    def deb_in_repository(self):
        return all(self.packages.get((self.release, self.arch, package)) == self.version
                   for package in rpki_packages)

    @property
    def src_in_repository(self):
        return self.packages.get((self.release, "source", rpki_source_package)) == self.version

    @property
    def version(self):
        return source_version + "~" + self.release

    @property
    def dsc(self):
        return os.path.join(dsc_dir, "rpki_{}.dsc".format(self.version))

    @property
    def tree(self):
        return os.path.join(args.apt_tree, self.distribution, "")

    @property
    def basefile(self):
        return os.path.expanduser("~/pbuilder/{0.release}{0.tag}-base.tgz".format(self))

    @property
    def result(self):
        return os.path.expanduser("~/pbuilder/{0.release}{0.tag}_result".format(self))

    @property
    def changes(self):
        return os.path.join(self.result, "rpki_{0.version}_{0.arch}.changes".format(self))

    def do_one_architecture(self):
        logging.info("Running build for %s %s %s", self.distribution, self.release, self.arch)

        if not os.path.exists(self.dsc):
            logging.info("Building source package %s", self.version)
            for fn in os.listdir(dsc_dir):
                fn = os.path.join(dsc_dir, fn)
                if not os.path.isdir(fn) and search_version not in fn:
                    os.unlink(fn)
            run("rm", "-rf", "debian", cwd = args.svn_tree)
            run(sys.executable, "buildtools/make-version.py", cwd = args.svn_tree)
            run(sys.executable, "buildtools/build-debian-packages.py", "--version-suffix", self.release, cwd = args.svn_tree)
            run("dpkg-buildpackage", "-S", "-us", "-uc", "-rfakeroot", cwd = args.svn_tree)

        if not os.path.exists(self.basefile):
            logging.info("Creating build environment %s %s", self.release, self.arch)
            run("pbuilder-dist", self.release, self.arch, "create", env = self.env)

        elif time.time() > os.stat(self.basefile).st_mtime + args.update_build_after:
            logging.info("Updating build environment %s %s", self.release, self.arch)
            run("pbuilder-dist", self.release, self.arch, "update", env = self.env)

        if not os.path.exists(self.changes):
            logging.info("Building binary packages %s %s %s", self.release, self.arch, self.version)
            for fn in os.listdir(self.result):
                os.unlink(os.path.join(self.result, fn))
            run("pbuilder-dist", self.release, self.arch, "build", "--keyring", args.keyring, self.dsc, env = self.env)

        if not self.deb_in_repository:
            logging.info("Updating repository for %s %s %s", self.release, self.arch, self.version)
            run("reprepro", "--ignore=wrongdistribution", "include", self.release, self.changes, cwd = self.tree)

        if not self.src_in_repository:
            logging.info("Updating repository for %s source %s", self.release, self.version)
            run("reprepro", "--ignore=wrongdistribution", "includedsc", self.release, self.dsc, cwd = self.tree)

    def setup_reprepro(self):

        logging.info("Configuring reprepro for %s/%s", self.distribution, self.release)

        dn = os.path.join(self.tree, "conf")
        if not os.path.isdir(dn):
            logging.info("Creating %s", dn)
            os.makedirs(dn)

        fn = os.path.join(self.tree, "conf", "distributions")
        distributions = open(fn, "r").read() if os.path.exists(fn) else ""
        if "Codename: {0.release}\n".format(self) not in distributions:
            logging.info("%s %s", "Editing" if distributions else "Creating", fn)
            with open(fn, "w") as f:
                if distributions:
                    f.write(distributions)
                    f.write("\n")
                f.write(dedent("""\
                               Origin: rpki.net
                               Label: rpki.net {self.distribution} repository
                               Codename: {self.release}
                               Architectures: {architectures} source
                               Components: main
                               Description: rpki.net {Distribution} APT Repository
                               SignWith: yes
                               DebOverride: override.{self.release}
                               DscOverride: override.{self.release}
                               """.format(
                            self = self,
                            Distribution  = self.distribution.capitalize(),
                            architectures = " ".join(self.architectures))))

        fn = os.path.join(self.tree, "conf", "options")
        if not os.path.exists(fn):
            logging.info("Creating %s", fn)
            with open(fn, "w") as f:
                f.write(dedent("""\
                               verbose
                               ask-passphrase
                               basedir .
                               """))

        fn = os.path.join(self.tree, "conf", "override." + self.release)
        if not os.path.exists(fn):
            logging.info("Creating %s", fn)
            with open(fn, "w") as f:
                for pkg in self.backports:
                    f.write(dedent("""\
                                   {pkg:<30}   Priority        optional
                                   {pkg:<30}   Section         python
                                   """.format(pkg = pkg)))
                f.write(dedent("""\
                               rpki-ca                 Priority        extra
                               rpki-ca                 Section         net
                               rpki-rp                 Priority        extra
                               rpki-rp                 Section         net
                               """))

        fn = os.path.join(args.apt_tree, "rpki.{}.list".format(self.release))
        if not os.path.exists(fn):
            logging.info("Creating %s", fn)
            with open(fn, "w") as f:
                f.write(dedent("""\
                               deb {self.apt_source}
                               deb-src {self.apt_source}
                               """.format(self = self)))

# Load whatever releases the user specified

for r in args.releases:
    Release(r, args.backports)

# Do all the real work.

Release.do_all_releases()

# Upload results, maybe.

# This should change to use the rsync:// over ssh hack so server can provide an rsyncd.conf
# tuning access.  See {bob,bikeshed}.cryptech.is configuration.

srv_path = "{user}@{host}:/usr/local/www/data/{host}/{path}/".format(user = args.apt_user,
                                                                     host = args.url_host,
                                                                     path = args.url_path.strip("/"))

if upload:
    logging.info("Synching repository to %s", srv_path)
    run("rsync", "-ai4",
        "--ignore-existing",
        args.apt_tree, srv_path)
    run("rsync", "-ai4",
        "--exclude", "HEADER.html",
        "--exclude", "HEADER.css",
        "--delete", "--delete-delay",
        args.apt_tree, srv_path)
else:
    logging.info("Would have synched repository to %s", srv_path)

logging.info("Done")

context:
space:
mode: