rpki-pbuilder.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351
  1. #!/usr/bin/python
  2. #
  3. # $Id$
  4. #
  5. # Copyright (C) 2014 Dragon Research Labs ("DRL")
  6. # Portions copyright (C) 2013 Internet Systems Consortium ("ISC")
  7. #
  8. # Permission to use, copy, modify, and distribute this software for any
  9. # purpose with or without fee is hereby granted, provided that the above
  10. # copyright notices and this permission notice appear in all copies.
  11. #
  12. # THE SOFTWARE IS PROVIDED "AS IS" AND DRL AND ISC DISCLAIM ALL
  13. # WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
  14. # WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL OR
  15. # ISC BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
  16. # DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA
  17. # OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
  18. # TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
  19. # PERFORMANCE OF THIS SOFTWARE.
  20. """
  21. Debian/Ubuntu package build tool, based on pbuilder-dist and reprepro.
  22. """
  23. import os
  24. import sys
  25. import time
  26. import fcntl
  27. import errno
  28. import socket
  29. import logging
  30. import argparse
  31. import subprocess
  32. from textwrap import dedent
  33. rpki_packages = ("rpki-rp", "rpki-ca")
  34. rpki_source_package = "rpki"
  35. parser = argparse.ArgumentParser(description = __doc__,
  36. formatter_class = argparse.ArgumentDefaultsHelpFormatter)
  37. parser.add_argument("--debug", action = "store_true",
  38. help = "enable debugging code")
  39. parser.add_argument("--update-build-after", type = int, default = 7 * 24 * 60 * 60,
  40. help = "interval (in seconds) after which we should update the pbuilder environment")
  41. parser.add_argument("--lockfile", default = os.path.expanduser("~/builder.lock"),
  42. help = "avoid collisions between multiple instances of this script")
  43. parser.add_argument("--keyring", default = os.path.expanduser("~/.gnupg/pubring.gpg"),
  44. help = "PGP keyring")
  45. parser.add_argument("--svn-tree", default = os.path.expanduser("~/source/trunk/"),
  46. help = "subversion tree")
  47. parser.add_argument("--apt-tree", default = os.path.expanduser("~/repository/"),
  48. help = "reprepro repository")
  49. parser.add_argument("--apt-user", default = "aptbot",
  50. help = "username for uploading apt repository to public web server")
  51. parser.add_argument("--url-host", default = "download.rpki.net",
  52. help = "hostname of public web server")
  53. parser.add_argument("--url-path", default = "/APT",
  54. help = "path of apt repository on public web server")
  55. parser.add_argument("--backports", nargs = "+", default = ["python-django", "python-tornado"],
  56. help = "backports needed for this build")
  57. parser.add_argument("--releases", nargs = "+", default = ["ubuntu/trusty", "debian/wheezy", "ubuntu/precise"],
  58. help = "releases for which to build")
  59. args = parser.parse_args()
  60. # Maybe logging should be configurable too. Later.
  61. logging.basicConfig(level = logging.INFO, timefmt = "%Y-%m-%dT%H:%M:%S",
  62. format = "%(asctime)s [%(process)d] %(levelname)s %(message)s")
  63. upload = socket.getfqdn() == "build-u.rpki.net"
  64. def run(*cmd, **kwargs):
  65. if args.debug:
  66. #logging.info("Running %r %r", cmd, kwargs)
  67. logging.info("Running %s", " ".join(cmd))
  68. subprocess.check_call(cmd, **kwargs)
  69. # Getting this to work right also required adding:
  70. #
  71. # DEBBUILDOPTS="-b"
  72. #
  73. # to /etc/pbuilderrc; without this, reprepro (eventually, a year after
  74. # we set this up) started failing to incorporate some of the built
  75. # packages, because the regenerated source packages had different
  76. # checksums than the ones loaded initially. See:
  77. #
  78. # http://stackoverflow.com/questions/21563872/reprepro-complains-about-the-generated-pbuilder-debian-tar-gz-archive-md5
  79. #
  80. # Putting stuff in ~/.pbuilderrc didn't work with pbuilder-dist when I
  81. # tried it last year, this may just be that sudo isn't configured to
  82. # pass HOME through, thus pbuilder is looking for ~root/.pbuilderrc.
  83. # Worth trying again at some point but not all that critical.
  84. logging.info("Starting")
  85. try:
  86. lock = os.open(args.lockfile, os.O_RDONLY | os.O_CREAT | os.O_NONBLOCK, 0666)
  87. fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
  88. except (IOError, OSError), e:
  89. sys.exit(0 if e.errno == errno.EAGAIN else "Error {!r} opening lock {!r}".format(e, args.lockfile))
  90. run("svn", "--quiet", "update", cwd = args.svn_tree)
  91. source_version = subprocess.check_output(("svnversion", "-c"), cwd = args.svn_tree).strip().split(":")[-1]
  92. logging.info("Source version is %s", source_version)
  93. if not source_version.isdigit() and not args.debug:
  94. sys.exit("Sources don't look pristine, not building ({!r})".format(source_version))
  95. source_version = "0." + source_version
  96. search_version = "_" + source_version + "~"
  97. dsc_dir = os.path.abspath(os.path.join(args.svn_tree, ".."))
  98. if not os.path.isdir(args.apt_tree):
  99. logging.info("Creating %s", args.apt_tree)
  100. os.makedirs(args.apt_tree)
  101. fn = os.path.join(args.apt_tree, "apt-gpg-key.asc")
  102. if not os.path.exists(fn):
  103. logging.info("Creating %s", fn)
  104. run("gpg", "--export", "--armor", "--keyring", args.keyring, stdout = open(fn, "w"))
  105. class Release(object):
  106. architectures = dict(amd64 = "", i386 = "-i386")
  107. releases = []
  108. packages = {}
  109. def __init__(self, distribution_release, backports):
  110. self.distribution, self.release = distribution_release.split("/")
  111. self.backports = backports
  112. self.apt_source = "http://{host}/{path}/{distribution} {release} main".format(
  113. host = args.url_host,
  114. path = args.url_path.strip("/"),
  115. distribution = self.distribution,
  116. release = self.release)
  117. self.env = dict(os.environ)
  118. if backports:
  119. self.env.update(OTHERMIRROR = "deb " + self.apt_source)
  120. self.releases.append(self)
  121. @classmethod
  122. def do_all_releases(cls):
  123. for release in cls.releases:
  124. release.setup_reprepro()
  125. for release in cls.releases:
  126. release.list_repository()
  127. for release in cls.releases:
  128. for release.arch, release.tag in cls.architectures.iteritems():
  129. release.do_one_architecture()
  130. del release.arch, release.tag
  131. @staticmethod
  132. def repokey(release, architecture, package):
  133. return (release, architecture, package)
  134. def list_repository(self):
  135. cmd = ("reprepro", "list", self.release)
  136. logging.info("Running %s", " ".join(cmd))
  137. listing = subprocess.check_output(cmd, cwd = self.tree)
  138. for line in listing.replace(":", " ").replace("|", " ").splitlines():
  139. rel, comp, arch, pkg, ver = line.split()
  140. key = (rel, arch, pkg)
  141. assert key not in self.packages
  142. self.packages[key] = ver
  143. @property
  144. def deb_in_repository(self):
  145. ret = all(self.packages.get((self.release, self.arch, package)) == self.version
  146. for package in rpki_packages)
  147. #logging.info("Got %s looking for %r in %r", ret, self.version, self.packages)
  148. return ret
  149. @property
  150. def src_in_repository(self):
  151. ret = self.packages.get((self.release, "source", rpki_source_package)) == self.version
  152. #logging.info("Got %s looking for %r in %r", ret, self.version, self.packages)
  153. return ret
  154. @property
  155. def version(self):
  156. return source_version + "~" + self.release
  157. @property
  158. def dsc(self):
  159. return os.path.join(dsc_dir, "rpki_{}.dsc".format(self.version))
  160. @property
  161. def tree(self):
  162. return os.path.join(args.apt_tree, self.distribution, "")
  163. @property
  164. def basefile(self):
  165. return os.path.expanduser("~/pbuilder/{0.release}{0.tag}-base.tgz".format(self))
  166. @property
  167. def result(self):
  168. return os.path.expanduser("~/pbuilder/{0.release}{0.tag}_result".format(self))
  169. @property
  170. def changes(self):
  171. return os.path.join(self.result, "rpki_{0.version}_{0.arch}.changes".format(self))
  172. def do_one_architecture(self):
  173. logging.info("Running build for %s %s %s", self.distribution, self.release, self.arch)
  174. if not os.path.exists(self.dsc):
  175. logging.info("%s does not exist", self.dsc)
  176. for fn in os.listdir(dsc_dir):
  177. fn = os.path.join(dsc_dir, fn)
  178. if not os.path.isdir(fn) and search_version not in fn:
  179. logging.info("Removing %s", fn)
  180. os.unlink(fn)
  181. run("rm", "-rf", "debian", cwd = args.svn_tree)
  182. logging.info("Building source package %s", self.version)
  183. run(sys.executable, "buildtools/make-version.py", cwd = args.svn_tree)
  184. run(sys.executable, "buildtools/build-debian-packages.py", "--version-suffix", self.release, cwd = args.svn_tree)
  185. run("dpkg-buildpackage", "-S", "-us", "-uc", "-rfakeroot", cwd = args.svn_tree)
  186. if not os.path.exists(self.basefile):
  187. logging.info("Creating build environment %s %s", self.release, self.arch)
  188. run("pbuilder-dist", self.release, self.arch, "create", env = self.env)
  189. elif time.time() > os.stat(self.basefile).st_mtime + args.update_build_after:
  190. logging.info("Updating build environment %s %s", self.release, self.arch)
  191. run("pbuilder-dist", self.release, self.arch, "update", env = self.env)
  192. if not self.deb_in_repository:
  193. for fn in os.listdir(self.result):
  194. fn = os.path.join(self.result, fn)
  195. logging.info("Removing %s", fn)
  196. os.unlink(fn)
  197. logging.info("Building binary packages %s %s %s", self.release, self.arch, self.version)
  198. run("pbuilder-dist", self.release, self.arch, "build", "--keyring", args.keyring, self.dsc, env = self.env)
  199. logging.info("Updating repository for %s %s %s", self.release, self.arch, self.version)
  200. run("reprepro", "--ignore=wrongdistribution", "include", self.release, self.changes, cwd = self.tree)
  201. if not self.src_in_repository:
  202. logging.info("Updating repository for %s source %s", self.release, self.version)
  203. run("reprepro", "--ignore=wrongdistribution", "includedsc", self.release, self.dsc, cwd = self.tree)
  204. def setup_reprepro(self):
  205. logging.info("Configuring reprepro for %s/%s", self.distribution, self.release)
  206. dn = os.path.join(self.tree, "conf")
  207. if not os.path.isdir(dn):
  208. logging.info("Creating %s", dn)
  209. os.makedirs(dn)
  210. fn = os.path.join(self.tree, "conf", "distributions")
  211. distributions = open(fn, "r").read() if os.path.exists(fn) else ""
  212. if "Codename: {0.release}\n".format(self) not in distributions:
  213. logging.info("%s %s", "Editing" if distributions else "Creating", fn)
  214. with open(fn, "w") as f:
  215. if distributions:
  216. f.write(distributions)
  217. f.write("\n")
  218. f.write(dedent("""\
  219. Origin: rpki.net
  220. Label: rpki.net {self.distribution} repository
  221. Codename: {self.release}
  222. Architectures: {architectures} source
  223. Components: main
  224. Description: rpki.net {Distribution} APT Repository
  225. SignWith: yes
  226. DebOverride: override.{self.release}
  227. DscOverride: override.{self.release}
  228. """.format(
  229. self = self,
  230. Distribution = self.distribution.capitalize(),
  231. architectures = " ".join(self.architectures))))
  232. fn = os.path.join(self.tree, "conf", "options")
  233. if not os.path.exists(fn):
  234. logging.info("Creating %s", fn)
  235. with open(fn, "w") as f:
  236. f.write(dedent("""\
  237. verbose
  238. ask-passphrase
  239. basedir .
  240. """))
  241. fn = os.path.join(self.tree, "conf", "override." + self.release)
  242. if not os.path.exists(fn):
  243. logging.info("Creating %s", fn)
  244. with open(fn, "w") as f:
  245. for pkg in self.backports:
  246. f.write(dedent("""\
  247. {pkg:<30} Priority optional
  248. {pkg:<30} Section python
  249. """.format(pkg = pkg)))
  250. f.write(dedent("""\
  251. rpki-ca Priority extra
  252. rpki-ca Section net
  253. rpki-rp Priority extra
  254. rpki-rp Section net
  255. """))
  256. fn = os.path.join(args.apt_tree, "rpki.{}.list".format(self.release))
  257. if not os.path.exists(fn):
  258. logging.info("Creating %s", fn)
  259. with open(fn, "w") as f:
  260. f.write(dedent("""\
  261. deb {self.apt_source}
  262. deb-src {self.apt_source}
  263. """.format(self = self)))
  264. # Load whatever releases the user specified
  265. for r in args.releases:
  266. Release(r, args.backports)
  267. # Do all the real work.
  268. Release.do_all_releases()
  269. # Upload results, maybe. We do this in two stages, to minimize the window
  270. # during which the uploaded repository might be in an inconsistent state.
  271. def rsync(*flags):
  272. cmd = ["rsync", "--archive", "--itemize-changes",
  273. "--rsh", "ssh -l {}".format(args.apt_user)]
  274. cmd.extend(flags)
  275. cmd.append(args.apt_tree)
  276. cmd.append("rsync://{host}/{path}/".format(host = args.url_host,
  277. path = args.url_path.strip("/")))
  278. if upload:
  279. logging.info("Synching repository to %s with flags %s",
  280. cmd[-1], " ".join(flags))
  281. run(*cmd)
  282. else:
  283. logging.info("Would have synched repository to %s with flags %",
  284. cmd[-1], " ".join(flags))
  285. rsync("--ignore-existing")
  286. rsync("--exclude", "HEADER.html",
  287. "--exclude", "HEADER.css",
  288. "--delete", "--delete-delay")
  289. logging.info("Done")