rpki-pbuilder.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363
  1. #!/usr/bin/python
  2. #
  3. # $Id$
  4. #
  5. # Copyright (C) 2014 Dragon Research Labs ("DRL")
  6. # Portions copyright (C) 2013 Internet Systems Consortium ("ISC")
  7. #
  8. # Permission to use, copy, modify, and distribute this software for any
  9. # purpose with or without fee is hereby granted, provided that the above
  10. # copyright notices and this permission notice appear in all copies.
  11. #
  12. # THE SOFTWARE IS PROVIDED "AS IS" AND DRL AND ISC DISCLAIM ALL
  13. # WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
  14. # WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL OR
  15. # ISC BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
  16. # DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA
  17. # OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
  18. # TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
  19. # PERFORMANCE OF THIS SOFTWARE.
  20. """
  21. Debian/Ubuntu package build tool, based on pbuilder-dist and reprepro.
  22. """
  23. import os
  24. import sys
  25. import time
  26. import fcntl
  27. import errno
  28. import socket
  29. import logging
  30. import argparse
  31. import subprocess
  32. from textwrap import dedent
  33. rpki_packages = ("rpki-rp", "rpki-ca")
  34. rpki_source_package = "rpki"
  35. parser = argparse.ArgumentParser(description = __doc__,
  36. formatter_class = argparse.ArgumentDefaultsHelpFormatter)
  37. parser.add_argument("--debug", action = "store_true",
  38. help = "enable debugging code")
  39. parser.add_argument("--update-build-after", type = int, default = 7 * 24 * 60 * 60,
  40. help = "interval (in seconds) after which we should update the pbuilder environment")
  41. parser.add_argument("--lockfile", default = os.path.expanduser("~/builder.lock"),
  42. help = "avoid collisions between multiple instances of this script")
  43. parser.add_argument("--keyring", default = os.path.expanduser("~/.gnupg/pubring.gpg"),
  44. help = "PGP keyring")
  45. parser.add_argument("--git-tree", default = os.path.expanduser("~/source/master/"),
  46. help = "git tree")
  47. parser.add_argument("--apt-tree", default = os.path.expanduser("~/repository/"),
  48. help = "reprepro repository")
  49. parser.add_argument("--url-host", default = "download.rpki.net",
  50. help = "hostname of public web server")
  51. parser.add_argument("--url-scheme", default = "http",
  52. help = "URL scheme of public web server")
  53. parser.add_argument("--url-path", default = "/APT",
  54. help = "path of apt repository on public web server")
  55. parser.add_argument("--backports", nargs = "+", default = ["python-django", "python-tornado"],
  56. help = "backports needed for this build")
  57. parser.add_argument("--releases", nargs = "+", default = ["debian/jessie", "ubuntu/xenial"],
  58. help = "releases for which to build")
  59. args = parser.parse_args()
  60. # Maybe logging should be configurable too. Later.
  61. logging.basicConfig(level = logging.INFO, timefmt = "%Y-%m-%dT%H:%M:%S",
  62. format = "%(asctime)s [%(process)d] %(levelname)s %(message)s")
  63. upload = socket.getfqdn() == "build-u.rpki.net"
  64. def run(*cmd, **kwargs):
  65. if args.debug:
  66. #logging.info("Running %r %r", cmd, kwargs)
  67. logging.info("Running %s", " ".join(cmd))
  68. subprocess.check_call(cmd, **kwargs)
  69. # Getting this to work right also required adding:
  70. #
  71. # DEBBUILDOPTS="-b"
  72. #
  73. # to /etc/pbuilderrc; without this, reprepro (eventually, a year after
  74. # we set this up) started failing to incorporate some of the built
  75. # packages, because the regenerated source packages had different
  76. # checksums than the ones loaded initially. See:
  77. #
  78. # http://stackoverflow.com/questions/21563872/reprepro-complains-about-the-generated-pbuilder-debian-tar-gz-archive-md5
  79. #
  80. # Putting stuff in ~/.pbuilderrc didn't work with pbuilder-dist when I
  81. # tried it last year, this may just be that sudo isn't configured to
  82. # pass HOME through, thus pbuilder is looking for ~root/.pbuilderrc.
  83. # Worth trying again at some point but not all that critical.
  84. logging.info("Starting")
  85. try:
  86. lock = os.open(args.lockfile, os.O_RDONLY | os.O_CREAT | os.O_NONBLOCK, 0666)
  87. fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
  88. except (IOError, OSError), e:
  89. sys.exit(0 if e.errno == errno.EAGAIN else "Error {!r} opening lock {!r}".format(e, args.lockfile))
  90. run("git", "fetch", "--all", "--prune", cwd = args.git_tree)
  91. run("git", "pull", cwd = args.git_tree)
  92. source_version = subprocess.check_output((sys.executable, os.path.join(args.git_tree, "buildtools/make-version.py"),
  93. "--build-tag", "--stdout"), cwd = args.git_tree).strip()
  94. assert source_version.startswith("buildbot-")
  95. source_version = source_version[len("buildbot-"):].replace("-", ".")
  96. logging.info("Source version is %s", source_version)
  97. if not args.debug:
  98. try:
  99. run("git", "diff-index", "--quiet", "HEAD", cwd = args.git_tree)
  100. except subprocess.CalledProcessError:
  101. sys.exit("Sources don't look pristine, not building ({!r})".format(source_version))
  102. search_version = "_" + source_version + "~"
  103. dsc_dir = os.path.abspath(os.path.join(args.git_tree, ".."))
  104. if not os.path.isdir(args.apt_tree):
  105. logging.info("Creating %s", args.apt_tree)
  106. os.makedirs(args.apt_tree)
  107. fn = os.path.join(args.apt_tree, "apt-gpg-key.asc")
  108. if not os.path.exists(fn):
  109. logging.info("Creating %s", fn)
  110. run("gpg", "--export", "--armor", "--keyring", args.keyring, stdout = open(fn, "w"))
  111. class Release(object):
  112. architectures = dict(amd64 = "", i386 = "-i386")
  113. releases = []
  114. packages = {}
  115. def __init__(self, distribution_release, backports):
  116. self.distribution, self.release = distribution_release.split("/")
  117. self.backports = backports
  118. self.apt_source = "{scheme}://{host}/{path}/{distribution} {release} main".format(
  119. scheme = args.url_scheme,
  120. host = args.url_host,
  121. path = args.url_path.strip("/"),
  122. distribution = self.distribution,
  123. release = self.release)
  124. self.env = dict(os.environ)
  125. if backports:
  126. self.env.update(OTHERMIRROR = "deb " + self.apt_source)
  127. self.releases.append(self)
  128. @classmethod
  129. def do_all_releases(cls):
  130. for release in cls.releases:
  131. release.setup_reprepro()
  132. for release in cls.releases:
  133. release.list_repository()
  134. for release in cls.releases:
  135. for release.arch, release.tag in cls.architectures.iteritems():
  136. release.do_one_architecture()
  137. del release.arch, release.tag
  138. @staticmethod
  139. def repokey(release, architecture, package):
  140. return (release, architecture, package)
  141. def list_repository(self):
  142. cmd = ("reprepro", "list", self.release)
  143. logging.info("Running %s", " ".join(cmd))
  144. listing = subprocess.check_output(cmd, cwd = self.tree)
  145. for line in listing.replace(":", " ").replace("|", " ").splitlines():
  146. rel, comp, arch, pkg, ver = line.split()
  147. key = (rel, arch, pkg)
  148. assert key not in self.packages
  149. self.packages[key] = ver
  150. @property
  151. def deb_in_repository(self):
  152. ret = all(self.packages.get((self.release, self.arch, package)) == self.version
  153. for package in rpki_packages)
  154. #logging.info("Got %s looking for %r in %r", ret, self.version, self.packages)
  155. return ret
  156. @property
  157. def src_in_repository(self):
  158. ret = self.packages.get((self.release, "source", rpki_source_package)) == self.version
  159. #logging.info("Got %s looking for %r in %r", ret, self.version, self.packages)
  160. return ret
  161. @property
  162. def version(self):
  163. return source_version + "~" + self.release
  164. @property
  165. def dsc(self):
  166. return os.path.join(dsc_dir, "rpki_{}.dsc".format(self.version))
  167. @property
  168. def tree(self):
  169. return os.path.join(args.apt_tree, self.distribution, "")
  170. @property
  171. def basefile(self):
  172. return os.path.expanduser("~/pbuilder/{0.release}{0.tag}-base.tgz".format(self))
  173. @property
  174. def result(self):
  175. return os.path.expanduser("~/pbuilder/{0.release}{0.tag}_result".format(self))
  176. @property
  177. def changes(self):
  178. return os.path.join(self.result, "rpki_{0.version}_{0.arch}.changes".format(self))
  179. def do_one_architecture(self):
  180. logging.info("Running build for %s %s %s", self.distribution, self.release, self.arch)
  181. if not os.path.exists(self.dsc):
  182. logging.info("%s does not exist", self.dsc)
  183. for fn in os.listdir(dsc_dir):
  184. fn = os.path.join(dsc_dir, fn)
  185. if not os.path.isdir(fn) and search_version not in fn:
  186. logging.info("Removing %s", fn)
  187. os.unlink(fn)
  188. run("rm", "-rf", "debian", cwd = args.git_tree)
  189. logging.info("Building source package %s", self.version)
  190. run(sys.executable, "buildtools/build-debian-packages.py", "--version-suffix", self.release, cwd = args.git_tree)
  191. run("dpkg-buildpackage", "-S", "-us", "-uc", "-rfakeroot", cwd = args.git_tree)
  192. if not os.path.exists(self.basefile):
  193. logging.info("Creating build environment %s %s", self.release, self.arch)
  194. run("pbuilder-dist", self.release, self.arch, "create", env = self.env)
  195. elif time.time() > os.stat(self.basefile).st_mtime + args.update_build_after:
  196. logging.info("Updating build environment %s %s", self.release, self.arch)
  197. run("pbuilder-dist", self.release, self.arch, "update", env = self.env)
  198. if not self.deb_in_repository:
  199. for fn in os.listdir(self.result):
  200. fn = os.path.join(self.result, fn)
  201. logging.info("Removing %s", fn)
  202. os.unlink(fn)
  203. logging.info("Building binary packages %s %s %s", self.release, self.arch, self.version)
  204. run("pbuilder-dist", self.release, self.arch, "build", "--keyring", args.keyring, self.dsc, env = self.env)
  205. logging.info("Updating repository for %s %s %s", self.release, self.arch, self.version)
  206. run("reprepro", "--ignore=wrongdistribution", "include", self.release, self.changes, cwd = self.tree)
  207. if not self.src_in_repository:
  208. logging.info("Updating repository for %s source %s", self.release, self.version)
  209. run("reprepro", "--ignore=wrongdistribution", "includedsc", self.release, self.dsc, cwd = self.tree)
  210. def setup_reprepro(self):
  211. logging.info("Configuring reprepro for %s/%s", self.distribution, self.release)
  212. dn = os.path.join(self.tree, "conf")
  213. if not os.path.isdir(dn):
  214. logging.info("Creating %s", dn)
  215. os.makedirs(dn)
  216. fn = os.path.join(self.tree, "conf", "distributions")
  217. distributions = open(fn, "r").read() if os.path.exists(fn) else ""
  218. if "Codename: {0.release}\n".format(self) not in distributions:
  219. logging.info("%s %s", "Editing" if distributions else "Creating", fn)
  220. with open(fn, "w") as f:
  221. if distributions:
  222. f.write(distributions)
  223. f.write("\n")
  224. f.write(dedent("""\
  225. Origin: rpki.net
  226. Label: rpki.net {self.distribution} repository
  227. Codename: {self.release}
  228. Architectures: {architectures} source
  229. Components: main
  230. Description: rpki.net {Distribution} APT Repository
  231. SignWith: yes
  232. DebOverride: override.{self.release}
  233. DscOverride: override.{self.release}
  234. """.format(
  235. self = self,
  236. Distribution = self.distribution.capitalize(),
  237. architectures = " ".join(self.architectures))))
  238. fn = os.path.join(self.tree, "conf", "options")
  239. if not os.path.exists(fn):
  240. logging.info("Creating %s", fn)
  241. with open(fn, "w") as f:
  242. f.write(dedent("""\
  243. verbose
  244. ask-passphrase
  245. basedir .
  246. """))
  247. fn = os.path.join(self.tree, "conf", "override." + self.release)
  248. if not os.path.exists(fn):
  249. logging.info("Creating %s", fn)
  250. with open(fn, "w") as f:
  251. for pkg in self.backports:
  252. f.write(dedent("""\
  253. {pkg:<30} Priority optional
  254. {pkg:<30} Section python
  255. """.format(pkg = pkg)))
  256. f.write(dedent("""\
  257. rpki-ca Priority extra
  258. rpki-ca Section net
  259. rpki-rp Priority extra
  260. rpki-rp Section net
  261. """))
  262. fn = os.path.join(args.apt_tree, "rpki.{}.list".format(self.release))
  263. if not os.path.exists(fn):
  264. logging.info("Creating %s", fn)
  265. with open(fn, "w") as f:
  266. f.write(dedent("""\
  267. deb {self.apt_source}
  268. deb-src {self.apt_source}
  269. """.format(self = self)))
  270. # Load whatever releases the user specified
  271. for r in args.releases:
  272. Release(r, args.backports)
  273. # Do all the real work.
  274. Release.do_all_releases()
  275. # Push any tags created above to the public git repository.
  276. if upload:
  277. run("git", "push", "--tags", cwd = args.git_tree)
  278. # Upload results, maybe. We do this in two stages, to minimize the window
  279. # during which the uploaded repository might be in an inconsistent state.
  280. def rsync(*flags):
  281. cmd = ["rsync", "--archive", "--itemize-changes", "--rsh", "ssh"]
  282. cmd.extend(flags)
  283. cmd.append(args.apt_tree)
  284. cmd.append("rsync://{host}/{path}/".format(host = args.url_host,
  285. path = args.url_path.strip("/")))
  286. if upload:
  287. logging.info("Synching repository to %s with flags %s",
  288. cmd[-1], " ".join(flags))
  289. run(*cmd)
  290. else:
  291. logging.info("Would have synched repository to %s with flags %s",
  292. cmd[-1], " ".join(flags))
  293. rsync("--ignore-existing")
  294. rsync("--exclude", "HEADER.html",
  295. "--exclude", "HEADER.css",
  296. "--delete", "--delete-delay")
  297. logging.info("Done")