diff options
Diffstat (limited to 'buildtools')
| -rw-r--r-- | buildtools/build-freebsd-ports.py | 172 | ||||
| -rw-r--r-- | buildtools/build-ubuntu-ports.py | 4 | ||||
| -rw-r--r-- | buildtools/defstack.py | 26 | ||||
| -rw-r--r-- | buildtools/freebsd-skeleton/rpki-ca/Makefile | 2 | ||||
| -rw-r--r-- | buildtools/freebsd-skeleton/rpki-ca/files/rpki-ca.in | 9 | ||||
| -rw-r--r-- | buildtools/freebsd-skeleton/rpki-rp/Makefile | 2 | ||||
| -rw-r--r-- | buildtools/make-rcynic-script.py | 4 | ||||
| -rw-r--r-- | buildtools/make-relaxng.py | 16 | ||||
| -rw-r--r-- | buildtools/make-sql-schemas.py | 18 | ||||
| -rw-r--r-- | buildtools/make-version.py | 38 | ||||
| -rw-r--r-- | buildtools/pull-doc-from-wiki.py | 188 | ||||
| -rw-r--r-- | buildtools/pylint.rc | 32 | ||||
| -rw-r--r-- | buildtools/rpki-pbuilder.py | 24 |
13 files changed, 284 insertions, 251 deletions
diff --git a/buildtools/build-freebsd-ports.py b/buildtools/build-freebsd-ports.py index fc35c94b..bf0b2c47 100644 --- a/buildtools/build-freebsd-ports.py +++ b/buildtools/build-freebsd-ports.py @@ -2,11 +2,11 @@ # # Copyright (C) 2014 Dragon Research Labs ("DRL") # Portions copyright (C) 2012-2013 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notices and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND DRL AND ISC DISCLAIM ALL # WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL OR @@ -23,124 +23,134 @@ This is a script because we need to generate package lists and update version numbers in the Makefiles. """ -import sys import os import re -import subprocess -import errno +import sys import glob +import errno import shutil import argparse +import subprocess def check_dir(s): - if not os.path.isdir(s): - raise argparse.ArgumentTypeError("%r is not a directory" % s) - return s - -parser = argparse.ArgumentParser(description = __doc__) -parser.add_argument("--allow-dirty", action = "store_true", - help = "don't insist on pristine subversion checkout") + if not os.path.isdir(s): + raise argparse.ArgumentTypeError("%r is not a directory" % s) + return s + +parser = argparse.ArgumentParser(description = __doc__, + formatter_class = argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument("--local-dist", action = "store_true", + help = "generate local distribution from subversion working tree (implies --make-package)") parser.add_argument("--make-package", action = "store_true", help = "build binary package") parser.add_argument("--no-clean", action = "store_true", help = "don't clean port after staging etc (implies --no-tarball)") parser.add_argument("--no-tarball", action = "store_true", help = "don't create tarball of generated port") +parser.add_argument("--portsdir", type = os.path.abspath, + default = os.path.abspath("freebsd-ports"), + help = "where to build FreeBSD port trees") parser.add_argument("svndir", metavar = "subversion-working-directory", type = check_dir, help = "directory containing subversion working tree") args = parser.parse_args() svnversion = subprocess.check_output(("svnversion", "-c", args.svndir)).strip().split(":")[-1] -if args.allow_dirty: - svnversion = svnversion.translate(None, "M") +if args.local_dist: + svnversion = svnversion.translate(None, "M") if not svnversion.isdigit(): - sys.exit("Sources don't look pristine, not building (%r)" % svnversion) + sys.exit("Sources don't look pristine, not building (%r)" % svnversion) branch = os.path.basename(args.svndir.rstrip(os.path.sep)) if branch != "trunk" and (branch[:2] != "tk" or not branch[2:].isdigit()): - sys.exit("Could not parse branch from working directory name, not building (%r)" % branch) + sys.exit("Could not parse branch from working directory name, not building (%r)" % branch) version = "0." + svnversion tarname = "rpki-%s-r%s" % (branch, svnversion) tarball = tarname + ".tar.xz" -url = "http://download.rpki.net/" + tarball - -portsdir = os.path.abspath("freebsd-ports") -portsdir_old = portsdir + ".old" -# Could perhaps use distutils.sysconfig.get_python_lib() instead of -# this regexp hack, but would be just as complicated in its own way, -# so just go with this for the moment. - -py_lib = re.compile(r"^lib/python\d+\.\d+") -py_sitelib = re.compile(r"^lib/python\d+\.\d+/site-packages") +portsdir_old = args.portsdir + ".old" if os.path.isdir(portsdir_old): - shutil.rmtree(portsdir_old) - -if os.path.isdir(portsdir): - os.rename(portsdir, portsdir_old) - -shutil.copytree(os.path.join(args.svndir, "buildtools", "freebsd-skeleton"), portsdir) - -if os.path.exists(os.path.join(portsdir_old, tarball)): - os.link(os.path.join(portsdir_old, tarball), os.path.join(portsdir, tarball)) + shutil.rmtree(portsdir_old) + +if os.path.isdir(args.portsdir): + os.rename(args.portsdir, portsdir_old) + +shutil.copytree(os.path.join(args.svndir, "buildtools", "freebsd-skeleton"), args.portsdir) + +if args.local_dist: + subprocess.check_call(("svn", "export", args.svndir, os.path.join(args.portsdir, tarname))) + for fn, fmt in (("VERSION", "%s\n"), ("rpki/version.py", "VERSION = \"%s\"\n")): + with open(os.path.join(args.portsdir, tarname, fn), "w") as f: + f.write(fmt % version) + subprocess.check_call(("tar", "cJvvf", tarball, tarname), cwd = args.portsdir) + shutil.rmtree(os.path.join(args.portsdir, tarname)) +elif os.path.exists(os.path.join(portsdir_old, tarball)): + os.link(os.path.join(portsdir_old, tarball), os.path.join(args.portsdir, tarball)) elif os.path.exists(os.path.join("/usr/ports/distfiles", tarball)): - shutil.copy(os.path.join("/usr/ports/distfiles", tarball), os.path.join(portsdir, tarball)) + shutil.copy(os.path.join("/usr/ports/distfiles", tarball), os.path.join(args.portsdir, tarball)) if os.path.isdir(portsdir_old): - shutil.rmtree(portsdir_old) + shutil.rmtree(portsdir_old) + +if args.make_package or args.local_dist: + pkgdir = os.path.join(args.portsdir, "packages") + os.mkdir(pkgdir) + +py_lib = re.compile(r"^lib/python\d+\.\d+") +py_sitelib = re.compile(r"^lib/python\d+\.\d+/site-packages") -if args.make_package: - pkgdir = os.path.join(portsdir, "packages") - os.mkdir(pkgdir) +if args.local_dist: + master_site = "file://" + args.portsdir + "/" +else: + master_site = "http://download.rpki.net/" -formatdict = dict(SVNVERSION = svnversion, SVNBRANCH = branch) +formatdict = dict(SVNVERSION = svnversion, SVNBRANCH = branch, MASTER_SITE = master_site) keepdirs = ("usr", "etc", "bin", "var", "lib", "libexec", "sbin", "share", "etc/rc.d", "%%PYTHON_SITELIBDIR%%") for port in ("rpki-rp", "rpki-ca"): - base = os.path.join(portsdir, port) - stage = os.path.join(base, "work", "stage") - fn = os.path.join(portsdir, port, "Makefile") - with open(fn, "r") as f: - template = f.read() - with open(fn, "w") as f: - f.write(template % formatdict) - - subprocess.check_call(("make", "makesum", "stage", "DISTDIR=" + portsdir, "NO_DEPENDS=yes"), - cwd = base) - - with open(os.path.join(base, "pkg-plist"), "w") as f: - usr_local = None - for dirpath, dirnames, filenames in os.walk(stage, topdown = False): - dn = dirpath[len(stage)+1:] - if dn.startswith("usr/local"): - if not usr_local and usr_local is not None: - f.write("@cwd\n") - usr_local = True - dn = dn[len("usr/local/"):] - dn = py_sitelib.sub("%%PYTHON_SITELIBDIR%%", dn) - if dn == "etc/rc.d": - continue - else: - if usr_local: - f.write("@cwd /\n") - usr_local = False - for fn in filenames: - f.write(os.path.join(dn, fn) + "\n") - if dn and dn not in keepdirs and not py_lib.match(dn): - f.write("@dirrm %s\n" % dn) - - if args.make_package: - subprocess.check_call(("make", "clean", "package", "PKGREPOSITORY=" + pkgdir), cwd = base) - - if not args.no_clean: - subprocess.check_call(("make", "clean"), cwd = base) - - if not args.no_tarball and not args.no_clean: - subprocess.check_call(("tar", "czf", "%s-port.tgz" % port, port), cwd = portsdir) + base = os.path.join(args.portsdir, port) + stage = os.path.join(base, "work", "stage") + fn = os.path.join(args.portsdir, port, "Makefile") + with open(fn, "r") as f: + template = f.read() + with open(fn, "w") as f: + f.write(template % formatdict) + + subprocess.check_call(("make", "makesum", "stage", "DISTDIR=" + args.portsdir, "NO_DEPENDS=yes"), + cwd = base) + + with open(os.path.join(base, "pkg-plist"), "w") as f: + usr_local = None + for dirpath, dirnames, filenames in os.walk(stage, topdown = False): + dn = dirpath[len(stage)+1:] + if dn.startswith("usr/local"): + if not usr_local and usr_local is not None: + f.write("@cwd\n") + usr_local = True + dn = dn[len("usr/local/"):] + dn = py_sitelib.sub("%%PYTHON_SITELIBDIR%%", dn) + if dn == "etc/rc.d": + continue + else: + if usr_local: + f.write("@cwd /\n") + usr_local = False + for fn in filenames: + f.write(os.path.join(dn, fn) + "\n") + if dn and dn not in keepdirs and not py_lib.match(dn): + f.write("@dirrm %s\n" % dn) + + if args.make_package or args.local_dist: + subprocess.check_call(("make", "clean", "package", "DISTDIR=" + args.portsdir, "PKGREPOSITORY=" + pkgdir), cwd = base) + + if not args.no_clean: + subprocess.check_call(("make", "clean"), cwd = base) + + if not args.no_tarball and not args.no_clean: + subprocess.check_call(("tar", "czf", "%s-port.tgz" % port, port), cwd = args.portsdir) diff --git a/buildtools/build-ubuntu-ports.py b/buildtools/build-ubuntu-ports.py index 0a326da8..19f61f6d 100644 --- a/buildtools/build-ubuntu-ports.py +++ b/buildtools/build-ubuntu-ports.py @@ -2,11 +2,11 @@ # # Copyright (C) 2014 Dragon Research Labs ("DRL") # Portions copyright (C) 2013 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notices and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND DRL AND ISC DISCLAIM ALL # WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL OR diff --git a/buildtools/defstack.py b/buildtools/defstack.py index 757516f3..b3df0777 100644 --- a/buildtools/defstack.py +++ b/buildtools/defstack.py @@ -8,11 +8,11 @@ # code with code maintained by humans, so "nasty" is a relative term. # # Copyright (C) 2011-2012 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -68,7 +68,7 @@ template = ''' ''' if len(sys.argv) < 2: - sys.exit("Usage: %s source.c [source.c ...]" % sys.argv[0]) + sys.exit("Usage: %s source.c [source.c ...]" % sys.argv[0]) splitter = re.compile("[() \t]+").split @@ -76,16 +76,16 @@ token = None for line in fileinput.input(): - if token is None: - path = fileinput.filename().split(os.path.sep) - path = os.path.join(path[-2], path[-1]) if len(path) > 1 else path[-1] - token = "".join(c if c.isalnum() else "_" for c in path.upper()) - sys.stdout.write(header.replace("%", token)) + if token is None: + path = fileinput.filename().split(os.path.sep) + path = os.path.join(path[-2], path[-1]) if len(path) > 1 else path[-1] + token = "".join(c if c.isalnum() else "_" for c in path.upper()) + sys.stdout.write(header.replace("%", token)) - if "DECLARE_STACK_OF" in line: - words = splitter(line) - if len(words) > 1 and words[0] == "DECLARE_STACK_OF": - sys.stdout.write(template.replace("%", words[1])) + if "DECLARE_STACK_OF" in line: + words = splitter(line) + if len(words) > 1 and words[0] == "DECLARE_STACK_OF": + sys.stdout.write(template.replace("%", words[1])) if token is not None: - sys.stdout.write(footer.replace("%", token)) + sys.stdout.write(footer.replace("%", token)) diff --git a/buildtools/freebsd-skeleton/rpki-ca/Makefile b/buildtools/freebsd-skeleton/rpki-ca/Makefile index 1bbc1355..c8ceaea1 100644 --- a/buildtools/freebsd-skeleton/rpki-ca/Makefile +++ b/buildtools/freebsd-skeleton/rpki-ca/Makefile @@ -1,7 +1,7 @@ PORTNAME= rpki-ca PORTVERSION= 0.%(SVNVERSION)s CATEGORIES= net -MASTER_SITES= http://download.rpki.net/ +MASTER_SITES= %(MASTER_SITE)s DISTFILES= rpki-%(SVNBRANCH)s-r%(SVNVERSION)s.tar.xz WRKSRC= ${WRKDIR}/rpki-%(SVNBRANCH)s-r%(SVNVERSION)s MAINTAINER= sra@hactrn.net diff --git a/buildtools/freebsd-skeleton/rpki-ca/files/rpki-ca.in b/buildtools/freebsd-skeleton/rpki-ca/files/rpki-ca.in index d6234a12..0c021e6d 100644 --- a/buildtools/freebsd-skeleton/rpki-ca/files/rpki-ca.in +++ b/buildtools/freebsd-skeleton/rpki-ca/files/rpki-ca.in @@ -22,13 +22,12 @@ stop_cmd="rpkica_stop" load_rc_config $name : ${rpkica_enable="NO"} - : ${rpkica_pid_dir="/var/run/rpki"} rpkica_start() { - /usr/bin/install -m 755 -d $rpkica_pid_dir - /usr/local/sbin/rpki-start-servers + /usr/bin/install -m 755 -d ${rpkica_pid_dir} + /usr/local/sbin/rpki-start-servers ${rpkica_flags} return 0 } @@ -36,9 +35,9 @@ rpkica_stop() { for i in rpkid pubd irdbd rootd do - if /bin/test -f $rpkica_pid_dir/$i.pid + if /bin/test -f ${rpkica_pid_dir}/${i}.pid then - /bin/kill `/bin/cat $rpkica_pid_dir/$i.pid` + /bin/kill `/bin/cat ${rpkica_pid_dir}/${i}.pid` fi done return 0 diff --git a/buildtools/freebsd-skeleton/rpki-rp/Makefile b/buildtools/freebsd-skeleton/rpki-rp/Makefile index 16537fdc..e020eaf3 100644 --- a/buildtools/freebsd-skeleton/rpki-rp/Makefile +++ b/buildtools/freebsd-skeleton/rpki-rp/Makefile @@ -1,7 +1,7 @@ PORTNAME= rpki-rp PORTVERSION= 0.%(SVNVERSION)s CATEGORIES= net -MASTER_SITES= http://download.rpki.net/ +MASTER_SITES= %(MASTER_SITE)s DISTFILES= rpki-%(SVNBRANCH)s-r%(SVNVERSION)s.tar.xz WRKSRC= ${WRKDIR}/rpki-%(SVNBRANCH)s-r%(SVNVERSION)s MAINTAINER= sra@hactrn.net diff --git a/buildtools/make-rcynic-script.py b/buildtools/make-rcynic-script.py index 94fb6f32..fdfb3d6b 100644 --- a/buildtools/make-rcynic-script.py +++ b/buildtools/make-rcynic-script.py @@ -24,8 +24,8 @@ sys.stdout.write('''\ ''' % os.environ) for k, v in os.environ.iteritems(): - if k.startswith("AC_") and k != "AC_PYTHON_INTERPRETER": - sys.stdout.write("%s = '''%s'''\n" % (k.lower(), v)) + if k.startswith("AC_") and k != "AC_PYTHON_INTERPRETER": + sys.stdout.write("%s = '''%s'''\n" % (k.lower(), v)) sys.stdout.write('''\ diff --git a/buildtools/make-relaxng.py b/buildtools/make-relaxng.py index d540fa9a..a8b562fa 100644 --- a/buildtools/make-relaxng.py +++ b/buildtools/make-relaxng.py @@ -1,5 +1,5 @@ # $Id$ -# +# # Copyright (C) 2014 Dragon Research Labs ("DRL") # Portions copyright (C) 2009--2012 Internet Systems Consortium ("ISC") # Portions copyright (C) 2007--2008 American Registry for Internet Numbers ("ARIN") @@ -26,6 +26,8 @@ import sys header = """\ # Automatically generated, do not edit. +# pylint: skip-file + from rpki.relaxng_parser import RelaxNGParser """ @@ -40,13 +42,13 @@ del RelaxNGParser """ def symbol(s): - for suffix in (".rng", "-schema"): - if s.endswith(suffix): - s = s[:-len(suffix)] - return s.replace("-", "_") + for suffix in (".rng", "-schema"): + if s.endswith(suffix): + s = s[:-len(suffix)] + return s.replace("-", "_") sys.stdout.write(header) for fn in sys.argv[1:]: - with open(fn, "r") as f: - sys.stdout.write(format % dict(name = symbol(fn), rng = f.read())) + with open(fn, "r") as f: + sys.stdout.write(format % dict(name = symbol(fn), rng = f.read())) sys.stdout.write(footer) diff --git a/buildtools/make-sql-schemas.py b/buildtools/make-sql-schemas.py index 0df775c2..051f17e8 100644 --- a/buildtools/make-sql-schemas.py +++ b/buildtools/make-sql-schemas.py @@ -1,11 +1,11 @@ # $Id$ -# +# # Copyright (C) 2009-2012 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -13,13 +13,13 @@ # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE # OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR # PERFORMANCE OF THIS SOFTWARE. -# +# # Portions copyright (C) 2007--2008 American Registry for Internet Numbers ("ARIN") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ARIN DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -47,6 +47,6 @@ format_2 = """\ print format_1 for name in schemas: - print format_2 % { - "name" : name, - "sql" : open(name + ".sql").read() } + print format_2 % { + "name" : name, + "sql" : open(name + ".sql").read() } diff --git a/buildtools/make-version.py b/buildtools/make-version.py index a73a89ab..09d43801 100644 --- a/buildtools/make-version.py +++ b/buildtools/make-version.py @@ -2,11 +2,11 @@ # $Id$ # Copyright (C) 2013 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, @@ -37,33 +37,33 @@ import sys unknown = "Unknown" try: - v = subprocess.Popen(("svnversion", "-c"), stdout = subprocess.PIPE).communicate()[0] - err = None + v = subprocess.Popen(("svnversion", "-c"), stdout = subprocess.PIPE).communicate()[0] + err = None except Exception, e: - v = unknown - err = e + v = unknown + err = e if any(s in v for s in ("Unversioned", "Uncommitted", unknown)): - v = unknown + v = unknown else: - v = "0." + v.strip().split(":")[-1].translate(None, "SMP") + v = "0." + v.strip().split(":")[-1].translate(None, "SMP") try: - old = open("VERSION", "r").read().strip() + old = open("VERSION", "r").read().strip() except: - old = None + old = None if err is not None and (old is None or old == unknown): - sys.stderr.write("Warning: No saved version and svnversion failed: %s\n" % err) + sys.stderr.write("Warning: No saved version and svnversion failed: %s\n" % err) if v == unknown: - if old is not None and old != unknown: - v = old - else: - sys.stderr.write("Warning: Could not determine software version\n") + if old is not None and old != unknown: + v = old + else: + sys.stderr.write("Warning: Could not determine software version\n") if old is None or v != old: - with open("rpki/version.py", "w") as f: - f.write("VERSION = \"%s\"\n" % v) - with open("VERSION", "w") as f: - f.write(v + "\n") + with open("rpki/version.py", "w") as f: + f.write("VERSION = \"%s\"\n" % v) + with open("VERSION", "w") as f: + f.write(v + "\n") diff --git a/buildtools/pull-doc-from-wiki.py b/buildtools/pull-doc-from-wiki.py index e3b61b53..5995823a 100644 --- a/buildtools/pull-doc-from-wiki.py +++ b/buildtools/pull-doc-from-wiki.py @@ -1,12 +1,12 @@ # $Id$ -# +# # Copyright (C) 2014 Dragon Research Labs ("DRL") # Portions copyright (C) 2012 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notices and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND DRL AND ISC DISCLAIM ALL # WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL OR @@ -45,94 +45,94 @@ import tempfile def main(): - base = "https://trac.rpki.net" - - parser = argparse.ArgumentParser(description = __doc__) - parser.add_argument("-b", "--base_url", - default = base, - help = "base URL for documentation web site") - parser.add_argument("-t", "--toc", - default = base + "/wiki/doc/RPKI/TOC", - help = "table of contents URL") - parser.add_argument("-d", "--directory", - default = ".", - help = "output directory") - parser.add_argument("-p", "--pdf_file", - default = "manual.pdf", - help = "output PDF file") - parser.add_argument("-r", "--html2textrc", - default = os.path.join(os.path.dirname(sys.argv[0]), "html2textrc"), - help = "html2textrc rules file") - args = parser.parse_args() - - urls = str(xsl_get_toc(lxml.etree.parse(urllib.urlopen(args.toc)).getroot(), - basename = repr(args.base_url))).splitlines() - - assert all(urlparse.urlparse(url).path.startswith("/wiki/") for url in urls) - - htmldoc = subprocess.Popen( - ("htmldoc", "--book", "--title", "--outfile", args.pdf_file, "--format", "pdf", - "--firstpage", "p1", "--size", "Universal", "--no-duplex", - "--fontsize", "11.0", "--fontspacing", "1.1", "--headfootsize", "11.0", - "--headingfont", "Helvetica", "--bodyfont", "Times", "--headfootfont", "Helvetica-Oblique", - "-"), stdin = subprocess.PIPE) - - lxml.etree.ElementTree(xml_title).write(htmldoc.stdin) - - png_fns = [] - - for url in urls: - path = urlparse.urlparse(url).path - page = xsl_get_page(lxml.etree.parse(urllib.urlopen(url)).getroot(), - basename = repr(args.base_url), - path = repr(path)) - - for img in page.xpath("//img | //object | //embed"): - attr = "data" if img.tag == "object" else "src" - img_url = img.get(attr) - if img_url.endswith(".svg"): - #sys.stderr.write("Converting %s to PNG\n" % img_url) - png_fd, png_fn = tempfile.mkstemp(suffix = ".png") - subprocess.Popen(("svg2png", "-h", "700", "-w", "600", "-", "-"), - stdout = png_fd, - stdin = subprocess.PIPE).communicate(urllib.urlopen(img_url).read()) - os.close(png_fd) - img.set(attr, png_fn) - png_fns.append(png_fn) - - page.write(htmldoc.stdin) - - html2text = subprocess.Popen(("html2text", "-rcfile", args.html2textrc, "-nobs", "-ascii"), - stdin = subprocess.PIPE, - stdout = subprocess.PIPE) - page.write(html2text.stdin) - html2text.stdin.close() - lines = html2text.stdout.readlines() - html2text.stdout.close() - html2text.wait() - - while lines and lines[0].isspace(): - del lines[0] - - fn = os.path.join(args.directory, path[len("/wiki/"):].replace("/", ".")) - f = open(fn, "w") - want_blank = False - for line in lines: - blank = line.isspace() - if want_blank and not blank: - f.write("\n") - if not blank: - f.write(line) - want_blank = blank - f.close() - sys.stderr.write("Wrote %s\n" % fn) - - htmldoc.stdin.close() - htmldoc.wait() - sys.stderr.write("Wrote %s\n" % args.pdf_file) - - for png_fn in png_fns: - os.unlink(png_fn) + base = "https://trac.rpki.net" + + parser = argparse.ArgumentParser(description = __doc__) + parser.add_argument("-b", "--base_url", + default = base, + help = "base URL for documentation web site") + parser.add_argument("-t", "--toc", + default = base + "/wiki/doc/RPKI/TOC", + help = "table of contents URL") + parser.add_argument("-d", "--directory", + default = ".", + help = "output directory") + parser.add_argument("-p", "--pdf_file", + default = "manual.pdf", + help = "output PDF file") + parser.add_argument("-r", "--html2textrc", + default = os.path.join(os.path.dirname(sys.argv[0]), "html2textrc"), + help = "html2textrc rules file") + args = parser.parse_args() + + urls = str(xsl_get_toc(lxml.etree.parse(urllib.urlopen(args.toc)).getroot(), + basename = repr(args.base_url))).splitlines() + + assert all(urlparse.urlparse(url).path.startswith("/wiki/") for url in urls) + + htmldoc = subprocess.Popen( + ("htmldoc", "--book", "--title", "--outfile", args.pdf_file, "--format", "pdf", + "--firstpage", "p1", "--size", "Universal", "--no-duplex", + "--fontsize", "11.0", "--fontspacing", "1.1", "--headfootsize", "11.0", + "--headingfont", "Helvetica", "--bodyfont", "Times", "--headfootfont", "Helvetica-Oblique", + "-"), stdin = subprocess.PIPE) + + lxml.etree.ElementTree(xml_title).write(htmldoc.stdin) + + png_fns = [] + + for url in urls: + path = urlparse.urlparse(url).path + page = xsl_get_page(lxml.etree.parse(urllib.urlopen(url)).getroot(), + basename = repr(args.base_url), + path = repr(path)) + + for img in page.xpath("//img | //object | //embed"): + attr = "data" if img.tag == "object" else "src" + img_url = img.get(attr) + if img_url.endswith(".svg"): + #sys.stderr.write("Converting %s to PNG\n" % img_url) + png_fd, png_fn = tempfile.mkstemp(suffix = ".png") + subprocess.Popen(("svg2png", "-h", "700", "-w", "600", "-", "-"), + stdout = png_fd, + stdin = subprocess.PIPE).communicate(urllib.urlopen(img_url).read()) + os.close(png_fd) + img.set(attr, png_fn) + png_fns.append(png_fn) + + page.write(htmldoc.stdin) + + html2text = subprocess.Popen(("html2text", "-rcfile", args.html2textrc, "-nobs", "-ascii"), + stdin = subprocess.PIPE, + stdout = subprocess.PIPE) + page.write(html2text.stdin) + html2text.stdin.close() + lines = html2text.stdout.readlines() + html2text.stdout.close() + html2text.wait() + + while lines and lines[0].isspace(): + del lines[0] + + fn = os.path.join(args.directory, path[len("/wiki/"):].replace("/", ".")) + f = open(fn, "w") + want_blank = False + for line in lines: + blank = line.isspace() + if want_blank and not blank: + f.write("\n") + if not blank: + f.write(line) + want_blank = blank + f.close() + sys.stderr.write("Wrote %s\n" % fn) + + htmldoc.stdin.close() + htmldoc.wait() + sys.stderr.write("Wrote %s\n" % args.pdf_file) + + for png_fn in png_fns: + os.unlink(png_fn) # HTMLDOC title page. At some point we might want to generate this # dynamically as an ElementTree, but static content will do for the @@ -188,7 +188,7 @@ xsl_get_toc = lxml.etree.XSLT(lxml.etree.XML('''\ # we care, and this seems to work. # # Original author's explanation: -# +# # The rather convoluted XPath expression for selecting the following # sibling aaa nodes which are merged with the current one: # @@ -321,12 +321,12 @@ xsl_get_page = lxml.etree.XSLT(lxml.etree.XML('''\ <xsl:otherwise> <xsl:value-of select="$s"/> </xsl:otherwise> - </xsl:choose> + </xsl:choose> </xsl:template> <xsl:template match="ol"> <xsl:if test="not(preceding-sibling::*[1]/self::ol)"> - <xsl:variable name="following" + <xsl:variable name="following" select="following-sibling::ol[ not(preceding-sibling::*[ not(self::ol) and diff --git a/buildtools/pylint.rc b/buildtools/pylint.rc index ed296108..ac893ad6 100644 --- a/buildtools/pylint.rc +++ b/buildtools/pylint.rc @@ -19,10 +19,22 @@ [MASTER] profile=no -ignore=.svn + +# Including "gui" here is a temporary measure: it's risky, but so is +# making ten zillion cosmetic changes in a co-worker's code on a +# long-running development branch. + +ignore=.svn,.git,migrations,south_migrations,gui + persistent=yes cache-size=500 -load-plugins= + +# We make heavy use of Django, which confuses pylint. Fortunately, there's a plug-in. +load-plugins=pylint_django + +# Extension (C, etc) modules that pylint should trust enough to import. + +extension-pkg-whitelist=lxml,rpki.POW [MESSAGES CONTROL] @@ -44,14 +56,24 @@ disable-msg-cat= #enable-msg= # Disable the message(s) with the given id(s). -disable=R0801,R0903,R0913,C0321,R0904,W0201,E1101,W0614,C0301,R0901,C0302,R0902,R0201,W0613,R0912,R0915,W0703,W0212,R0914,W0603,W0142,I0011,C0111,C0103,R0401,C0326,R0911,C0325 +# +# I0011 is (sort of) special, in that it marks places where we've used +# inline overrides in the code to control pylint's behavior. +# Ordinarily we leave this turned off, but it's a good idea to run +# with it enabled every once in a while to see what we've overriden. +# +disable=I0011,I0013,R0801,C0111,C0301,C0326,W0702,R0902,R0913,W0703,R0912,R0903,R0915,R0914,C0302,W0613,R0201,R0901,R0904,C0325,R0911,C0103,R0401 +# +# Additional messages we used to have disabled but now appear to be +# able to leave alone. +# +#disable=C0321,W0201,E1101,W0614,W0212,W0603,W0142,C0330,W0311,E1124 [REPORTS] #output-format=parseable msg-template={path}:{line}: [{msg_id}({symbol}), {obj}] {msg} -include-ids=yes files-output=no reports=no evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) @@ -196,7 +218,7 @@ max-line-length=80 max-module-lines=1000 # String used as indentation unit. -indent-string=' ' +indent-string=' ' [SIMILARITIES] diff --git a/buildtools/rpki-pbuilder.py b/buildtools/rpki-pbuilder.py index 5043c60e..32247ff8 100644 --- a/buildtools/rpki-pbuilder.py +++ b/buildtools/rpki-pbuilder.py @@ -4,11 +4,11 @@ # # Copyright (C) 2014 Dragon Research Labs ("DRL") # Portions copyright (C) 2013 Internet Systems Consortium ("ISC") -# +# # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notices and this permission notice appear in all copies. -# +# # THE SOFTWARE IS PROVIDED "AS IS" AND DRL AND ISC DISCLAIM ALL # WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DRL OR @@ -254,10 +254,10 @@ class Release(object): DebOverride: override.%(release)s DscOverride: override.%(release)s """ % dict( - distribution = self.distribution, - Distribution = self.distribution.capitalize(), - architectures = " ".join(self.architectures), - release = self.release))) + distribution = self.distribution, + Distribution = self.distribution.capitalize(), + architectures = " ".join(self.architectures), + release = self.release))) fn = os.path.join(self.tree, "conf", "options") if not os.path.exists(fn): @@ -275,14 +275,14 @@ class Release(object): with open(fn, "w") as f: for pkg in self.backports: f.write(dedent("""\ - %-30s Priority optional - %-30s Section python + %-30s Priority optional + %-30s Section python """ % (pkg, pkg))) f.write(dedent("""\ - rpki-ca Priority extra - rpki-ca Section net - rpki-rp Priority extra - rpki-rp Section net + rpki-ca Priority extra + rpki-ca Section net + rpki-rp Priority extra + rpki-rp Section net """)) fn = os.path.join(args.apt_tree, "rpki.%s.list" % self.release) |