diff options
| -rw-r--r-- | rpkid/Makefile.in | 7 | ||||
| -rw-r--r-- | rpkid/rpki/irdb/models.py | 1 | ||||
| -rw-r--r-- | rpkid/rpki/rpkic.py | 2029 | ||||
| -rw-r--r-- | rpkid/rpki/x509.py | 7 | ||||
| -rw-r--r-- | rpkid/rpkic.py | 21 | ||||
| -rw-r--r-- | rpkid/tests/smoketest.py | 12 | ||||
| -rw-r--r-- | rpkid/tests/sql-cleaner.py | 33 | ||||
| -rw-r--r-- | rpkid/tests/yamltest.py | 40 | ||||
| -rw-r--r-- | scripts/convert-from-entitydb-to-sql.py | 3 |
9 files changed, 2116 insertions, 37 deletions
diff --git a/rpkid/Makefile.in b/rpkid/Makefile.in index 67a6cbe4..aba6872b 100644 --- a/rpkid/Makefile.in +++ b/rpkid/Makefile.in @@ -42,7 +42,7 @@ SETUP_PY = \ POW_SO = rpki/POW/_POW.so SCRIPTS = rpki-sql-backup rpki-sql-setup rpki-start-servers irbe_cli irdbd myrpki \ - pubd rootd rpkid portal-gui/scripts/rpkigui-load-csv \ + pubd rootd rpkic rpkid portal-gui/scripts/rpkigui-load-csv \ portal-gui/scripts/rpkigui-add-user portal-gui/scripts/rpkigui-response \ portal-gui/scripts/rpkigui-rcynic @@ -123,7 +123,7 @@ tags: Makefile find . -type f \( -name '*.py' -o -name '*.sql' -o -name '*.rnc' -o -name '*.py.in' \) ! -name relaxng.py ! -name sql_schemas.py ! -name __doc__.py | etags - lint: - pylint --rcfile ${abs_top_srcdir}/buildtools/pylint.rc rpki/[a-z]*.py *d.py rpki-*.py myrpki.py irbe_cli.py tests/*.py + pylint --rcfile ${abs_top_srcdir}/buildtools/pylint.rc rpki/[a-z]*.py *d.py rpki-*.py myrpki.py rpkic.py irbe_cli.py tests/*.py # Documentation @@ -235,6 +235,9 @@ pubd: pubd.py rootd: rootd.py ${COMPILE_PYTHON} +rpkic: rpkic.py + ${COMPILE_PYTHON} + rpkid: rpkid.py ${COMPILE_PYTHON} diff --git a/rpkid/rpki/irdb/models.py b/rpkid/rpki/irdb/models.py index 107d2f5a..e05f3f03 100644 --- a/rpkid/rpki/irdb/models.py +++ b/rpkid/rpki/irdb/models.py @@ -121,7 +121,6 @@ class PKCS10Field(DERField): class SignedReferral(rpki.x509.XML_CMS_object): encoding = "us-ascii" schema = rpki.relaxng.myrpki - saxify = staticmethod(lambda x: x) class SignedReferralField(DERField): description = "CMS signed object containing XML" diff --git a/rpkid/rpki/rpkic.py b/rpkid/rpki/rpkic.py new file mode 100644 index 00000000..54427ebb --- /dev/null +++ b/rpkid/rpki/rpkic.py @@ -0,0 +1,2029 @@ +""" +This (oversized) module used to be an (oversized) program. +Refactoring in progress, some doc still needs updating. + + +This program is now the merger of three different tools: the old +myrpki.py script, the old myirbe.py script, and the newer setup.py CLI +tool. As such, it is still in need of some cleanup, but the need to +provide a saner user interface is more urgent than internal code +prettiness at the moment. In the long run, 90% of the code in this +file probably ought to move to well-designed library modules. + +Overall goal here is to build up the configuration necessary to run +rpkid and friends, by reading a config file, a collection of .CSV +files, and the results of a few out-of-band XML setup messages +exchanged with one's parents, children, and so forth. + +The config file is in an OpenSSL-compatible format, the CSV files are +simple tab-delimited text. The XML files are all generated by this +program, either the local instance or an instance being run by another +player in the system; the mechanism used to exchange these setup +messages is outside the scope of this program, feel free to use +PGP-signed mail, a web interface (not provided), USB stick, carrier +pigeons, whatever works. + +With one exception, the commands in this program avoid using any +third-party Python code other than the rpki libraries themselves; with +the same one exception, all OpenSSL work is done with the OpenSSL +command line tool (the one built as a side effect of building rcynic +will do, if your platform has no system copy or the system copy is too +old). This is all done in an attempt to make the code more portable, +so one can run most of the RPKI back end software on a laptop or +whatever. The one exception is the configure_daemons command, which +must, of necessity, use the same communication libraries as the +daemons with which it is conversing. So that one command will not +work if the correct Python modules are not available. + + +$Id$ + +Copyright (C) 2009--2011 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import subprocess, csv, re, os, getopt, sys, base64, time, glob, copy, warnings +import rpki.config, rpki.cli, rpki.sundial, rpki.log, rpki.oids +import rpki.http, rpki.resource_set, rpki.relaxng, rpki.exceptions +import rpki.left_right, rpki.x509, rpki.async + +from lxml.etree import (Element, SubElement, ElementTree, + fromstring as ElementFromString, + tostring as ElementToString) + + + +# Our XML namespace and protocol version. + +namespace = "http://www.hactrn.net/uris/rpki/myrpki/" +version = "2" +namespaceQName = "{" + namespace + "}" + +# Whether to include incomplete entries when rendering to XML. + +allow_incomplete = False + +# Whether to whine about incomplete entries while rendering to XML. + +whine = True + +class BadCommandSyntax(Exception): + """ + Bad command line syntax. + """ + +class BadPrefixSyntax(Exception): + """ + Bad prefix syntax. + """ + +class CouldntTalkToDaemon(Exception): + """ + Couldn't talk to daemon. + """ + +class BadCSVSyntax(Exception): + """ + Bad CSV syntax. + """ + +class BadXMLMessage(Exception): + """ + Bad XML message. + """ + +class PastExpiration(Exception): + """ + Expiration date has already passed. + """ + +class CantRunRootd(Exception): + """ + Can't run rootd. + """ + +class comma_set(set): + """ + Minor customization of set(), to provide a print syntax. + """ + + def __str__(self): + return ",".join(self) + +class EntityDB(object): + """ + Wrapper for entitydb path lookups and iterations. + """ + + def __init__(self, cfg): + self.dir = cfg.get("entitydb_dir", "entitydb") + self.identity = os.path.join(self.dir, "identity.xml") + + def __call__(self, dirname, filebase = None): + if filebase is None: + return os.path.join(self.dir, dirname) + else: + return os.path.join(self.dir, dirname, filebase + ".xml") + + def iterate(self, dir, base = "*"): + return glob.iglob(os.path.join(self.dir, dir, base + ".xml")) + +class roa_request(object): + """ + Representation of a ROA request. + """ + + v4re = re.compile("^([0-9]{1,3}\.){3}[0-9]{1,3}/[0-9]+(-[0-9]+)?$", re.I) + v6re = re.compile("^([0-9a-f]{0,4}:){0,15}[0-9a-f]{0,4}/[0-9]+(-[0-9]+)?$", re.I) + + def __init__(self, asn, group): + self.asn = asn + self.group = group + self.v4 = comma_set() + self.v6 = comma_set() + + def __repr__(self): + s = "<%s asn %s group %s" % (self.__class__.__name__, self.asn, self.group) + if self.v4: + s += " v4 %s" % self.v4 + if self.v6: + s += " v6 %s" % self.v6 + return s + ">" + + def add(self, prefix): + """ + Add one prefix to this ROA request. + """ + if self.v4re.match(prefix): + self.v4.add(prefix) + elif self.v6re.match(prefix): + self.v6.add(prefix) + else: + raise BadPrefixSyntax, "Bad prefix syntax: %r" % (prefix,) + + def xml(self, e): + """ + Generate XML element represeting representing this ROA request. + """ + e = SubElement(e, "roa_request", + asn = self.asn, + v4 = str(self.v4), + v6 = str(self.v6)) + e.tail = "\n" + +class roa_requests(dict): + """ + Database of ROA requests. + """ + + def add(self, asn, group, prefix): + """ + Add one <ASN, group, prefix> set to ROA request database. + """ + key = (asn, group) + if key not in self: + self[key] = roa_request(asn, group) + self[key].add(prefix) + + def xml(self, e): + """ + Render ROA requests as XML elements. + """ + for r in self.itervalues(): + r.xml(e) + + @classmethod + def from_csv(cls, roa_csv_file): + """ + Parse ROA requests from CSV file. + """ + self = cls() + # format: p/n-m asn group + for pnm, asn, group in csv_reader(roa_csv_file, columns = 3): + self.add(asn = asn, group = group, prefix = pnm) + return self + +class child(object): + """ + Representation of one child entity. + """ + + v4re = re.compile("^(([0-9]{1,3}\.){3}[0-9]{1,3}/[0-9]+)|(([0-9]{1,3}\.){3}[0-9]{1,3}-([0-9]{1,3}\.){3}[0-9]{1,3})$", re.I) + v6re = re.compile("^(([0-9a-f]{0,4}:){0,15}[0-9a-f]{0,4}/[0-9]+)|(([0-9a-f]{0,4}:){0,15}[0-9a-f]{0,4}-([0-9a-f]{0,4}:){0,15}[0-9a-f]{0,4})$", re.I) + + def __init__(self, handle): + self.handle = handle + self.asns = comma_set() + self.v4 = comma_set() + self.v6 = comma_set() + self.validity = None + self.bpki_certificate = None + + def __repr__(self): + s = "<%s %s" % (self.__class__.__name__, self.handle) + if self.asns: + s += " asn %s" % self.asns + if self.v4: + s += " v4 %s" % self.v4 + if self.v6: + s += " v6 %s" % self.v6 + if self.validity: + s += " valid %s" % self.validity + if self.bpki_certificate: + s += " cert %s" % self.bpki_certificate + return s + ">" + + def add(self, prefix = None, asn = None, validity = None, bpki_certificate = None): + """ + Add prefix, autonomous system number, validity date, or BPKI + certificate for this child. + """ + if prefix is not None: + if self.v4re.match(prefix): + self.v4.add(prefix) + elif self.v6re.match(prefix): + self.v6.add(prefix) + else: + raise BadPrefixSyntax, "Bad prefix syntax: %r" % (prefix,) + if asn is not None: + self.asns.add(asn) + if validity is not None: + self.validity = validity + if bpki_certificate is not None: + self.bpki_certificate = bpki_certificate + + def xml(self, e): + """ + Render this child as an XML element. + """ + complete = self.bpki_certificate and self.validity + if whine and not complete: + print "Incomplete child entry %s" % self + if complete or allow_incomplete: + e = SubElement(e, "child", + handle = self.handle, + valid_until = self.validity, + asns = str(self.asns), + v4 = str(self.v4), + v6 = str(self.v6)) + e.tail = "\n" + if self.bpki_certificate: + PEMElement(e, "bpki_certificate", self.bpki_certificate) + +class children(dict): + """ + Database of children. + """ + + def add(self, handle, prefix = None, asn = None, validity = None, bpki_certificate = None): + """ + Add resources to a child, creating the child object if necessary. + """ + if handle not in self: + self[handle] = child(handle) + self[handle].add(prefix = prefix, asn = asn, validity = validity, bpki_certificate = bpki_certificate) + + def xml(self, e): + """ + Render children database to XML. + """ + for c in self.itervalues(): + c.xml(e) + + @classmethod + def from_entitydb(cls, prefix_csv_file, asn_csv_file, fxcert, entitydb): + """ + Parse child data from entitydb. + """ + self = cls() + for f in entitydb.iterate("children"): + c = etree_read(f) + self.add(handle = os.path.splitext(os.path.split(f)[-1])[0], + validity = c.get("valid_until"), + bpki_certificate = fxcert(c.findtext("bpki_child_ta"))) + # childname p/n + for handle, pn in csv_reader(prefix_csv_file, columns = 2): + self.add(handle = handle, prefix = pn) + # childname asn + for handle, asn in csv_reader(asn_csv_file, columns = 2): + self.add(handle = handle, asn = asn) + return self + +class parent(object): + """ + Representation of one parent entity. + """ + + def __init__(self, handle): + self.handle = handle + self.service_uri = None + self.bpki_cms_certificate = None + self.myhandle = None + self.sia_base = None + + def __repr__(self): + s = "<%s %s" % (self.__class__.__name__, self.handle) + if self.myhandle: + s += " myhandle %s" % self.myhandle + if self.service_uri: + s += " uri %s" % self.service_uri + if self.sia_base: + s += " sia %s" % self.sia_base + if self.bpki_cms_certificate: + s += " cms %s" % self.bpki_cms_certificate + return s + ">" + + def add(self, service_uri = None, + bpki_cms_certificate = None, + myhandle = None, + sia_base = None): + """ + Add service URI or BPKI certificates to this parent object. + """ + if service_uri is not None: + self.service_uri = service_uri + if bpki_cms_certificate is not None: + self.bpki_cms_certificate = bpki_cms_certificate + if myhandle is not None: + self.myhandle = myhandle + if sia_base is not None: + self.sia_base = sia_base + + def xml(self, e): + """ + Render this parent object to XML. + """ + complete = self.bpki_cms_certificate and self.myhandle and self.service_uri and self.sia_base + if whine and not complete: + print "Incomplete parent entry %s" % self + if complete or allow_incomplete: + e = SubElement(e, "parent", + handle = self.handle, + myhandle = self.myhandle, + service_uri = self.service_uri, + sia_base = self.sia_base) + e.tail = "\n" + if self.bpki_cms_certificate: + PEMElement(e, "bpki_cms_certificate", self.bpki_cms_certificate) + +class parents(dict): + """ + Database of parent objects. + """ + + def add(self, handle, + service_uri = None, + bpki_cms_certificate = None, + myhandle = None, + sia_base = None): + """ + Add service URI or certificates to parent object, creating it if necessary. + """ + if handle not in self: + self[handle] = parent(handle) + self[handle].add(service_uri = service_uri, + bpki_cms_certificate = bpki_cms_certificate, + myhandle = myhandle, + sia_base = sia_base) + + def xml(self, e): + for c in self.itervalues(): + c.xml(e) + + @classmethod + def from_entitydb(cls, fxcert, entitydb): + """ + Parse parent data from entitydb. + """ + self = cls() + for f in entitydb.iterate("parents"): + h = os.path.splitext(os.path.split(f)[-1])[0] + p = etree_read(f) + r = etree_read(f.replace(os.path.sep + "parents" + os.path.sep, + os.path.sep + "repositories" + os.path.sep)) + if r.get("type") == "confirmed": + self.add(handle = h, + service_uri = p.get("service_uri"), + bpki_cms_certificate = fxcert(p.findtext("bpki_resource_ta")), + myhandle = p.get("child_handle"), + sia_base = r.get("sia_base")) + elif whine: + print "Parent %s's repository entry in state %s, skipping this parent" % (h, r.get("type")) + return self + +class repository(object): + """ + Representation of one repository entity. + """ + + def __init__(self, handle): + self.handle = handle + self.service_uri = None + self.bpki_certificate = None + + def __repr__(self): + s = "<%s %s" % (self.__class__.__name__, self.handle) + if self.service_uri: + s += " uri %s" % self.service_uri + if self.bpki_certificate: + s += " cert %s" % self.bpki_certificate + return s + ">" + + def add(self, service_uri = None, bpki_certificate = None): + """ + Add service URI or BPKI certificates to this repository object. + """ + if service_uri is not None: + self.service_uri = service_uri + if bpki_certificate is not None: + self.bpki_certificate = bpki_certificate + + def xml(self, e): + """ + Render this repository object to XML. + """ + complete = self.bpki_certificate and self.service_uri + if whine and not complete: + print "Incomplete repository entry %s" % self + if complete or allow_incomplete: + e = SubElement(e, "repository", + handle = self.handle, + service_uri = self.service_uri) + e.tail = "\n" + if self.bpki_certificate: + PEMElement(e, "bpki_certificate", self.bpki_certificate) + +class repositories(dict): + """ + Database of repository objects. + """ + + def add(self, handle, + service_uri = None, + bpki_certificate = None): + """ + Add service URI or certificate to repository object, creating it if necessary. + """ + if handle not in self: + self[handle] = repository(handle) + self[handle].add(service_uri = service_uri, + bpki_certificate = bpki_certificate) + + def xml(self, e): + for c in self.itervalues(): + c.xml(e) + + @classmethod + def from_entitydb(cls, fxcert, entitydb): + """ + Parse repository data from entitydb. + """ + self = cls() + for f in entitydb.iterate("repositories"): + h = os.path.splitext(os.path.split(f)[-1])[0] + r = etree_read(f) + if r.get("type") == "confirmed": + self.add(handle = h, + service_uri = r.get("service_uri"), + bpki_certificate = fxcert(r.findtext("bpki_server_ta"))) + elif whine: + print "Repository %s in state %s, skipping this repository" % (h, r.get("type")) + + return self + +class csv_reader(object): + """ + Reader for tab-delimited text that's (slightly) friendlier than the + stock Python csv module (which isn't intended for direct use by + humans anyway, and neither was this package originally, but that + seems to be the way that it has evolved...). + + Columns parameter specifies how many columns users of the reader + expect to see; lines with fewer columns will be padded with None + values. + + Original API design for this class courtesy of Warren Kumari, but + don't blame him if you don't like what I did with his ideas. + """ + + def __init__(self, filename, columns = None, min_columns = None, comment_characters = "#;"): + assert columns is None or isinstance(columns, int) + assert min_columns is None or isinstance(min_columns, int) + if columns is not None and min_columns is None: + min_columns = columns + self.filename = filename + self.columns = columns + self.min_columns = min_columns + self.comment_characters = comment_characters + self.file = open(filename, "r") + + def __iter__(self): + line_number = 0 + for line in self.file: + line_number += 1 + line = line.strip() + if not line or line[0] in self.comment_characters: + continue + fields = line.split() + if self.min_columns is not None and len(fields) < self.min_columns: + raise BadCSVSyntax, "%s:%d: Not enough columns in line %r" % (self.filename, line_number, line) + if self.columns is not None and len(fields) > self.columns: + raise BadCSVSyntax, "%s:%d: Too many columns in line %r" % (self.filename, line_number, line) + if self.columns is not None and len(fields) < self.columns: + fields += tuple(None for i in xrange(self.columns - len(fields))) + yield fields + +class csv_writer(object): + """ + Writer object for tab delimited text. We just use the stock CSV + module in excel-tab mode for this. + + If "renmwo" is set (default), the file will be written to + a temporary name and renamed to the real filename after closing. + """ + + def __init__(self, filename, renmwo = True): + self.filename = filename + self.renmwo = "%s.~renmwo%d~" % (filename, os.getpid()) if renmwo else filename + self.file = open(self.renmwo, "w") + self.writer = csv.writer(self.file, dialect = csv.get_dialect("excel-tab")) + + def close(self): + """ + Close this writer. + """ + if self.file is not None: + self.file.close() + self.file = None + if self.filename != self.renmwo: + os.rename(self.renmwo, self.filename) + + def __getattr__(self, attr): + """ + Fake inheritance from whatever object csv.writer deigns to give us. + """ + return getattr(self.writer, attr) + +def PEMBase64(filename): + """ + Extract Base64 encoded data from a PEM file. + """ + lines = open(filename).readlines() + while lines: + if lines.pop(0).startswith("-----BEGIN "): + break + while lines: + if lines.pop(-1).startswith("-----END "): + break + return "".join(lines) + +def PEMElement(e, tag, filename, **kwargs): + """ + Create an XML element containing Base64 encoded data taken from a + PEM file. + """ + if e.text is None: + e.text = "\n" + se = SubElement(e, tag, **kwargs) + se.text = "\n" + PEMBase64(filename) + se.tail = "\n" + return se + +class CA(object): + """ + Representation of one certification authority. + """ + + # Mapping of path restriction values we use to OpenSSL config file + # section names. + + path_restriction = { 0 : "ca_x509_ext_xcert0", + 1 : "ca_x509_ext_xcert1" } + + def __init__(self, cfg_file, dir): + self.cfg = cfg_file + self.dir = dir + self.cer = dir + "/ca.cer" + self.key = dir + "/ca.key" + self.req = dir + "/ca.req" + self.crl = dir + "/ca.crl" + self.index = dir + "/index" + self.serial = dir + "/serial" + self.crlnum = dir + "/crl_number" + + cfg = rpki.config.parser(cfg_file, "myrpki") + self.openssl = cfg.get("openssl", "openssl") + + self.env = { "PATH" : os.environ["PATH"], + "BPKI_DIRECTORY" : dir, + "RANDFILE" : ".OpenSSL.whines.unless.I.set.this", + "OPENSSL_CONF" : cfg_file } + + def run_openssl(self, *cmd, **kwargs): + """ + Run an OpenSSL command, suppresses stderr unless OpenSSL returns + failure, and returns stdout. + """ + stdin = kwargs.pop("stdin", None) + env = self.env.copy() + env.update(kwargs) + cmd = (self.openssl,) + cmd + p = subprocess.Popen(cmd, env = env, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE) + stdout, stderr = p.communicate(stdin) + if p.wait() != 0: + sys.stderr.write("OpenSSL command failed: " + stderr + "\n") + raise subprocess.CalledProcessError(returncode = p.returncode, cmd = cmd) + return stdout + + def run_ca(self, *args): + """ + Run OpenSSL "ca" command with common initial arguments. + """ + self.run_openssl("ca", "-batch", "-config", self.cfg, *args) + + def run_req(self, key_file, req_file, log_key = sys.stdout): + """ + Run OpenSSL "genrsa" and "req" commands. + """ + if not os.path.exists(key_file): + if log_key: + log_key.write("Generating 2048-bit RSA key %s\n" % os.path.realpath(key_file)) + self.run_openssl("genrsa", "-out", key_file, "2048") + if not os.path.exists(req_file): + self.run_openssl("req", "-new", "-sha256", "-config", self.cfg, "-key", key_file, "-out", req_file) + + def run_dgst(self, input, algorithm = "md5"): + """ + Run OpenSSL "dgst" command, return cleaned-up result. + """ + hash = self.run_openssl("dgst", "-" + algorithm, stdin = input) + # + # Twits just couldn't leave well enough alone, grr. + hash = "".join(hash.split()) + if hash.startswith("(stdin)="): + hash = hash[len("(stdin)="):] + return hash + + @staticmethod + def touch_file(filename, content = None): + """ + Create dumb little text files expected by OpenSSL "ca" utility. + """ + if not os.path.exists(filename): + f = open(filename, "w") + if content is not None: + f.write(content) + f.close() + + def setup(self, ca_name): + """ + Set up this CA. ca_name is an X.509 distinguished name in + /tag=val/tag=val format. + """ + + modified = False + + if not os.path.exists(self.dir): + os.makedirs(self.dir) + self.touch_file(self.index) + self.touch_file(self.serial, "01\n") + self.touch_file(self.crlnum, "01\n") + + self.run_req(key_file = self.key, req_file = self.req) + + if not os.path.exists(self.cer): + modified = True + self.run_ca("-selfsign", "-extensions", "ca_x509_ext_ca", "-subj", ca_name, "-in", self.req, "-out", self.cer) + + if not os.path.exists(self.crl): + modified = True + self.run_ca("-gencrl", "-out", self.crl) + + return modified + + def ee(self, ee_name, base_name): + """ + Issue an end-enity certificate. + """ + key_file = "%s/%s.key" % (self.dir, base_name) + req_file = "%s/%s.req" % (self.dir, base_name) + cer_file = "%s/%s.cer" % (self.dir, base_name) + self.run_req(key_file = key_file, req_file = req_file) + if not os.path.exists(cer_file): + self.run_ca("-extensions", "ca_x509_ext_ee", "-subj", ee_name, "-in", req_file, "-out", cer_file) + return True + else: + return False + + def cms_xml_sign(self, ee_name, base_name, elt): + """ + Sign an XML object with CMS, return Base64 text. + """ + self.ee(ee_name, base_name) + return base64.b64encode(self.run_openssl( + "cms", "-sign", "-binary", "-outform", "DER", + "-keyid", "-md", "sha256", "-nodetach", "-nosmimecap", + "-econtent_type", ".".join(str(i) for i in rpki.oids.name2oid["id-ct-xml"]), + "-inkey", "%s/%s.key" % (self.dir, base_name), + "-signer", "%s/%s.cer" % (self.dir, base_name), + stdin = ElementToString(etree_pre_write(elt)))) + + def cms_xml_verify(self, b64, ca): + """ + Attempt to verify and extract XML from a Base64-encoded signed CMS + object. CA is the filename of a certificate that we expect to be + the issuer of the EE certificate bundled with the CMS, and must + previously have been cross-certified under our trust anchor. + """ + # In theory, we should be able to use the -certfile parameter to + # pass in the CA certificate, but in practice, I have never gotten + # this to work, either with the command line tool or in the + # OpenSSL C API. Dunno why. Passing both TA and CA via -CAfile + # does work, so we do that, using a temporary file, sigh. + CAfile = os.path.join(self.dir, "temp.%s.pem" % os.getpid()) + try: + f = open(CAfile, "w") + f.write(open(self.cer).read()) + f.write(open(ca).read()) + f.close() + return etree_post_read(ElementFromString(self.run_openssl( + "cms", "-verify", "-inform", "DER", "-CAfile", CAfile, + stdin = base64.b64decode(b64)))) + finally: + if os.path.exists(CAfile): + os.unlink(CAfile) + + def bsc(self, pkcs10): + """ + Issue BSC certificate, if we have a PKCS #10 request for it. + """ + + if pkcs10 is None: + return None, None + + pkcs10 = base64.b64decode(pkcs10) + + hash = self.run_dgst(pkcs10) + + req_file = "%s/bsc.%s.req" % (self.dir, hash) + cer_file = "%s/bsc.%s.cer" % (self.dir, hash) + + if not os.path.exists(cer_file): + self.run_openssl("req", "-inform", "DER", "-out", req_file, stdin = pkcs10) + self.run_ca("-extensions", "ca_x509_ext_ee", "-in", req_file, "-out", cer_file) + + return req_file, cer_file + + def fxcert(self, b64, filename = None, path_restriction = 0): + """ + Write PEM certificate to file, then cross-certify. + """ + fn = os.path.join(self.dir, filename or "temp.%s.cer" % os.getpid()) + der = base64.b64decode(b64) + if False: + try: + text = self.run_openssl("x509", "-inform", "DER", "-noout", + "-issuer", "-subject", stdin = der) + except: + text = "" + print "fxcert():", self.dir, filename, text + try: + self.run_openssl("x509", "-inform", "DER", "-out", fn, + stdin = der) + return self.xcert(fn, path_restriction) + finally: + if not filename and os.path.exists(fn): + os.unlink(fn) + + def xcert_filename(self, cert): + """ + Generate filename for a cross-certification. + + Extracts public key and subject name from PEM file and hash it so + we can use the result as a tag for cross-certifying this cert. + """ + + if cert and os.path.exists(cert): + return "%s/xcert.%s.cer" % (self.dir, self.run_dgst(self.run_openssl( + "x509", "-noout", "-pubkey", "-subject", "-in", cert)).strip()) + else: + return None + + def xcert(self, cert, path_restriction = 0): + """ + Cross-certify a certificate represented as a PEM file, if we + haven't already. This only works for self-signed certs, due to + limitations of the OpenSSL command line tool, but that suffices + for our purposes. + """ + + xcert = self.xcert_filename(cert) + if not os.path.exists(xcert): + self.run_ca("-ss_cert", cert, "-out", xcert, "-extensions", self.path_restriction[path_restriction]) + return xcert + + def xcert_revoke(self, cert): + """ + Revoke a cross-certification and regenerate CRL. + """ + + xcert = self.xcert_filename(cert) + if xcert: + self.run_ca("-revoke", xcert) + self.run_ca("-gencrl", "-out", self.crl) + +def etree_validate(e): + rpki.relaxng.myrpki.assertValid(e) + +def etree_write(e, filename, verbose = False, msg = None): + """ + Write out an etree to a file, safely. + + I still miss SYSCAL(RENMWO). + """ + filename = os.path.realpath(filename) + tempname = filename + if not filename.startswith("/dev/"): + tempname += ".tmp" + if verbose or msg: + print "Writing", filename + if msg: + print msg + e = etree_pre_write(e) + ElementTree(e).write(tempname) + if tempname != filename: + os.rename(tempname, filename) + +def etree_pre_write(e): + """ + Do the namespace frobbing needed on write; broken out of + etree_write() because also needed with ElementToString(). + """ + e = copy.deepcopy(e) + e.set("version", version) + for i in e.getiterator(): + if i.tag[0] != "{": + i.tag = namespaceQName + i.tag + assert i.tag.startswith(namespaceQName) + etree_validate(e) + return e + +def etree_read(filename, verbose = False): + """ + Read an etree from a file, verifying then stripping XML namespace + cruft. + """ + if verbose: + print "Reading", filename + e = ElementTree(file = filename).getroot() + return etree_post_read(e) + +def etree_post_read(e): + """ + Do the namespace frobbing needed on read; broken out of etree_read() + beause also needed by ElementFromString(). + """ + etree_validate(e) + for i in e.getiterator(): + if i.tag.startswith(namespaceQName): + i.tag = i.tag[len(namespaceQName):] + else: + raise BadXMLMessage, "XML tag %r is not in namespace %r" % (i.tag, namespace) + return e + +def b64_equal(thing1, thing2): + """ + Compare two Base64-encoded values for equality. + """ + return "".join(thing1.split()) == "".join(thing2.split()) + + + +class IRDB(object): + """ + Front-end to the IRDB. This is broken out from class main so + that other applications (namely, the portal-gui) can reuse it. + """ + def __init__(self, cfg): + """ + Opens a new connection to the IRDB, using the configuration + information from a rpki.config.parser object. + """ + + from rpki.mysql_import import MySQLdb + + irdbd_cfg = rpki.config.parser(cfg.get("irdbd_conf", cfg.filename), "irdbd") + + self.db = MySQLdb.connect(user = irdbd_cfg.get("sql-username"), + db = irdbd_cfg.get("sql-database"), + passwd = irdbd_cfg.get("sql-password")) + + def update(self, handle, roa_requests, children, ghostbusters=None): + """ + Update the IRDB for a given resource handle. Removes all + existing data and replaces it with that specified in the + argument list. + + The "roa_requests" argument is a sequence of tuples of the form + (asID, v4_addresses, v6_addresses), where "v*_addresses" are + instances of rpki.resource_set.roa_prefix_set_ipv*. + + The "children" argument is a sequence of tuples of the form + (child_handle, asns, v4addrs, v6addrs, valid_until), + where "asns" is an instance of rpki.resource_set.resource_set_asn, + "v*addrs" are instances of rpki.resource_set.resource_set_ipv*, + and "valid_until" is an instance of rpki.sundial.datetime. + + The "ghostbusters" argument is a sequence of tuples of the form + (parent_handle, vcard_string). "parent_handle" may be value None, + in which case the specified vcard object will be used for all + parents. + """ + + cur = self.db.cursor() + + cur.execute( + """ + DELETE + FROM roa_request_prefix + USING roa_request, roa_request_prefix + WHERE roa_request.roa_request_id = roa_request_prefix.roa_request_id AND roa_request.roa_request_handle = %s + """, (handle,)) + + cur.execute("DELETE FROM roa_request WHERE roa_request.roa_request_handle = %s", (handle,)) + + for asID, v4addrs, v6addrs in roa_requests: + assert isinstance(v4addrs, rpki.resource_set.roa_prefix_set_ipv4) + assert isinstance(v6addrs, rpki.resource_set.roa_prefix_set_ipv6) + cur.execute("INSERT roa_request (roa_request_handle, asn) VALUES (%s, %s)", (handle, asID)) + roa_request_id = cur.lastrowid + for version, prefix_set in ((4, v4addrs), (6, v6addrs)): + if prefix_set: + cur.executemany("INSERT roa_request_prefix (roa_request_id, prefix, prefixlen, max_prefixlen, version) VALUES (%s, %s, %s, %s, %s)", + ((roa_request_id, p.prefix, p.prefixlen, p.max_prefixlen, version) for p in prefix_set)) + + cur.execute( + """ + DELETE + FROM registrant_asn + USING registrant, registrant_asn + WHERE registrant.registrant_id = registrant_asn.registrant_id AND registrant.registry_handle = %s + """ , (handle,)) + + cur.execute( + """ + DELETE FROM registrant_net USING registrant, registrant_net + WHERE registrant.registrant_id = registrant_net.registrant_id AND registrant.registry_handle = %s + """ , (handle,)) + + cur.execute("DELETE FROM registrant WHERE registrant.registry_handle = %s" , (handle,)) + + for child_handle, asns, ipv4, ipv6, valid_until in children: + cur.execute("INSERT registrant (registrant_handle, registry_handle, registrant_name, valid_until) VALUES (%s, %s, %s, %s)", + (child_handle, handle, child_handle, valid_until.to_sql())) + child_id = cur.lastrowid + if asns: + cur.executemany("INSERT registrant_asn (start_as, end_as, registrant_id) VALUES (%s, %s, %s)", + ((a.min, a.max, child_id) for a in asns)) + if ipv4: + cur.executemany("INSERT registrant_net (start_ip, end_ip, version, registrant_id) VALUES (%s, %s, 4, %s)", + ((a.min, a.max, child_id) for a in ipv4)) + if ipv6: + cur.executemany("INSERT registrant_net (start_ip, end_ip, version, registrant_id) VALUES (%s, %s, 6, %s)", + ((a.min, a.max, child_id) for a in ipv6)) + + # don't munge the ghostbuster_request table when the arg is None. + # this allows the cli to safely run configure_resources without + # stomping on GBRs created by the portal gui. + if ghostbusters is not None: + cur.execute("DELETE FROM ghostbuster_request WHERE self_handle = %s", (handle,)) + if ghostbusters: + cur.executemany("INSERT INTO ghostbuster_request (self_handle, parent_handle, vcard) VALUES (%s, %s, %s)", + ((handle, parent_handle, vcard) for parent_handle, vcard in ghostbusters)) + + self.db.commit() + + def close(self): + """ + Close the connection to the IRDB. + """ + self.db.close() + + + +class main(rpki.cli.Cmd): + + prompt = "rpkic> " + + completedefault = rpki.cli.Cmd.filename_complete + + show_xml = False + + def __init__(self): + os.environ["TZ"] = "UTC" + time.tzset() + + rpki.log.use_syslog = False + + self.cfg_file = None + + opts, argv = getopt.getopt(sys.argv[1:], "c:h?", ["config=", "help"]) + for o, a in opts: + if o in ("-c", "--config"): + self.cfg_file = a + elif o in ("-h", "--help", "-?"): + argv = ["help"] + + if not argv or argv[0] != "help": + rpki.log.init("rpkic") + self.read_config() + + rpki.cli.Cmd.__init__(self, argv) + + + def help_overview(self): + """ + Show program __doc__ string. Perhaps there's some clever way to + do this using the textwrap module, but for now something simple + and crude will suffice. + """ + for line in __doc__.splitlines(True): + self.stdout.write(" " * 4 + line) + self.stdout.write("\n") + + def entitydb_complete(self, prefix, text, line, begidx, endidx): + """ + Completion helper for entitydb filenames. + """ + names = [] + for name in self.entitydb.iterate(prefix): + name = os.path.splitext(os.path.basename(name))[0] + if name.startswith(text): + names.append(name) + return names + + def read_config(self): + + self.cfg = rpki.config.parser(self.cfg_file, "myrpki") + + self.histfile = self.cfg.get("history_file", ".rpkic_history") + self.handle = self.cfg.get("handle") + self.run_rpkid = self.cfg.getboolean("run_rpkid") + self.run_pubd = self.cfg.getboolean("run_pubd") + self.run_rootd = self.cfg.getboolean("run_rootd") + self.entitydb = EntityDB(self.cfg) + + if self.run_rootd and (not self.run_pubd or not self.run_rpkid): + raise CantRunRootd, "Can't run rootd unless also running rpkid and pubd" + + self.bpki_resources = CA(self.cfg.filename, self.cfg.get("bpki_resources_directory")) + if self.run_rpkid or self.run_pubd or self.run_rootd: + self.bpki_servers = CA(self.cfg.filename, self.cfg.get("bpki_servers_directory")) + else: + self.bpki_servers = None + + self.default_repository = self.cfg.get("default_repository", "") + self.pubd_contact_info = self.cfg.get("pubd_contact_info", "") + + self.rsync_module = self.cfg.get("publication_rsync_module") + self.rsync_server = self.cfg.get("publication_rsync_server") + + + def do_initialize(self, arg): + """ + Initialize an RPKI installation. This command reads the + configuration file, creates the BPKI and EntityDB directories, + generates the initial BPKI certificates, and creates an XML file + describing the resource-holding aspect of this RPKI installation. + """ + + if arg: + raise BadCommandSyntax, "This command takes no arguments" + + self.bpki_resources.setup(self.cfg.get("bpki_resources_ta_dn", + "/CN=%s BPKI Resource Trust Anchor" % self.handle)) + if self.run_rpkid or self.run_pubd or self.run_rootd: + self.bpki_servers.setup(self.cfg.get("bpki_servers_ta_dn", + "/CN=%s BPKI Server Trust Anchor" % self.handle)) + + # Create entitydb directories. + + for i in ("parents", "children", "repositories", "pubclients"): + d = self.entitydb(i) + if not os.path.exists(d): + os.makedirs(d) + + if self.run_rpkid or self.run_pubd or self.run_rootd: + + if self.run_rpkid: + self.bpki_servers.ee(self.cfg.get("bpki_rpkid_ee_dn", + "/CN=%s rpkid server certificate" % self.handle), "rpkid") + self.bpki_servers.ee(self.cfg.get("bpki_irdbd_ee_dn", + "/CN=%s irdbd server certificate" % self.handle), "irdbd") + if self.run_pubd: + self.bpki_servers.ee(self.cfg.get("bpki_pubd_ee_dn", + "/CN=%s pubd server certificate" % self.handle), "pubd") + if self.run_rpkid or self.run_pubd: + self.bpki_servers.ee(self.cfg.get("bpki_irbe_ee_dn", + "/CN=%s irbe client certificate" % self.handle), "irbe") + if self.run_rootd: + self.bpki_servers.ee(self.cfg.get("bpki_rootd_ee_dn", + "/CN=%s rootd server certificate" % self.handle), "rootd") + + # Build the identity.xml file. Need to check for existing file so we don't + # overwrite? Worry about that later. + + e = Element("identity", handle = self.handle) + PEMElement(e, "bpki_ta", self.bpki_resources.cer) + etree_write(e, self.entitydb.identity, + msg = None if self.run_rootd else 'This is the "identity" file you will need to send to your parent') + + # If we're running rootd, construct a fake parent to go with it, + # and cross-certify in both directions so we can talk to rootd. + + if self.run_rootd: + + e = Element("parent", parent_handle = self.handle, child_handle = self.handle, + service_uri = "http://localhost:%s/" % self.cfg.get("rootd_server_port"), + valid_until = str(rpki.sundial.now() + rpki.sundial.timedelta(days = 365))) + PEMElement(e, "bpki_resource_ta", self.bpki_servers.cer) + PEMElement(e, "bpki_child_ta", self.bpki_resources.cer) + SubElement(e, "repository", type = "offer") + etree_write(e, self.entitydb("parents", self.handle)) + + self.bpki_resources.xcert(self.bpki_servers.cer) + + rootd_child_fn = self.cfg.get("child-bpki-cert", None, "rootd") + if not os.path.exists(rootd_child_fn): + os.link(self.bpki_servers.xcert(self.bpki_resources.cer), rootd_child_fn) + + repo_file_name = self.entitydb("repositories", self.handle) + + try: + want_offer = etree_read(repo_file_name).get("type") != "confirmed" + except IOError: + want_offer = True + + if want_offer: + e = Element("repository", type = "offer", handle = self.handle, parent_handle = self.handle) + PEMElement(e, "bpki_client_ta", self.bpki_resources.cer) + etree_write(e, repo_file_name, + msg = 'This is the "repository offer" file for you to use if you want to publish in your own repository') + + + def do_update_bpki(self, arg): + """ + Update BPKI certificates. Assumes an existing RPKI installation. + + Basic plan here is to reissue all BPKI certificates we can, right + now. In the long run we might want to be more clever about only + touching ones that need maintenance, but this will do for a start. + + Most likely this should be run under cron. + """ + + if self.bpki_servers: + bpkis = (self.bpki_resources, self.bpki_servers) + else: + bpkis = (self.bpki_resources,) + + for bpki in bpkis: + for cer in glob.iglob("%s/*.cer" % bpki.dir): + key = cer[0:-4] + ".key" + req = cer[0:-4] + ".req" + if os.path.exists(key): + print "Regenerating BPKI PKCS #10", req + bpki.run_openssl("x509", "-x509toreq", "-in", cer, "-out", req, "-signkey", key) + print "Clearing BPKI certificate", cer + os.unlink(cer) + if cer == bpki.cer: + assert req == bpki.req + print "Regenerating certificate", cer + bpki.run_ca("-selfsign", "-extensions", "ca_x509_ext_ca", "-in", req, "-out", cer) + + print "Regenerating CRLs" + for bpki in bpkis: + bpki.run_ca("-gencrl", "-out", bpki.crl) + + self.do_initialize(None) + if self.run_rpkid or self.run_pubd or self.run_rootd: + self.do_configure_daemons(arg) + else: + self.do_configure_resources(None) + + + def do_configure_child(self, arg): + """ + Configure a new child of this RPKI entity, given the child's XML + identity file as an input. This command extracts the child's data + from the XML, cross-certifies the child's resource-holding BPKI + certificate, and generates an XML file describing the relationship + between the child and this parent, including this parent's BPKI + data and up-down protocol service URI. + """ + + child_handle = None + + opts, argv = getopt.getopt(arg.split(), "", ["child_handle="]) + for o, a in opts: + if o == "--child_handle": + child_handle = a + + if len(argv) != 1: + raise BadCommandSyntax, "Need to specify filename for child.xml" + + c = etree_read(argv[0]) + + if child_handle is None: + child_handle = c.get("handle") + + try: + e = etree_read(self.cfg.get("xml_filename")) + service_uri_base = e.get("service_uri") + + except IOError: + if self.run_rpkid: + service_uri_base = "http://%s:%s/up-down/%s" % (self.cfg.get("rpkid_server_host"), + self.cfg.get("rpkid_server_port"), + self.handle) + else: + service_uri_base = None + + if not service_uri_base: + print "Sorry, you can't set up children of a hosted config that itself has not yet been set up" + return + + print "Child calls itself %r, we call it %r" % (c.get("handle"), child_handle) + + if self.run_rpkid or self.run_pubd or self.run_rootd: + self.bpki_servers.fxcert(c.findtext("bpki_ta")) + + e = Element("parent", parent_handle = self.handle, child_handle = child_handle, + service_uri = "%s/%s" % (service_uri_base, child_handle), + valid_until = str(rpki.sundial.now() + rpki.sundial.timedelta(days = 365))) + + PEMElement(e, "bpki_resource_ta", self.bpki_resources.cer) + SubElement(e, "bpki_child_ta").text = c.findtext("bpki_ta") + + repo = None + for f in self.entitydb.iterate("repositories"): + r = etree_read(f) + if r.get("type") == "confirmed": + h = os.path.splitext(os.path.split(f)[-1])[0] + if repo is None or h == self.default_repository: + repo_handle = h + repo = r + + if repo is None: + print "Couldn't find any usable repositories, not giving referral" + + elif repo_handle == self.handle: + SubElement(e, "repository", type = "offer") + + else: + proposed_sia_base = repo.get("sia_base") + child_handle + "/" + r = Element("referral", authorized_sia_base = proposed_sia_base) + r.text = c.findtext("bpki_ta") + auth = self.bpki_resources.cms_xml_sign( + "/CN=%s Publication Referral" % self.handle, "referral", r) + + r = SubElement(e, "repository", type = "referral") + SubElement(r, "authorization", referrer = repo.get("client_handle")).text = auth + SubElement(r, "contact_info").text = repo.findtext("contact_info") + + etree_write(e, self.entitydb("children", child_handle), + msg = "Send this file back to the child you just configured") + + + def do_delete_child(self, arg): + """ + Delete a child of this RPKI entity. + + This should check that the XML file it's deleting really is a + child, but doesn't, yet. + """ + + try: + os.unlink(self.entitydb("children", arg)) + except OSError: + print "No such child \"%s\"" % arg + + def complete_delete_child(self, *args): + return self.entitydb_complete("children", *args) + + + def do_configure_parent(self, arg): + """ + Configure a new parent of this RPKI entity, given the output of + the parent's configure_child command as input. This command reads + the parent's response XML, extracts the parent's BPKI and service + URI information, cross-certifies the parent's BPKI data into this + entity's BPKI, and checks for offers or referrals of publication + service. If a publication offer or referral is present, we + generate a request-for-service message to that repository, in case + the user wants to avail herself of the referral or offer. + """ + + parent_handle = None + + opts, argv = getopt.getopt(arg.split(), "", ["parent_handle="]) + for o, a in opts: + if o == "--parent_handle": + parent_handle = a + + if len(argv) != 1: + raise BadCommandSyntax, "Need to specify filename for parent.xml on command line" + + p = etree_read(argv[0]) + + if parent_handle is None: + parent_handle = p.get("parent_handle") + + print "Parent calls itself %r, we call it %r" % (p.get("parent_handle"), parent_handle) + print "Parent calls us %r" % p.get("child_handle") + + self.bpki_resources.fxcert(p.findtext("bpki_resource_ta")) + + etree_write(p, self.entitydb("parents", parent_handle)) + + r = p.find("repository") + + if r is None or r.get("type") not in ("offer", "referral"): + r = Element("repository", type = "none") + + r.set("handle", self.handle) + r.set("parent_handle", parent_handle) + PEMElement(r, "bpki_client_ta", self.bpki_resources.cer) + etree_write(r, self.entitydb("repositories", parent_handle), + msg = "This is the file to send to the repository operator") + + + def do_delete_parent(self, arg): + """ + Delete a parent of this RPKI entity. + + This should check that the XML file it's deleting really is a + parent, but doesn't, yet. + """ + + try: + os.unlink(self.entitydb("parents", arg)) + except OSError: + print "No such parent \"%s\"" % arg + + def complete_delete_parent(self, *args): + return self.entitydb_complete("parents", *args) + + + def do_configure_publication_client(self, arg): + """ + Configure publication server to know about a new client, given the + client's request-for-service message as input. This command reads + the client's request for service, cross-certifies the client's + BPKI data, and generates a response message containing the + repository's BPKI data and service URI. + """ + + sia_base = None + + opts, argv = getopt.getopt(arg.split(), "", ["sia_base="]) + for o, a in opts: + if o == "--sia_base": + sia_base = a + + if len(argv) != 1: + raise BadCommandSyntax, "Need to specify filename for client.xml" + + client = etree_read(argv[0]) + + if sia_base is None and client.get("handle") == self.handle and b64_equal(PEMBase64(self.bpki_resources.cer), client.findtext("bpki_client_ta")): + print "This looks like self-hosted publication" + sia_base = "rsync://%s/%s/%s/" % (self.rsync_server, self.rsync_module, self.handle) + + if sia_base is None and client.get("type") == "referral": + print "This looks like a referral, checking" + try: + auth = client.find("authorization") + if auth is None: + raise BadXMLMessage, "Malformed referral, couldn't find <auth/> element" + referrer = etree_read(self.entitydb("pubclients", auth.get("referrer").replace("/","."))) + referrer = self.bpki_servers.fxcert(referrer.findtext("bpki_client_ta")) + referral = self.bpki_servers.cms_xml_verify(auth.text, referrer) + if not b64_equal(referral.text, client.findtext("bpki_client_ta")): + raise BadXMLMessage, "Referral trust anchor does not match" + sia_base = referral.get("authorized_sia_base") + except IOError: + print "We have no record of client (%s) alleged to have made this referral" % auth.get("referrer") + + if sia_base is None and client.get("type") == "offer" and client.get("parent_handle") == self.handle: + print "This looks like an offer, client claims to be our child, checking" + client_ta = client.findtext("bpki_client_ta") + if not client_ta: + raise BadXMLMessage, "Malformed offer, couldn't find <bpki_client_ta/> element" + for child in self.entitydb.iterate("children"): + c = etree_read(child) + if b64_equal(c.findtext("bpki_child_ta"), client_ta): + sia_base = "rsync://%s/%s/%s/%s/" % (self.rsync_server, self.rsync_module, + self.handle, client.get("handle")) + break + + # If we still haven't figured out what to do with this client, it + # gets a top-level tree of its own, no attempt at nesting. + + if sia_base is None: + print "Don't know where to nest this client, defaulting to top-level" + sia_base = "rsync://%s/%s/%s/" % (self.rsync_server, self.rsync_module, client.get("handle")) + + if not sia_base.startswith("rsync://"): + raise BadXMLMessage, "Malformed sia_base parameter %r, should start with 'rsync://'" % sia_base + + client_handle = "/".join(sia_base.rstrip("/").split("/")[4:]) + + parent_handle = client.get("parent_handle") + + print "Client calls itself %r, we call it %r" % (client.get("handle"), client_handle) + print "Client says its parent handle is %r" % parent_handle + + self.bpki_servers.fxcert(client.findtext("bpki_client_ta")) + + e = Element("repository", type = "confirmed", + client_handle = client_handle, + parent_handle = parent_handle, + sia_base = sia_base, + service_uri = "http://%s:%s/client/%s" % (self.cfg.get("pubd_server_host"), + self.cfg.get("pubd_server_port"), + client_handle)) + + PEMElement(e, "bpki_server_ta", self.bpki_servers.cer) + SubElement(e, "bpki_client_ta").text = client.findtext("bpki_client_ta") + SubElement(e, "contact_info").text = self.pubd_contact_info + etree_write(e, self.entitydb("pubclients", client_handle.replace("/", ".")), + msg = "Send this file back to the publication client you just configured") + + + def do_delete_publication_client(self, arg): + """ + Delete a publication client of this RPKI entity. + + This should check that the XML file it's deleting really is a + client, but doesn't, yet. + """ + + try: + os.unlink(self.entitydb("pubclients", arg)) + except OSError: + print "No such client \"%s\"" % arg + + def complete_delete_publication_client(self, *args): + return self.entitydb_complete("pubclients", *args) + + + def do_configure_repository(self, arg): + """ + Configure a publication repository for this RPKI entity, given the + repository's response to our request-for-service message as input. + This command reads the repository's response, extracts and + cross-certifies the BPKI data and service URI, and links the + repository data with the corresponding parent data in our local + database. + """ + + parent_handle = None + + opts, argv = getopt.getopt(arg.split(), "", ["parent_handle="]) + for o, a in opts: + if o == "--parent_handle": + parent_handle = a + + if len(argv) != 1: + raise BadCommandSyntax, "Need to specify filename for repository.xml on command line" + + r = etree_read(argv[0]) + + if parent_handle is None: + parent_handle = r.get("parent_handle") + + print "Repository calls us %r" % (r.get("client_handle")) + print "Repository response associated with parent_handle %r" % parent_handle + + etree_write(r, self.entitydb("repositories", parent_handle)) + + + def do_delete_repository(self, arg): + """ + Delete a repository of this RPKI entity. + + This should check that the XML file it's deleting really is a + repository, but doesn't, yet. + """ + + try: + os.unlink(self.entitydb("repositories", arg)) + except OSError: + print "No such repository \"%s\"" % arg + + def complete_delete_repository(self, *args): + return self.entitydb_complete("repositories", *args) + + + def renew_children_common(self, arg, plural): + """ + Common code for renew_child and renew_all_children commands. + """ + + valid_until = None + + opts, argv = getopt.getopt(arg.split(), "", ["valid_until"]) + for o, a in opts: + if o == "--valid_until": + valid_until = a + + if plural: + if len(argv) != 0: + raise BadCommandSyntax, "Unexpected arguments" + children = "*" + else: + if len(argv) != 1: + raise BadCommandSyntax, "Need to specify child handle" + children = argv[0] + + if valid_until is None: + valid_until = rpki.sundial.now() + rpki.sundial.timedelta(days = 365) + else: + valid_until = rpki.sundial.fromXMLtime(valid_until) + if valid_until < rpki.sundial.now(): + raise PastExpiration, "Specified new expiration time %s has passed" % valid_until + + print "New validity date", valid_until + + for f in self.entitydb.iterate("children", children): + c = etree_read(f) + c.set("valid_until", str(valid_until)) + etree_write(c, f) + + def do_renew_child(self, arg): + """ + Update validity period for one child entity. + """ + return self.renew_children_common(arg, False) + + def complete_renew_child(self, *args): + return self.entitydb_complete("children", *args) + + def do_renew_all_children(self, arg): + """ + Update validity period for all child entities. + """ + return self.renew_children_common(arg, True) + + + + + def configure_resources_main(self, msg = None): + """ + Main program of old myrpki.py script. This remains separate + because it's called from more than one place. + """ + + roa_csv_file = self.cfg.get("roa_csv") + prefix_csv_file = self.cfg.get("prefix_csv") + asn_csv_file = self.cfg.get("asn_csv") + + # This probably should become an argument instead of (or in + # addition to a default from?) a config file option. + xml_filename = self.cfg.get("xml_filename") + + try: + e = etree_read(xml_filename) + bsc_req, bsc_cer = self.bpki_resources.bsc(e.findtext("bpki_bsc_pkcs10")) + service_uri = e.get("service_uri") + except IOError: + bsc_req, bsc_cer = None, None + service_uri = None + + e = Element("myrpki", handle = self.handle) + + if service_uri: + e.set("service_uri", service_uri) + + roa_requests.from_csv(roa_csv_file).xml(e) + + children.from_entitydb( + prefix_csv_file = prefix_csv_file, + asn_csv_file = asn_csv_file, + fxcert = self.bpki_resources.fxcert, + entitydb = self.entitydb).xml(e) + + parents.from_entitydb( + fxcert = self.bpki_resources.fxcert, + entitydb = self.entitydb).xml(e) + + repositories.from_entitydb( + fxcert = self.bpki_resources.fxcert, + entitydb = self.entitydb).xml(e) + + PEMElement(e, "bpki_ca_certificate", self.bpki_resources.cer) + PEMElement(e, "bpki_crl", self.bpki_resources.crl) + + if bsc_cer: + PEMElement(e, "bpki_bsc_certificate", bsc_cer) + + if bsc_req: + PEMElement(e, "bpki_bsc_pkcs10", bsc_req) + + etree_write(e, xml_filename, msg = msg) + + + def do_configure_resources(self, arg): + """ + Read CSV files and all the descriptions of parents and children + that we've built up, package the result up as a single XML file to + be shipped to a hosting rpkid. + """ + + if arg: + raise BadCommandSyntax, "Unexpected argument %r" % arg + self.configure_resources_main(msg = "Send this file to the rpkid operator who is hosting you") + + + + def do_configure_daemons(self, arg): + """ + Configure RPKI daemons with the data built up by the other + commands in this program. + + The basic model here is that each entity with resources to certify + runs the rpkic tool, but not all of them necessarily run their + own RPKI engines. The entities that do run RPKI engines get data + from the entities they host via the XML files output by the + configure_resources command. Those XML files are the input to + this command, which uses them to do all the work of configuring + daemons, populating SQL databases, and so forth. A few operations + (eg, BSC construction) generate data which has to be shipped back + to the resource holder, which we do by updating the same XML file. + + In essence, the XML files are a sneakernet (or email, or carrier + pigeon) communication channel between the resource holders and the + RPKI engine operators. + + As a convenience, for the normal case where the RPKI engine + operator is itself a resource holder, this command in effect runs + the configure_resources command automatically to process the RPKI + engine operator's own resources. + + Note that, due to the back and forth nature of some of these + operations, it may take several cycles for data structures to stablize + and everything to reach a steady state. This is normal. + """ + + argv = arg.split() + + def findbase64(tree, name, b64type = rpki.x509.X509): + x = tree.findtext(name) + return b64type(Base64 = x) if x else None + + # We can use a single BSC for everything -- except BSC key + # rollovers. Drive off that bridge when we get to it. + + bsc_handle = "bsc" + + self.cfg.set_global_flags() + + # Default values for CRL parameters are low, for testing. Not + # quite as low as they once were, too much expired CRL whining. + + self_crl_interval = self.cfg.getint("self_crl_interval", 2 * 60 * 60) + self_regen_margin = self.cfg.getint("self_regen_margin", self_crl_interval / 4) + pubd_base = "http://%s:%s/" % (self.cfg.get("pubd_server_host"), self.cfg.get("pubd_server_port")) + rpkid_base = "http://%s:%s/" % (self.cfg.get("rpkid_server_host"), self.cfg.get("rpkid_server_port")) + + # Wrappers to simplify calling rpkid and pubd. + + call_rpkid = rpki.async.sync_wrapper(rpki.http.caller( + proto = rpki.left_right, + client_key = rpki.x509.RSA( PEM_file = self.bpki_servers.dir + "/irbe.key"), + client_cert = rpki.x509.X509(PEM_file = self.bpki_servers.dir + "/irbe.cer"), + server_ta = rpki.x509.X509(PEM_file = self.bpki_servers.cer), + server_cert = rpki.x509.X509(PEM_file = self.bpki_servers.dir + "/rpkid.cer"), + url = rpkid_base + "left-right", + debug = self.show_xml)) + + if self.run_pubd: + + call_pubd = rpki.async.sync_wrapper(rpki.http.caller( + proto = rpki.publication, + client_key = rpki.x509.RSA( PEM_file = self.bpki_servers.dir + "/irbe.key"), + client_cert = rpki.x509.X509(PEM_file = self.bpki_servers.dir + "/irbe.cer"), + server_ta = rpki.x509.X509(PEM_file = self.bpki_servers.cer), + server_cert = rpki.x509.X509(PEM_file = self.bpki_servers.dir + "/pubd.cer"), + url = pubd_base + "control", + debug = self.show_xml)) + + # Make sure that pubd's BPKI CRL is up to date. + + call_pubd(rpki.publication.config_elt.make_pdu( + action = "set", + bpki_crl = rpki.x509.CRL(PEM_file = self.bpki_servers.crl))) + + irdb = IRDB(self.cfg) + + xmlfiles = [] + + # If [myrpki] section includes an "xml_filename" setting, run + # myrpki.py internally, as a convenience, and include its output at + # the head of our list of XML files to process. + + my_xmlfile = self.cfg.get("xml_filename", "") + if my_xmlfile: + self.configure_resources_main() + xmlfiles.append(my_xmlfile) + else: + my_xmlfile = None + + # Add any other XML files specified on the command line + + xmlfiles.extend(argv) + + for xmlfile in xmlfiles: + + # Parse XML file and validate it against our scheme + + tree = etree_read(xmlfile) + + handle = tree.get("handle") + + # Update IRDB with parsed resource and roa-request data. + + roa_requests = [( + x.get('asn'), + rpki.resource_set.roa_prefix_set_ipv4(x.get("v4")), + rpki.resource_set.roa_prefix_set_ipv6(x.get("v6"))) for x in tree.getiterator("roa_request")] + + children = [( + x.get("handle"), + rpki.resource_set.resource_set_as(x.get("asns")), + rpki.resource_set.resource_set_ipv4(x.get("v4")), + rpki.resource_set.resource_set_ipv6(x.get("v6")), + rpki.sundial.datetime.fromXMLtime(x.get("valid_until"))) for x in tree.getiterator("child")] + + # ghostbusters are ignored for now + irdb.update(handle, roa_requests, children) + + # Check for certificates before attempting anything else + + hosted_cacert = findbase64(tree, "bpki_ca_certificate") + if not hosted_cacert: + print "Nothing else I can do without a trust anchor for the entity I'm hosting." + continue + + rpkid_xcert = rpki.x509.X509(PEM_file = self.bpki_servers.fxcert(b64 = hosted_cacert.get_Base64(), + #filename = handle + ".cacert.cer", + path_restriction = 1)) + + # See what rpkid and pubd already have on file for this entity. + + if self.run_pubd: + client_pdus = dict((x.client_handle, x) + for x in call_pubd(rpki.publication.client_elt.make_pdu(action = "list")) + if isinstance(x, rpki.publication.client_elt)) + + rpkid_reply = call_rpkid( + rpki.left_right.self_elt.make_pdu( action = "get", tag = "self", self_handle = handle), + rpki.left_right.bsc_elt.make_pdu( action = "list", tag = "bsc", self_handle = handle), + rpki.left_right.repository_elt.make_pdu(action = "list", tag = "repository", self_handle = handle), + rpki.left_right.parent_elt.make_pdu( action = "list", tag = "parent", self_handle = handle), + rpki.left_right.child_elt.make_pdu( action = "list", tag = "child", self_handle = handle)) + + self_pdu = rpkid_reply[0] + bsc_pdus = dict((x.bsc_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.bsc_elt)) + repository_pdus = dict((x.repository_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.repository_elt)) + parent_pdus = dict((x.parent_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.parent_elt)) + child_pdus = dict((x.child_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.child_elt)) + + pubd_query = [] + rpkid_query = [] + + # There should be exactly one <self/> object per hosted entity, by definition + + if (isinstance(self_pdu, rpki.left_right.report_error_elt) or + self_pdu.crl_interval != self_crl_interval or + self_pdu.regen_margin != self_regen_margin or + self_pdu.bpki_cert != rpkid_xcert): + rpkid_query.append(rpki.left_right.self_elt.make_pdu( + action = "create" if isinstance(self_pdu, rpki.left_right.report_error_elt) else "set", + tag = "self", + self_handle = handle, + bpki_cert = rpkid_xcert, + crl_interval = self_crl_interval, + regen_margin = self_regen_margin)) + + # In general we only need one <bsc/> per <self/>. BSC objects are a + # little unusual in that the PKCS #10 subelement is generated by rpkid + # in response to generate_keypair, so there's more of a separation + # between create and set than with other objects. + + bsc_cert = findbase64(tree, "bpki_bsc_certificate") + bsc_crl = findbase64(tree, "bpki_crl", rpki.x509.CRL) + + bsc_pdu = bsc_pdus.pop(bsc_handle, None) + + if bsc_pdu is None: + rpkid_query.append(rpki.left_right.bsc_elt.make_pdu( + action = "create", + tag = "bsc", + self_handle = handle, + bsc_handle = bsc_handle, + generate_keypair = "yes")) + elif bsc_pdu.signing_cert != bsc_cert or bsc_pdu.signing_cert_crl != bsc_crl: + rpkid_query.append(rpki.left_right.bsc_elt.make_pdu( + action = "set", + tag = "bsc", + self_handle = handle, + bsc_handle = bsc_handle, + signing_cert = bsc_cert, + signing_cert_crl = bsc_crl)) + + rpkid_query.extend(rpki.left_right.bsc_elt.make_pdu( + action = "destroy", self_handle = handle, bsc_handle = b) for b in bsc_pdus) + + bsc_req = None + + if bsc_pdu and bsc_pdu.pkcs10_request: + bsc_req = bsc_pdu.pkcs10_request + + # At present we need one <repository/> per <parent/>, not because + # rpkid requires that, but because pubd does. pubd probably should + # be fixed to support a single client allowed to update multiple + # trees, but for the moment the easiest way forward is just to + # enforce a 1:1 mapping between <parent/> and <repository/> objects + + for repository in tree.getiterator("repository"): + + repository_handle = repository.get("handle") + repository_pdu = repository_pdus.pop(repository_handle, None) + repository_uri = repository.get("service_uri") + repository_cert = findbase64(repository, "bpki_certificate") + + if (repository_pdu is None or + repository_pdu.bsc_handle != bsc_handle or + repository_pdu.peer_contact_uri != repository_uri or + repository_pdu.bpki_cert != repository_cert): + rpkid_query.append(rpki.left_right.repository_elt.make_pdu( + action = "create" if repository_pdu is None else "set", + tag = repository_handle, + self_handle = handle, + repository_handle = repository_handle, + bsc_handle = bsc_handle, + peer_contact_uri = repository_uri, + bpki_cert = repository_cert)) + + rpkid_query.extend(rpki.left_right.repository_elt.make_pdu( + action = "destroy", self_handle = handle, repository_handle = r) for r in repository_pdus) + + # <parent/> setup code currently assumes 1:1 mapping between + # <repository/> and <parent/>, and further assumes that the handles + # for an associated pair are the identical (that is: + # parent.repository_handle == parent.parent_handle). + + for parent in tree.getiterator("parent"): + + parent_handle = parent.get("handle") + parent_pdu = parent_pdus.pop(parent_handle, None) + parent_uri = parent.get("service_uri") + parent_myhandle = parent.get("myhandle") + parent_sia_base = parent.get("sia_base") + parent_cms_cert = findbase64(parent, "bpki_cms_certificate") + + if (parent_pdu is None or + parent_pdu.bsc_handle != bsc_handle or + parent_pdu.repository_handle != parent_handle or + parent_pdu.peer_contact_uri != parent_uri or + parent_pdu.sia_base != parent_sia_base or + parent_pdu.sender_name != parent_myhandle or + parent_pdu.recipient_name != parent_handle or + parent_pdu.bpki_cms_cert != parent_cms_cert): + rpkid_query.append(rpki.left_right.parent_elt.make_pdu( + action = "create" if parent_pdu is None else "set", + tag = parent_handle, + self_handle = handle, + parent_handle = parent_handle, + bsc_handle = bsc_handle, + repository_handle = parent_handle, + peer_contact_uri = parent_uri, + sia_base = parent_sia_base, + sender_name = parent_myhandle, + recipient_name = parent_handle, + bpki_cms_cert = parent_cms_cert)) + + rpkid_query.extend(rpki.left_right.parent_elt.make_pdu( + action = "destroy", self_handle = handle, parent_handle = p) for p in parent_pdus) + + # Children are simpler than parents, because they call us, so no URL + # to construct and figuring out what certificate to use is their + # problem, not ours. + + for child in tree.getiterator("child"): + + child_handle = child.get("handle") + child_pdu = child_pdus.pop(child_handle, None) + child_cert = findbase64(child, "bpki_certificate") + + if (child_pdu is None or + child_pdu.bsc_handle != bsc_handle or + child_pdu.bpki_cert != child_cert): + rpkid_query.append(rpki.left_right.child_elt.make_pdu( + action = "create" if child_pdu is None else "set", + tag = child_handle, + self_handle = handle, + child_handle = child_handle, + bsc_handle = bsc_handle, + bpki_cert = child_cert)) + + rpkid_query.extend(rpki.left_right.child_elt.make_pdu( + action = "destroy", self_handle = handle, child_handle = c) for c in child_pdus) + + # Publication setup. + + if self.run_pubd: + + for f in self.entitydb.iterate("pubclients"): + c = etree_read(f) + + client_handle = c.get("client_handle") + client_base_uri = c.get("sia_base") + client_bpki_cert = rpki.x509.X509(PEM_file = self.bpki_servers.fxcert(c.findtext("bpki_client_ta"))) + client_pdu = client_pdus.pop(client_handle, None) + + if (client_pdu is None or + client_pdu.base_uri != client_base_uri or + client_pdu.bpki_cert != client_bpki_cert): + pubd_query.append(rpki.publication.client_elt.make_pdu( + action = "create" if client_pdu is None else "set", + client_handle = client_handle, + bpki_cert = client_bpki_cert, + base_uri = client_base_uri)) + + pubd_query.extend(rpki.publication.client_elt.make_pdu( + action = "destroy", client_handle = p) for p in client_pdus) + + # If we changed anything, ship updates off to daemons + + failed = False + + if rpkid_query: + rpkid_reply = call_rpkid(*rpkid_query) + bsc_pdus = dict((x.bsc_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.bsc_elt)) + if bsc_handle in bsc_pdus and bsc_pdus[bsc_handle].pkcs10_request: + bsc_req = bsc_pdus[bsc_handle].pkcs10_request + for r in rpkid_reply: + if isinstance(r, rpki.left_right.report_error_elt): + failed = True + print "rpkid reported failure:", r.error_code + if r.error_text: + print r.error_text + + if failed: + raise CouldntTalkToDaemon + + if pubd_query: + assert self.run_pubd + pubd_reply = call_pubd(*pubd_query) + for r in pubd_reply: + if isinstance(r, rpki.publication.report_error_elt): + failed = True + print "pubd reported failure:", r.error_code + if r.error_text: + print r.error_text + + if failed: + raise CouldntTalkToDaemon + + # Rewrite XML. + + e = tree.find("bpki_bsc_pkcs10") + if e is not None: + tree.remove(e) + if bsc_req is not None: + SubElement(tree, "bpki_bsc_pkcs10").text = bsc_req.get_Base64() + + tree.set("service_uri", rpkid_base + "up-down/" + handle) + + etree_write(tree, xmlfile, + msg = None if xmlfile is my_xmlfile else 'Send this file back to the hosted entity ("%s")' % handle) + + irdb.close() + + # We used to run event loop again to give TLS connections a chance to shut down cleanly. + # Seems not to be needed (and sometimes hangs forever, which is odd) with TLS out of the picture. + #rpki.async.event_loop() diff --git a/rpkid/rpki/x509.py b/rpkid/rpki/x509.py index 2d5505d5..29470c31 100644 --- a/rpkid/rpki/x509.py +++ b/rpkid/rpki/x509.py @@ -1402,6 +1402,13 @@ class XML_CMS_object(CMS_object): self.schema_check() return self.saxify(self.get_content()) + ## @var saxify + # SAX handler hook. Subclasses can set this to a SAX handler, in + # which case .unwrap() will call it and return the result. + # Otherwise, .unwrap() just returns a verified element tree. + + saxify = staticmethod(lambda x: x) + class Ghostbuster(CMS_object): """ Class to hold Ghostbusters record (CMS-wrapped VCard). This is diff --git a/rpkid/rpkic.py b/rpkid/rpkic.py new file mode 100644 index 00000000..d8f68627 --- /dev/null +++ b/rpkid/rpkic.py @@ -0,0 +1,21 @@ +""" +$Id$ + +Copyright (C) 2010 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +if __name__ == "__main__": + import rpki.myrpki + rpki.myrpki.main() diff --git a/rpkid/tests/smoketest.py b/rpkid/tests/smoketest.py index 189f6d6a..4c888f67 100644 --- a/rpkid/tests/smoketest.py +++ b/rpkid/tests/smoketest.py @@ -1269,16 +1269,12 @@ def mangle_sql(filename): """ Mangle an SQL file into a sequence of SQL statements. """ - - # There is no pretty way to do this. Just shut your eyes, it'll be - # over soon. - + words = [] f = open(filename) - statements = " ".join(" ".join(word for word in line.expandtabs().split(" ") if word) - for line in [line.strip(" \t\n") for line in f.readlines()] - if line and not line.startswith("--")).rstrip(";").split(";") + for line in f: + words.extend(line.partition("--")[0].split()) f.close() - return [stmt.strip() for stmt in statements] + return " ".join(words).strip(";").split(";") bpki_cert_fmt_1 = '''\ [ req ] diff --git a/rpkid/tests/sql-cleaner.py b/rpkid/tests/sql-cleaner.py index 5c772bc4..5d11781f 100644 --- a/rpkid/tests/sql-cleaner.py +++ b/rpkid/tests/sql-cleaner.py @@ -3,7 +3,7 @@ $Id$ -Copyright (C) 2009--2010 Internet Systems Consortium ("ISC") +Copyright (C) 2009--2011 Internet Systems Consortium ("ISC") Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above @@ -18,7 +18,8 @@ OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """ -import subprocess, rpki.config +import rpki.config, rpki.sql_schemas +from rpki.mysql_import import MySQLdb cfg = rpki.config.parser(None, "yamltest", allow_missing = True) @@ -26,8 +27,30 @@ for name in ("rpkid", "irdbd", "pubd"): username = cfg.get("%s_sql_username" % name, name[:4]) password = cfg.get("%s_sql_password" % name, "fnord") + + schema = [] + for line in getattr(rpki.sql_schemas, name).splitlines(): + schema.extend(line.partition("--")[0].split()) + schema = " ".join(schema).strip(";").split(";") + schema = [statement.strip() for statement in schema if "DROP TABLE" not in statement] for i in xrange(12): - subprocess.check_call( - ("mysql", "-u", username, "-p" + password, "%s%d" % (name[:4], i)), - stdin = open("../%s.sql" % name)) + + database = "%s%d" % (name[:4], i) + + db = MySQLdb.connect(user = username, db = database, passwd = password) + cur = db.cursor() + + cur.execute("SHOW TABLES") + tables = [r[0] for r in cur.fetchall()] + + cur.execute("SET foreign_key_checks = 0") + for table in tables: + cur.execute("DROP TABLE %s" % table) + cur.execute("SET foreign_key_checks = 1") + + for statement in schema: + cur.execute(statement) + + cur.close() + db.close() diff --git a/rpkid/tests/yamltest.py b/rpkid/tests/yamltest.py index ecd00af2..403076f1 100644 --- a/rpkid/tests/yamltest.py +++ b/rpkid/tests/yamltest.py @@ -1,6 +1,6 @@ """ Test framework, using the same YAML test description format as -smoketest.py, but using the myrpki.py tool to do all the back-end +smoketest.py, but using the rpkic.py tool to do all the back-end work. Reads YAML file, generates .csv and .conf files, runs daemons and waits for one of them to exit. @@ -10,7 +10,7 @@ Still to do: - Implement smoketest.py-style delta actions, that is, modify the allocation database under control of the YAML file, dump out new - .csv files, and run myrpki.py again to feed resulting changes into + .csv files, and run rpkic.py again to feed resulting changes into running daemons. $Id$ @@ -46,7 +46,7 @@ PERFORMANCE OF THIS SOFTWARE. """ import subprocess, re, os, getopt, sys, yaml, signal, time -import rpki.resource_set, rpki.sundial, rpki.config, rpki.log, rpki.myrpki +import rpki.resource_set, rpki.sundial, rpki.config, rpki.log, rpki.rpkic # Nasty regular expressions for parsing config files. Sadly, while # the Python ConfigParser supports writing config files, it does so in @@ -67,7 +67,7 @@ this_dir = os.getcwd() test_dir = cleanpath(this_dir, "yamltest.dir") rpkid_dir = cleanpath(this_dir, "..") -prog_myrpki = cleanpath(rpkid_dir, "myrpki.py") +prog_rpkic = cleanpath(rpkid_dir, "rpkic.py") prog_rpkid = cleanpath(rpkid_dir, "rpkid.py") prog_irdbd = cleanpath(rpkid_dir, "irdbd.py") prog_pubd = cleanpath(rpkid_dir, "pubd.py") @@ -154,7 +154,7 @@ class allocation_db(list): class allocation(object): """ One entity in our allocation database. Every entity in the database - is assumed to hold resources, so needs at least myrpki services. + is assumed to hold resources, so needs at least rpkic services. Entities that don't have the hosted_by property run their own copies of rpkid, irdbd, and pubd, so they also need myirbe services. """ @@ -290,12 +290,12 @@ class allocation(object): def csvout(self, fn): """ Open and log a CSV output file. We use delimiter and dialect - settings imported from the myrpki module, so that we automatically + settings imported from the rpkic module, so that we automatically write CSV files in the right format. """ path = self.path(fn) print "Writing", path - return rpki.myrpki.csv_writer(path) + return rpki.rpkic.csv_writer(path) def up_down_url(self): """ @@ -465,20 +465,20 @@ class allocation(object): print "%s is hosted, skipping configure_daemons" % self.name else: files = [h.path("myrpki.xml") for h in self.hosts] - self.run_myrpki("configure_daemons", *[f for f in files if os.path.exists(f)]) + self.run_rpkic("configure_daemons", *[f for f in files if os.path.exists(f)]) def run_configure_resources(self): """ Run configure_resources for this entity. """ - self.run_myrpki("configure_resources") + self.run_rpkic("configure_resources") - def run_myrpki(self, *args): + def run_rpkic(self, *args): """ - Run myrpki.py for this entity. + Run rpkic.py for this entity. """ - print 'Running "%s" for %s' % (" ".join(("myrpki",) + args), self.name) - subprocess.check_call((sys.executable, prog_myrpki) + args, cwd = self.path()) + print 'Running "%s" for %s' % (" ".join(("rpkic",) + args), self.name) + subprocess.check_call((sys.executable, prog_rpkic) + args, cwd = self.path()) def run_python_daemon(self, prog): """ @@ -623,7 +623,7 @@ try: # Initialize BPKI and generate self-descriptor for each entity. for d in db: - d.run_myrpki("initialize") + d.run_rpkic("initialize") # Create publication directories. @@ -660,19 +660,19 @@ try: print "Configuring", d.name print if d.is_root(): - d.run_myrpki("configure_publication_client", d.path("entitydb", "repositories", "%s.xml" % d.name)) + d.run_rpkic("configure_publication_client", d.path("entitydb", "repositories", "%s.xml" % d.name)) print - d.run_myrpki("configure_repository", d.path("entitydb", "pubclients", "%s.xml" % d.name)) + d.run_rpkic("configure_repository", d.path("entitydb", "pubclients", "%s.xml" % d.name)) print else: - d.parent.run_myrpki("configure_child", d.path("entitydb", "identity.xml")) + d.parent.run_rpkic("configure_child", d.path("entitydb", "identity.xml")) print - d.run_myrpki("configure_parent", d.parent.path("entitydb", "children", "%s.xml" % d.name)) + d.run_rpkic("configure_parent", d.parent.path("entitydb", "children", "%s.xml" % d.name)) print publisher, path = d.find_pubd() - publisher.run_myrpki("configure_publication_client", d.path("entitydb", "repositories", "%s.xml" % d.parent.name)) + publisher.run_rpkic("configure_publication_client", d.path("entitydb", "repositories", "%s.xml" % d.parent.name)) print - d.run_myrpki("configure_repository", publisher.path("entitydb", "pubclients", "%s.xml" % path)) + d.run_rpkic("configure_repository", publisher.path("entitydb", "pubclients", "%s.xml" % path)) print parent_host = d.parent.find_host() if d.parent is not parent_host: diff --git a/scripts/convert-from-entitydb-to-sql.py b/scripts/convert-from-entitydb-to-sql.py index 1ab5201d..1fb1bbea 100644 --- a/scripts/convert-from-entitydb-to-sql.py +++ b/scripts/convert-from-entitydb-to-sql.py @@ -404,7 +404,8 @@ for row in cur.fetchall(): for filename in sorted(xcert_filenames): cer = rpki.x509.X509(Auto_file = filename) - print "Unused cross-certificate:", filename, cer.getSubject() + #print "Unused cross-certificate:", filename, cer.getSubject() + print "Unused cross-certificate:", filename, cer.get_POW().pprint() # Done! |